From b6e8b9df35155e8f8fbc242961d09eded0ccc3bf Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 20 Aug 2022 13:00:25 +0200 Subject: [PATCH 1/4] Use cached leverage tiers --- freqtrade/exchange/exchange.py | 50 +++++++++++++++++++++++++++++----- 1 file changed, 43 insertions(+), 7 deletions(-) diff --git a/freqtrade/exchange/exchange.py b/freqtrade/exchange/exchange.py index c15481ca5..cfe4eab34 100644 --- a/freqtrade/exchange/exchange.py +++ b/freqtrade/exchange/exchange.py @@ -17,6 +17,7 @@ import ccxt import ccxt.async_support as ccxt_async from cachetools import TTLCache from ccxt import ROUND_DOWN, ROUND_UP, TICK_SIZE, TRUNCATE, decimal_to_precision +from dateutil import parser from pandas import DataFrame from freqtrade.constants import (DEFAULT_AMOUNT_RESERVE_PERCENT, NON_OPEN_EXCHANGE_STATES, BuySell, @@ -30,7 +31,8 @@ from freqtrade.exchange.common import (API_FETCH_ORDER_RETRY_COUNT, BAD_EXCHANGE EXCHANGE_HAS_OPTIONAL, EXCHANGE_HAS_REQUIRED, SUPPORTED_EXCHANGES, remove_credentials, retrier, retrier_async) -from freqtrade.misc import chunks, deep_merge_dicts, safe_value_fallback2 +from freqtrade.misc import (chunks, deep_merge_dicts, file_dump_json, file_load_json, + safe_value_fallback2) from freqtrade.plugins.pairlist.pairlist_helpers import expand_pairlist from freqtrade.util import FtPrecise @@ -2207,6 +2209,7 @@ class Exchange: @retrier_async async def get_market_leverage_tiers(self, symbol: str) -> Tuple[str, List[Dict]]: + """ Leverage tiers per symbol """ try: tier = await self._api_async.fetch_market_leverage_tiers(symbol) return symbol, tier @@ -2238,12 +2241,21 @@ class Exchange: tiers: Dict[str, List[Dict]] = {} - # Be verbose here, as this delays startup by ~1 minute. - logger.info( - f"Initializing leverage_tiers for {len(symbols)} markets. " - "This will take about a minute.") + tiers_cached = self.load_cached_leverage_tiers(self._config['stake_currency']) + if tiers_cached: + tiers = tiers_cached - coros = [self.get_market_leverage_tiers(symbol) for symbol in sorted(symbols)] + coros = [ + self.get_market_leverage_tiers(symbol) + for symbol in sorted(symbols) if symbol not in tiers] + + # Be verbose here, as this delays startup by ~1 minute. + if coros: + logger.info( + f"Initializing leverage_tiers for {len(symbols)} markets. " + "This will take about a minute.") + else: + logger.info("Using cached leverage_tiers.") async def gather_results(): return await asyncio.gather(*input_coro, return_exceptions=True) @@ -2255,7 +2267,8 @@ class Exchange: for symbol, res in results: tiers[symbol] = res - + if len(coros) > 0: + self.cache_leverage_tiers(tiers, self._config['stake_currency']) logger.info(f"Done initializing {len(symbols)} markets.") return tiers @@ -2264,6 +2277,29 @@ class Exchange: else: return {} + def cache_leverage_tiers(self, tiers: Dict[str, List[Dict]], stake_currency: str) -> None: + + filename = self._config['datadir'] / "futures" / f"leverage_tiers_{stake_currency}.json" + data = { + "updated": datetime.now(timezone.utc), + "data": tiers, + } + file_dump_json(filename, data) + + def load_cached_leverage_tiers(self, stake_currency: str) -> Optional[Dict[str, List[Dict]]]: + filename = self._config['datadir'] / "futures" / f"leverage_tiers_{stake_currency}.json" + if filename.is_file(): + tiers = file_load_json(filename) + updated = tiers.get('updated') + if updated: + updated_dt = parser.parse(updated) + print(updated_dt) + if updated_dt < datetime.now(timezone.utc) - timedelta(days=1): + logger.info("Cached leverage tiers are outdated. Will update.") + return None + return tiers['data'] + return None + def fill_leverage_tiers(self) -> None: """ Assigns property _leverage_tiers to a dictionary of information about the leverage From 738e95b8753bd6bbdc53eca58789b0dcc092d6bb Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 20 Aug 2022 13:47:34 +0200 Subject: [PATCH 2/4] Add tests for leverage tiers caching --- freqtrade/exchange/exchange.py | 3 ++- tests/exchange/test_okx.py | 26 ++++++++++++++++++++++++-- 2 files changed, 26 insertions(+), 3 deletions(-) diff --git a/freqtrade/exchange/exchange.py b/freqtrade/exchange/exchange.py index cfe4eab34..7ef6858a5 100644 --- a/freqtrade/exchange/exchange.py +++ b/freqtrade/exchange/exchange.py @@ -2280,6 +2280,8 @@ class Exchange: def cache_leverage_tiers(self, tiers: Dict[str, List[Dict]], stake_currency: str) -> None: filename = self._config['datadir'] / "futures" / f"leverage_tiers_{stake_currency}.json" + if not filename.parent.is_dir(): + filename.parent.mkdir(parents=True) data = { "updated": datetime.now(timezone.utc), "data": tiers, @@ -2293,7 +2295,6 @@ class Exchange: updated = tiers.get('updated') if updated: updated_dt = parser.parse(updated) - print(updated_dt) if updated_dt < datetime.now(timezone.utc) - timedelta(days=1): logger.info("Cached leverage tiers are outdated. Will update.") return None diff --git a/tests/exchange/test_okx.py b/tests/exchange/test_okx.py index 91c4a3368..10e087ced 100644 --- a/tests/exchange/test_okx.py +++ b/tests/exchange/test_okx.py @@ -1,4 +1,5 @@ from datetime import datetime, timedelta, timezone +from pathlib import Path from unittest.mock import MagicMock, PropertyMock import pytest @@ -6,7 +7,7 @@ import pytest from freqtrade.enums import MarginMode, TradingMode from freqtrade.enums.candletype import CandleType from freqtrade.exchange.exchange import timeframe_to_minutes -from tests.conftest import get_mock_coro, get_patched_exchange +from tests.conftest import get_mock_coro, get_patched_exchange, log_has from tests.exchange.test_exchange import ccxt_exceptionhandlers @@ -267,7 +268,10 @@ def test_additional_exchange_init_okx(default_conf, mocker): "additional_exchange_init", "fetch_accounts") -def test_load_leverage_tiers_okx(default_conf, mocker, markets): +def test_load_leverage_tiers_okx(default_conf, mocker, markets, tmpdir, caplog, time_machine): + + default_conf['datadir'] = Path(tmpdir) + # fd_mock = mocker.patch('freqtrade.exchange.exchange.file_dump_json') api_mock = MagicMock() type(api_mock).has = PropertyMock(return_value={ 'fetchLeverageTiers': False, @@ -455,3 +459,21 @@ def test_load_leverage_tiers_okx(default_conf, mocker, markets): }, ], } + filename = (default_conf['datadir'] / + f"futures/leverage_tiers_{default_conf['stake_currency']}.json") + assert filename.is_file() + + logmsg = 'Cached leverage tiers are outdated. Will update.' + assert not log_has(logmsg, caplog) + + api_mock.fetch_market_leverage_tiers.reset_mock() + + exchange.load_leverage_tiers() + assert not log_has(logmsg, caplog) + + api_mock.fetch_market_leverage_tiers.call_count == 0 + # 2 day passes ... + time_machine.move_to(datetime.now() + timedelta(days=2)) + exchange.load_leverage_tiers() + + assert log_has(logmsg, caplog) From 4511634f3a5012266fc49ef4cd33f9833235c805 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 20 Aug 2022 14:03:47 +0200 Subject: [PATCH 3/4] improve test coverage --- tests/exchange/test_exchange.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/tests/exchange/test_exchange.py b/tests/exchange/test_exchange.py index 6ad4a72c6..ec259d703 100644 --- a/tests/exchange/test_exchange.py +++ b/tests/exchange/test_exchange.py @@ -4791,6 +4791,20 @@ def test_load_leverage_tiers(mocker, default_conf, leverage_tiers, exchange_name ) +@pytest.mark.asyncio +@pytest.mark.parametrize('exchange_name', EXCHANGES) +async def test_get_market_leverage_tiers(mocker, default_conf, exchange_name): + default_conf['exchange']['name'] = exchange_name + await async_ccxt_exception( + mocker, + default_conf, + MagicMock(), + "get_market_leverage_tiers", + "fetch_market_leverage_tiers", + symbol='BTC/USDT:USDT' + ) + + def test_parse_leverage_tier(mocker, default_conf): exchange = get_patched_exchange(mocker, default_conf) From 1fb2e9558ff9511d2c8a3f4e0d43caace0581a4d Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 20 Aug 2022 14:36:19 +0200 Subject: [PATCH 4/4] Disable caching of leverage tiers in ccxt compat methods --- tests/exchange/test_ccxt_compat.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tests/exchange/test_ccxt_compat.py b/tests/exchange/test_ccxt_compat.py index 7bb52ccaf..c3a5fe17f 100644 --- a/tests/exchange/test_ccxt_compat.py +++ b/tests/exchange/test_ccxt_compat.py @@ -137,6 +137,10 @@ def exchange_futures(request, exchange_conf, class_mocker): 'freqtrade.exchange.binance.Binance.fill_leverage_tiers') class_mocker.patch('freqtrade.exchange.exchange.Exchange.fetch_trading_fees') class_mocker.patch('freqtrade.exchange.okx.Okx.additional_exchange_init') + class_mocker.patch('freqtrade.exchange.exchange.Exchange.load_cached_leverage_tiers', + return_value=None) + class_mocker.patch('freqtrade.exchange.exchange.Exchange.cache_leverage_tiers') + exchange = ExchangeResolver.load_exchange( request.param, exchange_conf, validate=True, load_leverage_tiers=True)