From c8f125dbb99dd2042898c4509064ed692379abee Mon Sep 17 00:00:00 2001
From: misagh <misagh.shakeri@gmail.com>
Date: Tue, 31 Jul 2018 12:47:32 +0200
Subject: [PATCH 01/47] ccxt async POC

---
 freqtrade/exchange/__init__.py | 42 ++++++++++++++++++++++++++++++----
 freqtrade/freqtradebot.py      | 33 +++++++++++++++++++++-----
 2 files changed, 65 insertions(+), 10 deletions(-)

diff --git a/freqtrade/exchange/__init__.py b/freqtrade/exchange/__init__.py
index 972ff49ca..6b9f000eb 100644
--- a/freqtrade/exchange/__init__.py
+++ b/freqtrade/exchange/__init__.py
@@ -6,7 +6,9 @@ from typing import List, Dict, Any, Optional
 from datetime import datetime
 
 import ccxt
+import ccxt.async_support as ccxt_async
 import arrow
+import asyncio
 
 from freqtrade import constants, OperationalException, DependencyException, TemporaryError
 
@@ -44,6 +46,7 @@ class Exchange(object):
 
     # Current selected exchange
     _api: ccxt.Exchange = None
+    _api_async: ccxt_async.Exchange = None
     _conf: Dict = {}
     _cached_ticker: Dict[str, Any] = {}
 
@@ -64,6 +67,7 @@ class Exchange(object):
 
         exchange_config = config['exchange']
         self._api = self._init_ccxt(exchange_config)
+        self._api_async = self._init_ccxt(exchange_config, ccxt_async)
 
         logger.info('Using Exchange "%s"', self.name)
 
@@ -74,7 +78,7 @@ class Exchange(object):
             # Check if timeframe is available
             self.validate_timeframes(config['ticker_interval'])
 
-    def _init_ccxt(self, exchange_config: dict) -> ccxt.Exchange:
+    def _init_ccxt(self, exchange_config: dict, ccxt_module=ccxt) -> ccxt.Exchange:
         """
         Initialize ccxt with given config and return valid
         ccxt instance.
@@ -82,15 +86,16 @@ class Exchange(object):
         # Find matching class for the given exchange name
         name = exchange_config['name']
 
-        if name not in ccxt.exchanges:
+        if name not in ccxt_module.exchanges:
             raise OperationalException(f'Exchange {name} is not supported')
         try:
-            api = getattr(ccxt, name.lower())({
+            api = getattr(ccxt_module, name.lower())({
                 'apiKey': exchange_config.get('key'),
                 'secret': exchange_config.get('secret'),
                 'password': exchange_config.get('password'),
                 'uid': exchange_config.get('uid', ''),
-                'enableRateLimit': True,
+                #'enableRateLimit': True,
+                'enableRateLimit': False,
             })
         except (KeyError, AttributeError):
             raise OperationalException(f'Exchange {name} is not supported')
@@ -286,6 +291,35 @@ class Exchange(object):
             logger.info("returning cached ticker-data for %s", pair)
             return self._cached_ticker[pair]
 
+
+    async def async_get_tickers_history(self, pairs, tick_interval):
+        # COMMENTED CODE IS FOR DISCUSSION: where should we close the loop on async ? 
+        #loop = asyncio.new_event_loop()
+        #asyncio.set_event_loop(loop)
+        input_coroutines = [self.async_get_ticker_history(symbol, tick_interval) for symbol in pairs]
+        tickers = await asyncio.gather(*input_coroutines, return_exceptions=True)
+        #await self._api_async.close()
+        return tickers
+
+    async def async_get_ticker_history(self, pair: str, tick_interval: str,
+                           since_ms: Optional[int] = None) -> List[Dict]:
+        try:
+            # fetch ohlcv asynchronously
+            print("fetching %s ..." % pair)
+            data = await self._api_async.fetch_ohlcv(pair, timeframe=tick_interval, since=since_ms)
+            print("done fetching %s ..." % pair)
+            return pair, data
+
+        except ccxt.NotSupported as e:
+            raise OperationalException(
+                f'Exchange {self._api.name} does not support fetching historical candlestick data.'
+                f'Message: {e}')
+        except (ccxt.NetworkError, ccxt.ExchangeError) as e:
+            raise TemporaryError(
+                f'Could not load ticker history due to {e.__class__.__name__}. Message: {e}')
+        except ccxt.BaseError as e:
+            raise OperationalException(f'Could not fetch ticker data. Msg: {e}')
+
     @retrier
     def get_ticker_history(self, pair: str, tick_interval: str,
                            since_ms: Optional[int] = None) -> List[Dict]:
diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py
index 46fbb3a38..93977ab16 100644
--- a/freqtrade/freqtradebot.py
+++ b/freqtrade/freqtradebot.py
@@ -8,11 +8,14 @@ import time
 import traceback
 from datetime import datetime
 from typing import Any, Callable, Dict, List, Optional
+import asyncio
 
 import arrow
 import requests
+
 from cachetools import TTLCache, cached
 
+
 from freqtrade import (DependencyException, OperationalException,
                        TemporaryError, __version__, constants, persistence)
 from freqtrade.exchange import Exchange
@@ -301,6 +304,12 @@ class FreqtradeBot(object):
         amount_reserve_percent = max(amount_reserve_percent, 0.5)
         return min(min_stake_amounts)/amount_reserve_percent
 
+    async def async_get_tickers(self, exchange, pairs):
+        input_coroutines = [exchange.async_get_ticker_history(symbol, self.strategy.ticker_interval) for symbol in pairs]
+        tickers = await asyncio.gather(*input_coroutines, return_exceptions=True)
+        return tickers
+        #await exchange.close()
+
     def create_trade(self) -> bool:
         """
         Checks the implemented trading indicator(s) for a randomly picked pair,
@@ -328,13 +337,25 @@ class FreqtradeBot(object):
         if not whitelist:
             raise DependencyException('No currency pairs in whitelist')
 
-        # Pick pair based on buy signals
-        for _pair in whitelist:
-            thistory = self.exchange.get_ticker_history(_pair, interval)
-            (buy, sell) = self.strategy.get_signal(_pair, interval, thistory)
+        
+        # fetching kline history for all pairs asynchronously and wait till all done
+        data = asyncio.get_event_loop().run_until_complete(self.exchange.async_get_tickers_history(whitelist, self.strategy.ticker_interval))
+        
+        # list of pairs having buy signals
+        buy_pairs = []
 
-            if buy and not sell:
-                return self.execute_buy(_pair, stake_amount)
+        # running get_signal on historical data fetched 
+        # to find buy signals
+        for _pair, thistory in data: 
+            (buy, sell) = self.strategy.get_signal(_pair, interval, thistory)
+            if buy and not sell: 
+                buy_pairs.append(_pair)
+
+        # If there is at least one buy signal then 
+        # Go ahead and buy the first pair 
+        if buy_pairs:
+            return self.execute_buy(buy_pairs[0], stake_amount)
+        
         return False
 
     def execute_buy(self, pair: str, stake_amount: float) -> bool:

From a486b1d01c631fe0b4c0334fc3e6e7fe61e84206 Mon Sep 17 00:00:00 2001
From: Matthias <xmatthias@outlook.com>
Date: Tue, 31 Jul 2018 20:25:10 +0200
Subject: [PATCH 02/47] Use Dict instead of tuplelist, run in _process

---
 freqtrade/freqtradebot.py | 25 +++++++++++++++----------
 1 file changed, 15 insertions(+), 10 deletions(-)

diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py
index 93977ab16..e17498b51 100644
--- a/freqtrade/freqtradebot.py
+++ b/freqtrade/freqtradebot.py
@@ -149,6 +149,10 @@ class FreqtradeBot(object):
             final_list = sanitized_list[:nb_assets] if nb_assets else sanitized_list
             self.config['exchange']['pair_whitelist'] = final_list
 
+            datatups = asyncio.get_event_loop().run_until_complete(
+                self.exchange.async_get_tickers_history(final_list, self.strategy.ticker_interval))
+            self._klines = {pair: data for (pair, data) in datatups}
+
             # Query trades from persistence layer
             trades = Trade.query.filter(Trade.is_open.is_(True)).all()
 
@@ -337,25 +341,25 @@ class FreqtradeBot(object):
         if not whitelist:
             raise DependencyException('No currency pairs in whitelist')
 
-        
+
         # fetching kline history for all pairs asynchronously and wait till all done
-        data = asyncio.get_event_loop().run_until_complete(self.exchange.async_get_tickers_history(whitelist, self.strategy.ticker_interval))
-        
+        # data = asyncio.get_event_loop().run_until_complete(self.exchange.async_get_tickers_history(whitelist, self.strategy.ticker_interval))
+
         # list of pairs having buy signals
         buy_pairs = []
 
-        # running get_signal on historical data fetched 
+        # running get_signal on historical data fetched
         # to find buy signals
-        for _pair, thistory in data: 
+        for _pair, thistory in self._klines.items():
             (buy, sell) = self.strategy.get_signal(_pair, interval, thistory)
-            if buy and not sell: 
+            if buy and not sell:
                 buy_pairs.append(_pair)
 
-        # If there is at least one buy signal then 
-        # Go ahead and buy the first pair 
+        # If there is at least one buy signal then
+        # Go ahead and buy the first pair
         if buy_pairs:
             return self.execute_buy(buy_pairs[0], stake_amount)
-        
+
         return False
 
     def execute_buy(self, pair: str, stake_amount: float) -> bool:
@@ -518,7 +522,8 @@ class FreqtradeBot(object):
         (buy, sell) = (False, False)
         experimental = self.config.get('experimental', {})
         if experimental.get('use_sell_signal') or experimental.get('ignore_roi_if_buy_signal'):
-            ticker = self.exchange.get_ticker_history(trade.pair, self.strategy.ticker_interval)
+            # ticker = self.exchange.get_ticker_history(trade.pair, self.strategy.ticker_interval)
+            ticker = self._klines[trade.pair]
             (buy, sell) = self.strategy.get_signal(trade.pair, self.strategy.ticker_interval,
                                                    ticker)
 

From 31870abd251f1cf19fd0acf8890fb69443445006 Mon Sep 17 00:00:00 2001
From: Matthias <xmatthias@outlook.com>
Date: Tue, 31 Jul 2018 20:43:32 +0200
Subject: [PATCH 03/47] Refactor async-refresh to it's own function

---
 freqtrade/exchange/__init__.py | 28 +++++++++++++++++-----------
 freqtrade/freqtradebot.py      | 21 ++++-----------------
 2 files changed, 21 insertions(+), 28 deletions(-)

diff --git a/freqtrade/exchange/__init__.py b/freqtrade/exchange/__init__.py
index abbc9808b..8c5793b83 100644
--- a/freqtrade/exchange/__init__.py
+++ b/freqtrade/exchange/__init__.py
@@ -2,7 +2,7 @@
 """ Cryptocurrency Exchanges support """
 import logging
 from random import randint
-from typing import List, Dict, Any, Optional
+from typing import List, Dict, Tuple, Any, Optional
 from datetime import datetime
 from math import floor, ceil
 
@@ -95,7 +95,7 @@ class Exchange(object):
                 'secret': exchange_config.get('secret'),
                 'password': exchange_config.get('password'),
                 'uid': exchange_config.get('uid', ''),
-                #'enableRateLimit': True,
+                # 'enableRateLimit': True,
                 'enableRateLimit': False,
             })
         except (KeyError, AttributeError):
@@ -334,23 +334,23 @@ class Exchange(object):
             logger.info("returning cached ticker-data for %s", pair)
             return self._cached_ticker[pair]
 
-
     async def async_get_tickers_history(self, pairs, tick_interval):
-        # COMMENTED CODE IS FOR DISCUSSION: where should we close the loop on async ? 
-        #loop = asyncio.new_event_loop()
-        #asyncio.set_event_loop(loop)
-        input_coroutines = [self.async_get_ticker_history(symbol, tick_interval) for symbol in pairs]
+        # COMMENTED CODE IS FOR DISCUSSION: where should we close the loop on async ?
+        # loop = asyncio.new_event_loop()
+        # asyncio.set_event_loop(loop)
+        input_coroutines = [self.async_get_ticker_history(
+            symbol, tick_interval) for symbol in pairs]
         tickers = await asyncio.gather(*input_coroutines, return_exceptions=True)
-        #await self._api_async.close()
+        # await self._api_async.close()
         return tickers
 
     async def async_get_ticker_history(self, pair: str, tick_interval: str,
-                           since_ms: Optional[int] = None) -> List[Dict]:
+                                       since_ms: Optional[int] = None) -> Tuple[str, List]:
         try:
             # fetch ohlcv asynchronously
-            print("fetching %s ..." % pair)
+            logger.debug("fetching %s ...",  pair)
             data = await self._api_async.fetch_ohlcv(pair, timeframe=tick_interval, since=since_ms)
-            print("done fetching %s ..." % pair)
+            logger.debug("done fetching %s ...", pair)
             return pair, data
 
         except ccxt.NotSupported as e:
@@ -363,6 +363,12 @@ class Exchange(object):
         except ccxt.BaseError as e:
             raise OperationalException(f'Could not fetch ticker data. Msg: {e}')
 
+    def refresh_tickers(self, pair_list: List[str], ticker_interval: str) -> Dict:
+        logger.debug("Refreshing klines for %d pairs", len(pair_list))
+        datatups = asyncio.get_event_loop().run_until_complete(
+            self.async_get_tickers_history(pair_list, ticker_interval))
+        return {pair: data for (pair, data) in datatups}
+
     @retrier
     def get_ticker_history(self, pair: str, tick_interval: str,
                            since_ms: Optional[int] = None) -> List[Dict]:
diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py
index e17498b51..eaa2184ac 100644
--- a/freqtrade/freqtradebot.py
+++ b/freqtrade/freqtradebot.py
@@ -8,7 +8,6 @@ import time
 import traceback
 from datetime import datetime
 from typing import Any, Callable, Dict, List, Optional
-import asyncio
 
 import arrow
 import requests
@@ -149,9 +148,7 @@ class FreqtradeBot(object):
             final_list = sanitized_list[:nb_assets] if nb_assets else sanitized_list
             self.config['exchange']['pair_whitelist'] = final_list
 
-            datatups = asyncio.get_event_loop().run_until_complete(
-                self.exchange.async_get_tickers_history(final_list, self.strategy.ticker_interval))
-            self._klines = {pair: data for (pair, data) in datatups}
+            self._klines = self.exchange.refresh_tickers(final_list, self.strategy.ticker_interval)
 
             # Query trades from persistence layer
             trades = Trade.query.filter(Trade.is_open.is_(True)).all()
@@ -306,13 +303,7 @@ class FreqtradeBot(object):
             amount_reserve_percent += self.strategy.stoploss
         # it should not be more than 50%
         amount_reserve_percent = max(amount_reserve_percent, 0.5)
-        return min(min_stake_amounts)/amount_reserve_percent
-
-    async def async_get_tickers(self, exchange, pairs):
-        input_coroutines = [exchange.async_get_ticker_history(symbol, self.strategy.ticker_interval) for symbol in pairs]
-        tickers = await asyncio.gather(*input_coroutines, return_exceptions=True)
-        return tickers
-        #await exchange.close()
+        return min(min_stake_amounts) / amount_reserve_percent
 
     def create_trade(self) -> bool:
         """
@@ -341,17 +332,13 @@ class FreqtradeBot(object):
         if not whitelist:
             raise DependencyException('No currency pairs in whitelist')
 
-
-        # fetching kline history for all pairs asynchronously and wait till all done
-        # data = asyncio.get_event_loop().run_until_complete(self.exchange.async_get_tickers_history(whitelist, self.strategy.ticker_interval))
-
         # list of pairs having buy signals
         buy_pairs = []
 
         # running get_signal on historical data fetched
         # to find buy signals
-        for _pair, thistory in self._klines.items():
-            (buy, sell) = self.strategy.get_signal(_pair, interval, thistory)
+        for _pair in whitelist:
+            (buy, sell) = self.strategy.get_signal(_pair, interval, self._klines[_pair])
             if buy and not sell:
                 buy_pairs.append(_pair)
 

From b45d465ed833264aaf1832b8f048fbbd4c6fc306 Mon Sep 17 00:00:00 2001
From: Matthias <xmatthias@outlook.com>
Date: Tue, 31 Jul 2018 20:50:59 +0200
Subject: [PATCH 04/47] init _klines properly

---
 freqtrade/freqtradebot.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py
index eaa2184ac..4313b310d 100644
--- a/freqtrade/freqtradebot.py
+++ b/freqtrade/freqtradebot.py
@@ -55,6 +55,7 @@ class FreqtradeBot(object):
         self.persistence = None
         self.exchange = Exchange(self.config)
         self._init_modules()
+        self._klines = {}
 
     def _init_modules(self) -> None:
         """
@@ -338,7 +339,7 @@ class FreqtradeBot(object):
         # running get_signal on historical data fetched
         # to find buy signals
         for _pair in whitelist:
-            (buy, sell) = self.strategy.get_signal(_pair, interval, self._klines[_pair])
+            (buy, sell) = self.strategy.get_signal(_pair, interval, self._klines.get(_pair))
             if buy and not sell:
                 buy_pairs.append(_pair)
 

From 52065178e16ad2e84a1774e73e6de5b878f3dbe1 Mon Sep 17 00:00:00 2001
From: Matthias <xmatthias@outlook.com>
Date: Tue, 31 Jul 2018 20:53:32 +0200
Subject: [PATCH 05/47] use .get all the time

---
 freqtrade/freqtradebot.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py
index 4313b310d..365d145ca 100644
--- a/freqtrade/freqtradebot.py
+++ b/freqtrade/freqtradebot.py
@@ -511,7 +511,7 @@ class FreqtradeBot(object):
         experimental = self.config.get('experimental', {})
         if experimental.get('use_sell_signal') or experimental.get('ignore_roi_if_buy_signal'):
             # ticker = self.exchange.get_ticker_history(trade.pair, self.strategy.ticker_interval)
-            ticker = self._klines[trade.pair]
+            ticker = self._klines.get(trade.pair)
             (buy, sell) = self.strategy.get_signal(trade.pair, self.strategy.ticker_interval,
                                                    ticker)
 

From 12417cc303715244dc1701d13b5d5ed77d12b6ed Mon Sep 17 00:00:00 2001
From: Matthias <xmatthias@outlook.com>
Date: Tue, 31 Jul 2018 20:54:51 +0200
Subject: [PATCH 06/47] fix tests

---
 freqtrade/tests/test_freqtradebot.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/freqtrade/tests/test_freqtradebot.py b/freqtrade/tests/test_freqtradebot.py
index 69f349107..fc84464e4 100644
--- a/freqtrade/tests/test_freqtradebot.py
+++ b/freqtrade/tests/test_freqtradebot.py
@@ -44,6 +44,7 @@ def patch_get_signal(freqtrade: FreqtradeBot, value=(True, False)) -> None:
     """
     freqtrade.strategy.get_signal = lambda e, s, t: value
     freqtrade.exchange.get_ticker_history = lambda p, i: None
+    freqtrade.exchange.refresh_tickers = lambda pl, i: {}
 
 
 def patch_RPCManager(mocker) -> MagicMock:

From 136442245c850c83bb314230306a27ef1924992d Mon Sep 17 00:00:00 2001
From: Matthias <xmatthias@outlook.com>
Date: Tue, 31 Jul 2018 21:01:44 +0200
Subject: [PATCH 07/47] Add todo's and dockstring

---
 freqtrade/exchange/__init__.py | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/freqtrade/exchange/__init__.py b/freqtrade/exchange/__init__.py
index 8c5793b83..8e88a7f5f 100644
--- a/freqtrade/exchange/__init__.py
+++ b/freqtrade/exchange/__init__.py
@@ -364,6 +364,12 @@ class Exchange(object):
             raise OperationalException(f'Could not fetch ticker data. Msg: {e}')
 
     def refresh_tickers(self, pair_list: List[str], ticker_interval: str) -> Dict:
+        """
+        Refresh tickers asyncronously and return the result.
+        """
+        # TODO: maybe add since_ms to use async in the download-script?
+        # TODO: only refresh once per interval ? *may require this to move to freqtradebot.py
+        # TODO@ Add tests for this and the async stuff above
         logger.debug("Refreshing klines for %d pairs", len(pair_list))
         datatups = asyncio.get_event_loop().run_until_complete(
             self.async_get_tickers_history(pair_list, ticker_interval))

From c466a028e00341965acff78143f203a7a2abb13e Mon Sep 17 00:00:00 2001
From: Matthias <xmatthias@outlook.com>
Date: Wed, 1 Aug 2018 21:19:49 +0200
Subject: [PATCH 08/47] Add a first async test

---
 .travis.yml                               |  4 +-
 freqtrade/tests/exchange/test_exchange.py | 53 +++++++++++++++++++++++
 requirements.txt                          |  1 +
 3 files changed, 56 insertions(+), 2 deletions(-)

diff --git a/.travis.yml b/.travis.yml
index 88121945f..981eedcf8 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -13,12 +13,12 @@ addons:
 install:
 - ./install_ta-lib.sh
 - export LD_LIBRARY_PATH=/usr/local/lib:$LD_LIBRARY_PATH
-- pip install --upgrade flake8 coveralls pytest-random-order mypy
+- pip install --upgrade flake8 coveralls pytest-random-order pytest-asyncio mypy
 - pip install -r requirements.txt
 - pip install -e .
 jobs:
   include:
-    - script: 
+    - script:
       - pytest --cov=freqtrade --cov-config=.coveragerc freqtrade/tests/
       - coveralls
     - script:
diff --git a/freqtrade/tests/exchange/test_exchange.py b/freqtrade/tests/exchange/test_exchange.py
index d327b97c7..b2c3b6a2e 100644
--- a/freqtrade/tests/exchange/test_exchange.py
+++ b/freqtrade/tests/exchange/test_exchange.py
@@ -27,6 +27,20 @@ def ccxt_exceptionhandlers(mocker, default_conf, api_mock, fun, mock_ccxt_fun, *
     assert api_mock.__dict__[mock_ccxt_fun].call_count == 1
 
 
+async def async_ccxt_exception(mocker, default_conf, api_mock, fun, mock_ccxt_fun, **kwargs):
+    with pytest.raises(TemporaryError):
+        api_mock.__dict__[mock_ccxt_fun] = MagicMock(side_effect=ccxt.NetworkError)
+        exchange = get_patched_exchange(mocker, default_conf, api_mock)
+        await getattr(exchange, fun)(**kwargs)
+    assert api_mock.__dict__[mock_ccxt_fun].call_count == 1
+
+    with pytest.raises(OperationalException):
+        api_mock.__dict__[mock_ccxt_fun] = MagicMock(side_effect=ccxt.BaseError)
+        exchange = get_patched_exchange(mocker, default_conf, api_mock)
+        await getattr(exchange, fun)(**kwargs)
+    assert api_mock.__dict__[mock_ccxt_fun].call_count == 1
+
+
 def test_init(default_conf, mocker, caplog):
     caplog.set_level(logging.INFO)
     get_patched_exchange(mocker, default_conf)
@@ -515,6 +529,45 @@ def test_get_ticker(default_conf, mocker):
     exchange.get_ticker(pair='ETH/BTC', refresh=True)
 
 
+@pytest.mark.asyncio
+async def test_async_get_ticker_history(default_conf, mocker):
+    tick = [
+        [
+            1511686200000,  # unix timestamp ms
+            1,  # open
+            2,  # high
+            3,  # low
+            4,  # close
+            5,  # volume (in quote currency)
+        ]
+    ]
+
+    async def async_fetch_ohlcv(pair, timeframe, since):
+        return tick
+
+    exchange = get_patched_exchange(mocker, default_conf)
+    # Monkey-patch async function
+    exchange._api_async.fetch_ohlcv = async_fetch_ohlcv
+
+    exchange = Exchange(default_conf)
+    pair = 'ETH/BTC'
+    res = await exchange.async_get_ticker_history(pair, "5m")
+    assert type(res) is tuple
+    assert len(res) == 2
+    assert res[0] == pair
+    assert res[1] == tick
+
+    await async_ccxt_exception(mocker, default_conf, MagicMock(),
+                               "async_get_ticker_history", "fetch_ohlcv",
+                               pair='ABCD/BTC', tick_interval=default_conf['ticker_interval'])
+
+    api_mock = MagicMock()
+    with pytest.raises(OperationalException, match=r'Could not fetch ticker data*'):
+        api_mock.fetch_ohlcv = MagicMock(side_effect=ccxt.BaseError)
+        exchange = get_patched_exchange(mocker, default_conf, api_mock)
+        await exchange.async_get_ticker_history(pair, "5m")
+
+
 def make_fetch_ohlcv_mock(data):
     def fetch_ohlcv_mock(pair, timeframe, since):
         if since:
diff --git a/requirements.txt b/requirements.txt
index 964da51e3..57dcf78e9 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -15,6 +15,7 @@ TA-Lib==0.4.17
 pytest==3.6.4
 pytest-mock==1.10.0
 pytest-cov==2.5.1
+pytest-asyncio==0.9.0
 tabulate==0.8.2
 coinmarketcap==5.0.3
 

From 915160f21f88730ebcd0b09571e8c978078c9053 Mon Sep 17 00:00:00 2001
From: Matthias <xmatthias@outlook.com>
Date: Wed, 1 Aug 2018 21:44:02 +0200
Subject: [PATCH 09/47] Add tests for tickers-history

---
 freqtrade/exchange/__init__.py            |  2 +-
 freqtrade/tests/exchange/test_exchange.py | 47 +++++++++++++++++++++++
 2 files changed, 48 insertions(+), 1 deletion(-)

diff --git a/freqtrade/exchange/__init__.py b/freqtrade/exchange/__init__.py
index 8e88a7f5f..522636d22 100644
--- a/freqtrade/exchange/__init__.py
+++ b/freqtrade/exchange/__init__.py
@@ -334,7 +334,7 @@ class Exchange(object):
             logger.info("returning cached ticker-data for %s", pair)
             return self._cached_ticker[pair]
 
-    async def async_get_tickers_history(self, pairs, tick_interval):
+    async def async_get_tickers_history(self, pairs, tick_interval) -> List[Tuple[str, List]]:
         # COMMENTED CODE IS FOR DISCUSSION: where should we close the loop on async ?
         # loop = asyncio.new_event_loop()
         # asyncio.set_event_loop(loop)
diff --git a/freqtrade/tests/exchange/test_exchange.py b/freqtrade/tests/exchange/test_exchange.py
index b2c3b6a2e..cf62a48c8 100644
--- a/freqtrade/tests/exchange/test_exchange.py
+++ b/freqtrade/tests/exchange/test_exchange.py
@@ -568,6 +568,53 @@ async def test_async_get_ticker_history(default_conf, mocker):
         await exchange.async_get_ticker_history(pair, "5m")
 
 
+@pytest.mark.asyncio
+async def test_async_get_tickers_history(default_conf, mocker):
+    tick = [
+        [
+            1511686200000,  # unix timestamp ms
+            1,  # open
+            2,  # high
+            3,  # low
+            4,  # close
+            5,  # volume (in quote currency)
+        ]
+    ]
+
+    async def async_fetch_ohlcv(pair, timeframe, since):
+        return tick
+
+    exchange = get_patched_exchange(mocker, default_conf)
+    # Monkey-patch async function
+    exchange._api_async.fetch_ohlcv = async_fetch_ohlcv
+
+    exchange = Exchange(default_conf)
+    pairs = ['ETH/BTC', 'XRP/BTC']
+    res = await exchange.async_get_tickers_history(pairs, "5m")
+    assert type(res) is list
+    assert len(res) == 2
+    assert type(res[0]) is tuple
+    assert res[0][0] == pairs[0]
+    assert res[0][1] == tick
+    assert res[1][0] == pairs[1]
+    assert res[1][1] == tick
+
+    # await async_ccxt_exception(mocker, default_conf, MagicMock(),
+    #                            "async_get_tickers_history", "fetch_ohlcv",
+    #                            pairs=pairs, tick_interval=default_conf['ticker_interval'])
+
+    # api_mock = MagicMock()
+    # with pytest.raises(OperationalException, match=r'Could not fetch ticker data*'):
+    #     api_mock.fetch_ohlcv = MagicMock(side_effect=ccxt.BaseError)
+    #     exchange = get_patched_exchange(mocker, default_conf, api_mock)
+    #     await exchange.async_get_tickers_history('ETH/BTC', "5m")
+
+
+def test_refresh_tickers():
+    # TODO: Implement test for this
+    pass
+
+
 def make_fetch_ohlcv_mock(data):
     def fetch_ohlcv_mock(pair, timeframe, since):
         if since:

From 9c08cdc81dec53be85d1d58f41ec56008e46f319 Mon Sep 17 00:00:00 2001
From: Matthias <xmatthias@outlook.com>
Date: Wed, 1 Aug 2018 21:58:32 +0200
Subject: [PATCH 10/47] Fix typehints

---
 freqtrade/freqtradebot.py       | 2 +-
 freqtrade/strategy/interface.py | 4 ++--
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py
index 365d145ca..f53bc2836 100644
--- a/freqtrade/freqtradebot.py
+++ b/freqtrade/freqtradebot.py
@@ -55,7 +55,7 @@ class FreqtradeBot(object):
         self.persistence = None
         self.exchange = Exchange(self.config)
         self._init_modules()
-        self._klines = {}
+        self._klines: Dict[str, List[Dict]] = {}
 
     def _init_modules(self) -> None:
         """
diff --git a/freqtrade/strategy/interface.py b/freqtrade/strategy/interface.py
index dfd624393..488281eee 100644
--- a/freqtrade/strategy/interface.py
+++ b/freqtrade/strategy/interface.py
@@ -6,7 +6,7 @@ import logging
 from abc import ABC, abstractmethod
 from datetime import datetime
 from enum import Enum
-from typing import Dict, List, NamedTuple, Tuple
+from typing import Dict, List, NamedTuple, Optional, Tuple
 import warnings
 
 import arrow
@@ -118,7 +118,7 @@ class IStrategy(ABC):
         dataframe = self.advise_sell(dataframe, metadata)
         return dataframe
 
-    def get_signal(self, pair: str, interval: str, ticker_hist: List[Dict]) -> Tuple[bool, bool]:
+    def get_signal(self, pair: str, interval: str, ticker_hist: Optional[List[Dict]]) -> Tuple[bool, bool]:
         """
         Calculates current signal based several technical analysis indicators
         :param pair: pair in format ANT/BTC

From 05ca78d2a3b9473b43836d3aab8d7f3335a27537 Mon Sep 17 00:00:00 2001
From: misagh <misagh.shakeri@gmail.com>
Date: Thu, 2 Aug 2018 17:10:38 +0200
Subject: [PATCH 11/47] ticker_history changed to candle_history naming

---
 freqtrade/tests/exchange/test_exchange.py | 16 ++++++++--------
 1 file changed, 8 insertions(+), 8 deletions(-)

diff --git a/freqtrade/tests/exchange/test_exchange.py b/freqtrade/tests/exchange/test_exchange.py
index 7cbead128..3a7573f70 100644
--- a/freqtrade/tests/exchange/test_exchange.py
+++ b/freqtrade/tests/exchange/test_exchange.py
@@ -530,7 +530,7 @@ def test_get_ticker(default_conf, mocker):
 
 
 @pytest.mark.asyncio
-async def test_async_get_ticker_history(default_conf, mocker):
+async def test_async_get_candle_history(default_conf, mocker):
     tick = [
         [
             1511686200000,  # unix timestamp ms
@@ -551,25 +551,25 @@ async def test_async_get_ticker_history(default_conf, mocker):
 
     exchange = Exchange(default_conf)
     pair = 'ETH/BTC'
-    res = await exchange.async_get_ticker_history(pair, "5m")
+    res = await exchange.async_get_candle_history(pair, "5m")
     assert type(res) is tuple
     assert len(res) == 2
     assert res[0] == pair
     assert res[1] == tick
 
     await async_ccxt_exception(mocker, default_conf, MagicMock(),
-                               "async_get_ticker_history", "fetch_ohlcv",
+                               "async_get_candle_history", "fetch_ohlcv",
                                pair='ABCD/BTC', tick_interval=default_conf['ticker_interval'])
 
     api_mock = MagicMock()
     with pytest.raises(OperationalException, match=r'Could not fetch ticker data*'):
         api_mock.fetch_ohlcv = MagicMock(side_effect=ccxt.BaseError)
         exchange = get_patched_exchange(mocker, default_conf, api_mock)
-        await exchange.async_get_ticker_history(pair, "5m")
+        await exchange.async_get_candle_history(pair, "5m")
 
 
 @pytest.mark.asyncio
-async def test_async_get_tickers_history(default_conf, mocker):
+async def test_async_get_candles_history(default_conf, mocker):
     tick = [
         [
             1511686200000,  # unix timestamp ms
@@ -590,7 +590,7 @@ async def test_async_get_tickers_history(default_conf, mocker):
 
     exchange = Exchange(default_conf)
     pairs = ['ETH/BTC', 'XRP/BTC']
-    res = await exchange.async_get_tickers_history(pairs, "5m")
+    res = await exchange.async_get_candles_history(pairs, "5m")
     assert type(res) is list
     assert len(res) == 2
     assert type(res[0]) is tuple
@@ -600,14 +600,14 @@ async def test_async_get_tickers_history(default_conf, mocker):
     assert res[1][1] == tick
 
     # await async_ccxt_exception(mocker, default_conf, MagicMock(),
-    #                            "async_get_tickers_history", "fetch_ohlcv",
+    #                            "async_get_candles_history", "fetch_ohlcv",
     #                            pairs=pairs, tick_interval=default_conf['ticker_interval'])
 
     # api_mock = MagicMock()
     # with pytest.raises(OperationalException, match=r'Could not fetch ticker data*'):
     #     api_mock.fetch_ohlcv = MagicMock(side_effect=ccxt.BaseError)
     #     exchange = get_patched_exchange(mocker, default_conf, api_mock)
-    #     await exchange.async_get_tickers_history('ETH/BTC', "5m")
+    #     await exchange.async_get_candles_history('ETH/BTC', "5m")
 
 
 def test_refresh_tickers():

From 337d9174d93aac86ec1e33110d020996fd29dff9 Mon Sep 17 00:00:00 2001
From: Matthias <xmatthias@outlook.com>
Date: Thu, 2 Aug 2018 20:11:27 +0200
Subject: [PATCH 12/47] Flake8 fixes

---
 freqtrade/strategy/interface.py      | 3 ++-
 freqtrade/tests/test_freqtradebot.py | 1 -
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/freqtrade/strategy/interface.py b/freqtrade/strategy/interface.py
index 488281eee..6d7b8dba7 100644
--- a/freqtrade/strategy/interface.py
+++ b/freqtrade/strategy/interface.py
@@ -118,7 +118,8 @@ class IStrategy(ABC):
         dataframe = self.advise_sell(dataframe, metadata)
         return dataframe
 
-    def get_signal(self, pair: str, interval: str, ticker_hist: Optional[List[Dict]]) -> Tuple[bool, bool]:
+    def get_signal(self, pair: str, interval: str,
+                   ticker_hist: Optional[List[Dict]]) -> Tuple[bool, bool]:
         """
         Calculates current signal based several technical analysis indicators
         :param pair: pair in format ANT/BTC
diff --git a/freqtrade/tests/test_freqtradebot.py b/freqtrade/tests/test_freqtradebot.py
index f3cb2be3f..3b72c3521 100644
--- a/freqtrade/tests/test_freqtradebot.py
+++ b/freqtrade/tests/test_freqtradebot.py
@@ -47,7 +47,6 @@ def patch_get_signal(freqtrade: FreqtradeBot, value=(True, False)) -> None:
     freqtrade.exchange.refresh_tickers = lambda pl, i: {}
 
 
-
 def patch_RPCManager(mocker) -> MagicMock:
     """
     This function mock RPC manager to avoid repeating this code in almost every tests

From 59b9a6d94d57db1addfe11c07ceb116401cda9db Mon Sep 17 00:00:00 2001
From: misagh <misagh.shakeri@gmail.com>
Date: Fri, 3 Aug 2018 14:49:55 +0200
Subject: [PATCH 13/47] Break the loop as soon as one buy signal is found.

---
 freqtrade/freqtradebot.py | 10 +---------
 1 file changed, 1 insertion(+), 9 deletions(-)

diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py
index 4df3a327a..872f6dc5c 100644
--- a/freqtrade/freqtradebot.py
+++ b/freqtrade/freqtradebot.py
@@ -333,20 +333,12 @@ class FreqtradeBot(object):
         if not whitelist:
             raise DependencyException('No currency pairs in whitelist')
 
-        # list of pairs having buy signals
-        buy_pairs = []
-
         # running get_signal on historical data fetched
         # to find buy signals
         for _pair in whitelist:
             (buy, sell) = self.strategy.get_signal(_pair, interval, self._klines.get(_pair))
             if buy and not sell:
-                buy_pairs.append(_pair)
-
-        # If there is at least one buy signal then
-        # Go ahead and buy the first pair
-        if buy_pairs:
-            return self.execute_buy(buy_pairs[0], stake_amount)
+                return self.execute_buy(_pair, stake_amount)
 
         return False
 

From af93b18475f0eaaaa25996fc8ee92ca7e20c0956 Mon Sep 17 00:00:00 2001
From: misagh <misagh.shakeri@gmail.com>
Date: Fri, 3 Aug 2018 18:10:03 +0200
Subject: [PATCH 14/47] Do not refresh candles on "process_throttle_secs" but
 on intervals

---
 freqtrade/exchange/__init__.py | 15 ++-------------
 freqtrade/freqtradebot.py      | 33 ++++++++++++++++++++++++++++++++-
 2 files changed, 34 insertions(+), 14 deletions(-)

diff --git a/freqtrade/exchange/__init__.py b/freqtrade/exchange/__init__.py
index 897cb547a..46a56860b 100644
--- a/freqtrade/exchange/__init__.py
+++ b/freqtrade/exchange/__init__.py
@@ -6,10 +6,11 @@ from typing import List, Dict, Tuple, Any, Optional
 from datetime import datetime
 from math import floor, ceil
 
+import asyncio
 import ccxt
 import ccxt.async_support as ccxt_async
 import arrow
-import asyncio
+
 
 from freqtrade import constants, OperationalException, DependencyException, TemporaryError
 
@@ -362,18 +363,6 @@ class Exchange(object):
         except ccxt.BaseError as e:
             raise OperationalException(f'Could not fetch ticker data. Msg: {e}')
 
-    def refresh_tickers(self, pair_list: List[str], ticker_interval: str) -> Dict:
-        """
-        Refresh tickers asyncronously and return the result.
-        """
-        # TODO: maybe add since_ms to use async in the download-script?
-        # TODO: only refresh once per interval ? *may require this to move to freqtradebot.py
-        # TODO: Add tests for this and the async stuff above
-        logger.debug("Refreshing klines for %d pairs", len(pair_list))
-        datatups = asyncio.get_event_loop().run_until_complete(
-            self.async_get_candles_history(pair_list, ticker_interval))
-        return {pair: data for (pair, data) in datatups}
-
     @retrier
     def get_candle_history(self, pair: str, tick_interval: str,
                            since_ms: Optional[int] = None) -> List[Dict]:
diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py
index 872f6dc5c..854b42c69 100644
--- a/freqtrade/freqtradebot.py
+++ b/freqtrade/freqtradebot.py
@@ -9,6 +9,7 @@ import traceback
 from datetime import datetime
 from typing import Any, Callable, Dict, List, Optional
 
+import asyncio
 import arrow
 import requests
 
@@ -56,6 +57,7 @@ class FreqtradeBot(object):
         self.exchange = Exchange(self.config)
         self._init_modules()
         self._klines: Dict[str, List[Dict]] = {}
+        self._klines_last_fetched_time = 0
 
     def _init_modules(self) -> None:
         """
@@ -129,6 +131,34 @@ class FreqtradeBot(object):
         time.sleep(duration)
         return result
 
+    def refresh_tickers(self, pair_list: List[str]) -> Dict:
+        """
+        Refresh tickers asyncronously and return the result.
+        """
+        # TODO: maybe add since_ms to use async in the download-script?
+        # TODO: only refresh once per interval ? *may require this to move to freqtradebot.py
+        # TODO: Add tests for this and the async stuff above
+         
+        ticker_interval = self.strategy.ticker_interval
+        interval_in_seconds = int(ticker_interval[:-1]) * 60
+
+        should_not_update = ((self._klines_last_fetched_time + interval_in_seconds +1) > round(time.time()))
+
+        if should_not_update:
+            return False
+
+        logger.debug("Refreshing klines for %d pairs", len(pair_list))
+        datatups = asyncio.get_event_loop().run_until_complete(
+            self.exchange.async_get_candles_history(pair_list, ticker_interval))
+
+        # fetching the timestamp of last candle
+        self._klines_last_fetched_time = datatups[0][1][-1][0] / 1000
+
+        # updating klines
+        self._klines = {pair: data for (pair, data) in datatups}
+
+        return True
+
     def _process(self, nb_assets: Optional[int] = 0) -> bool:
         """
         Queries the persistence layer for open trades and handles them,
@@ -149,7 +179,8 @@ class FreqtradeBot(object):
             final_list = sanitized_list[:nb_assets] if nb_assets else sanitized_list
             self.config['exchange']['pair_whitelist'] = final_list
 
-            self._klines = self.exchange.refresh_tickers(final_list, self.strategy.ticker_interval)
+            # Refreshing candles
+            self.refresh_tickers(final_list)
 
             # Query trades from persistence layer
             trades = Trade.query.filter(Trade.is_open.is_(True)).all()

From 3ce4d20ab9932b3695ce3bf8e4106c6f19430524 Mon Sep 17 00:00:00 2001
From: misagh <misagh.shakeri@gmail.com>
Date: Sat, 4 Aug 2018 13:04:16 +0200
Subject: [PATCH 15/47] using constants instead of stripping the string

---
 freqtrade/freqtradebot.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py
index 854b42c69..c14411893 100644
--- a/freqtrade/freqtradebot.py
+++ b/freqtrade/freqtradebot.py
@@ -140,7 +140,7 @@ class FreqtradeBot(object):
         # TODO: Add tests for this and the async stuff above
          
         ticker_interval = self.strategy.ticker_interval
-        interval_in_seconds = int(ticker_interval[:-1]) * 60
+        interval_in_seconds = constants.TICKER_INTERVAL_MINUTES[ticker_interval] * 60
 
         should_not_update = ((self._klines_last_fetched_time + interval_in_seconds +1) > round(time.time()))
 

From 255f30385031003bd2915809fa8c3bd9fc769323 Mon Sep 17 00:00:00 2001
From: Matthias <xmatthias@outlook.com>
Date: Tue, 7 Aug 2018 08:56:06 +0200
Subject: [PATCH 16/47] Fix tests and flake8

---
 freqtrade/freqtradebot.py            | 8 ++++----
 freqtrade/tests/test_freqtradebot.py | 2 +-
 2 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py
index c14411893..f4e9c1d5b 100644
--- a/freqtrade/freqtradebot.py
+++ b/freqtrade/freqtradebot.py
@@ -131,18 +131,18 @@ class FreqtradeBot(object):
         time.sleep(duration)
         return result
 
-    def refresh_tickers(self, pair_list: List[str]) -> Dict:
+    def refresh_tickers(self, pair_list: List[str]) -> bool:
         """
         Refresh tickers asyncronously and return the result.
         """
         # TODO: maybe add since_ms to use async in the download-script?
-        # TODO: only refresh once per interval ? *may require this to move to freqtradebot.py
         # TODO: Add tests for this and the async stuff above
-         
+
         ticker_interval = self.strategy.ticker_interval
         interval_in_seconds = constants.TICKER_INTERVAL_MINUTES[ticker_interval] * 60
 
-        should_not_update = ((self._klines_last_fetched_time + interval_in_seconds +1) > round(time.time()))
+        should_not_update = ((self._klines_last_fetched_time +
+                              interval_in_seconds + 1) > round(time.time()))
 
         if should_not_update:
             return False
diff --git a/freqtrade/tests/test_freqtradebot.py b/freqtrade/tests/test_freqtradebot.py
index 3b72c3521..3bf6ad037 100644
--- a/freqtrade/tests/test_freqtradebot.py
+++ b/freqtrade/tests/test_freqtradebot.py
@@ -44,7 +44,7 @@ def patch_get_signal(freqtrade: FreqtradeBot, value=(True, False)) -> None:
     """
     freqtrade.strategy.get_signal = lambda e, s, t: value
     freqtrade.exchange.get_candle_history = lambda p, i: None
-    freqtrade.exchange.refresh_tickers = lambda pl, i: {}
+    freqtrade.refresh_tickers = lambda i: True
 
 
 def patch_RPCManager(mocker) -> MagicMock:

From cef09f49a63676d451571007a202cd52bda42021 Mon Sep 17 00:00:00 2001
From: misagh <misagh.shakeri@gmail.com>
Date: Thu, 9 Aug 2018 11:51:38 +0200
Subject: [PATCH 17/47] wait for markets to be loaded before looping in
 symbols.

---
 freqtrade/exchange/__init__.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/freqtrade/exchange/__init__.py b/freqtrade/exchange/__init__.py
index 46a56860b..50d759936 100644
--- a/freqtrade/exchange/__init__.py
+++ b/freqtrade/exchange/__init__.py
@@ -338,6 +338,7 @@ class Exchange(object):
         # COMMENTED CODE IS FOR DISCUSSION: where should we close the loop on async ?
         # loop = asyncio.new_event_loop()
         # asyncio.set_event_loop(loop)
+        await self._api_async.load_markets()
         input_coroutines = [self.async_get_candle_history(
             symbol, tick_interval) for symbol in pairs]
         tickers = await asyncio.gather(*input_coroutines, return_exceptions=True)

From cb2608522914df2af27f30c51e3a0b6a452e6a54 Mon Sep 17 00:00:00 2001
From: misagh <misagh.shakeri@gmail.com>
Date: Thu, 9 Aug 2018 12:47:26 +0200
Subject: [PATCH 18/47] =?UTF-8?q?Moving=20should=5Fnot=5Fupdate=20logic=20?=
 =?UTF-8?q?to=20async=20function=20per=20pair.=20if=20there=20is=20no=20ne?=
 =?UTF-8?q?w=20candle,=20async=20function=20will=20just=20return=20the=20l?=
 =?UTF-8?q?ast=20cached=20candle=20locally=20and=20doesn=E2=80=99t=20hit?=
 =?UTF-8?q?=20the=20API?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 freqtrade/exchange/__init__.py | 28 ++++++++++++++++++++++++++--
 freqtrade/freqtradebot.py      | 15 +--------------
 2 files changed, 27 insertions(+), 16 deletions(-)

diff --git a/freqtrade/exchange/__init__.py b/freqtrade/exchange/__init__.py
index 50d759936..449a8270d 100644
--- a/freqtrade/exchange/__init__.py
+++ b/freqtrade/exchange/__init__.py
@@ -5,6 +5,7 @@ from random import randint
 from typing import List, Dict, Tuple, Any, Optional
 from datetime import datetime
 from math import floor, ceil
+import time
 
 import asyncio
 import ccxt
@@ -52,6 +53,12 @@ class Exchange(object):
     _conf: Dict = {}
     _cached_ticker: Dict[str, Any] = {}
 
+    # Holds last candle refreshed time of each pair
+    _pairs_last_refreshed_time = {}
+
+    # Holds candles
+    _cached_klines: Dict[str, Any] = {}
+
     # Holds all open sell orders for dry_run
     _dry_run_open_orders: Dict[str, Any] = {}
 
@@ -349,8 +356,25 @@ class Exchange(object):
                                        since_ms: Optional[int] = None) -> Tuple[str, List]:
         try:
             # fetch ohlcv asynchronously
-            logger.debug("fetching %s ...",  pair)
-            data = await self._api_async.fetch_ohlcv(pair, timeframe=tick_interval, since=since_ms)
+            logger.debug("fetching %s ...", pair)
+
+            # Calculating ticker interval in second
+            interval_in_seconds = constants.TICKER_INTERVAL_MINUTES[tick_interval] * 60
+
+            # If (last update time) + (interval in second) + (1 second) is greater than now
+            # that means we don't have to hit the API as there is no new candle
+            # so we fetch it from local cache
+            if self._pairs_last_refreshed_time.get(pair, 0) + interval_in_seconds + 1 > round(time.time()):
+                data = self._cached_klines[pair]
+            else: 
+                data = await self._api_async.fetch_ohlcv(pair, timeframe=tick_interval, since=since_ms)
+
+            # keeping last candle time as last refreshed time of the pair
+            self._pairs_last_refreshed_time[pair] = data[-1][0] / 1000
+
+            # keeping candles in cache
+            self._cached_klines[pair] = data
+
             logger.debug("done fetching %s ...", pair)
             return pair, data
 
diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py
index f4e9c1d5b..d552dc65f 100644
--- a/freqtrade/freqtradebot.py
+++ b/freqtrade/freqtradebot.py
@@ -137,22 +137,9 @@ class FreqtradeBot(object):
         """
         # TODO: maybe add since_ms to use async in the download-script?
         # TODO: Add tests for this and the async stuff above
-
-        ticker_interval = self.strategy.ticker_interval
-        interval_in_seconds = constants.TICKER_INTERVAL_MINUTES[ticker_interval] * 60
-
-        should_not_update = ((self._klines_last_fetched_time +
-                              interval_in_seconds + 1) > round(time.time()))
-
-        if should_not_update:
-            return False
-
         logger.debug("Refreshing klines for %d pairs", len(pair_list))
         datatups = asyncio.get_event_loop().run_until_complete(
-            self.exchange.async_get_candles_history(pair_list, ticker_interval))
-
-        # fetching the timestamp of last candle
-        self._klines_last_fetched_time = datatups[0][1][-1][0] / 1000
+            self.exchange.async_get_candles_history(pair_list, self.strategy.ticker_interval))
 
         # updating klines
         self._klines = {pair: data for (pair, data) in datatups}

From e654b76bc8ed24fe6bce239de1cfe6dd927de6b7 Mon Sep 17 00:00:00 2001
From: Matthias <xmatthias@outlook.com>
Date: Fri, 10 Aug 2018 09:44:03 +0200
Subject: [PATCH 19/47] Fix async test

---
 freqtrade/tests/exchange/test_exchange.py | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/freqtrade/tests/exchange/test_exchange.py b/freqtrade/tests/exchange/test_exchange.py
index 3a7573f70..ab25a5a7a 100644
--- a/freqtrade/tests/exchange/test_exchange.py
+++ b/freqtrade/tests/exchange/test_exchange.py
@@ -584,11 +584,15 @@ async def test_async_get_candles_history(default_conf, mocker):
     async def async_fetch_ohlcv(pair, timeframe, since):
         return tick
 
+    async def async_load_markets():
+        return {}
+
     exchange = get_patched_exchange(mocker, default_conf)
     # Monkey-patch async function
     exchange._api_async.fetch_ohlcv = async_fetch_ohlcv
 
-    exchange = Exchange(default_conf)
+    exchange._api_async.load_markets = async_load_markets
+
     pairs = ['ETH/BTC', 'XRP/BTC']
     res = await exchange.async_get_candles_history(pairs, "5m")
     assert type(res) is list

From 36f05af79ad112b6d7dcaf6ecc38cd1c91eddbed Mon Sep 17 00:00:00 2001
From: Matthias <xmatthias@outlook.com>
Date: Fri, 10 Aug 2018 09:44:15 +0200
Subject: [PATCH 20/47] sort fetch_olvhc result, refactor some

* add exception for since_ms - if this is set it should always download
---
 freqtrade/exchange/__init__.py | 22 +++++++++++++++-------
 1 file changed, 15 insertions(+), 7 deletions(-)

diff --git a/freqtrade/exchange/__init__.py b/freqtrade/exchange/__init__.py
index 449a8270d..3a6fdcf35 100644
--- a/freqtrade/exchange/__init__.py
+++ b/freqtrade/exchange/__init__.py
@@ -54,7 +54,7 @@ class Exchange(object):
     _cached_ticker: Dict[str, Any] = {}
 
     # Holds last candle refreshed time of each pair
-    _pairs_last_refreshed_time = {}
+    _pairs_last_refresh_time = {}
 
     # Holds candles
     _cached_klines: Dict[str, Any] = {}
@@ -359,18 +359,26 @@ class Exchange(object):
             logger.debug("fetching %s ...", pair)
 
             # Calculating ticker interval in second
-            interval_in_seconds = constants.TICKER_INTERVAL_MINUTES[tick_interval] * 60
+            interval_in_sec = constants.TICKER_INTERVAL_MINUTES[tick_interval] * 60
 
-            # If (last update time) + (interval in second) + (1 second) is greater than now
+            # If (last update time) + (interval in second) is greater or equal than now
             # that means we don't have to hit the API as there is no new candle
             # so we fetch it from local cache
-            if self._pairs_last_refreshed_time.get(pair, 0) + interval_in_seconds + 1 > round(time.time()):
+            if (not since_ms and
+                    self._pairs_last_refresh_time.get(pair, 0) + interval_in_sec >=
+                    int(time.time())):
                 data = self._cached_klines[pair]
-            else: 
-                data = await self._api_async.fetch_ohlcv(pair, timeframe=tick_interval, since=since_ms)
+            else:
+                data = await self._api_async.fetch_ohlcv(pair, timeframe=tick_interval,
+                                                         since=since_ms)
+
+            # Because some exchange sort Tickers ASC and other DESC.
+            # Ex: Bittrex returns a list of tickers ASC (oldest first, newest last)
+            # when GDAX returns a list of tickers DESC (newest first, oldest last)
+            data = sorted(data, key=lambda x: x[0])
 
             # keeping last candle time as last refreshed time of the pair
-            self._pairs_last_refreshed_time[pair] = data[-1][0] / 1000
+            self._pairs_last_refresh_time[pair] = data[-1][0] // 1000
 
             # keeping candles in cache
             self._cached_klines[pair] = data

From 8a0fc888d64510d5946c647e3fd0a5965c0013b8 Mon Sep 17 00:00:00 2001
From: Matthias <xmatthias@outlook.com>
Date: Fri, 10 Aug 2018 09:48:54 +0200
Subject: [PATCH 21/47] log if using cached data

---
 freqtrade/exchange/__init__.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/freqtrade/exchange/__init__.py b/freqtrade/exchange/__init__.py
index 3a6fdcf35..bd4745a52 100644
--- a/freqtrade/exchange/__init__.py
+++ b/freqtrade/exchange/__init__.py
@@ -368,6 +368,7 @@ class Exchange(object):
                     self._pairs_last_refresh_time.get(pair, 0) + interval_in_sec >=
                     int(time.time())):
                 data = self._cached_klines[pair]
+                logger.debug("Using cached klines data for %s ...", pair)
             else:
                 data = await self._api_async.fetch_ohlcv(pair, timeframe=tick_interval,
                                                          since=since_ms)

From e34f2abc3afa78998d4f9fa10c595204b0eaba5a Mon Sep 17 00:00:00 2001
From: Matthias <xmatthias@outlook.com>
Date: Fri, 10 Aug 2018 09:56:54 +0200
Subject: [PATCH 22/47] Add some typehints

---
 freqtrade/exchange/__init__.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/freqtrade/exchange/__init__.py b/freqtrade/exchange/__init__.py
index bd4745a52..6165b7493 100644
--- a/freqtrade/exchange/__init__.py
+++ b/freqtrade/exchange/__init__.py
@@ -54,7 +54,7 @@ class Exchange(object):
     _cached_ticker: Dict[str, Any] = {}
 
     # Holds last candle refreshed time of each pair
-    _pairs_last_refresh_time = {}
+    _pairs_last_refresh_time : Dict[str, int] = {}
 
     # Holds candles
     _cached_klines: Dict[str, Any] = {}
@@ -341,7 +341,8 @@ class Exchange(object):
             logger.info("returning cached ticker-data for %s", pair)
             return self._cached_ticker[pair]
 
-    async def async_get_candles_history(self, pairs, tick_interval) -> List[Tuple[str, List]]:
+    async def async_get_candles_history(self, pairs: List[str],
+                                        tick_interval: str) -> List[Tuple[str, List]]:
         # COMMENTED CODE IS FOR DISCUSSION: where should we close the loop on async ?
         # loop = asyncio.new_event_loop()
         # asyncio.set_event_loop(loop)

From 74d6816a1a0809e7b981748201216f8aea1eeb6a Mon Sep 17 00:00:00 2001
From: Matthias <xmatthias@outlook.com>
Date: Fri, 10 Aug 2018 10:19:26 +0200
Subject: [PATCH 23/47] Fix some comments

---
 freqtrade/exchange/__init__.py | 2 +-
 freqtrade/optimize/__init__.py | 7 +++----
 2 files changed, 4 insertions(+), 5 deletions(-)

diff --git a/freqtrade/exchange/__init__.py b/freqtrade/exchange/__init__.py
index 6165b7493..0cc707772 100644
--- a/freqtrade/exchange/__init__.py
+++ b/freqtrade/exchange/__init__.py
@@ -54,7 +54,7 @@ class Exchange(object):
     _cached_ticker: Dict[str, Any] = {}
 
     # Holds last candle refreshed time of each pair
-    _pairs_last_refresh_time : Dict[str, int] = {}
+    _pairs_last_refresh_time: Dict[str, int] = {}
 
     # Holds candles
     _cached_klines: Dict[str, Any] = {}
diff --git a/freqtrade/optimize/__init__.py b/freqtrade/optimize/__init__.py
index 8d5350fe5..502407f07 100644
--- a/freqtrade/optimize/__init__.py
+++ b/freqtrade/optimize/__init__.py
@@ -191,19 +191,18 @@ def download_backtesting_testdata(datadir: str,
                                   timerange: Optional[TimeRange] = None) -> None:
 
     """
-    Download the latest ticker intervals from the exchange for the pairs passed in parameters
+    Download the latest ticker intervals from the exchange for the pair passed in parameters
     The data is downloaded starting from the last correct ticker interval data that
-    esists in a cache. If timerange starts earlier than the data in the cache,
+    exists in a cache. If timerange starts earlier than the data in the cache,
     the full data will be redownloaded
 
     Based on @Rybolov work: https://github.com/rybolov/freqtrade-data
-    :param pairs: list of pairs to download
+    :param pair: pair to download
     :param tick_interval: ticker interval
     :param timerange: range of time to download
     :return: None
 
     """
-
     path = make_testdata_path(datadir)
     filepair = pair.replace("/", "_")
     filename = os.path.join(path, f'{filepair}-{tick_interval}.json')

From a107c4c7b4203c2f0232faf6f6040d03e7477293 Mon Sep 17 00:00:00 2001
From: Matthias <xmatthias@outlook.com>
Date: Fri, 10 Aug 2018 11:08:28 +0200
Subject: [PATCH 24/47] Download using asyncio

---
 freqtrade/exchange/__init__.py    | 36 +++++++++++++++++++++++++++++--
 freqtrade/optimize/__init__.py    |  4 ++--
 scripts/download_backtest_data.py |  9 ++++----
 3 files changed, 41 insertions(+), 8 deletions(-)

diff --git a/freqtrade/exchange/__init__.py b/freqtrade/exchange/__init__.py
index 0cc707772..2cfcfbde7 100644
--- a/freqtrade/exchange/__init__.py
+++ b/freqtrade/exchange/__init__.py
@@ -142,6 +142,7 @@ class Exchange(object):
 
         try:
             markets = self._api.load_markets()
+            asyncio.get_event_loop().run_until_complete(self._api_async.load_markets())
         except ccxt.BaseError as e:
             logger.warning('Unable to validate pairs (assuming they are correct). Reason: %s', e)
             return
@@ -341,12 +342,43 @@ class Exchange(object):
             logger.info("returning cached ticker-data for %s", pair)
             return self._cached_ticker[pair]
 
+    def get_history(self, pair: str, tick_interval: str,
+                    since_ms: int) -> List:
+        """
+        Gets candle history using asyncio and returns the list of candles.
+        Handles all async doing.
+        """
+        return asyncio.get_event_loop().run_until_complete(
+            self._async_get_history(pair=pair, tick_interval=tick_interval,
+                                    since_ms=since_ms))
+
+    async def _async_get_history(self, pair: str,
+                                 tick_interval: str,
+                                 since_ms: int) -> List:
+        # Assume exchange returns 500 candles
+        _LIMIT = 500
+
+        one_call = constants.TICKER_INTERVAL_MINUTES[tick_interval] * 60 * _LIMIT * 1000
+        logger.debug("one_call: %s", one_call)
+        input_coroutines = [self.async_get_candle_history(
+            pair, tick_interval, since) for since in
+                range(since_ms, int(time.time() * 1000), one_call)]
+        tickers = await asyncio.gather(*input_coroutines, return_exceptions=True)
+
+        # Combine tickers
+        data = []
+        for tick in tickers:
+            if tick[0] == pair:
+                data.extend(tick[1])
+        logger.info("downloaded %s with length %s.", pair, len(data))
+        return data
+
     async def async_get_candles_history(self, pairs: List[str],
                                         tick_interval: str) -> List[Tuple[str, List]]:
         # COMMENTED CODE IS FOR DISCUSSION: where should we close the loop on async ?
         # loop = asyncio.new_event_loop()
         # asyncio.set_event_loop(loop)
-        await self._api_async.load_markets()
+        # await self._api_async.load_markets()
         input_coroutines = [self.async_get_candle_history(
             symbol, tick_interval) for symbol in pairs]
         tickers = await asyncio.gather(*input_coroutines, return_exceptions=True)
@@ -357,7 +389,7 @@ class Exchange(object):
                                        since_ms: Optional[int] = None) -> Tuple[str, List]:
         try:
             # fetch ohlcv asynchronously
-            logger.debug("fetching %s ...", pair)
+            logger.debug("fetching %s since %s ...", pair, since_ms)
 
             # Calculating ticker interval in second
             interval_in_sec = constants.TICKER_INTERVAL_MINUTES[tick_interval] * 60
diff --git a/freqtrade/optimize/__init__.py b/freqtrade/optimize/__init__.py
index 502407f07..49b286fe8 100644
--- a/freqtrade/optimize/__init__.py
+++ b/freqtrade/optimize/__init__.py
@@ -218,8 +218,8 @@ def download_backtesting_testdata(datadir: str,
     logger.debug("Current Start: %s", misc.format_ms_time(data[1][0]) if data else 'None')
     logger.debug("Current End: %s", misc.format_ms_time(data[-1][0]) if data else 'None')
 
-    new_data = exchange.get_candle_history(pair=pair, tick_interval=tick_interval,
-                                           since_ms=since_ms)
+    new_data = exchange.get_history(pair=pair, tick_interval=tick_interval,
+                                    since_ms=since_ms)
     data.extend(new_data)
 
     logger.debug("New Start: %s", misc.format_ms_time(data[0][0]))
diff --git a/scripts/download_backtest_data.py b/scripts/download_backtest_data.py
index 686098f94..27c4c1e1c 100755
--- a/scripts/download_backtest_data.py
+++ b/scripts/download_backtest_data.py
@@ -1,6 +1,6 @@
 #!/usr/bin/env python3
 
-"""This script generate json data from bittrex"""
+"""This script generate json data"""
 import json
 import sys
 from pathlib import Path
@@ -52,9 +52,10 @@ exchange = Exchange({'key': '',
                      'stake_currency': '',
                      'dry_run': True,
                      'exchange': {
-                        'name': args.exchange,
-                        'pair_whitelist': []
-                        }
+                         'name': args.exchange,
+                         'pair_whitelist': [],
+                         'ccxt_rate_limit': False
+                     }
                      })
 pairs_not_available = []
 

From a852d2ff32627ac1ffc56e3038319d5511f675c0 Mon Sep 17 00:00:00 2001
From: Matthias <xmatthias@outlook.com>
Date: Fri, 10 Aug 2018 11:15:02 +0200
Subject: [PATCH 25/47] default since_ms to 30 days if no timerange is given

---
 freqtrade/exchange/__init__.py | 2 +-
 freqtrade/optimize/__init__.py | 5 ++++-
 2 files changed, 5 insertions(+), 2 deletions(-)

diff --git a/freqtrade/exchange/__init__.py b/freqtrade/exchange/__init__.py
index 2cfcfbde7..1df92b874 100644
--- a/freqtrade/exchange/__init__.py
+++ b/freqtrade/exchange/__init__.py
@@ -366,7 +366,7 @@ class Exchange(object):
         tickers = await asyncio.gather(*input_coroutines, return_exceptions=True)
 
         # Combine tickers
-        data = []
+        data: List = []
         for tick in tickers:
             if tick[0] == pair:
                 data.extend(tick[1])
diff --git a/freqtrade/optimize/__init__.py b/freqtrade/optimize/__init__.py
index 49b286fe8..4332f84a4 100644
--- a/freqtrade/optimize/__init__.py
+++ b/freqtrade/optimize/__init__.py
@@ -218,8 +218,11 @@ def download_backtesting_testdata(datadir: str,
     logger.debug("Current Start: %s", misc.format_ms_time(data[1][0]) if data else 'None')
     logger.debug("Current End: %s", misc.format_ms_time(data[-1][0]) if data else 'None')
 
+    # Default since_ms to 30 days if nothing is given
     new_data = exchange.get_history(pair=pair, tick_interval=tick_interval,
-                                    since_ms=since_ms)
+                                    since_ms=since_ms if since_ms
+                                    else
+                                    int(arrow.utcnow().shift(days=-30).float_timestamp) * 1000)
     data.extend(new_data)
 
     logger.debug("New Start: %s", misc.format_ms_time(data[0][0]))

From fce071843dcb370e8e46a7783249d07d13ebaa17 Mon Sep 17 00:00:00 2001
From: Matthias <xmatthias@outlook.com>
Date: Fri, 10 Aug 2018 13:04:43 +0200
Subject: [PATCH 26/47] Move async-load to seperate function

---
 freqtrade/exchange/__init__.py | 11 ++++++++++-
 1 file changed, 10 insertions(+), 1 deletion(-)

diff --git a/freqtrade/exchange/__init__.py b/freqtrade/exchange/__init__.py
index 1df92b874..e8889ca17 100644
--- a/freqtrade/exchange/__init__.py
+++ b/freqtrade/exchange/__init__.py
@@ -78,6 +78,7 @@ class Exchange(object):
         self._api = self._init_ccxt(exchange_config)
         self._api_async = self._init_ccxt(exchange_config, ccxt_async)
 
+        self._load_async_markets()
         logger.info('Using Exchange "%s"', self.name)
 
         # Check if all pairs are available
@@ -132,6 +133,15 @@ class Exchange(object):
                                      "Please check your config.json")
                 raise OperationalException(f'Exchange {name} does not provide a sandbox api')
 
+    def _load_async_markets(self) -> None:
+        try:
+            if self._api_async:
+                asyncio.get_event_loop().run_until_complete(self._api_async.load_markets())
+
+        except ccxt.BaseError as e:
+            logger.warning('Could not load async markets. Reason: %s', e)
+            return
+
     def validate_pairs(self, pairs: List[str]) -> None:
         """
         Checks if all given pairs are tradable on the current exchange.
@@ -142,7 +152,6 @@ class Exchange(object):
 
         try:
             markets = self._api.load_markets()
-            asyncio.get_event_loop().run_until_complete(self._api_async.load_markets())
         except ccxt.BaseError as e:
             logger.warning('Unable to validate pairs (assuming they are correct). Reason: %s', e)
             return

From 88e85e8d33083b4e7efa9147a9b415d1416d6f93 Mon Sep 17 00:00:00 2001
From: Matthias <xmatthias@outlook.com>
Date: Fri, 10 Aug 2018 13:11:04 +0200
Subject: [PATCH 27/47] fix tests - move load_async_markets call to
 validate_pairs

---
 freqtrade/exchange/__init__.py            |  2 +-
 freqtrade/tests/exchange/test_exchange.py | 16 +++++++++++++---
 freqtrade/tests/optimize/test_optimize.py | 16 ++++++++--------
 3 files changed, 22 insertions(+), 12 deletions(-)

diff --git a/freqtrade/exchange/__init__.py b/freqtrade/exchange/__init__.py
index e8889ca17..de1310751 100644
--- a/freqtrade/exchange/__init__.py
+++ b/freqtrade/exchange/__init__.py
@@ -78,7 +78,6 @@ class Exchange(object):
         self._api = self._init_ccxt(exchange_config)
         self._api_async = self._init_ccxt(exchange_config, ccxt_async)
 
-        self._load_async_markets()
         logger.info('Using Exchange "%s"', self.name)
 
         # Check if all pairs are available
@@ -152,6 +151,7 @@ class Exchange(object):
 
         try:
             markets = self._api.load_markets()
+            self._load_async_markets()
         except ccxt.BaseError as e:
             logger.warning('Unable to validate pairs (assuming they are correct). Reason: %s', e)
             return
diff --git a/freqtrade/tests/exchange/test_exchange.py b/freqtrade/tests/exchange/test_exchange.py
index ab25a5a7a..8fa7a6fec 100644
--- a/freqtrade/tests/exchange/test_exchange.py
+++ b/freqtrade/tests/exchange/test_exchange.py
@@ -13,6 +13,10 @@ from freqtrade.exchange import API_RETRY_COUNT, Exchange
 from freqtrade.tests.conftest import get_patched_exchange, log_has
 
 
+async def async_load_markets():
+    return {}
+
+
 def ccxt_exceptionhandlers(mocker, default_conf, api_mock, fun, mock_ccxt_fun, **kwargs):
     with pytest.raises(TemporaryError):
         api_mock.__dict__[mock_ccxt_fun] = MagicMock(side_effect=ccxt.NetworkError)
@@ -78,6 +82,7 @@ def test_symbol_amount_prec(default_conf, mocker):
 
     mocker.patch('freqtrade.exchange.Exchange._init_ccxt', MagicMock(return_value=api_mock))
     mocker.patch('freqtrade.exchange.Exchange.validate_timeframes', MagicMock())
+    mocker.patch('freqtrade.exchange.Exchange._load_async_markets', MagicMock())
     exchange = Exchange(default_conf)
 
     amount = 2.34559
@@ -101,6 +106,7 @@ def test_symbol_price_prec(default_conf, mocker):
 
     mocker.patch('freqtrade.exchange.Exchange._init_ccxt', MagicMock(return_value=api_mock))
     mocker.patch('freqtrade.exchange.Exchange.validate_timeframes', MagicMock())
+    mocker.patch('freqtrade.exchange.Exchange._load_async_markets', MagicMock())
     exchange = Exchange(default_conf)
 
     price = 2.34559
@@ -122,6 +128,7 @@ def test_set_sandbox(default_conf, mocker):
     type(api_mock).urls = url_mock
     mocker.patch('freqtrade.exchange.Exchange._init_ccxt', MagicMock(return_value=api_mock))
     mocker.patch('freqtrade.exchange.Exchange.validate_timeframes', MagicMock())
+    mocker.patch('freqtrade.exchange.Exchange._load_async_markets', MagicMock())
 
     exchange = Exchange(default_conf)
     liveurl = exchange._api.urls['api']
@@ -143,6 +150,7 @@ def test_set_sandbox_exception(default_conf, mocker):
 
     mocker.patch('freqtrade.exchange.Exchange._init_ccxt', MagicMock(return_value=api_mock))
     mocker.patch('freqtrade.exchange.Exchange.validate_timeframes', MagicMock())
+    mocker.patch('freqtrade.exchange.Exchange._load_async_markets', MagicMock())
 
     with pytest.raises(OperationalException, match=r'does not provide a sandbox api'):
         exchange = Exchange(default_conf)
@@ -160,6 +168,7 @@ def test_validate_pairs(default_conf, mocker):
 
     mocker.patch('freqtrade.exchange.Exchange._init_ccxt', MagicMock(return_value=api_mock))
     mocker.patch('freqtrade.exchange.Exchange.validate_timeframes', MagicMock())
+    mocker.patch('freqtrade.exchange.Exchange._load_async_markets', MagicMock())
     Exchange(default_conf)
 
 
@@ -168,6 +177,7 @@ def test_validate_pairs_not_available(default_conf, mocker):
     api_mock.load_markets = MagicMock(return_value={})
     mocker.patch('freqtrade.exchange.Exchange._init_ccxt', MagicMock(return_value=api_mock))
     mocker.patch('freqtrade.exchange.Exchange.validate_timeframes', MagicMock())
+    mocker.patch('freqtrade.exchange.Exchange._load_async_markets', MagicMock())
 
     with pytest.raises(OperationalException, match=r'not available'):
         Exchange(default_conf)
@@ -181,6 +191,7 @@ def test_validate_pairs_not_compatible(default_conf, mocker):
     default_conf['stake_currency'] = 'ETH'
     mocker.patch('freqtrade.exchange.Exchange._init_ccxt', MagicMock(return_value=api_mock))
     mocker.patch('freqtrade.exchange.Exchange.validate_timeframes', MagicMock())
+    mocker.patch('freqtrade.exchange.Exchange._load_async_markets', MagicMock())
     with pytest.raises(OperationalException, match=r'not compatible'):
         Exchange(default_conf)
 
@@ -193,6 +204,7 @@ def test_validate_pairs_exception(default_conf, mocker, caplog):
     api_mock.load_markets = MagicMock(return_value={})
     mocker.patch('freqtrade.exchange.Exchange._init_ccxt', api_mock)
     mocker.patch('freqtrade.exchange.Exchange.validate_timeframes', MagicMock())
+    mocker.patch('freqtrade.exchange.Exchange._load_async_markets', MagicMock())
 
     with pytest.raises(OperationalException, match=r'Pair ETH/BTC is not available at Binance'):
         Exchange(default_conf)
@@ -212,6 +224,7 @@ def test_validate_pairs_stake_exception(default_conf, mocker, caplog):
     api_mock.name = MagicMock(return_value='binance')
     mocker.patch('freqtrade.exchange.Exchange._init_ccxt', api_mock)
     mocker.patch('freqtrade.exchange.Exchange.validate_timeframes', MagicMock())
+    mocker.patch('freqtrade.exchange.Exchange._load_async_markets', MagicMock())
 
     with pytest.raises(
         OperationalException,
@@ -584,9 +597,6 @@ async def test_async_get_candles_history(default_conf, mocker):
     async def async_fetch_ohlcv(pair, timeframe, since):
         return tick
 
-    async def async_load_markets():
-        return {}
-
     exchange = get_patched_exchange(mocker, default_conf)
     # Monkey-patch async function
     exchange._api_async.fetch_ohlcv = async_fetch_ohlcv
diff --git a/freqtrade/tests/optimize/test_optimize.py b/freqtrade/tests/optimize/test_optimize.py
index 13f65fbf5..77fa3e3b1 100644
--- a/freqtrade/tests/optimize/test_optimize.py
+++ b/freqtrade/tests/optimize/test_optimize.py
@@ -53,7 +53,7 @@ def _clean_test_file(file: str) -> None:
 
 
 def test_load_data_30min_ticker(ticker_history, mocker, caplog, default_conf) -> None:
-    mocker.patch('freqtrade.exchange.Exchange.get_candle_history', return_value=ticker_history)
+    mocker.patch('freqtrade.exchange.Exchange.get_history', return_value=ticker_history)
     file = os.path.join(os.path.dirname(__file__), '..', 'testdata', 'UNITTEST_BTC-30m.json')
     _backup_file(file, copy_file=True)
     optimize.load_data(None, pairs=['UNITTEST/BTC'], ticker_interval='30m')
@@ -63,7 +63,7 @@ def test_load_data_30min_ticker(ticker_history, mocker, caplog, default_conf) ->
 
 
 def test_load_data_5min_ticker(ticker_history, mocker, caplog, default_conf) -> None:
-    mocker.patch('freqtrade.exchange.Exchange.get_candle_history', return_value=ticker_history)
+    mocker.patch('freqtrade.exchange.Exchange.get_history', return_value=ticker_history)
 
     file = os.path.join(os.path.dirname(__file__), '..', 'testdata', 'UNITTEST_BTC-5m.json')
     _backup_file(file, copy_file=True)
@@ -74,7 +74,7 @@ def test_load_data_5min_ticker(ticker_history, mocker, caplog, default_conf) ->
 
 
 def test_load_data_1min_ticker(ticker_history, mocker, caplog) -> None:
-    mocker.patch('freqtrade.exchange.Exchange.get_candle_history', return_value=ticker_history)
+    mocker.patch('freqtrade.exchange.Exchange.get_history', return_value=ticker_history)
     file = os.path.join(os.path.dirname(__file__), '..', 'testdata', 'UNITTEST_BTC-1m.json')
     _backup_file(file, copy_file=True)
     optimize.load_data(None, ticker_interval='1m', pairs=['UNITTEST/BTC'])
@@ -87,7 +87,7 @@ def test_load_data_with_new_pair_1min(ticker_history, mocker, caplog, default_co
     """
     Test load_data() with 1 min ticker
     """
-    mocker.patch('freqtrade.exchange.Exchange.get_candle_history', return_value=ticker_history)
+    mocker.patch('freqtrade.exchange.Exchange.get_history', return_value=ticker_history)
     exchange = get_patched_exchange(mocker, default_conf)
     file = os.path.join(os.path.dirname(__file__), '..', 'testdata', 'MEME_BTC-1m.json')
 
@@ -118,7 +118,7 @@ def test_testdata_path() -> None:
 
 
 def test_download_pairs(ticker_history, mocker, default_conf) -> None:
-    mocker.patch('freqtrade.exchange.Exchange.get_candle_history', return_value=ticker_history)
+    mocker.patch('freqtrade.exchange.Exchange.get_history', return_value=ticker_history)
     exchange = get_patched_exchange(mocker, default_conf)
     file1_1 = os.path.join(os.path.dirname(__file__), '..', 'testdata', 'MEME_BTC-1m.json')
     file1_5 = os.path.join(os.path.dirname(__file__), '..', 'testdata', 'MEME_BTC-5m.json')
@@ -261,7 +261,7 @@ def test_load_cached_data_for_updating(mocker) -> None:
 
 
 def test_download_pairs_exception(ticker_history, mocker, caplog, default_conf) -> None:
-    mocker.patch('freqtrade.exchange.Exchange.get_candle_history', return_value=ticker_history)
+    mocker.patch('freqtrade.exchange.Exchange.get_history', return_value=ticker_history)
     mocker.patch('freqtrade.optimize.__init__.download_backtesting_testdata',
                  side_effect=BaseException('File Error'))
     exchange = get_patched_exchange(mocker, default_conf)
@@ -279,7 +279,7 @@ def test_download_pairs_exception(ticker_history, mocker, caplog, default_conf)
 
 
 def test_download_backtesting_testdata(ticker_history, mocker, default_conf) -> None:
-    mocker.patch('freqtrade.exchange.Exchange.get_candle_history', return_value=ticker_history)
+    mocker.patch('freqtrade.exchange.Exchange.get_history', return_value=ticker_history)
     exchange = get_patched_exchange(mocker, default_conf)
 
     # Download a 1 min ticker file
@@ -304,7 +304,7 @@ def test_download_backtesting_testdata2(mocker, default_conf) -> None:
         [1509836580000, 0.00161, 0.00161, 0.00161, 0.00161, 82.390199]
     ]
     json_dump_mock = mocker.patch('freqtrade.misc.file_dump_json', return_value=None)
-    mocker.patch('freqtrade.exchange.Exchange.get_candle_history', return_value=tick)
+    mocker.patch('freqtrade.exchange.Exchange.get_history', return_value=tick)
     exchange = get_patched_exchange(mocker, default_conf)
     download_backtesting_testdata(None, exchange, pair="UNITTEST/BTC", tick_interval='1m')
     download_backtesting_testdata(None, exchange, pair="UNITTEST/BTC", tick_interval='3m')

From 69cc6aa9587f9e1b4d8a0b630911df578aa91b5c Mon Sep 17 00:00:00 2001
From: Matthias <xmatthias@outlook.com>
Date: Tue, 14 Aug 2018 15:59:45 +0200
Subject: [PATCH 28/47] Add test to async

---
 freqtrade/freqtradebot.py                 |  1 -
 freqtrade/tests/exchange/test_exchange.py |  5 ----
 freqtrade/tests/test_freqtradebot.py      | 30 +++++++++++++++++++++++
 3 files changed, 30 insertions(+), 6 deletions(-)

diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py
index d552dc65f..52ada40fa 100644
--- a/freqtrade/freqtradebot.py
+++ b/freqtrade/freqtradebot.py
@@ -135,7 +135,6 @@ class FreqtradeBot(object):
         """
         Refresh tickers asyncronously and return the result.
         """
-        # TODO: maybe add since_ms to use async in the download-script?
         # TODO: Add tests for this and the async stuff above
         logger.debug("Refreshing klines for %d pairs", len(pair_list))
         datatups = asyncio.get_event_loop().run_until_complete(
diff --git a/freqtrade/tests/exchange/test_exchange.py b/freqtrade/tests/exchange/test_exchange.py
index 8fa7a6fec..f6b6b105f 100644
--- a/freqtrade/tests/exchange/test_exchange.py
+++ b/freqtrade/tests/exchange/test_exchange.py
@@ -624,11 +624,6 @@ async def test_async_get_candles_history(default_conf, mocker):
     #     await exchange.async_get_candles_history('ETH/BTC', "5m")
 
 
-def test_refresh_tickers():
-    # TODO: Implement test for this
-    pass
-
-
 def make_fetch_ohlcv_mock(data):
     def fetch_ohlcv_mock(pair, timeframe, since):
         if since:
diff --git a/freqtrade/tests/test_freqtradebot.py b/freqtrade/tests/test_freqtradebot.py
index 3bf6ad037..489392438 100644
--- a/freqtrade/tests/test_freqtradebot.py
+++ b/freqtrade/tests/test_freqtradebot.py
@@ -137,6 +137,36 @@ def test_throttle_with_assets(mocker, default_conf) -> None:
     assert result == -1
 
 
+def test_refresh_tickers(mocker, default_conf, caplog) -> None:
+    tick = [
+         [
+             1511686200000,  # unix timestamp ms
+             1,  # open
+             2,  # high
+             3,  # low
+             4,  # close
+             5,  # volume (in quote currency)
+         ]
+     ]
+
+    async def async_get_candles_history(pairlist, timeframe):
+        return [(pair, tick) for pair in pairlist]
+
+    caplog.set_level(logging.DEBUG)
+    freqtrade = get_patched_freqtradebot(mocker, default_conf)
+    freqtrade.exchange.async_get_candles_history = async_get_candles_history
+
+    pairs = ['IOTA/ETH', 'XRP/ETH']
+    # empty dicts
+    assert not freqtrade._klines
+    freqtrade.refresh_tickers(['IOTA/ETH', 'XRP/ETH'])
+
+    assert log_has(f'Refreshing klines for {len(pairs)} pairs', caplog.record_tuples)
+    assert freqtrade._klines
+    for pair in pairs:
+        assert freqtrade._klines[pair]
+
+
 def test_gen_pair_whitelist(mocker, default_conf, tickers) -> None:
     freqtrade = get_patched_freqtradebot(mocker, default_conf)
     mocker.patch('freqtrade.exchange.Exchange.get_tickers', tickers)

From 8528143ffa37941eddb8104148991fdf0bae99e8 Mon Sep 17 00:00:00 2001
From: Matthias <xmatthias@outlook.com>
Date: Tue, 14 Aug 2018 19:51:49 +0200
Subject: [PATCH 29/47] Properly close async exchange as requested by ccxt

---
 freqtrade/exchange/__init__.py            | 9 +++++++++
 freqtrade/tests/exchange/test_exchange.py | 6 ++++++
 2 files changed, 15 insertions(+)

diff --git a/freqtrade/exchange/__init__.py b/freqtrade/exchange/__init__.py
index de1310751..07ab4d3c6 100644
--- a/freqtrade/exchange/__init__.py
+++ b/freqtrade/exchange/__init__.py
@@ -1,6 +1,7 @@
 # pragma pylint: disable=W0603
 """ Cryptocurrency Exchanges support """
 import logging
+import inspect
 from random import randint
 from typing import List, Dict, Tuple, Any, Optional
 from datetime import datetime
@@ -87,6 +88,14 @@ class Exchange(object):
             # Check if timeframe is available
             self.validate_timeframes(config['ticker_interval'])
 
+    def __del__(self):
+        """
+        Destructor - clean up async stuff
+        """
+        logger.debug("Exchange object destroyed, closing async loop")
+        if self._api_async and inspect.iscoroutinefunction(self._api_async.close):
+            asyncio.get_event_loop().run_until_complete(self._api_async.close())
+
     def _init_ccxt(self, exchange_config: dict, ccxt_module=ccxt) -> ccxt.Exchange:
         """
         Initialize ccxt with given config and return valid
diff --git a/freqtrade/tests/exchange/test_exchange.py b/freqtrade/tests/exchange/test_exchange.py
index f6b6b105f..f379ee689 100644
--- a/freqtrade/tests/exchange/test_exchange.py
+++ b/freqtrade/tests/exchange/test_exchange.py
@@ -51,6 +51,12 @@ def test_init(default_conf, mocker, caplog):
     assert log_has('Instance is running with dry_run enabled', caplog.record_tuples)
 
 
+def test_destroy(default_conf, mocker, caplog):
+    caplog.set_level(logging.DEBUG)
+    get_patched_exchange(mocker, default_conf)
+    assert log_has('Exchange object destroyed, closing async loop', caplog.record_tuples)
+
+
 def test_init_exception(default_conf, mocker):
     default_conf['exchange']['name'] = 'wrong_exchange_name'
 

From 37e504610a44c839889b752fe474719752be5abc Mon Sep 17 00:00:00 2001
From: Matthias <xmatthias@outlook.com>
Date: Tue, 14 Aug 2018 20:33:03 +0200
Subject: [PATCH 30/47] refactor private method - improve some async tests

---
 freqtrade/exchange/__init__.py            |  8 +--
 freqtrade/tests/exchange/test_exchange.py | 61 +++++++++++++----------
 2 files changed, 39 insertions(+), 30 deletions(-)

diff --git a/freqtrade/exchange/__init__.py b/freqtrade/exchange/__init__.py
index 07ab4d3c6..ae96a1d00 100644
--- a/freqtrade/exchange/__init__.py
+++ b/freqtrade/exchange/__init__.py
@@ -378,7 +378,7 @@ class Exchange(object):
 
         one_call = constants.TICKER_INTERVAL_MINUTES[tick_interval] * 60 * _LIMIT * 1000
         logger.debug("one_call: %s", one_call)
-        input_coroutines = [self.async_get_candle_history(
+        input_coroutines = [self._async_get_candle_history(
             pair, tick_interval, since) for since in
                 range(since_ms, int(time.time() * 1000), one_call)]
         tickers = await asyncio.gather(*input_coroutines, return_exceptions=True)
@@ -397,14 +397,14 @@ class Exchange(object):
         # loop = asyncio.new_event_loop()
         # asyncio.set_event_loop(loop)
         # await self._api_async.load_markets()
-        input_coroutines = [self.async_get_candle_history(
+        input_coroutines = [self._async_get_candle_history(
             symbol, tick_interval) for symbol in pairs]
         tickers = await asyncio.gather(*input_coroutines, return_exceptions=True)
         # await self._api_async.close()
         return tickers
 
-    async def async_get_candle_history(self, pair: str, tick_interval: str,
-                                       since_ms: Optional[int] = None) -> Tuple[str, List]:
+    async def _async_get_candle_history(self, pair: str, tick_interval: str,
+                                        since_ms: Optional[int] = None) -> Tuple[str, List]:
         try:
             # fetch ohlcv asynchronously
             logger.debug("fetching %s since %s ...", pair, since_ms)
diff --git a/freqtrade/tests/exchange/test_exchange.py b/freqtrade/tests/exchange/test_exchange.py
index f379ee689..e881e5590 100644
--- a/freqtrade/tests/exchange/test_exchange.py
+++ b/freqtrade/tests/exchange/test_exchange.py
@@ -3,7 +3,8 @@
 import logging
 from datetime import datetime
 from random import randint
-from unittest.mock import MagicMock, PropertyMock
+import time
+from unittest.mock import Mock, MagicMock, PropertyMock
 
 import ccxt
 import pytest
@@ -13,6 +14,14 @@ from freqtrade.exchange import API_RETRY_COUNT, Exchange
 from freqtrade.tests.conftest import get_patched_exchange, log_has
 
 
+# Source: https://stackoverflow.com/questions/29881236/how-to-mock-asyncio-coroutines
+def get_mock_coro(return_value):
+    async def mock_coro(*args, **kwargs):
+        return return_value
+
+    return Mock(wraps=mock_coro)
+
+
 async def async_load_markets():
     return {}
 
@@ -549,10 +558,10 @@ def test_get_ticker(default_conf, mocker):
 
 
 @pytest.mark.asyncio
-async def test_async_get_candle_history(default_conf, mocker):
+async def test__async_get_candle_history(default_conf, mocker, caplog):
     tick = [
         [
-            1511686200000,  # unix timestamp ms
+            int(time.time() * 1000),  # unix timestamp ms
             1,  # open
             2,  # high
             3,  # low
@@ -563,28 +572,37 @@ async def test_async_get_candle_history(default_conf, mocker):
 
     async def async_fetch_ohlcv(pair, timeframe, since):
         return tick
-
+    caplog.set_level(logging.DEBUG)
     exchange = get_patched_exchange(mocker, default_conf)
     # Monkey-patch async function
-    exchange._api_async.fetch_ohlcv = async_fetch_ohlcv
+    exchange._api_async.fetch_ohlcv = get_mock_coro(tick)
 
     exchange = Exchange(default_conf)
     pair = 'ETH/BTC'
-    res = await exchange.async_get_candle_history(pair, "5m")
+    res = await exchange._async_get_candle_history(pair, "5m")
     assert type(res) is tuple
     assert len(res) == 2
     assert res[0] == pair
     assert res[1] == tick
+    assert exchange._api_async.fetch_ohlcv.call_count == 1
+    assert not log_has(f"Using cached klines data for {pair} ...", caplog.record_tuples)
+    # test caching
+    res = await exchange._async_get_candle_history(pair, "5m")
+    assert exchange._api_async.fetch_ohlcv.call_count == 1
+    assert log_has(f"Using cached klines data for {pair} ...", caplog.record_tuples)
 
+    # exchange = Exchange(default_conf)
     await async_ccxt_exception(mocker, default_conf, MagicMock(),
-                               "async_get_candle_history", "fetch_ohlcv",
+                               "_async_get_candle_history", "fetch_ohlcv",
                                pair='ABCD/BTC', tick_interval=default_conf['ticker_interval'])
+    # # reinit exchange
+    # del exchange
 
-    api_mock = MagicMock()
-    with pytest.raises(OperationalException, match=r'Could not fetch ticker data*'):
-        api_mock.fetch_ohlcv = MagicMock(side_effect=ccxt.BaseError)
-        exchange = get_patched_exchange(mocker, default_conf, api_mock)
-        await exchange.async_get_candle_history(pair, "5m")
+    # api_mock = MagicMock()
+    # with pytest.raises(OperationalException, match=r'Could not fetch ticker data*'):
+    #     api_mock.fetch_ohlcv = MagicMock(side_effect=ccxt.BaseError)
+    #     exchange = get_patched_exchange(mocker, default_conf, api_mock)
+    #     await exchange._async_get_candle_history(pair, "5m")
 
 
 @pytest.mark.asyncio
@@ -600,14 +618,14 @@ async def test_async_get_candles_history(default_conf, mocker):
         ]
     ]
 
-    async def async_fetch_ohlcv(pair, timeframe, since):
-        return tick
+    async def mock_get_candle_hist(pair, tick_interval, since_ms=None):
+        return (pair, tick)
 
     exchange = get_patched_exchange(mocker, default_conf)
     # Monkey-patch async function
-    exchange._api_async.fetch_ohlcv = async_fetch_ohlcv
+    exchange._api_async.fetch_ohlcv = get_mock_coro(tick)
 
-    exchange._api_async.load_markets = async_load_markets
+    exchange._async_get_candle_history = Mock(wraps=mock_get_candle_hist)
 
     pairs = ['ETH/BTC', 'XRP/BTC']
     res = await exchange.async_get_candles_history(pairs, "5m")
@@ -618,16 +636,7 @@ async def test_async_get_candles_history(default_conf, mocker):
     assert res[0][1] == tick
     assert res[1][0] == pairs[1]
     assert res[1][1] == tick
-
-    # await async_ccxt_exception(mocker, default_conf, MagicMock(),
-    #                            "async_get_candles_history", "fetch_ohlcv",
-    #                            pairs=pairs, tick_interval=default_conf['ticker_interval'])
-
-    # api_mock = MagicMock()
-    # with pytest.raises(OperationalException, match=r'Could not fetch ticker data*'):
-    #     api_mock.fetch_ohlcv = MagicMock(side_effect=ccxt.BaseError)
-    #     exchange = get_patched_exchange(mocker, default_conf, api_mock)
-    #     await exchange.async_get_candles_history('ETH/BTC', "5m")
+    assert exchange._async_get_candle_history.call_count == 2
 
 
 def make_fetch_ohlcv_mock(data):

From 67cbbc86f27dc8f71d37680048f182badfccbb2e Mon Sep 17 00:00:00 2001
From: Matthias <xmatthias@outlook.com>
Date: Tue, 14 Aug 2018 20:35:12 +0200
Subject: [PATCH 31/47] Add test for exception

---
 freqtrade/tests/exchange/test_exchange.py | 12 +++++-------
 1 file changed, 5 insertions(+), 7 deletions(-)

diff --git a/freqtrade/tests/exchange/test_exchange.py b/freqtrade/tests/exchange/test_exchange.py
index e881e5590..04b505584 100644
--- a/freqtrade/tests/exchange/test_exchange.py
+++ b/freqtrade/tests/exchange/test_exchange.py
@@ -595,14 +595,12 @@ async def test__async_get_candle_history(default_conf, mocker, caplog):
     await async_ccxt_exception(mocker, default_conf, MagicMock(),
                                "_async_get_candle_history", "fetch_ohlcv",
                                pair='ABCD/BTC', tick_interval=default_conf['ticker_interval'])
-    # # reinit exchange
-    # del exchange
 
-    # api_mock = MagicMock()
-    # with pytest.raises(OperationalException, match=r'Could not fetch ticker data*'):
-    #     api_mock.fetch_ohlcv = MagicMock(side_effect=ccxt.BaseError)
-    #     exchange = get_patched_exchange(mocker, default_conf, api_mock)
-    #     await exchange._async_get_candle_history(pair, "5m")
+    api_mock = MagicMock()
+    with pytest.raises(OperationalException, match=r'Could not fetch ticker data*'):
+        api_mock.fetch_ohlcv = MagicMock(side_effect=ccxt.BaseError)
+        exchange = get_patched_exchange(mocker, default_conf, api_mock)
+        await exchange._async_get_candle_history(pair, "5m", int((time.time() - 2000) * 1000))
 
 
 @pytest.mark.asyncio

From e37cb49dc26ec94815078d22d7d932be52303351 Mon Sep 17 00:00:00 2001
From: Matthias <xmatthias@outlook.com>
Date: Tue, 14 Aug 2018 20:42:13 +0200
Subject: [PATCH 32/47] Ad test for async_load_markets

---
 freqtrade/tests/exchange/test_exchange.py | 14 ++++++++++++++
 1 file changed, 14 insertions(+)

diff --git a/freqtrade/tests/exchange/test_exchange.py b/freqtrade/tests/exchange/test_exchange.py
index 04b505584..68a9aba23 100644
--- a/freqtrade/tests/exchange/test_exchange.py
+++ b/freqtrade/tests/exchange/test_exchange.py
@@ -173,6 +173,20 @@ def test_set_sandbox_exception(default_conf, mocker):
         exchange.set_sandbox(exchange._api, default_conf['exchange'], 'Logname')
 
 
+def test__load_async_markets(default_conf, mocker, caplog):
+    exchange = get_patched_exchange(mocker, default_conf)
+    exchange._api_async.load_markets = get_mock_coro(None)
+    exchange._load_async_markets()
+    assert exchange._api_async.load_markets.call_count == 1
+    caplog.set_level(logging.DEBUG)
+
+    exchange._api_async.load_markets = Mock(side_effect=ccxt.BaseError("deadbeef"))
+    exchange._load_async_markets()
+
+    assert log_has('Could not load async markets. Reason: deadbeef',
+                   caplog.record_tuples)
+
+
 def test_validate_pairs(default_conf, mocker):
     api_mock = MagicMock()
     api_mock.load_markets = MagicMock(return_value={

From 3aa210cf93f781009a9d62aa50d7956ff5d09979 Mon Sep 17 00:00:00 2001
From: Matthias <xmatthias@outlook.com>
Date: Tue, 14 Aug 2018 20:53:58 +0200
Subject: [PATCH 33/47] Add test for get_history

---
 freqtrade/tests/exchange/test_exchange.py | 28 +++++++++++++++++++++++
 1 file changed, 28 insertions(+)

diff --git a/freqtrade/tests/exchange/test_exchange.py b/freqtrade/tests/exchange/test_exchange.py
index 68a9aba23..376324757 100644
--- a/freqtrade/tests/exchange/test_exchange.py
+++ b/freqtrade/tests/exchange/test_exchange.py
@@ -571,6 +571,34 @@ def test_get_ticker(default_conf, mocker):
     exchange.get_ticker(pair='ETH/BTC', refresh=True)
 
 
+def test_get_history(default_conf, mocker, caplog):
+    exchange = get_patched_exchange(mocker, default_conf)
+    tick = [
+        [
+            int(time.time() * 1000),  # unix timestamp ms
+            1,  # open
+            2,  # high
+            3,  # low
+            4,  # close
+            5,  # volume (in quote currency)
+        ]
+    ]
+    pair = 'ETH/BTC'
+
+    async def mock_cacndle_hist(pair, tick_interval, since_ms):
+        return pair, tick
+
+    exchange._async_get_candle_history = Mock(wraps=mock_cacndle_hist)
+    # one_call calculation * 1.8 should do 2 calls
+    since = 5 * 60 * 500 * 1.8
+    print(f"since = {since}")
+    ret = exchange.get_history(pair, "5m", int((time.time() - since) * 1000))
+
+    assert exchange._async_get_candle_history.call_count == 2
+    # Returns twice the above tick
+    assert len(ret) == 2
+
+
 @pytest.mark.asyncio
 async def test__async_get_candle_history(default_conf, mocker, caplog):
     tick = [

From ca6594cd24fdbb2dcb2791eafdea99c02e0477fe Mon Sep 17 00:00:00 2001
From: Matthias <xmatthias@outlook.com>
Date: Wed, 15 Aug 2018 12:46:45 +0200
Subject: [PATCH 34/47] remove comment, add docstring

---
 freqtrade/exchange/__init__.py | 7 +------
 1 file changed, 1 insertion(+), 6 deletions(-)

diff --git a/freqtrade/exchange/__init__.py b/freqtrade/exchange/__init__.py
index ae96a1d00..ce562e056 100644
--- a/freqtrade/exchange/__init__.py
+++ b/freqtrade/exchange/__init__.py
@@ -13,7 +13,6 @@ import ccxt
 import ccxt.async_support as ccxt_async
 import arrow
 
-
 from freqtrade import constants, OperationalException, DependencyException, TemporaryError
 
 logger = logging.getLogger(__name__)
@@ -393,14 +392,10 @@ class Exchange(object):
 
     async def async_get_candles_history(self, pairs: List[str],
                                         tick_interval: str) -> List[Tuple[str, List]]:
-        # COMMENTED CODE IS FOR DISCUSSION: where should we close the loop on async ?
-        # loop = asyncio.new_event_loop()
-        # asyncio.set_event_loop(loop)
-        # await self._api_async.load_markets()
+        """Download ohlcv history for pair-list asyncronously """
         input_coroutines = [self._async_get_candle_history(
             symbol, tick_interval) for symbol in pairs]
         tickers = await asyncio.gather(*input_coroutines, return_exceptions=True)
-        # await self._api_async.close()
         return tickers
 
     async def _async_get_candle_history(self, pair: str, tick_interval: str,

From 76914c2c07e70e9de6d87d270d7c96990ff97104 Mon Sep 17 00:00:00 2001
From: Matthias <xmatthias@outlook.com>
Date: Wed, 15 Aug 2018 12:57:27 +0200
Subject: [PATCH 35/47] remove todo comment as this is actually done

---
 freqtrade/freqtradebot.py | 6 ++----
 1 file changed, 2 insertions(+), 4 deletions(-)

diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py
index 52ada40fa..4e9677b35 100644
--- a/freqtrade/freqtradebot.py
+++ b/freqtrade/freqtradebot.py
@@ -2,6 +2,7 @@
 Freqtrade is the main module of this bot. It contains the class Freqtrade()
 """
 
+import asyncio
 import copy
 import logging
 import time
@@ -9,13 +10,11 @@ import traceback
 from datetime import datetime
 from typing import Any, Callable, Dict, List, Optional
 
-import asyncio
 import arrow
 import requests
 
 from cachetools import TTLCache, cached
 
-
 from freqtrade import (DependencyException, OperationalException,
                        TemporaryError, __version__, constants, persistence)
 from freqtrade.exchange import Exchange
@@ -135,12 +134,11 @@ class FreqtradeBot(object):
         """
         Refresh tickers asyncronously and return the result.
         """
-        # TODO: Add tests for this and the async stuff above
         logger.debug("Refreshing klines for %d pairs", len(pair_list))
         datatups = asyncio.get_event_loop().run_until_complete(
             self.exchange.async_get_candles_history(pair_list, self.strategy.ticker_interval))
 
-        # updating klines
+        # updating cached klines available to bot
         self._klines = {pair: data for (pair, data) in datatups}
 
         return True

From baeffee80d0bfb7df6431d95cf65f54aa2472d9c Mon Sep 17 00:00:00 2001
From: Matthias <xmatthias@outlook.com>
Date: Wed, 15 Aug 2018 13:18:52 +0200
Subject: [PATCH 36/47] Replace time.time with arrow.utcnow().timestamp

arrow is imported already
---
 freqtrade/exchange/__init__.py            |  5 ++---
 freqtrade/tests/exchange/test_exchange.py | 11 ++++++-----
 2 files changed, 8 insertions(+), 8 deletions(-)

diff --git a/freqtrade/exchange/__init__.py b/freqtrade/exchange/__init__.py
index ce562e056..88e403906 100644
--- a/freqtrade/exchange/__init__.py
+++ b/freqtrade/exchange/__init__.py
@@ -6,7 +6,6 @@ from random import randint
 from typing import List, Dict, Tuple, Any, Optional
 from datetime import datetime
 from math import floor, ceil
-import time
 
 import asyncio
 import ccxt
@@ -379,7 +378,7 @@ class Exchange(object):
         logger.debug("one_call: %s", one_call)
         input_coroutines = [self._async_get_candle_history(
             pair, tick_interval, since) for since in
-                range(since_ms, int(time.time() * 1000), one_call)]
+            range(since_ms, arrow.utcnow().timestamp * 1000, one_call)]
         tickers = await asyncio.gather(*input_coroutines, return_exceptions=True)
 
         # Combine tickers
@@ -412,7 +411,7 @@ class Exchange(object):
             # so we fetch it from local cache
             if (not since_ms and
                     self._pairs_last_refresh_time.get(pair, 0) + interval_in_sec >=
-                    int(time.time())):
+                    arrow.utcnow().timestamp):
                 data = self._cached_klines[pair]
                 logger.debug("Using cached klines data for %s ...", pair)
             else:
diff --git a/freqtrade/tests/exchange/test_exchange.py b/freqtrade/tests/exchange/test_exchange.py
index 376324757..ca06a4a70 100644
--- a/freqtrade/tests/exchange/test_exchange.py
+++ b/freqtrade/tests/exchange/test_exchange.py
@@ -3,9 +3,9 @@
 import logging
 from datetime import datetime
 from random import randint
-import time
 from unittest.mock import Mock, MagicMock, PropertyMock
 
+import arrow
 import ccxt
 import pytest
 
@@ -575,7 +575,7 @@ def test_get_history(default_conf, mocker, caplog):
     exchange = get_patched_exchange(mocker, default_conf)
     tick = [
         [
-            int(time.time() * 1000),  # unix timestamp ms
+            arrow.utcnow().timestamp * 1000,  # unix timestamp ms
             1,  # open
             2,  # high
             3,  # low
@@ -592,7 +592,7 @@ def test_get_history(default_conf, mocker, caplog):
     # one_call calculation * 1.8 should do 2 calls
     since = 5 * 60 * 500 * 1.8
     print(f"since = {since}")
-    ret = exchange.get_history(pair, "5m", int((time.time() - since) * 1000))
+    ret = exchange.get_history(pair, "5m", int((arrow.utcnow().timestamp - since) * 1000))
 
     assert exchange._async_get_candle_history.call_count == 2
     # Returns twice the above tick
@@ -603,7 +603,7 @@ def test_get_history(default_conf, mocker, caplog):
 async def test__async_get_candle_history(default_conf, mocker, caplog):
     tick = [
         [
-            int(time.time() * 1000),  # unix timestamp ms
+            arrow.utcnow().timestamp * 1000,  # unix timestamp ms
             1,  # open
             2,  # high
             3,  # low
@@ -642,7 +642,8 @@ async def test__async_get_candle_history(default_conf, mocker, caplog):
     with pytest.raises(OperationalException, match=r'Could not fetch ticker data*'):
         api_mock.fetch_ohlcv = MagicMock(side_effect=ccxt.BaseError)
         exchange = get_patched_exchange(mocker, default_conf, api_mock)
-        await exchange._async_get_candle_history(pair, "5m", int((time.time() - 2000) * 1000))
+        await exchange._async_get_candle_history(pair, "5m",
+                                                 (arrow.utcnow().timestamp - 2000) * 1000)
 
 
 @pytest.mark.asyncio

From e6e2799f03803cc3dee46daa5f46d5e98e400a63 Mon Sep 17 00:00:00 2001
From: misagh <misagh.shakeri@gmail.com>
Date: Thu, 16 Aug 2018 11:37:31 +0200
Subject: [PATCH 37/47] Keeping cached Klines only in exchange and renaming
 _cached_klines to klines.

---
 freqtrade/exchange/__init__.py       | 6 +++---
 freqtrade/freqtradebot.py            | 9 ++++-----
 freqtrade/tests/test_freqtradebot.py | 6 +++---
 3 files changed, 10 insertions(+), 11 deletions(-)

diff --git a/freqtrade/exchange/__init__.py b/freqtrade/exchange/__init__.py
index 69ad4130e..f6bc39239 100644
--- a/freqtrade/exchange/__init__.py
+++ b/freqtrade/exchange/__init__.py
@@ -56,7 +56,7 @@ class Exchange(object):
     _pairs_last_refresh_time: Dict[str, int] = {}
 
     # Holds candles
-    _cached_klines: Dict[str, Any] = {}
+    klines: Dict[str, Any] = {}
 
     # Holds all open sell orders for dry_run
     _dry_run_open_orders: Dict[str, Any] = {}
@@ -412,7 +412,7 @@ class Exchange(object):
             if (not since_ms and
                     self._pairs_last_refresh_time.get(pair, 0) + interval_in_sec >=
                     arrow.utcnow().timestamp):
-                data = self._cached_klines[pair]
+                data = self.klines[pair]
                 logger.debug("Using cached klines data for %s ...", pair)
             else:
                 data = await self._api_async.fetch_ohlcv(pair, timeframe=tick_interval,
@@ -427,7 +427,7 @@ class Exchange(object):
             self._pairs_last_refresh_time[pair] = data[-1][0] // 1000
 
             # keeping candles in cache
-            self._cached_klines[pair] = data
+            self.klines[pair] = data
 
             logger.debug("done fetching %s ...", pair)
             return pair, data
diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py
index adff27b1d..40c665261 100644
--- a/freqtrade/freqtradebot.py
+++ b/freqtrade/freqtradebot.py
@@ -55,8 +55,6 @@ class FreqtradeBot(object):
         self.persistence = None
         self.exchange = Exchange(self.config)
         self._init_modules()
-        self._klines: Dict[str, List[Dict]] = {}
-        self._klines_last_fetched_time = 0
 
     def _init_modules(self) -> None:
         """
@@ -173,7 +171,8 @@ class FreqtradeBot(object):
             self.exchange.async_get_candles_history(pair_list, self.strategy.ticker_interval))
 
         # updating cached klines available to bot
-        self._klines = {pair: data for (pair, data) in datatups}
+        #self.exchange.klines = {pair: data for (pair, data) in datatups}
+        # self.exchange.klines = datatups
 
         return True
 
@@ -385,7 +384,7 @@ class FreqtradeBot(object):
         # running get_signal on historical data fetched
         # to find buy signals
         for _pair in whitelist:
-            (buy, sell) = self.strategy.get_signal(_pair, interval, self._klines.get(_pair))
+            (buy, sell) = self.strategy.get_signal(_pair, interval, self.exchange.klines.get(_pair))
             if buy and not sell:
                 return self.execute_buy(_pair, stake_amount)
 
@@ -551,7 +550,7 @@ class FreqtradeBot(object):
         (buy, sell) = (False, False)
         experimental = self.config.get('experimental', {})
         if experimental.get('use_sell_signal') or experimental.get('ignore_roi_if_buy_signal'):
-            ticker = self._klines.get(trade.pair)
+            ticker = self.exchange.klines.get(trade.pair)
             (buy, sell) = self.strategy.get_signal(trade.pair, self.strategy.ticker_interval,
                                                    ticker)
 
diff --git a/freqtrade/tests/test_freqtradebot.py b/freqtrade/tests/test_freqtradebot.py
index 6e77f6341..42b348892 100644
--- a/freqtrade/tests/test_freqtradebot.py
+++ b/freqtrade/tests/test_freqtradebot.py
@@ -158,13 +158,13 @@ def test_refresh_tickers(mocker, default_conf, caplog) -> None:
 
     pairs = ['IOTA/ETH', 'XRP/ETH']
     # empty dicts
-    assert not freqtrade._klines
+    assert not freqtrade.exchange.klines
     freqtrade.refresh_tickers(['IOTA/ETH', 'XRP/ETH'])
 
     assert log_has(f'Refreshing klines for {len(pairs)} pairs', caplog.record_tuples)
-    assert freqtrade._klines
+    assert freqtrade.exchange.klines
     for pair in pairs:
-        assert freqtrade._klines[pair]
+        assert freqtrade.exchange.klines[pair]
 
 
 def test_gen_pair_whitelist(mocker, default_conf, tickers) -> None:

From ff8ed564f1d4c930aea39cf376e52765d1f6a220 Mon Sep 17 00:00:00 2001
From: Matthias <xmatthias@outlook.com>
Date: Thu, 16 Aug 2018 12:15:09 +0200
Subject: [PATCH 38/47] Refactor refresh_pairs to exchange and fix tests

---
 freqtrade/exchange/__init__.py            | 10 +++++++
 freqtrade/freqtradebot.py                 | 17 +-----------
 freqtrade/tests/exchange/test_exchange.py | 27 +++++++++++++++++++
 freqtrade/tests/test_freqtradebot.py      | 32 +----------------------
 4 files changed, 39 insertions(+), 47 deletions(-)

diff --git a/freqtrade/exchange/__init__.py b/freqtrade/exchange/__init__.py
index f6bc39239..1f9f147d8 100644
--- a/freqtrade/exchange/__init__.py
+++ b/freqtrade/exchange/__init__.py
@@ -389,6 +389,16 @@ class Exchange(object):
         logger.info("downloaded %s with length %s.", pair, len(data))
         return data
 
+    def refresh_tickers(self, pair_list: List[str], ticker_interval: str) -> bool:
+        """
+        Refresh tickers asyncronously and return the result.
+        """
+        logger.debug("Refreshing klines for %d pairs", len(pair_list))
+        asyncio.get_event_loop().run_until_complete(
+            self.async_get_candles_history(pair_list, ticker_interval))
+
+        return True
+
     async def async_get_candles_history(self, pairs: List[str],
                                         tick_interval: str) -> List[Tuple[str, List]]:
         """Download ohlcv history for pair-list asyncronously """
diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py
index 40c665261..3cb7ade9a 100644
--- a/freqtrade/freqtradebot.py
+++ b/freqtrade/freqtradebot.py
@@ -2,7 +2,6 @@
 Freqtrade is the main module of this bot. It contains the class Freqtrade()
 """
 
-import asyncio
 import copy
 import logging
 import time
@@ -162,20 +161,6 @@ class FreqtradeBot(object):
         time.sleep(duration)
         return result
 
-    def refresh_tickers(self, pair_list: List[str]) -> bool:
-        """
-        Refresh tickers asyncronously and return the result.
-        """
-        logger.debug("Refreshing klines for %d pairs", len(pair_list))
-        datatups = asyncio.get_event_loop().run_until_complete(
-            self.exchange.async_get_candles_history(pair_list, self.strategy.ticker_interval))
-
-        # updating cached klines available to bot
-        #self.exchange.klines = {pair: data for (pair, data) in datatups}
-        # self.exchange.klines = datatups
-
-        return True
-
     def _process(self, nb_assets: Optional[int] = 0) -> bool:
         """
         Queries the persistence layer for open trades and handles them,
@@ -197,7 +182,7 @@ class FreqtradeBot(object):
             self.config['exchange']['pair_whitelist'] = final_list
 
             # Refreshing candles
-            self.refresh_tickers(final_list)
+            self.exchange.refresh_tickers(final_list, self.strategy.ticker_interval)
 
             # Query trades from persistence layer
             trades = Trade.query.filter(Trade.is_open.is_(True)).all()
diff --git a/freqtrade/tests/exchange/test_exchange.py b/freqtrade/tests/exchange/test_exchange.py
index ca06a4a70..a9b786cb8 100644
--- a/freqtrade/tests/exchange/test_exchange.py
+++ b/freqtrade/tests/exchange/test_exchange.py
@@ -599,6 +599,33 @@ def test_get_history(default_conf, mocker, caplog):
     assert len(ret) == 2
 
 
+def test_refresh_tickers(mocker, default_conf, caplog) -> None:
+    tick = [
+        [
+            1511686200000,  # unix timestamp ms
+            1,  # open
+            2,  # high
+            3,  # low
+            4,  # close
+            5,  # volume (in quote currency)
+        ]
+    ]
+
+    caplog.set_level(logging.DEBUG)
+    exchange = get_patched_exchange(mocker, default_conf)
+    exchange._api_async.fetch_ohlcv = get_mock_coro(tick)
+
+    pairs = ['IOTA/ETH', 'XRP/ETH']
+    # empty dicts
+    assert not exchange.klines
+    exchange.refresh_tickers(['IOTA/ETH', 'XRP/ETH'], '5m')
+
+    assert log_has(f'Refreshing klines for {len(pairs)} pairs', caplog.record_tuples)
+    assert exchange.klines
+    for pair in pairs:
+        assert exchange.klines[pair]
+
+
 @pytest.mark.asyncio
 async def test__async_get_candle_history(default_conf, mocker, caplog):
     tick = [
diff --git a/freqtrade/tests/test_freqtradebot.py b/freqtrade/tests/test_freqtradebot.py
index 42b348892..5764c5b0f 100644
--- a/freqtrade/tests/test_freqtradebot.py
+++ b/freqtrade/tests/test_freqtradebot.py
@@ -44,7 +44,7 @@ def patch_get_signal(freqtrade: FreqtradeBot, value=(True, False)) -> None:
     """
     freqtrade.strategy.get_signal = lambda e, s, t: value
     freqtrade.exchange.get_candle_history = lambda p, i: None
-    freqtrade.refresh_tickers = lambda i: True
+    freqtrade.exchange.refresh_tickers = lambda p, i: True
 
 
 def patch_RPCManager(mocker) -> MagicMock:
@@ -137,36 +137,6 @@ def test_throttle_with_assets(mocker, default_conf) -> None:
     assert result == -1
 
 
-def test_refresh_tickers(mocker, default_conf, caplog) -> None:
-    tick = [
-         [
-             1511686200000,  # unix timestamp ms
-             1,  # open
-             2,  # high
-             3,  # low
-             4,  # close
-             5,  # volume (in quote currency)
-         ]
-     ]
-
-    async def async_get_candles_history(pairlist, timeframe):
-        return [(pair, tick) for pair in pairlist]
-
-    caplog.set_level(logging.DEBUG)
-    freqtrade = get_patched_freqtradebot(mocker, default_conf)
-    freqtrade.exchange.async_get_candles_history = async_get_candles_history
-
-    pairs = ['IOTA/ETH', 'XRP/ETH']
-    # empty dicts
-    assert not freqtrade.exchange.klines
-    freqtrade.refresh_tickers(['IOTA/ETH', 'XRP/ETH'])
-
-    assert log_has(f'Refreshing klines for {len(pairs)} pairs', caplog.record_tuples)
-    assert freqtrade.exchange.klines
-    for pair in pairs:
-        assert freqtrade.exchange.klines[pair]
-
-
 def test_gen_pair_whitelist(mocker, default_conf, tickers) -> None:
     freqtrade = get_patched_freqtradebot(mocker, default_conf)
     mocker.patch('freqtrade.exchange.Exchange.get_tickers', tickers)

From d556f669b0e220bb09b68c7ff5b1fee717e14bad Mon Sep 17 00:00:00 2001
From: Matthias <xmatthias@outlook.com>
Date: Sat, 18 Aug 2018 21:05:38 +0200
Subject: [PATCH 39/47] Add async retrier

---
 freqtrade/exchange/__init__.py            | 19 +++++++++++++++++++
 freqtrade/tests/exchange/test_exchange.py |  2 +-
 2 files changed, 20 insertions(+), 1 deletion(-)

diff --git a/freqtrade/exchange/__init__.py b/freqtrade/exchange/__init__.py
index 1f9f147d8..16ef549a0 100644
--- a/freqtrade/exchange/__init__.py
+++ b/freqtrade/exchange/__init__.py
@@ -26,6 +26,24 @@ _EXCHANGE_URLS = {
 }
 
 
+def retrier_async(f):
+    async def wrapper(*args, **kwargs):
+        count = kwargs.pop('count', API_RETRY_COUNT)
+        try:
+            return await f(*args, **kwargs)
+        except (TemporaryError, DependencyException) as ex:
+            logger.warning('%s() returned exception: "%s"', f.__name__, ex)
+            if count > 0:
+                count -= 1
+                kwargs.update({'count': count})
+                logger.warning('retrying %s() still for %s times', f.__name__, count)
+                return await wrapper(*args, **kwargs)
+            else:
+                logger.warning('Giving up retrying: %s()', f.__name__)
+                raise ex
+    return wrapper
+
+
 def retrier(f):
     def wrapper(*args, **kwargs):
         count = kwargs.pop('count', API_RETRY_COUNT)
@@ -407,6 +425,7 @@ class Exchange(object):
         tickers = await asyncio.gather(*input_coroutines, return_exceptions=True)
         return tickers
 
+    @retrier_async
     async def _async_get_candle_history(self, pair: str, tick_interval: str,
                                         since_ms: Optional[int] = None) -> Tuple[str, List]:
         try:
diff --git a/freqtrade/tests/exchange/test_exchange.py b/freqtrade/tests/exchange/test_exchange.py
index a9b786cb8..61a3f3efc 100644
--- a/freqtrade/tests/exchange/test_exchange.py
+++ b/freqtrade/tests/exchange/test_exchange.py
@@ -45,7 +45,7 @@ async def async_ccxt_exception(mocker, default_conf, api_mock, fun, mock_ccxt_fu
         api_mock.__dict__[mock_ccxt_fun] = MagicMock(side_effect=ccxt.NetworkError)
         exchange = get_patched_exchange(mocker, default_conf, api_mock)
         await getattr(exchange, fun)(**kwargs)
-    assert api_mock.__dict__[mock_ccxt_fun].call_count == 1
+    assert api_mock.__dict__[mock_ccxt_fun].call_count == API_RETRY_COUNT + 1
 
     with pytest.raises(OperationalException):
         api_mock.__dict__[mock_ccxt_fun] = MagicMock(side_effect=ccxt.BaseError)

From d722c12109281660e07615d178b262620b719ef4 Mon Sep 17 00:00:00 2001
From: Matthias <xmatthias@outlook.com>
Date: Sat, 18 Aug 2018 21:08:59 +0200
Subject: [PATCH 40/47] fix bug in async download script

---
 freqtrade/exchange/__init__.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/freqtrade/exchange/__init__.py b/freqtrade/exchange/__init__.py
index 16ef549a0..1196cb8fc 100644
--- a/freqtrade/exchange/__init__.py
+++ b/freqtrade/exchange/__init__.py
@@ -404,6 +404,8 @@ class Exchange(object):
         for tick in tickers:
             if tick[0] == pair:
                 data.extend(tick[1])
+        # Sort data again after extending the result - above calls return in "async order" order
+        data = sorted(data, key=lambda x: x[0])
         logger.info("downloaded %s with length %s.", pair, len(data))
         return data
 

From 088c54b88c6d8a08c7e0cefc2878834281bf1264 Mon Sep 17 00:00:00 2001
From: Matthias <xmatthias@outlook.com>
Date: Sun, 19 Aug 2018 09:17:17 +0200
Subject: [PATCH 41/47] remove unnecessary function

---
 freqtrade/tests/exchange/test_exchange.py | 4 ----
 1 file changed, 4 deletions(-)

diff --git a/freqtrade/tests/exchange/test_exchange.py b/freqtrade/tests/exchange/test_exchange.py
index 61a3f3efc..367b7d778 100644
--- a/freqtrade/tests/exchange/test_exchange.py
+++ b/freqtrade/tests/exchange/test_exchange.py
@@ -22,10 +22,6 @@ def get_mock_coro(return_value):
     return Mock(wraps=mock_coro)
 
 
-async def async_load_markets():
-    return {}
-
-
 def ccxt_exceptionhandlers(mocker, default_conf, api_mock, fun, mock_ccxt_fun, **kwargs):
     with pytest.raises(TemporaryError):
         api_mock.__dict__[mock_ccxt_fun] = MagicMock(side_effect=ccxt.NetworkError)

From 9403248e4d60f3dbf455c3060965d77e7ab3c8a4 Mon Sep 17 00:00:00 2001
From: Matthias <xmatthias@outlook.com>
Date: Sun, 19 Aug 2018 19:48:13 +0200
Subject: [PATCH 42/47] have plot-script use async ticker-refresh

---
 freqtrade/tests/optimize/test_backtesting.py | 2 +-
 scripts/plot_dataframe.py                    | 3 ++-
 2 files changed, 3 insertions(+), 2 deletions(-)

diff --git a/freqtrade/tests/optimize/test_backtesting.py b/freqtrade/tests/optimize/test_backtesting.py
index 32a5229c0..1625cc5a3 100644
--- a/freqtrade/tests/optimize/test_backtesting.py
+++ b/freqtrade/tests/optimize/test_backtesting.py
@@ -110,7 +110,7 @@ def mocked_load_data(datadir, pairs=[], ticker_interval='0m', refresh_pairs=Fals
     return pairdata
 
 
-# use for mock freqtrade.exchange.get_candle_history'
+# use for mock ccxt.fetch_ohlvc'
 def _load_pair_as_ticks(pair, tickfreq):
     ticks = optimize.load_data(None, ticker_interval=tickfreq, pairs=[pair])
     ticks = trim_dictlist(ticks, -201)
diff --git a/scripts/plot_dataframe.py b/scripts/plot_dataframe.py
index f2f2e0c7f..0f0a3d4cb 100755
--- a/scripts/plot_dataframe.py
+++ b/scripts/plot_dataframe.py
@@ -138,7 +138,8 @@ def plot_analyzed_dataframe(args: Namespace) -> None:
     tickers = {}
     if args.live:
         logger.info('Downloading pair.')
-        tickers[pair] = exchange.get_candle_history(pair, tick_interval)
+        exchange.refresh_tickers([pair], tick_interval)
+        tickers[pair] = exchange.klines[pair]
     else:
         tickers = optimize.load_data(
             datadir=_CONF.get("datadir"),

From 694b8be32f8de101a103c0ecda903d10604a8cd4 Mon Sep 17 00:00:00 2001
From: Matthias <xmatthias@outlook.com>
Date: Sun, 19 Aug 2018 19:37:48 +0200
Subject: [PATCH 43/47] Move variables from class to instance

---
 freqtrade/exchange/__init__.py | 15 ++++++++-------
 1 file changed, 8 insertions(+), 7 deletions(-)

diff --git a/freqtrade/exchange/__init__.py b/freqtrade/exchange/__init__.py
index 1f9f147d8..2f1aca962 100644
--- a/freqtrade/exchange/__init__.py
+++ b/freqtrade/exchange/__init__.py
@@ -50,13 +50,6 @@ class Exchange(object):
     _api: ccxt.Exchange = None
     _api_async: ccxt_async.Exchange = None
     _conf: Dict = {}
-    _cached_ticker: Dict[str, Any] = {}
-
-    # Holds last candle refreshed time of each pair
-    _pairs_last_refresh_time: Dict[str, int] = {}
-
-    # Holds candles
-    klines: Dict[str, Any] = {}
 
     # Holds all open sell orders for dry_run
     _dry_run_open_orders: Dict[str, Any] = {}
@@ -70,6 +63,14 @@ class Exchange(object):
         """
         self._conf.update(config)
 
+        self._cached_ticker: Dict[str, Any] = {}
+
+        # Holds last candle refreshed time of each pair
+        self._pairs_last_refresh_time: Dict[str, int] = {}
+
+        # Holds candles
+        self.klines: Dict[str, Any] = {}
+
         if config['dry_run']:
             logger.info('Instance is running with dry_run enabled')
 

From de0f3e43bf68f9b037875032b03febe1e40e190d Mon Sep 17 00:00:00 2001
From: Matthias <xmatthias@outlook.com>
Date: Sun, 19 Aug 2018 19:44:40 +0200
Subject: [PATCH 44/47] remove unused mocks

---
 freqtrade/tests/strategy/test_interface.py | 1 -
 freqtrade/tests/test_freqtradebot.py       | 2 --
 2 files changed, 3 deletions(-)

diff --git a/freqtrade/tests/strategy/test_interface.py b/freqtrade/tests/strategy/test_interface.py
index ec4ab0fd4..c821891b1 100644
--- a/freqtrade/tests/strategy/test_interface.py
+++ b/freqtrade/tests/strategy/test_interface.py
@@ -88,7 +88,6 @@ def test_get_signal_old_dataframe(default_conf, mocker, caplog):
 
 
 def test_get_signal_handles_exceptions(mocker, default_conf):
-    mocker.patch('freqtrade.exchange.Exchange.get_candle_history', return_value=MagicMock())
     exchange = get_patched_exchange(mocker, default_conf)
     mocker.patch.object(
         _STRATEGY, 'analyze_ticker',
diff --git a/freqtrade/tests/test_freqtradebot.py b/freqtrade/tests/test_freqtradebot.py
index 5764c5b0f..fb03cd464 100644
--- a/freqtrade/tests/test_freqtradebot.py
+++ b/freqtrade/tests/test_freqtradebot.py
@@ -43,7 +43,6 @@ def patch_get_signal(freqtrade: FreqtradeBot, value=(True, False)) -> None:
     :return: None
     """
     freqtrade.strategy.get_signal = lambda e, s, t: value
-    freqtrade.exchange.get_candle_history = lambda p, i: None
     freqtrade.exchange.refresh_tickers = lambda p, i: True
 
 
@@ -545,7 +544,6 @@ def test_create_trade_no_signal(default_conf, fee, mocker) -> None:
     mocker.patch.multiple(
         'freqtrade.exchange.Exchange',
         validate_pairs=MagicMock(),
-        get_candle_history=MagicMock(return_value=20),
         get_balance=MagicMock(return_value=20),
         get_fee=fee,
     )

From 6d1c82a5faf6e575707e861275ae6dde414868f7 Mon Sep 17 00:00:00 2001
From: Matthias <xmatthias@outlook.com>
Date: Sun, 19 Aug 2018 19:39:22 +0200
Subject: [PATCH 45/47] Remove last refreence to `get_candle_history`

---
 freqtrade/optimize/backtesting.py            |  6 ++---
 freqtrade/tests/optimize/test_backtesting.py | 25 +++++++++++++-------
 2 files changed, 20 insertions(+), 11 deletions(-)

diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py
index 9e68318f7..d0b70afc7 100644
--- a/freqtrade/optimize/backtesting.py
+++ b/freqtrade/optimize/backtesting.py
@@ -330,15 +330,15 @@ class Backtesting(object):
         Run a backtesting end-to-end
         :return: None
         """
-        data = {}
+        data: Dict[str, Any] = {}
         pairs = self.config['exchange']['pair_whitelist']
         logger.info('Using stake_currency: %s ...', self.config['stake_currency'])
         logger.info('Using stake_amount: %s ...', self.config['stake_amount'])
 
         if self.config.get('live'):
             logger.info('Downloading data for all pairs in whitelist ...')
-            for pair in pairs:
-                data[pair] = self.exchange.get_candle_history(pair, self.ticker_interval)
+            self.exchange.refresh_tickers(pairs, self.ticker_interval)
+            data = self.exchange.klines
         else:
             logger.info('Using local backtesting data (using whitelist in given config) ...')
 
diff --git a/freqtrade/tests/optimize/test_backtesting.py b/freqtrade/tests/optimize/test_backtesting.py
index 1625cc5a3..a17867b3a 100644
--- a/freqtrade/tests/optimize/test_backtesting.py
+++ b/freqtrade/tests/optimize/test_backtesting.py
@@ -455,7 +455,7 @@ def test_backtesting_start(default_conf, mocker, caplog) -> None:
         return Arrow(2017, 11, 14, 21, 17), Arrow(2017, 11, 14, 22, 59)
 
     mocker.patch('freqtrade.optimize.load_data', mocked_load_data)
-    mocker.patch('freqtrade.exchange.Exchange.get_candle_history')
+    mocker.patch('freqtrade.exchange.Exchange.refresh_tickers', MagicMock())
     patch_exchange(mocker)
     mocker.patch.multiple(
         'freqtrade.optimize.backtesting.Backtesting',
@@ -490,7 +490,7 @@ def test_backtesting_start_no_data(default_conf, mocker, caplog) -> None:
         return Arrow(2017, 11, 14, 21, 17), Arrow(2017, 11, 14, 22, 59)
 
     mocker.patch('freqtrade.optimize.load_data', MagicMock(return_value={}))
-    mocker.patch('freqtrade.exchange.Exchange.get_candle_history')
+    mocker.patch('freqtrade.exchange.Exchange.refresh_tickers', MagicMock())
     patch_exchange(mocker)
     mocker.patch.multiple(
         'freqtrade.optimize.backtesting.Backtesting',
@@ -733,9 +733,14 @@ def test_backtest_record(default_conf, fee, mocker):
 
 def test_backtest_start_live(default_conf, mocker, caplog):
     default_conf['exchange']['pair_whitelist'] = ['UNITTEST/BTC']
-    mocker.patch('freqtrade.exchange.Exchange.get_candle_history',
-                 new=lambda s, n, i: _load_pair_as_ticks(n, i))
-    patch_exchange(mocker)
+
+    async def load_pairs(pair, timeframe, since):
+        return _load_pair_as_ticks(pair, timeframe)
+
+    api_mock = MagicMock()
+    api_mock.fetch_ohlcv = load_pairs
+
+    patch_exchange(mocker, api_mock)
     mocker.patch('freqtrade.optimize.backtesting.Backtesting.backtest', MagicMock())
     mocker.patch('freqtrade.optimize.backtesting.Backtesting._generate_text_table', MagicMock())
     mocker.patch('freqtrade.configuration.open', mocker.mock_open(
@@ -776,9 +781,13 @@ def test_backtest_start_live(default_conf, mocker, caplog):
 
 def test_backtest_start_multi_strat(default_conf, mocker, caplog):
     default_conf['exchange']['pair_whitelist'] = ['UNITTEST/BTC']
-    mocker.patch('freqtrade.exchange.Exchange.get_candle_history',
-                 new=lambda s, n, i: _load_pair_as_ticks(n, i))
-    patch_exchange(mocker)
+
+    async def load_pairs(pair, timeframe, since):
+        return _load_pair_as_ticks(pair, timeframe)
+    api_mock = MagicMock()
+    api_mock.fetch_ohlcv = load_pairs
+
+    patch_exchange(mocker, api_mock)
     backtestmock = MagicMock()
     mocker.patch('freqtrade.optimize.backtesting.Backtesting.backtest', backtestmock)
     gen_table_mock = MagicMock()

From ffd4469c1d4317510f8f279a8a16812ae218ef2f Mon Sep 17 00:00:00 2001
From: Matthias <xmatthias@outlook.com>
Date: Wed, 29 Aug 2018 19:56:38 +0200
Subject: [PATCH 46/47] fix typo, refresh_tickers does not need a return value

---
 freqtrade/exchange/__init__.py            | 4 +---
 freqtrade/tests/exchange/test_exchange.py | 4 ++--
 freqtrade/tests/test_freqtradebot.py      | 2 +-
 3 files changed, 4 insertions(+), 6 deletions(-)

diff --git a/freqtrade/exchange/__init__.py b/freqtrade/exchange/__init__.py
index bcb539996..fbc84c9b9 100644
--- a/freqtrade/exchange/__init__.py
+++ b/freqtrade/exchange/__init__.py
@@ -410,7 +410,7 @@ class Exchange(object):
         logger.info("downloaded %s with length %s.", pair, len(data))
         return data
 
-    def refresh_tickers(self, pair_list: List[str], ticker_interval: str) -> bool:
+    def refresh_tickers(self, pair_list: List[str], ticker_interval: str) -> None:
         """
         Refresh tickers asyncronously and return the result.
         """
@@ -418,8 +418,6 @@ class Exchange(object):
         asyncio.get_event_loop().run_until_complete(
             self.async_get_candles_history(pair_list, ticker_interval))
 
-        return True
-
     async def async_get_candles_history(self, pairs: List[str],
                                         tick_interval: str) -> List[Tuple[str, List]]:
         """Download ohlcv history for pair-list asyncronously """
diff --git a/freqtrade/tests/exchange/test_exchange.py b/freqtrade/tests/exchange/test_exchange.py
index f752da0a3..3c90f425c 100644
--- a/freqtrade/tests/exchange/test_exchange.py
+++ b/freqtrade/tests/exchange/test_exchange.py
@@ -581,10 +581,10 @@ def test_get_history(default_conf, mocker, caplog):
     ]
     pair = 'ETH/BTC'
 
-    async def mock_cacndle_hist(pair, tick_interval, since_ms):
+    async def mock_candle_hist(pair, tick_interval, since_ms):
         return pair, tick
 
-    exchange._async_get_candle_history = Mock(wraps=mock_cacndle_hist)
+    exchange._async_get_candle_history = Mock(wraps=mock_candle_hist)
     # one_call calculation * 1.8 should do 2 calls
     since = 5 * 60 * 500 * 1.8
     print(f"since = {since}")
diff --git a/freqtrade/tests/test_freqtradebot.py b/freqtrade/tests/test_freqtradebot.py
index 6971ec0dd..5e982f11a 100644
--- a/freqtrade/tests/test_freqtradebot.py
+++ b/freqtrade/tests/test_freqtradebot.py
@@ -43,7 +43,7 @@ def patch_get_signal(freqtrade: FreqtradeBot, value=(True, False)) -> None:
     :return: None
     """
     freqtrade.strategy.get_signal = lambda e, s, t: value
-    freqtrade.exchange.refresh_tickers = lambda p, i: True
+    freqtrade.exchange.refresh_tickers = lambda p, i: None
 
 
 def patch_RPCManager(mocker) -> MagicMock:

From e9deb928f6938375c2d3a008f789191c03ce326b Mon Sep 17 00:00:00 2001
From: Matthias <xmatthias@outlook.com>
Date: Sun, 2 Sep 2018 19:15:23 +0200
Subject: [PATCH 47/47] Fix bug when exchange result is empty

---
 freqtrade/exchange/__init__.py            |  3 ++-
 freqtrade/tests/exchange/test_exchange.py | 22 ++++++++++++++++++++--
 2 files changed, 22 insertions(+), 3 deletions(-)

diff --git a/freqtrade/exchange/__init__.py b/freqtrade/exchange/__init__.py
index fbc84c9b9..f663420e0 100644
--- a/freqtrade/exchange/__init__.py
+++ b/freqtrade/exchange/__init__.py
@@ -454,7 +454,8 @@ class Exchange(object):
             data = sorted(data, key=lambda x: x[0])
 
             # keeping last candle time as last refreshed time of the pair
-            self._pairs_last_refresh_time[pair] = data[-1][0] // 1000
+            if data:
+                self._pairs_last_refresh_time[pair] = data[-1][0] // 1000
 
             # keeping candles in cache
             self.klines[pair] = data
diff --git a/freqtrade/tests/exchange/test_exchange.py b/freqtrade/tests/exchange/test_exchange.py
index 3c90f425c..de720b3d9 100644
--- a/freqtrade/tests/exchange/test_exchange.py
+++ b/freqtrade/tests/exchange/test_exchange.py
@@ -635,8 +635,6 @@ async def test__async_get_candle_history(default_conf, mocker, caplog):
         ]
     ]
 
-    async def async_fetch_ohlcv(pair, timeframe, since):
-        return tick
     caplog.set_level(logging.DEBUG)
     exchange = get_patched_exchange(mocker, default_conf)
     # Monkey-patch async function
@@ -669,6 +667,26 @@ async def test__async_get_candle_history(default_conf, mocker, caplog):
                                                  (arrow.utcnow().timestamp - 2000) * 1000)
 
 
+@pytest.mark.asyncio
+async def test__async_get_candle_history_empty(default_conf, mocker, caplog):
+    """ Test empty exchange result """
+    tick = []
+
+    caplog.set_level(logging.DEBUG)
+    exchange = get_patched_exchange(mocker, default_conf)
+    # Monkey-patch async function
+    exchange._api_async.fetch_ohlcv = get_mock_coro([])
+
+    exchange = Exchange(default_conf)
+    pair = 'ETH/BTC'
+    res = await exchange._async_get_candle_history(pair, "5m")
+    assert type(res) is tuple
+    assert len(res) == 2
+    assert res[0] == pair
+    assert res[1] == tick
+    assert exchange._api_async.fetch_ohlcv.call_count == 1
+
+
 @pytest.mark.asyncio
 async def test_async_get_candles_history(default_conf, mocker):
     tick = [