diff --git a/.pyup.yml b/.pyup.yml index 01d4bba2a..462ae5783 100644 --- a/.pyup.yml +++ b/.pyup.yml @@ -22,6 +22,7 @@ requirements: - requirements.txt - requirements-dev.txt - requirements-plot.txt + - requirements-pi.txt # configure the branch prefix the bot is using diff --git a/Dockerfile b/Dockerfile index aeefc0722..e36766530 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM python:3.7.2-slim-stretch +FROM python:3.7.3-slim-stretch RUN apt-get update \ && apt-get -y install curl build-essential libssl-dev \ diff --git a/Dockerfile.pi b/Dockerfile.pi new file mode 100644 index 000000000..5184e2d37 --- /dev/null +++ b/Dockerfile.pi @@ -0,0 +1,40 @@ +FROM balenalib/raspberrypi3-debian:stretch + +RUN [ "cross-build-start" ] + +RUN apt-get update \ + && apt-get -y install wget curl build-essential libssl-dev libffi-dev \ + && apt-get clean + +# Prepare environment +RUN mkdir /freqtrade +WORKDIR /freqtrade + +# Install TA-lib +COPY build_helpers/ta-lib-0.4.0-src.tar.gz /freqtrade/ +RUN tar -xzf /freqtrade/ta-lib-0.4.0-src.tar.gz \ + && cd /freqtrade/ta-lib/ \ + && ./configure \ + && make \ + && make install \ + && rm /freqtrade/ta-lib-0.4.0-src.tar.gz + +ENV LD_LIBRARY_PATH /usr/local/lib + +# Install berryconda +RUN wget https://github.com/jjhelmus/berryconda/releases/download/v2.0.0/Berryconda3-2.0.0-Linux-armv7l.sh \ + && bash ./Berryconda3-2.0.0-Linux-armv7l.sh -b \ + && rm Berryconda3-2.0.0-Linux-armv7l.sh + +# Install dependencies +COPY requirements-pi.txt /freqtrade/ +RUN ~/berryconda3/bin/conda install -y numpy pandas scipy \ + && ~/berryconda3/bin/pip install -r requirements-pi.txt --no-cache-dir + +# Install and execute +COPY . /freqtrade/ +RUN ~/berryconda3/bin/pip install -e . --no-cache-dir + +RUN [ "cross-build-end" ] + +ENTRYPOINT ["/root/berryconda3/bin/python","./freqtrade/main.py"] diff --git a/config.json.example b/config.json.example index 323ff711e..94084434a 100644 --- a/config.json.example +++ b/config.json.example @@ -30,7 +30,8 @@ "secret": "your_exchange_secret", "ccxt_config": {"enableRateLimit": true}, "ccxt_async_config": { - "enableRateLimit": false + "enableRateLimit": true, + "rateLimit": 500 }, "pair_whitelist": [ "ETH/BTC", diff --git a/config_binance.json.example b/config_binance.json.example index 3d11f317a..ab57db88f 100644 --- a/config_binance.json.example +++ b/config_binance.json.example @@ -30,7 +30,8 @@ "secret": "your_exchange_secret", "ccxt_config": {"enableRateLimit": true}, "ccxt_async_config": { - "enableRateLimit": false + "enableRateLimit": true, + "rateLimit": 200 }, "pair_whitelist": [ "AST/BTC", diff --git a/config_full.json.example b/config_full.json.example index 7fc088585..806b5f718 100644 --- a/config_full.json.example +++ b/config_full.json.example @@ -63,6 +63,7 @@ "ccxt_config": {"enableRateLimit": true}, "ccxt_async_config": { "enableRateLimit": false, + "rateLimit": 500, "aiohttp_trust_env": false }, "pair_whitelist": [ diff --git a/docs/configuration.md b/docs/configuration.md index 81bac42cf..dc623a728 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -46,7 +46,6 @@ Mandatory Parameters are marked as **Required**. | `exchange.secret` | '' | API secret to use for the exchange. Only required when you are in production mode. | `exchange.pair_whitelist` | [] | List of currency to use by the bot. Can be overrided with `--dynamic-whitelist` param. | `exchange.pair_blacklist` | [] | List of currency the bot must avoid. Useful when using `--dynamic-whitelist` param. -| `exchange.ccxt_rate_limit` | True | DEPRECATED!! Have CCXT handle Exchange rate limits. Depending on the exchange, having this to false can lead to temporary bans from the exchange. | `exchange.ccxt_config` | None | Additional CCXT parameters passed to the regular ccxt instance. Parameters may differ from exchange to exchange and are documented in the [ccxt documentation](https://ccxt.readthedocs.io/en/latest/manual.html#instantiation) | `exchange.ccxt_async_config` | None | Additional CCXT parameters passed to the async ccxt instance. Parameters may differ from exchange to exchange and are documented in the [ccxt documentation](https://ccxt.readthedocs.io/en/latest/manual.html#instantiation) | `exchange.markets_refresh_interval` | 60 | The interval in minutes in which markets are reloaded. @@ -70,8 +69,8 @@ Mandatory Parameters are marked as **Required**. | `strategy` | DefaultStrategy | Defines Strategy class to use. | `strategy_path` | null | Adds an additional strategy lookup path (must be a folder). | `internals.process_throttle_secs` | 5 | **Required.** Set the process throttle. Value in second. -| `logfile` | | Specify Logfile. Uses a rolling strategy of 10 files, with 1Mb per file. | `internals.sd_notify` | false | Enables use of the sd_notify protocol to tell systemd service manager about changes in the bot state and issue keep-alive pings. See [here](installation.md#7-optional-configure-freqtrade-as-a-systemd-service) for more details. +| `logfile` | | Specify Logfile. Uses a rolling strategy of 10 files, with 1Mb per file. ### Parameters in the strategy @@ -217,6 +216,7 @@ The below is the default which is used if this is not configured in either strat the bot would recreate one. ### Understand order_time_in_force + The `order_time_in_force` configuration parameter defines the policy by which the order is executed on the exchange. Three commonly used time in force are: @@ -252,9 +252,9 @@ The possible values are: `gtc` (default), `fok` or `ioc`. This is an ongoing work. For now it is supported only for binance and only for buy orders. Please don't change the default value unless you know what you are doing. -### What values for exchange.name? +### Exchange configuration -Freqtrade is based on [CCXT library](https://github.com/ccxt/ccxt) that supports 115 cryptocurrency +Freqtrade is based on [CCXT library](https://github.com/ccxt/ccxt) that supports over 100 cryptocurrency exchange markets and trading APIs. The complete up-to-date list can be found in the [CCXT repo homepage](https://github.com/ccxt/ccxt/tree/master/python). However, the bot was tested with only Bittrex and Binance. @@ -266,6 +266,30 @@ The bot was tested with the following exchanges: Feel free to test other exchanges and submit your PR to improve the bot. +#### Sample exchange configuration + +A exchange configuration for "binance" would look as follows: + +```json +"exchange": { + "name": "binance", + "key": "your_exchange_key", + "secret": "your_exchange_secret", + "ccxt_config": {"enableRateLimit": true}, + "ccxt_async_config": { + "enableRateLimit": true, + "rateLimit": 200 + }, +``` + +This configuration enables binance, as well as rate limiting to avoid bans from the exchange. +`"rateLimit": 200` defines a wait-event of 0.2s between each call. This can also be completely disabled by setting `"enableRateLimit"` to false. + +!!! Note + Optimal settings for rate limiting depend on the exchange and the size of the whitelist, so an ideal parameter will vary on many other settings. + We try to provide sensible defaults per exchange where possible, if you encounter bans please make sure that `"enableRateLimit"` is enabled and increase the `"rateLimit"` parameter step by step. + + ### What values can be used for fiat_display_currency? The `fiat_display_currency` configuration parameter sets the base currency to use for the diff --git a/docs/developer.md b/docs/developer.md index 6fbcdc812..e7f79bc1c 100644 --- a/docs/developer.md +++ b/docs/developer.md @@ -95,9 +95,9 @@ git checkout develop git checkout -b new_release ``` -* edit `freqtrade/__init__.py` and add the desired version (for example `0.18.0`) +* Edit `freqtrade/__init__.py` and add the desired version (for example `0.18.0`) * Commit this part -* push that branch to the remote and create a PR +* push that branch to the remote and create a PR against the master branch ### create changelog from git commits @@ -108,10 +108,12 @@ git log --oneline --no-decorate --no-merges master..develop ### Create github release / tag +* Use the button "Draft a new release" in the Github UI (subsection releases) * Use the version-number specified as tag. * Use "master" as reference (this step comes after the above PR is merged). -* use the above changelog as release comment (as codeblock) +* Use the above changelog as release comment (as codeblock) ### After-release -* update version in develop to next valid version and postfix that with `-dev` (`0.18.0 -> 0.18.1-dev`) +* Update version in develop to next valid version and postfix that with `-dev` (`0.18.0 -> 0.18.1-dev`). +* Create a PR against develop to update that branch. diff --git a/docs/faq.md b/docs/faq.md index 60c1509e0..c551e3638 100644 --- a/docs/faq.md +++ b/docs/faq.md @@ -19,8 +19,7 @@ of course constantly aim to improve the bot but it will _always_ be a gamble, which should leave you with modest wins on monthly basis but you can't say much from few trades. -#### I’d like to change the stake amount. Can I just stop the bot with -/stop and then change the config.json and run it again? +#### I’d like to change the stake amount. Can I just stop the bot with /stop and then change the config.json and run it again? Not quite. Trades are persisted to a database but the configuration is currently only read when the bot is killed and restarted. `/stop` more @@ -31,16 +30,16 @@ like pauses. You can stop your bot, adjust settings and start it again. That's great. We have a nice backtesting and hyperoptimizing setup. See the tutorial [here|Testing-new-strategies-with-Hyperopt](bot-usage.md#hyperopt-commands). -#### Is there a setting to only SELL the coins being held and not -perform anymore BUYS? +#### Is there a setting to only SELL the coins being held and not perform anymore BUYS? You can use the `/forcesell all` command from Telegram. ### Hyperopt module #### How many epoch do I need to get a good Hyperopt result? + Per default Hyperopts without `-e` or `--epochs` parameter will only -run 100 epochs, means 100 evals of your triggers, guards, .... Too few +run 100 epochs, means 100 evals of your triggers, guards, ... Too few to find a great result (unless if you are very lucky), so you probably have to run it for 10.000 or more. But it will take an eternity to compute. @@ -64,10 +63,10 @@ Finding a great Hyperopt results takes time. If you wonder why it takes a while to find great hyperopt results This answer was written during the under the release 0.15.1, when we had: + - 8 triggers - 9 guards: let's say we evaluate even 10 values from each -- 1 stoploss calculation: let's say we want 10 values from that too to -be evaluated +- 1 stoploss calculation: let's say we want 10 values from that too to be evaluated The following calculation is still very rough and not very precise but it will give the idea. With only these triggers and guards there is @@ -82,10 +81,9 @@ of the search space. The Edge module is mostly a result of brainstorming of [@mishaker](https://github.com/mishaker) and [@creslinux](https://github.com/creslinux) freqtrade team members. You can find further info on expectancy, winrate, risk management and position size in the following sources: -* https://www.tradeciety.com/ultimate-math-guide-for-traders/ -* http://www.vantharp.com/tharp-concepts/expectancy.asp -* https://samuraitradingacademy.com/trading-expectancy/ -* https://www.learningmarkets.com/determining-expectancy-in-your-trading/ -* http://www.lonestocktrader.com/make-money-trading-positive-expectancy/ -* https://www.babypips.com/trading/trade-expectancy-matter - +- https://www.tradeciety.com/ultimate-math-guide-for-traders/ +- http://www.vantharp.com/tharp-concepts/expectancy.asp +- https://samuraitradingacademy.com/trading-expectancy/ +- https://www.learningmarkets.com/determining-expectancy-in-your-trading/ +- http://www.lonestocktrader.com/make-money-trading-positive-expectancy/ +- https://www.babypips.com/trading/trade-expectancy-matter diff --git a/docs/installation.md b/docs/installation.md index 995bc561b..23a6cbd23 100644 --- a/docs/installation.md +++ b/docs/installation.md @@ -315,7 +315,6 @@ Before installing FreqTrade on a Raspberry Pi running the official Raspbian Imag The following assumes that miniconda3 is installed and available in your environment. Last miniconda3 installation file use python 3.4, we will update to python 3.6 on this installation. It's recommended to use (mini)conda for this as installation/compilation of `numpy`, `scipy` and `pandas` takes a long time. -If you have installed it from (mini)conda, you can remove `numpy`, `scipy`, and `pandas` from `requirements.txt` before you install it with `pip`. Additional package to install on your Raspbian, `libffi-dev` required by cryptography (from python-telegram-bot). @@ -327,7 +326,7 @@ conda activate freqtrade conda install scipy pandas numpy sudo apt install libffi-dev -python3 -m pip install -r requirements.txt +python3 -m pip install -r requirements-pi.txt python3 -m pip install -e . ``` diff --git a/freqtrade/__init__.py b/freqtrade/__init__.py index 292613297..f84f5240e 100644 --- a/freqtrade/__init__.py +++ b/freqtrade/__init__.py @@ -1,5 +1,5 @@ """ FreqTrade bot """ -__version__ = '0.18.2-dev' +__version__ = '0.18.5-dev' class DependencyException(BaseException): diff --git a/freqtrade/arguments.py b/freqtrade/arguments.py index 8d7dac4bc..96f080bd2 100644 --- a/freqtrade/arguments.py +++ b/freqtrade/arguments.py @@ -247,6 +247,22 @@ class Arguments(object): dest='timerange', ) + parser.add_argument( + '--max_open_trades', + help='Specify max_open_trades to use.', + default=None, + type=int, + dest='max_open_trades', + ) + + parser.add_argument( + '--stake_amount', + help='Specify stake_amount.', + default=None, + type=float, + dest='stake_amount', + ) + @staticmethod def hyperopt_options(parser: argparse.ArgumentParser) -> None: """ @@ -267,7 +283,6 @@ class Arguments(object): dest='position_stacking', default=False ) - parser.add_argument( '--dmmp', '--disable-max-market-positions', help='Disable applying `max_open_trades` during backtest ' @@ -293,6 +308,13 @@ class Arguments(object): nargs='+', dest='spaces', ) + parser.add_argument( + '--print-all', + help='Print all results, not only the best ones.', + action='store_true', + dest='print_all', + default=False + ) def _build_subcommands(self) -> None: """ diff --git a/freqtrade/configuration.py b/freqtrade/configuration.py index b0dc0b212..a1128088d 100644 --- a/freqtrade/configuration.py +++ b/freqtrade/configuration.py @@ -9,11 +9,11 @@ from argparse import Namespace from logging.handlers import RotatingFileHandler from typing import Any, Dict, List, Optional -import ccxt from jsonschema import Draft4Validator, validators from jsonschema.exceptions import ValidationError, best_match from freqtrade import OperationalException, constants +from freqtrade.exchange import is_exchange_supported, supported_exchanges from freqtrade.misc import deep_merge_dicts from freqtrade.state import RunMode @@ -216,7 +216,7 @@ class Configuration(object): logger.info(f'Created data directory: {datadir}') return datadir - def _load_backtesting_config(self, config: Dict[str, Any]) -> Dict[str, Any]: + def _load_backtesting_config(self, config: Dict[str, Any]) -> Dict[str, Any]: # noqa: C901 """ Extract information for sys.argv and load Backtesting configuration :return: configuration as dictionary @@ -239,14 +239,24 @@ class Configuration(object): config.update({'position_stacking': True}) logger.info('Parameter --enable-position-stacking detected ...') - # If --disable-max-market-positions is used we add it to the configuration + # If --disable-max-market-positions or --max_open_trades is used we update configuration if 'use_max_market_positions' in self.args and not self.args.use_max_market_positions: config.update({'use_max_market_positions': False}) logger.info('Parameter --disable-max-market-positions detected ...') logger.info('max_open_trades set to unlimited ...') + elif 'max_open_trades' in self.args and self.args.max_open_trades: + config.update({'max_open_trades': self.args.max_open_trades}) + logger.info('Parameter --max_open_trades detected, ' + 'overriding max_open_trades to: %s ...', config.get('max_open_trades')) else: logger.info('Using max_open_trades: %s ...', config.get('max_open_trades')) + # If --stake_amount is used we update configuration + if 'stake_amount' in self.args and self.args.stake_amount: + config.update({'stake_amount': self.args.stake_amount}) + logger.info('Parameter --stake_amount detected, overriding stake_amount to: %s ...', + config.get('stake_amount')) + # If --timerange is used we add it to the configuration if 'timerange' in self.args and self.args.timerange: config.update({'timerange': self.args.timerange}) @@ -331,6 +341,10 @@ class Configuration(object): config.update({'spaces': self.args.spaces}) logger.info('Parameter -s/--spaces detected: %s', config.get('spaces')) + if 'print_all' in self.args and self.args.print_all: + config.update({'print_all': self.args.print_all}) + logger.info('Parameter --print-all detected: %s', config.get('print_all')) + return config def _validate_config_schema(self, conf: Dict[str, Any]) -> Dict[str, Any]: @@ -396,20 +410,16 @@ class Configuration(object): :return: True or raised an exception if the exchange if not supported """ exchange = config.get('exchange', {}).get('name').lower() - if exchange not in ccxt.exchanges: + if not is_exchange_supported(exchange): exception_msg = f'Exchange "{exchange}" not supported.\n' \ - f'The following exchanges are supported: {", ".join(ccxt.exchanges)}' + f'The following exchanges are supported: ' \ + f'{", ".join(supported_exchanges())}' logger.critical(exception_msg) raise OperationalException( exception_msg ) - # Depreciation warning - if 'ccxt_rate_limit' in config.get('exchange', {}): - logger.warning("`ccxt_rate_limit` has been deprecated in favor of " - "`ccxt_config` and `ccxt_async_config` and will be removed " - "in a future version.") logger.debug('Exchange "%s" supported', exchange) return True diff --git a/freqtrade/data/converter.py b/freqtrade/data/converter.py index 28749293b..77a3447da 100644 --- a/freqtrade/data/converter.py +++ b/freqtrade/data/converter.py @@ -2,9 +2,9 @@ Functions to convert data from one format to another """ import logging + import pandas as pd from pandas import DataFrame, to_datetime -from freqtrade.misc import timeframe_to_minutes logger = logging.getLogger(__name__) @@ -58,6 +58,8 @@ def ohlcv_fill_up_missing_data(dataframe: DataFrame, ticker_interval: str) -> Da using the previous close as price for "open", "high" "low" and "close", volume is set to 0 """ + from freqtrade.exchange import timeframe_to_minutes + ohlc_dict = { 'open': 'first', 'high': 'max', diff --git a/freqtrade/data/dataprovider.py b/freqtrade/data/dataprovider.py index 375b8bf5b..df4accf93 100644 --- a/freqtrade/data/dataprovider.py +++ b/freqtrade/data/dataprovider.py @@ -37,23 +37,23 @@ class DataProvider(object): @property def available_pairs(self) -> List[Tuple[str, str]]: """ - Return a list of tuples containing pair, tick_interval for which data is currently cached. + Return a list of tuples containing pair, ticker_interval for which data is currently cached. Should be whitelist + open trades. """ return list(self._exchange._klines.keys()) - def ohlcv(self, pair: str, tick_interval: str = None, copy: bool = True) -> DataFrame: + def ohlcv(self, pair: str, ticker_interval: str = None, copy: bool = True) -> DataFrame: """ get ohlcv data for the given pair as DataFrame Please check `available_pairs` to verify which pairs are currently cached. :param pair: pair to get the data for - :param tick_interval: ticker_interval to get pair for + :param ticker_interval: ticker_interval to get pair for :param copy: copy dataframe before returning. Use false only for RO operations (where the dataframe is not modified) """ if self.runmode in (RunMode.DRY_RUN, RunMode.LIVE): - if tick_interval: - pairtick = (pair, tick_interval) + if ticker_interval: + pairtick = (pair, ticker_interval) else: pairtick = (pair, self._config['ticker_interval']) @@ -65,7 +65,7 @@ class DataProvider(object): """ get stored historic ohlcv data :param pair: pair to get the data for - :param tick_interval: ticker_interval to get pair for + :param ticker_interval: ticker_interval to get pair for """ return load_pair_history(pair=pair, ticker_interval=ticker_interval, diff --git a/freqtrade/data/history.py b/freqtrade/data/history.py index 0ecc632e4..07a7f0b66 100644 --- a/freqtrade/data/history.py +++ b/freqtrade/data/history.py @@ -1,10 +1,10 @@ """ Handle historic data (ohlcv). -includes: + +Includes: * load data for a pair (or a list of pairs) from disk * download data from exchange and store to disk """ - import logging from pathlib import Path from typing import Optional, List, Dict, Tuple, Any @@ -15,8 +15,7 @@ from pandas import DataFrame from freqtrade import misc, OperationalException from freqtrade.arguments import TimeRange from freqtrade.data.converter import parse_ticker_dataframe -from freqtrade.exchange import Exchange -from freqtrade.misc import timeframe_to_minutes +from freqtrade.exchange import Exchange, timeframe_to_minutes logger = logging.getLogger(__name__) @@ -101,7 +100,7 @@ def load_pair_history(pair: str, download_pair_history(datadir=datadir, exchange=exchange, pair=pair, - tick_interval=ticker_interval, + ticker_interval=ticker_interval, timerange=timerange) pairdata = load_tickerdata_file(datadir, pair, ticker_interval, timerange=timerange) @@ -151,7 +150,7 @@ def make_testdata_path(datadir: Optional[Path]) -> Path: return datadir or (Path(__file__).parent.parent / "tests" / "testdata").resolve() -def load_cached_data_for_updating(filename: Path, tick_interval: str, +def load_cached_data_for_updating(filename: Path, ticker_interval: str, timerange: Optional[TimeRange]) -> Tuple[List[Any], Optional[int]]: """ @@ -165,7 +164,7 @@ def load_cached_data_for_updating(filename: Path, tick_interval: str, if timerange.starttype == 'date': since_ms = timerange.startts * 1000 elif timerange.stoptype == 'line': - num_minutes = timerange.stopts * timeframe_to_minutes(tick_interval) + num_minutes = timerange.stopts * timeframe_to_minutes(ticker_interval) since_ms = arrow.utcnow().shift(minutes=num_minutes).timestamp * 1000 # read the cached file @@ -192,7 +191,7 @@ def load_cached_data_for_updating(filename: Path, tick_interval: str, def download_pair_history(datadir: Optional[Path], exchange: Exchange, pair: str, - tick_interval: str = '5m', + ticker_interval: str = '5m', timerange: Optional[TimeRange] = None) -> bool: """ Download the latest ticker intervals from the exchange for the pair passed in parameters @@ -202,7 +201,7 @@ def download_pair_history(datadir: Optional[Path], Based on @Rybolov work: https://github.com/rybolov/freqtrade-data :param pair: pair to download - :param tick_interval: ticker interval + :param ticker_interval: ticker interval :param timerange: range of time to download :return: bool with success state @@ -210,17 +209,17 @@ def download_pair_history(datadir: Optional[Path], try: path = make_testdata_path(datadir) filepair = pair.replace("/", "_") - filename = path.joinpath(f'{filepair}-{tick_interval}.json') + filename = path.joinpath(f'{filepair}-{ticker_interval}.json') - logger.info('Download the pair: "%s", Interval: %s', pair, tick_interval) + logger.info('Download the pair: "%s", Interval: %s', pair, ticker_interval) - data, since_ms = load_cached_data_for_updating(filename, tick_interval, timerange) + data, since_ms = load_cached_data_for_updating(filename, ticker_interval, timerange) logger.debug("Current Start: %s", misc.format_ms_time(data[1][0]) if data else 'None') logger.debug("Current End: %s", misc.format_ms_time(data[-1][0]) if data else 'None') # Default since_ms to 30 days if nothing is given - new_data = exchange.get_history(pair=pair, tick_interval=tick_interval, + new_data = exchange.get_history(pair=pair, ticker_interval=ticker_interval, since_ms=since_ms if since_ms else int(arrow.utcnow().shift(days=-30).float_timestamp) * 1000) @@ -233,5 +232,5 @@ def download_pair_history(datadir: Optional[Path], return True except BaseException: logger.info('Failed to download the pair: "%s", Interval: %s', - pair, tick_interval) + pair, ticker_interval) return False diff --git a/freqtrade/exchange/__init__.py b/freqtrade/exchange/__init__.py index f6db04da6..3c90e69ee 100644 --- a/freqtrade/exchange/__init__.py +++ b/freqtrade/exchange/__init__.py @@ -1,3 +1,8 @@ from freqtrade.exchange.exchange import Exchange # noqa: F401 +from freqtrade.exchange.exchange import (is_exchange_supported, # noqa: F401 + supported_exchanges) +from freqtrade.exchange.exchange import (timeframe_to_seconds, # noqa: F401 + timeframe_to_minutes, + timeframe_to_msecs) from freqtrade.exchange.kraken import Kraken # noqa: F401 from freqtrade.exchange.binance import Binance # noqa: F401 diff --git a/freqtrade/exchange/exchange.py b/freqtrade/exchange/exchange.py index 03f6767d4..94673fdc3 100644 --- a/freqtrade/exchange/exchange.py +++ b/freqtrade/exchange/exchange.py @@ -1,5 +1,7 @@ # pragma pylint: disable=W0603 -""" Cryptocurrency Exchanges support """ +""" +Cryptocurrency Exchanges support +""" import logging import inspect from random import randint @@ -16,7 +18,6 @@ from pandas import DataFrame from freqtrade import (constants, DependencyException, OperationalException, TemporaryError, InvalidOrderException) from freqtrade.data.converter import parse_ticker_dataframe -from freqtrade.misc import timeframe_to_seconds, timeframe_to_msecs logger = logging.getLogger(__name__) @@ -138,7 +139,7 @@ class Exchange(object): # Find matching class for the given exchange name name = exchange_config['name'] - if name not in ccxt_module.exchanges: + if not is_exchange_supported(name, ccxt_module): raise OperationalException(f'Exchange {name} is not supported') ex_config = { @@ -146,7 +147,6 @@ class Exchange(object): 'secret': exchange_config.get('secret'), 'password': exchange_config.get('password'), 'uid': exchange_config.get('uid', ''), - 'enableRateLimit': exchange_config.get('ccxt_rate_limit', True) } if ccxt_kwargs: logger.info('Applying additional ccxt config: %s', ccxt_kwargs) @@ -490,26 +490,26 @@ class Exchange(object): logger.info("returning cached ticker-data for %s", pair) return self._cached_ticker[pair] - def get_history(self, pair: str, tick_interval: str, + def get_history(self, pair: str, ticker_interval: str, since_ms: int) -> List: """ Gets candle history using asyncio and returns the list of candles. Handles all async doing. """ return asyncio.get_event_loop().run_until_complete( - self._async_get_history(pair=pair, tick_interval=tick_interval, + self._async_get_history(pair=pair, ticker_interval=ticker_interval, since_ms=since_ms)) async def _async_get_history(self, pair: str, - tick_interval: str, + ticker_interval: str, since_ms: int) -> List: # Assume exchange returns 500 candles _LIMIT = 500 - one_call = timeframe_to_msecs(tick_interval) * _LIMIT + one_call = timeframe_to_msecs(ticker_interval) * _LIMIT logger.debug("one_call: %s msecs", one_call) input_coroutines = [self._async_get_candle_history( - pair, tick_interval, since) for since in + pair, ticker_interval, since) for since in range(since_ms, arrow.utcnow().timestamp * 1000, one_call)] tickers = await asyncio.gather(*input_coroutines, return_exceptions=True) @@ -549,14 +549,14 @@ class Exchange(object): logger.warning("Async code raised an exception: %s", res.__class__.__name__) continue pair = res[0] - tick_interval = res[1] + ticker_interval = res[1] ticks = res[2] # keeping last candle time as last refreshed time of the pair if ticks: - self._pairs_last_refresh_time[(pair, tick_interval)] = ticks[-1][0] // 1000 + self._pairs_last_refresh_time[(pair, ticker_interval)] = ticks[-1][0] // 1000 # keeping parsed dataframe in cache - self._klines[(pair, tick_interval)] = parse_ticker_dataframe( - ticks, tick_interval, fill_missing=True) + self._klines[(pair, ticker_interval)] = parse_ticker_dataframe( + ticks, ticker_interval, fill_missing=True) return tickers def _now_is_time_to_refresh(self, pair: str, ticker_interval: str) -> bool: @@ -567,17 +567,17 @@ class Exchange(object): + interval_in_sec) >= arrow.utcnow().timestamp) @retrier_async - async def _async_get_candle_history(self, pair: str, tick_interval: str, + async def _async_get_candle_history(self, pair: str, ticker_interval: str, since_ms: Optional[int] = None) -> Tuple[str, str, List]: """ Asyncronously gets candle histories using fetch_ohlcv - returns tuple: (pair, tick_interval, ohlcv_list) + returns tuple: (pair, ticker_interval, ohlcv_list) """ try: # fetch ohlcv asynchronously - logger.debug("fetching %s, %s since %s ...", pair, tick_interval, since_ms) + logger.debug("fetching %s, %s since %s ...", pair, ticker_interval, since_ms) - data = await self._api_async.fetch_ohlcv(pair, timeframe=tick_interval, + data = await self._api_async.fetch_ohlcv(pair, timeframe=ticker_interval, since=since_ms) # Because some exchange sort Tickers ASC and other DESC. @@ -589,9 +589,9 @@ class Exchange(object): data = sorted(data, key=lambda x: x[0]) except IndexError: logger.exception("Error loading %s. Result was %s.", pair, data) - return pair, tick_interval, [] - logger.debug("done fetching %s, %s ...", pair, tick_interval) - return pair, tick_interval, data + return pair, ticker_interval, [] + logger.debug("done fetching %s, %s ...", pair, ticker_interval) + return pair, ticker_interval, data except ccxt.NotSupported as e: raise OperationalException( @@ -690,3 +690,34 @@ class Exchange(object): f'Could not get fee info due to {e.__class__.__name__}. Message: {e}') except ccxt.BaseError as e: raise OperationalException(e) + + +def is_exchange_supported(exchange: str, ccxt_module=None) -> bool: + return exchange in supported_exchanges(ccxt_module) + + +def supported_exchanges(ccxt_module=None) -> List[str]: + return ccxt_module.exchanges if ccxt_module is not None else ccxt.exchanges + + +def timeframe_to_seconds(ticker_interval: str) -> int: + """ + Translates the timeframe interval value written in the human readable + form ('1m', '5m', '1h', '1d', '1w', etc.) to the number + of seconds for one timeframe interval. + """ + return ccxt.Exchange.parse_timeframe(ticker_interval) + + +def timeframe_to_minutes(ticker_interval: str) -> int: + """ + Same as above, but returns minutes. + """ + return ccxt.Exchange.parse_timeframe(ticker_interval) // 60 + + +def timeframe_to_msecs(ticker_interval: str) -> int: + """ + Same as above, but returns milliseconds. + """ + return ccxt.Exchange.parse_timeframe(ticker_interval) * 1000 diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index bceab8b23..8b29d6d40 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -16,7 +16,7 @@ from freqtrade import (DependencyException, OperationalException, InvalidOrderEx from freqtrade.data.converter import order_book_to_dataframe from freqtrade.data.dataprovider import DataProvider from freqtrade.edge import Edge -from freqtrade.misc import timeframe_to_minutes +from freqtrade.exchange import timeframe_to_minutes from freqtrade.persistence import Trade from freqtrade.rpc import RPCManager, RPCMessageType from freqtrade.resolvers import ExchangeResolver, StrategyResolver, PairListResolver @@ -460,7 +460,7 @@ class FreqtradeBot(object): def get_real_amount(self, trade: Trade, order: Dict) -> float: """ Get real amount for the trade - Necessary for self.exchanges which charge fees in base currency (e.g. binance) + Necessary for exchanges which charge fees in base currency (e.g. binance) """ order_amount = order['amount'] # Only run for closed orders @@ -522,6 +522,10 @@ class FreqtradeBot(object): trade.update(order) + # Updating wallets when order is closed + if not trade.is_open: + self.wallets.update() + def get_sell_rate(self, pair: str, refresh: bool) -> float: """ Get sell rate - either using get-ticker bid or first bid based on orderbook diff --git a/freqtrade/misc.py b/freqtrade/misc.py index d066878be..9d37214e4 100644 --- a/freqtrade/misc.py +++ b/freqtrade/misc.py @@ -1,18 +1,17 @@ """ Various tool function for Freqtrade and scripts """ - import gzip import logging import re from datetime import datetime from typing import Dict -from ccxt import Exchange import numpy as np from pandas import DataFrame import rapidjson + logger = logging.getLogger(__name__) @@ -132,26 +131,3 @@ def deep_merge_dicts(source, destination): destination[key] = value return destination - - -def timeframe_to_seconds(ticker_interval: str) -> int: - """ - Translates the timeframe interval value written in the human readable - form ('1m', '5m', '1h', '1d', '1w', etc.) to the number - of seconds for one timeframe interval. - """ - return Exchange.parse_timeframe(ticker_interval) - - -def timeframe_to_minutes(ticker_interval: str) -> int: - """ - Same as above, but returns minutes. - """ - return Exchange.parse_timeframe(ticker_interval) // 60 - - -def timeframe_to_msecs(ticker_interval: str) -> int: - """ - Same as above, but returns milliseconds. - """ - return Exchange.parse_timeframe(ticker_interval) * 1000 diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index 513498766..a4324a021 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -19,12 +19,14 @@ from freqtrade.arguments import Arguments from freqtrade.configuration import Configuration from freqtrade.data import history from freqtrade.data.dataprovider import DataProvider -from freqtrade.misc import file_dump_json, timeframe_to_minutes +from freqtrade.exchange import timeframe_to_minutes +from freqtrade.misc import file_dump_json from freqtrade.persistence import Trade from freqtrade.resolvers import ExchangeResolver, StrategyResolver from freqtrade.state import RunMode from freqtrade.strategy.interface import SellType, IStrategy + logger = logging.getLogger(__name__) diff --git a/freqtrade/optimize/hyperopt.py b/freqtrade/optimize/hyperopt.py index f6d39f11c..b37027244 100644 --- a/freqtrade/optimize/hyperopt.py +++ b/freqtrade/optimize/hyperopt.py @@ -115,7 +115,7 @@ class Hyperopt(Backtesting): """ Log results if it is better than any previous evaluation """ - if results['loss'] < self.current_best_loss: + if self.config.get('print_all', False) or results['loss'] < self.current_best_loss: current = results['current_tries'] total = results['total_tries'] res = results['result'] diff --git a/freqtrade/resolvers/strategy_resolver.py b/freqtrade/resolvers/strategy_resolver.py index 44cc3fe76..b2743a417 100644 --- a/freqtrade/resolvers/strategy_resolver.py +++ b/freqtrade/resolvers/strategy_resolver.py @@ -157,12 +157,15 @@ class StrategyResolver(IResolver): getfullargspec(strategy.populate_indicators).args) strategy._buy_fun_len = len(getfullargspec(strategy.populate_buy_trend).args) strategy._sell_fun_len = len(getfullargspec(strategy.populate_sell_trend).args) - - return import_strategy(strategy, config=config) + try: + return import_strategy(strategy, config=config) + except TypeError as e: + logger.warning( + f"Impossible to load strategy '{strategy}' from {_path}. Error: {e}") except FileNotFoundError: logger.warning('Path "%s" does not exist', _path.relative_to(Path.cwd())) raise ImportError( - "Impossible to load Strategy '{}'. This class does not exist" - " or contains Python code errors".format(strategy_name) + f"Impossible to load Strategy '{strategy_name}'. This class does not exist" + " or contains Python code errors" ) diff --git a/freqtrade/rpc/telegram.py b/freqtrade/rpc/telegram.py index ca108b17e..c61193d29 100644 --- a/freqtrade/rpc/telegram.py +++ b/freqtrade/rpc/telegram.py @@ -20,6 +20,9 @@ logger = logging.getLogger(__name__) logger.debug('Included module rpc.telegram ...') +MAX_TELEGRAM_MESSAGE_LENGTH = 4096 + + def authorized_only(command_handler: Callable[..., None]) -> Callable[..., Any]: """ Decorator to check if the message comes from the correct chat_id @@ -266,7 +269,8 @@ class Telegram(RPC): headers=[ 'Day', f'Profit {stake_cur}', - f'Profit {fiat_disp_cur}' + f'Profit {fiat_disp_cur}', + f'Trades' ], tablefmt='simple') message = f'Daily Profit over the last {timescale} days:\n
{stats_tab}
' @@ -327,13 +331,20 @@ class Telegram(RPC): output = '' for currency in result['currencies']: if currency['est_btc'] > 0.0001: - output += "*{currency}:*\n" \ + curr_output = "*{currency}:*\n" \ "\t`Available: {available: .8f}`\n" \ "\t`Balance: {balance: .8f}`\n" \ "\t`Pending: {pending: .8f}`\n" \ "\t`Est. BTC: {est_btc: .8f}`\n".format(**currency) else: - output += "*{currency}:* not showing <1$ amount \n".format(**currency) + curr_output = "*{currency}:* not showing <1$ amount \n".format(**currency) + + # Handle overflowing messsage length + if len(output + curr_output) >= MAX_TELEGRAM_MESSAGE_LENGTH: + self._send_msg(output, bot=bot) + output = curr_output + else: + output += curr_output output += "\n*Estimated Value*:\n" \ "\t`BTC: {total: .8f}`\n" \ diff --git a/freqtrade/strategy/__init__.py b/freqtrade/strategy/__init__.py index b29e26ef9..c62bfe5dc 100644 --- a/freqtrade/strategy/__init__.py +++ b/freqtrade/strategy/__init__.py @@ -6,6 +6,7 @@ from freqtrade.strategy.interface import IStrategy # Import Default-Strategy to have hyperopt correctly resolve from freqtrade.strategy.default_strategy import DefaultStrategy # noqa: F401 + logger = logging.getLogger(__name__) @@ -16,7 +17,6 @@ def import_strategy(strategy: IStrategy, config: dict) -> IStrategy: """ # Copy all attributes from base class and class - comb = {**strategy.__class__.__dict__, **strategy.__dict__} # Delete '_abc_impl' from dict as deepcopy fails on 3.7 with @@ -26,6 +26,7 @@ def import_strategy(strategy: IStrategy, config: dict) -> IStrategy: del comb['_abc_impl'] attr = deepcopy(comb) + # Adjust module name attr['__module__'] = 'freqtrade.strategy' diff --git a/freqtrade/strategy/interface.py b/freqtrade/strategy/interface.py index 646bd2a94..caf56f13e 100644 --- a/freqtrade/strategy/interface.py +++ b/freqtrade/strategy/interface.py @@ -13,10 +13,11 @@ import arrow from pandas import DataFrame from freqtrade.data.dataprovider import DataProvider -from freqtrade.misc import timeframe_to_minutes +from freqtrade.exchange import timeframe_to_minutes from freqtrade.persistence import Trade from freqtrade.wallets import Wallets + logger = logging.getLogger(__name__) diff --git a/freqtrade/tests/data/test_dataprovider.py b/freqtrade/tests/data/test_dataprovider.py index b17bba273..993f0b59b 100644 --- a/freqtrade/tests/data/test_dataprovider.py +++ b/freqtrade/tests/data/test_dataprovider.py @@ -9,31 +9,31 @@ from freqtrade.tests.conftest import get_patched_exchange def test_ohlcv(mocker, default_conf, ticker_history): default_conf["runmode"] = RunMode.DRY_RUN - tick_interval = default_conf["ticker_interval"] + ticker_interval = default_conf["ticker_interval"] exchange = get_patched_exchange(mocker, default_conf) - exchange._klines[("XRP/BTC", tick_interval)] = ticker_history - exchange._klines[("UNITTEST/BTC", tick_interval)] = ticker_history + exchange._klines[("XRP/BTC", ticker_interval)] = ticker_history + exchange._klines[("UNITTEST/BTC", ticker_interval)] = ticker_history dp = DataProvider(default_conf, exchange) assert dp.runmode == RunMode.DRY_RUN - assert ticker_history.equals(dp.ohlcv("UNITTEST/BTC", tick_interval)) - assert isinstance(dp.ohlcv("UNITTEST/BTC", tick_interval), DataFrame) - assert dp.ohlcv("UNITTEST/BTC", tick_interval) is not ticker_history - assert dp.ohlcv("UNITTEST/BTC", tick_interval, copy=False) is ticker_history - assert not dp.ohlcv("UNITTEST/BTC", tick_interval).empty - assert dp.ohlcv("NONESENSE/AAA", tick_interval).empty + assert ticker_history.equals(dp.ohlcv("UNITTEST/BTC", ticker_interval)) + assert isinstance(dp.ohlcv("UNITTEST/BTC", ticker_interval), DataFrame) + assert dp.ohlcv("UNITTEST/BTC", ticker_interval) is not ticker_history + assert dp.ohlcv("UNITTEST/BTC", ticker_interval, copy=False) is ticker_history + assert not dp.ohlcv("UNITTEST/BTC", ticker_interval).empty + assert dp.ohlcv("NONESENSE/AAA", ticker_interval).empty # Test with and without parameter - assert dp.ohlcv("UNITTEST/BTC", tick_interval).equals(dp.ohlcv("UNITTEST/BTC")) + assert dp.ohlcv("UNITTEST/BTC", ticker_interval).equals(dp.ohlcv("UNITTEST/BTC")) default_conf["runmode"] = RunMode.LIVE dp = DataProvider(default_conf, exchange) assert dp.runmode == RunMode.LIVE - assert isinstance(dp.ohlcv("UNITTEST/BTC", tick_interval), DataFrame) + assert isinstance(dp.ohlcv("UNITTEST/BTC", ticker_interval), DataFrame) default_conf["runmode"] = RunMode.BACKTEST dp = DataProvider(default_conf, exchange) assert dp.runmode == RunMode.BACKTEST - assert dp.ohlcv("UNITTEST/BTC", tick_interval).empty + assert dp.ohlcv("UNITTEST/BTC", ticker_interval).empty def test_historic_ohlcv(mocker, default_conf, ticker_history): @@ -54,15 +54,15 @@ def test_historic_ohlcv(mocker, default_conf, ticker_history): def test_available_pairs(mocker, default_conf, ticker_history): exchange = get_patched_exchange(mocker, default_conf) - tick_interval = default_conf["ticker_interval"] - exchange._klines[("XRP/BTC", tick_interval)] = ticker_history - exchange._klines[("UNITTEST/BTC", tick_interval)] = ticker_history + ticker_interval = default_conf["ticker_interval"] + exchange._klines[("XRP/BTC", ticker_interval)] = ticker_history + exchange._klines[("UNITTEST/BTC", ticker_interval)] = ticker_history dp = DataProvider(default_conf, exchange) assert len(dp.available_pairs) == 2 assert dp.available_pairs == [ - ("XRP/BTC", tick_interval), - ("UNITTEST/BTC", tick_interval), + ("XRP/BTC", ticker_interval), + ("UNITTEST/BTC", ticker_interval), ] @@ -71,10 +71,10 @@ def test_refresh(mocker, default_conf, ticker_history): mocker.patch("freqtrade.exchange.Exchange.refresh_latest_ohlcv", refresh_mock) exchange = get_patched_exchange(mocker, default_conf, id="binance") - tick_interval = default_conf["ticker_interval"] - pairs = [("XRP/BTC", tick_interval), ("UNITTEST/BTC", tick_interval)] + ticker_interval = default_conf["ticker_interval"] + pairs = [("XRP/BTC", ticker_interval), ("UNITTEST/BTC", ticker_interval)] - pairs_non_trad = [("ETH/USDT", tick_interval), ("BTC/TUSD", "1h")] + pairs_non_trad = [("ETH/USDT", ticker_interval), ("BTC/TUSD", "1h")] dp = DataProvider(default_conf, exchange) dp.refresh(pairs) diff --git a/freqtrade/tests/data/test_history.py b/freqtrade/tests/data/test_history.py index bc859b325..c0b1cade3 100644 --- a/freqtrade/tests/data/test_history.py +++ b/freqtrade/tests/data/test_history.py @@ -242,10 +242,10 @@ def test_download_pair_history(ticker_history_list, mocker, default_conf) -> Non assert download_pair_history(datadir=None, exchange=exchange, pair='MEME/BTC', - tick_interval='1m') + ticker_interval='1m') assert download_pair_history(datadir=None, exchange=exchange, pair='CFI/BTC', - tick_interval='1m') + ticker_interval='1m') assert not exchange._pairs_last_refresh_time assert os.path.isfile(file1_1) is True assert os.path.isfile(file2_1) is True @@ -259,10 +259,10 @@ def test_download_pair_history(ticker_history_list, mocker, default_conf) -> Non assert download_pair_history(datadir=None, exchange=exchange, pair='MEME/BTC', - tick_interval='5m') + ticker_interval='5m') assert download_pair_history(datadir=None, exchange=exchange, pair='CFI/BTC', - tick_interval='5m') + ticker_interval='5m') assert not exchange._pairs_last_refresh_time assert os.path.isfile(file1_5) is True assert os.path.isfile(file2_5) is True @@ -280,8 +280,8 @@ def test_download_pair_history2(mocker, default_conf) -> None: json_dump_mock = mocker.patch('freqtrade.misc.file_dump_json', return_value=None) mocker.patch('freqtrade.exchange.Exchange.get_history', return_value=tick) exchange = get_patched_exchange(mocker, default_conf) - download_pair_history(None, exchange, pair="UNITTEST/BTC", tick_interval='1m') - download_pair_history(None, exchange, pair="UNITTEST/BTC", tick_interval='3m') + download_pair_history(None, exchange, pair="UNITTEST/BTC", ticker_interval='1m') + download_pair_history(None, exchange, pair="UNITTEST/BTC", ticker_interval='3m') assert json_dump_mock.call_count == 2 @@ -298,7 +298,7 @@ def test_download_backtesting_data_exception(ticker_history, mocker, caplog, def assert not download_pair_history(datadir=None, exchange=exchange, pair='MEME/BTC', - tick_interval='1m') + ticker_interval='1m') # clean files freshly downloaded _clean_test_file(file1_1) _clean_test_file(file1_5) diff --git a/freqtrade/tests/exchange/test_exchange.py b/freqtrade/tests/exchange/test_exchange.py index 66bc47405..9e471d551 100644 --- a/freqtrade/tests/exchange/test_exchange.py +++ b/freqtrade/tests/exchange/test_exchange.py @@ -941,8 +941,8 @@ def test_get_history(default_conf, mocker, caplog, exchange_name): ] pair = 'ETH/BTC' - async def mock_candle_hist(pair, tick_interval, since_ms): - return pair, tick_interval, tick + async def mock_candle_hist(pair, ticker_interval, since_ms): + return pair, ticker_interval, tick exchange._async_get_candle_history = Mock(wraps=mock_candle_hist) # one_call calculation * 1.8 should do 2 calls @@ -1038,7 +1038,7 @@ async def test__async_get_candle_history(default_conf, mocker, caplog, exchange_ # exchange = Exchange(default_conf) await async_ccxt_exception(mocker, default_conf, MagicMock(), "_async_get_candle_history", "fetch_ohlcv", - pair='ABCD/BTC', tick_interval=default_conf['ticker_interval']) + pair='ABCD/BTC', ticker_interval=default_conf['ticker_interval']) api_mock = MagicMock() with pytest.raises(OperationalException, match=r'Could not fetch ticker data*'): diff --git a/freqtrade/tests/optimize/__init__.py b/freqtrade/tests/optimize/__init__.py index 227050770..457113cb7 100644 --- a/freqtrade/tests/optimize/__init__.py +++ b/freqtrade/tests/optimize/__init__.py @@ -3,7 +3,7 @@ from typing import NamedTuple, List import arrow from pandas import DataFrame -from freqtrade.misc import timeframe_to_minutes +from freqtrade.exchange import timeframe_to_minutes from freqtrade.strategy.interface import SellType ticker_start_time = arrow.get(2018, 10, 3) diff --git a/freqtrade/tests/optimize/test_optimize.py b/freqtrade/tests/optimize/test_optimize.py index 088743038..d746aa44f 100644 --- a/freqtrade/tests/optimize/test_optimize.py +++ b/freqtrade/tests/optimize/test_optimize.py @@ -2,7 +2,7 @@ from freqtrade import optimize from freqtrade.arguments import TimeRange from freqtrade.data import history -from freqtrade.misc import timeframe_to_minutes +from freqtrade.exchange import timeframe_to_minutes from freqtrade.strategy.default_strategy import DefaultStrategy from freqtrade.tests.conftest import log_has, patch_exchange diff --git a/freqtrade/tests/rpc/test_rpc_telegram.py b/freqtrade/tests/rpc/test_rpc_telegram.py index 8f43d7ed0..f2f3f3945 100644 --- a/freqtrade/tests/rpc/test_rpc_telegram.py +++ b/freqtrade/tests/rpc/test_rpc_telegram.py @@ -4,7 +4,8 @@ import re from datetime import datetime -from random import randint +from random import choice, randint +from string import ascii_uppercase from unittest.mock import MagicMock, PropertyMock import arrow @@ -20,7 +21,8 @@ from freqtrade.rpc import RPCMessageType from freqtrade.rpc.telegram import Telegram, authorized_only from freqtrade.state import State from freqtrade.strategy.interface import SellType -from freqtrade.tests.conftest import (get_patched_freqtradebot, log_has, patch_exchange) +from freqtrade.tests.conftest import (get_patched_freqtradebot, log_has, + patch_exchange) from freqtrade.tests.test_freqtradebot import patch_get_signal @@ -587,6 +589,45 @@ def test_balance_handle_empty_response(default_conf, update, mocker) -> None: assert 'all balances are zero' in result +def test_balance_handle_too_large_response(default_conf, update, mocker) -> None: + balances = [] + for i in range(100): + curr = choice(ascii_uppercase) + choice(ascii_uppercase) + choice(ascii_uppercase) + balances.append({ + 'currency': curr, + 'available': 1.0, + 'pending': 0.5, + 'balance': i, + 'est_btc': 1 + }) + mocker.patch('freqtrade.rpc.rpc.RPC._rpc_balance', return_value={ + 'currencies': balances, + 'total': 100.0, + 'symbol': 100.0, + 'value': 1000.0, + }) + + msg_mock = MagicMock() + mocker.patch.multiple( + 'freqtrade.rpc.telegram.Telegram', + _init=MagicMock(), + _send_msg=msg_mock + ) + + freqtradebot = get_patched_freqtradebot(mocker, default_conf) + patch_get_signal(freqtradebot, (True, False)) + + telegram = Telegram(freqtradebot) + + telegram._balance(bot=MagicMock(), update=update) + assert msg_mock.call_count > 1 + # Test if wrap happens around 4000 - + # and each single currency-output is around 120 characters long so we need + # an offset to avoid random test failures + assert len(msg_mock.call_args_list[0][0][0]) < 4096 + assert len(msg_mock.call_args_list[0][0][0]) > (4096 - 120) + + def test_start_handle(default_conf, update, mocker) -> None: msg_mock = MagicMock() mocker.patch.multiple( diff --git a/freqtrade/tests/strategy/test_strategy.py b/freqtrade/tests/strategy/test_strategy.py index b63180d1e..2ed2567f9 100644 --- a/freqtrade/tests/strategy/test_strategy.py +++ b/freqtrade/tests/strategy/test_strategy.py @@ -1,17 +1,19 @@ # pragma pylint: disable=missing-docstring, protected-access, C0103 import logging +import warnings from base64 import urlsafe_b64encode from os import path from pathlib import Path -import warnings +from unittest.mock import Mock import pytest from pandas import DataFrame +from freqtrade.resolvers import StrategyResolver from freqtrade.strategy import import_strategy from freqtrade.strategy.default_strategy import DefaultStrategy from freqtrade.strategy.interface import IStrategy -from freqtrade.resolvers import StrategyResolver +from freqtrade.tests.conftest import log_has_re def test_import_strategy(caplog): @@ -94,6 +96,16 @@ def test_load_not_found_strategy(): strategy._load_strategy(strategy_name='NotFoundStrategy', config={}) +def test_load_staticmethod_importerror(mocker, caplog): + mocker.patch("freqtrade.resolvers.strategy_resolver.import_strategy", Mock( + side_effect=TypeError("can't pickle staticmethod objects"))) + with pytest.raises(ImportError, + match=r"Impossible to load Strategy 'DefaultStrategy'." + r" This class does not exist or contains Python code errors"): + StrategyResolver() + assert log_has_re(r".*Error: can't pickle staticmethod objects", caplog.record_tuples) + + def test_strategy(result): config = {'strategy': 'DefaultStrategy'} diff --git a/freqtrade/tests/test_configuration.py b/freqtrade/tests/test_configuration.py index af94506b7..6fcae6aaa 100644 --- a/freqtrade/tests/test_configuration.py +++ b/freqtrade/tests/test_configuration.py @@ -494,15 +494,6 @@ def test_check_exchange(default_conf, caplog) -> None: ): configuration.check_exchange(default_conf) - # Test ccxt_rate_limit depreciation - default_conf.get('exchange').update({'name': 'binance'}) - default_conf['exchange']['ccxt_rate_limit'] = True - configuration.check_exchange(default_conf) - assert log_has("`ccxt_rate_limit` has been deprecated in favor of " - "`ccxt_config` and `ccxt_async_config` and will be removed " - "in a future version.", - caplog.record_tuples) - def test_cli_verbose_with_params(default_conf, mocker, caplog) -> None: mocker.patch('freqtrade.configuration.open', mocker.mock_open( diff --git a/freqtrade/tests/test_freqtradebot.py b/freqtrade/tests/test_freqtradebot.py index 103c0777e..67b05ac3e 100644 --- a/freqtrade/tests/test_freqtradebot.py +++ b/freqtrade/tests/test_freqtradebot.py @@ -1407,7 +1407,8 @@ def test_update_trade_state_withorderdict(default_conf, trades_for_order, limit_ amount=amount, exchange='binance', open_rate=0.245441, - open_order_id="123456" + open_order_id="123456", + is_open=True, ) freqtrade.update_trade_state(trade, limit_buy_order) assert trade.amount != amount @@ -1432,6 +1433,35 @@ def test_update_trade_state_exception(mocker, default_conf, assert log_has('Could not update trade amount: ', caplog.record_tuples) +def test_update_trade_state_sell(default_conf, trades_for_order, limit_sell_order, mocker): + mocker.patch('freqtrade.exchange.Exchange.get_trades_for_order', return_value=trades_for_order) + # get_order should not be called!! + mocker.patch('freqtrade.exchange.Exchange.get_order', MagicMock(side_effect=ValueError)) + wallet_mock = MagicMock() + mocker.patch('freqtrade.wallets.Wallets.update', wallet_mock) + + patch_exchange(mocker) + Trade.session = MagicMock() + amount = limit_sell_order["amount"] + freqtrade = get_patched_freqtradebot(mocker, default_conf) + wallet_mock.reset_mock() + trade = Trade( + pair='LTC/ETH', + amount=amount, + exchange='binance', + open_rate=0.245441, + fee_open=0.0025, + fee_close=0.0025, + open_order_id="123456", + is_open=True, + ) + freqtrade.update_trade_state(trade, limit_sell_order) + assert trade.amount == limit_sell_order['amount'] + # Wallet needs to be updated after closing a limit-sell order to reenable buying + assert wallet_mock.call_count == 1 + assert not trade.is_open + + def test_handle_trade(default_conf, limit_buy_order, limit_sell_order, fee, markets, mocker) -> None: patch_RPCManager(mocker) diff --git a/requirements-dev.txt b/requirements-dev.txt index c59923ed2..123531f23 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -4,9 +4,9 @@ flake8==3.7.7 flake8-type-annotations==0.1.0 flake8-tidy-imports==2.0.0 -pytest==4.4.0 -pytest-mock==1.10.3 +pytest==4.4.1 +pytest-mock==1.10.4 pytest-asyncio==0.10.0 pytest-cov==2.6.1 coveralls==1.7.0 -mypy==0.700 +mypy==0.701 diff --git a/requirements-pi.txt b/requirements-pi.txt new file mode 100644 index 000000000..80935e9ef --- /dev/null +++ b/requirements-pi.txt @@ -0,0 +1,23 @@ +ccxt==1.18.486 +SQLAlchemy==1.3.3 +python-telegram-bot==11.1.0 +arrow==0.13.1 +cachetools==3.1.0 +requests==2.21.0 +urllib3==1.25 +wrapt==1.11.1 +scikit-learn==0.20.3 +joblib==0.13.2 +jsonschema==3.0.1 +TA-Lib==0.4.17 +tabulate==0.8.3 +coinmarketcap==5.0.3 + +# Required for hyperopt +scikit-optimize==0.5.2 + +# find first, C search in arrays +py_find_1st==1.1.3 + +#Load ticker files 30% faster +python-rapidjson==0.7.0 diff --git a/requirements-plot.txt b/requirements-plot.txt index e582fddf6..4cdd6ec8f 100644 --- a/requirements-plot.txt +++ b/requirements-plot.txt @@ -1,5 +1,5 @@ # Include all requirements to run the bot. -r requirements.txt -plotly==3.7.1 +plotly==3.8.1 diff --git a/requirements.txt b/requirements.txt index 7006c52c7..ea3792583 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,12 +1,12 @@ -ccxt==1.18.435 -SQLAlchemy==1.3.2 +ccxt==1.18.486 +SQLAlchemy==1.3.3 python-telegram-bot==11.1.0 arrow==0.13.1 cachetools==3.1.0 requests==2.21.0 -urllib3==1.24.1 +urllib3==1.25 wrapt==1.11.1 -numpy==1.16.2 +numpy==1.16.3 pandas==0.24.2 scikit-learn==0.20.3 joblib==0.13.2 diff --git a/scripts/download_backtest_data.py b/scripts/download_backtest_data.py index 5dee41bdd..42b305778 100755 --- a/scripts/download_backtest_data.py +++ b/scripts/download_backtest_data.py @@ -1,6 +1,7 @@ #!/usr/bin/env python3 - -"""This script generate json data""" +""" +This script generates json data +""" import json import sys from pathlib import Path @@ -35,7 +36,7 @@ if args.config: config: Dict[str, Any] = {} # Now expecting a list of config filenames here, not a string for path in args.config: - print('Using config: %s ...', path) + print(f"Using config: {path}...") # Merge config options, overwriting old values config = deep_merge_dicts(configuration._load_config_file(path), config) @@ -44,18 +45,20 @@ if args.config: config['exchange']['key'] = '' config['exchange']['secret'] = '' else: - config = {'stake_currency': '', - 'dry_run': True, - 'exchange': { - 'name': args.exchange, - 'key': '', - 'secret': '', - 'pair_whitelist': [], - 'ccxt_async_config': { - "enableRateLimit": False - } - } - } + config = { + 'stake_currency': '', + 'dry_run': True, + 'exchange': { + 'name': args.exchange, + 'key': '', + 'secret': '', + 'pair_whitelist': [], + 'ccxt_async_config': { + 'enableRateLimit': True, + 'rateLimit': 200 + } + } + } dl_path = Path(DEFAULT_DL_PATH).joinpath(config['exchange']['name']) @@ -92,18 +95,18 @@ for pair in PAIRS: pairs_not_available.append(pair) print(f"skipping pair {pair}") continue - for tick_interval in timeframes: + for ticker_interval in timeframes: pair_print = pair.replace('/', '_') - filename = f'{pair_print}-{tick_interval}.json' + filename = f'{pair_print}-{ticker_interval}.json' dl_file = dl_path.joinpath(filename) if args.erase and dl_file.exists(): - print(f'Deleting existing data for pair {pair}, interval {tick_interval}') + print(f'Deleting existing data for pair {pair}, interval {ticker_interval}') dl_file.unlink() - print(f'downloading pair {pair}, interval {tick_interval}') + print(f'downloading pair {pair}, interval {ticker_interval}') download_pair_history(datadir=dl_path, exchange=exchange, pair=pair, - tick_interval=tick_interval, + ticker_interval=ticker_interval, timerange=timerange) diff --git a/scripts/get_market_pairs.py b/scripts/get_market_pairs.py index 1a4c593f9..cd38bf2fa 100644 --- a/scripts/get_market_pairs.py +++ b/scripts/get_market_pairs.py @@ -1,5 +1,10 @@ +""" +This script was adapted from ccxt here: +https://github.com/ccxt/ccxt/blob/master/examples/py/arbitrage-pairs.py +""" import os import sys +import traceback root = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) sys.path.append(root + '/python') @@ -49,6 +54,11 @@ def print_supported_exchanges(): try: + if len(sys.argv) < 2: + dump("Usage: python " + sys.argv[0], green('id')) + print_supported_exchanges() + sys.exit(1) + id = sys.argv[1] # get exchange id from command line arguments # check if the exchange is supported by ccxt @@ -87,5 +97,7 @@ try: except Exception as e: dump('[' + type(e).__name__ + ']', str(e)) + dump(traceback.format_exc()) dump("Usage: python " + sys.argv[0], green('id')) print_supported_exchanges() + sys.exit(1) diff --git a/scripts/plot_dataframe.py b/scripts/plot_dataframe.py index 14d57265e..7fdc607e0 100755 --- a/scripts/plot_dataframe.py +++ b/scripts/plot_dataframe.py @@ -82,7 +82,7 @@ def load_trades(args: Namespace, pair: str, timerange: TimeRange) -> pd.DataFram return trades -def generate_plot_file(fig, pair, tick_interval, is_last) -> None: +def generate_plot_file(fig, pair, ticker_interval, is_last) -> None: """ Generate a plot html file from pre populated fig plotly object :return: None @@ -90,7 +90,7 @@ def generate_plot_file(fig, pair, tick_interval, is_last) -> None: logger.info('Generate plot file for %s', pair) pair_name = pair.replace("/", "_") - file_name = 'freqtrade-plot-' + pair_name + '-' + tick_interval + '.html' + file_name = 'freqtrade-plot-' + pair_name + '-' + ticker_interval + '.html' Path("user_data/plots").mkdir(parents=True, exist_ok=True) @@ -135,20 +135,20 @@ def get_tickers_data(strategy, exchange, pairs: List[str], args): :return: dictinnary of tickers. output format: {'pair': tickersdata} """ - tick_interval = strategy.ticker_interval + ticker_interval = strategy.ticker_interval timerange = Arguments.parse_timerange(args.timerange) tickers = {} if args.live: logger.info('Downloading pairs.') - exchange.refresh_latest_ohlcv([(pair, tick_interval) for pair in pairs]) + exchange.refresh_latest_ohlcv([(pair, ticker_interval) for pair in pairs]) for pair in pairs: - tickers[pair] = exchange.klines((pair, tick_interval)) + tickers[pair] = exchange.klines((pair, ticker_interval)) else: tickers = history.load_data( datadir=Path(str(_CONF.get("datadir"))), pairs=pairs, - ticker_interval=tick_interval, + ticker_interval=ticker_interval, refresh_pairs=_CONF.get('refresh_pairs', False), timerange=timerange, exchange=Exchange(_CONF) @@ -399,7 +399,7 @@ def analyse_and_plot_pairs(args: Namespace): strategy, exchange, pairs = get_trading_env(args) # Set timerange to use timerange = Arguments.parse_timerange(args.timerange) - tick_interval = strategy.ticker_interval + ticker_interval = strategy.ticker_interval tickers = get_tickers_data(strategy, exchange, pairs, args) pair_counter = 0 @@ -422,7 +422,7 @@ def analyse_and_plot_pairs(args: Namespace): ) is_last = (False, True)[pair_counter == len(tickers)] - generate_plot_file(fig, pair, tick_interval, is_last) + generate_plot_file(fig, pair, ticker_interval, is_last) logger.info('End of ploting process %s plots generated', pair_counter) diff --git a/scripts/plot_profit.py b/scripts/plot_profit.py index 2b6360bec..5f7d42c87 100755 --- a/scripts/plot_profit.py +++ b/scripts/plot_profit.py @@ -27,10 +27,12 @@ from plotly.offline import plot from freqtrade.arguments import Arguments from freqtrade.configuration import Configuration from freqtrade.data import history -from freqtrade.misc import common_datearray, timeframe_to_seconds +from freqtrade.exchange import timeframe_to_seconds +from freqtrade.misc import common_datearray from freqtrade.resolvers import StrategyResolver from freqtrade.state import RunMode + logger = logging.getLogger(__name__) @@ -76,7 +78,7 @@ def plot_profit(args: Namespace) -> None: in helping out to find a good algorithm. """ - # We need to use the same pairs, same tick_interval + # We need to use the same pairs, same ticker_interval # and same timeperiod as used in backtesting # to match the tickerdata against the profits-results timerange = Arguments.parse_timerange(args.timerange) @@ -112,7 +114,7 @@ def plot_profit(args: Namespace) -> None: else: filter_pairs = config['exchange']['pair_whitelist'] - tick_interval = strategy.ticker_interval + ticker_interval = strategy.ticker_interval pairs = config['exchange']['pair_whitelist'] if filter_pairs: @@ -122,7 +124,7 @@ def plot_profit(args: Namespace) -> None: tickers = history.load_data( datadir=Path(str(config.get('datadir'))), pairs=pairs, - ticker_interval=tick_interval, + ticker_interval=ticker_interval, refresh_pairs=False, timerange=timerange ) @@ -134,7 +136,7 @@ def plot_profit(args: Namespace) -> None: dates = common_datearray(dataframes) min_date = int(min(dates).timestamp()) max_date = int(max(dates).timestamp()) - num_iterations = define_index(min_date, max_date, tick_interval) + 1 + num_iterations = define_index(min_date, max_date, ticker_interval) + 1 # Make an average close price of all the pairs that was involved. # this could be useful to gauge the overall market trend @@ -154,7 +156,7 @@ def plot_profit(args: Namespace) -> None: avgclose /= num # make an profits-growth array - pg = make_profit_array(data, num_iterations, min_date, tick_interval, filter_pairs) + pg = make_profit_array(data, num_iterations, min_date, ticker_interval, filter_pairs) # # Plot the pairs average close prices, and total profit growth @@ -178,7 +180,7 @@ def plot_profit(args: Namespace) -> None: fig.append_trace(profit, 2, 1) for pair in pairs: - pg = make_profit_array(data, num_iterations, min_date, tick_interval, [pair]) + pg = make_profit_array(data, num_iterations, min_date, ticker_interval, [pair]) pair_profit = go.Scattergl( x=dates, y=pg, @@ -189,11 +191,11 @@ def plot_profit(args: Namespace) -> None: plot(fig, filename=str(Path('user_data').joinpath('freqtrade-profit-plot.html'))) -def define_index(min_date: int, max_date: int, interval: str) -> int: +def define_index(min_date: int, max_date: int, ticker_interval: str) -> int: """ Return the index of a specific date """ - interval_seconds = timeframe_to_seconds(interval) + interval_seconds = timeframe_to_seconds(ticker_interval) return int((max_date - min_date) / interval_seconds)