From 6fbdd6bee9520afc13450cb63c3fba4591537ce3 Mon Sep 17 00:00:00 2001 From: Matthias Date: Thu, 2 Jan 2020 09:51:24 +0100 Subject: [PATCH 01/16] Remove unused directory from user_data --- user_data/backtest_data/.gitkeep | 0 1 file changed, 0 insertions(+), 0 deletions(-) delete mode 100644 user_data/backtest_data/.gitkeep diff --git a/user_data/backtest_data/.gitkeep b/user_data/backtest_data/.gitkeep deleted file mode 100644 index e69de29bb..000000000 From 2c8e8d8ef65021fbb3ee3446768fe3e3fa34c36f Mon Sep 17 00:00:00 2001 From: Matthias Date: Thu, 2 Jan 2020 09:51:47 +0100 Subject: [PATCH 02/16] Align columns for btanalysis loading --- freqtrade/data/btanalysis.py | 2 +- tests/data/test_btanalysis.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/freqtrade/data/btanalysis.py b/freqtrade/data/btanalysis.py index 2fc931a9b..04b2ca980 100644 --- a/freqtrade/data/btanalysis.py +++ b/freqtrade/data/btanalysis.py @@ -47,7 +47,7 @@ def load_backtest_data(filename) -> pd.DataFrame: utc=True, infer_datetime_format=True ) - df['profitabs'] = df['close_rate'] - df['open_rate'] + df['profit'] = df['close_rate'] - df['open_rate'] df = df.sort_values("open_time").reset_index(drop=True) return df diff --git a/tests/data/test_btanalysis.py b/tests/data/test_btanalysis.py index 13711c63e..60d9c3ea5 100644 --- a/tests/data/test_btanalysis.py +++ b/tests/data/test_btanalysis.py @@ -20,7 +20,7 @@ def test_load_backtest_data(testdatadir): filename = testdatadir / "backtest-result_test.json" bt_data = load_backtest_data(filename) assert isinstance(bt_data, DataFrame) - assert list(bt_data.columns) == BT_DATA_COLUMNS + ["profitabs"] + assert list(bt_data.columns) == BT_DATA_COLUMNS + ["profit"] assert len(bt_data) == 179 # Test loading from string (must yield same result) From 9325880fe5c84973a6553f47993639042f574ac9 Mon Sep 17 00:00:00 2001 From: Matthias Date: Thu, 2 Jan 2020 10:38:59 +0100 Subject: [PATCH 03/16] Split config-validation requires --- freqtrade/configuration/config_validation.py | 11 +++++-- freqtrade/constants.py | 30 ++++++++++++-------- tests/test_configuration.py | 1 + 3 files changed, 28 insertions(+), 14 deletions(-) diff --git a/freqtrade/configuration/config_validation.py b/freqtrade/configuration/config_validation.py index 43eead46a..9afa39ca7 100644 --- a/freqtrade/configuration/config_validation.py +++ b/freqtrade/configuration/config_validation.py @@ -1,4 +1,5 @@ import logging +from copy import deepcopy from typing import Any, Dict from jsonschema import Draft4Validator, validators @@ -42,15 +43,21 @@ def validate_config_schema(conf: Dict[str, Any]) -> Dict[str, Any]: :param conf: Config in JSON format :return: Returns the config if valid, otherwise throw an exception """ + conf_schema = deepcopy(constants.CONF_SCHEMA) + if conf.get('runmode', RunMode.OTHER) in (RunMode.DRY_RUN, RunMode.LIVE): + conf_schema['required'] = constants.SCHEMA_TRADE_REQUIRED + else: + conf_schema['required'] = constants.SCHEMA_MINIMAL_REQUIRED + try: - FreqtradeValidator(constants.CONF_SCHEMA).validate(conf) + FreqtradeValidator(conf_schema).validate(conf) return conf except ValidationError as e: logger.critical( f"Invalid configuration. See config.json.example. Reason: {e}" ) raise ValidationError( - best_match(Draft4Validator(constants.CONF_SCHEMA).iter_errors(conf)).message + best_match(Draft4Validator(conf_schema).iter_errors(conf)).message ) diff --git a/freqtrade/constants.py b/freqtrade/constants.py index d7c6249d5..b21279bb4 100644 --- a/freqtrade/constants.py +++ b/freqtrade/constants.py @@ -269,16 +269,22 @@ CONF_SCHEMA = { 'required': ['process_throttle_secs', 'allowed_risk', 'capital_available_percentage'] } }, - 'required': [ - 'exchange', - 'max_open_trades', - 'stake_currency', - 'stake_amount', - 'dry_run', - 'dry_run_wallet', - 'bid_strategy', - 'unfilledtimeout', - 'stoploss', - 'minimal_roi', - ] } + +SCHEMA_TRADE_REQUIRED = [ + 'exchange', + 'max_open_trades', + 'stake_currency', + 'stake_amount', + 'dry_run', + 'dry_run_wallet', + 'bid_strategy', + 'unfilledtimeout', + 'stoploss', + 'minimal_roi', +] + +SCHEMA_MINIMAL_REQUIRED = [ + 'exchange', + 'dry_run', +] diff --git a/tests/test_configuration.py b/tests/test_configuration.py index ee3d23131..531a88688 100644 --- a/tests/test_configuration.py +++ b/tests/test_configuration.py @@ -49,6 +49,7 @@ def test_load_config_missing_attributes(default_conf) -> None: conf = deepcopy(default_conf) conf.pop('stake_currency') + conf['runmode'] = RunMode.DRY_RUN with pytest.raises(ValidationError, match=r".*'stake_currency' is a required property.*"): validate_config_schema(conf) From 20fc3b7978d036da11a56df2b8c81622854e2a6e Mon Sep 17 00:00:00 2001 From: Matthias Date: Thu, 2 Jan 2020 10:41:10 +0100 Subject: [PATCH 04/16] validate config for utils too --- freqtrade/utils.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/freqtrade/utils.py b/freqtrade/utils.py index 45520ecf7..9fe15aea6 100644 --- a/freqtrade/utils.py +++ b/freqtrade/utils.py @@ -12,7 +12,8 @@ from colorama import init as colorama_init from tabulate import tabulate from freqtrade.configuration import (Configuration, TimeRange, - remove_credentials) + remove_credentials, + validate_config_consistency) from freqtrade.configuration.directory_operations import (copy_sample_files, create_userdata_dir) from freqtrade.constants import USERPATH_HYPEROPTS, USERPATH_STRATEGY @@ -40,6 +41,7 @@ def setup_utils_configuration(args: Dict[str, Any], method: RunMode) -> Dict[str # Ensure we do not use Exchange credentials remove_credentials(config) + validate_config_consistency(config) return config From 22fcf7b4dc64c82058e4eb1349422ac83199c043 Mon Sep 17 00:00:00 2001 From: Matthias Date: Thu, 2 Jan 2020 10:47:37 +0100 Subject: [PATCH 05/16] Allow empty stake currency in certain cases --- freqtrade/configuration/config_validation.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/freqtrade/configuration/config_validation.py b/freqtrade/configuration/config_validation.py index 9afa39ca7..f7886eec2 100644 --- a/freqtrade/configuration/config_validation.py +++ b/freqtrade/configuration/config_validation.py @@ -48,6 +48,10 @@ def validate_config_schema(conf: Dict[str, Any]) -> Dict[str, Any]: conf_schema['required'] = constants.SCHEMA_TRADE_REQUIRED else: conf_schema['required'] = constants.SCHEMA_MINIMAL_REQUIRED + # Dynamically allow empty stake-currency + # Since the minimal config specifies this too. + # It's not allowed for Dry-run or live modes + conf_schema['properties']['stake_currency']['enum'] += [''] try: FreqtradeValidator(conf_schema).validate(conf) From 9382b38c41509e51495d48248b968d58bdf1e6df Mon Sep 17 00:00:00 2001 From: Matthias Date: Thu, 2 Jan 2020 10:56:00 +0100 Subject: [PATCH 06/16] Fix mypy error --- freqtrade/configuration/config_validation.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/configuration/config_validation.py b/freqtrade/configuration/config_validation.py index f7886eec2..6f56790f4 100644 --- a/freqtrade/configuration/config_validation.py +++ b/freqtrade/configuration/config_validation.py @@ -51,7 +51,7 @@ def validate_config_schema(conf: Dict[str, Any]) -> Dict[str, Any]: # Dynamically allow empty stake-currency # Since the minimal config specifies this too. # It's not allowed for Dry-run or live modes - conf_schema['properties']['stake_currency']['enum'] += [''] + conf_schema['properties']['stake_currency']['enum'] += [''] # type: ignore try: FreqtradeValidator(conf_schema).validate(conf) From f15e5e9d5715c233d49adbe692dd38215e71a769 Mon Sep 17 00:00:00 2001 From: hroff-1902 Date: Thu, 2 Jan 2020 13:51:25 +0300 Subject: [PATCH 07/16] Add _notify_buy() --- freqtrade/freqtradebot.py | 42 +++++++++++++++++++++++++-------------- 1 file changed, 27 insertions(+), 15 deletions(-) diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index d5d918585..9570a80ef 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -400,8 +400,6 @@ class FreqtradeBot: :param pair: pair for which we want to create a LIMIT_BUY :return: None """ - stake_currency = self.config['stake_currency'] - fiat_currency = self.config.get('fiat_display_currency', None) time_in_force = self.strategy.order_time_in_force['buy'] if price: @@ -458,17 +456,6 @@ class FreqtradeBot: amount = order['amount'] buy_limit_filled_price = order['price'] - self.rpc.send_msg({ - 'type': RPCMessageType.BUY_NOTIFICATION, - 'exchange': self.exchange.name.capitalize(), - 'pair': pair, - 'limit': buy_limit_filled_price, - 'order_type': order_type, - 'stake_amount': stake_amount, - 'stake_currency': stake_currency, - 'fiat_currency': fiat_currency - }) - # Fee is applied twice because we make a LIMIT_BUY and LIMIT_SELL fee = self.exchange.get_fee(symbol=pair, taker_or_maker='maker') trade = Trade( @@ -486,6 +473,8 @@ class FreqtradeBot: ticker_interval=timeframe_to_minutes(self.config['ticker_interval']) ) + self._notify_buy(trade, order_type) + # Update fees if order is closed if order_status == 'closed': self.update_trade_state(trade, order) @@ -498,6 +487,30 @@ class FreqtradeBot: return True + def _notify_buy(self, trade: Trade, order_type: str): + """ + Sends rpc notification when a buy occured. + """ + msg = { + 'type': RPCMessageType.BUY_NOTIFICATION, + 'exchange': self.exchange.name.capitalize(), + 'pair': trade.pair, + 'limit': trade.open_rate, + 'order_type': order_type, + 'stake_amount': trade.stake_amount, + } + + if 'stake_currency' in self.config and 'fiat_display_currency' in self.config: + stake_currency = self.config['stake_currency'] + fiat_currency = self.config['fiat_display_currency'] + msg.update({ + 'stake_currency': stake_currency, + 'fiat_currency': fiat_currency, + }) + + # Send the message + self.rpc.send_msg(msg) + # # SELL / exit positions / close trades logic and methods # @@ -952,10 +965,9 @@ class FreqtradeBot: 'profit_percent': profit_percent, 'sell_reason': trade.sell_reason, 'open_date': trade.open_date, - 'close_date': trade.close_date or datetime.utcnow() + 'close_date': trade.close_date or datetime.utcnow(), } - # For regular case, when the configuration exists if 'stake_currency' in self.config and 'fiat_display_currency' in self.config: stake_currency = self.config['stake_currency'] fiat_currency = self.config['fiat_display_currency'] From 88efa4065bdcb222091d9594c2cb86a5a8b03461 Mon Sep 17 00:00:00 2001 From: hroff-1902 Date: Thu, 2 Jan 2020 13:56:16 +0300 Subject: [PATCH 08/16] Align the name of a variable to be same for buy and sell parts --- freqtrade/freqtradebot.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index 9570a80ef..5e6ffbfde 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -913,16 +913,16 @@ class FreqtradeBot: except InvalidOrderException: logger.exception(f"Could not cancel stoploss order {trade.stoploss_order_id}") - ordertype = self.strategy.order_types[sell_type] + order_type = self.strategy.order_types[sell_type] if sell_reason == SellType.EMERGENCY_SELL: # Emergencysells (default to market!) - ordertype = self.strategy.order_types.get("emergencysell", "market") + order_type = self.strategy.order_types.get("emergencysell", "market") amount = self._safe_sell_amount(trade.pair, trade.amount) # Execute sell and update trade record order = self.exchange.sell(pair=str(trade.pair), - ordertype=ordertype, + ordertype=order_type, amount=amount, rate=limit, time_in_force=self.strategy.order_time_in_force['sell'] ) @@ -938,7 +938,7 @@ class FreqtradeBot: # Lock pair for one candle to prevent immediate rebuys self.strategy.lock_pair(trade.pair, timeframe_to_next_date(self.config['ticker_interval'])) - self._notify_sell(trade, ordertype) + self._notify_sell(trade, order_type) def _notify_sell(self, trade: Trade, order_type: str): """ From a47a25ca8889667bfec6cd948a2773b471cff87a Mon Sep 17 00:00:00 2001 From: hroff-1902 Date: Thu, 2 Jan 2020 14:38:25 +0300 Subject: [PATCH 09/16] Refine passing msg params --- freqtrade/freqtradebot.py | 15 ++++----------- 1 file changed, 4 insertions(+), 11 deletions(-) diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index 5e6ffbfde..c18d9131f 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -498,16 +498,10 @@ class FreqtradeBot: 'limit': trade.open_rate, 'order_type': order_type, 'stake_amount': trade.stake_amount, + 'stake_currency': self.config['stake_currency'], + 'fiat_currency': self.config.get('fiat_display_currency', None), } - if 'stake_currency' in self.config and 'fiat_display_currency' in self.config: - stake_currency = self.config['stake_currency'] - fiat_currency = self.config['fiat_display_currency'] - msg.update({ - 'stake_currency': stake_currency, - 'fiat_currency': fiat_currency, - }) - # Send the message self.rpc.send_msg(msg) @@ -966,13 +960,12 @@ class FreqtradeBot: 'sell_reason': trade.sell_reason, 'open_date': trade.open_date, 'close_date': trade.close_date or datetime.utcnow(), + 'stake_currency': self.config['stake_currency'], } - if 'stake_currency' in self.config and 'fiat_display_currency' in self.config: - stake_currency = self.config['stake_currency'] + if 'fiat_display_currency' in self.config: fiat_currency = self.config['fiat_display_currency'] msg.update({ - 'stake_currency': stake_currency, 'fiat_currency': fiat_currency, }) From b48bf035f6758bedf5773ec9b1b1ed98ca3477ae Mon Sep 17 00:00:00 2001 From: Matthias Date: Thu, 2 Jan 2020 14:52:34 +0100 Subject: [PATCH 10/16] Add note about MacOS installation --- docs/installation.md | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/docs/installation.md b/docs/installation.md index 27b7a94c5..f7dce8787 100644 --- a/docs/installation.md +++ b/docs/installation.md @@ -270,3 +270,19 @@ The easiest way is to download install Microsoft Visual Studio Community [here]( Now you have an environment ready, the next step is [Bot Configuration](configuration.md). + + +## Trouble shooting + +### MacOS installation error + +Newer versions of MacOS may have installation fail with errors like `error: command 'g++' failed with exit status 1`. + +This error will require an explicit installation of the SDK Headers, which are not installed by default in this version of MacOS. +For MacOS 10.14, this can be accomplished with the below command. + +``` bash +open /Library/Developer/CommandLineTools/Packages/macOS_SDK_headers_for_macOS_10.14.pkg +``` + +If this file is inexistant, then you're probably on a different version of MacOS, so you may need to consult the internet for specific resolution details. From 90744ff5abe262ef4d0731d966c5d8ad74ade076 Mon Sep 17 00:00:00 2001 From: Matthias Date: Thu, 2 Jan 2020 19:36:31 +0100 Subject: [PATCH 11/16] show percent instead of ratio (!) --- freqtrade/plot/plotting.py | 4 ++-- tests/test_plotting.py | 1 + 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/freqtrade/plot/plotting.py b/freqtrade/plot/plotting.py index db4637ee5..e1989b249 100644 --- a/freqtrade/plot/plotting.py +++ b/freqtrade/plot/plotting.py @@ -120,8 +120,8 @@ def plot_trades(fig, trades: pd.DataFrame) -> make_subplots: ) ) # Create description for sell summarizing the trade - desc = trades.apply(lambda row: f"{round(row['profitperc'], 3)}%, {row['sell_reason']}, " - f"{row['duration']} min", + desc = trades.apply(lambda row: f"{round(row['profitperc'] * 100, 1)}%, " + f"{row['sell_reason']}, {row['duration']} min", axis=1) trade_sells = go.Scatter( x=trades["close_time"], diff --git a/tests/test_plotting.py b/tests/test_plotting.py index 9934d2493..271246517 100644 --- a/tests/test_plotting.py +++ b/tests/test_plotting.py @@ -119,6 +119,7 @@ def test_plot_trades(testdatadir, caplog): assert trade_sell.yaxis == 'y' assert len(trades) == len(trade_sell.x) assert trade_sell.marker.color == 'red' + assert trade_sell.text[0] == "4.0%, roi, 15 min" def test_generate_candlestick_graph_no_signals_no_trades(default_conf, mocker, testdatadir, caplog): From 560aea876efa6aaaec65f7def6c58a9c4613d33a Mon Sep 17 00:00:00 2001 From: Matthias Date: Thu, 2 Jan 2020 20:20:29 +0100 Subject: [PATCH 12/16] Remove fiat_currency temporary variable --- freqtrade/freqtradebot.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index c18d9131f..373055165 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -964,9 +964,8 @@ class FreqtradeBot: } if 'fiat_display_currency' in self.config: - fiat_currency = self.config['fiat_display_currency'] msg.update({ - 'fiat_currency': fiat_currency, + 'fiat_currency': self.config['fiat_display_currency'], }) # Send the message From da1fea6582790e99b3cd69688dd6dfb7c3b57e00 Mon Sep 17 00:00:00 2001 From: Matthias Date: Fri, 3 Jan 2020 06:37:36 +0100 Subject: [PATCH 13/16] Minor correction to wording of MacOS Specific install doc --- docs/installation.md | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/docs/installation.md b/docs/installation.md index f7dce8787..267d91c8d 100644 --- a/docs/installation.md +++ b/docs/installation.md @@ -271,14 +271,13 @@ The easiest way is to download install Microsoft Visual Studio Community [here]( Now you have an environment ready, the next step is [Bot Configuration](configuration.md). - -## Trouble shooting +## Troubleshooting ### MacOS installation error -Newer versions of MacOS may have installation fail with errors like `error: command 'g++' failed with exit status 1`. +Newer versions of MacOS may have installation failed with errors like `error: command 'g++' failed with exit status 1`. -This error will require an explicit installation of the SDK Headers, which are not installed by default in this version of MacOS. +This error will require explicit installation of the SDK Headers, which are not installed by default in this version of MacOS. For MacOS 10.14, this can be accomplished with the below command. ``` bash From 55041878ae7da0758a8e58d2c1add3139e54fcbb Mon Sep 17 00:00:00 2001 From: Matthias Date: Fri, 3 Jan 2020 11:20:08 +0100 Subject: [PATCH 14/16] Update Backtesting fee documentation --- docs/backtesting.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/docs/backtesting.md b/docs/backtesting.md index ac7c8e11a..c2359b370 100644 --- a/docs/backtesting.md +++ b/docs/backtesting.md @@ -79,12 +79,14 @@ Please also read about the [strategy startup period](strategy-customization.md#s Sometimes your account has certain fee rebates (fee reductions starting with a certain account size or monthly volume), which are not visible to ccxt. To account for this in backtesting, you can use `--fee 0.001` to supply this value to backtesting. -This fee must be a percentage, and will be applied twice (once for trade entry, and once for trade exit). +This fee must be a ratio, and will be applied twice (once for trade entry, and once for trade exit). ```bash freqtrade backtesting --fee 0.001 ``` +!!! Note + Only supply this parameter if you want to experiment with different fee values. By default, Backtesting fetches the exchange's default fee from the exchange. #### Running backtest with smaller testset by using timerange From e1f89e3ad34187caf0385a841f13ec1c710fbd17 Mon Sep 17 00:00:00 2001 From: Matthias Date: Fri, 3 Jan 2020 20:11:58 +0100 Subject: [PATCH 15/16] Reword Note in backtesting fee docs --- docs/backtesting.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/backtesting.md b/docs/backtesting.md index c2359b370..45759e2aa 100644 --- a/docs/backtesting.md +++ b/docs/backtesting.md @@ -86,7 +86,7 @@ freqtrade backtesting --fee 0.001 ``` !!! Note - Only supply this parameter if you want to experiment with different fee values. By default, Backtesting fetches the exchange's default fee from the exchange. + Only supply this option (or the corresponding configuration parameter) if you want to experiment with different fee values. By default, Backtesting fetches the default fee from the exchange pair/market info. #### Running backtest with smaller testset by using timerange From 24aa596e3c956df3e3b8967e0334246574f8e434 Mon Sep 17 00:00:00 2001 From: hroff-1902 <47309513+hroff-1902@users.noreply.github.com> Date: Sat, 4 Jan 2020 01:08:37 +0300 Subject: [PATCH 16/16] Minor: Refine fee example in the docs Taken from https://github.com/freqtrade/freqtrade/issues/2738#issuecomment-570687230. slightly reworded. --- docs/backtesting.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/docs/backtesting.md b/docs/backtesting.md index 45759e2aa..41428085d 100644 --- a/docs/backtesting.md +++ b/docs/backtesting.md @@ -78,9 +78,11 @@ Please also read about the [strategy startup period](strategy-customization.md#s #### Supplying custom fee value Sometimes your account has certain fee rebates (fee reductions starting with a certain account size or monthly volume), which are not visible to ccxt. -To account for this in backtesting, you can use `--fee 0.001` to supply this value to backtesting. +To account for this in backtesting, you can use the `--fee` command line option to supply this value to backtesting. This fee must be a ratio, and will be applied twice (once for trade entry, and once for trade exit). +For example, if the buying and selling commission fee is 0.1% (i.e., 0.001 written as ratio), then you would run backtesting as the following: + ```bash freqtrade backtesting --fee 0.001 ```