Merge pull request #9986 from freqtrade/feat/show_config

add show-config command

docs/configuration.md

@@ -49,6 +49,9 @@ FREQTRADE__EXCHANGE__SECRET=<yourExchangeSecret>
 !!! Note
     Environment variables detected are logged at startup - so if you can't find why a value is not what you think it should be based on the configuration, make sure it's not loaded from an environment variable.
 
+!!! Tip "Validate combined result"
+    You can use the [show-config subcommand](utils.md#show-config) to see the final, combined configuration.
+
 ### Multiple configuration files
 
 Multiple configuration files can be specified and used by the bot or the bot can read its configuration parameters from the process standard input stream.
@@ -56,6 +59,9 @@ Multiple configuration files can be specified and used by the bot or the bot can
 You can specify additional configuration files in `add_config_files`. Files specified in this parameter will be loaded and merged with the initial config file. The files are resolved relative to the initial configuration file.
 This is similar to using multiple `--config` parameters, but simpler in usage as you don't have to specify all files for all commands.
 
+!!! Tip "Validate combined result"
+    You can use the [show-config subcommand](utils.md#show-config) to see the final, combined configuration.
+
 !!! Tip "Use multiple configuration files to keep secrets secret"
     You can use a 2nd configuration file containing your secrets. That way you can share your "primary" configuration file, while still keeping your API keys for yourself.
     The 2nd file should only specify what you intend to override.

docs/utils.md

@@ -66,6 +66,53 @@ $ freqtrade new-config --config user_data/config_binance.json
 ? Do you want to enable Telegram?  No
 ```
 
+## Show config
+
+Show configuration file (with sensitive values redacted by default).
+Especially useful with [split configuration files](configuration.md#multiple-configuration-files) or [environment variables](configuration.md#environment-variables), where this command will show the merged configuration.
+
+![Show config output](assets/show-config-output.png)
+
+```
+usage: freqtrade show-config [-h] [--userdir PATH] [-c PATH]
+                             [--show-sensitive]
+
+options:
+  -h, --help            show this help message and exit
+  --userdir PATH, --user-data-dir PATH
+                        Path to userdata directory.
+  -c PATH, --config PATH
+                        Specify configuration file (default:
+                        `userdir/config.json` or `config.json` whichever
+                        exists). Multiple --config options may be used. Can be
+                        set to `-` to read config from stdin.
+  --show-sensitive      Show secrets in the output.
+```
+
+``` output
+Your combined configuration is:
+{
+  "exit_pricing": {
+    "price_side": "other",
+    "use_order_book": true,
+    "order_book_top": 1
+  },
+  "stake_currency": "USDT",
+  "exchange": {
+    "name": "binance",
+    "key": "REDACTED",
+    "secret": "REDACTED",
+    "ccxt_config": {},
+    "ccxt_async_config": {},
+  }
+  // ...
+}
+```
+
+!!! Warning "Sharing information provided by this command"
+    We try to remove all known sensitive information from the default output (without `--show-sensitive`). 
+    Yet, please do double-check for sensitive values in your output to make sure you're not accidentally exposing some private info.
+
 ## Create new strategy
 
 Creates a new strategy from a template similar to SampleStrategy.

freqtrade/commands/__init__.py

@@ -8,7 +8,7 @@ Note: Be careful with file-scoped imports in these subfiles.
 """
 from freqtrade.commands.analyze_commands import start_analysis_entries_exits
 from freqtrade.commands.arguments import Arguments
-from freqtrade.commands.build_config_commands import start_new_config
+from freqtrade.commands.build_config_commands import start_new_config, start_show_config
 from freqtrade.commands.data_commands import (start_convert_data, start_convert_trades,
                                               start_download_data, start_list_data)
 from freqtrade.commands.db_commands import start_convert_db

freqtrade/commands/arguments.py

@@ -62,6 +62,7 @@ ARGS_TEST_PAIRLIST = ["user_data_dir", "verbosity", "config", "quote_currencies"
 ARGS_CREATE_USERDIR = ["user_data_dir", "reset"]
 
 ARGS_BUILD_CONFIG = ["config"]
+ARGS_SHOW_CONFIG = ["user_data_dir", "config", "show_sensitive"]
 
 ARGS_BUILD_STRATEGY = ["user_data_dir", "strategy", "template"]
 
@@ -209,9 +210,9 @@ class Arguments:
                                         start_list_strategies, start_list_timeframes,
                                         start_lookahead_analysis, start_new_config,
                                         start_new_strategy, start_plot_dataframe, start_plot_profit,
-                                        start_recursive_analysis, start_show_trades,
-                                        start_strategy_update, start_test_pairlist, start_trading,
-                                        start_webserver)
+                                        start_recursive_analysis, start_show_config,
+                                        start_show_trades, start_strategy_update,
+                                        start_test_pairlist, start_trading, start_webserver)
 
         subparsers = self.parser.add_subparsers(dest='command',
                                                 # Use custom message when no subhandler is added
@@ -244,6 +245,14 @@ class Arguments:
         build_config_cmd.set_defaults(func=start_new_config)
         self._build_args(optionlist=ARGS_BUILD_CONFIG, parser=build_config_cmd)
 
+        # add show-config subcommand
+        show_config_cmd = subparsers.add_parser(
+            'show-config',
+            help="Show resolved config",
+        )
+        show_config_cmd.set_defaults(func=start_show_config)
+        self._build_args(optionlist=ARGS_SHOW_CONFIG, parser=show_config_cmd)
+
         # add new-strategy subcommand
         build_strategy_cmd = subparsers.add_parser(
             'new-strategy',

freqtrade/commands/build_config_commands.py

@@ -5,9 +5,12 @@ from typing import Any, Dict, List
 
 from questionary import Separator, prompt
 
+from freqtrade.configuration import sanitize_config
+from freqtrade.configuration.config_setup import setup_utils_configuration
 from freqtrade.configuration.detect_environment import running_in_docker
 from freqtrade.configuration.directory_operations import chown_user_directory
 from freqtrade.constants import UNLIMITED_STAKE_AMOUNT
+from freqtrade.enums import RunMode
 from freqtrade.exceptions import OperationalException
 from freqtrade.exchange import MAP_EXCHANGE_CHILDCLASS, available_exchanges
 from freqtrade.util import render_template
@@ -264,3 +267,19 @@ def start_new_config(args: Dict[str, Any]) -> None:
                 "Please delete it or use a different configuration file name.")
     selections = ask_user_config()
     deploy_new_config(config_path, selections)
+
+
+def start_show_config(args: Dict[str, Any]) -> None:
+
+    config = setup_utils_configuration(args, RunMode.UTIL_EXCHANGE, set_dry=False)
+
+    # TODO: Sanitize from sensitive info before printing
+
+    print("Your combined configuration is:")
+    config_sanitized = sanitize_config(
+        config['original_config'],
+        show_sensitive=args.get('show_sensitive', False)
+    )
+
+    from rich import print_json
+    print_json(data=config_sanitized)

freqtrade/commands/cli_options.py

@@ -716,4 +716,10 @@ AVAILABLE_CLI_OPTIONS = {
         help='Specify startup candles to be checked (`199`, `499`, `999`, `1999`).',
         nargs='+',
     ),
+    "show_sensitive": Arg(
+        '--show-sensitive',
+        help='Show secrets in the output.',
+        action='store_true',
+        default=False,
+    ),
 }

freqtrade/configuration/__init__.py

@@ -1,5 +1,6 @@
 # flake8: noqa: F401
 
+from freqtrade.configuration.config_secrets import sanitize_config
 from freqtrade.configuration.config_setup import setup_utils_configuration
 from freqtrade.configuration.config_validation import validate_config_consistency
 from freqtrade.configuration.configuration import Configuration

freqtrade/configuration/config_secrets.py

@@ -0,0 +1,36 @@
+from copy import deepcopy
+
+from freqtrade.constants import Config
+
+
+def sanitize_config(config: Config, *, show_sensitive: bool = False) -> Config:
+    """
+    Remove sensitive information from the config.
+    :param config: Configuration
+    :param show_sensitive: Show sensitive information
+    :return: Configuration
+    """
+    if show_sensitive:
+        return config
+    keys_to_remove = [
+        "exchange.key",
+        "exchange.secret",
+        "exchange.password",
+        "exchange.uid",
+        "telegram.token",
+        "telegram.chat_id",
+        "discord.webhook_url",
+        "api_server.password",
+    ]
+    config = deepcopy(config)
+    for key in keys_to_remove:
+        if '.' in key:
+            nested_keys = key.split('.')
+            nested_config = config
+            for nested_key in nested_keys[:-1]:
+                nested_config = nested_config.get(nested_key, {})
+            nested_config[nested_keys[-1]] = 'REDACTED'
+        else:
+            config[key] = 'REDACTED'
+
+    return config

freqtrade/configuration/config_setup.py

@@ -10,7 +10,8 @@ from .configuration import Configuration
 logger = logging.getLogger(__name__)
 
 
-def setup_utils_configuration(args: Dict[str, Any], method: RunMode) -> Dict[str, Any]:
+def setup_utils_configuration(
+        args: Dict[str, Any], method: RunMode, *, set_dry: bool = True) -> Dict[str, Any]:
     """
     Prepare the configuration for utils subcommands
     :param args: Cli args from Arguments()
@@ -21,6 +22,7 @@ def setup_utils_configuration(args: Dict[str, Any], method: RunMode) -> Dict[str
     config = configuration.get_config()
 
     # Ensure these modes are using Dry-run
+    if set_dry:
         config['dry_run'] = True
     validate_config_consistency(config, preliminary=True)
 

freqtrade/configuration/configuration.py

@@ -200,6 +200,12 @@ class Configuration:
             config['exportfilename'] = (config['user_data_dir']
                                         / 'backtest_results')
 
+        if self.args.get('show_sensitive'):
+            logger.warning(
+                "Sensitive information will be shown in the upcomming output. "
+                "Please make sure to never share this output without redacting "
+                "the information yourself.")
+
     def _process_optimize_options(self, config: Config) -> None:
 
         # This will override the strategy configuration

tests/commands/test_commands.py

@@ -12,9 +12,9 @@ from freqtrade.commands import (start_backtesting_show, start_convert_data, star
                                 start_create_userdir, start_download_data, start_hyperopt_list,
                                 start_hyperopt_show, start_install_ui, start_list_data,
                                 start_list_exchanges, start_list_markets, start_list_strategies,
-                                start_list_timeframes, start_new_strategy, start_show_trades,
-                                start_strategy_update, start_test_pairlist, start_trading,
-                                start_webserver)
+                                start_list_timeframes, start_new_strategy, start_show_config,
+                                start_show_trades, start_strategy_update, start_test_pairlist,
+                                start_trading, start_webserver)
 from freqtrade.commands.db_commands import start_convert_db
 from freqtrade.commands.deploy_commands import (clean_ui_subdir, download_and_install_ui,
                                                 get_ui_download_url, read_ui_version)
@@ -39,6 +39,14 @@ def test_setup_utils_configuration():
     assert "exchange" in config
     assert config['dry_run'] is True
 
+    args = [
+        'list-exchanges', '--config', 'tests/testdata/testconfigs/testconfig.json',
+    ]
+
+    config = setup_utils_configuration(get_args(args), RunMode.OTHER, set_dry=False)
+    assert "exchange" in config
+    assert config['dry_run'] is False
+
 
 def test_start_trading_fail(mocker, caplog):
 
@@ -1572,3 +1580,33 @@ def test_start_strategy_updater(mocker, tmp_path):
     start_strategy_update(pargs)
     # Number of strategies in the test directory
     assert sc_mock.call_count == 2
+
+
+def test_start_show_config(capsys, caplog):
+    args = [
+        "show-config",
+        "--config",
+        "tests/testdata/testconfigs/main_test_config.json",
+    ]
+    pargs = get_args(args)
+    start_show_config(pargs)
+
+    captured = capsys.readouterr()
+    assert "Your combined configuration is:" in captured.out
+    assert '"max_open_trades":' in captured.out
+    assert '"secret": "REDACTED"' in captured.out
+
+    args = [
+        "show-config",
+        "--config",
+        "tests/testdata/testconfigs/main_test_config.json",
+        "--show-sensitive"
+    ]
+    pargs = get_args(args)
+    start_show_config(pargs)
+
+    captured = capsys.readouterr()
+    assert "Your combined configuration is:" in captured.out
+    assert '"max_open_trades":' in captured.out
+    assert '"secret": "REDACTED"' not in captured.out
+    assert log_has_re(r'Sensitive information will be shown in the upcomming output.*', caplog)

tests/test_configuration.py

@@ -10,6 +10,7 @@ from jsonschema import ValidationError
 
 from freqtrade.commands import Arguments
 from freqtrade.configuration import Configuration, validate_config_consistency
+from freqtrade.configuration.config_secrets import sanitize_config
 from freqtrade.configuration.config_validation import validate_config_schema
 from freqtrade.configuration.deprecated_settings import (check_conflicting_settings,
                                                          process_deprecated_setting,
@@ -1426,7 +1427,7 @@ def test_flat_vars_to_nested_dict(caplog):
     assert not log_has("Loading variable 'NOT_RELEVANT'", caplog)
 
 
-def test_setup_hyperopt_freqai(mocker, default_conf, caplog) -> None:
+def test_setup_hyperopt_freqai(mocker, default_conf) -> None:
     patched_configuration_load_config_file(mocker, default_conf)
     mocker.patch(
         'freqtrade.configuration.configuration.create_datadir',
@@ -1459,7 +1460,7 @@ def test_setup_hyperopt_freqai(mocker, default_conf, caplog) -> None:
         validate_config_consistency(config)
 
 
-def test_setup_freqai_backtesting(mocker, default_conf, caplog) -> None:
+def test_setup_freqai_backtesting(mocker, default_conf) -> None:
     patched_configuration_load_config_file(mocker, default_conf)
     mocker.patch(
         'freqtrade.configuration.configuration.create_datadir',
@@ -1506,3 +1507,17 @@ def test_setup_freqai_backtesting(mocker, default_conf, caplog) -> None:
         OperationalException, match=r".* pass --timerange if you intend to use FreqAI .*"
     ):
         validate_config_consistency(conf)
+
+
+def test_sanitize_config(default_conf_usdt):
+    assert default_conf_usdt['exchange']['key'] != 'REDACTED'
+    res = sanitize_config(default_conf_usdt)
+    # Didn't modify original dict
+    assert default_conf_usdt['exchange']['key'] != 'REDACTED'
+
+    assert res['exchange']['key'] == 'REDACTED'
+    assert res['exchange']['secret'] == 'REDACTED'
+
+    res = sanitize_config(default_conf_usdt, show_sensitive=True)
+    assert res['exchange']['key'] == default_conf_usdt['exchange']['key']
+    assert res['exchange']['secret'] == default_conf_usdt['exchange']['secret']

tests/testdata/testconfigs/test_base_config.json

@@ -1,6 +1,6 @@
 {
     "stake_currency": "",
-    "dry_run": true,
+    "dry_run": false,
     "exchange": {
         "name": "",
         "key": "",