From f94dbcd0853e87e6cb6996cbf8587ed5732b0138 Mon Sep 17 00:00:00 2001 From: anasyusef Date: Mon, 12 Jul 2021 12:02:10 +0000 Subject: [PATCH 1/6] feat: censor password from logs --- .terraform/modules/vpc | 1 + freqtrade/misc.py | 14 ++++++++++++++ 2 files changed, 15 insertions(+) create mode 160000 .terraform/modules/vpc diff --git a/.terraform/modules/vpc b/.terraform/modules/vpc new file mode 160000 index 000000000..88e1c6ec7 --- /dev/null +++ b/.terraform/modules/vpc @@ -0,0 +1 @@ +Subproject commit 88e1c6ec7a8f6337b0d6bb9840bb70c8b3f14051 diff --git a/freqtrade/misc.py b/freqtrade/misc.py index 967f08299..79c6aec67 100644 --- a/freqtrade/misc.py +++ b/freqtrade/misc.py @@ -8,6 +8,7 @@ from datetime import datetime from pathlib import Path from typing import Any, Iterator, List from typing.io import IO +from urllib.parse import urlparse import rapidjson @@ -214,3 +215,16 @@ def chunks(lst: List[Any], n: int) -> Iterator[List[Any]]: """ for chunk in range(0, len(lst), n): yield (lst[chunk:chunk + n]) + + +def parse_db_uri_for_logging(uri: str): + """ + Helper method to parse the DB URI and return the same DB URI with the password censored + if it contains it. Otherwise, return the DB URI unchanged + :param uri: DB URI to parse for logging + """ + parsed_db_uri = urlparse(uri) + if not parsed_db_uri.netloc: # No need for censoring as no password was provided + return uri + pwd = parsed_db_uri.netloc.split(':')[1].split('@')[0] + return parsed_db_uri.geturl().replace(f':{pwd}@', ':****@') From 6a53e2c764ff2aaad68e83269604f364b11cdf82 Mon Sep 17 00:00:00 2001 From: anasyusef Date: Mon, 12 Jul 2021 12:08:01 +0000 Subject: [PATCH 2/6] feat: apply censoring to logging --- freqtrade/commands/list_commands.py | 4 ++-- freqtrade/configuration/configuration.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/freqtrade/commands/list_commands.py b/freqtrade/commands/list_commands.py index cd26aa60e..410b9b72b 100644 --- a/freqtrade/commands/list_commands.py +++ b/freqtrade/commands/list_commands.py @@ -14,7 +14,7 @@ from freqtrade.constants import USERPATH_HYPEROPTS, USERPATH_STRATEGIES from freqtrade.enums import RunMode from freqtrade.exceptions import OperationalException from freqtrade.exchange import market_is_active, validate_exchanges -from freqtrade.misc import plural +from freqtrade.misc import parse_db_uri_for_logging, plural from freqtrade.resolvers import ExchangeResolver, StrategyResolver @@ -225,7 +225,7 @@ def start_show_trades(args: Dict[str, Any]) -> None: if 'db_url' not in config: raise OperationalException("--db-url is required for this command.") - logger.info(f'Using DB: "{config["db_url"]}"') + logger.info(f'Using DB: "{parse_db_uri_for_logging(config["db_url"])}"') init_db(config['db_url'], clean_open_orders=False) tfilter = [] diff --git a/freqtrade/configuration/configuration.py b/freqtrade/configuration/configuration.py index 1d2e3f802..ea202355a 100644 --- a/freqtrade/configuration/configuration.py +++ b/freqtrade/configuration/configuration.py @@ -15,7 +15,7 @@ from freqtrade.configuration.load_config import load_config_file, load_file from freqtrade.enums import NON_UTIL_MODES, TRADING_MODES, RunMode from freqtrade.exceptions import OperationalException from freqtrade.loggers import setup_logging -from freqtrade.misc import deep_merge_dicts +from freqtrade.misc import deep_merge_dicts, parse_db_uri_for_logging logger = logging.getLogger(__name__) @@ -144,7 +144,7 @@ class Configuration: config['db_url'] = constants.DEFAULT_DB_PROD_URL logger.info('Dry run is disabled') - logger.info(f'Using DB: "{config["db_url"]}"') + logger.info(f'Using DB: "{parse_db_uri_for_logging(config["db_url"])}"') def _process_common_options(self, config: Dict[str, Any]) -> None: From c78b2075d84bb1bcd685e141c4bcd7149fdca95c Mon Sep 17 00:00:00 2001 From: anasyusef Date: Mon, 12 Jul 2021 12:27:59 +0000 Subject: [PATCH 3/6] feat: add one additional asterisk --- freqtrade/misc.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/misc.py b/freqtrade/misc.py index 79c6aec67..6f439866b 100644 --- a/freqtrade/misc.py +++ b/freqtrade/misc.py @@ -227,4 +227,4 @@ def parse_db_uri_for_logging(uri: str): if not parsed_db_uri.netloc: # No need for censoring as no password was provided return uri pwd = parsed_db_uri.netloc.split(':')[1].split('@')[0] - return parsed_db_uri.geturl().replace(f':{pwd}@', ':****@') + return parsed_db_uri.geturl().replace(f':{pwd}@', ':*****@') From 313cf6a01307f9d7826781c13e184b31374ffee4 Mon Sep 17 00:00:00 2001 From: anasyusef Date: Mon, 12 Jul 2021 12:28:34 +0000 Subject: [PATCH 4/6] test: add test for parsing db uri --- tests/test_misc.py | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/tests/test_misc.py b/tests/test_misc.py index e6ba70aee..5b5a1400f 100644 --- a/tests/test_misc.py +++ b/tests/test_misc.py @@ -9,7 +9,7 @@ import pytest from freqtrade.misc import (decimals_per_coin, file_dump_json, file_load_json, format_ms_time, pair_to_filename, plural, render_template, render_template_with_fallback, round_coin_value, safe_value_fallback, - safe_value_fallback2, shorten_date) + safe_value_fallback2, shorten_date, parse_db_uri_for_logging) def test_decimals_per_coin(): @@ -179,3 +179,18 @@ def test_render_template_fallback(mocker): ) assert isinstance(val, str) assert 'if self.dp' in val + +def test_parse_db_uri_for_logging() -> None: + postgresql_conn_uri = "postgresql+psycopg2://scott123:scott123@host/dbname" + mariadb_conn_uri = "mariadb+mariadbconnector://app_user:Password123!@127.0.0.1:3306/company" + mysql_conn_uri = "mysql+pymysql://user:pass@some_mariadb/dbname?charset=utf8mb4" + sqlite_conn_uri = "sqlite:////freqtrade/user_data/tradesv3.sqlite" + censored_pwd = "*****" + + get_pwd = lambda x: x.split(':')[2].split('@')[0] + + assert get_pwd(parse_db_uri_for_logging(postgresql_conn_uri)) == censored_pwd + assert get_pwd(parse_db_uri_for_logging(mariadb_conn_uri)) == censored_pwd + assert get_pwd(parse_db_uri_for_logging(mysql_conn_uri)) == censored_pwd + assert sqlite_conn_uri == parse_db_uri_for_logging(sqlite_conn_uri) + From 8def18b002a524a6206a4dd88519717879d41259 Mon Sep 17 00:00:00 2001 From: anasyusef Date: Mon, 12 Jul 2021 12:31:13 +0000 Subject: [PATCH 5/6] style: apply flake8 formatting --- tests/test_misc.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/test_misc.py b/tests/test_misc.py index 5b5a1400f..221c7b712 100644 --- a/tests/test_misc.py +++ b/tests/test_misc.py @@ -7,9 +7,9 @@ from unittest.mock import MagicMock import pytest from freqtrade.misc import (decimals_per_coin, file_dump_json, file_load_json, format_ms_time, - pair_to_filename, plural, render_template, + pair_to_filename, parse_db_uri_for_logging, plural, render_template, render_template_with_fallback, round_coin_value, safe_value_fallback, - safe_value_fallback2, shorten_date, parse_db_uri_for_logging) + safe_value_fallback2, shorten_date) def test_decimals_per_coin(): @@ -180,6 +180,7 @@ def test_render_template_fallback(mocker): assert isinstance(val, str) assert 'if self.dp' in val + def test_parse_db_uri_for_logging() -> None: postgresql_conn_uri = "postgresql+psycopg2://scott123:scott123@host/dbname" mariadb_conn_uri = "mariadb+mariadbconnector://app_user:Password123!@127.0.0.1:3306/company" @@ -187,10 +188,9 @@ def test_parse_db_uri_for_logging() -> None: sqlite_conn_uri = "sqlite:////freqtrade/user_data/tradesv3.sqlite" censored_pwd = "*****" - get_pwd = lambda x: x.split(':')[2].split('@')[0] + def get_pwd(x): return x.split(':')[2].split('@')[0] assert get_pwd(parse_db_uri_for_logging(postgresql_conn_uri)) == censored_pwd assert get_pwd(parse_db_uri_for_logging(mariadb_conn_uri)) == censored_pwd assert get_pwd(parse_db_uri_for_logging(mysql_conn_uri)) == censored_pwd assert sqlite_conn_uri == parse_db_uri_for_logging(sqlite_conn_uri) - From 91e5562ae01e68a6f5b98f24c9926d76cc205066 Mon Sep 17 00:00:00 2001 From: anasyusef Date: Mon, 12 Jul 2021 12:31:13 +0000 Subject: [PATCH 6/6] style: apply flake8 formatting --- .terraform/modules/vpc | 1 - tests/test_misc.py | 8 ++++---- 2 files changed, 4 insertions(+), 5 deletions(-) delete mode 160000 .terraform/modules/vpc diff --git a/.terraform/modules/vpc b/.terraform/modules/vpc deleted file mode 160000 index 88e1c6ec7..000000000 --- a/.terraform/modules/vpc +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 88e1c6ec7a8f6337b0d6bb9840bb70c8b3f14051 diff --git a/tests/test_misc.py b/tests/test_misc.py index 5b5a1400f..221c7b712 100644 --- a/tests/test_misc.py +++ b/tests/test_misc.py @@ -7,9 +7,9 @@ from unittest.mock import MagicMock import pytest from freqtrade.misc import (decimals_per_coin, file_dump_json, file_load_json, format_ms_time, - pair_to_filename, plural, render_template, + pair_to_filename, parse_db_uri_for_logging, plural, render_template, render_template_with_fallback, round_coin_value, safe_value_fallback, - safe_value_fallback2, shorten_date, parse_db_uri_for_logging) + safe_value_fallback2, shorten_date) def test_decimals_per_coin(): @@ -180,6 +180,7 @@ def test_render_template_fallback(mocker): assert isinstance(val, str) assert 'if self.dp' in val + def test_parse_db_uri_for_logging() -> None: postgresql_conn_uri = "postgresql+psycopg2://scott123:scott123@host/dbname" mariadb_conn_uri = "mariadb+mariadbconnector://app_user:Password123!@127.0.0.1:3306/company" @@ -187,10 +188,9 @@ def test_parse_db_uri_for_logging() -> None: sqlite_conn_uri = "sqlite:////freqtrade/user_data/tradesv3.sqlite" censored_pwd = "*****" - get_pwd = lambda x: x.split(':')[2].split('@')[0] + def get_pwd(x): return x.split(':')[2].split('@')[0] assert get_pwd(parse_db_uri_for_logging(postgresql_conn_uri)) == censored_pwd assert get_pwd(parse_db_uri_for_logging(mariadb_conn_uri)) == censored_pwd assert get_pwd(parse_db_uri_for_logging(mysql_conn_uri)) == censored_pwd assert sqlite_conn_uri == parse_db_uri_for_logging(sqlite_conn_uri) -