From 6b98bfe9b73ecacd25b0b9ade94723ecb70ef1fe Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 6 Jul 2024 11:42:40 +0200 Subject: [PATCH 01/38] Simplify output --- freqtrade/commands/list_commands.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/freqtrade/commands/list_commands.py b/freqtrade/commands/list_commands.py index 97be7bac6..153b1054d 100644 --- a/freqtrade/commands/list_commands.py +++ b/freqtrade/commands/list_commands.py @@ -57,11 +57,12 @@ def start_list_exchanges(args: Dict[str, Any]) -> None: if args["list_exchanges_all"]: exchanges = [build_entry(e, True) for e in exchanges] - print(f"All exchanges supported by the ccxt library ({len(exchanges)} exchanges):") + title = f"All exchanges supported by the ccxt library ({len(exchanges)} exchanges):" else: exchanges = [build_entry(e, False) for e in exchanges if e["valid"] is not False] - print(f"Exchanges available for Freqtrade ({len(exchanges)} exchanges):") + title = f"Exchanges available for Freqtrade ({len(exchanges)} exchanges):" + print(title) print( tabulate( exchanges, From 95097d1f51fe65e96f1ea7dde7527596ba583f50 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 6 Jul 2024 11:49:26 +0200 Subject: [PATCH 02/38] chore: don't overwrite exchanges variable --- freqtrade/commands/list_commands.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/freqtrade/commands/list_commands.py b/freqtrade/commands/list_commands.py index 153b1054d..b8e479462 100644 --- a/freqtrade/commands/list_commands.py +++ b/freqtrade/commands/list_commands.py @@ -26,10 +26,10 @@ def start_list_exchanges(args: Dict[str, Any]) -> None: :param args: Cli args from Arguments() :return: None """ - exchanges = list_available_exchanges(args["list_exchanges_all"]) + available_exchanges = list_available_exchanges(args["list_exchanges_all"]) if args["print_one_column"]: - print("\n".join([e["name"] for e in exchanges])) + print("\n".join([e["name"] for e in available_exchanges])) else: headers = { "name": "Exchange name", @@ -56,10 +56,12 @@ def start_list_exchanges(args: Dict[str, Any]) -> None: return result if args["list_exchanges_all"]: - exchanges = [build_entry(e, True) for e in exchanges] + exchanges = [build_entry(e, True) for e in available_exchanges] title = f"All exchanges supported by the ccxt library ({len(exchanges)} exchanges):" else: - exchanges = [build_entry(e, False) for e in exchanges if e["valid"] is not False] + exchanges = [ + build_entry(e, False) for e in available_exchanges if e["valid"] is not False + ] title = f"Exchanges available for Freqtrade ({len(exchanges)} exchanges):" print(title) From b6b589b1f01004ef598bb64f507ab7b52ba14f6f Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 6 Jul 2024 12:59:50 +0200 Subject: [PATCH 03/38] Update list-exchanges to use rich tables --- freqtrade/commands/list_commands.py | 85 ++++++++++++++++------------- tests/commands/test_commands.py | 4 +- 2 files changed, 48 insertions(+), 41 deletions(-) diff --git a/freqtrade/commands/list_commands.py b/freqtrade/commands/list_commands.py index b8e479462..e3c598786 100644 --- a/freqtrade/commands/list_commands.py +++ b/freqtrade/commands/list_commands.py @@ -1,11 +1,14 @@ import csv import logging import sys -from typing import Any, Dict, List, Union +from typing import Any, Dict, List import rapidjson from colorama import Fore, Style from colorama import init as colorama_init +from rich.console import Console +from rich.table import Table +from rich.text import Text from tabulate import tabulate from freqtrade.configuration import setup_utils_configuration @@ -14,7 +17,7 @@ from freqtrade.exceptions import ConfigurationError, OperationalException from freqtrade.exchange import list_available_exchanges, market_is_active from freqtrade.misc import parse_db_uri_for_logging, plural from freqtrade.resolvers import ExchangeResolver, StrategyResolver -from freqtrade.types import ValidExchangesType +from freqtrade.types.valid_exchanges_type import ValidExchangesType logger = logging.getLogger(__name__) @@ -26,51 +29,55 @@ def start_list_exchanges(args: Dict[str, Any]) -> None: :param args: Cli args from Arguments() :return: None """ - available_exchanges = list_available_exchanges(args["list_exchanges_all"]) + available_exchanges: List[ValidExchangesType] = list_available_exchanges( + args["list_exchanges_all"] + ) if args["print_one_column"]: print("\n".join([e["name"] for e in available_exchanges])) else: - headers = { - "name": "Exchange name", - "supported": "Supported", - "trade_modes": "Markets", - "comment": "Reason", - } - headers.update({"valid": "Valid"} if args["list_exchanges_all"] else {}) + if args["list_exchanges_all"]: + title = ( + f"All exchanges supported by the ccxt library " + f"({len(available_exchanges)} exchanges):" + ) + else: + available_exchanges = [e for e in available_exchanges if e["valid"] is not False] + title = f"Exchanges available for Freqtrade ({len(available_exchanges)} exchanges):" - def build_entry(exchange: ValidExchangesType, valid: bool): - valid_entry = {"valid": exchange["valid"]} if valid else {} - result: Dict[str, Union[str, bool]] = { - "name": exchange["name"], - **valid_entry, - "supported": "Official" if exchange["supported"] else "", - "trade_modes": ("DEX: " if exchange["dex"] else "") - + ", ".join( - (f"{a['margin_mode']} " if a["margin_mode"] else "") + a["trading_mode"] + table = Table(title=title) + + table.add_column("Exchange Name") + table.add_column("Markets") + table.add_column("Reason") + + for exchange in available_exchanges: + name = Text(exchange["name"]) + if exchange["supported"]: + name.append(" (Official)", style="italic") + name.stylize("green bold") + + trade_modes = Text( + ", ".join( + (f"{a.get('margin_mode', '')} {a["trading_mode"]}").lstrip() for a in exchange["trade_modes"] ), - "comment": exchange["comment"], - } - - return result - - if args["list_exchanges_all"]: - exchanges = [build_entry(e, True) for e in available_exchanges] - title = f"All exchanges supported by the ccxt library ({len(exchanges)} exchanges):" - else: - exchanges = [ - build_entry(e, False) for e in available_exchanges if e["valid"] is not False - ] - title = f"Exchanges available for Freqtrade ({len(exchanges)} exchanges):" - - print(title) - print( - tabulate( - exchanges, - headers=headers, + style="", ) - ) + if exchange["dex"]: + trade_modes = Text("DEX: ") + trade_modes + trade_modes.stylize("bold", 0, 3) + + table.add_row( + name, + trade_modes, + exchange["comment"], + style=None if exchange["valid"] else "red", + ) + # table.add_row(*[exchange[header] for header in headers]) + + console = Console() + console.print(table) def _print_objs_tabular(objs: List, print_colorized: bool) -> None: diff --git a/tests/commands/test_commands.py b/tests/commands/test_commands.py index 02b234b6c..818533630 100644 --- a/tests/commands/test_commands.py +++ b/tests/commands/test_commands.py @@ -116,7 +116,7 @@ def test_list_exchanges(capsys): start_list_exchanges(get_args(args)) captured = capsys.readouterr() - assert re.match(r"Exchanges available for Freqtrade.*", captured.out) + assert re.search(r".*Exchanges available for Freqtrade.*", captured.out) assert re.search(r".*binance.*", captured.out) assert re.search(r".*bybit.*", captured.out) @@ -139,7 +139,7 @@ def test_list_exchanges(capsys): start_list_exchanges(get_args(args)) captured = capsys.readouterr() - assert re.match(r"All exchanges supported by the ccxt library.*", captured.out) + assert re.search(r"All exchanges supported by the ccxt library.*", captured.out) assert re.search(r".*binance.*", captured.out) assert re.search(r".*bingx.*", captured.out) assert re.search(r".*bitmex.*", captured.out) From bafb6507c459c9a328860d3754c6b3bb783daec2 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 6 Jul 2024 13:09:37 +0200 Subject: [PATCH 04/38] Use Rich table for pair list output --- freqtrade/commands/list_commands.py | 11 ++++++++--- tests/commands/test_commands.py | 4 ++-- 2 files changed, 10 insertions(+), 5 deletions(-) diff --git a/freqtrade/commands/list_commands.py b/freqtrade/commands/list_commands.py index e3c598786..d6fdd3111 100644 --- a/freqtrade/commands/list_commands.py +++ b/freqtrade/commands/list_commands.py @@ -280,9 +280,14 @@ def start_list_markets(args: Dict[str, Any], pairs_only: bool = False) -> None: writer.writeheader() writer.writerows(tabular_data) else: - # print data as a table, with the human-readable summary - print(f"{summary_str}:") - print(tabulate(tabular_data, headers="keys", tablefmt="psql", stralign="right")) + table = Table(title=summary_str) + for header in headers: + table.add_column(header, justify="right") + for row in tabular_data: + table.add_row(*[str(row[header]) for header in headers]) + + console = Console() + console.print(table) elif not ( args.get("print_one_column", False) or args.get("list_pairs_print_json", False) diff --git a/tests/commands/test_commands.py b/tests/commands/test_commands.py index 818533630..7e4d2e2cc 100644 --- a/tests/commands/test_commands.py +++ b/tests/commands/test_commands.py @@ -293,7 +293,7 @@ def test_list_markets(mocker, markets_static, capsys): pargs["config"] = None start_list_markets(pargs, False) captured = capsys.readouterr() - assert re.match("\nExchange Binance has 12 active markets:\n", captured.out) + assert re.search(r".*Exchange Binance has 12 active markets.*", captured.out) patch_exchange(mocker, api_mock=api_mock, exchange="binance", mock_markets=markets_static) # Test with --all: all markets @@ -491,7 +491,7 @@ def test_list_markets(mocker, markets_static, capsys): ] start_list_markets(get_args(args), False) captured = capsys.readouterr() - assert "Exchange Binance has 12 active markets:\n" in captured.out + assert "Exchange Binance has 12 active markets" in captured.out # Test tabular output, no markets found args = [ From 2e0372d24739c2913281cd9210c2c7fcb16a2cbd Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 6 Jul 2024 13:12:16 +0200 Subject: [PATCH 05/38] extract print_rich_table --- freqtrade/commands/list_commands.py | 22 ++++++++++++++-------- 1 file changed, 14 insertions(+), 8 deletions(-) diff --git a/freqtrade/commands/list_commands.py b/freqtrade/commands/list_commands.py index d6fdd3111..4260ed87b 100644 --- a/freqtrade/commands/list_commands.py +++ b/freqtrade/commands/list_commands.py @@ -80,6 +80,19 @@ def start_list_exchanges(args: Dict[str, Any]) -> None: console.print(table) +def _print_rich_table(summary: str, headers: List[str], tabular_data: List[Dict[str, Any]]) -> None: + table = Table(title=summary) + + for header in headers: + table.add_column(header, justify="right") + + for row in tabular_data: + table.add_row(*[str(row[header]) for header in headers]) + + console = Console() + console.print(table) + + def _print_objs_tabular(objs: List, print_colorized: bool) -> None: if print_colorized: colorama_init(autoreset=True) @@ -280,14 +293,7 @@ def start_list_markets(args: Dict[str, Any], pairs_only: bool = False) -> None: writer.writeheader() writer.writerows(tabular_data) else: - table = Table(title=summary_str) - for header in headers: - table.add_column(header, justify="right") - for row in tabular_data: - table.add_row(*[str(row[header]) for header in headers]) - - console = Console() - console.print(table) + _print_rich_table(summary_str, headers, tabular_data) elif not ( args.get("print_one_column", False) or args.get("list_pairs_print_json", False) From 8d00e1d929a7ded656f6e5876f2135cd3e595255 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 6 Jul 2024 18:09:03 +0200 Subject: [PATCH 06/38] feat: Use Rich for list-strategies command --- freqtrade/commands/list_commands.py | 40 +++++++++++++++-------------- 1 file changed, 21 insertions(+), 19 deletions(-) diff --git a/freqtrade/commands/list_commands.py b/freqtrade/commands/list_commands.py index 4260ed87b..c7122afc2 100644 --- a/freqtrade/commands/list_commands.py +++ b/freqtrade/commands/list_commands.py @@ -59,7 +59,7 @@ def start_list_exchanges(args: Dict[str, Any]) -> None: trade_modes = Text( ", ".join( - (f"{a.get('margin_mode', '')} {a["trading_mode"]}").lstrip() + (f"{a.get('margin_mode', '')} {a['trading_mode']}").lstrip() for a in exchange["trade_modes"] ), style="", @@ -94,27 +94,17 @@ def _print_rich_table(summary: str, headers: List[str], tabular_data: List[Dict[ def _print_objs_tabular(objs: List, print_colorized: bool) -> None: - if print_colorized: - colorama_init(autoreset=True) - red = Fore.RED - yellow = Fore.YELLOW - reset = Style.RESET_ALL - else: - red = "" - yellow = "" - reset = "" - names = [s["name"] for s in objs] objs_to_print = [ { - "name": s["name"] if s["name"] else "--", - "location": s["location_rel"], + "name": Text(s["name"] if s["name"] else "--"), + "location": Text(s["location_rel"]), "status": ( - red + "LOAD FAILED" + reset + Text("LOAD FAILED", style="bold red") if s["class"] is None - else "OK" + else Text("OK", style="bold green") if names.count(s["name"]) == 1 - else yellow + "DUPLICATE NAME" + reset + else Text("DUPLICATE NAME", style="bold yellow") ), } for s in objs @@ -124,11 +114,23 @@ def _print_objs_tabular(objs: List, print_colorized: bool) -> None: objs_to_print[idx].update( { "hyperoptable": "Yes" if s["hyperoptable"]["count"] > 0 else "No", - "buy-Params": len(s["hyperoptable"].get("buy", [])), - "sell-Params": len(s["hyperoptable"].get("sell", [])), + "buy-Params": str(len(s["hyperoptable"].get("buy", []))), + "sell-Params": str(len(s["hyperoptable"].get("sell", []))), } ) - print(tabulate(objs_to_print, headers="keys", tablefmt="psql", stralign="right")) + table = Table(title="Available:") + + for header in objs_to_print[0].keys(): + table.add_column(header.capitalize(), justify="right") + + for row in objs_to_print: + table.add_row(*[row[header] for header in objs_to_print[0].keys()]) + + console = Console( + color_system="auto" if print_colorized else None, + width=200 if "pytest" in sys.modules else None, + ) + console.print(table) def start_list_strategies(args: Dict[str, Any]) -> None: From 2d8470b2546fe3e47a00f20b21b28c9d4ade62c3 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 6 Jul 2024 18:26:08 +0200 Subject: [PATCH 07/38] Remove unused imports --- freqtrade/commands/list_commands.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/freqtrade/commands/list_commands.py b/freqtrade/commands/list_commands.py index c7122afc2..e473d6e90 100644 --- a/freqtrade/commands/list_commands.py +++ b/freqtrade/commands/list_commands.py @@ -4,12 +4,9 @@ import sys from typing import Any, Dict, List import rapidjson -from colorama import Fore, Style -from colorama import init as colorama_init from rich.console import Console from rich.table import Table from rich.text import Text -from tabulate import tabulate from freqtrade.configuration import setup_utils_configuration from freqtrade.enums import RunMode From 768a51cb9bebb70e3bd2b3b468d7e8d8a4896d1e Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 6 Jul 2024 18:26:36 +0200 Subject: [PATCH 08/38] Extract rich_table print to utils --- freqtrade/commands/list_commands.py | 16 ++-------------- freqtrade/util/__init__.py | 2 ++ freqtrade/util/rich_tables.py | 17 +++++++++++++++++ 3 files changed, 21 insertions(+), 14 deletions(-) create mode 100644 freqtrade/util/rich_tables.py diff --git a/freqtrade/commands/list_commands.py b/freqtrade/commands/list_commands.py index e473d6e90..ea2c84ae9 100644 --- a/freqtrade/commands/list_commands.py +++ b/freqtrade/commands/list_commands.py @@ -15,6 +15,7 @@ from freqtrade.exchange import list_available_exchanges, market_is_active from freqtrade.misc import parse_db_uri_for_logging, plural from freqtrade.resolvers import ExchangeResolver, StrategyResolver from freqtrade.types.valid_exchanges_type import ValidExchangesType +from freqtrade.util import print_rich_table logger = logging.getLogger(__name__) @@ -77,19 +78,6 @@ def start_list_exchanges(args: Dict[str, Any]) -> None: console.print(table) -def _print_rich_table(summary: str, headers: List[str], tabular_data: List[Dict[str, Any]]) -> None: - table = Table(title=summary) - - for header in headers: - table.add_column(header, justify="right") - - for row in tabular_data: - table.add_row(*[str(row[header]) for header in headers]) - - console = Console() - console.print(table) - - def _print_objs_tabular(objs: List, print_colorized: bool) -> None: names = [s["name"] for s in objs] objs_to_print = [ @@ -292,7 +280,7 @@ def start_list_markets(args: Dict[str, Any], pairs_only: bool = False) -> None: writer.writeheader() writer.writerows(tabular_data) else: - _print_rich_table(summary_str, headers, tabular_data) + print_rich_table(summary_str, headers, tabular_data) elif not ( args.get("print_one_column", False) or args.get("list_pairs_print_json", False) diff --git a/freqtrade/util/__init__.py b/freqtrade/util/__init__.py index 503f5861a..f478829e6 100644 --- a/freqtrade/util/__init__.py +++ b/freqtrade/util/__init__.py @@ -15,6 +15,7 @@ from freqtrade.util.formatters import decimals_per_coin, fmt_coin, round_value from freqtrade.util.ft_precise import FtPrecise from freqtrade.util.measure_time import MeasureTime from freqtrade.util.periodic_cache import PeriodicCache +from freqtrade.util.rich_tables import print_rich_table from freqtrade.util.template_renderer import render_template, render_template_with_fallback # noqa @@ -36,4 +37,5 @@ __all__ = [ "round_value", "fmt_coin", "MeasureTime", + "print_rich_table", ] diff --git a/freqtrade/util/rich_tables.py b/freqtrade/util/rich_tables.py new file mode 100644 index 000000000..f773a4fd1 --- /dev/null +++ b/freqtrade/util/rich_tables.py @@ -0,0 +1,17 @@ +from typing import Any, Dict, List + +from rich.console import Console +from rich.table import Table + + +def print_rich_table(summary: str, headers: List[str], tabular_data: List[Dict[str, Any]]) -> None: + table = Table(title=summary) + + for header in headers: + table.add_column(header, justify="right") + + for row in tabular_data: + table.add_row(*[str(row[header]) for header in headers]) + + console = Console() + console.print(table) From c9b3987d334d4e4bc9d1bd1a7d02c9b0d7bd5090 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 6 Jul 2024 18:30:48 +0200 Subject: [PATCH 09/38] chore: update rich-table print helper --- freqtrade/commands/list_commands.py | 2 +- freqtrade/util/rich_tables.py | 11 ++++++++--- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/freqtrade/commands/list_commands.py b/freqtrade/commands/list_commands.py index ea2c84ae9..e0f5d2d62 100644 --- a/freqtrade/commands/list_commands.py +++ b/freqtrade/commands/list_commands.py @@ -280,7 +280,7 @@ def start_list_markets(args: Dict[str, Any], pairs_only: bool = False) -> None: writer.writeheader() writer.writerows(tabular_data) else: - print_rich_table(summary_str, headers, tabular_data) + print_rich_table(tabular_data, headers, summary_str) elif not ( args.get("print_one_column", False) or args.get("list_pairs_print_json", False) diff --git a/freqtrade/util/rich_tables.py b/freqtrade/util/rich_tables.py index f773a4fd1..b8ec59d58 100644 --- a/freqtrade/util/rich_tables.py +++ b/freqtrade/util/rich_tables.py @@ -1,17 +1,22 @@ -from typing import Any, Dict, List +from typing import Any, Dict, List, Optional from rich.console import Console from rich.table import Table -def print_rich_table(summary: str, headers: List[str], tabular_data: List[Dict[str, Any]]) -> None: +def print_rich_table( + tabular_data: List[Dict[str, Any]], headers: List[str], summary: Optional[str] = None +) -> None: table = Table(title=summary) for header in headers: table.add_column(header, justify="right") for row in tabular_data: - table.add_row(*[str(row[header]) for header in headers]) + if isinstance(row, dict): + table.add_row(*[str(row[header]) for header in headers]) + else: + table.add_row(*row) console = Console() console.print(table) From c296a8cf829f4868e3cb30fc50db8ff2bea7605b Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 6 Jul 2024 20:59:50 +0200 Subject: [PATCH 10/38] Improve rich tables interface --- freqtrade/util/rich_tables.py | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/freqtrade/util/rich_tables.py b/freqtrade/util/rich_tables.py index b8ec59d58..00c3302de 100644 --- a/freqtrade/util/rich_tables.py +++ b/freqtrade/util/rich_tables.py @@ -1,3 +1,4 @@ +import sys from typing import Any, Dict, List, Optional from rich.console import Console @@ -5,9 +6,13 @@ from rich.table import Table def print_rich_table( - tabular_data: List[Dict[str, Any]], headers: List[str], summary: Optional[str] = None + tabular_data: List[Dict[str, Any]], + headers: List[str], + summary: Optional[str] = None, + *, + table_kwargs: Optional[Dict[str, Any]] = None, ) -> None: - table = Table(title=summary) + table = Table(title=summary, **(table_kwargs or {})) for header in headers: table.add_column(header, justify="right") @@ -18,5 +23,7 @@ def print_rich_table( else: table.add_row(*row) - console = Console() + console = Console( + width=200 if "pytest" in sys.modules else None, + ) console.print(table) From 9f628309e92ce48fd894cbd926f06ca000f11dc2 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 6 Jul 2024 21:00:34 +0200 Subject: [PATCH 11/38] feat: Update list-pairs command to use rich tables --- freqtrade/commands/data_commands.py | 61 +++++++++++++---------------- tests/commands/test_commands.py | 27 ++++++------- 2 files changed, 39 insertions(+), 49 deletions(-) diff --git a/freqtrade/commands/data_commands.py b/freqtrade/commands/data_commands.py index 92e60daa4..f3f56c7b2 100644 --- a/freqtrade/commands/data_commands.py +++ b/freqtrade/commands/data_commands.py @@ -16,6 +16,7 @@ from freqtrade.exceptions import ConfigurationError from freqtrade.exchange import timeframe_to_minutes from freqtrade.plugins.pairlist.pairlist_helpers import dynamic_expand_pairlist from freqtrade.resolvers import ExchangeResolver +from freqtrade.util import print_rich_table from freqtrade.util.migrations import migrate_data @@ -119,8 +120,6 @@ def start_list_data(args: Dict[str, Any]) -> None: config = setup_utils_configuration(args, RunMode.UTIL_NO_EXCHANGE) - from tabulate import tabulate - from freqtrade.data.history import get_datahandler dhc = get_datahandler(config["datadir"], config["dataformat_ohlcv"]) @@ -131,8 +130,7 @@ def start_list_data(args: Dict[str, Any]) -> None: if args["pairs"]: paircombs = [comb for comb in paircombs if comb[0] in args["pairs"]] - - print(f"Found {len(paircombs)} pair / timeframe combinations.") + title = f"Found {len(paircombs)} pair / timeframe combinations." if not config.get("show_timerange"): groupedpair = defaultdict(list) for pair, timeframe, candle_type in sorted( @@ -141,40 +139,35 @@ def start_list_data(args: Dict[str, Any]) -> None: groupedpair[(pair, candle_type)].append(timeframe) if groupedpair: - print( - tabulate( - [ - (pair, ", ".join(timeframes), candle_type) - for (pair, candle_type), timeframes in groupedpair.items() - ], - headers=("Pair", "Timeframe", "Type"), - tablefmt="psql", - stralign="right", - ) + print_rich_table( + [ + (pair, ", ".join(timeframes), candle_type) + for (pair, candle_type), timeframes in groupedpair.items() + ], + ("Pair", "Timeframe", "Type"), + title, + table_kwargs={"min_width": 50}, ) else: paircombs1 = [ (pair, timeframe, candle_type, *dhc.ohlcv_data_min_max(pair, timeframe, candle_type)) for pair, timeframe, candle_type in paircombs ] - - print( - tabulate( - [ - ( - pair, - timeframe, - candle_type, - start.strftime(DATETIME_PRINT_FORMAT), - end.strftime(DATETIME_PRINT_FORMAT), - length, - ) - for pair, timeframe, candle_type, start, end, length in sorted( - paircombs1, key=lambda x: (x[0], timeframe_to_minutes(x[1]), x[2]) - ) - ], - headers=("Pair", "Timeframe", "Type", "From", "To", "Candles"), - tablefmt="psql", - stralign="right", - ) + print_rich_table( + [ + ( + pair, + timeframe, + candle_type, + start.strftime(DATETIME_PRINT_FORMAT), + end.strftime(DATETIME_PRINT_FORMAT), + str(length), + ) + for pair, timeframe, candle_type, start, end, length in sorted( + paircombs1, key=lambda x: (x[0], timeframe_to_minutes(x[1]), x[2]) + ) + ], + ("Pair", "Timeframe", "Type", "From", "To", "Candles"), + summary=title, + table_kwargs={"min_width": 50}, ) diff --git a/tests/commands/test_commands.py b/tests/commands/test_commands.py index 7e4d2e2cc..687bff69f 100644 --- a/tests/commands/test_commands.py +++ b/tests/commands/test_commands.py @@ -1633,8 +1633,8 @@ def test_start_list_data(testdatadir, capsys): start_list_data(pargs) captured = capsys.readouterr() assert "Found 16 pair / timeframe combinations." in captured.out - assert "\n| Pair | Timeframe | Type |\n" in captured.out - assert "\n| UNITTEST/BTC | 1m, 5m, 8m, 30m | spot |\n" in captured.out + assert re.search(r".*Pair.*Timeframe.*Type.*\n", captured.out) + assert re.search(r"\n.* UNITTEST/BTC .* 1m, 5m, 8m, 30m .* spot |\n", captured.out) args = [ "list-data", @@ -1650,9 +1650,9 @@ def test_start_list_data(testdatadir, capsys): start_list_data(pargs) captured = capsys.readouterr() assert "Found 2 pair / timeframe combinations." in captured.out - assert "\n| Pair | Timeframe | Type |\n" in captured.out + assert re.search(r".*Pair.*Timeframe.*Type.*\n", captured.out) assert "UNITTEST/BTC" not in captured.out - assert "\n| XRP/ETH | 1m, 5m | spot |\n" in captured.out + assert re.search(r"\n.* XRP/ETH .* 1m, 5m .* spot |\n", captured.out) args = [ "list-data", @@ -1667,9 +1667,9 @@ def test_start_list_data(testdatadir, capsys): captured = capsys.readouterr() assert "Found 6 pair / timeframe combinations." in captured.out - assert "\n| Pair | Timeframe | Type |\n" in captured.out - assert "\n| XRP/USDT:USDT | 5m, 1h | futures |\n" in captured.out - assert "\n| XRP/USDT:USDT | 1h, 8h | mark |\n" in captured.out + assert re.search(r".*Pair.*Timeframe.*Type.*\n", captured.out) + assert re.search(r"\n.* XRP/USDT:USDT .* 5m, 1h .* futures |\n", captured.out) + assert re.search(r"\n.* XRP/USDT:USDT .* 1h, 8h .* mark |\n", captured.out) args = [ "list-data", @@ -1684,15 +1684,12 @@ def test_start_list_data(testdatadir, capsys): start_list_data(pargs) captured = capsys.readouterr() assert "Found 2 pair / timeframe combinations." in captured.out - assert ( - "\n| Pair | Timeframe | Type " - "| From | To | Candles |\n" - ) in captured.out + assert re.search(r".*Pair.*Timeframe.*Type.*From .* To .* Candles .*\n", captured.out) assert "UNITTEST/BTC" not in captured.out - assert ( - "\n| XRP/ETH | 1m | spot | " - "2019-10-11 00:00:00 | 2019-10-13 11:19:00 | 2469 |\n" - ) in captured.out + assert re.search( + r"\n.* XRP/USDT .* 1m .* spot .* 2019-10-11 00:00:00 .* 2019-10-13 11:19:00 .* 2469 |\n", + captured.out, + ) @pytest.mark.usefixtures("init_persistence") From ffb0cf1a2c2217d08362c8d08ba91397a1718386 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 7 Jul 2024 08:36:51 +0200 Subject: [PATCH 12/38] chore: Improve typing --- freqtrade/commands/list_commands.py | 8 ++++---- freqtrade/util/rich_tables.py | 10 +++++++--- 2 files changed, 11 insertions(+), 7 deletions(-) diff --git a/freqtrade/commands/list_commands.py b/freqtrade/commands/list_commands.py index e0f5d2d62..1696fc8f0 100644 --- a/freqtrade/commands/list_commands.py +++ b/freqtrade/commands/list_commands.py @@ -1,7 +1,7 @@ import csv import logging import sys -from typing import Any, Dict, List +from typing import Any, Dict, List, Union import rapidjson from rich.console import Console @@ -80,10 +80,10 @@ def start_list_exchanges(args: Dict[str, Any]) -> None: def _print_objs_tabular(objs: List, print_colorized: bool) -> None: names = [s["name"] for s in objs] - objs_to_print = [ + objs_to_print: List[Dict[str, Union[Text, str]]] = [ { "name": Text(s["name"] if s["name"] else "--"), - "location": Text(s["location_rel"]), + "location": s["location_rel"], "status": ( Text("LOAD FAILED", style="bold red") if s["class"] is None @@ -103,7 +103,7 @@ def _print_objs_tabular(objs: List, print_colorized: bool) -> None: "sell-Params": str(len(s["hyperoptable"].get("sell", []))), } ) - table = Table(title="Available:") + table = Table() for header in objs_to_print[0].keys(): table.add_column(header.capitalize(), justify="right") diff --git a/freqtrade/util/rich_tables.py b/freqtrade/util/rich_tables.py index 00c3302de..d34162d66 100644 --- a/freqtrade/util/rich_tables.py +++ b/freqtrade/util/rich_tables.py @@ -1,13 +1,17 @@ import sys -from typing import Any, Dict, List, Optional +from typing import Any, Dict, Optional, Sequence, Union from rich.console import Console from rich.table import Table +from rich.text import Text + + +TextOrString = Union[str, Text] def print_rich_table( - tabular_data: List[Dict[str, Any]], - headers: List[str], + tabular_data: Sequence[Union[Dict[str, Any], Sequence[TextOrString]]], + headers: Sequence[str], summary: Optional[str] = None, *, table_kwargs: Optional[Dict[str, Any]] = None, From cdae61e155c64db70ca6726a6cf579e08417e0b7 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 7 Jul 2024 09:38:02 +0200 Subject: [PATCH 13/38] feat: use rich tables for entryexitanalysis --- freqtrade/data/entryexitanalysis.py | 3 ++- freqtrade/util/__init__.py | 3 ++- freqtrade/util/rich_tables.py | 35 ++++++++++++++++++++++++++++ tests/data/test_entryexitanalysis.py | 6 ++--- 4 files changed, 42 insertions(+), 5 deletions(-) diff --git a/freqtrade/data/entryexitanalysis.py b/freqtrade/data/entryexitanalysis.py index 9d936d295..3981423f7 100644 --- a/freqtrade/data/entryexitanalysis.py +++ b/freqtrade/data/entryexitanalysis.py @@ -14,6 +14,7 @@ from freqtrade.data.btanalysis import ( load_backtest_stats, ) from freqtrade.exceptions import OperationalException +from freqtrade.util import print_df_rich_table logger = logging.getLogger(__name__) @@ -307,7 +308,7 @@ def _print_table( if name is not None: print(name) - print(tabulate(data, headers="keys", tablefmt="psql", showindex=show_index)) + print_df_rich_table(data, data.keys(), show_index=show_index) def process_entry_exit_reasons(config: Config): diff --git a/freqtrade/util/__init__.py b/freqtrade/util/__init__.py index f478829e6..5052e17dd 100644 --- a/freqtrade/util/__init__.py +++ b/freqtrade/util/__init__.py @@ -15,7 +15,7 @@ from freqtrade.util.formatters import decimals_per_coin, fmt_coin, round_value from freqtrade.util.ft_precise import FtPrecise from freqtrade.util.measure_time import MeasureTime from freqtrade.util.periodic_cache import PeriodicCache -from freqtrade.util.rich_tables import print_rich_table +from freqtrade.util.rich_tables import print_df_rich_table, print_rich_table from freqtrade.util.template_renderer import render_template, render_template_with_fallback # noqa @@ -38,4 +38,5 @@ __all__ = [ "fmt_coin", "MeasureTime", "print_rich_table", + "print_df_rich_table", ] diff --git a/freqtrade/util/rich_tables.py b/freqtrade/util/rich_tables.py index d34162d66..63715b70e 100644 --- a/freqtrade/util/rich_tables.py +++ b/freqtrade/util/rich_tables.py @@ -1,6 +1,7 @@ import sys from typing import Any, Dict, Optional, Sequence, Union +from pandas import DataFrame from rich.console import Console from rich.table import Table from rich.text import Text @@ -31,3 +32,37 @@ def print_rich_table( width=200 if "pytest" in sys.modules else None, ) console.print(table) + + +def _format_value(value: Any, *, floatfmt: str) -> str: + if isinstance(value, float): + return f"{value:{floatfmt}}" + return str(value) + + +def print_df_rich_table( + tabular_data: DataFrame, + headers: Sequence[str], + summary: Optional[str] = None, + *, + show_index=False, + index_name: Optional[str] = None, + table_kwargs: Optional[Dict[str, Any]] = None, +) -> None: + table = Table(title=summary, **(table_kwargs or {})) + + if show_index: + index_name = str(index_name) if index_name else tabular_data.index.name + table.add_column(index_name) + + for header in headers: + table.add_column(header, justify="right") + + for value_list in tabular_data.itertuples(index=show_index): + row = [_format_value(x, floatfmt=".3f") for x in value_list] + table.add_row(*row) + + console = Console( + width=200 if "pytest" in sys.modules else None, + ) + console.print(table) diff --git a/tests/data/test_entryexitanalysis.py b/tests/data/test_entryexitanalysis.py index 1a5309190..e7909c339 100644 --- a/tests/data/test_entryexitanalysis.py +++ b/tests/data/test_entryexitanalysis.py @@ -154,10 +154,10 @@ def test_backtest_analysis_nomock(default_conf, mocker, caplog, testdatadir, use assert "-3.5" in captured.out assert "50" in captured.out assert "0" in captured.out - assert "0.01616" in captured.out + assert "0.016" in captured.out assert "34.049" in captured.out - assert "0.104411" in captured.out - assert "52.8292" in captured.out + assert "0.104" in captured.out + assert "52.829" in captured.out # test group 1 args = get_args(base_args + ["--analysis-groups", "1"]) From e705471946d103bdc78495dc95538060fe41f69f Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 7 Jul 2024 09:43:44 +0200 Subject: [PATCH 14/38] chore: don't return what ain't being needed --- freqtrade/optimize/analysis/lookahead_helpers.py | 2 +- freqtrade/optimize/analysis/recursive_helpers.py | 4 ++-- tests/optimize/test_lookahead_analysis.py | 10 +++++----- tests/optimize/test_recursive_analysis.py | 8 ++------ 4 files changed, 10 insertions(+), 14 deletions(-) diff --git a/freqtrade/optimize/analysis/lookahead_helpers.py b/freqtrade/optimize/analysis/lookahead_helpers.py index c0e6fa1ba..b502716dc 100644 --- a/freqtrade/optimize/analysis/lookahead_helpers.py +++ b/freqtrade/optimize/analysis/lookahead_helpers.py @@ -64,7 +64,7 @@ class LookaheadAnalysisSubFunctions: table = tabulate(data, headers=headers, tablefmt="orgtbl") print(table) - return table, headers, data + return data @staticmethod def export_to_csv(config: Dict[str, Any], lookahead_analysis: List[LookaheadAnalysis]): diff --git a/freqtrade/optimize/analysis/recursive_helpers.py b/freqtrade/optimize/analysis/recursive_helpers.py index cde1a214e..90ac8ed1d 100644 --- a/freqtrade/optimize/analysis/recursive_helpers.py +++ b/freqtrade/optimize/analysis/recursive_helpers.py @@ -34,9 +34,9 @@ class RecursiveAnalysisSubFunctions: table = tabulate(data, headers=headers, tablefmt="orgtbl") print(table) - return table, headers, data + return data - return None, None, data + return data @staticmethod def calculate_config_overrides(config: Config): diff --git a/tests/optimize/test_lookahead_analysis.py b/tests/optimize/test_lookahead_analysis.py index 88e3ad877..f7d38b24b 100644 --- a/tests/optimize/test_lookahead_analysis.py +++ b/tests/optimize/test_lookahead_analysis.py @@ -147,7 +147,7 @@ def test_lookahead_helper_text_table_lookahead_analysis_instances(lookahead_conf instance = LookaheadAnalysis(lookahead_conf, strategy_obj) instance.current_analysis = analysis - _table, _headers, data = LookaheadAnalysisSubFunctions.text_table_lookahead_analysis_instances( + data = LookaheadAnalysisSubFunctions.text_table_lookahead_analysis_instances( lookahead_conf, [instance] ) @@ -163,14 +163,14 @@ def test_lookahead_helper_text_table_lookahead_analysis_instances(lookahead_conf analysis.false_exit_signals = 10 instance = LookaheadAnalysis(lookahead_conf, strategy_obj) instance.current_analysis = analysis - _table, _headers, data = LookaheadAnalysisSubFunctions.text_table_lookahead_analysis_instances( + data = LookaheadAnalysisSubFunctions.text_table_lookahead_analysis_instances( lookahead_conf, [instance] ) assert data[0][2].__contains__("error") # edit it into not showing an error instance.failed_bias_check = False - _table, _headers, data = LookaheadAnalysisSubFunctions.text_table_lookahead_analysis_instances( + data = LookaheadAnalysisSubFunctions.text_table_lookahead_analysis_instances( lookahead_conf, [instance] ) assert data[0][0] == "strategy_test_v3_with_lookahead_bias.py" @@ -183,7 +183,7 @@ def test_lookahead_helper_text_table_lookahead_analysis_instances(lookahead_conf analysis.false_indicators.append("falseIndicator1") analysis.false_indicators.append("falseIndicator2") - _table, _headers, data = LookaheadAnalysisSubFunctions.text_table_lookahead_analysis_instances( + data = LookaheadAnalysisSubFunctions.text_table_lookahead_analysis_instances( lookahead_conf, [instance] ) @@ -193,7 +193,7 @@ def test_lookahead_helper_text_table_lookahead_analysis_instances(lookahead_conf assert len(data) == 1 # check amount of multiple rows - _table, _headers, data = LookaheadAnalysisSubFunctions.text_table_lookahead_analysis_instances( + data = LookaheadAnalysisSubFunctions.text_table_lookahead_analysis_instances( lookahead_conf, [instance, instance, instance] ) assert len(data) == 3 diff --git a/tests/optimize/test_recursive_analysis.py b/tests/optimize/test_recursive_analysis.py index 2969b4153..e16c82d24 100644 --- a/tests/optimize/test_recursive_analysis.py +++ b/tests/optimize/test_recursive_analysis.py @@ -105,9 +105,7 @@ def test_recursive_helper_text_table_recursive_analysis_instances(recursive_conf instance = RecursiveAnalysis(recursive_conf, strategy_obj) instance.dict_recursive = dict_diff - _table, _headers, data = RecursiveAnalysisSubFunctions.text_table_recursive_analysis_instances( - [instance] - ) + data = RecursiveAnalysisSubFunctions.text_table_recursive_analysis_instances([instance]) # check row contents for a try that has too few signals assert data[0][0] == "rsi" @@ -118,9 +116,7 @@ def test_recursive_helper_text_table_recursive_analysis_instances(recursive_conf dict_diff = dict() instance = RecursiveAnalysis(recursive_conf, strategy_obj) instance.dict_recursive = dict_diff - _table, _headers, data = RecursiveAnalysisSubFunctions.text_table_recursive_analysis_instances( - [instance] - ) + data = RecursiveAnalysisSubFunctions.text_table_recursive_analysis_instances([instance]) assert len(data) == 0 From 5e88bd231dd0f5d347c2939e04e8ee1646a278ce Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 7 Jul 2024 09:56:49 +0200 Subject: [PATCH 15/38] feat: lookahead-heplpers -> rich table --- freqtrade/optimize/analysis/lookahead_helpers.py | 10 ++++++---- freqtrade/util/rich_tables.py | 2 +- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/freqtrade/optimize/analysis/lookahead_helpers.py b/freqtrade/optimize/analysis/lookahead_helpers.py index b502716dc..a8fb1cd35 100644 --- a/freqtrade/optimize/analysis/lookahead_helpers.py +++ b/freqtrade/optimize/analysis/lookahead_helpers.py @@ -4,11 +4,13 @@ from pathlib import Path from typing import Any, Dict, List import pandas as pd +from rich.text import Text from freqtrade.constants import Config from freqtrade.exceptions import OperationalException from freqtrade.optimize.analysis.lookahead import LookaheadAnalysis from freqtrade.resolvers import StrategyResolver +from freqtrade.util import print_rich_table logger = logging.getLogger(__name__) @@ -53,17 +55,17 @@ class LookaheadAnalysisSubFunctions: [ inst.strategy_obj["location"].parts[-1], inst.strategy_obj["name"], - inst.current_analysis.has_bias, + Text("Yes", style="bold red") + if inst.current_analysis.has_bias + else Text("No", style="bold green"), inst.current_analysis.total_signals, inst.current_analysis.false_entry_signals, inst.current_analysis.false_exit_signals, ", ".join(inst.current_analysis.false_indicators), ] ) - from tabulate import tabulate - table = tabulate(data, headers=headers, tablefmt="orgtbl") - print(table) + print_rich_table(data, headers, summary="Lookahead Analysis") return data @staticmethod diff --git a/freqtrade/util/rich_tables.py b/freqtrade/util/rich_tables.py index 63715b70e..926eba916 100644 --- a/freqtrade/util/rich_tables.py +++ b/freqtrade/util/rich_tables.py @@ -26,7 +26,7 @@ def print_rich_table( if isinstance(row, dict): table.add_row(*[str(row[header]) for header in headers]) else: - table.add_row(*row) + table.add_row(*[r if isinstance(r, Text) else str(r) for r in row]) console = Console( width=200 if "pytest" in sys.modules else None, From 8e2f28955ee04d74f85fdc2facefdbe119acfdf1 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 7 Jul 2024 10:05:17 +0200 Subject: [PATCH 16/38] feat: rich table for recursive analysis --- freqtrade/optimize/analysis/recursive_helpers.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/freqtrade/optimize/analysis/recursive_helpers.py b/freqtrade/optimize/analysis/recursive_helpers.py index 90ac8ed1d..be596fa68 100644 --- a/freqtrade/optimize/analysis/recursive_helpers.py +++ b/freqtrade/optimize/analysis/recursive_helpers.py @@ -7,6 +7,7 @@ from freqtrade.constants import Config from freqtrade.exceptions import OperationalException from freqtrade.optimize.analysis.recursive import RecursiveAnalysis from freqtrade.resolvers import StrategyResolver +from freqtrade.util import print_rich_table logger = logging.getLogger(__name__) @@ -16,9 +17,9 @@ class RecursiveAnalysisSubFunctions: @staticmethod def text_table_recursive_analysis_instances(recursive_instances: List[RecursiveAnalysis]): startups = recursive_instances[0]._startup_candle - headers = ["indicators"] + headers = ["Indicators"] for candle in startups: - headers.append(candle) + headers.append(str(candle)) data = [] for inst in recursive_instances: @@ -30,10 +31,8 @@ class RecursiveAnalysisSubFunctions: data.append(temp_data) if len(data) > 0: - from tabulate import tabulate + print_rich_table(data, headers, summary="Recursive Analysis") - table = tabulate(data, headers=headers, tablefmt="orgtbl") - print(table) return data return data From 7adc3c2ef57663158490a1bb2ef306cde689cd06 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 7 Jul 2024 12:47:27 +0200 Subject: [PATCH 17/38] Improve rich_tables generic --- freqtrade/util/rich_tables.py | 21 ++++++++++++++------- 1 file changed, 14 insertions(+), 7 deletions(-) diff --git a/freqtrade/util/rich_tables.py b/freqtrade/util/rich_tables.py index 926eba916..66e2d70dc 100644 --- a/freqtrade/util/rich_tables.py +++ b/freqtrade/util/rich_tables.py @@ -3,7 +3,7 @@ from typing import Any, Dict, Optional, Sequence, Union from pandas import DataFrame from rich.console import Console -from rich.table import Table +from rich.table import Column, Table from rich.text import Text @@ -17,16 +17,23 @@ def print_rich_table( *, table_kwargs: Optional[Dict[str, Any]] = None, ) -> None: - table = Table(title=summary, **(table_kwargs or {})) - - for header in headers: - table.add_column(header, justify="right") + table = Table( + *[c if isinstance(c, Column) else Column(c, justify="right") for c in headers], + title=summary, + **(table_kwargs or {}), + ) for row in tabular_data: if isinstance(row, dict): - table.add_row(*[str(row[header]) for header in headers]) + table.add_row( + *[ + row[header] if isinstance(row[header], (Text, Table)) else str(row[header]) + for header in headers + ] + ) + else: - table.add_row(*[r if isinstance(r, Text) else str(r) for r in row]) + table.add_row(*[r if isinstance(r, (Text, Table)) else str(r) for r in row]) console = Console( width=200 if "pytest" in sys.modules else None, From dba7f9968b7d219757abee6270e2e9c60f57d520 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 7 Jul 2024 12:58:44 +0200 Subject: [PATCH 18/38] chore: fix minor type gotcha --- freqtrade/optimize/hyperopt.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/freqtrade/optimize/hyperopt.py b/freqtrade/optimize/hyperopt.py index b19fca9dc..99b78d09f 100644 --- a/freqtrade/optimize/hyperopt.py +++ b/freqtrade/optimize/hyperopt.py @@ -260,7 +260,7 @@ class Hyperopt: result["max_open_trades"] = {"max_open_trades": strategy.max_open_trades} return result - def print_results(self, results) -> None: + def print_results(self, results: Dict[str, Any]) -> None: """ Log results if it is better than any previous evaluation TODO: this should be moved to HyperoptTools too @@ -271,7 +271,7 @@ class Hyperopt: print( HyperoptTools.get_result_table( self.config, - results, + [results], self.total_epochs, self.print_all, self.print_colorized, From 296bf9dc1d86ffb5accbc9fd7e85c67ac5f7b288 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 7 Jul 2024 13:05:37 +0200 Subject: [PATCH 19/38] chore: Fix unused import --- freqtrade/data/entryexitanalysis.py | 1 - 1 file changed, 1 deletion(-) diff --git a/freqtrade/data/entryexitanalysis.py b/freqtrade/data/entryexitanalysis.py index 3981423f7..e76f2dff7 100644 --- a/freqtrade/data/entryexitanalysis.py +++ b/freqtrade/data/entryexitanalysis.py @@ -4,7 +4,6 @@ from typing import List import joblib import pandas as pd -from tabulate import tabulate from freqtrade.configuration import TimeRange from freqtrade.constants import Config From 4d5b330b777b7b3c6d8fb7ec14fce9519bfb2ff4 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 7 Jul 2024 13:08:52 +0200 Subject: [PATCH 20/38] Improve rich generic --- freqtrade/util/rich_tables.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/freqtrade/util/rich_tables.py b/freqtrade/util/rich_tables.py index 66e2d70dc..23fb15d1f 100644 --- a/freqtrade/util/rich_tables.py +++ b/freqtrade/util/rich_tables.py @@ -27,13 +27,15 @@ def print_rich_table( if isinstance(row, dict): table.add_row( *[ - row[header] if isinstance(row[header], (Text, Table)) else str(row[header]) + row[header] if isinstance(row[header], Text) else str(row[header]) for header in headers ] ) else: - table.add_row(*[r if isinstance(r, (Text, Table)) else str(r) for r in row]) + table.add_row( + *[r if isinstance(r, Text) else str(r) for r in row], # type: ignore[arg-type] + ) console = Console( width=200 if "pytest" in sys.modules else None, From bc60855b93d2b66c75ad89ae3ac4b4d33700aea9 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 7 Jul 2024 13:18:50 +0200 Subject: [PATCH 21/38] chore: pre-commit now needs rich types --- .pre-commit-config.yaml | 1 + build_helpers/pre_commit_update.py | 4 +++- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 8fa4bf9ee..44134826c 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -20,6 +20,7 @@ repos: - types-tabulate==0.9.0.20240106 - types-python-dateutil==2.9.0.20240316 - SQLAlchemy==2.0.31 + - rich==13.7.1 # stages: [push] - repo: https://github.com/pycqa/isort diff --git a/build_helpers/pre_commit_update.py b/build_helpers/pre_commit_update.py index 9d313efd2..2e97e3ed0 100644 --- a/build_helpers/pre_commit_update.py +++ b/build_helpers/pre_commit_update.py @@ -18,7 +18,9 @@ with require.open("r") as rfile: # Extract types only type_reqs = [ - r.strip("\n") for r in requirements if r.startswith("types-") or r.startswith("SQLAlchemy") + r.strip("\n") + for r in requirements + if r.startswith("types-") or r.startswith("SQLAlchemy") or r.startswith("rich") ] with pre_commit_file.open("r") as file: From 8c807f00bbd0b3d161525fd5d30ecd494142a925 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 7 Jul 2024 15:10:14 +0200 Subject: [PATCH 22/38] Extract row before exploding for better typing --- freqtrade/util/rich_tables.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/freqtrade/util/rich_tables.py b/freqtrade/util/rich_tables.py index 23fb15d1f..d9762c9fb 100644 --- a/freqtrade/util/rich_tables.py +++ b/freqtrade/util/rich_tables.py @@ -1,5 +1,5 @@ import sys -from typing import Any, Dict, Optional, Sequence, Union +from typing import Any, Dict, List, Optional, Sequence, Union from pandas import DataFrame from rich.console import Console @@ -33,9 +33,8 @@ def print_rich_table( ) else: - table.add_row( - *[r if isinstance(r, Text) else str(r) for r in row], # type: ignore[arg-type] - ) + row_to_add: List[Union[str, Text]] = [r if isinstance(r, Text) else str(r) for r in row] + table.add_row(*row_to_add) console = Console( width=200 if "pytest" in sys.modules else None, From 483a829d0e750bf0a8a14dc9201f17df7267dce2 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 7 Jul 2024 15:10:33 +0200 Subject: [PATCH 23/38] Revert "chore: pre-commit now needs rich types" This reverts commit bc60855b93d2b66c75ad89ae3ac4b4d33700aea9. --- .pre-commit-config.yaml | 1 - build_helpers/pre_commit_update.py | 4 +--- 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 44134826c..8fa4bf9ee 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -20,7 +20,6 @@ repos: - types-tabulate==0.9.0.20240106 - types-python-dateutil==2.9.0.20240316 - SQLAlchemy==2.0.31 - - rich==13.7.1 # stages: [push] - repo: https://github.com/pycqa/isort diff --git a/build_helpers/pre_commit_update.py b/build_helpers/pre_commit_update.py index 2e97e3ed0..9d313efd2 100644 --- a/build_helpers/pre_commit_update.py +++ b/build_helpers/pre_commit_update.py @@ -18,9 +18,7 @@ with require.open("r") as rfile: # Extract types only type_reqs = [ - r.strip("\n") - for r in requirements - if r.startswith("types-") or r.startswith("SQLAlchemy") or r.startswith("rich") + r.strip("\n") for r in requirements if r.startswith("types-") or r.startswith("SQLAlchemy") ] with pre_commit_file.open("r") as file: From 69628736b287e9e96e9de92c051b9f0b87073c35 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 7 Jul 2024 15:29:58 +0200 Subject: [PATCH 24/38] feat: add hyperopt output functionality --- freqtrade/commands/hyperopt_commands.py | 10 +++ freqtrade/optimize/hyperopt_output.py | 107 ++++++++++++++++++++++++ 2 files changed, 117 insertions(+) create mode 100644 freqtrade/optimize/hyperopt_output.py diff --git a/freqtrade/commands/hyperopt_commands.py b/freqtrade/commands/hyperopt_commands.py index ac0b8453f..746fafe92 100644 --- a/freqtrade/commands/hyperopt_commands.py +++ b/freqtrade/commands/hyperopt_commands.py @@ -18,6 +18,7 @@ def start_hyperopt_list(args: Dict[str, Any]) -> None: """ List hyperopt epochs previously evaluated """ + from freqtrade.optimize.hyperopt_output import HyperoptOutput from freqtrade.optimize.hyperopt_tools import HyperoptTools config = setup_utils_configuration(args, RunMode.UTIL_NO_EXCHANGE) @@ -50,6 +51,15 @@ def start_hyperopt_list(args: Dict[str, Any]) -> None: 0, ) ) + hot = HyperoptOutput() + hot.add_data( + config, + epochs, + total_epochs, + not config.get("hyperopt_list_best", False), + ) + hot.print(print_colorized=print_colorized) + except KeyboardInterrupt: print("User interrupted..") diff --git a/freqtrade/optimize/hyperopt_output.py b/freqtrade/optimize/hyperopt_output.py new file mode 100644 index 000000000..f7cc217ad --- /dev/null +++ b/freqtrade/optimize/hyperopt_output.py @@ -0,0 +1,107 @@ +import sys +from typing import List, Optional, Union + +from rich.console import Console +from rich.table import Table +from rich.text import Text + +from freqtrade.constants import Config +from freqtrade.optimize.optimize_reports import generate_wins_draws_losses +from freqtrade.util import fmt_coin + + +class HyperoptOutput: + def __init__(self): + self._table = Table( + title="Hyperopt results", + ) + # Headers + self._table.add_column("Best", justify="left") + self._table.add_column("Epoch", justify="right") + self._table.add_column("Trades", justify="right") + self._table.add_column("Win Draw Loss Win%", justify="right") + self._table.add_column("Avg profit", justify="right") + self._table.add_column("Profit", justify="right") + self._table.add_column("Avg duration", justify="right") + self._table.add_column("Objective", justify="right") + self._table.add_column("Max Drawdown (Acct)", justify="right") + + def _add_row(self, data: List[Union[str, Text]]): + """Add single row""" + row_to_add: List[Union[str, Text]] = [r if isinstance(r, Text) else str(r) for r in data] + + self._table.add_row(*row_to_add) + + def _add_rows(self, data: List[List[Union[str, Text]]]): + """add multiple rows""" + for row in data: + self._add_row(row) + + def print(self, console: Optional[Console] = None, *, print_colorized=True): + if not console: + console = Console( + color_system="auto" if print_colorized else None, + width=200 if "pytest" in sys.modules else None, + ) + + console.print(self._table) + + def add_data( + self, + config: Config, + results: list, + total_epochs: int, + highlight_best: bool, + ) -> str: + """Format one or multiple rows and add them""" + stake_currency = config["stake_currency"] + + res = [ + [ + # "Best": + ( + ("*" if r["is_initial_point"] or r["is_random"] else "") + + (" Best" if r["is_best"] else "") + ).lstrip(), + # "Epoch": + f"{r['current_epoch']}/{total_epochs}", + # "Trades": + r["results_metrics"]["total_trades"], + # "Win Draw Loss Win%": + generate_wins_draws_losses( + r["results_metrics"]["wins"], + r["results_metrics"]["draws"], + r["results_metrics"]["losses"], + ), + # "Avg profit": + f"{r['results_metrics']['profit_mean']:.2%}", + # "Profit": + "{} {}".format( + fmt_coin( + r["results_metrics"]["profit_total_abs"], + stake_currency, + keep_trailing_zeros=True, + ), + f"({r['results_metrics']['profit_total']:,.2%})".rjust(10, " "), + ) + if r["results_metrics"]["profit_total_abs"] != 0.0 + else "--", + # "Avg duration": + r["results_metrics"]["holding_avg"], + # "Objective": + f"{r["loss"]:,.5f}" if r["loss"] != 100000 else "N/A", + # "Max Drawdown (Acct)": + "{} {}".format( + fmt_coin( + r["results_metrics"]["max_drawdown_abs"], + stake_currency, + keep_trailing_zeros=True, + ), + (f"({r["results_metrics"]['max_drawdown_account']:,.2%})").rjust(10, " "), + ) + if r["results_metrics"]["max_drawdown_account"] != 0.0 + else "--", + ] + for r in results + ] + self._add_rows(res) From befefd449c5dbb93e39a6dacbbe78cc6f876202f Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 7 Jul 2024 16:13:55 +0200 Subject: [PATCH 25/38] Add a generic Progressbar which allows printing additional objects --- freqtrade/util/__init__.py | 2 ++ freqtrade/util/rich_progress.py | 12 ++++++++++++ 2 files changed, 14 insertions(+) create mode 100644 freqtrade/util/rich_progress.py diff --git a/freqtrade/util/__init__.py b/freqtrade/util/__init__.py index 5052e17dd..76902b176 100644 --- a/freqtrade/util/__init__.py +++ b/freqtrade/util/__init__.py @@ -15,6 +15,7 @@ from freqtrade.util.formatters import decimals_per_coin, fmt_coin, round_value from freqtrade.util.ft_precise import FtPrecise from freqtrade.util.measure_time import MeasureTime from freqtrade.util.periodic_cache import PeriodicCache +from freqtrade.util.rich_progress import CustomProgress from freqtrade.util.rich_tables import print_df_rich_table, print_rich_table from freqtrade.util.template_renderer import render_template, render_template_with_fallback # noqa @@ -39,4 +40,5 @@ __all__ = [ "MeasureTime", "print_rich_table", "print_df_rich_table", + "CustomProgress", ] diff --git a/freqtrade/util/rich_progress.py b/freqtrade/util/rich_progress.py new file mode 100644 index 000000000..6cf138bf5 --- /dev/null +++ b/freqtrade/util/rich_progress.py @@ -0,0 +1,12 @@ +from rich.console import ConsoleRenderable, Group, RichCast +from rich.progress import Progress + + +class CustomProgress(Progress): + def __init__(self, *args, cust_objs, **kwargs) -> None: + self._cust_objs = cust_objs + super().__init__(*args, **kwargs) + + def get_renderable(self) -> ConsoleRenderable | RichCast | str: + renderable = Group(*self._cust_objs, *self.get_renderables()) + return renderable From f05c019757d6647c4a6a1f2467ff664009830a1d Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 7 Jul 2024 16:15:09 +0200 Subject: [PATCH 26/38] feat: Update output for hyperopt to Rich --- freqtrade/optimize/hyperopt.py | 24 ++++++++++++------------ freqtrade/optimize/hyperopt_output.py | 26 +++++++++++++------------- 2 files changed, 25 insertions(+), 25 deletions(-) diff --git a/freqtrade/optimize/hyperopt.py b/freqtrade/optimize/hyperopt.py index 99b78d09f..8d2f8846a 100644 --- a/freqtrade/optimize/hyperopt.py +++ b/freqtrade/optimize/hyperopt.py @@ -18,10 +18,10 @@ from colorama import init as colorama_init from joblib import Parallel, cpu_count, delayed, dump, load, wrap_non_picklable_objects from joblib.externals import cloudpickle from pandas import DataFrame +from rich.align import Align from rich.progress import ( BarColumn, MofNCompleteColumn, - Progress, TaskProgressColumn, TextColumn, TimeElapsedColumn, @@ -40,6 +40,7 @@ from freqtrade.optimize.backtesting import Backtesting # Import IHyperOpt and IHyperOptLoss to allow unpickling classes from these modules from freqtrade.optimize.hyperopt_auto import HyperOptAuto from freqtrade.optimize.hyperopt_loss_interface import IHyperOptLoss +from freqtrade.optimize.hyperopt_output import HyperoptOutput from freqtrade.optimize.hyperopt_tools import ( HyperoptStateContainer, HyperoptTools, @@ -47,6 +48,7 @@ from freqtrade.optimize.hyperopt_tools import ( ) from freqtrade.optimize.optimize_reports import generate_strategy_stats from freqtrade.resolvers.hyperopt_resolver import HyperOptLossResolver +from freqtrade.util import CustomProgress # Suppress scikit-learn FutureWarnings from skopt @@ -86,6 +88,8 @@ class Hyperopt: self.max_open_trades_space: List[Dimension] = [] self.dimensions: List[Dimension] = [] + self._hyper_out: HyperoptOutput = HyperoptOutput() + self.config = config self.min_date: datetime self.max_date: datetime @@ -268,17 +272,12 @@ class Hyperopt: is_best = results["is_best"] if self.print_all or is_best: - print( - HyperoptTools.get_result_table( - self.config, - [results], - self.total_epochs, - self.print_all, - self.print_colorized, - self.hyperopt_table_header, - ) + self._hyper_out.add_data( + self.config, + [results], + self.total_epochs, + self.print_all, ) - self.hyperopt_table_header = 2 def init_spaces(self): """ @@ -635,7 +634,7 @@ class Hyperopt: logger.info(f"Effective number of parallel workers used: {jobs}") # Define progressbar - with Progress( + with CustomProgress( TextColumn("[progress.description]{task.description}"), BarColumn(bar_width=None), MofNCompleteColumn(), @@ -645,6 +644,7 @@ class Hyperopt: "•", TimeRemainingColumn(), expand=True, + cust_objs=[Align.center(self._hyper_out.table)], ) as pbar: task = pbar.add_task("Epochs", total=self.total_epochs) diff --git a/freqtrade/optimize/hyperopt_output.py b/freqtrade/optimize/hyperopt_output.py index f7cc217ad..2f4e9aff8 100644 --- a/freqtrade/optimize/hyperopt_output.py +++ b/freqtrade/optimize/hyperopt_output.py @@ -12,25 +12,25 @@ from freqtrade.util import fmt_coin class HyperoptOutput: def __init__(self): - self._table = Table( + self.table = Table( title="Hyperopt results", ) # Headers - self._table.add_column("Best", justify="left") - self._table.add_column("Epoch", justify="right") - self._table.add_column("Trades", justify="right") - self._table.add_column("Win Draw Loss Win%", justify="right") - self._table.add_column("Avg profit", justify="right") - self._table.add_column("Profit", justify="right") - self._table.add_column("Avg duration", justify="right") - self._table.add_column("Objective", justify="right") - self._table.add_column("Max Drawdown (Acct)", justify="right") + self.table.add_column("Best", justify="left") + self.table.add_column("Epoch", justify="right") + self.table.add_column("Trades", justify="right") + self.table.add_column("Win Draw Loss Win%", justify="right") + self.table.add_column("Avg profit", justify="right") + self.table.add_column("Profit", justify="right") + self.table.add_column("Avg duration", justify="right") + self.table.add_column("Objective", justify="right") + self.table.add_column("Max Drawdown (Acct)", justify="right") def _add_row(self, data: List[Union[str, Text]]): """Add single row""" row_to_add: List[Union[str, Text]] = [r if isinstance(r, Text) else str(r) for r in data] - self._table.add_row(*row_to_add) + self.table.add_row(*row_to_add) def _add_rows(self, data: List[List[Union[str, Text]]]): """add multiple rows""" @@ -44,7 +44,7 @@ class HyperoptOutput: width=200 if "pytest" in sys.modules else None, ) - console.print(self._table) + console.print(self.table) def add_data( self, @@ -52,7 +52,7 @@ class HyperoptOutput: results: list, total_epochs: int, highlight_best: bool, - ) -> str: + ) -> None: """Format one or multiple rows and add them""" stake_currency = config["stake_currency"] From 8f0ac0aaeae1db678f6c69136c4c375cbab2a0be Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 7 Jul 2024 16:27:53 +0200 Subject: [PATCH 27/38] Remove old output from hyperopt-list --- freqtrade/commands/hyperopt_commands.py | 16 +++------------- 1 file changed, 3 insertions(+), 13 deletions(-) diff --git a/freqtrade/commands/hyperopt_commands.py b/freqtrade/commands/hyperopt_commands.py index 746fafe92..e6c264051 100644 --- a/freqtrade/commands/hyperopt_commands.py +++ b/freqtrade/commands/hyperopt_commands.py @@ -41,24 +41,14 @@ def start_hyperopt_list(args: Dict[str, Any]) -> None: if not export_csv: try: - print( - HyperoptTools.get_result_table( - config, - epochs, - total_epochs, - not config.get("hyperopt_list_best", False), - print_colorized, - 0, - ) - ) - hot = HyperoptOutput() - hot.add_data( + h_out = HyperoptOutput() + h_out.add_data( config, epochs, total_epochs, not config.get("hyperopt_list_best", False), ) - hot.print(print_colorized=print_colorized) + h_out.print(print_colorized=print_colorized) except KeyboardInterrupt: print("User interrupted..") From 4d6f399131b72cca6bf36102fa35d2b6e672879d Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 7 Jul 2024 16:28:02 +0200 Subject: [PATCH 28/38] Implement colors for hyperopt-output --- freqtrade/optimize/hyperopt_output.py | 104 ++++++++++++++------------ 1 file changed, 56 insertions(+), 48 deletions(-) diff --git a/freqtrade/optimize/hyperopt_output.py b/freqtrade/optimize/hyperopt_output.py index 2f4e9aff8..145ac758e 100644 --- a/freqtrade/optimize/hyperopt_output.py +++ b/freqtrade/optimize/hyperopt_output.py @@ -56,52 +56,60 @@ class HyperoptOutput: """Format one or multiple rows and add them""" stake_currency = config["stake_currency"] - res = [ - [ - # "Best": - ( - ("*" if r["is_initial_point"] or r["is_random"] else "") - + (" Best" if r["is_best"] else "") - ).lstrip(), - # "Epoch": - f"{r['current_epoch']}/{total_epochs}", - # "Trades": - r["results_metrics"]["total_trades"], - # "Win Draw Loss Win%": - generate_wins_draws_losses( - r["results_metrics"]["wins"], - r["results_metrics"]["draws"], - r["results_metrics"]["losses"], + for r in results: + self.table.add_row( + *[ + # "Best": + ( + ("*" if r["is_initial_point"] or r["is_random"] else "") + + (" Best" if r["is_best"] else "") + ).lstrip(), + # "Epoch": + f"{r['current_epoch']}/{total_epochs}", + # "Trades": + str(r["results_metrics"]["total_trades"]), + # "Win Draw Loss Win%": + generate_wins_draws_losses( + r["results_metrics"]["wins"], + r["results_metrics"]["draws"], + r["results_metrics"]["losses"], + ), + # "Avg profit": + f"{r['results_metrics']['profit_mean']:.2%}", + # "Profit": + Text( + "{} {}".format( + fmt_coin( + r["results_metrics"]["profit_total_abs"], + stake_currency, + keep_trailing_zeros=True, + ), + f"({r['results_metrics']['profit_total']:,.2%})".rjust(10, " "), + ) + if r["results_metrics"]["profit_total_abs"] != 0.0 + else "--", + style="green" if r["results_metrics"]["profit_total_abs"] > 0 else "red", + ), + # "Avg duration": + r["results_metrics"]["holding_avg"], + # "Objective": + f"{r["loss"]:,.5f}" if r["loss"] != 100000 else "N/A", + # "Max Drawdown (Acct)": + "{} {}".format( + fmt_coin( + r["results_metrics"]["max_drawdown_abs"], + stake_currency, + keep_trailing_zeros=True, + ), + (f"({r["results_metrics"]['max_drawdown_account']:,.2%})").rjust(10, " "), + ) + if r["results_metrics"]["max_drawdown_account"] != 0.0 + else "--", + ], + style=" ".join( + [ + "bold " if r["is_best"] and highlight_best else "", + "italic " if r["is_initial_point"] else "", + ] ), - # "Avg profit": - f"{r['results_metrics']['profit_mean']:.2%}", - # "Profit": - "{} {}".format( - fmt_coin( - r["results_metrics"]["profit_total_abs"], - stake_currency, - keep_trailing_zeros=True, - ), - f"({r['results_metrics']['profit_total']:,.2%})".rjust(10, " "), - ) - if r["results_metrics"]["profit_total_abs"] != 0.0 - else "--", - # "Avg duration": - r["results_metrics"]["holding_avg"], - # "Objective": - f"{r["loss"]:,.5f}" if r["loss"] != 100000 else "N/A", - # "Max Drawdown (Acct)": - "{} {}".format( - fmt_coin( - r["results_metrics"]["max_drawdown_abs"], - stake_currency, - keep_trailing_zeros=True, - ), - (f"({r["results_metrics"]['max_drawdown_account']:,.2%})").rjust(10, " "), - ) - if r["results_metrics"]["max_drawdown_account"] != 0.0 - else "--", - ] - for r in results - ] - self._add_rows(res) + ) From f51b63fc37e7b41331e5fb353e46ef5addde8c3b Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 7 Jul 2024 16:33:47 +0200 Subject: [PATCH 29/38] Fix wrong type for live running --- freqtrade/optimize/hyperopt_output.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/optimize/hyperopt_output.py b/freqtrade/optimize/hyperopt_output.py index 145ac758e..cd89cf035 100644 --- a/freqtrade/optimize/hyperopt_output.py +++ b/freqtrade/optimize/hyperopt_output.py @@ -91,7 +91,7 @@ class HyperoptOutput: style="green" if r["results_metrics"]["profit_total_abs"] > 0 else "red", ), # "Avg duration": - r["results_metrics"]["holding_avg"], + str(r["results_metrics"]["holding_avg"]), # "Objective": f"{r["loss"]:,.5f}" if r["loss"] != 100000 else "N/A", # "Max Drawdown (Acct)": From 879797e7c5014e61411c5c53c7fcfac471b7d2fd Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 7 Jul 2024 16:58:46 +0200 Subject: [PATCH 30/38] chore: remove no longer used result formatting methods --- freqtrade/optimize/hyperopt_tools.py | 174 --------------------------- 1 file changed, 174 deletions(-) diff --git a/freqtrade/optimize/hyperopt_tools.py b/freqtrade/optimize/hyperopt_tools.py index 50c55c43d..975338cd5 100644 --- a/freqtrade/optimize/hyperopt_tools.py +++ b/freqtrade/optimize/hyperopt_tools.py @@ -5,10 +5,7 @@ from pathlib import Path from typing import Any, Dict, Iterator, List, Optional, Tuple import numpy as np -import pandas as pd import rapidjson -import tabulate -from colorama import Fore, Style from pandas import isna, json_normalize from freqtrade.constants import FTHYPT_FILEVERSION, Config @@ -16,8 +13,6 @@ from freqtrade.enums import HyperoptState from freqtrade.exceptions import OperationalException from freqtrade.misc import deep_merge_dicts, round_dict, safe_value_fallback2 from freqtrade.optimize.hyperopt_epoch_filters import hyperopt_filter_epochs -from freqtrade.optimize.optimize_reports import generate_wins_draws_losses -from freqtrade.util import fmt_coin logger = logging.getLogger(__name__) @@ -357,175 +352,6 @@ class HyperoptTools: + f"Objective: {results['loss']:.5f}" ) - @staticmethod - def prepare_trials_columns(trials: pd.DataFrame) -> pd.DataFrame: - trials["Best"] = "" - - if "results_metrics.winsdrawslosses" not in trials.columns: - # Ensure compatibility with older versions of hyperopt results - trials["results_metrics.winsdrawslosses"] = "N/A" - - has_account_drawdown = "results_metrics.max_drawdown_account" in trials.columns - if not has_account_drawdown: - # Ensure compatibility with older versions of hyperopt results - trials["results_metrics.max_drawdown_account"] = None - if "is_random" not in trials.columns: - trials["is_random"] = False - - # New mode, using backtest result for metrics - trials["results_metrics.winsdrawslosses"] = trials.apply( - lambda x: generate_wins_draws_losses( - x["results_metrics.wins"], x["results_metrics.draws"], x["results_metrics.losses"] - ), - axis=1, - ) - - trials = trials[ - [ - "Best", - "current_epoch", - "results_metrics.total_trades", - "results_metrics.winsdrawslosses", - "results_metrics.profit_mean", - "results_metrics.profit_total_abs", - "results_metrics.profit_total", - "results_metrics.holding_avg", - "results_metrics.max_drawdown_account", - "results_metrics.max_drawdown_abs", - "loss", - "is_initial_point", - "is_random", - "is_best", - ] - ] - - trials.columns = [ - "Best", - "Epoch", - "Trades", - " Win Draw Loss Win%", - "Avg profit", - "Total profit", - "Profit", - "Avg duration", - "max_drawdown_account", - "max_drawdown_abs", - "Objective", - "is_initial_point", - "is_random", - "is_best", - ] - - return trials - - @staticmethod - def get_result_table( - config: Config, - results: list, - total_epochs: int, - highlight_best: bool, - print_colorized: bool, - remove_header: int, - ) -> str: - """ - Log result table - """ - if not results: - return "" - - tabulate.PRESERVE_WHITESPACE = True - trials = json_normalize(results, max_level=1) - - trials = HyperoptTools.prepare_trials_columns(trials) - - trials["is_profit"] = False - trials.loc[trials["is_initial_point"] | trials["is_random"], "Best"] = "* " - trials.loc[trials["is_best"], "Best"] = "Best" - trials.loc[ - (trials["is_initial_point"] | trials["is_random"]) & trials["is_best"], "Best" - ] = "* Best" - trials.loc[trials["Total profit"] > 0, "is_profit"] = True - trials["Trades"] = trials["Trades"].astype(str) - # perc_multi = 1 if legacy_mode else 100 - trials["Epoch"] = trials["Epoch"].apply( - lambda x: "{}/{}".format(str(x).rjust(len(str(total_epochs)), " "), total_epochs) - ) - trials["Avg profit"] = trials["Avg profit"].apply( - lambda x: f"{x:,.2%}".rjust(7, " ") if not isna(x) else "--".rjust(7, " ") - ) - trials["Avg duration"] = trials["Avg duration"].apply( - lambda x: ( - f"{x:,.1f} m".rjust(7, " ") - if isinstance(x, float) - else f"{x}" - if not isna(x) - else "--".rjust(7, " ") - ) - ) - trials["Objective"] = trials["Objective"].apply( - lambda x: f"{x:,.5f}".rjust(8, " ") if x != 100000 else "N/A".rjust(8, " ") - ) - - stake_currency = config["stake_currency"] - - trials["Max Drawdown (Acct)"] = trials.apply( - lambda x: ( - "{} {}".format( - fmt_coin(x["max_drawdown_abs"], stake_currency, keep_trailing_zeros=True), - (f"({x['max_drawdown_account']:,.2%})").rjust(10, " "), - ).rjust(25 + len(stake_currency)) - if x["max_drawdown_account"] != 0.0 - else "--".rjust(25 + len(stake_currency)) - ), - axis=1, - ) - - trials = trials.drop(columns=["max_drawdown_abs", "max_drawdown_account"]) - - trials["Profit"] = trials.apply( - lambda x: ( - "{} {}".format( - fmt_coin(x["Total profit"], stake_currency, keep_trailing_zeros=True), - f"({x['Profit']:,.2%})".rjust(10, " "), - ).rjust(25 + len(stake_currency)) - if x["Total profit"] != 0.0 - else "--".rjust(25 + len(stake_currency)) - ), - axis=1, - ) - trials = trials.drop(columns=["Total profit"]) - - if print_colorized: - trials2 = trials.astype(str) - for i in range(len(trials)): - if trials.loc[i]["is_profit"]: - for j in range(len(trials.loc[i]) - 3): - trials2.iat[i, j] = f"{Fore.GREEN}{str(trials.iloc[i, j])}{Fore.RESET}" - if trials.loc[i]["is_best"] and highlight_best: - for j in range(len(trials.loc[i]) - 3): - trials2.iat[i, j] = ( - f"{Style.BRIGHT}{str(trials.iloc[i, j])}{Style.RESET_ALL}" - ) - trials = trials2 - del trials2 - trials = trials.drop(columns=["is_initial_point", "is_best", "is_profit", "is_random"]) - if remove_header > 0: - table = tabulate.tabulate( - trials.to_dict(orient="list"), tablefmt="orgtbl", headers="keys", stralign="right" - ) - - table = table.split("\n", remove_header)[remove_header] - elif remove_header < 0: - table = tabulate.tabulate( - trials.to_dict(orient="list"), tablefmt="psql", headers="keys", stralign="right" - ) - table = "\n".join(table.split("\n")[0:remove_header]) - else: - table = tabulate.tabulate( - trials.to_dict(orient="list"), tablefmt="psql", headers="keys", stralign="right" - ) - return table - @staticmethod def export_csv_file(config: Config, results: list, csv_file: str) -> None: """ From 62320a361ebc3ea47b2d7e118facdd134aad1928 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 7 Jul 2024 17:04:22 +0200 Subject: [PATCH 31/38] chore: fix now failing test --- tests/optimize/test_hyperopt.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/optimize/test_hyperopt.py b/tests/optimize/test_hyperopt.py index 2e9ae8d35..a9f697629 100644 --- a/tests/optimize/test_hyperopt.py +++ b/tests/optimize/test_hyperopt.py @@ -291,9 +291,10 @@ def test_log_results_if_loss_improves(hyperopt, capsys) -> None: "is_best": True, } ) + hyperopt._hyper_out.print() out, _err = capsys.readouterr() assert all( - x in out for x in ["Best", "2/2", " 1", "0.10%", "0.00100000 BTC (1.00%)", "00:20:00"] + x in out for x in ["Best", "2/2", "1", "0.10%", "0.00100000 BTC (1.00%)", "0:20:00"] ) From 004e1101e7cbfba3c8493d6f08527591a49b7dd0 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 7 Jul 2024 17:11:54 +0200 Subject: [PATCH 32/38] Improve resiliance, drop compatibility test --- freqtrade/optimize/hyperopt_output.py | 10 +++++++--- tests/conftest_hyperopt.py | 3 ++- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/freqtrade/optimize/hyperopt_output.py b/freqtrade/optimize/hyperopt_output.py index cd89cf035..81a7ec6bd 100644 --- a/freqtrade/optimize/hyperopt_output.py +++ b/freqtrade/optimize/hyperopt_output.py @@ -75,7 +75,9 @@ class HyperoptOutput: r["results_metrics"]["losses"], ), # "Avg profit": - f"{r['results_metrics']['profit_mean']:.2%}", + f"{r['results_metrics']['profit_mean']:.2%}" + if r["results_metrics"]["profit_mean"] is not None + else "--", # "Profit": Text( "{} {}".format( @@ -86,9 +88,11 @@ class HyperoptOutput: ), f"({r['results_metrics']['profit_total']:,.2%})".rjust(10, " "), ) - if r["results_metrics"]["profit_total_abs"] != 0.0 + if r["results_metrics"].get("profit_total_abs", 0) != 0.0 else "--", - style="green" if r["results_metrics"]["profit_total_abs"] > 0 else "red", + style="green" + if r["results_metrics"].get("profit_total_abs", 0) > 0 + else "red", ), # "Avg duration": str(r["results_metrics"]["holding_avg"]), diff --git a/tests/conftest_hyperopt.py b/tests/conftest_hyperopt.py index af4039a3c..315b138cf 100644 --- a/tests/conftest_hyperopt.py +++ b/tests/conftest_hyperopt.py @@ -324,7 +324,8 @@ def hyperopt_test_result(): "profit_mean": None, "profit_median": None, "profit_total": 0, - "profit": 0.0, + "max_drawdown_account": 0.0, + "max_drawdown_abs": 0.0, "holding_avg": timedelta(), }, # noqa: E501 "results_explanation": " 0 trades. Avg profit nan%. Total profit 0.00000000 BTC ( 0.00Σ%). Avg duration nan min.", # noqa: E501 From 94565d0d39808f023e63f47b6b69eb902a43af42 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 7 Jul 2024 20:03:49 +0200 Subject: [PATCH 33/38] "best" should be shown in gold --- freqtrade/optimize/hyperopt_output.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/freqtrade/optimize/hyperopt_output.py b/freqtrade/optimize/hyperopt_output.py index 81a7ec6bd..9e0bbbb5b 100644 --- a/freqtrade/optimize/hyperopt_output.py +++ b/freqtrade/optimize/hyperopt_output.py @@ -90,9 +90,13 @@ class HyperoptOutput: ) if r["results_metrics"].get("profit_total_abs", 0) != 0.0 else "--", - style="green" - if r["results_metrics"].get("profit_total_abs", 0) > 0 - else "red", + style=( + "green" + if r["results_metrics"].get("profit_total_abs", 0) > 0 + else "red" + ) + if not r["is_best"] + else "", ), # "Avg duration": str(r["results_metrics"]["holding_avg"]), @@ -112,7 +116,7 @@ class HyperoptOutput: ], style=" ".join( [ - "bold " if r["is_best"] and highlight_best else "", + "bold gold1" if r["is_best"] and highlight_best else "", "italic " if r["is_initial_point"] else "", ] ), From 28f4e1c06898daba2b1c907343f84d96c9c2cc6c Mon Sep 17 00:00:00 2001 From: Matthias Date: Mon, 8 Jul 2024 06:38:29 +0200 Subject: [PATCH 34/38] Colorama is not necessary for hyperopt commands anymore --- freqtrade/commands/hyperopt_commands.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/freqtrade/commands/hyperopt_commands.py b/freqtrade/commands/hyperopt_commands.py index e6c264051..d89d25796 100644 --- a/freqtrade/commands/hyperopt_commands.py +++ b/freqtrade/commands/hyperopt_commands.py @@ -2,8 +2,6 @@ import logging from operator import itemgetter from typing import Any, Dict -from colorama import init as colorama_init - from freqtrade.configuration import setup_utils_configuration from freqtrade.data.btanalysis import get_latest_hyperopt_file from freqtrade.enums import RunMode @@ -36,9 +34,6 @@ def start_hyperopt_list(args: Dict[str, Any]) -> None: # Previous evaluations epochs, total_epochs = HyperoptTools.load_filtered_results(results_file, config) - if print_colorized: - colorama_init(autoreset=True) - if not export_csv: try: h_out = HyperoptOutput() From 49a60fa67f6be852eee390bbc7aa40bc6db6763d Mon Sep 17 00:00:00 2001 From: Matthias Date: Mon, 8 Jul 2024 06:44:21 +0200 Subject: [PATCH 35/38] Properly support "--no-color" for hyperopt --- freqtrade/optimize/hyperopt.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/freqtrade/optimize/hyperopt.py b/freqtrade/optimize/hyperopt.py index 8d2f8846a..b411e7752 100644 --- a/freqtrade/optimize/hyperopt.py +++ b/freqtrade/optimize/hyperopt.py @@ -14,11 +14,11 @@ from pathlib import Path from typing import Any, Dict, List, Optional, Tuple import rapidjson -from colorama import init as colorama_init from joblib import Parallel, cpu_count, delayed, dump, load, wrap_non_picklable_objects from joblib.externals import cloudpickle from pandas import DataFrame from rich.align import Align +from rich.console import Console from rich.progress import ( BarColumn, MofNCompleteColumn, @@ -625,13 +625,13 @@ class Hyperopt: self.opt = self.get_optimizer(self.dimensions, config_jobs) - if self.print_colorized: - colorama_init(autoreset=True) - try: with Parallel(n_jobs=config_jobs) as parallel: jobs = parallel._effective_n_jobs() logger.info(f"Effective number of parallel workers used: {jobs}") + console = Console( + color_system="auto" if self.print_colorized else None, + ) # Define progressbar with CustomProgress( @@ -644,6 +644,7 @@ class Hyperopt: "•", TimeRemainingColumn(), expand=True, + console=console, cust_objs=[Align.center(self._hyper_out.table)], ) as pbar: task = pbar.add_task("Epochs", total=self.total_epochs) From b208f978dbf8ebc00757fdfb73bc10be6fadafc6 Mon Sep 17 00:00:00 2001 From: Matthias Date: Mon, 8 Jul 2024 06:44:52 +0200 Subject: [PATCH 36/38] Remove dependency on colorama --- requirements.txt | 2 -- setup.py | 1 - 2 files changed, 3 deletions(-) diff --git a/requirements.txt b/requirements.txt index 170e5d805..791e7fe08 100644 --- a/requirements.txt +++ b/requirements.txt @@ -45,8 +45,6 @@ pyjwt==2.8.0 aiofiles==24.1.0 psutil==6.0.0 -# Support for colorized terminal output -colorama==0.4.6 # Building config files interactively questionary==2.0.1 prompt-toolkit==3.0.36 diff --git a/setup.py b/setup.py index 82e529767..6963862e0 100644 --- a/setup.py +++ b/setup.py @@ -88,7 +88,6 @@ setup( "py_find_1st", "python-rapidjson", "orjson", - "colorama", "jinja2", "questionary", "prompt-toolkit", From 64d22bbd89a4590847c075111d2f8d3218446412 Mon Sep 17 00:00:00 2001 From: Matthias Date: Mon, 8 Jul 2024 18:20:39 +0200 Subject: [PATCH 37/38] chore: Fix fstring incompatibility with older python versions --- freqtrade/optimize/hyperopt_output.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/freqtrade/optimize/hyperopt_output.py b/freqtrade/optimize/hyperopt_output.py index 9e0bbbb5b..72e049745 100644 --- a/freqtrade/optimize/hyperopt_output.py +++ b/freqtrade/optimize/hyperopt_output.py @@ -101,7 +101,7 @@ class HyperoptOutput: # "Avg duration": str(r["results_metrics"]["holding_avg"]), # "Objective": - f"{r["loss"]:,.5f}" if r["loss"] != 100000 else "N/A", + f"{r['loss']:,.5f}" if r["loss"] != 100000 else "N/A", # "Max Drawdown (Acct)": "{} {}".format( fmt_coin( @@ -109,7 +109,7 @@ class HyperoptOutput: stake_currency, keep_trailing_zeros=True, ), - (f"({r["results_metrics"]['max_drawdown_account']:,.2%})").rjust(10, " "), + (f"({r['results_metrics']['max_drawdown_account']:,.2%})").rjust(10, " "), ) if r["results_metrics"]["max_drawdown_account"] != 0.0 else "--", From 0e870ab47c27e053f785ae9dfbb167b988e96102 Mon Sep 17 00:00:00 2001 From: Matthias Date: Mon, 8 Jul 2024 19:15:05 +0200 Subject: [PATCH 38/38] chore: don't use pipe operator, not supported on 3.9 --- freqtrade/util/rich_progress.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/freqtrade/util/rich_progress.py b/freqtrade/util/rich_progress.py index 6cf138bf5..d295dafd5 100644 --- a/freqtrade/util/rich_progress.py +++ b/freqtrade/util/rich_progress.py @@ -1,3 +1,5 @@ +from typing import Union + from rich.console import ConsoleRenderable, Group, RichCast from rich.progress import Progress @@ -7,6 +9,6 @@ class CustomProgress(Progress): self._cust_objs = cust_objs super().__init__(*args, **kwargs) - def get_renderable(self) -> ConsoleRenderable | RichCast | str: + def get_renderable(self) -> Union[ConsoleRenderable, RichCast, str]: renderable = Group(*self._cust_objs, *self.get_renderables()) return renderable