diff --git a/freqtrade/data/history/history_utils.py b/freqtrade/data/history/history_utils.py index d13fa6fca..f71fcc874 100644 --- a/freqtrade/data/history/history_utils.py +++ b/freqtrade/data/history/history_utils.py @@ -26,8 +26,7 @@ from freqtrade.enums import CandleType, TradingMode from freqtrade.exceptions import OperationalException from freqtrade.exchange import Exchange from freqtrade.plugins.pairlist.pairlist_helpers import dynamic_expand_pairlist -from freqtrade.util import dt_ts, format_ms_time -from freqtrade.util.datetime_helpers import dt_now +from freqtrade.util import dt_now, dt_ts, format_ms_time, get_progress_tracker from freqtrade.util.migrations import migrate_data @@ -155,11 +154,9 @@ def refresh_data( :param candle_type: Any of the enum CandleType (must match trading mode!) """ data_handler = get_datahandler(datadir, data_format) - for idx, pair in enumerate(pairs): - process = f"{idx}/{len(pairs)}" + for pair in pairs: _download_pair_history( pair=pair, - process=process, timeframe=timeframe, datadir=datadir, timerange=timerange, @@ -223,7 +220,6 @@ def _download_pair_history( datadir: Path, exchange: Exchange, timeframe: str = "5m", - process: str = "", new_pairs_days: int = 30, data_handler: Optional[IDataHandler] = None, timerange: Optional[TimeRange] = None, @@ -261,7 +257,7 @@ def _download_pair_history( ) logger.info( - f'({process}) - Download history data for "{pair}", {timeframe}, ' + f'Download history data for "{pair}", {timeframe}, ' f"{candle_type} and store in {datadir}. " f'From {format_ms_time(since_ms) if since_ms else "start"} to ' f'{format_ms_time(until_ms) if until_ms else "now"}' @@ -345,53 +341,65 @@ def refresh_backtest_ohlcv_data( pairs_not_available = [] data_handler = get_datahandler(datadir, data_format) candle_type = CandleType.get_default(trading_mode) - process = "" - for idx, pair in enumerate(pairs, start=1): - if pair not in exchange.markets: - pairs_not_available.append(pair) - logger.info(f"Skipping pair {pair}...") - continue - for timeframe in timeframes: - logger.debug(f"Downloading pair {pair}, {candle_type}, interval {timeframe}.") - process = f"{idx}/{len(pairs)}" - _download_pair_history( - pair=pair, - process=process, - datadir=datadir, - exchange=exchange, - timerange=timerange, - data_handler=data_handler, - timeframe=str(timeframe), - new_pairs_days=new_pairs_days, - candle_type=candle_type, - erase=erase, - prepend=prepend, - ) - if trading_mode == "futures": - # Predefined candletype (and timeframe) depending on exchange - # Downloads what is necessary to backtest based on futures data. - tf_mark = exchange.get_option("mark_ohlcv_timeframe") - tf_funding_rate = exchange.get_option("funding_fee_timeframe") + with get_progress_tracker() as progress: + tf_length = len(timeframes) if trading_mode != "futures" else len(timeframes) + 2 + timeframe_task = progress.add_task("Timeframe", total=tf_length) + pair_task = progress.add_task("Downloading data...", total=len(pairs)) - fr_candle_type = CandleType.from_string(exchange.get_option("mark_ohlcv_price")) - # All exchanges need FundingRate for futures trading. - # The timeframe is aligned to the mark-price timeframe. - combs = ((CandleType.FUNDING_RATE, tf_funding_rate), (fr_candle_type, tf_mark)) - for candle_type_f, tf in combs: - logger.debug(f"Downloading pair {pair}, {candle_type_f}, interval {tf}.") + for pair in pairs: + progress.update(pair_task, description=f"Downloading {pair}") + progress.update(timeframe_task, completed=0) + + if pair not in exchange.markets: + pairs_not_available.append(pair) + logger.info(f"Skipping pair {pair}...") + continue + for timeframe in timeframes: + progress.update(timeframe_task, description=f"Timeframe {timeframe}") + logger.debug(f"Downloading pair {pair}, {candle_type}, interval {timeframe}.") _download_pair_history( pair=pair, - process=process, datadir=datadir, exchange=exchange, timerange=timerange, data_handler=data_handler, - timeframe=str(tf), + timeframe=str(timeframe), new_pairs_days=new_pairs_days, - candle_type=candle_type_f, + candle_type=candle_type, erase=erase, prepend=prepend, ) + progress.update(timeframe_task, advance=1) + if trading_mode == "futures": + # Predefined candletype (and timeframe) depending on exchange + # Downloads what is necessary to backtest based on futures data. + tf_mark = exchange.get_option("mark_ohlcv_timeframe") + tf_funding_rate = exchange.get_option("funding_fee_timeframe") + + fr_candle_type = CandleType.from_string(exchange.get_option("mark_ohlcv_price")) + # All exchanges need FundingRate for futures trading. + # The timeframe is aligned to the mark-price timeframe. + combs = ((CandleType.FUNDING_RATE, tf_funding_rate), (fr_candle_type, tf_mark)) + for candle_type_f, tf in combs: + logger.debug(f"Downloading pair {pair}, {candle_type_f}, interval {tf}.") + _download_pair_history( + pair=pair, + datadir=datadir, + exchange=exchange, + timerange=timerange, + data_handler=data_handler, + timeframe=str(tf), + new_pairs_days=new_pairs_days, + candle_type=candle_type_f, + erase=erase, + prepend=prepend, + ) + progress.update( + timeframe_task, advance=1, description=f"Timeframe {candle_type_f}, {tf}" + ) + + progress.update(pair_task, advance=1) + progress.update(timeframe_task, description="Timeframe") return pairs_not_available @@ -501,25 +509,30 @@ def refresh_backtest_trades_data( """ pairs_not_available = [] data_handler = get_datahandler(datadir, data_format=data_format) - for pair in pairs: - if pair not in exchange.markets: - pairs_not_available.append(pair) - logger.info(f"Skipping pair {pair}...") - continue + with get_progress_tracker() as progress: + pair_task = progress.add_task("Downloading data...", total=len(pairs)) + for pair in pairs: + progress.update(pair_task, description=f"Downloading trades [{pair}]") + if pair not in exchange.markets: + pairs_not_available.append(pair) + logger.info(f"Skipping pair {pair}...") + continue - if erase: - if data_handler.trades_purge(pair, trading_mode): - logger.info(f"Deleting existing data for pair {pair}.") + if erase: + if data_handler.trades_purge(pair, trading_mode): + logger.info(f"Deleting existing data for pair {pair}.") + + logger.info(f"Downloading trades for pair {pair}.") + _download_trades_history( + exchange=exchange, + pair=pair, + new_pairs_days=new_pairs_days, + timerange=timerange, + data_handler=data_handler, + trading_mode=trading_mode, + ) + progress.update(pair_task, advance=1) - logger.info(f"Downloading trades for pair {pair}.") - _download_trades_history( - exchange=exchange, - pair=pair, - new_pairs_days=new_pairs_days, - timerange=timerange, - data_handler=data_handler, - trading_mode=trading_mode, - ) return pairs_not_available diff --git a/freqtrade/optimize/hyperopt.py b/freqtrade/optimize/hyperopt.py index b411e7752..2006cee58 100644 --- a/freqtrade/optimize/hyperopt.py +++ b/freqtrade/optimize/hyperopt.py @@ -19,14 +19,6 @@ from joblib.externals import cloudpickle from pandas import DataFrame from rich.align import Align from rich.console import Console -from rich.progress import ( - BarColumn, - MofNCompleteColumn, - TaskProgressColumn, - TextColumn, - TimeElapsedColumn, - TimeRemainingColumn, -) from freqtrade.constants import DATETIME_PRINT_FORMAT, FTHYPT_FILEVERSION, LAST_BT_RESULT_FN, Config from freqtrade.data.converter import trim_dataframes @@ -48,7 +40,7 @@ from freqtrade.optimize.hyperopt_tools import ( ) from freqtrade.optimize.optimize_reports import generate_strategy_stats from freqtrade.resolvers.hyperopt_resolver import HyperOptLossResolver -from freqtrade.util import CustomProgress +from freqtrade.util import get_progress_tracker # Suppress scikit-learn FutureWarnings from skopt @@ -634,16 +626,7 @@ class Hyperopt: ) # Define progressbar - with CustomProgress( - TextColumn("[progress.description]{task.description}"), - BarColumn(bar_width=None), - MofNCompleteColumn(), - TaskProgressColumn(), - "•", - TimeElapsedColumn(), - "•", - TimeRemainingColumn(), - expand=True, + with get_progress_tracker( console=console, cust_objs=[Align.center(self._hyper_out.table)], ) as pbar: diff --git a/freqtrade/util/__init__.py b/freqtrade/util/__init__.py index 76902b176..8343790a0 100644 --- a/freqtrade/util/__init__.py +++ b/freqtrade/util/__init__.py @@ -15,6 +15,7 @@ from freqtrade.util.formatters import decimals_per_coin, fmt_coin, round_value from freqtrade.util.ft_precise import FtPrecise from freqtrade.util.measure_time import MeasureTime from freqtrade.util.periodic_cache import PeriodicCache +from freqtrade.util.progress_tracker import get_progress_tracker # noqa F401 from freqtrade.util.rich_progress import CustomProgress from freqtrade.util.rich_tables import print_df_rich_table, print_rich_table from freqtrade.util.template_renderer import render_template, render_template_with_fallback # noqa diff --git a/freqtrade/util/progress_tracker.py b/freqtrade/util/progress_tracker.py new file mode 100644 index 000000000..ca8f5d3e9 --- /dev/null +++ b/freqtrade/util/progress_tracker.py @@ -0,0 +1,28 @@ +from rich.progress import ( + BarColumn, + MofNCompleteColumn, + TaskProgressColumn, + TextColumn, + TimeElapsedColumn, + TimeRemainingColumn, +) + +from freqtrade.util.rich_progress import CustomProgress + + +def get_progress_tracker(**kwargs): + """ + Get progress Bar with custom columns. + """ + return CustomProgress( + TextColumn("[progress.description]{task.description}"), + BarColumn(bar_width=None), + MofNCompleteColumn(), + TaskProgressColumn(), + "•", + TimeElapsedColumn(), + "•", + TimeRemainingColumn(), + expand=True, + **kwargs, + ) diff --git a/freqtrade/util/rich_progress.py b/freqtrade/util/rich_progress.py index d295dafd5..afa26683e 100644 --- a/freqtrade/util/rich_progress.py +++ b/freqtrade/util/rich_progress.py @@ -5,7 +5,7 @@ from rich.progress import Progress class CustomProgress(Progress): - def __init__(self, *args, cust_objs, **kwargs) -> None: + def __init__(self, *args, cust_objs=[], **kwargs) -> None: self._cust_objs = cust_objs super().__init__(*args, **kwargs) diff --git a/tests/data/test_history.py b/tests/data/test_history.py index e3f92b1f9..e613bc589 100644 --- a/tests/data/test_history.py +++ b/tests/data/test_history.py @@ -151,9 +151,7 @@ def test_load_data_with_new_pair_1min( ) load_pair_history(datadir=tmp_path, timeframe="1m", pair="MEME/BTC", candle_type=candle_type) assert file.is_file() - assert log_has_re( - r'\(0/1\) - Download history data for "MEME/BTC", 1m, ' r"spot and store in .*", caplog - ) + assert log_has_re(r'Download history data for "MEME/BTC", 1m, ' r"spot and store in .*", caplog) def test_testdata_path(testdatadir) -> None: