diff --git a/README.md b/README.md index d7ab7c05c..137078214 100644 --- a/README.md +++ b/README.md @@ -86,41 +86,50 @@ For further (native) installation methods, please refer to the [Installation doc ``` usage: freqtrade [-h] [-V] - {trade,create-userdir,new-config,new-strategy,download-data,convert-data,convert-trade-data,list-data,backtesting,edge,hyperopt,hyperopt-list,hyperopt-show,list-exchanges,list-hyperopts,list-markets,list-pairs,list-strategies,list-timeframes,show-trades,test-pairlist,install-ui,plot-dataframe,plot-profit,webserver} + {trade,create-userdir,new-config,show-config,new-strategy,download-data,convert-data,convert-trade-data,trades-to-ohlcv,list-data,backtesting,backtesting-show,backtesting-analysis,edge,hyperopt,hyperopt-list,hyperopt-show,list-exchanges,list-markets,list-pairs,list-strategies,list-freqaimodels,list-timeframes,show-trades,test-pairlist,convert-db,install-ui,plot-dataframe,plot-profit,webserver,strategy-updater,lookahead-analysis,recursive-analysis} ... Free, open source crypto trading bot positional arguments: - {trade,create-userdir,new-config,new-strategy,download-data,convert-data,convert-trade-data,list-data,backtesting,edge,hyperopt,hyperopt-list,hyperopt-show,list-exchanges,list-hyperopts,list-markets,list-pairs,list-strategies,list-timeframes,show-trades,test-pairlist,install-ui,plot-dataframe,plot-profit,webserver} + {trade,create-userdir,new-config,show-config,new-strategy,download-data,convert-data,convert-trade-data,trades-to-ohlcv,list-data,backtesting,backtesting-show,backtesting-analysis,edge,hyperopt,hyperopt-list,hyperopt-show,list-exchanges,list-markets,list-pairs,list-strategies,list-freqaimodels,list-timeframes,show-trades,test-pairlist,convert-db,install-ui,plot-dataframe,plot-profit,webserver,strategy-updater,lookahead-analysis,recursive-analysis} trade Trade module. create-userdir Create user-data directory. new-config Create new config + show-config Show resolved config new-strategy Create new strategy download-data Download backtesting data. convert-data Convert candle (OHLCV) data from one format to another. convert-trade-data Convert trade data from one format to another. + trades-to-ohlcv Convert trade data to OHLCV data. list-data List downloaded data. backtesting Backtesting module. + backtesting-show Show past Backtest results + backtesting-analysis + Backtest Analysis module. edge Edge module. hyperopt Hyperopt module. hyperopt-list List Hyperopt results hyperopt-show Show details of Hyperopt results list-exchanges Print available exchanges. - list-hyperopts Print available hyperopt classes. list-markets Print markets on exchange. list-pairs Print pairs on exchange. list-strategies Print available strategies. + list-freqaimodels Print available freqAI models. list-timeframes Print available timeframes for the exchange. show-trades Show trades. test-pairlist Test your pairlist configuration. + convert-db Migrate database to different system install-ui Install FreqUI plot-dataframe Plot candles with indicators. plot-profit Generate plot showing profits. webserver Webserver module. + strategy-updater updates outdated strategy files to the current version + lookahead-analysis Check for potential look ahead bias. + recursive-analysis Check for potential recursive formula issue. -optional arguments: +options: -h, --help show this help message and exit -V, --version show program's version number and exit diff --git a/docs/bot-usage.md b/docs/bot-usage.md index 7aeda0c42..8ebc82552 100644 --- a/docs/bot-usage.md +++ b/docs/bot-usage.md @@ -12,41 +12,50 @@ This page explains the different parameters of the bot and how to run it. ``` usage: freqtrade [-h] [-V] - {trade,create-userdir,new-config,new-strategy,download-data,convert-data,convert-trade-data,list-data,backtesting,edge,hyperopt,hyperopt-list,hyperopt-show,list-exchanges,list-hyperopts,list-markets,list-pairs,list-strategies,list-timeframes,show-trades,test-pairlist,install-ui,plot-dataframe,plot-profit,webserver} + {trade,create-userdir,new-config,show-config,new-strategy,download-data,convert-data,convert-trade-data,trades-to-ohlcv,list-data,backtesting,backtesting-show,backtesting-analysis,edge,hyperopt,hyperopt-list,hyperopt-show,list-exchanges,list-markets,list-pairs,list-strategies,list-freqaimodels,list-timeframes,show-trades,test-pairlist,convert-db,install-ui,plot-dataframe,plot-profit,webserver,strategy-updater,lookahead-analysis,recursive-analysis} ... Free, open source crypto trading bot positional arguments: - {trade,create-userdir,new-config,new-strategy,download-data,convert-data,convert-trade-data,list-data,backtesting,edge,hyperopt,hyperopt-list,hyperopt-show,list-exchanges,list-hyperopts,list-markets,list-pairs,list-strategies,list-timeframes,show-trades,test-pairlist,install-ui,plot-dataframe,plot-profit,webserver} + {trade,create-userdir,new-config,show-config,new-strategy,download-data,convert-data,convert-trade-data,trades-to-ohlcv,list-data,backtesting,backtesting-show,backtesting-analysis,edge,hyperopt,hyperopt-list,hyperopt-show,list-exchanges,list-markets,list-pairs,list-strategies,list-freqaimodels,list-timeframes,show-trades,test-pairlist,convert-db,install-ui,plot-dataframe,plot-profit,webserver,strategy-updater,lookahead-analysis,recursive-analysis} trade Trade module. create-userdir Create user-data directory. new-config Create new config + show-config Show resolved config new-strategy Create new strategy download-data Download backtesting data. convert-data Convert candle (OHLCV) data from one format to another. convert-trade-data Convert trade data from one format to another. + trades-to-ohlcv Convert trade data to OHLCV data. list-data List downloaded data. backtesting Backtesting module. + backtesting-show Show past Backtest results + backtesting-analysis + Backtest Analysis module. edge Edge module. hyperopt Hyperopt module. hyperopt-list List Hyperopt results hyperopt-show Show details of Hyperopt results list-exchanges Print available exchanges. - list-hyperopts Print available hyperopt classes. list-markets Print markets on exchange. list-pairs Print pairs on exchange. list-strategies Print available strategies. + list-freqaimodels Print available freqAI models. list-timeframes Print available timeframes for the exchange. show-trades Show trades. test-pairlist Test your pairlist configuration. + convert-db Migrate database to different system install-ui Install FreqUI plot-dataframe Plot candles with indicators. plot-profit Generate plot showing profits. webserver Webserver module. + strategy-updater updates outdated strategy files to the current version + lookahead-analysis Check for potential look ahead bias. + recursive-analysis Check for potential recursive formula issue. -optional arguments: +options: -h, --help show this help message and exit -V, --version show program's version number and exit diff --git a/docs/data-download.md b/docs/data-download.md index 2a51edb0b..59fa23b97 100644 --- a/docs/data-download.md +++ b/docs/data-download.md @@ -423,7 +423,8 @@ You can get a list of downloaded data using the `list-data` sub-command. usage: freqtrade list-data [-h] [-v] [--logfile FILE] [-V] [-c PATH] [-d PATH] [--userdir PATH] [--exchange EXCHANGE] [--data-format-ohlcv {json,jsongz,hdf5,feather,parquet}] - [-p PAIRS [PAIRS ...]] + [--data-format-trades {json,jsongz,hdf5,feather,parquet}] + [--trades] [-p PAIRS [PAIRS ...]] [--trading-mode {spot,margin,futures}] [--show-timerange] @@ -433,6 +434,10 @@ options: --data-format-ohlcv {json,jsongz,hdf5,feather,parquet} Storage format for downloaded candle (OHLCV) data. (default: `feather`). + --data-format-trades {json,jsongz,hdf5,feather,parquet} + Storage format for downloaded trades data. (default: + `feather`). + --trades Work on trades data instead of OHLCV data. -p PAIRS [PAIRS ...], --pairs PAIRS [PAIRS ...] Limit command to these pairs. Pairs are space- separated. @@ -465,13 +470,29 @@ Common arguments: ```bash > freqtrade list-data --userdir ~/.freqtrade/user_data/ -Found 33 pair / timeframe combinations. -pairs timeframe ----------- ----------------------------------------- -ADA/BTC 5m, 15m, 30m, 1h, 2h, 4h, 6h, 12h, 1d -ADA/ETH 5m, 15m, 30m, 1h, 2h, 4h, 6h, 12h, 1d -ETH/BTC 5m, 15m, 30m, 1h, 2h, 4h, 6h, 12h, 1d -ETH/USDT 5m, 15m, 30m, 1h, 2h, 4h + Found 33 pair / timeframe combinations. +┏━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━┓ +┃ Pair ┃ Timeframe ┃ Type ┃ +┡━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━┩ +│ ADA/BTC │ 5m, 15m, 30m, 1h, 2h, 4h, 6h, 12h, 1d │ spot │ +│ ADA/ETH │ 5m, 15m, 30m, 1h, 2h, 4h, 6h, 12h, 1d │ spot │ +│ ETH/BTC │ 5m, 15m, 30m, 1h, 2h, 4h, 6h, 12h, 1d │ spot │ +│ ETH/USDT │ 5m, 15m, 30m, 1h, 2h, 4h │ spot │ +└───────────────┴───────────────────────────────────────────┴──────┘ + +``` + +Show all trades data including from/to timerange + +``` bash +> freqtrade list-data --show --trades + Found trades data for 1 pair. +┏━━━━━━━━━┳━━━━━━┳━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━┓ +┃ Pair ┃ Type ┃ From ┃ To ┃ Trades ┃ +┡━━━━━━━━━╇━━━━━━╇━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━┩ +│ XRP/ETH │ spot │ 2019-10-11 00:00:11 │ 2019-10-13 11:19:28 │ 12477 │ +└─────────┴──────┴─────────────────────┴─────────────────────┴────────┘ + ``` ## Trades (tick) data diff --git a/freqtrade/commands/__init__.py b/freqtrade/commands/__init__.py index 48ee18e93..ec145eb5f 100644 --- a/freqtrade/commands/__init__.py +++ b/freqtrade/commands/__init__.py @@ -15,6 +15,7 @@ from freqtrade.commands.data_commands import ( start_convert_trades, start_download_data, start_list_data, + start_list_trades_data, ) from freqtrade.commands.db_commands import start_convert_db from freqtrade.commands.deploy_commands import ( diff --git a/freqtrade/commands/arguments.py b/freqtrade/commands/arguments.py index 0c93af78a..62a79b0e8 100755 --- a/freqtrade/commands/arguments.py +++ b/freqtrade/commands/arguments.py @@ -132,7 +132,15 @@ ARGS_CONVERT_TRADES = [ "trading_mode", ] -ARGS_LIST_DATA = ["exchange", "dataformat_ohlcv", "pairs", "trading_mode", "show_timerange"] +ARGS_LIST_DATA = [ + "exchange", + "dataformat_ohlcv", + "dataformat_trades", + "trades", + "pairs", + "trading_mode", + "show_timerange", +] ARGS_DOWNLOAD_DATA = [ "pairs", diff --git a/freqtrade/commands/cli_options.py b/freqtrade/commands/cli_options.py index b9236a0ab..54e139443 100755 --- a/freqtrade/commands/cli_options.py +++ b/freqtrade/commands/cli_options.py @@ -446,8 +446,12 @@ AVAILABLE_CLI_OPTIONS = { ), "download_trades": Arg( "--dl-trades", - help="Download trades instead of OHLCV data. The bot will resample trades to the " - "desired timeframe as specified as --timeframes/-t.", + help="Download trades instead of OHLCV data.", + action="store_true", + ), + "trades": Arg( + "--trades", + help="Work on trades data instead of OHLCV data.", action="store_true", ), "convert_trades": Arg( diff --git a/freqtrade/commands/data_commands.py b/freqtrade/commands/data_commands.py index f3f56c7b2..a114444b3 100644 --- a/freqtrade/commands/data_commands.py +++ b/freqtrade/commands/data_commands.py @@ -14,6 +14,7 @@ from freqtrade.data.history import download_data_main from freqtrade.enums import CandleType, RunMode, TradingMode from freqtrade.exceptions import ConfigurationError from freqtrade.exchange import timeframe_to_minutes +from freqtrade.misc import plural from freqtrade.plugins.pairlist.pairlist_helpers import dynamic_expand_pairlist from freqtrade.resolvers import ExchangeResolver from freqtrade.util import print_rich_table @@ -115,9 +116,13 @@ def start_convert_data(args: Dict[str, Any], ohlcv: bool = True) -> None: def start_list_data(args: Dict[str, Any]) -> None: """ - List available backtest data + List available OHLCV data """ + if args["trades"]: + start_list_trades_data(args) + return + config = setup_utils_configuration(args, RunMode.UTIL_NO_EXCHANGE) from freqtrade.data.history import get_datahandler @@ -127,7 +132,6 @@ def start_list_data(args: Dict[str, Any]) -> None: paircombs = dhc.ohlcv_get_available_data( config["datadir"], config.get("trading_mode", TradingMode.SPOT) ) - if args["pairs"]: paircombs = [comb for comb in paircombs if comb[0] in args["pairs"]] title = f"Found {len(paircombs)} pair / timeframe combinations." @@ -171,3 +175,51 @@ def start_list_data(args: Dict[str, Any]) -> None: summary=title, table_kwargs={"min_width": 50}, ) + + +def start_list_trades_data(args: Dict[str, Any]) -> None: + """ + List available Trades data + """ + + config = setup_utils_configuration(args, RunMode.UTIL_NO_EXCHANGE) + + from freqtrade.data.history import get_datahandler + + dhc = get_datahandler(config["datadir"], config["dataformat_trades"]) + + paircombs = dhc.trades_get_available_data( + config["datadir"], config.get("trading_mode", TradingMode.SPOT) + ) + + if args["pairs"]: + paircombs = [comb for comb in paircombs if comb in args["pairs"]] + + title = f"Found trades data for {len(paircombs)} {plural(len(paircombs), 'pair')}." + if not config.get("show_timerange"): + print_rich_table( + [(pair, config.get("candle_type_def", CandleType.SPOT)) for pair in sorted(paircombs)], + ("Pair", "Type"), + title, + table_kwargs={"min_width": 50}, + ) + else: + paircombs1 = [ + (pair, *dhc.trades_data_min_max(pair, config.get("trading_mode", TradingMode.SPOT))) + for pair in paircombs + ] + print_rich_table( + [ + ( + pair, + config.get("candle_type_def", CandleType.SPOT), + start.strftime(DATETIME_PRINT_FORMAT), + end.strftime(DATETIME_PRINT_FORMAT), + str(length), + ) + for pair, start, end, length in sorted(paircombs1, key=lambda x: (x[0])) + ], + ("Pair", "Type", "From", "To", "Trades"), + summary=title, + table_kwargs={"min_width": 50}, + ) diff --git a/freqtrade/data/history/datahandlers/idatahandler.py b/freqtrade/data/history/datahandlers/idatahandler.py index aea7ea14a..db1660dc8 100644 --- a/freqtrade/data/history/datahandlers/idatahandler.py +++ b/freqtrade/data/history/datahandlers/idatahandler.py @@ -12,7 +12,7 @@ from datetime import datetime, timezone from pathlib import Path from typing import List, Optional, Tuple, Type -from pandas import DataFrame +from pandas import DataFrame, to_datetime from freqtrade import misc from freqtrade.configuration import TimeRange @@ -32,6 +32,7 @@ logger = logging.getLogger(__name__) class IDataHandler(ABC): _OHLCV_REGEX = r"^([a-zA-Z_\d-]+)\-(\d+[a-zA-Z]{1,2})\-?([a-zA-Z_]*)?(?=\.)" + _TRADES_REGEX = r"^([a-zA-Z_\d-]+)\-(trades)?(?=\.)" def __init__(self, datadir: Path) -> None: self._datadir = datadir @@ -166,6 +167,50 @@ class IDataHandler(ABC): :param candle_type: Any of the enum CandleType (must match trading mode!) """ + @classmethod + def trades_get_available_data(cls, datadir: Path, trading_mode: TradingMode) -> List[str]: + """ + Returns a list of all pairs with ohlcv data available in this datadir + :param datadir: Directory to search for ohlcv files + :param trading_mode: trading-mode to be used + :return: List of Tuples of (pair, timeframe, CandleType) + """ + if trading_mode == TradingMode.FUTURES: + datadir = datadir.joinpath("futures") + _tmp = [ + re.search(cls._TRADES_REGEX, p.name) + for p in datadir.glob(f"*.{cls._get_file_extension()}") + ] + return [ + cls.rebuild_pair_from_filename(match[1]) + for match in _tmp + if match and len(match.groups()) > 1 + ] + + def trades_data_min_max( + self, + pair: str, + trading_mode: TradingMode, + ) -> Tuple[datetime, datetime, int]: + """ + Returns the min and max timestamp for the given pair's trades data. + :param pair: Pair to get min/max for + :param trading_mode: Trading mode to use (used to determine the filename) + :return: (min, max, len) + """ + df = self._trades_load(pair, trading_mode) + if df.empty: + return ( + datetime.fromtimestamp(0, tz=timezone.utc), + datetime.fromtimestamp(0, tz=timezone.utc), + 0, + ) + return ( + to_datetime(df.iloc[0]["timestamp"], unit="ms", utc=True).to_pydatetime(), + to_datetime(df.iloc[-1]["timestamp"], unit="ms", utc=True).to_pydatetime(), + len(df), + ) + @classmethod def trades_get_pairs(cls, datadir: Path) -> List[str]: """ diff --git a/tests/commands/test_commands.py b/tests/commands/test_commands.py index 687bff69f..c55126db1 100644 --- a/tests/commands/test_commands.py +++ b/tests/commands/test_commands.py @@ -1692,6 +1692,54 @@ def test_start_list_data(testdatadir, capsys): ) +def test_start_list_trades_data(testdatadir, capsys): + args = [ + "list-data", + "--datadir", + str(testdatadir), + "--trades", + ] + pargs = get_args(args) + pargs["config"] = None + start_list_data(pargs) + captured = capsys.readouterr() + assert "Found trades data for 1 pair." in captured.out + assert re.search(r".*Pair.*Type.*\n", captured.out) + assert re.search(r"\n.* XRP/ETH .* spot |\n", captured.out) + + args = [ + "list-data", + "--datadir", + str(testdatadir), + "--trades", + "--show-timerange", + ] + pargs = get_args(args) + pargs["config"] = None + start_list_data(pargs) + captured = capsys.readouterr() + assert "Found trades data for 1 pair." in captured.out + assert re.search(r".*Pair.*Type.*From.*To.*Trades.*\n", captured.out) + assert re.search( + r"\n.* XRP/ETH .* spot .* 2019-10-11 00:00:01 .* 2019-10-13 11:19:28 .* 12477 .*|\n", + captured.out, + ) + + args = [ + "list-data", + "--datadir", + str(testdatadir), + "--trades", + "--trading-mode", + "futures", + ] + pargs = get_args(args) + pargs["config"] = None + start_list_data(pargs) + captured = capsys.readouterr() + assert "Found trades data for 0 pairs." in captured.out + + @pytest.mark.usefixtures("init_persistence") def test_show_trades(mocker, fee, capsys, caplog): mocker.patch("freqtrade.persistence.init_db") diff --git a/tests/data/test_datahandler.py b/tests/data/test_datahandler.py index b8bb5661f..99af63eca 100644 --- a/tests/data/test_datahandler.py +++ b/tests/data/test_datahandler.py @@ -519,6 +519,39 @@ def test_datahandler_trades_purge(mocker, testdatadir, datahandler): assert unlinkmock.call_count == 1 +def test_datahandler_trades_get_available_data(testdatadir): + paircombs = FeatherDataHandler.trades_get_available_data(testdatadir, TradingMode.SPOT) + # Convert to set to avoid failures due to sorting + assert set(paircombs) == {"XRP/ETH"} + + paircombs = FeatherDataHandler.trades_get_available_data(testdatadir, TradingMode.FUTURES) + # Convert to set to avoid failures due to sorting + assert set(paircombs) == set() + + paircombs = JsonGzDataHandler.trades_get_available_data(testdatadir, TradingMode.SPOT) + assert set(paircombs) == {"XRP/ETH", "XRP/OLD"} + paircombs = HDF5DataHandler.trades_get_available_data(testdatadir, TradingMode.SPOT) + assert set(paircombs) == {"XRP/ETH"} + + +def test_datahandler_trades_data_min_max(testdatadir): + dh = FeatherDataHandler(testdatadir) + min_max = dh.trades_data_min_max("XRP/ETH", TradingMode.SPOT) + assert len(min_max) == 3 + + # Empty pair + min_max = dh.trades_data_min_max("NADA/ETH", TradingMode.SPOT) + assert len(min_max) == 3 + assert min_max[0] == datetime.fromtimestamp(0, tz=timezone.utc) + assert min_max[0] == min_max[1] + + # Existing pair ... + min_max = dh.trades_data_min_max("XRP/ETH", TradingMode.SPOT) + assert len(min_max) == 3 + assert min_max[0] == datetime(2019, 10, 11, 0, 0, 11, 620000, tzinfo=timezone.utc) + assert min_max[1] == datetime(2019, 10, 13, 11, 19, 28, 844000, tzinfo=timezone.utc) + + def test_gethandlerclass(): cl = get_datahandlerclass("json") assert cl == JsonDataHandler