CTYPTO/freqtrade
1
0
Fork 0
mirror of https://github.com/freqtrade/freqtrade.git synced 2025-11-29 00:23:07 +00:00
Code Issues Packages Projects Releases Wiki Activity

Update download-trades method to work with dataframes

Browse Source
This commit is contained in:
Matthias
2023-08-18 09:07:51 +02:00
parent 7577613882
commit fa5f7d290b
2 changed files with 22 additions and 19 deletions
Download Patch File Download Diff File Expand all files Collapse all files

37
freqtrade/data/history/history_utils.py
View File

@@ -10,14 +10,16 @@ from freqtrade.configuration import TimeRange
from freqtrade.constants import (DATETIME_PRINT_FORMAT, DEFAULT_DATAFRAME_COLUMNS,
DL_DATA_TIMEFRAMES, Config)
from freqtrade.data.converter import (clean_ohlcv_dataframe, ohlcv_to_dataframe,
trades_remove_duplicates, trades_to_ohlcv)
trades_df_remove_duplicates, trades_list_to_df,
trades_to_ohlcv)
from freqtrade.data.history.idatahandler import IDataHandler, get_datahandler
from freqtrade.enums import CandleType
from freqtrade.exceptions import OperationalException
from freqtrade.exchange import Exchange
from freqtrade.plugins.pairlist.pairlist_helpers import dynamic_expand_pairlist
from freqtrade.util import format_ms_time
from freqtrade.util import dt_ts, format_ms_time
from freqtrade.util.binance_mig import migrate_binance_futures_data
from freqtrade.util.datetime_helpers import dt_now
logger = logging.getLogger(__name__)
@@ -343,30 +345,32 @@ def _download_trades_history(exchange: Exchange,
if timerange.stoptype == 'date':
until = timerange.stopts * 1000
trades = data_handler.trades_load_aslist(pair)
trades = data_handler.trades_load(pair)
# TradesList columns are defined in constants.DEFAULT_TRADES_COLUMNS
# DEFAULT_TRADES_COLUMNS: 0 -> timestamp
# DEFAULT_TRADES_COLUMNS: 1 -> id
if trades and since < trades[0][0]:
if not trades.empty and since < trades.iloc[0]['timestamp']:
# since is before the first trade
logger.info(f"Start earlier than available data. Redownloading trades for {pair}...")
trades = []
trades = trades_list_to_df([])
if not since:
since = int((datetime.now() - timedelta(days=new_pairs_days)).timestamp()) * 1000
since = dt_ts(dt_now() - timedelta(days=new_pairs_days))
from_id = trades[-1][1] if trades else None
if trades and since < trades[-1][0]:
from_id = trades.iloc[-1]['id'] if not trades.empty else None
if not trades.empty and since < trades.iloc[-1]['timestamp']:
# Reset since to the last available point
# - 5 seconds (to ensure we're getting all trades)
since = trades[-1][0] - (5 * 1000)
since = trades.iloc[-1]['timestamp'] - (5 * 1000)
logger.info(f"Using last trade date -5s - Downloading trades for {pair} "
f"since: {format_ms_time(since)}.")
logger.debug(f"Current Start: {format_ms_time(trades[0][0]) if trades else 'None'}")
logger.debug(f"Current End: {format_ms_time(trades[-1][0]) if trades else 'None'}")
logger.debug("Current Start: %s", 'None' if trades.empty else
f"{trades.iloc[0]['date']:{DATETIME_PRINT_FORMAT}}")
logger.debug("Current End: %s", 'None' if trades.empty else
f"{trades.iloc[-1]['date']:{DATETIME_PRINT_FORMAT}}")
logger.info(f"Current Amount of trades: {len(trades)}")
# Default since_ms to 30 days if nothing is given
@@ -375,13 +379,16 @@ def _download_trades_history(exchange: Exchange,
until=until,
from_id=from_id,
)
trades.extend(new_trades[1])
new_trades_df = trades_list_to_df(new_trades[1])
trades = concat([trades, new_trades_df], axis=0)
# Remove duplicates to make sure we're not storing data we don't need
trades = trades_remove_duplicates(trades)
trades = trades_df_remove_duplicates(trades)
data_handler.trades_store(pair, data=trades)
logger.debug(f"New Start: {format_ms_time(trades[0][0])}")
logger.debug(f"New End: {format_ms_time(trades[-1][0])}")
logger.debug("New Start: %s", 'None' if trades.empty else
f"{trades.iloc[0]['date']:{DATETIME_PRINT_FORMAT}}")
logger.debug("New End: %s", 'None' if trades.empty else
f"{trades.iloc[-1]['date']:{DATETIME_PRINT_FORMAT}}")
logger.info(f"New Amount of trades: {len(trades)}")
return True

4
freqtrade/data/history/idatahandler.py
View File

@@ -222,10 +222,6 @@ class IDataHandler(ABC):
trades = trades_convert_types(trades)
return trades
def trades_load_aslist(self, pair: str, timerange: Optional[TimeRange] = None) -> TradeList:
trades = trades_df_remove_duplicates(self._trades_load(pair, timerange=timerange))
return trades.values.tolist()
@classmethod
def create_dir_if_needed(cls, datadir: Path):
"""