CTYPTO/freqtrade
1
0
Fork 0
mirror of https://github.com/freqtrade/freqtrade.git synced 2026-01-20 05:50:36 +00:00
Code Issues Packages Projects Releases Wiki Activity

Datahandlers should store data from dataframes

Browse Source
This commit is contained in:
Matthias
2023-08-17 19:45:40 +02:00
parent 6fc1ee9831
commit 357b04202c
7 changed files with 27 additions and 51 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
freqtrade/data/converter.py
View File

@@ -264,7 +264,7 @@ def convert_trades_format(config: Config, convert_from: str, convert_to: str, er
logger.info(f"Converting trades for {config['pairs']}")
for pair in config['pairs']:
data = src.trades_load_aslist(pair=pair)
data = src.trades_load(pair=pair)
logger.info(f"Converting {len(data)} trades for {pair}")
trg.trades_store(pair, data)
if erase and convert_from != convert_to:

9
freqtrade/data/history/featherdatahandler.py
View File

@@ -4,7 +4,7 @@ from typing import Optional
from pandas import DataFrame, read_feather, to_datetime
from freqtrade.configuration import TimeRange
from freqtrade.constants import DEFAULT_DATAFRAME_COLUMNS, DEFAULT_TRADES_COLUMNS, TradeList
from freqtrade.constants import DEFAULT_DATAFRAME_COLUMNS, DEFAULT_TRADES_COLUMNS
from freqtrade.enums import CandleType
from .idatahandler import IDataHandler
@@ -82,18 +82,17 @@ class FeatherDataHandler(IDataHandler):
"""
raise NotImplementedError()
def trades_store(self, pair: str, data: TradeList) -> None:
def trades_store(self, pair: str, data: DataFrame) -> None:
"""
Store trades data (list of Dicts) to file
:param pair: Pair - used for filename
:param data: List of Lists containing trade data,
:param data: Dataframe containing trades
column sequence as in DEFAULT_TRADES_COLUMNS
"""
filename = self._pair_trades_filename(self._datadir, pair)
self.create_dir_if_needed(filename)
tradesdata = DataFrame(data, columns=DEFAULT_TRADES_COLUMNS)
tradesdata.to_feather(filename, compression_level=9, compression='lz4')
data.to_feather(filename, compression_level=9, compression='lz4')
def trades_append(self, pair: str, data: DataFrame):
"""

8
freqtrade/data/history/hdf5datahandler.py
View File

@@ -5,7 +5,7 @@ import numpy as np
import pandas as pd
from freqtrade.configuration import TimeRange
from freqtrade.constants import DEFAULT_DATAFRAME_COLUMNS, DEFAULT_TRADES_COLUMNS, TradeList
from freqtrade.constants import DEFAULT_DATAFRAME_COLUMNS, DEFAULT_TRADES_COLUMNS
from freqtrade.enums import CandleType
from .idatahandler import IDataHandler
@@ -100,16 +100,16 @@ class HDF5DataHandler(IDataHandler):
"""
raise NotImplementedError()
def trades_store(self, pair: str, data: TradeList) -> None:
def trades_store(self, pair: str, data: pd.DataFrame) -> None:
"""
Store trades data (list of Dicts) to file
:param pair: Pair - used for filename
:param data: List of Lists containing trade data,
:param data: Dataframe containing trades
column sequence as in DEFAULT_TRADES_COLUMNS
"""
key = self._pair_trades_key(pair)
pd.DataFrame(data, columns=DEFAULT_TRADES_COLUMNS).to_hdf(
data.to_hdf(
self._pair_trades_filename(self._datadir, pair), key,
mode='a', complevel=9, complib='blosc',
format='table', data_columns=['timestamp']

4
freqtrade/data/history/idatahandler.py
View File

@@ -171,11 +171,11 @@ class IDataHandler(ABC):
return [cls.rebuild_pair_from_filename(match[0]) for match in _tmp if match]
@abstractmethod
def trades_store(self, pair: str, data: TradeList) -> None:
def trades_store(self, pair: str, data: DataFrame) -> None:
"""
Store trades data (list of Dicts) to file
:param pair: Pair - used for filename
:param data: List of Lists containing trade data,
:param data: Dataframe containing trades
column sequence as in DEFAULT_TRADES_COLUMNS
"""

10
freqtrade/data/history/jsondatahandler.py
View File

@@ -6,7 +6,7 @@ from pandas import DataFrame, read_json, to_datetime
from freqtrade import misc
from freqtrade.configuration import TimeRange
from freqtrade.constants import DEFAULT_DATAFRAME_COLUMNS, DEFAULT_TRADES_COLUMNS, TradeList
from freqtrade.constants import DEFAULT_DATAFRAME_COLUMNS, DEFAULT_TRADES_COLUMNS
from freqtrade.data.converter import trades_dict_to_list
from freqtrade.enums import CandleType
@@ -94,15 +94,17 @@ class JsonDataHandler(IDataHandler):
"""
raise NotImplementedError()
def trades_store(self, pair: str, data: TradeList) -> None:
def trades_store(self, pair: str, data: DataFrame) -> None:
"""
Store trades data (list of Dicts) to file
:param pair: Pair - used for filename
:param data: List of Lists containing trade data,
:param data: Dataframe containing trades
column sequence as in DEFAULT_TRADES_COLUMNS
"""
filename = self._pair_trades_filename(self._datadir, pair)
misc.file_dump_json(filename, data, is_zip=self._use_zip)
data.loc[:, 'timestamp'] = data.loc[:, 'timestamp'].view(np.int64) // 1000 // 1000
trades = data[DEFAULT_TRADES_COLUMNS].values.tolist()
misc.file_dump_json(filename, trades, is_zip=self._use_zip)
def trades_append(self, pair: str, data: DataFrame):
"""

4
freqtrade/data/history/parquetdatahandler.py
View File

@@ -81,11 +81,11 @@ class ParquetDataHandler(IDataHandler):
"""
raise NotImplementedError()
def trades_store(self, pair: str, data: TradeList) -> None:
def trades_store(self, pair: str, data: DataFrame) -> None:
"""
Store trades data (list of Dicts) to file
:param pair: Pair - used for filename
:param data: List of Lists containing trade data,
:param data: Dataframe containing trades
column sequence as in DEFAULT_TRADES_COLUMNS
"""
# filename = self._pair_trades_filename(self._datadir, pair)

41
tests/data/test_datahandler.py
View File

@@ -7,6 +7,7 @@ from unittest.mock import MagicMock
import pytest
from pandas import DataFrame, Timestamp
from pandas.testing import assert_frame_equal
from freqtrade.configuration import TimeRange
from freqtrade.constants import AVAILABLE_DATAHANDLERS
@@ -331,30 +332,18 @@ def test_hdf5datahandler_trades_load(testdatadir):
def test_hdf5datahandler_trades_store(testdatadir, tmpdir):
tmpdir1 = Path(tmpdir)
dh = get_datahandler(testdatadir, 'hdf5')
trades = dh.trades_load_aslist('XRP/ETH')
trades = dh.trades_load('XRP/ETH')
dh1 = get_datahandler(tmpdir1, 'hdf5')
dh1.trades_store('XRP/NEW', trades)
file = tmpdir1 / 'XRP_NEW-trades.h5'
assert file.is_file()
# Load trades back
trades_new = dh1.trades_load_aslist('XRP/NEW')
trades_new = dh1.trades_load('XRP/NEW')
assert len(trades_new) == len(trades)
assert trades[0][0] == trades_new[0][0]
assert trades[0][1] == trades_new[0][1]
# assert trades[0][2] == trades_new[0][2] # This is nan - so comparison does not make sense
assert trades[0][3] == trades_new[0][3]
assert trades[0][4] == trades_new[0][4]
assert trades[0][5] == trades_new[0][5]
assert trades[0][6] == trades_new[0][6]
assert trades[-1][0] == trades_new[-1][0]
assert trades[-1][1] == trades_new[-1][1]
# assert trades[-1][2] == trades_new[-1][2] # This is nan - so comparison does not make sense
assert trades[-1][3] == trades_new[-1][3]
assert trades[-1][4] == trades_new[-1][4]
assert trades[-1][5] == trades_new[-1][5]
assert trades[-1][6] == trades_new[-1][6]
assert_frame_equal(trades, trades_new, check_exact=True)
assert len(trades_new) == len(trades)
def test_hdf5datahandler_trades_purge(mocker, testdatadir):
@@ -508,30 +497,16 @@ def test_featherdatahandler_trades_load(testdatadir):
def test_featherdatahandler_trades_store(testdatadir, tmpdir):
tmpdir1 = Path(tmpdir)
dh = get_datahandler(testdatadir, 'feather')
trades = dh.trades_load_aslist('XRP/ETH')
trades = dh.trades_load('XRP/ETH')
dh1 = get_datahandler(tmpdir1, 'feather')
dh1.trades_store('XRP/NEW', trades)
file = tmpdir1 / 'XRP_NEW-trades.feather'
assert file.is_file()
# Load trades back
trades_new = dh1.trades_load_aslist('XRP/NEW')
trades_new = dh1.trades_load('XRP/NEW')
assert_frame_equal(trades, trades_new, check_exact=True)
assert len(trades_new) == len(trades)
assert trades[0][0] == trades_new[0][0]
assert trades[0][1] == trades_new[0][1]
# assert trades[0][2] == trades_new[0][2] # This is nan - so comparison does not make sense
assert trades[0][3] == trades_new[0][3]
assert trades[0][4] == trades_new[0][4]
assert trades[0][5] == trades_new[0][5]
assert trades[0][6] == trades_new[0][6]
assert trades[-1][0] == trades_new[-1][0]
assert trades[-1][1] == trades_new[-1][1]
# assert trades[-1][2] == trades_new[-1][2] # This is nan - so comparison does not make sense
assert trades[-1][3] == trades_new[-1][3]
assert trades[-1][4] == trades_new[-1][4]
assert trades[-1][5] == trades_new[-1][5]
assert trades[-1][6] == trades_new[-1][6]
def test_featherdatahandler_trades_purge(mocker, testdatadir):