CTYPTO/freqtrade
1
0
Fork 0
mirror of https://github.com/freqtrade/freqtrade.git synced 2026-04-29 05:20:10 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge remote-tracking branch 'upstream/develop' into feature/fetch-public-trades

Browse Source
This commit is contained in:
Joe Schr
2023-12-18 10:34:20 +01:00
parent 1a4eabc0bb 99359a7f85
commit 18be79c906
124 changed files with 7802 additions and 2380 deletions
Download Patch File Download Diff File Expand all files Collapse all files

52
tests/data/test_converter.py
View File

@@ -1,6 +1,5 @@
# pragma pylint: disable=missing-docstring, C0103
import logging
from pathlib import Path
from shutil import copyfile
import numpy as np
@@ -50,8 +49,8 @@ def test_trades_to_ohlcv(trades_history_df, caplog):
assert 'high' in df.columns
assert 'low' in df.columns
assert 'close' in df.columns
assert df.loc[:, 'high'][0] == 0.019627
assert df.loc[:, 'low'][0] == 0.019626
assert df.iloc[0, :]['high'] == 0.019627
assert df.iloc[0, :]['low'] == 0.019626
def test_ohlcv_fill_up_missing_data(testdatadir, caplog):
@@ -65,7 +64,7 @@ def test_ohlcv_fill_up_missing_data(testdatadir, caplog):
# Column names should not change
assert (data.columns == data2.columns).all()
assert log_has_re(f"Missing data fillup for UNITTEST/BTC: before: "
assert log_has_re(f"Missing data fillup for UNITTEST/BTC, 1m: before: "
f"{len(data)} - after: {len(data2)}.*", caplog)
# Test fillup actually fixes invalid backtest data
@@ -129,7 +128,7 @@ def test_ohlcv_fill_up_missing_data2(caplog):
# Column names should not change
assert (data.columns == data2.columns).all()
assert log_has_re(f"Missing data fillup for UNITTEST/BTC: before: "
assert log_has_re(f"Missing data fillup for UNITTEST/BTC, {timeframe}: before: "
f"{len(data)} - after: {len(data2)}.*", caplog)
@@ -323,18 +322,17 @@ def test_trades_dict_to_list(fetch_trades_result):
assert t[6] == fetch_trades_result[i]['cost']
def test_convert_trades_format(default_conf, testdatadir, tmpdir):
tmpdir1 = Path(tmpdir)
files = [{'old': tmpdir1 / "XRP_ETH-trades.json.gz",
'new': tmpdir1 / "XRP_ETH-trades.json"},
{'old': tmpdir1 / "XRP_OLD-trades.json.gz",
'new': tmpdir1 / "XRP_OLD-trades.json"},
def test_convert_trades_format(default_conf, testdatadir, tmp_path):
files = [{'old': tmp_path / "XRP_ETH-trades.json.gz",
'new': tmp_path / "XRP_ETH-trades.json"},
{'old': tmp_path / "XRP_OLD-trades.json.gz",
'new': tmp_path / "XRP_OLD-trades.json"},
]
for file in files:
copyfile(testdatadir / file['old'].name, file['old'])
assert not file['new'].exists()
default_conf['datadir'] = tmpdir1
default_conf['datadir'] = tmp_path
convert_trades_format(default_conf, convert_from='jsongz',
convert_to='json', erase=False)
@@ -362,16 +360,15 @@ def test_convert_trades_format(default_conf, testdatadir, tmpdir):
(['UNITTEST_USDT_USDT-1h-mark', 'XRP_USDT_USDT-1h-mark'], CandleType.MARK),
(['XRP_USDT_USDT-1h-futures'], CandleType.FUTURES),
])
def test_convert_ohlcv_format(default_conf, testdatadir, tmpdir, file_base, candletype):
tmpdir1 = Path(tmpdir)
def test_convert_ohlcv_format(default_conf, testdatadir, tmp_path, file_base, candletype):
prependix = '' if candletype == CandleType.SPOT else 'futures/'
files_orig = []
files_temp = []
files_new = []
for file in file_base:
file_orig = testdatadir / f"{prependix}{file}.feather"
file_temp = tmpdir1 / f"{prependix}{file}.feather"
file_new = tmpdir1 / f"{prependix}{file}.json.gz"
file_temp = tmp_path / f"{prependix}{file}.feather"
file_new = tmp_path / f"{prependix}{file}.json.gz"
IDataHandler.create_dir_if_needed(file_temp)
copyfile(file_orig, file_temp)
@@ -379,7 +376,7 @@ def test_convert_ohlcv_format(default_conf, testdatadir, tmpdir, file_base, cand
files_temp.append(file_temp)
files_new.append(file_new)
default_conf['datadir'] = tmpdir1
default_conf['datadir'] = tmp_path
default_conf['candle_types'] = [candletype]
if candletype == CandleType.SPOT:
@@ -445,30 +442,29 @@ def test_reduce_dataframe_footprint():
assert df2['close_copy'].dtype == np.float32
def test_convert_trades_to_ohlcv(testdatadir, tmpdir, caplog):
tmpdir1 = Path(tmpdir)
def test_convert_trades_to_ohlcv(testdatadir, tmp_path, caplog):
pair = 'XRP/ETH'
file1 = tmpdir1 / 'XRP_ETH-1m.feather'
file5 = tmpdir1 / 'XRP_ETH-5m.feather'
filetrades = tmpdir1 / 'XRP_ETH-trades.json.gz'
file1 = tmp_path / 'XRP_ETH-1m.feather'
file5 = tmp_path / 'XRP_ETH-5m.feather'
filetrades = tmp_path / 'XRP_ETH-trades.json.gz'
copyfile(testdatadir / file1.name, file1)
copyfile(testdatadir / file5.name, file5)
copyfile(testdatadir / filetrades.name, filetrades)
# Compare downloaded dataset with converted dataset
dfbak_1m = load_pair_history(datadir=tmpdir1, timeframe="1m", pair=pair)
dfbak_5m = load_pair_history(datadir=tmpdir1, timeframe="5m", pair=pair)
dfbak_1m = load_pair_history(datadir=tmp_path, timeframe="1m", pair=pair)
dfbak_5m = load_pair_history(datadir=tmp_path, timeframe="5m", pair=pair)
tr = TimeRange.parse_timerange('20191011-20191012')
convert_trades_to_ohlcv([pair], timeframes=['1m', '5m'],
data_format_trades='jsongz',
datadir=tmpdir1, timerange=tr, erase=True)
datadir=tmp_path, timerange=tr, erase=True)
assert log_has("Deleting existing data for pair XRP/ETH, interval 1m.", caplog)
# Load new data
df_1m = load_pair_history(datadir=tmpdir1, timeframe="1m", pair=pair)
df_5m = load_pair_history(datadir=tmpdir1, timeframe="5m", pair=pair)
df_1m = load_pair_history(datadir=tmp_path, timeframe="1m", pair=pair)
df_5m = load_pair_history(datadir=tmp_path, timeframe="5m", pair=pair)
assert_frame_equal(dfbak_1m, df_1m, check_exact=True)
assert_frame_equal(dfbak_5m, df_5m, check_exact=True)
@@ -477,5 +473,5 @@ def test_convert_trades_to_ohlcv(testdatadir, tmpdir, caplog):
convert_trades_to_ohlcv(['NoDatapair'], timeframes=['1m', '5m'],
data_format_trades='jsongz',
datadir=tmpdir1, timerange=tr, erase=True)
datadir=tmp_path, timerange=tr, erase=True)
assert log_has(msg, caplog)

25
tests/data/test_datahandler.py
View File

@@ -328,17 +328,16 @@ def test_hdf5datahandler_trades_load(testdatadir):
])
def test_hdf5datahandler_ohlcv_load_and_resave(
testdatadir,
tmpdir,
tmp_path,
pair,
timeframe,
candle_type,
candle_append,
startdt, enddt
):
tmpdir1 = Path(tmpdir)
tmpdir2 = tmpdir1
tmpdir2 = tmp_path
if candle_type not in ('', 'spot'):
tmpdir2 = tmpdir1 / 'futures'
tmpdir2 = tmp_path / 'futures'
tmpdir2.mkdir()
dh = get_datahandler(testdatadir, 'hdf5')
ohlcv = dh._ohlcv_load(pair, timeframe, None, candle_type=candle_type)
@@ -348,7 +347,7 @@ def test_hdf5datahandler_ohlcv_load_and_resave(
file = tmpdir2 / f"UNITTEST_NEW-{timeframe}{candle_append}.h5"
assert not file.is_file()
dh1 = get_datahandler(tmpdir1, 'hdf5')
dh1 = get_datahandler(tmp_path, 'hdf5')
dh1.ohlcv_store('UNITTEST/NEW', timeframe, ohlcv, candle_type=candle_type)
assert file.is_file()
@@ -379,17 +378,16 @@ def test_hdf5datahandler_ohlcv_load_and_resave(
def test_generic_datahandler_ohlcv_load_and_resave(
datahandler,
testdatadir,
tmpdir,
tmp_path,
pair,
timeframe,
candle_type,
candle_append,
startdt, enddt
):
tmpdir1 = Path(tmpdir)
tmpdir2 = tmpdir1
tmpdir2 = tmp_path
if candle_type not in ('', 'spot'):
tmpdir2 = tmpdir1 / 'futures'
tmpdir2 = tmp_path / 'futures'
tmpdir2.mkdir()
# Load data from one common file
dhbase = get_datahandler(testdatadir, 'feather')
@@ -403,7 +401,7 @@ def test_generic_datahandler_ohlcv_load_and_resave(
file = tmpdir2 / f"UNITTEST_NEW-{timeframe}{candle_append}.{dh._get_file_extension()}"
assert not file.is_file()
dh1 = get_datahandler(tmpdir1, datahandler)
dh1 = get_datahandler(tmp_path, datahandler)
dh1.ohlcv_store('UNITTEST/NEW', timeframe, ohlcv, candle_type=candle_type)
assert file.is_file()
@@ -459,15 +457,14 @@ def test_datahandler_trades_load(testdatadir, datahandler):
@pytest.mark.parametrize('datahandler', ['jsongz', 'hdf5', 'feather', 'parquet'])
def test_datahandler_trades_store(testdatadir, tmpdir, datahandler):
tmpdir1 = Path(tmpdir)
def test_datahandler_trades_store(testdatadir, tmp_path, datahandler):
dh = get_datahandler(testdatadir, datahandler)
trades = dh.trades_load('XRP/ETH')
dh1 = get_datahandler(tmpdir1, datahandler)
dh1 = get_datahandler(tmp_path, datahandler)
dh1.trades_store('XRP/NEW', trades)
file = tmpdir1 / f'XRP_NEW-trades.{dh1._get_file_extension()}'
file = tmp_path / f'XRP_NEW-trades.{dh1._get_file_extension()}'
assert file.is_file()
# Load trades back
trades_new = dh1.trades_load('XRP/NEW')

86
tests/data/test_dataprovider.py
View File

@@ -500,3 +500,89 @@ def test_dp__add_external_df(default_conf_usdt):
# 36 hours - from 2022-01-03 12:00:00+00:00 to 2022-01-05 00:00:00+00:00
assert isinstance(res[1], int)
assert res[1] == 0
def test_dp_get_required_startup(default_conf_usdt):
timeframe = '1h'
default_conf_usdt["timeframe"] = timeframe
dp = DataProvider(default_conf_usdt, None)
# No FreqAI config
assert dp.get_required_startup('5m', False) == 0
assert dp.get_required_startup('1h', False) == 0
assert dp.get_required_startup('1d', False) == 0
assert dp.get_required_startup('1d', True) == 0
assert dp.get_required_startup('1d') == 0
dp._config['startup_candle_count'] = 20
assert dp.get_required_startup('5m', False) == 20
assert dp.get_required_startup('5m', True) == 20
assert dp.get_required_startup('1h', False) == 20
assert dp.get_required_startup('1h') == 20
# With freqAI config
dp._config['freqai'] = {
'enabled': True,
'train_period_days': 20,
'feature_parameters': {
'indicator_periods_candles': [
5,
20,
]
}
}
assert dp.get_required_startup('5m', False) == 20
assert dp.get_required_startup('5m', True) == 5780
assert dp.get_required_startup('1h', False) == 20
assert dp.get_required_startup('1h', True) == 500
assert dp.get_required_startup('1d', False) == 20
assert dp.get_required_startup('1d', True) == 40
assert dp.get_required_startup('1d') == 40
# FreqAI kindof ignores startup_candle_count if it's below indicator_periods_candles
dp._config['startup_candle_count'] = 0
assert dp.get_required_startup('5m', False) == 20
assert dp.get_required_startup('5m', True) == 5780
assert dp.get_required_startup('1h', False) == 20
assert dp.get_required_startup('1h', True) == 500
assert dp.get_required_startup('1d', False) == 20
assert dp.get_required_startup('1d', True) == 40
assert dp.get_required_startup('1d') == 40
dp._config['freqai']['feature_parameters']['indicator_periods_candles'][1] = 50
assert dp.get_required_startup('5m', False) == 50
assert dp.get_required_startup('5m', True) == 5810
assert dp.get_required_startup('1h', False) == 50
assert dp.get_required_startup('1h', True) == 530
assert dp.get_required_startup('1d', False) == 50
assert dp.get_required_startup('1d', True) == 70
assert dp.get_required_startup('1d') == 70
# scenario from issue https://github.com/freqtrade/freqtrade/issues/9432
dp._config['freqai'] = {
'enabled': True,
'train_period_days': 180,
'feature_parameters': {
'indicator_periods_candles': [
10,
20,
]
}
}
dp._config['startup_candle_count'] = 40
assert dp.get_required_startup('5m', False) == 40
assert dp.get_required_startup('5m', True) == 51880
assert dp.get_required_startup('1h', False) == 40
assert dp.get_required_startup('1h', True) == 4360
assert dp.get_required_startup('1d', False) == 40
assert dp.get_required_startup('1d', True) == 220
assert dp.get_required_startup('1d') == 220

44
tests/data/test_history.py
View File

@@ -106,17 +106,16 @@ def test_load_data_startup_candles(mocker, testdatadir) -> None:
@pytest.mark.parametrize('candle_type', ['mark', ''])
def test_load_data_with_new_pair_1min(ohlcv_history_list, mocker, caplog,
default_conf, tmpdir, candle_type) -> None:
default_conf, tmp_path, candle_type) -> None:
"""
Test load_pair_history() with 1 min timeframe
"""
tmpdir1 = Path(tmpdir)
mocker.patch(f'{EXMS}.get_historic_ohlcv', return_value=ohlcv_history_list)
exchange = get_patched_exchange(mocker, default_conf)
file = tmpdir1 / 'MEME_BTC-1m.feather'
file = tmp_path / 'MEME_BTC-1m.feather'
# do not download a new pair if refresh_pairs isn't set
load_pair_history(datadir=tmpdir1, timeframe='1m', pair='MEME/BTC', candle_type=candle_type)
load_pair_history(datadir=tmp_path, timeframe='1m', pair='MEME/BTC', candle_type=candle_type)
assert not file.is_file()
assert log_has(
f"No history for MEME/BTC, {candle_type}, 1m found. "
@@ -124,10 +123,10 @@ def test_load_data_with_new_pair_1min(ohlcv_history_list, mocker, caplog,
)
# download a new pair if refresh_pairs is set
refresh_data(datadir=tmpdir1, timeframe='1m', pairs=['MEME/BTC'],
refresh_data(datadir=tmp_path, timeframe='1m', pairs=['MEME/BTC'],
exchange=exchange, candle_type=CandleType.SPOT
)
load_pair_history(datadir=tmpdir1, timeframe='1m', pair='MEME/BTC', candle_type=candle_type)
load_pair_history(datadir=tmp_path, timeframe='1m', pair='MEME/BTC', candle_type=candle_type)
assert file.is_file()
assert log_has_re(
r'\(0/1\) - Download history data for "MEME/BTC", 1m, '
@@ -273,27 +272,26 @@ def test_download_pair_history(
ohlcv_history_list,
mocker,
default_conf,
tmpdir,
tmp_path,
candle_type,
subdir,
file_tail
) -> None:
mocker.patch(f'{EXMS}.get_historic_ohlcv', return_value=ohlcv_history_list)
exchange = get_patched_exchange(mocker, default_conf)
tmpdir1 = Path(tmpdir)
file1_1 = tmpdir1 / f'{subdir}MEME_BTC-1m{file_tail}.feather'
file1_5 = tmpdir1 / f'{subdir}MEME_BTC-5m{file_tail}.feather'
file2_1 = tmpdir1 / f'{subdir}CFI_BTC-1m{file_tail}.feather'
file2_5 = tmpdir1 / f'{subdir}CFI_BTC-5m{file_tail}.feather'
file1_1 = tmp_path / f'{subdir}MEME_BTC-1m{file_tail}.feather'
file1_5 = tmp_path / f'{subdir}MEME_BTC-5m{file_tail}.feather'
file2_1 = tmp_path / f'{subdir}CFI_BTC-1m{file_tail}.feather'
file2_5 = tmp_path / f'{subdir}CFI_BTC-5m{file_tail}.feather'
assert not file1_1.is_file()
assert not file2_1.is_file()
assert _download_pair_history(datadir=tmpdir1, exchange=exchange,
assert _download_pair_history(datadir=tmp_path, exchange=exchange,
pair='MEME/BTC',
timeframe='1m',
candle_type=candle_type)
assert _download_pair_history(datadir=tmpdir1, exchange=exchange,
assert _download_pair_history(datadir=tmp_path, exchange=exchange,
pair='CFI/BTC',
timeframe='1m',
candle_type=candle_type)
@@ -308,11 +306,11 @@ def test_download_pair_history(
assert not file1_5.is_file()
assert not file2_5.is_file()
assert _download_pair_history(datadir=tmpdir1, exchange=exchange,
assert _download_pair_history(datadir=tmp_path, exchange=exchange,
pair='MEME/BTC',
timeframe='5m',
candle_type=candle_type)
assert _download_pair_history(datadir=tmpdir1, exchange=exchange,
assert _download_pair_history(datadir=tmp_path, exchange=exchange,
pair='CFI/BTC',
timeframe='5m',
candle_type=candle_type)
@@ -340,13 +338,12 @@ def test_download_pair_history2(mocker, default_conf, testdatadir) -> None:
assert json_dump_mock.call_count == 3
def test_download_backtesting_data_exception(mocker, caplog, default_conf, tmpdir) -> None:
def test_download_backtesting_data_exception(mocker, caplog, default_conf, tmp_path) -> None:
mocker.patch(f'{EXMS}.get_historic_ohlcv',
side_effect=Exception('File Error'))
tmpdir1 = Path(tmpdir)
exchange = get_patched_exchange(mocker, default_conf)
assert not _download_pair_history(datadir=tmpdir1, exchange=exchange,
assert not _download_pair_history(datadir=tmp_path, exchange=exchange,
pair='MEME/BTC',
timeframe='1m', candle_type='spot')
assert log_has('Failed to download history data for pair: "MEME/BTC", timeframe: 1m.', caplog)
@@ -570,16 +567,15 @@ def test_refresh_backtest_trades_data(mocker, default_conf, markets, caplog, tes
def test_download_trades_history(trades_history, mocker, default_conf, testdatadir, caplog,
tmpdir, time_machine) -> None:
tmp_path, time_machine) -> None:
start_dt = dt_utc(2023, 1, 1)
time_machine.move_to(start_dt, tick=False)
tmpdir1 = Path(tmpdir)
ght_mock = MagicMock(side_effect=lambda pair, *args, **kwargs: (pair, trades_history))
mocker.patch(f'{EXMS}.get_historic_trades', ght_mock)
exchange = get_patched_exchange(mocker, default_conf)
file1 = tmpdir1 / 'ETH_BTC-trades.json.gz'
data_handler = get_datahandler(tmpdir1, data_format='jsongz')
file1 = tmp_path / 'ETH_BTC-trades.json.gz'
data_handler = get_datahandler(tmp_path, data_format='jsongz')
assert not file1.is_file()
@@ -614,7 +610,7 @@ def test_download_trades_history(trades_history, mocker, default_conf, testdatad
pair='ETH/BTC')
assert log_has_re('Failed to download historic trades for pair: "ETH/BTC".*', caplog)
file2 = tmpdir1 / 'XRP_ETH-trades.json.gz'
file2 = tmp_path / 'XRP_ETH-trades.json.gz'
copyfile(testdatadir / file2.name, file2)
ght_mock.reset_mock()

12
tests/data/test_trade_converter_kraken.py
View File

@@ -1,5 +1,4 @@
from datetime import datetime, timezone
from pathlib import Path
from shutil import copytree
from unittest.mock import PropertyMock
@@ -11,7 +10,7 @@ from freqtrade.exceptions import OperationalException
from tests.conftest import EXMS, log_has, log_has_re, patch_exchange
def test_import_kraken_trades_from_csv(testdatadir, tmpdir, caplog, default_conf_usdt, mocker):
def test_import_kraken_trades_from_csv(testdatadir, tmp_path, caplog, default_conf_usdt, mocker):
with pytest.raises(OperationalException, match="This function is only for the kraken exchange"):
import_kraken_trades_from_csv(default_conf_usdt, 'feather')
@@ -21,10 +20,9 @@ def test_import_kraken_trades_from_csv(testdatadir, tmpdir, caplog, default_conf
mocker.patch(f'{EXMS}.markets', PropertyMock(return_value={
'BCH/EUR': {'symbol': 'BCH/EUR', 'id': 'BCHEUR', 'altname': 'BCHEUR'},
}))
tmpdir1 = Path(tmpdir)
dstfile = tmpdir1 / 'BCH_EUR-trades.feather'
dstfile = tmp_path / 'BCH_EUR-trades.feather'
assert not dstfile.is_file()
default_conf_usdt['datadir'] = tmpdir1
default_conf_usdt['datadir'] = tmp_path
# There's 2 files in this tree, containing a total of 2 days.
# tests/testdata/kraken/
# └── trades_csv
@@ -32,7 +30,7 @@ def test_import_kraken_trades_from_csv(testdatadir, tmpdir, caplog, default_conf
# └── incremental_q2
# └── BCHEUR.csv <-- 2023-01-02
copytree(testdatadir / 'kraken/trades_csv', tmpdir1 / 'trades_csv')
copytree(testdatadir / 'kraken/trades_csv', tmp_path / 'trades_csv')
import_kraken_trades_from_csv(default_conf_usdt, 'feather')
assert log_has("Found csv files for BCHEUR.", caplog)
@@ -40,7 +38,7 @@ def test_import_kraken_trades_from_csv(testdatadir, tmpdir, caplog, default_conf
assert dstfile.is_file()
dh = get_datahandler(tmpdir1, 'feather')
dh = get_datahandler(tmp_path, 'feather')
trades = dh.trades_load('BCH_EUR')
assert len(trades) == 340