ruff format: orderflow / public trades

This commit is contained in:
Joe Schr
2024-05-15 17:09:32 +02:00
parent 6c696e14f0
commit a56faf503b
13 changed files with 420 additions and 314 deletions

View File

@@ -11,49 +11,68 @@ from freqtrade.data.converter.trade_converter import trades_list_to_df
BIN_SIZE_SCALE = 0.5
def read_csv(filename, converter_columns: list = ['side', 'type']):
return pd.read_csv(filename, skipinitialspace=True, infer_datetime_format=True, index_col=0,
parse_dates=True, converters={col: str.strip for col in converter_columns})
def read_csv(filename, converter_columns: list = ["side", "type"]):
return pd.read_csv(
filename,
skipinitialspace=True,
infer_datetime_format=True,
index_col=0,
parse_dates=True,
converters={col: str.strip for col in converter_columns},
)
@pytest.fixture
def populate_dataframe_with_trades_dataframe(testdatadir):
return pd.read_feather(testdatadir / 'orderflow/populate_dataframe_with_trades_DF.feather')
return pd.read_feather(testdatadir / "orderflow/populate_dataframe_with_trades_DF.feather")
@pytest.fixture
def populate_dataframe_with_trades_trades(testdatadir):
return pd.read_feather(testdatadir / 'orderflow/populate_dataframe_with_trades_TRADES.feather')
return pd.read_feather(testdatadir / "orderflow/populate_dataframe_with_trades_TRADES.feather")
@pytest.fixture
def candles(testdatadir):
return pd.read_json(testdatadir / 'orderflow/candles.json').copy()
return pd.read_json(testdatadir / "orderflow/candles.json").copy()
@pytest.fixture
def public_trades_list(testdatadir):
return read_csv(testdatadir / 'orderflow/public_trades_list.csv').copy()
return read_csv(testdatadir / "orderflow/public_trades_list.csv").copy()
@pytest.fixture
def public_trades_list_simple(testdatadir):
return read_csv(testdatadir / 'orderflow/public_trades_list_simple_example.csv').copy()
return read_csv(testdatadir / "orderflow/public_trades_list_simple_example.csv").copy()
def test_public_trades_columns_before_change(
populate_dataframe_with_trades_dataframe,
populate_dataframe_with_trades_trades):
populate_dataframe_with_trades_dataframe, populate_dataframe_with_trades_trades
):
assert populate_dataframe_with_trades_dataframe.columns.tolist() == [
'date', 'open', 'high', 'low', 'close', 'volume']
"date",
"open",
"high",
"low",
"close",
"volume",
]
assert populate_dataframe_with_trades_trades.columns.tolist() == [
'timestamp', 'id', 'type', 'side', 'price',
'amount', 'cost', 'date']
"timestamp",
"id",
"type",
"side",
"price",
"amount",
"cost",
"date",
]
def test_public_trades_mock_populate_dataframe_with_trades__check_orderflow(
populate_dataframe_with_trades_dataframe,
populate_dataframe_with_trades_trades):
populate_dataframe_with_trades_dataframe, populate_dataframe_with_trades_trades
):
"""
Tests the `populate_dataframe_with_trades` function's order flow calculation.
@@ -64,24 +83,25 @@ def test_public_trades_mock_populate_dataframe_with_trades__check_orderflow(
dataframe = populate_dataframe_with_trades_dataframe.copy()
trades = populate_dataframe_with_trades_trades.copy()
# Convert the 'date' column to datetime format with milliseconds
dataframe['date'] = pd.to_datetime(
dataframe['date'], unit='ms')
dataframe["date"] = pd.to_datetime(dataframe["date"], unit="ms")
# Select the last rows and reset the index (optional, depends on usage)
dataframe = dataframe.copy().tail().reset_index(drop=True)
# Define the configuration for order flow calculation
config = {'timeframe': '5m',
'orderflow': {
'scale': 0.005,
'imbalance_volume': 0,
'imbalance_ratio': 300,
'stacked_imbalance_range': 3
}}
config = {
"timeframe": "5m",
"orderflow": {
"scale": 0.005,
"imbalance_volume": 0,
"imbalance_ratio": 300,
"stacked_imbalance_range": 3,
},
}
# Apply the function to populate the data frame with order flow data
df = populate_dataframe_with_trades(config, dataframe, trades)
# Extract results from the first row of the DataFrame
results = df.iloc[0]
t = results['trades']
of = results['orderflow']
t = results["trades"]
of = results["orderflow"]
# Assert basic properties of the results
assert 0 != len(results)
@@ -98,7 +118,7 @@ def test_public_trades_mock_populate_dataframe_with_trades__check_orderflow(
assert [0.0, 1.0, 0.103, 0.0, 0.103, 0.103, 1.0] == of.iloc[-1].values.tolist()
# Extract order flow from the last row of the DataFrame
of = df.iloc[-1]['orderflow']
of = df.iloc[-1]["orderflow"]
# Assert number of order flow data points in the last row
assert 19 == len(of) # Assert expected number of data points
@@ -107,42 +127,49 @@ def test_public_trades_mock_populate_dataframe_with_trades__check_orderflow(
assert [1.0, 0.0, -12.536, 12.536, 0.0, 12.536, 1.0] == of.iloc[0].values.tolist()
# Assert specific order flow values at the end of the last row
assert [4.0, 3.0, -40.94800000000001, 59.18200000000001,
18.233999999999998, 77.41600000000001, 7.0] == of.iloc[-1].values.tolist()
assert [
4.0,
3.0,
-40.94800000000001,
59.18200000000001,
18.233999999999998,
77.41600000000001,
7.0,
] == of.iloc[-1].values.tolist()
# --- Delta and Other Results ---
# Assert delta value from the first row
assert -50.519000000000005 == results['delta']
assert -50.519000000000005 == results["delta"]
# Assert min and max delta values from the first row
assert -79.469 == results['min_delta']
assert 17.298 == results['max_delta']
assert -79.469 == results["min_delta"]
assert 17.298 == results["max_delta"]
# Assert that stacked imbalances are NaN (not applicable in this test)
assert np.isnan(results['stacked_imbalances_bid'])
assert np.isnan(results['stacked_imbalances_ask'])
assert np.isnan(results["stacked_imbalances_bid"])
assert np.isnan(results["stacked_imbalances_ask"])
# Repeat assertions for the third from last row
results = df.iloc[-2]
assert -20.86200000000008 == results['delta']
assert -54.55999999999999 == results['min_delta']
assert 82.842 == results['max_delta']
assert 234.99 == results['stacked_imbalances_bid']
assert 234.96 == results['stacked_imbalances_ask']
assert -20.86200000000008 == results["delta"]
assert -54.55999999999999 == results["min_delta"]
assert 82.842 == results["max_delta"]
assert 234.99 == results["stacked_imbalances_bid"]
assert 234.96 == results["stacked_imbalances_ask"]
# Repeat assertions for the last row
results = df.iloc[-1]
assert -49.30200000000002 == results['delta']
assert -70.222 == results['min_delta']
assert 11.213000000000003 == results['max_delta']
assert np.isnan(results['stacked_imbalances_bid'])
assert np.isnan(results['stacked_imbalances_ask'])
assert -49.30200000000002 == results["delta"]
assert -70.222 == results["min_delta"]
assert 11.213000000000003 == results["max_delta"]
assert np.isnan(results["stacked_imbalances_bid"])
assert np.isnan(results["stacked_imbalances_ask"])
def test_public_trades_trades_mock_populate_dataframe_with_trades__check_trades(
populate_dataframe_with_trades_dataframe,
populate_dataframe_with_trades_trades):
populate_dataframe_with_trades_dataframe, populate_dataframe_with_trades_trades
):
"""
Tests the `populate_dataframe_with_trades` function's handling of trades,
ensuring correct integration of trades data into the generated DataFrame.
@@ -155,7 +182,7 @@ def test_public_trades_trades_mock_populate_dataframe_with_trades__check_trades(
# --- Data Preparation ---
# Convert the 'date' column to datetime format with milliseconds
dataframe['date'] = pd.to_datetime(dataframe['date'], unit='ms')
dataframe["date"] = pd.to_datetime(dataframe["date"], unit="ms")
# Select the final row of the DataFrame
dataframe = dataframe.tail().reset_index(drop=True)
@@ -165,19 +192,19 @@ def test_public_trades_trades_mock_populate_dataframe_with_trades__check_trades(
trades.reset_index(inplace=True, drop=True) # Reset index for clarity
# Assert the first trade ID to ensure filtering worked correctly
assert trades['id'][0] == '313881442'
assert trades["id"][0] == "313881442"
# --- Configuration and Function Call ---
# Define configuration for order flow calculation (used for context)
config = {
'timeframe': '5m',
'orderflow': {
'scale': 0.5,
'imbalance_volume': 0,
'imbalance_ratio': 300,
'stacked_imbalance_range': 3
}
"timeframe": "5m",
"orderflow": {
"scale": 0.5,
"imbalance_volume": 0,
"imbalance_ratio": 300,
"stacked_imbalance_range": 3,
},
}
# Populate the DataFrame with trades and order flow data
@@ -190,28 +217,38 @@ def test_public_trades_trades_mock_populate_dataframe_with_trades__check_trades(
# Assert DataFrame structure
assert list(df.columns) == [
# ... (list of expected column names)
'date', 'open', 'high', 'low',
'close', 'volume', 'trades', 'orderflow',
'bid', 'ask', 'delta', 'min_delta',
'max_delta', 'total_trades',
'stacked_imbalances_bid',
'stacked_imbalances_ask'
"date",
"open",
"high",
"low",
"close",
"volume",
"trades",
"orderflow",
"bid",
"ask",
"delta",
"min_delta",
"max_delta",
"total_trades",
"stacked_imbalances_bid",
"stacked_imbalances_ask",
]
# Assert delta, bid, and ask values
assert -50.519 == pytest.approx(row['delta'])
assert 219.961 == row['bid']
assert 169.442 == row['ask']
assert -50.519 == pytest.approx(row["delta"])
assert 219.961 == row["bid"]
assert 169.442 == row["ask"]
# Assert the number of trades
assert 151 == len(row.trades)
# Assert specific details of the first trade
t = row['trades'].iloc[0]
assert trades['id'][0] == t["id"]
assert int(trades['timestamp'][0]) == int(t['timestamp'])
assert 'sell' == t['side']
assert '313881442' == t['id']
assert 234.72 == t['price']
t = row["trades"].iloc[0]
assert trades["id"][0] == t["id"]
assert int(trades["timestamp"][0]) == int(t["timestamp"])
assert "sell" == t["side"]
assert "313881442" == t["id"]
assert 234.72 == t["price"]
def test_public_trades_put_volume_profile_into_ohlcv_candles(public_trades_list_simple, candles):
@@ -232,18 +269,19 @@ def test_public_trades_put_volume_profile_into_ohlcv_candles(public_trades_list_
df = trades_to_volumeprofile_with_total_delta_bid_ask(df, scale=BIN_SIZE_SCALE)
# Initialize the 'vp' column in the candles DataFrame with NaNs
candles['vp'] = np.nan
candles["vp"] = np.nan
# Select the second candle (index 1) and attempt to assign the volume profile data
# (as a DataFrame) to the 'vp' element.
candles.loc[candles.index == 1, ['vp']] = candles.loc[candles.index == 1, [
'vp']].applymap(lambda x: pd.DataFrame(df.to_dict()))
candles.loc[candles.index == 1, ["vp"]] = candles.loc[candles.index == 1, ["vp"]].applymap(
lambda x: pd.DataFrame(df.to_dict())
)
# Assert the delta value in the 'vp' element of the second candle
assert 0.14 == candles['vp'][1].values.tolist()[1][2]
assert 0.14 == candles["vp"][1].values.tolist()[1][2]
# Alternative assertion using `.iat` accessor (assuming correct assignment logic)
assert 0.14 == candles['vp'][1]['delta'].iat[1]
assert 0.14 == candles["vp"][1]["delta"].iat[1]
def test_public_trades_binned_big_sample_list(public_trades_list):
@@ -262,8 +300,15 @@ def test_public_trades_binned_big_sample_list(public_trades_list):
df = trades_to_volumeprofile_with_total_delta_bid_ask(trades, scale=BIN_SIZE_SCALE)
# Assert that the DataFrame has the expected columns
assert df.columns.tolist() == ['bid', 'ask', 'delta', 'bid_amount',
'ask_amount', 'total_volume', 'total_trades']
assert df.columns.tolist() == [
"bid",
"ask",
"delta",
"bid_amount",
"ask_amount",
"total_volume",
"total_trades",
]
# Assert the number of rows in the DataFrame (expected 23 for this bin size)
assert len(df) == 23
@@ -271,22 +316,22 @@ def test_public_trades_binned_big_sample_list(public_trades_list):
# Assert that the index values are in ascending order and spaced correctly
assert all(df.index[i] < df.index[i + 1] for i in range(len(df) - 1))
assert df.index[0] + BIN_SIZE_SCALE == df.index[1]
assert (trades['price'].min() - BIN_SIZE_SCALE) < df.index[0] < trades['price'].max()
assert (trades["price"].min() - BIN_SIZE_SCALE) < df.index[0] < trades["price"].max()
assert (df.index[0] + BIN_SIZE_SCALE) >= df.index[1]
assert (trades['price'].max() - BIN_SIZE_SCALE) < df.index[-1] < trades['price'].max()
assert (trades["price"].max() - BIN_SIZE_SCALE) < df.index[-1] < trades["price"].max()
# Assert specific values in the first and last rows of the DataFrame
assert 32 == df['bid'].iloc[0] # bid price
assert 197.512 == df['bid_amount'].iloc[0] # total bid amount
assert 88.98 == df['ask_amount'].iloc[0] # total ask amount
assert 26 == df['ask'].iloc[0] # ask price
assert -108.532 == pytest.approx(df['delta'].iloc[0]) # delta (bid amount - ask amount)
assert 32 == df["bid"].iloc[0] # bid price
assert 197.512 == df["bid_amount"].iloc[0] # total bid amount
assert 88.98 == df["ask_amount"].iloc[0] # total ask amount
assert 26 == df["ask"].iloc[0] # ask price
assert -108.532 == pytest.approx(df["delta"].iloc[0]) # delta (bid amount - ask amount)
assert 3 == df['bid'].iloc[-1] # bid price
assert 50.659 == df['bid_amount'].iloc[-1] # total bid amount
assert 108.21 == df['ask_amount'].iloc[-1] # total ask amount
assert 44 == df['ask'].iloc[-1] # ask price
assert 57.551 == df['delta'].iloc[-1] # delta (bid amount - ask amount)
assert 3 == df["bid"].iloc[-1] # bid price
assert 50.659 == df["bid_amount"].iloc[-1] # total bid amount
assert 108.21 == df["ask_amount"].iloc[-1] # total ask amount
assert 44 == df["ask"].iloc[-1] # ask price
assert 57.551 == df["delta"].iloc[-1] # delta (bid amount - ask amount)
# Repeat the process with a larger bin size
BIN_SIZE_SCALE = 1
@@ -299,43 +344,89 @@ def test_public_trades_binned_big_sample_list(public_trades_list):
# Repeat similar assertions for index ordering and spacing
assert all(df.index[i] < df.index[i + 1] for i in range(len(df) - 1))
assert (trades['price'].min() - BIN_SIZE_SCALE) < df.index[0] < trades['price'].max()
assert (trades["price"].min() - BIN_SIZE_SCALE) < df.index[0] < trades["price"].max()
assert (df.index[0] + BIN_SIZE_SCALE) >= df.index[1]
assert (trades['price'].max() - BIN_SIZE_SCALE) < df.index[-1] < trades['price'].max()
assert (trades["price"].max() - BIN_SIZE_SCALE) < df.index[-1] < trades["price"].max()
# Assert the value in the last row of the DataFrame with the larger bin size
assert 1667.0 == df.index[-1]
assert 710.98 == df['bid_amount'].iat[0]
assert 111 == df['bid'].iat[0]
assert 52.7199999 == pytest.approx(df['delta'].iat[0]) # delta
assert 710.98 == df["bid_amount"].iat[0]
assert 111 == df["bid"].iat[0]
assert 52.7199999 == pytest.approx(df["delta"].iat[0]) # delta
def test_public_trades_testdata_sanity(
candles,
public_trades_list,
public_trades_list_simple,
populate_dataframe_with_trades_dataframe,
populate_dataframe_with_trades_trades):
candles,
public_trades_list,
public_trades_list_simple,
populate_dataframe_with_trades_dataframe,
populate_dataframe_with_trades_trades,
):
assert 10999 == len(candles)
assert 1000 == len(public_trades_list)
assert 999 == len(populate_dataframe_with_trades_dataframe)
assert 293532 == len(populate_dataframe_with_trades_trades)
assert 7 == len(public_trades_list_simple)
assert 5 == public_trades_list_simple.loc[
public_trades_list_simple['side'].str.contains('sell'), 'id'].count()
assert 2 == public_trades_list_simple.loc[
public_trades_list_simple['side'].str.contains('buy'), 'id'].count()
assert (
5
== public_trades_list_simple.loc[
public_trades_list_simple["side"].str.contains("sell"), "id"
].count()
)
assert (
2
== public_trades_list_simple.loc[
public_trades_list_simple["side"].str.contains("buy"), "id"
].count()
)
assert public_trades_list.columns.tolist() == [
'timestamp', 'id', 'type', 'side', 'price',
'amount', 'cost', 'date']
"timestamp",
"id",
"type",
"side",
"price",
"amount",
"cost",
"date",
]
assert public_trades_list.columns.tolist() == [
'timestamp', 'id', 'type', 'side', 'price', 'amount', 'cost', 'date']
"timestamp",
"id",
"type",
"side",
"price",
"amount",
"cost",
"date",
]
assert public_trades_list_simple.columns.tolist() == [
'timestamp', 'id', 'type', 'side', 'price', 'amount', 'cost', 'date']
"timestamp",
"id",
"type",
"side",
"price",
"amount",
"cost",
"date",
]
assert populate_dataframe_with_trades_dataframe.columns.tolist() == [
'date', 'open', 'high', 'low', 'close', 'volume']
"date",
"open",
"high",
"low",
"close",
"volume",
]
assert populate_dataframe_with_trades_trades.columns.tolist() == [
'timestamp', 'id', 'type', 'side', 'price', 'amount', 'cost', 'date']
"timestamp",
"id",
"type",
"side",
"price",
"amount",
"cost",
"date",
]

View File

@@ -66,7 +66,7 @@ def test_historic_trades(mocker, default_conf, trades_history_df):
historymock = MagicMock(return_value=trades_history_df)
mocker.patch(
"freqtrade.data.history.datahandlers.featherdatahandler.FeatherDataHandler._trades_load",
historymock
historymock,
)
dp = DataProvider(default_conf, None)
@@ -82,8 +82,8 @@ def test_historic_trades(mocker, default_conf, trades_history_df):
assert len(data) == 0
# Switch to backtest mode
default_conf['runmode'] = RunMode.BACKTEST
default_conf['dataformat_trades'] = 'feather'
default_conf["runmode"] = RunMode.BACKTEST
default_conf["dataformat_trades"] = "feather"
exchange = get_patched_exchange(mocker, default_conf)
dp = DataProvider(default_conf, exchange)
data = dp.trades("UNITTEST/BTC", "5m")
@@ -91,7 +91,7 @@ def test_historic_trades(mocker, default_conf, trades_history_df):
assert len(data) == len(trades_history_df)
# Random other runmode
default_conf['runmode'] = RunMode.UTIL_EXCHANGE
default_conf["runmode"] = RunMode.UTIL_EXCHANGE
dp = DataProvider(default_conf, None)
data = dp.trades("UNITTEST/BTC", "5m")
assert isinstance(data, DataFrame)
@@ -311,7 +311,7 @@ def test_refresh(mocker, default_conf):
# Test with public trades
refresh_mock.reset_mock()
refresh_mock.reset_mock()
default_conf['exchange']['use_public_trades'] = True
default_conf["exchange"]["use_public_trades"] = True
dp.refresh(pairs, pairs_non_trad)
assert mock_refresh_trades.call_count == 1
assert refresh_mock.call_count == 1

View File

@@ -674,8 +674,9 @@ def test_download_trades_history(
mocker.patch(f"{EXMS}.get_historic_trades", MagicMock(side_effect=ValueError))
caplog.clear()
assert not _download_trades_history(data_handler=data_handler, exchange=exchange,
pair='ETH/BTC', trading_mode=TradingMode.SPOT)
assert not _download_trades_history(
data_handler=data_handler, exchange=exchange, pair="ETH/BTC", trading_mode=TradingMode.SPOT
)
assert log_has_re('Failed to download and store historic trades for pair: "ETH/BTC".*', caplog)
file2 = tmp_path / "XRP_ETH-trades.json.gz"