From 10917a280a2a72944e3425bc18b4ff2d04570331 Mon Sep 17 00:00:00 2001 From: eSeR1805 Date: Tue, 31 May 2022 12:26:07 +0300 Subject: [PATCH 01/46] Add initial structure and wrapping. --- freqtrade/optimize/backtesting.py | 5 +- freqtrade/persistence/__init__.py | 1 + freqtrade/persistence/keyvalue.py | 57 ++++++++++ freqtrade/persistence/keyvalue_middleware.py | 108 +++++++++++++++++++ freqtrade/persistence/models.py | 2 + freqtrade/persistence/trade_model.py | 19 ++++ 6 files changed, 191 insertions(+), 1 deletion(-) create mode 100644 freqtrade/persistence/keyvalue.py create mode 100644 freqtrade/persistence/keyvalue_middleware.py diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index 4e604898f..c552d8790 100755 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -30,7 +30,7 @@ from freqtrade.optimize.bt_progress import BTProgress from freqtrade.optimize.optimize_reports import (generate_backtest_stats, show_backtest_results, store_backtest_signal_candles, store_backtest_stats) -from freqtrade.persistence import LocalTrade, Order, PairLocks, Trade +from freqtrade.persistence import KeyValues, LocalTrade, Order, PairLocks, Trade from freqtrade.plugins.pairlistmanager import PairListManager from freqtrade.plugins.protectionmanager import ProtectionManager from freqtrade.resolvers import ExchangeResolver, StrategyResolver @@ -151,6 +151,7 @@ class Backtesting: LoggingMixin.show_output = True PairLocks.use_db = True Trade.use_db = True + KeyValues.use_db = True # ??? def init_backtest_detail(self): # Load detail timeframe if specified @@ -294,6 +295,8 @@ class Backtesting: Trade.use_db = False PairLocks.reset_locks() Trade.reset_trades() + KeyValues.use_db = False + KeyValues.reset_keyvalues() self.rejected_trades = 0 self.timedout_entry_orders = 0 self.timedout_exit_orders = 0 diff --git a/freqtrade/persistence/__init__.py b/freqtrade/persistence/__init__.py index ab6e2f6a5..0158f588c 100644 --- a/freqtrade/persistence/__init__.py +++ b/freqtrade/persistence/__init__.py @@ -1,5 +1,6 @@ # flake8: noqa: F401 +from freqtrade.persistence.keyvalue_middleware import KeyValues from freqtrade.persistence.models import clean_dry_run_db, cleanup_db, init_db from freqtrade.persistence.pairlock_middleware import PairLocks from freqtrade.persistence.trade_model import LocalTrade, Order, Trade diff --git a/freqtrade/persistence/keyvalue.py b/freqtrade/persistence/keyvalue.py new file mode 100644 index 000000000..60fa903a1 --- /dev/null +++ b/freqtrade/persistence/keyvalue.py @@ -0,0 +1,57 @@ +from datetime import datetime +from typing import Optional + +from sqlalchemy import Column, DateTime, ForeignKey, Integer, String, Text, UniqueConstraint +from sqlalchemy.orm import Query, relationship + +from freqtrade.constants import DATETIME_PRINT_FORMAT +from freqtrade.persistence.base import _DECL_BASE + + +class KeyValue(_DECL_BASE): + """ + KeyValue database model + Keeps records of metadata as key/value store + for trades or global persistant values + One to many relationship with Trades: + - One trade can have many metadata entries + - One metadata entry can only be associated with one Trade + """ + __tablename__ = 'keyvalue' + # Uniqueness should be ensured over pair, order_id + # its likely that order_id is unique per Pair on some exchanges. + __table_args__ = (UniqueConstraint('ft_trade_id', 'kv_key', name="_trade_id_kv_key"),) + + id = Column(Integer, primary_key=True) + ft_trade_id = Column(Integer, ForeignKey('trades.id'), index=True, default=0) + + trade = relationship("Trade", back_populates="keyvalues") + + kv_key = Column(String(255), nullable=False) + kv_type = Column(String(25), nullable=False) + kv_value = Column(Text, nullable=False) + created_at = Column(DateTime, nullable=False, default=datetime.utcnow) + updated_at = Column(DateTime, nullable=True) + + def __repr__(self): + create_time = (self.created_at.strftime(DATETIME_PRINT_FORMAT) + if self.created_at is not None else None) + update_time = (self.updated_at.strftime(DATETIME_PRINT_FORMAT) + if self.updated_at is not None else None) + return (f'KeyValue(id={self.id}, key={self.kv_key}, type={self.kv_type}, ', + f'value={self.kv_value}, trade_id={self.ft_trade_id}, created={create_time}, ', + f'updated={update_time})') + + @staticmethod + def query_kv(key: Optional[str] = None, trade_id: Optional[int] = None) -> Query: + """ + Get all keyvalues, if trade_id is not specified + return will be for generic values not tied to a trade + :param trade_id: id of the Trade + """ + key = key if key is not None else "%" + + filters = [KeyValue.ft_trade_id == trade_id if trade_id is not None else 0, + KeyValue.kv_key.ilike(key)] + + return KeyValue.query.filter(*filters) diff --git a/freqtrade/persistence/keyvalue_middleware.py b/freqtrade/persistence/keyvalue_middleware.py new file mode 100644 index 000000000..24c74610a --- /dev/null +++ b/freqtrade/persistence/keyvalue_middleware.py @@ -0,0 +1,108 @@ +import json +import logging +from datetime import datetime +from typing import Any, List, Optional + +from freqtrade.persistence.keyvalue import KeyValue + + +logger = logging.getLogger(__name__) + + +class KeyValues(): + """ + KeyValues middleware class + Abstracts the database layer away so it becomes optional - which will be necessary to support + backtesting and hyperopt in the future. + """ + + use_db = True + kvals: List[KeyValue] = [] + unserialized_types = ['bool', 'float', 'int', 'str'] + + @staticmethod + def reset_keyvalues() -> None: + """ + Resets all key-value pairs. Only active for backtesting mode. + """ + if not KeyValues.use_db: + KeyValues.kvals = [] + + @staticmethod + def get_kval(key: Optional[str] = None, trade_id: Optional[int] = None) -> List[KeyValue]: + if trade_id is None: + trade_id = 0 + + if KeyValues.use_db: + filtered_kvals = KeyValue.query_kv(trade_id=trade_id, key=key).all() + for index, kval in enumerate(filtered_kvals): + if kval.kv_type not in KeyValues.unserialized_types: + kval.kv_value = json.loads(kval.kv_value) + filtered_kvals[index] = kval + return filtered_kvals + else: + filtered_kvals = [kval for kval in KeyValues.kvals if (kval.ft_trade_id == trade_id)] + if key is not None: + filtered_kvals = [ + kval for kval in filtered_kvals if (kval.kv_key.casefold() == key.casefold())] + return filtered_kvals + + @staticmethod + def set_kval(key: str, value: Any, trade_id: Optional[int] = None) -> None: + + logger.warning(f"[set_kval] key: {key} trade_id: {trade_id} value: {value}") + value_type = type(value).__name__ + value_db = None + + if value_type not in KeyValues.unserialized_types: + try: + value_db = json.dumps(value) + except TypeError as e: + logger.warning(f"could not serialize {key} value due to {e}") + else: + value_db = str(value) + + if trade_id is None: + trade_id = 0 + + kvals = KeyValues.get_kval(key=key, trade_id=trade_id) + if kvals: + kv = kvals[0] + kv.kv_value = value + kv.updated_at = datetime.utcnow() + else: + kv = KeyValue( + ft_trade_id=trade_id, + kv_key=key, + kv_type=value_type, + kv_value=value, + created_at=datetime.utcnow() + ) + + if KeyValues.use_db and value_db is not None: + kv.kv_value = value_db + KeyValue.query.session.add(kv) + KeyValue.query.session.commit() + elif not KeyValues.use_db: + kv_index = -1 + for index, kval in enumerate(KeyValues.kvals): + if kval.ft_trade_id == trade_id and kval.kv_key == key: + kv_index = index + break + + if kv_index >= 0: + kval.kv_type = value_type + kval.value = value + kval.updated_at = datetime.utcnow() + + KeyValues.kvals[kv_index] = kval + else: + KeyValues.kvals.append(kv) + + @staticmethod + def get_all_kvals() -> List[KeyValue]: + + if KeyValues.use_db: + return KeyValue.query.all() + else: + return KeyValues.kvals diff --git a/freqtrade/persistence/models.py b/freqtrade/persistence/models.py index c31e50892..6a279ab5c 100644 --- a/freqtrade/persistence/models.py +++ b/freqtrade/persistence/models.py @@ -10,6 +10,7 @@ from sqlalchemy.pool import StaticPool from freqtrade.exceptions import OperationalException from freqtrade.persistence.base import _DECL_BASE +from freqtrade.persistence.keyvalue import KeyValue from freqtrade.persistence.migrations import check_migrate from freqtrade.persistence.pairlock import PairLock from freqtrade.persistence.trade_model import Order, Trade @@ -59,6 +60,7 @@ def init_db(db_url: str, clean_open_orders: bool = False) -> None: Trade.query = Trade._session.query_property() Order.query = Trade._session.query_property() PairLock.query = Trade._session.query_property() + KeyValue.query = Trade._session.query_property() previous_tables = inspect(engine).get_table_names() _DECL_BASE.metadata.create_all(engine) diff --git a/freqtrade/persistence/trade_model.py b/freqtrade/persistence/trade_model.py index 358e776e3..b097f6574 100644 --- a/freqtrade/persistence/trade_model.py +++ b/freqtrade/persistence/trade_model.py @@ -15,6 +15,8 @@ from freqtrade.enums import ExitType, TradingMode from freqtrade.exceptions import DependencyException, OperationalException from freqtrade.leverage import interest from freqtrade.persistence.base import _DECL_BASE +from freqtrade.persistence.keyvalue import KeyValue +from freqtrade.persistence.keyvalue_middleware import KeyValues logger = logging.getLogger(__name__) @@ -206,6 +208,7 @@ class LocalTrade(): id: int = 0 orders: List[Order] = [] + keyvalues: List[KeyValue] = [] exchange: str = '' pair: str = '' @@ -870,6 +873,12 @@ class LocalTrade(): (o.filled or 0) > 0 and o.status in NON_OPEN_EXCHANGE_STATES] + def set_kval(self, key: str, value: Any) -> None: + KeyValues.set_kval(key=key, value=value, trade_id=self.id) + + def get_kval(self, key: Optional[str]) -> List[KeyValue]: + return KeyValues.get_kval(key=key, trade_id=self.id) + @property def nr_of_successful_entries(self) -> int: """ @@ -1000,6 +1009,7 @@ class Trade(_DECL_BASE, LocalTrade): id = Column(Integer, primary_key=True) orders = relationship("Order", order_by="Order.id", cascade="all, delete-orphan", lazy="joined") + keyvalues = relationship("KeyValue", order_by="KeyValue.id", cascade="all, delete-orphan") exchange = Column(String(25), nullable=False) pair = Column(String(25), nullable=False, index=True) @@ -1070,6 +1080,9 @@ class Trade(_DECL_BASE, LocalTrade): for order in self.orders: Order.query.session.delete(order) + for kval in self.keyvalues: + KeyValue.query.session.delete(kval) + Trade.query.session.delete(self) Trade.commit() @@ -1345,3 +1358,9 @@ class Trade(_DECL_BASE, LocalTrade): .group_by(Trade.pair) \ .order_by(desc('profit_sum')).first() return best_pair + + def set_kval(self, key: str, value: Any) -> None: + super().set_kval(key=key, value=value) + + def get_kval(self, key: Optional[str]) -> List[KeyValue]: + return super().get_kval(key=key) From 096e98a68c1e6025da4ffc688928a6e6d27cb20f Mon Sep 17 00:00:00 2001 From: eSeR1805 Date: Tue, 31 May 2022 16:16:57 +0300 Subject: [PATCH 02/46] Remove stray debug message. --- freqtrade/persistence/keyvalue_middleware.py | 1 - 1 file changed, 1 deletion(-) diff --git a/freqtrade/persistence/keyvalue_middleware.py b/freqtrade/persistence/keyvalue_middleware.py index 24c74610a..8248143ce 100644 --- a/freqtrade/persistence/keyvalue_middleware.py +++ b/freqtrade/persistence/keyvalue_middleware.py @@ -50,7 +50,6 @@ class KeyValues(): @staticmethod def set_kval(key: str, value: Any, trade_id: Optional[int] = None) -> None: - logger.warning(f"[set_kval] key: {key} trade_id: {trade_id} value: {value}") value_type = type(value).__name__ value_db = None From de01aaf290a965e071b083ae28ce1db31d3f97bf Mon Sep 17 00:00:00 2001 From: eSeR1805 Date: Tue, 31 May 2022 16:17:31 +0300 Subject: [PATCH 03/46] Add documentation details. --- docs/strategy-advanced.md | 70 ++++++++++++++++++++++++++++++++++++++- 1 file changed, 69 insertions(+), 1 deletion(-) diff --git a/docs/strategy-advanced.md b/docs/strategy-advanced.md index 374c675a2..45961c59d 100644 --- a/docs/strategy-advanced.md +++ b/docs/strategy-advanced.md @@ -11,7 +11,7 @@ If you're just getting started, please be familiar with the methods described in !!! Tip You can get a strategy template containing all below methods by running `freqtrade new-strategy --strategy MyAwesomeStrategy --template advanced` -## Storing information +## Storing information (Non-Persistent) Storing information can be accomplished by creating a new dictionary within the strategy class. @@ -40,6 +40,74 @@ class AwesomeStrategy(IStrategy): !!! Note If the data is pair-specific, make sure to use pair as one of the keys in the dictionary. +## Storing information (Persistent) + +Storing information can also be performed in a persistent manner. Freqtrade allows storing/retrieving user custom information associated with a specific trade. +Using a trade object handle information can be stored using `trade_obj.set_kval(key='my_key', value=my_value)` and retrieved using `trade_obj.get_kval(key='my_key')`. +Each data entry is associated with a trade and a user supplied key (of type `string`). This means that this can only be used in callbacks that also provide a trade object handle. +For the data to be able to be stored within the database it must be serialized. This is done by converting it to a JSON formatted string. + +```python +from freqtrade.persistence import Trade +from datetime import timedelta + +class AwesomeStrategy(IStrategy): + + def bot_loop_start(self, **kwargs) -> None: + for trade in Trade.get_open_order_trades(): + fills = trade.select_filled_orders(trade.entry_side) + if trade.pair == 'ETH/USDT': + trade_entry_type = trade.get_kval(key='entry_type') + if trade_entry_type is None: + trade_entry_type = 'breakout' if 'entry_1' in trade.enter_tag else 'dip' + elif fills > 1: + trade_entry_type = 'buy_up' + trade.set_kval(key='entry_type', value=trade_entry_type) + return super().bot_loop_start(**kwargs) + + def adjust_entry_price(self, trade: Trade, order: Optional[Order], pair: str, + current_time: datetime, proposed_rate: float, current_order_rate: float, + entry_tag: Optional[str], side: str, **kwargs) -> float: + # Limit orders to use and follow SMA200 as price target for the first 10 minutes since entry trigger for BTC/USDT pair. + if pair == 'BTC/USDT' and entry_tag == 'long_sma200' and side == 'long' and (current_time - timedelta(minutes=10) > trade.open_date_utc and order.filled == 0.0: + dataframe, _ = self.dp.get_analyzed_dataframe(pair=pair, timeframe=self.timeframe) + current_candle = dataframe.iloc[-1].squeeze() + # store information about entry adjustment + existing_count = trade.get_kval(key='num_entry_adjustments') + if not existing_count: + existing_count = 1 + else: + existing_count += 1 + trade.set_kval(key='num_entry_adjustments', value=existing_count) + + # adjust order price + return current_candle['sma_200'] + + # default: maintain existing order + return current_order_rate + + def custom_exit(self, pair: str, trade: Trade, current_time: datetime, current_rate: float, current_profit: float, **kwargs): + + entry_adjustment_count = trade.get_kval(key='num_entry_adjustments') + trade_entry_type = trade.get_kval(key='entry_type') + if entry_adjustment_count is None: + if current_profit > 0.01 and (current_time - timedelta(minutes=100) > trade.open_date_utc): + return True, 'exit_1' + else + if entry_adjustment_count > 0 and if current_profit > 0.05: + return True, 'exit_2' + if trade_entry_type == 'breakout' and current_profit > 0.1: + return True, 'exit_3 + + return False, None +``` + +!!! Note + It is recommended that simple data types are used `[bool, int, float, str]` to ensure no issues when serializing the data that needs to be stored. + +!!! Warning + If supplied data cannot be serialized a warning is logged and the entry for the specified `key` will contain `None` as data. + ## Dataframe access You may access dataframe in various strategy functions by querying it from dataprovider. From abda02572b4248391dc2a9f4b9bc98f095735beb Mon Sep 17 00:00:00 2001 From: eSeR1805 Date: Sun, 5 Jun 2022 12:18:07 +0300 Subject: [PATCH 04/46] Fix KeyValue __repr__. --- freqtrade/persistence/keyvalue.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/freqtrade/persistence/keyvalue.py b/freqtrade/persistence/keyvalue.py index 60fa903a1..2ed64f255 100644 --- a/freqtrade/persistence/keyvalue.py +++ b/freqtrade/persistence/keyvalue.py @@ -38,8 +38,8 @@ class KeyValue(_DECL_BASE): if self.created_at is not None else None) update_time = (self.updated_at.strftime(DATETIME_PRINT_FORMAT) if self.updated_at is not None else None) - return (f'KeyValue(id={self.id}, key={self.kv_key}, type={self.kv_type}, ', - f'value={self.kv_value}, trade_id={self.ft_trade_id}, created={create_time}, ', + return (f'KeyValue(id={self.id}, key={self.kv_key}, type={self.kv_type}, ' + + f'value={self.kv_value}, trade_id={self.ft_trade_id}, created={create_time}, ' + f'updated={update_time})') @staticmethod From be169a23f4e3ed307871a45635b3da89f347a133 Mon Sep 17 00:00:00 2001 From: eSeR1805 Date: Mon, 13 Jun 2022 20:00:21 +0300 Subject: [PATCH 05/46] Add a new session for KeyValues. --- freqtrade/persistence/models.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/freqtrade/persistence/models.py b/freqtrade/persistence/models.py index f2d75fec7..5ba0a28bd 100644 --- a/freqtrade/persistence/models.py +++ b/freqtrade/persistence/models.py @@ -58,7 +58,8 @@ def init_db(db_url: str) -> None: Trade.query = Trade._session.query_property() Order.query = Trade._session.query_property() PairLock.query = Trade._session.query_property() - KeyValue.query = Trade._session.query_property() + KeyValue._session = scoped_session(sessionmaker(bind=engine, autoflush=True)) + KeyValue.query = KeyValue._session.query_property() previous_tables = inspect(engine).get_table_names() _DECL_BASE.metadata.create_all(engine) From f3dee5ec4f98592c1363f5c53c6f375f1476f4fd Mon Sep 17 00:00:00 2001 From: eSeR1805 Date: Mon, 13 Jun 2022 20:02:06 +0300 Subject: [PATCH 06/46] Update handling for query_kv when no Key is supplied. --- freqtrade/persistence/keyvalue.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/freqtrade/persistence/keyvalue.py b/freqtrade/persistence/keyvalue.py index 2ed64f255..d3d1454b7 100644 --- a/freqtrade/persistence/keyvalue.py +++ b/freqtrade/persistence/keyvalue.py @@ -49,9 +49,9 @@ class KeyValue(_DECL_BASE): return will be for generic values not tied to a trade :param trade_id: id of the Trade """ - key = key if key is not None else "%" - - filters = [KeyValue.ft_trade_id == trade_id if trade_id is not None else 0, - KeyValue.kv_key.ilike(key)] + filters = [] + filters.append(KeyValue.ft_trade_id == trade_id if trade_id is not None else 0) + if key is not None: + filters.append(KeyValue.kv_key.ilike(key)) return KeyValue.query.filter(*filters) From c719860a164a95a0718bc4437476396198e5b093 Mon Sep 17 00:00:00 2001 From: eSeR1805 Date: Mon, 13 Jun 2022 20:03:22 +0300 Subject: [PATCH 07/46] get_kval() -> get_kvals(). Update docs also. --- docs/strategy-advanced.md | 10 +++++----- freqtrade/persistence/trade_model.py | 13 +++++++------ 2 files changed, 12 insertions(+), 11 deletions(-) diff --git a/docs/strategy-advanced.md b/docs/strategy-advanced.md index 45961c59d..765dd3fab 100644 --- a/docs/strategy-advanced.md +++ b/docs/strategy-advanced.md @@ -43,7 +43,7 @@ class AwesomeStrategy(IStrategy): ## Storing information (Persistent) Storing information can also be performed in a persistent manner. Freqtrade allows storing/retrieving user custom information associated with a specific trade. -Using a trade object handle information can be stored using `trade_obj.set_kval(key='my_key', value=my_value)` and retrieved using `trade_obj.get_kval(key='my_key')`. +Using a trade object handle information can be stored using `trade_obj.set_kval(key='my_key', value=my_value)` and retrieved using `trade_obj.get_kvals(key='my_key')`. Each data entry is associated with a trade and a user supplied key (of type `string`). This means that this can only be used in callbacks that also provide a trade object handle. For the data to be able to be stored within the database it must be serialized. This is done by converting it to a JSON formatted string. @@ -57,7 +57,7 @@ class AwesomeStrategy(IStrategy): for trade in Trade.get_open_order_trades(): fills = trade.select_filled_orders(trade.entry_side) if trade.pair == 'ETH/USDT': - trade_entry_type = trade.get_kval(key='entry_type') + trade_entry_type = trade.get_kvals(key='entry_type').kv_value if trade_entry_type is None: trade_entry_type = 'breakout' if 'entry_1' in trade.enter_tag else 'dip' elif fills > 1: @@ -73,7 +73,7 @@ class AwesomeStrategy(IStrategy): dataframe, _ = self.dp.get_analyzed_dataframe(pair=pair, timeframe=self.timeframe) current_candle = dataframe.iloc[-1].squeeze() # store information about entry adjustment - existing_count = trade.get_kval(key='num_entry_adjustments') + existing_count = trade.get_kvals(key='num_entry_adjustments').kv_value if not existing_count: existing_count = 1 else: @@ -88,8 +88,8 @@ class AwesomeStrategy(IStrategy): def custom_exit(self, pair: str, trade: Trade, current_time: datetime, current_rate: float, current_profit: float, **kwargs): - entry_adjustment_count = trade.get_kval(key='num_entry_adjustments') - trade_entry_type = trade.get_kval(key='entry_type') + entry_adjustment_count = trade.get_kvals(key='num_entry_adjustments').kv_value + trade_entry_type = trade.get_kvals(key='entry_type').kv_value if entry_adjustment_count is None: if current_profit > 0.01 and (current_time - timedelta(minutes=100) > trade.open_date_utc): return True, 'exit_1' diff --git a/freqtrade/persistence/trade_model.py b/freqtrade/persistence/trade_model.py index 83d400412..ce9fde59e 100644 --- a/freqtrade/persistence/trade_model.py +++ b/freqtrade/persistence/trade_model.py @@ -923,7 +923,7 @@ class LocalTrade(): def set_kval(self, key: str, value: Any) -> None: KeyValues.set_kval(key=key, value=value, trade_id=self.id) - def get_kval(self, key: Optional[str]) -> List[KeyValue]: + def get_kvals(self, key: Optional[str]) -> List[KeyValue]: return KeyValues.get_kval(key=key, trade_id=self.id) @property @@ -1127,12 +1127,13 @@ class Trade(_DECL_BASE, LocalTrade): for order in self.orders: Order.query.session.delete(order) - for kval in self.keyvalues: - KeyValue.query.session.delete(kval) - Trade.query.session.delete(self) Trade.commit() + for kval in self.keyvalues: + KeyValue.query.session.delete(kval) + KeyValue.query.session.commit() + @staticmethod def commit(): Trade.query.session.commit() @@ -1409,5 +1410,5 @@ class Trade(_DECL_BASE, LocalTrade): def set_kval(self, key: str, value: Any) -> None: super().set_kval(key=key, value=value) - def get_kval(self, key: Optional[str]) -> List[KeyValue]: - return super().get_kval(key=key) + def get_kvals(self, key: Optional[str]) -> List[KeyValue]: + return super().get_kvals(key=key) From 4f799cc9db8ed061af8385a775f9c1df67e421f0 Mon Sep 17 00:00:00 2001 From: eSeR1805 Date: Mon, 13 Jun 2022 20:04:14 +0300 Subject: [PATCH 08/46] Add /list_kvals command for TG and underlying RPC. --- freqtrade/rpc/rpc.py | 20 +++++++++++++ freqtrade/rpc/telegram.py | 51 +++++++++++++++++++++++++++++++++- tests/rpc/test_rpc_telegram.py | 2 +- 3 files changed, 71 insertions(+), 2 deletions(-) diff --git a/freqtrade/rpc/rpc.py b/freqtrade/rpc/rpc.py index a98e3f96d..929ab4150 100644 --- a/freqtrade/rpc/rpc.py +++ b/freqtrade/rpc/rpc.py @@ -843,6 +843,26 @@ class RPC: 'cancel_order_count': c_count, } + def _rpc_list_kvals(self, trade_id: int, key: Optional[str]) -> List[Dict[str, Any]]: + # Query for trade + trade = Trade.get_trades(trade_filter=[Trade.id == trade_id]).first() + if trade is None: + return [] + # Query keyvals + keyvals = trade.get_kvals(key=key) + return [ + { + 'id': kval.id, + 'ft_trade_id': kval.ft_trade_id, + 'kv_key': kval.kv_key, + 'kv_type': kval.kv_type, + 'kv_value': kval.kv_value, + 'created_at': kval.created_at, + 'updated_at': kval.updated_at + } + for kval in keyvals + ] + def _rpc_performance(self) -> List[Dict[str, Any]]: """ Handler for performance. diff --git a/freqtrade/rpc/telegram.py b/freqtrade/rpc/telegram.py index e456b1eef..f5bed167d 100644 --- a/freqtrade/rpc/telegram.py +++ b/freqtrade/rpc/telegram.py @@ -182,6 +182,7 @@ class Telegram(RPCHandler): CommandHandler('health', self._health), CommandHandler('help', self._help), CommandHandler('version', self._version), + CommandHandler('list_kvals', self._list_kvals), ] callbacks = [ CallbackQueryHandler(self._status_table, pattern='update_status_table'), @@ -1459,7 +1460,9 @@ class Telegram(RPCHandler): "*/stats:* `Shows Wins / losses by Sell reason as well as " "Avg. holding durationsfor buys and sells.`\n" "*/help:* `This help message`\n" - "*/version:* `Show version`" + "*/version:* `Show version`\n" + "*/list_kvals :* `List key-value for Trade ID and Key combo.`\n" + "`If no Key is supplied it will list all key-value pairs found for that Trade ID.`" ) self._send_msg(message, parse_mode=ParseMode.MARKDOWN) @@ -1539,6 +1542,52 @@ class Telegram(RPCHandler): f"*Current state:* `{val['state']}`" ) + @authorized_only + def _list_kvals(self, update: Update, context: CallbackContext) -> None: + """ + Handler for /list_kvals . + List keyvalues for specified trade (and key if supplied). + :param bot: telegram bot + :param update: message update + :return: None + """ + try: + if not context.args or len(context.args) == 0: + raise RPCException("Trade-id not set.") + trade_id = int(context.args[0]) + key = None if len(context.args) < 2 else str(context.args[1]) + + results = self._rpc._rpc_list_kvals(trade_id, key) + logger.warning(len(results)) + logger.warning(results) + messages = [] + if len(results) > 0: + messages = ['Found key-value pair' + 's: \n' if key is None else ': \n'] + for result in results: + lines = [ + f"*Key:* `{result['kv_key']}`", + f"*ID:* `{result['id']}`", + f"*Trade ID:* `{result['ft_trade_id']}`", + f"*Type:* `{result['kv_type']}`", + f"*Value:* `{result['kv_value']}`", + f"*Create Date:* `{result['created_at']}`", + f"*Update Date:* `{result['updated_at']}`" + ] + # Filter empty lines using list-comprehension + messages.append("\n".join([line for line in lines if line])) + for msg in messages: + logger.warning(msg) + self._send_msg(msg) + else: + message = f"Didn't find any key-value pairs for Trade ID: `{trade_id}`" + logger.warning(message) + message += f" and Key: `{key}`." if key is not None else "" + logger.warning(message) + self._send_msg(message) + + except RPCException as e: + self._send_msg(str(e)) + def _update_msg(self, query: CallbackQuery, msg: str, callback_path: str = "", reload_able: bool = False, parse_mode: str = ParseMode.MARKDOWN) -> None: if reload_able: diff --git a/tests/rpc/test_rpc_telegram.py b/tests/rpc/test_rpc_telegram.py index 2bc4fc5c3..ee0bac9e5 100644 --- a/tests/rpc/test_rpc_telegram.py +++ b/tests/rpc/test_rpc_telegram.py @@ -102,7 +102,7 @@ def test_telegram_init(default_conf, mocker, caplog) -> None: "['count'], ['locks'], ['unlock', 'delete_locks'], " "['reload_config', 'reload_conf'], ['show_config', 'show_conf'], " "['stopbuy'], ['whitelist'], ['blacklist'], ['blacklist_delete', 'bl_delete'], " - "['logs'], ['edge'], ['health'], ['help'], ['version']" + "['logs'], ['edge'], ['health'], ['help'], ['version'], ['list_kvals']" "]") assert log_has(message_str, caplog) From 3ad8111d118437d697a95543b4b6bf3b8f7dcab4 Mon Sep 17 00:00:00 2001 From: eSeR1805 Date: Tue, 14 Jun 2022 13:26:45 +0300 Subject: [PATCH 09/46] Remove stray debug messages. --- freqtrade/rpc/telegram.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/freqtrade/rpc/telegram.py b/freqtrade/rpc/telegram.py index a64242511..c29ec6daa 100644 --- a/freqtrade/rpc/telegram.py +++ b/freqtrade/rpc/telegram.py @@ -1536,8 +1536,6 @@ class Telegram(RPCHandler): key = None if len(context.args) < 2 else str(context.args[1]) results = self._rpc._rpc_list_kvals(trade_id, key) - logger.warning(len(results)) - logger.warning(results) messages = [] if len(results) > 0: messages = ['Found key-value pair' + 's: \n' if key is None else ': \n'] @@ -1554,13 +1552,11 @@ class Telegram(RPCHandler): # Filter empty lines using list-comprehension messages.append("\n".join([line for line in lines if line])) for msg in messages: - logger.warning(msg) self._send_msg(msg) else: message = f"Didn't find any key-value pairs for Trade ID: `{trade_id}`" logger.warning(message) message += f" and Key: `{key}`." if key is not None else "" - logger.warning(message) self._send_msg(message) except RPCException as e: From 24b6ce450b3ce4c95bb808686bdbbba1ade3662a Mon Sep 17 00:00:00 2001 From: eSeR1805 Date: Tue, 14 Jun 2022 13:27:50 +0300 Subject: [PATCH 10/46] Further cleanup. --- freqtrade/rpc/telegram.py | 1 - 1 file changed, 1 deletion(-) diff --git a/freqtrade/rpc/telegram.py b/freqtrade/rpc/telegram.py index c29ec6daa..665621975 100644 --- a/freqtrade/rpc/telegram.py +++ b/freqtrade/rpc/telegram.py @@ -1555,7 +1555,6 @@ class Telegram(RPCHandler): self._send_msg(msg) else: message = f"Didn't find any key-value pairs for Trade ID: `{trade_id}`" - logger.warning(message) message += f" and Key: `{key}`." if key is not None else "" self._send_msg(message) From 9fdb8b07accd04b4f629d075d52d6339e02ebaa2 Mon Sep 17 00:00:00 2001 From: eSeR1805 Date: Sun, 19 Jun 2022 15:56:50 +0300 Subject: [PATCH 11/46] Rename persistant storage infrastructure. --- docs/strategy-advanced.md | 14 +-- freqtrade/optimize/backtesting.py | 8 +- freqtrade/persistence/__init__.py | 2 +- freqtrade/persistence/keyvalue.py | 30 +++--- freqtrade/persistence/keyvalue_middleware.py | 108 ++++++++++--------- freqtrade/persistence/models.py | 6 +- freqtrade/persistence/trade_model.py | 30 +++--- freqtrade/rpc/rpc.py | 22 ++-- freqtrade/rpc/telegram.py | 18 ++-- tests/rpc/test_rpc_telegram.py | 2 +- 10 files changed, 123 insertions(+), 117 deletions(-) diff --git a/docs/strategy-advanced.md b/docs/strategy-advanced.md index 765dd3fab..9cd05d4f6 100644 --- a/docs/strategy-advanced.md +++ b/docs/strategy-advanced.md @@ -43,7 +43,7 @@ class AwesomeStrategy(IStrategy): ## Storing information (Persistent) Storing information can also be performed in a persistent manner. Freqtrade allows storing/retrieving user custom information associated with a specific trade. -Using a trade object handle information can be stored using `trade_obj.set_kval(key='my_key', value=my_value)` and retrieved using `trade_obj.get_kvals(key='my_key')`. +Using a trade object handle information can be stored using `trade_obj.set_custom_data(key='my_key', value=my_value)` and retrieved using `trade_obj.get_custom_data(key='my_key')`. Each data entry is associated with a trade and a user supplied key (of type `string`). This means that this can only be used in callbacks that also provide a trade object handle. For the data to be able to be stored within the database it must be serialized. This is done by converting it to a JSON formatted string. @@ -57,12 +57,12 @@ class AwesomeStrategy(IStrategy): for trade in Trade.get_open_order_trades(): fills = trade.select_filled_orders(trade.entry_side) if trade.pair == 'ETH/USDT': - trade_entry_type = trade.get_kvals(key='entry_type').kv_value + trade_entry_type = trade.get_custom_data(key='entry_type').kv_value if trade_entry_type is None: trade_entry_type = 'breakout' if 'entry_1' in trade.enter_tag else 'dip' elif fills > 1: trade_entry_type = 'buy_up' - trade.set_kval(key='entry_type', value=trade_entry_type) + trade.set_custom_data(key='entry_type', value=trade_entry_type) return super().bot_loop_start(**kwargs) def adjust_entry_price(self, trade: Trade, order: Optional[Order], pair: str, @@ -73,12 +73,12 @@ class AwesomeStrategy(IStrategy): dataframe, _ = self.dp.get_analyzed_dataframe(pair=pair, timeframe=self.timeframe) current_candle = dataframe.iloc[-1].squeeze() # store information about entry adjustment - existing_count = trade.get_kvals(key='num_entry_adjustments').kv_value + existing_count = trade.get_custom_data(key='num_entry_adjustments').kv_value if not existing_count: existing_count = 1 else: existing_count += 1 - trade.set_kval(key='num_entry_adjustments', value=existing_count) + trade.set_custom_data(key='num_entry_adjustments', value=existing_count) # adjust order price return current_candle['sma_200'] @@ -88,8 +88,8 @@ class AwesomeStrategy(IStrategy): def custom_exit(self, pair: str, trade: Trade, current_time: datetime, current_rate: float, current_profit: float, **kwargs): - entry_adjustment_count = trade.get_kvals(key='num_entry_adjustments').kv_value - trade_entry_type = trade.get_kvals(key='entry_type').kv_value + entry_adjustment_count = trade.get_custom_data(key='num_entry_adjustments').kv_value + trade_entry_type = trade.get_custom_data(key='entry_type').kv_value if entry_adjustment_count is None: if current_profit > 0.01 and (current_time - timedelta(minutes=100) > trade.open_date_utc): return True, 'exit_1' diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index 14cc8d2ef..3071fb019 100755 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -30,7 +30,7 @@ from freqtrade.optimize.bt_progress import BTProgress from freqtrade.optimize.optimize_reports import (generate_backtest_stats, show_backtest_results, store_backtest_signal_candles, store_backtest_stats) -from freqtrade.persistence import KeyValues, LocalTrade, Order, PairLocks, Trade +from freqtrade.persistence import CustomDataWrapper, LocalTrade, Order, PairLocks, Trade from freqtrade.plugins.pairlistmanager import PairListManager from freqtrade.plugins.protectionmanager import ProtectionManager from freqtrade.resolvers import ExchangeResolver, StrategyResolver @@ -151,7 +151,7 @@ class Backtesting: LoggingMixin.show_output = True PairLocks.use_db = True Trade.use_db = True - KeyValues.use_db = True # ??? + CustomDataWrapper.use_db = True def init_backtest_detail(self): # Load detail timeframe if specified @@ -300,8 +300,8 @@ class Backtesting: Trade.use_db = False PairLocks.reset_locks() Trade.reset_trades() - KeyValues.use_db = False - KeyValues.reset_keyvalues() + CustomDataWrapper.use_db = False + CustomDataWrapper.reset_custom_data() self.rejected_trades = 0 self.timedout_entry_orders = 0 self.timedout_exit_orders = 0 diff --git a/freqtrade/persistence/__init__.py b/freqtrade/persistence/__init__.py index 635445e40..12cb68a10 100644 --- a/freqtrade/persistence/__init__.py +++ b/freqtrade/persistence/__init__.py @@ -1,6 +1,6 @@ # flake8: noqa: F401 -from freqtrade.persistence.keyvalue_middleware import KeyValues +from freqtrade.persistence.keyvalue_middleware import CustomDataWrapper from freqtrade.persistence.models import cleanup_db, init_db from freqtrade.persistence.pairlock_middleware import PairLocks from freqtrade.persistence.trade_model import LocalTrade, Order, Trade diff --git a/freqtrade/persistence/keyvalue.py b/freqtrade/persistence/keyvalue.py index d3d1454b7..1f85467dd 100644 --- a/freqtrade/persistence/keyvalue.py +++ b/freqtrade/persistence/keyvalue.py @@ -8,28 +8,28 @@ from freqtrade.constants import DATETIME_PRINT_FORMAT from freqtrade.persistence.base import _DECL_BASE -class KeyValue(_DECL_BASE): +class CustomData(_DECL_BASE): """ - KeyValue database model + CustomData database model Keeps records of metadata as key/value store for trades or global persistant values One to many relationship with Trades: - One trade can have many metadata entries - One metadata entry can only be associated with one Trade """ - __tablename__ = 'keyvalue' + __tablename__ = 'trade_custom_data' # Uniqueness should be ensured over pair, order_id # its likely that order_id is unique per Pair on some exchanges. - __table_args__ = (UniqueConstraint('ft_trade_id', 'kv_key', name="_trade_id_kv_key"),) + __table_args__ = (UniqueConstraint('ft_trade_id', 'cd_key', name="_trade_id_cd_key"),) id = Column(Integer, primary_key=True) ft_trade_id = Column(Integer, ForeignKey('trades.id'), index=True, default=0) - trade = relationship("Trade", back_populates="keyvalues") + trade = relationship("Trade", back_populates="custom_data") - kv_key = Column(String(255), nullable=False) - kv_type = Column(String(25), nullable=False) - kv_value = Column(Text, nullable=False) + cd_key = Column(String(255), nullable=False) + cd_type = Column(String(25), nullable=False) + cd_value = Column(Text, nullable=False) created_at = Column(DateTime, nullable=False, default=datetime.utcnow) updated_at = Column(DateTime, nullable=True) @@ -38,20 +38,20 @@ class KeyValue(_DECL_BASE): if self.created_at is not None else None) update_time = (self.updated_at.strftime(DATETIME_PRINT_FORMAT) if self.updated_at is not None else None) - return (f'KeyValue(id={self.id}, key={self.kv_key}, type={self.kv_type}, ' + - f'value={self.kv_value}, trade_id={self.ft_trade_id}, created={create_time}, ' + + return (f'CustomData(id={self.id}, key={self.cd_key}, type={self.cd_type}, ' + + f'value={self.cd_value}, trade_id={self.ft_trade_id}, created={create_time}, ' + f'updated={update_time})') @staticmethod - def query_kv(key: Optional[str] = None, trade_id: Optional[int] = None) -> Query: + def query_cd(key: Optional[str] = None, trade_id: Optional[int] = None) -> Query: """ - Get all keyvalues, if trade_id is not specified + Get all CustomData, if trade_id is not specified return will be for generic values not tied to a trade :param trade_id: id of the Trade """ filters = [] - filters.append(KeyValue.ft_trade_id == trade_id if trade_id is not None else 0) + filters.append(CustomData.ft_trade_id == trade_id if trade_id is not None else 0) if key is not None: - filters.append(KeyValue.kv_key.ilike(key)) + filters.append(CustomData.cd_key.ilike(key)) - return KeyValue.query.filter(*filters) + return CustomData.query.filter(*filters) diff --git a/freqtrade/persistence/keyvalue_middleware.py b/freqtrade/persistence/keyvalue_middleware.py index 8248143ce..0f3c745ad 100644 --- a/freqtrade/persistence/keyvalue_middleware.py +++ b/freqtrade/persistence/keyvalue_middleware.py @@ -3,57 +3,63 @@ import logging from datetime import datetime from typing import Any, List, Optional -from freqtrade.persistence.keyvalue import KeyValue +from freqtrade.persistence.keyvalue import CustomData logger = logging.getLogger(__name__) -class KeyValues(): +class CustomDataWrapper(): """ - KeyValues middleware class + CustomData middleware class Abstracts the database layer away so it becomes optional - which will be necessary to support backtesting and hyperopt in the future. """ use_db = True - kvals: List[KeyValue] = [] + custom_data: List[CustomData] = [] unserialized_types = ['bool', 'float', 'int', 'str'] @staticmethod - def reset_keyvalues() -> None: + def reset_custom_data() -> None: """ Resets all key-value pairs. Only active for backtesting mode. """ - if not KeyValues.use_db: - KeyValues.kvals = [] + if not CustomDataWrapper.use_db: + CustomDataWrapper.custom_data = [] @staticmethod - def get_kval(key: Optional[str] = None, trade_id: Optional[int] = None) -> List[KeyValue]: + def get_custom_data(key: Optional[str] = None, + trade_id: Optional[int] = None) -> List[CustomData]: if trade_id is None: trade_id = 0 - if KeyValues.use_db: - filtered_kvals = KeyValue.query_kv(trade_id=trade_id, key=key).all() - for index, kval in enumerate(filtered_kvals): - if kval.kv_type not in KeyValues.unserialized_types: - kval.kv_value = json.loads(kval.kv_value) - filtered_kvals[index] = kval - return filtered_kvals + if CustomDataWrapper.use_db: + filtered_custom_data = CustomData.query_cd(trade_id=trade_id, key=key).all() + for index, data_entry in enumerate(filtered_custom_data): + if data_entry.cd_type not in CustomDataWrapper.unserialized_types: + data_entry.cd_value = json.loads(data_entry.cd_value) + filtered_custom_data[index] = data_entry + return filtered_custom_data else: - filtered_kvals = [kval for kval in KeyValues.kvals if (kval.ft_trade_id == trade_id)] + filtered_custom_data = [ + data_entry for data_entry in CustomDataWrapper.custom_data + if (data_entry.ft_trade_id == trade_id) + ] if key is not None: - filtered_kvals = [ - kval for kval in filtered_kvals if (kval.kv_key.casefold() == key.casefold())] - return filtered_kvals + filtered_custom_data = [ + data_entry for data_entry in filtered_custom_data + if (data_entry.cd_key.casefold() == key.casefold()) + ] + return filtered_custom_data @staticmethod - def set_kval(key: str, value: Any, trade_id: Optional[int] = None) -> None: + def set_custom_data(key: str, value: Any, trade_id: Optional[int] = None) -> None: value_type = type(value).__name__ value_db = None - if value_type not in KeyValues.unserialized_types: + if value_type not in CustomDataWrapper.unserialized_types: try: value_db = json.dumps(value) except TypeError as e: @@ -64,44 +70,44 @@ class KeyValues(): if trade_id is None: trade_id = 0 - kvals = KeyValues.get_kval(key=key, trade_id=trade_id) - if kvals: - kv = kvals[0] - kv.kv_value = value - kv.updated_at = datetime.utcnow() + custom_data = CustomDataWrapper.get_custom_data(key=key, trade_id=trade_id) + if custom_data: + data_entry = custom_data[0] + data_entry.cd_value = value + data_entry.updated_at = datetime.utcnow() else: - kv = KeyValue( - ft_trade_id=trade_id, - kv_key=key, - kv_type=value_type, - kv_value=value, - created_at=datetime.utcnow() + data_entry = CustomData( + ft_trade_id=trade_id, + cd_key=key, + cd_type=value_type, + cd_value=value, + created_at=datetime.utcnow() ) - if KeyValues.use_db and value_db is not None: - kv.kv_value = value_db - KeyValue.query.session.add(kv) - KeyValue.query.session.commit() - elif not KeyValues.use_db: - kv_index = -1 - for index, kval in enumerate(KeyValues.kvals): - if kval.ft_trade_id == trade_id and kval.kv_key == key: - kv_index = index + if CustomDataWrapper.use_db and value_db is not None: + data_entry.cd_value = value_db + CustomData.query.session.add(data_entry) + CustomData.query.session.commit() + elif not CustomDataWrapper.use_db: + cd_index = -1 + for index, data_entry in enumerate(CustomDataWrapper.custom_data): + if data_entry.ft_trade_id == trade_id and data_entry.cd_key == key: + cd_index = index break - if kv_index >= 0: - kval.kv_type = value_type - kval.value = value - kval.updated_at = datetime.utcnow() + if cd_index >= 0: + data_entry.cd_type = value_type + data_entry.value = value + data_entry.updated_at = datetime.utcnow() - KeyValues.kvals[kv_index] = kval + CustomDataWrapper.custom_data[cd_index] = data_entry else: - KeyValues.kvals.append(kv) + CustomDataWrapper.custom_data.append(data_entry) @staticmethod - def get_all_kvals() -> List[KeyValue]: + def get_all_custom_data() -> List[CustomData]: - if KeyValues.use_db: - return KeyValue.query.all() + if CustomDataWrapper.use_db: + return CustomData.query.all() else: - return KeyValues.kvals + return CustomDataWrapper.custom_data diff --git a/freqtrade/persistence/models.py b/freqtrade/persistence/models.py index 5ba0a28bd..a4c01b119 100644 --- a/freqtrade/persistence/models.py +++ b/freqtrade/persistence/models.py @@ -10,7 +10,7 @@ from sqlalchemy.pool import StaticPool from freqtrade.exceptions import OperationalException from freqtrade.persistence.base import _DECL_BASE -from freqtrade.persistence.keyvalue import KeyValue +from freqtrade.persistence.keyvalue import CustomData from freqtrade.persistence.migrations import check_migrate from freqtrade.persistence.pairlock import PairLock from freqtrade.persistence.trade_model import Order, Trade @@ -58,8 +58,8 @@ def init_db(db_url: str) -> None: Trade.query = Trade._session.query_property() Order.query = Trade._session.query_property() PairLock.query = Trade._session.query_property() - KeyValue._session = scoped_session(sessionmaker(bind=engine, autoflush=True)) - KeyValue.query = KeyValue._session.query_property() + CustomData._session = scoped_session(sessionmaker(bind=engine, autoflush=True)) + CustomData.query = CustomData._session.query_property() previous_tables = inspect(engine).get_table_names() _DECL_BASE.metadata.create_all(engine) diff --git a/freqtrade/persistence/trade_model.py b/freqtrade/persistence/trade_model.py index 53abe638b..ac7ba4833 100644 --- a/freqtrade/persistence/trade_model.py +++ b/freqtrade/persistence/trade_model.py @@ -15,8 +15,8 @@ from freqtrade.enums import ExitType, TradingMode from freqtrade.exceptions import DependencyException, OperationalException from freqtrade.leverage import interest from freqtrade.persistence.base import _DECL_BASE -from freqtrade.persistence.keyvalue import KeyValue -from freqtrade.persistence.keyvalue_middleware import KeyValues +from freqtrade.persistence.keyvalue import CustomData +from freqtrade.persistence.keyvalue_middleware import CustomDataWrapper logger = logging.getLogger(__name__) @@ -240,7 +240,7 @@ class LocalTrade(): id: int = 0 orders: List[Order] = [] - keyvalues: List[KeyValue] = [] + custom_data: List[CustomData] = [] exchange: str = '' pair: str = '' @@ -880,11 +880,11 @@ class LocalTrade(): or (o.ft_is_open is True and o.status is not None) ] - def set_kval(self, key: str, value: Any) -> None: - KeyValues.set_kval(key=key, value=value, trade_id=self.id) + def set_custom_data(self, key: str, value: Any) -> None: + CustomDataWrapper.set_custom_data(key=key, value=value, trade_id=self.id) - def get_kvals(self, key: Optional[str]) -> List[KeyValue]: - return KeyValues.get_kval(key=key, trade_id=self.id) + def get_custom_data(self, key: Optional[str]) -> List[CustomData]: + return CustomDataWrapper.get_custom_data(key=key, trade_id=self.id) @property def nr_of_successful_entries(self) -> int: @@ -1016,7 +1016,7 @@ class Trade(_DECL_BASE, LocalTrade): id = Column(Integer, primary_key=True) orders = relationship("Order", order_by="Order.id", cascade="all, delete-orphan", lazy="joined") - keyvalues = relationship("KeyValue", order_by="KeyValue.id", cascade="all, delete-orphan") + custom_data = relationship("CustomData", order_by="CustomData.id", cascade="all, delete-orphan") exchange = Column(String(25), nullable=False) pair = Column(String(25), nullable=False, index=True) @@ -1090,9 +1090,9 @@ class Trade(_DECL_BASE, LocalTrade): Trade.query.session.delete(self) Trade.commit() - for kval in self.keyvalues: - KeyValue.query.session.delete(kval) - KeyValue.query.session.commit() + for entry in self.custom_data: + CustomData.query.session.delete(entry) + CustomData.query.session.commit() @staticmethod def commit(): @@ -1367,11 +1367,11 @@ class Trade(_DECL_BASE, LocalTrade): .order_by(desc('profit_sum')).first() return best_pair - def set_kval(self, key: str, value: Any) -> None: - super().set_kval(key=key, value=value) + def set_custom_data(self, key: str, value: Any) -> None: + super().set_custom_data(key=key, value=value) - def get_kvals(self, key: Optional[str]) -> List[KeyValue]: - return super().get_kvals(key=key) + def get_custom_data(self, key: Optional[str]) -> List[CustomData]: + return super().get_custom_data(key=key) @staticmethod def get_trading_volume(start_date: datetime = datetime.fromtimestamp(0)) -> float: diff --git a/freqtrade/rpc/rpc.py b/freqtrade/rpc/rpc.py index 608f51bcd..cee2007ff 100644 --- a/freqtrade/rpc/rpc.py +++ b/freqtrade/rpc/rpc.py @@ -805,24 +805,24 @@ class RPC: 'cancel_order_count': c_count, } - def _rpc_list_kvals(self, trade_id: int, key: Optional[str]) -> List[Dict[str, Any]]: + def _rpc_list_custom_data(self, trade_id: int, key: Optional[str]) -> List[Dict[str, Any]]: # Query for trade trade = Trade.get_trades(trade_filter=[Trade.id == trade_id]).first() if trade is None: return [] - # Query keyvals - keyvals = trade.get_kvals(key=key) + # Query custom_data + custom_data = trade.get_custom_data(key=key) return [ { - 'id': kval.id, - 'ft_trade_id': kval.ft_trade_id, - 'kv_key': kval.kv_key, - 'kv_type': kval.kv_type, - 'kv_value': kval.kv_value, - 'created_at': kval.created_at, - 'updated_at': kval.updated_at + 'id': data_entry.id, + 'ft_trade_id': data_entry.ft_trade_id, + 'cd_key': data_entry.cd_key, + 'cd_type': data_entry.cd_type, + 'cd_value': data_entry.cd_value, + 'created_at': data_entry.created_at, + 'updated_at': data_entry.updated_at } - for kval in keyvals + for data_entry in custom_data ] def _rpc_performance(self) -> List[Dict[str, Any]]: diff --git a/freqtrade/rpc/telegram.py b/freqtrade/rpc/telegram.py index ae4da9904..4af7c5d5d 100644 --- a/freqtrade/rpc/telegram.py +++ b/freqtrade/rpc/telegram.py @@ -192,7 +192,7 @@ class Telegram(RPCHandler): CommandHandler('health', self._health), CommandHandler('help', self._help), CommandHandler('version', self._version), - CommandHandler('list_kvals', self._list_kvals), + CommandHandler('list_custom_data', self._list_custom_data), ] callbacks = [ CallbackQueryHandler(self._status_table, pattern='update_status_table'), @@ -1453,7 +1453,7 @@ class Telegram(RPCHandler): "Avg. holding durationsfor buys and sells.`\n" "*/help:* `This help message`\n" "*/version:* `Show version`\n" - "*/list_kvals :* `List key-value for Trade ID and Key combo.`\n" + "*/list_custom_data :* `List custom_data for Trade ID & Key combo.`\n" "`If no Key is supplied it will list all key-value pairs found for that Trade ID.`" ) @@ -1535,10 +1535,10 @@ class Telegram(RPCHandler): ) @authorized_only - def _list_kvals(self, update: Update, context: CallbackContext) -> None: + def _list_custom_data(self, update: Update, context: CallbackContext) -> None: """ - Handler for /list_kvals . - List keyvalues for specified trade (and key if supplied). + Handler for /list_custom_data . + List custom_data for specified trade (and key if supplied). :param bot: telegram bot :param update: message update :return: None @@ -1549,17 +1549,17 @@ class Telegram(RPCHandler): trade_id = int(context.args[0]) key = None if len(context.args) < 2 else str(context.args[1]) - results = self._rpc._rpc_list_kvals(trade_id, key) + results = self._rpc._rpc_list_custom_data(trade_id, key) messages = [] if len(results) > 0: messages = ['Found key-value pair' + 's: \n' if key is None else ': \n'] for result in results: lines = [ - f"*Key:* `{result['kv_key']}`", + f"*Key:* `{result['cd_key']}`", f"*ID:* `{result['id']}`", f"*Trade ID:* `{result['ft_trade_id']}`", - f"*Type:* `{result['kv_type']}`", - f"*Value:* `{result['kv_value']}`", + f"*Type:* `{result['cd_type']}`", + f"*Value:* `{result['cd_value']}`", f"*Create Date:* `{result['created_at']}`", f"*Update Date:* `{result['updated_at']}`" ] diff --git a/tests/rpc/test_rpc_telegram.py b/tests/rpc/test_rpc_telegram.py index 789d10a02..39e33a8e3 100644 --- a/tests/rpc/test_rpc_telegram.py +++ b/tests/rpc/test_rpc_telegram.py @@ -103,7 +103,7 @@ def test_telegram_init(default_conf, mocker, caplog) -> None: "['count'], ['locks'], ['unlock', 'delete_locks'], " "['reload_config', 'reload_conf'], ['show_config', 'show_conf'], " "['stopbuy'], ['whitelist'], ['blacklist'], ['blacklist_delete', 'bl_delete'], " - "['logs'], ['edge'], ['health'], ['help'], ['version'], ['list_kvals']" + "['logs'], ['edge'], ['health'], ['help'], ['version'], ['list_custom_data']" "]") assert log_has(message_str, caplog) From 365527508bbded7a6fb42e0d3351c018c682820a Mon Sep 17 00:00:00 2001 From: eSeR1805 Date: Sun, 19 Jun 2022 15:59:21 +0300 Subject: [PATCH 12/46] Rename files. --- freqtrade/persistence/{keyvalue.py => custom_data.py} | 0 .../{keyvalue_middleware.py => custom_data_middleware.py} | 0 2 files changed, 0 insertions(+), 0 deletions(-) rename freqtrade/persistence/{keyvalue.py => custom_data.py} (100%) rename freqtrade/persistence/{keyvalue_middleware.py => custom_data_middleware.py} (100%) diff --git a/freqtrade/persistence/keyvalue.py b/freqtrade/persistence/custom_data.py similarity index 100% rename from freqtrade/persistence/keyvalue.py rename to freqtrade/persistence/custom_data.py diff --git a/freqtrade/persistence/keyvalue_middleware.py b/freqtrade/persistence/custom_data_middleware.py similarity index 100% rename from freqtrade/persistence/keyvalue_middleware.py rename to freqtrade/persistence/custom_data_middleware.py From ce9d9d7e60b03566ecfab0f5eae6bf72eff89c01 Mon Sep 17 00:00:00 2001 From: eSeR1805 Date: Sun, 19 Jun 2022 16:02:24 +0300 Subject: [PATCH 13/46] Finish renaming persistant storage infrastructure. --- freqtrade/persistence/__init__.py | 2 +- freqtrade/persistence/custom_data_middleware.py | 2 +- freqtrade/persistence/models.py | 2 +- freqtrade/persistence/trade_model.py | 4 ++-- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/freqtrade/persistence/__init__.py b/freqtrade/persistence/__init__.py index 12cb68a10..bf0a8dcbf 100644 --- a/freqtrade/persistence/__init__.py +++ b/freqtrade/persistence/__init__.py @@ -1,6 +1,6 @@ # flake8: noqa: F401 -from freqtrade.persistence.keyvalue_middleware import CustomDataWrapper +from freqtrade.persistence.custom_data_middleware import CustomDataWrapper from freqtrade.persistence.models import cleanup_db, init_db from freqtrade.persistence.pairlock_middleware import PairLocks from freqtrade.persistence.trade_model import LocalTrade, Order, Trade diff --git a/freqtrade/persistence/custom_data_middleware.py b/freqtrade/persistence/custom_data_middleware.py index 0f3c745ad..2fe4bd931 100644 --- a/freqtrade/persistence/custom_data_middleware.py +++ b/freqtrade/persistence/custom_data_middleware.py @@ -3,7 +3,7 @@ import logging from datetime import datetime from typing import Any, List, Optional -from freqtrade.persistence.keyvalue import CustomData +from freqtrade.persistence.custom_data import CustomData logger = logging.getLogger(__name__) diff --git a/freqtrade/persistence/models.py b/freqtrade/persistence/models.py index a4c01b119..16076adb9 100644 --- a/freqtrade/persistence/models.py +++ b/freqtrade/persistence/models.py @@ -10,7 +10,7 @@ from sqlalchemy.pool import StaticPool from freqtrade.exceptions import OperationalException from freqtrade.persistence.base import _DECL_BASE -from freqtrade.persistence.keyvalue import CustomData +from freqtrade.persistence.custom_data import CustomData from freqtrade.persistence.migrations import check_migrate from freqtrade.persistence.pairlock import PairLock from freqtrade.persistence.trade_model import Order, Trade diff --git a/freqtrade/persistence/trade_model.py b/freqtrade/persistence/trade_model.py index ac7ba4833..582e91d3d 100644 --- a/freqtrade/persistence/trade_model.py +++ b/freqtrade/persistence/trade_model.py @@ -15,8 +15,8 @@ from freqtrade.enums import ExitType, TradingMode from freqtrade.exceptions import DependencyException, OperationalException from freqtrade.leverage import interest from freqtrade.persistence.base import _DECL_BASE -from freqtrade.persistence.keyvalue import CustomData -from freqtrade.persistence.keyvalue_middleware import CustomDataWrapper +from freqtrade.persistence.custom_data import CustomData +from freqtrade.persistence.custom_data_middleware import CustomDataWrapper logger = logging.getLogger(__name__) From c8ba8106e668ea92c6f57d577f2d9e4ae750b4b3 Mon Sep 17 00:00:00 2001 From: eSeR1805 Date: Sun, 19 Jun 2022 17:24:13 +0300 Subject: [PATCH 14/46] Update telegram reporting. --- freqtrade/rpc/telegram.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/freqtrade/rpc/telegram.py b/freqtrade/rpc/telegram.py index 4af7c5d5d..6bd68fd3d 100644 --- a/freqtrade/rpc/telegram.py +++ b/freqtrade/rpc/telegram.py @@ -1552,7 +1552,9 @@ class Telegram(RPCHandler): results = self._rpc._rpc_list_custom_data(trade_id, key) messages = [] if len(results) > 0: - messages = ['Found key-value pair' + 's: \n' if key is None else ': \n'] + messages.append( + 'Found custom-data entr' + ('ies: ' if len(results) > 1 else 'y: ') + ) for result in results: lines = [ f"*Key:* `{result['cd_key']}`", @@ -1568,7 +1570,7 @@ class Telegram(RPCHandler): for msg in messages: self._send_msg(msg) else: - message = f"Didn't find any key-value pairs for Trade ID: `{trade_id}`" + message = f"Didn't find any custom-data entries for Trade ID: `{trade_id}`" message += f" and Key: `{key}`." if key is not None else "" self._send_msg(message) From 8494bea64f0b7534553b28c15bf1b5a1a791e383 Mon Sep 17 00:00:00 2001 From: eSeR1805 Date: Sun, 19 Jun 2022 19:59:14 +0300 Subject: [PATCH 15/46] Handle max message length. --- freqtrade/rpc/telegram.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/freqtrade/rpc/telegram.py b/freqtrade/rpc/telegram.py index 6bd68fd3d..6b3ccaac8 100644 --- a/freqtrade/rpc/telegram.py +++ b/freqtrade/rpc/telegram.py @@ -1568,6 +1568,10 @@ class Telegram(RPCHandler): # Filter empty lines using list-comprehension messages.append("\n".join([line for line in lines if line])) for msg in messages: + if len(msg) > MAX_TELEGRAM_MESSAGE_LENGTH: + msg = "Message dropped because length exceeds " + msg += f"maximum allowed characters: {MAX_TELEGRAM_MESSAGE_LENGTH}" + logger.warning(msg) self._send_msg(msg) else: message = f"Didn't find any custom-data entries for Trade ID: `{trade_id}`" From c420304b33b94f67533864345201893ea28f2b9b Mon Sep 17 00:00:00 2001 From: eSeR1805 Date: Sun, 19 Jun 2022 20:03:56 +0300 Subject: [PATCH 16/46] Delete custom data before the trade. --- freqtrade/persistence/trade_model.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/freqtrade/persistence/trade_model.py b/freqtrade/persistence/trade_model.py index 582e91d3d..7e0314738 100644 --- a/freqtrade/persistence/trade_model.py +++ b/freqtrade/persistence/trade_model.py @@ -1087,12 +1087,12 @@ class Trade(_DECL_BASE, LocalTrade): for order in self.orders: Order.query.session.delete(order) - Trade.query.session.delete(self) - Trade.commit() - for entry in self.custom_data: CustomData.query.session.delete(entry) + CustomData.query.session.commit() + Trade.query.session.delete(self) + Trade.commit() @staticmethod def commit(): From 8f9f4b40cdcad855cec449f9364a449a22cfad67 Mon Sep 17 00:00:00 2001 From: Matthias Date: Wed, 7 Feb 2024 07:21:16 +0100 Subject: [PATCH 17/46] Update model to new sqlalchemy version --- freqtrade/persistence/custom_data.py | 29 ++++++++++--------- .../persistence/custom_data_middleware.py | 10 ++++--- freqtrade/persistence/trade_model.py | 3 +- freqtrade/rpc/telegram.py | 8 ++--- 4 files changed, 28 insertions(+), 22 deletions(-) diff --git a/freqtrade/persistence/custom_data.py b/freqtrade/persistence/custom_data.py index 1f85467dd..beae8c478 100644 --- a/freqtrade/persistence/custom_data.py +++ b/freqtrade/persistence/custom_data.py @@ -1,14 +1,15 @@ from datetime import datetime -from typing import Optional +from typing import ClassVar, Optional -from sqlalchemy import Column, DateTime, ForeignKey, Integer, String, Text, UniqueConstraint -from sqlalchemy.orm import Query, relationship +from sqlalchemy import DateTime, ForeignKey, Integer, String, Text, UniqueConstraint, select +from sqlalchemy.orm import Mapped, Query, mapped_column, relationship from freqtrade.constants import DATETIME_PRINT_FORMAT -from freqtrade.persistence.base import _DECL_BASE +from freqtrade.persistence.base import ModelBase, SessionType +from freqtrade.util import dt_now -class CustomData(_DECL_BASE): +class CustomData(ModelBase): """ CustomData database model Keeps records of metadata as key/value store @@ -18,20 +19,22 @@ class CustomData(_DECL_BASE): - One metadata entry can only be associated with one Trade """ __tablename__ = 'trade_custom_data' + session: ClassVar[SessionType] + # Uniqueness should be ensured over pair, order_id # its likely that order_id is unique per Pair on some exchanges. __table_args__ = (UniqueConstraint('ft_trade_id', 'cd_key', name="_trade_id_cd_key"),) - id = Column(Integer, primary_key=True) - ft_trade_id = Column(Integer, ForeignKey('trades.id'), index=True, default=0) + id = mapped_column(Integer, primary_key=True) + ft_trade_id = mapped_column(Integer, ForeignKey('trades.id'), index=True, default=0) trade = relationship("Trade", back_populates="custom_data") - cd_key = Column(String(255), nullable=False) - cd_type = Column(String(25), nullable=False) - cd_value = Column(Text, nullable=False) - created_at = Column(DateTime, nullable=False, default=datetime.utcnow) - updated_at = Column(DateTime, nullable=True) + cd_key: Mapped[str] = mapped_column(String(255), nullable=False) + cd_type: Mapped[str] = mapped_column(String(25), nullable=False) + cd_value: Mapped[str] = mapped_column(Text, nullable=False) + created_at: Mapped[datetime] = mapped_column(DateTime, nullable=False, default=dt_now) + updated_at: Mapped[Optional[datetime]] = mapped_column(DateTime, nullable=True) def __repr__(self): create_time = (self.created_at.strftime(DATETIME_PRINT_FORMAT) @@ -54,4 +57,4 @@ class CustomData(_DECL_BASE): if key is not None: filters.append(CustomData.cd_key.ilike(key)) - return CustomData.query.filter(*filters) + return CustomData.session.scalars(select(CustomData)) diff --git a/freqtrade/persistence/custom_data_middleware.py b/freqtrade/persistence/custom_data_middleware.py index 64e8a47fd..cf7b83abc 100644 --- a/freqtrade/persistence/custom_data_middleware.py +++ b/freqtrade/persistence/custom_data_middleware.py @@ -3,6 +3,8 @@ import logging from datetime import datetime from typing import Any, List, Optional +from sqlalchemy import select + from freqtrade.persistence.custom_data import CustomData @@ -86,8 +88,8 @@ class CustomDataWrapper: if CustomDataWrapper.use_db and value_db is not None: data_entry.cd_value = value_db - CustomData.query.session.add(data_entry) - CustomData.query.session.commit() + CustomData.session.add(data_entry) + CustomData.session.commit() elif not CustomDataWrapper.use_db: cd_index = -1 for index, data_entry in enumerate(CustomDataWrapper.custom_data): @@ -97,7 +99,7 @@ class CustomDataWrapper: if cd_index >= 0: data_entry.cd_type = value_type - data_entry.value = value + data_entry.cd_value = value data_entry.updated_at = datetime.utcnow() CustomDataWrapper.custom_data[cd_index] = data_entry @@ -108,6 +110,6 @@ class CustomDataWrapper: def get_all_custom_data() -> List[CustomData]: if CustomDataWrapper.use_db: - return CustomData.query.all() + return CustomData.session.scalars(select(CustomData)).all() else: return CustomDataWrapper.custom_data diff --git a/freqtrade/persistence/trade_model.py b/freqtrade/persistence/trade_model.py index a3f71ee4f..d96378bf2 100644 --- a/freqtrade/persistence/trade_model.py +++ b/freqtrade/persistence/trade_model.py @@ -1467,7 +1467,8 @@ class Trade(ModelBase, LocalTrade): orders: Mapped[List[Order]] = relationship( "Order", order_by="Order.id", cascade="all, delete-orphan", lazy="selectin", innerjoin=True) # type: ignore - custom_data = relationship("CustomData", order_by="CustomData.id", cascade="all, delete-orphan") + custom_data: Mapped[List[CustomData]] = relationship( + "CustomData", order_by="CustomData.id", cascade="all, delete-orphan") # type: ignore exchange: Mapped[str] = mapped_column(String(25), nullable=False) # type: ignore pair: Mapped[str] = mapped_column(String(25), nullable=False, index=True) # type: ignore diff --git a/freqtrade/rpc/telegram.py b/freqtrade/rpc/telegram.py index 4bdb09f01..4d91d1bdb 100644 --- a/freqtrade/rpc/telegram.py +++ b/freqtrade/rpc/telegram.py @@ -1770,7 +1770,7 @@ class Telegram(RPCHandler): ) @authorized_only - def _list_custom_data(self, update: Update, context: CallbackContext) -> None: + async def _list_custom_data(self, update: Update, context: CallbackContext) -> None: """ Handler for /list_custom_data . List custom_data for specified trade (and key if supplied). @@ -1807,14 +1807,14 @@ class Telegram(RPCHandler): msg = "Message dropped because length exceeds " msg += f"maximum allowed characters: {MAX_MESSAGE_LENGTH}" logger.warning(msg) - self._send_msg(msg) + await self._send_msg(msg) else: message = f"Didn't find any custom-data entries for Trade ID: `{trade_id}`" message += f" and Key: `{key}`." if key is not None else "" - self._send_msg(message) + await self._send_msg(message) except RPCException as e: - self._send_msg(str(e)) + await self._send_msg(str(e)) async def _update_msg(self, query: CallbackQuery, msg: str, callback_path: str = "", reload_able: bool = False, parse_mode: str = ParseMode.MARKDOWN) -> None: From 2393a9fecf04eb284078ec1d09e6b0113138d5f4 Mon Sep 17 00:00:00 2001 From: Matthias Date: Wed, 7 Feb 2024 19:06:41 +0100 Subject: [PATCH 18/46] Fix some minor test failures --- freqtrade/persistence/models.py | 4 ++-- tests/rpc/test_rpc_telegram.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/freqtrade/persistence/models.py b/freqtrade/persistence/models.py index 3f1661ee2..189b80fa6 100644 --- a/freqtrade/persistence/models.py +++ b/freqtrade/persistence/models.py @@ -79,8 +79,8 @@ def init_db(db_url: str) -> None: Order.session = Trade.session PairLock.session = Trade.session _KeyValueStoreModel.session = Trade.session - CustomData._session = scoped_session(sessionmaker(bind=engine, autoflush=True), - scopefunc=get_request_or_thread_id) + CustomData.session = scoped_session(sessionmaker(bind=engine, autoflush=True), + scopefunc=get_request_or_thread_id) previous_tables = inspect(engine).get_table_names() ModelBase.metadata.create_all(engine) diff --git a/tests/rpc/test_rpc_telegram.py b/tests/rpc/test_rpc_telegram.py index 110cd7819..45268b23b 100644 --- a/tests/rpc/test_rpc_telegram.py +++ b/tests/rpc/test_rpc_telegram.py @@ -150,7 +150,7 @@ def test_telegram_init(default_conf, mocker, caplog) -> None: "['stopbuy', 'stopentry'], ['whitelist'], ['blacklist'], " "['bl_delete', 'blacklist_delete'], " "['logs'], ['edge'], ['health'], ['help'], ['version'], ['marketdir'], " - "['order']], ['list_custom_data']") + "['order'], ['list_custom_data']]") assert log_has(message_str, caplog) From 626c9041039212f7672dd3b5e769323de54f636f Mon Sep 17 00:00:00 2001 From: Matthias Date: Wed, 7 Feb 2024 19:28:06 +0100 Subject: [PATCH 19/46] Fix some issues with types --- freqtrade/persistence/custom_data.py | 14 ++++++++------ freqtrade/persistence/custom_data_middleware.py | 8 ++++---- 2 files changed, 12 insertions(+), 10 deletions(-) diff --git a/freqtrade/persistence/custom_data.py b/freqtrade/persistence/custom_data.py index beae8c478..42b267e95 100644 --- a/freqtrade/persistence/custom_data.py +++ b/freqtrade/persistence/custom_data.py @@ -1,8 +1,8 @@ from datetime import datetime -from typing import ClassVar, Optional +from typing import ClassVar, Optional, Self, Sequence from sqlalchemy import DateTime, ForeignKey, Integer, String, Text, UniqueConstraint, select -from sqlalchemy.orm import Mapped, Query, mapped_column, relationship +from sqlalchemy.orm import Mapped, mapped_column, relationship from freqtrade.constants import DATETIME_PRINT_FORMAT from freqtrade.persistence.base import ModelBase, SessionType @@ -45,16 +45,18 @@ class CustomData(ModelBase): f'value={self.cd_value}, trade_id={self.ft_trade_id}, created={create_time}, ' + f'updated={update_time})') - @staticmethod - def query_cd(key: Optional[str] = None, trade_id: Optional[int] = None) -> Query: + @classmethod + def query_cd(cls, key: Optional[str] = None, + trade_id: Optional[int] = None) -> Sequence['CustomData']: """ Get all CustomData, if trade_id is not specified return will be for generic values not tied to a trade :param trade_id: id of the Trade """ filters = [] - filters.append(CustomData.ft_trade_id == trade_id if trade_id is not None else 0) + if trade_id is not None: + filters.append(CustomData.ft_trade_id == trade_id) if key is not None: filters.append(CustomData.cd_key.ilike(key)) - return CustomData.session.scalars(select(CustomData)) + return CustomData.session.scalars(select(CustomData)).all() diff --git a/freqtrade/persistence/custom_data_middleware.py b/freqtrade/persistence/custom_data_middleware.py index cf7b83abc..2f99d9c75 100644 --- a/freqtrade/persistence/custom_data_middleware.py +++ b/freqtrade/persistence/custom_data_middleware.py @@ -37,11 +37,11 @@ class CustomDataWrapper: trade_id = 0 if CustomDataWrapper.use_db: - filtered_custom_data = CustomData.query_cd(trade_id=trade_id, key=key).all() - for index, data_entry in enumerate(filtered_custom_data): + filtered_custom_data = [] + for data_entry in CustomData.query_cd(trade_id=trade_id, key=key): if data_entry.cd_type not in CustomDataWrapper.unserialized_types: data_entry.cd_value = json.loads(data_entry.cd_value) - filtered_custom_data[index] = data_entry + filtered_custom_data.append(data_entry) return filtered_custom_data else: filtered_custom_data = [ @@ -110,6 +110,6 @@ class CustomDataWrapper: def get_all_custom_data() -> List[CustomData]: if CustomDataWrapper.use_db: - return CustomData.session.scalars(select(CustomData)).all() + return list(CustomData.query_cd()) else: return CustomDataWrapper.custom_data From d49da763824aec1ad5fefd38aee938ef34e2a6b8 Mon Sep 17 00:00:00 2001 From: Matthias Date: Mon, 12 Feb 2024 18:17:48 +0100 Subject: [PATCH 20/46] Slighlty improve docs --- docs/strategy-advanced.md | 63 ++++++++++++++++++++++++--------------- 1 file changed, 39 insertions(+), 24 deletions(-) diff --git a/docs/strategy-advanced.md b/docs/strategy-advanced.md index b2a0d8431..69d4ced34 100644 --- a/docs/strategy-advanced.md +++ b/docs/strategy-advanced.md @@ -13,38 +13,46 @@ The call sequence of the methods described here is covered under [bot execution ## Storing information (Non-Persistent) -Storing information can be accomplished by creating a new dictionary within the strategy class. +!!! Warning "Deprecated" + This method of storing information is deprecated, and we do advise against using non-persistent storage. + Please use the below [Persistent Storing Information Section](#storing-information-persistent) instead. -The name of the variable can be chosen at will, but should be prefixed with `custom_` to avoid naming collisions with predefined strategy variables. + It's content has therefore be collapsed. -```python -class AwesomeStrategy(IStrategy): - # Create custom dictionary - custom_info = {} +??? Abstract "Storing information" + Storing information can be accomplished by creating a new dictionary within the strategy class. - def populate_indicators(self, dataframe: DataFrame, metadata: dict) -> DataFrame: - # Check if the entry already exists - if not metadata["pair"] in self.custom_info: - # Create empty entry for this pair - self.custom_info[metadata["pair"]] = {} + The name of the variable can be chosen at will, but should be prefixed with `custom_` to avoid naming collisions with predefined strategy variables. - if "crosstime" in self.custom_info[metadata["pair"]]: - self.custom_info[metadata["pair"]]["crosstime"] += 1 - else: - self.custom_info[metadata["pair"]]["crosstime"] = 1 -``` + ```python + class AwesomeStrategy(IStrategy): + # Create custom dictionary + custom_info = {} -!!! Warning - The data is not persisted after a bot-restart (or config-reload). Also, the amount of data should be kept smallish (no DataFrames and such), otherwise the bot will start to consume a lot of memory and eventually run out of memory and crash. + def populate_indicators(self, dataframe: DataFrame, metadata: dict) -> DataFrame: + # Check if the entry already exists + if not metadata["pair"] in self.custom_info: + # Create empty entry for this pair + self.custom_info[metadata["pair"]] = {} -!!! Note - If the data is pair-specific, make sure to use pair as one of the keys in the dictionary. + if "crosstime" in self.custom_info[metadata["pair"]]: + self.custom_info[metadata["pair"]]["crosstime"] += 1 + else: + self.custom_info[metadata["pair"]]["crosstime"] = 1 + ``` + + !!! Warning + The data is not persisted after a bot-restart (or config-reload). Also, the amount of data should be kept smallish (no DataFrames and such), otherwise the bot will start to consume a lot of memory and eventually run out of memory and crash. + + !!! Note + If the data is pair-specific, make sure to use pair as one of the keys in the dictionary. ## Storing information (Persistent) Storing information can also be performed in a persistent manner. Freqtrade allows storing/retrieving user custom information associated with a specific trade. -Using a trade object handle information can be stored using `trade_obj.set_custom_data(key='my_key', value=my_value)` and retrieved using `trade_obj.get_custom_data(key='my_key')`. -Each data entry is associated with a trade and a user supplied key (of type `string`). This means that this can only be used in callbacks that also provide a trade object handle. + +Using a trade object, information can be stored using `trade_obj.set_custom_data(key='my_key', value=my_value)` and retrieved using `trade_obj.get_custom_data(key='my_key')`. Each data entry is associated with a trade and a user supplied key (of type `string`). This means that this can only be used in callbacks that also provide a trade object. + For the data to be able to be stored within the database it must be serialized. This is done by converting it to a JSON formatted string. ```python @@ -69,7 +77,13 @@ class AwesomeStrategy(IStrategy): current_time: datetime, proposed_rate: float, current_order_rate: float, entry_tag: Optional[str], side: str, **kwargs) -> float: # Limit orders to use and follow SMA200 as price target for the first 10 minutes since entry trigger for BTC/USDT pair. - if pair == 'BTC/USDT' and entry_tag == 'long_sma200' and side == 'long' and (current_time - timedelta(minutes=10) > trade.open_date_utc and order.filled == 0.0: + if ( + pair == 'BTC/USDT' + and entry_tag == 'long_sma200' + and side == 'long' + and (current_time - timedelta(minutes=10)) > trade.open_date_utc + and order.filled == 0.0 + ): dataframe, _ = self.dp.get_analyzed_dataframe(pair=pair, timeframe=self.timeframe) current_candle = dataframe.iloc[-1].squeeze() # store information about entry adjustment @@ -104,8 +118,9 @@ class AwesomeStrategy(IStrategy): !!! Note It is recommended that simple data types are used `[bool, int, float, str]` to ensure no issues when serializing the data that needs to be stored. + Storing big junks of data may lead to unintended side-effects, like a database becoming big pretty fast (and as a consequence, also slow). -!!! Warning +!!! Warning "Non-serializable data" If supplied data cannot be serialized a warning is logged and the entry for the specified `key` will contain `None` as data. ## Dataframe access From 85930946195444937accc653770655788e9fd32e Mon Sep 17 00:00:00 2001 From: Matthias Date: Mon, 12 Feb 2024 18:22:49 +0100 Subject: [PATCH 21/46] Ensure custom data access goes through the accessor functions --- freqtrade/persistence/trade_model.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/freqtrade/persistence/trade_model.py b/freqtrade/persistence/trade_model.py index d96378bf2..b0b4a58dc 100644 --- a/freqtrade/persistence/trade_model.py +++ b/freqtrade/persistence/trade_model.py @@ -1468,7 +1468,8 @@ class Trade(ModelBase, LocalTrade): "Order", order_by="Order.id", cascade="all, delete-orphan", lazy="selectin", innerjoin=True) # type: ignore custom_data: Mapped[List[CustomData]] = relationship( - "CustomData", order_by="CustomData.id", cascade="all, delete-orphan") # type: ignore + "CustomData", order_by="CustomData.id", cascade="all, delete-orphan", + lazy="raise") # type: ignore exchange: Mapped[str] = mapped_column(String(25), nullable=False) # type: ignore pair: Mapped[str] = mapped_column(String(25), nullable=False, index=True) # type: ignore From c67e451fe177c00f8148d0d66915e957ab15b8a2 Mon Sep 17 00:00:00 2001 From: Matthias Date: Mon, 12 Feb 2024 18:24:26 +0100 Subject: [PATCH 22/46] Remove unused imports --- freqtrade/persistence/custom_data.py | 2 +- freqtrade/persistence/custom_data_middleware.py | 2 -- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/freqtrade/persistence/custom_data.py b/freqtrade/persistence/custom_data.py index 42b267e95..004aa51df 100644 --- a/freqtrade/persistence/custom_data.py +++ b/freqtrade/persistence/custom_data.py @@ -1,5 +1,5 @@ from datetime import datetime -from typing import ClassVar, Optional, Self, Sequence +from typing import ClassVar, Optional, Sequence from sqlalchemy import DateTime, ForeignKey, Integer, String, Text, UniqueConstraint, select from sqlalchemy.orm import Mapped, mapped_column, relationship diff --git a/freqtrade/persistence/custom_data_middleware.py b/freqtrade/persistence/custom_data_middleware.py index 2f99d9c75..e40491b28 100644 --- a/freqtrade/persistence/custom_data_middleware.py +++ b/freqtrade/persistence/custom_data_middleware.py @@ -3,8 +3,6 @@ import logging from datetime import datetime from typing import Any, List, Optional -from sqlalchemy import select - from freqtrade.persistence.custom_data import CustomData From 83b22dedd5a6151edf67dcc0c96bcbf8207caccb Mon Sep 17 00:00:00 2001 From: Matthias Date: Mon, 12 Feb 2024 18:25:02 +0100 Subject: [PATCH 23/46] Fix non-reset of use_db --- freqtrade/persistence/usedb_context.py | 1 + 1 file changed, 1 insertion(+) diff --git a/freqtrade/persistence/usedb_context.py b/freqtrade/persistence/usedb_context.py index 795388af0..193f7021d 100644 --- a/freqtrade/persistence/usedb_context.py +++ b/freqtrade/persistence/usedb_context.py @@ -12,6 +12,7 @@ def disable_database_use(timeframe: str) -> None: PairLocks.use_db = False PairLocks.timeframe = timeframe Trade.use_db = False + CustomDataWrapper.use_db = False def enable_database_use() -> None: From 9699011cd9e607c4efacc30428c0ebe6f5048b67 Mon Sep 17 00:00:00 2001 From: Matthias Date: Mon, 12 Feb 2024 18:28:29 +0100 Subject: [PATCH 24/46] Remove pointless wrapper --- freqtrade/persistence/trade_model.py | 6 ------ 1 file changed, 6 deletions(-) diff --git a/freqtrade/persistence/trade_model.py b/freqtrade/persistence/trade_model.py index b0b4a58dc..ea03e2c29 100644 --- a/freqtrade/persistence/trade_model.py +++ b/freqtrade/persistence/trade_model.py @@ -1882,12 +1882,6 @@ class Trade(ModelBase, LocalTrade): return best_pair - def set_custom_data(self, key: str, value: Any) -> None: - super().set_custom_data(key=key, value=value) - - def get_custom_data(self, key: Optional[str]) -> List[CustomData]: - return super().get_custom_data(key=key) - @staticmethod def get_trading_volume(start_date: datetime = datetime.fromtimestamp(0)) -> float: """ From 7fd70b82fa61d8f1f68a8394f5b616eaff4861f5 Mon Sep 17 00:00:00 2001 From: Matthias Date: Mon, 12 Feb 2024 20:05:13 +0100 Subject: [PATCH 25/46] custom_data: Simplify and fix a few things --- freqtrade/persistence/custom_data.py | 2 +- .../persistence/custom_data_middleware.py | 22 +++++++++---------- 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/freqtrade/persistence/custom_data.py b/freqtrade/persistence/custom_data.py index 004aa51df..cfe150967 100644 --- a/freqtrade/persistence/custom_data.py +++ b/freqtrade/persistence/custom_data.py @@ -59,4 +59,4 @@ class CustomData(ModelBase): if key is not None: filters.append(CustomData.cd_key.ilike(key)) - return CustomData.session.scalars(select(CustomData)).all() + return CustomData.session.scalars(select(CustomData).filter(*filters)).all() diff --git a/freqtrade/persistence/custom_data_middleware.py b/freqtrade/persistence/custom_data_middleware.py index e40491b28..acc65606b 100644 --- a/freqtrade/persistence/custom_data_middleware.py +++ b/freqtrade/persistence/custom_data_middleware.py @@ -1,9 +1,9 @@ import json import logging -from datetime import datetime from typing import Any, List, Optional from freqtrade.persistence.custom_data import CustomData +from freqtrade.util import dt_now logger = logging.getLogger(__name__) @@ -30,7 +30,7 @@ class CustomDataWrapper: @staticmethod def get_custom_data(key: Optional[str] = None, - trade_id: Optional[int] = None) -> List[CustomData]: + trade_id: Optional[int] = None) -> CustomData: if trade_id is None: trade_id = 0 @@ -73,15 +73,15 @@ class CustomDataWrapper: custom_data = CustomDataWrapper.get_custom_data(key=key, trade_id=trade_id) if custom_data: data_entry = custom_data[0] - data_entry.cd_value = value - data_entry.updated_at = datetime.utcnow() + data_entry.cd_value = value_db + data_entry.updated_at = dt_now() else: data_entry = CustomData( - ft_trade_id=trade_id, - cd_key=key, - cd_type=value_type, - cd_value=value, - created_at=datetime.utcnow() + ft_trade_id=trade_id, + cd_key=key, + cd_type=value_type, + cd_value=value_db, + created_at=dt_now() ) if CustomDataWrapper.use_db and value_db is not None: @@ -97,8 +97,8 @@ class CustomDataWrapper: if cd_index >= 0: data_entry.cd_type = value_type - data_entry.cd_value = value - data_entry.updated_at = datetime.utcnow() + data_entry.cd_value = value_db + data_entry.updated_at = dt_now() CustomDataWrapper.custom_data[cd_index] = data_entry else: From b7904b8e805b57a7cd6b588c203ee981229b0b5e Mon Sep 17 00:00:00 2001 From: Matthias Date: Mon, 12 Feb 2024 20:14:37 +0100 Subject: [PATCH 26/46] Combine custom_data classes to one file --- freqtrade/persistence/__init__.py | 2 +- freqtrade/persistence/custom_data.py | 111 ++++++++++++++++- .../persistence/custom_data_middleware.py | 113 ------------------ freqtrade/persistence/trade_model.py | 13 +- freqtrade/persistence/usedb_context.py | 2 +- 5 files changed, 118 insertions(+), 123 deletions(-) delete mode 100644 freqtrade/persistence/custom_data_middleware.py diff --git a/freqtrade/persistence/__init__.py b/freqtrade/persistence/__init__.py index 5926f2ad3..d5584c22c 100644 --- a/freqtrade/persistence/__init__.py +++ b/freqtrade/persistence/__init__.py @@ -1,6 +1,6 @@ # flake8: noqa: F401 -from freqtrade.persistence.custom_data_middleware import CustomDataWrapper +from freqtrade.persistence.custom_data import CustomDataWrapper from freqtrade.persistence.key_value_store import KeyStoreKeys, KeyValueStore from freqtrade.persistence.models import init_db from freqtrade.persistence.pairlock_middleware import PairLocks diff --git a/freqtrade/persistence/custom_data.py b/freqtrade/persistence/custom_data.py index cfe150967..bf6056278 100644 --- a/freqtrade/persistence/custom_data.py +++ b/freqtrade/persistence/custom_data.py @@ -1,5 +1,7 @@ +import json +import logging from datetime import datetime -from typing import ClassVar, Optional, Sequence +from typing import Any, ClassVar, List, Optional, Sequence from sqlalchemy import DateTime, ForeignKey, Integer, String, Text, UniqueConstraint, select from sqlalchemy.orm import Mapped, mapped_column, relationship @@ -9,6 +11,9 @@ from freqtrade.persistence.base import ModelBase, SessionType from freqtrade.util import dt_now +logger = logging.getLogger(__name__) + + class CustomData(ModelBase): """ CustomData database model @@ -60,3 +65,107 @@ class CustomData(ModelBase): filters.append(CustomData.cd_key.ilike(key)) return CustomData.session.scalars(select(CustomData).filter(*filters)).all() + + +class CustomDataWrapper: + """ + CustomData middleware class + Abstracts the database layer away so it becomes optional - which will be necessary to support + backtesting and hyperopt in the future. + """ + + use_db = True + custom_data: List[CustomData] = [] + unserialized_types = ['bool', 'float', 'int', 'str'] + + @staticmethod + def reset_custom_data() -> None: + """ + Resets all key-value pairs. Only active for backtesting mode. + """ + if not CustomDataWrapper.use_db: + CustomDataWrapper.custom_data = [] + + @staticmethod + def get_custom_data(key: Optional[str] = None, + trade_id: Optional[int] = None) -> CustomData: + if trade_id is None: + trade_id = 0 + + if CustomDataWrapper.use_db: + filtered_custom_data = [] + for data_entry in CustomData.query_cd(trade_id=trade_id, key=key): + if data_entry.cd_type not in CustomDataWrapper.unserialized_types: + data_entry.cd_value = json.loads(data_entry.cd_value) + filtered_custom_data.append(data_entry) + return filtered_custom_data + else: + filtered_custom_data = [ + data_entry for data_entry in CustomDataWrapper.custom_data + if (data_entry.ft_trade_id == trade_id) + ] + if key is not None: + filtered_custom_data = [ + data_entry for data_entry in filtered_custom_data + if (data_entry.cd_key.casefold() == key.casefold()) + ] + return filtered_custom_data + + @staticmethod + def set_custom_data(key: str, value: Any, trade_id: Optional[int] = None) -> None: + + value_type = type(value).__name__ + value_db = None + + if value_type not in CustomDataWrapper.unserialized_types: + try: + value_db = json.dumps(value) + except TypeError as e: + logger.warning(f"could not serialize {key} value due to {e}") + else: + value_db = str(value) + + if trade_id is None: + trade_id = 0 + + custom_data = CustomDataWrapper.get_custom_data(key=key, trade_id=trade_id) + if custom_data: + data_entry = custom_data[0] + data_entry.cd_value = value_db + data_entry.updated_at = dt_now() + else: + data_entry = CustomData( + ft_trade_id=trade_id, + cd_key=key, + cd_type=value_type, + cd_value=value_db, + created_at=dt_now() + ) + + if CustomDataWrapper.use_db and value_db is not None: + data_entry.cd_value = value_db + CustomData.session.add(data_entry) + CustomData.session.commit() + elif not CustomDataWrapper.use_db: + cd_index = -1 + for index, data_entry in enumerate(CustomDataWrapper.custom_data): + if data_entry.ft_trade_id == trade_id and data_entry.cd_key == key: + cd_index = index + break + + if cd_index >= 0: + data_entry.cd_type = value_type + data_entry.cd_value = value_db + data_entry.updated_at = dt_now() + + CustomDataWrapper.custom_data[cd_index] = data_entry + else: + CustomDataWrapper.custom_data.append(data_entry) + + @staticmethod + def get_all_custom_data() -> List[CustomData]: + + if CustomDataWrapper.use_db: + return list(CustomData.query_cd()) + else: + return CustomDataWrapper.custom_data diff --git a/freqtrade/persistence/custom_data_middleware.py b/freqtrade/persistence/custom_data_middleware.py deleted file mode 100644 index acc65606b..000000000 --- a/freqtrade/persistence/custom_data_middleware.py +++ /dev/null @@ -1,113 +0,0 @@ -import json -import logging -from typing import Any, List, Optional - -from freqtrade.persistence.custom_data import CustomData -from freqtrade.util import dt_now - - -logger = logging.getLogger(__name__) - - -class CustomDataWrapper: - """ - CustomData middleware class - Abstracts the database layer away so it becomes optional - which will be necessary to support - backtesting and hyperopt in the future. - """ - - use_db = True - custom_data: List[CustomData] = [] - unserialized_types = ['bool', 'float', 'int', 'str'] - - @staticmethod - def reset_custom_data() -> None: - """ - Resets all key-value pairs. Only active for backtesting mode. - """ - if not CustomDataWrapper.use_db: - CustomDataWrapper.custom_data = [] - - @staticmethod - def get_custom_data(key: Optional[str] = None, - trade_id: Optional[int] = None) -> CustomData: - if trade_id is None: - trade_id = 0 - - if CustomDataWrapper.use_db: - filtered_custom_data = [] - for data_entry in CustomData.query_cd(trade_id=trade_id, key=key): - if data_entry.cd_type not in CustomDataWrapper.unserialized_types: - data_entry.cd_value = json.loads(data_entry.cd_value) - filtered_custom_data.append(data_entry) - return filtered_custom_data - else: - filtered_custom_data = [ - data_entry for data_entry in CustomDataWrapper.custom_data - if (data_entry.ft_trade_id == trade_id) - ] - if key is not None: - filtered_custom_data = [ - data_entry for data_entry in filtered_custom_data - if (data_entry.cd_key.casefold() == key.casefold()) - ] - return filtered_custom_data - - @staticmethod - def set_custom_data(key: str, value: Any, trade_id: Optional[int] = None) -> None: - - value_type = type(value).__name__ - value_db = None - - if value_type not in CustomDataWrapper.unserialized_types: - try: - value_db = json.dumps(value) - except TypeError as e: - logger.warning(f"could not serialize {key} value due to {e}") - else: - value_db = str(value) - - if trade_id is None: - trade_id = 0 - - custom_data = CustomDataWrapper.get_custom_data(key=key, trade_id=trade_id) - if custom_data: - data_entry = custom_data[0] - data_entry.cd_value = value_db - data_entry.updated_at = dt_now() - else: - data_entry = CustomData( - ft_trade_id=trade_id, - cd_key=key, - cd_type=value_type, - cd_value=value_db, - created_at=dt_now() - ) - - if CustomDataWrapper.use_db and value_db is not None: - data_entry.cd_value = value_db - CustomData.session.add(data_entry) - CustomData.session.commit() - elif not CustomDataWrapper.use_db: - cd_index = -1 - for index, data_entry in enumerate(CustomDataWrapper.custom_data): - if data_entry.ft_trade_id == trade_id and data_entry.cd_key == key: - cd_index = index - break - - if cd_index >= 0: - data_entry.cd_type = value_type - data_entry.cd_value = value_db - data_entry.updated_at = dt_now() - - CustomDataWrapper.custom_data[cd_index] = data_entry - else: - CustomDataWrapper.custom_data.append(data_entry) - - @staticmethod - def get_all_custom_data() -> List[CustomData]: - - if CustomDataWrapper.use_db: - return list(CustomData.query_cd()) - else: - return CustomDataWrapper.custom_data diff --git a/freqtrade/persistence/trade_model.py b/freqtrade/persistence/trade_model.py index ea03e2c29..7487c72b3 100644 --- a/freqtrade/persistence/trade_model.py +++ b/freqtrade/persistence/trade_model.py @@ -23,8 +23,7 @@ from freqtrade.exchange import (ROUND_DOWN, ROUND_UP, amount_to_contract_precisi from freqtrade.leverage import interest from freqtrade.misc import safe_value_fallback from freqtrade.persistence.base import ModelBase, SessionType -from freqtrade.persistence.custom_data import CustomData -from freqtrade.persistence.custom_data_middleware import CustomDataWrapper +from freqtrade.persistence.custom_data import CustomData, CustomDataWrapper from freqtrade.util import FtPrecise, dt_from_ts, dt_now, dt_ts @@ -345,7 +344,7 @@ class LocalTrade: id: int = 0 orders: List[Order] = [] - custom_data: List[CustomData] = [] + custom_data: List[_CustomData] = [] exchange: str = '' pair: str = '' @@ -1209,7 +1208,7 @@ class LocalTrade: def set_custom_data(self, key: str, value: Any) -> None: CustomDataWrapper.set_custom_data(key=key, value=value, trade_id=self.id) - def get_custom_data(self, key: Optional[str]) -> List[CustomData]: + def get_custom_data(self, key: Optional[str]) -> List[_CustomData]: return CustomDataWrapper.get_custom_data(key=key, trade_id=self.id) @property @@ -1467,7 +1466,7 @@ class Trade(ModelBase, LocalTrade): orders: Mapped[List[Order]] = relationship( "Order", order_by="Order.id", cascade="all, delete-orphan", lazy="selectin", innerjoin=True) # type: ignore - custom_data: Mapped[List[CustomData]] = relationship( + custom_data: Mapped[List[_CustomData]] = relationship( "CustomData", order_by="CustomData.id", cascade="all, delete-orphan", lazy="raise") # type: ignore @@ -1574,9 +1573,9 @@ class Trade(ModelBase, LocalTrade): Order.session.delete(order) for entry in self.custom_data: - CustomData.session.delete(entry) + _CustomData.session.delete(entry) - CustomData.session.commit() + _CustomData.session.commit() Trade.session.delete(self) Trade.commit() diff --git a/freqtrade/persistence/usedb_context.py b/freqtrade/persistence/usedb_context.py index 193f7021d..732f0b0f8 100644 --- a/freqtrade/persistence/usedb_context.py +++ b/freqtrade/persistence/usedb_context.py @@ -1,5 +1,5 @@ -from freqtrade.persistence.custom_data_middleware import CustomDataWrapper +from freqtrade.persistence.custom_data import CustomDataWrapper from freqtrade.persistence.pairlock_middleware import PairLocks from freqtrade.persistence.trade_model import Trade From 8dda28351e9e6728877a169553431e3f94ad3092 Mon Sep 17 00:00:00 2001 From: Matthias Date: Mon, 12 Feb 2024 20:25:26 +0100 Subject: [PATCH 27/46] Simplify custom_data stuff --- freqtrade/persistence/custom_data.py | 57 ++++++++++++++-------------- freqtrade/persistence/models.py | 6 +-- freqtrade/persistence/trade_model.py | 24 ++++++++++-- 3 files changed, 52 insertions(+), 35 deletions(-) diff --git a/freqtrade/persistence/custom_data.py b/freqtrade/persistence/custom_data.py index bf6056278..e8fa0d960 100644 --- a/freqtrade/persistence/custom_data.py +++ b/freqtrade/persistence/custom_data.py @@ -14,7 +14,7 @@ from freqtrade.util import dt_now logger = logging.getLogger(__name__) -class CustomData(ModelBase): +class _CustomData(ModelBase): """ CustomData database model Keeps records of metadata as key/value store @@ -41,6 +41,9 @@ class CustomData(ModelBase): created_at: Mapped[datetime] = mapped_column(DateTime, nullable=False, default=dt_now) updated_at: Mapped[Optional[datetime]] = mapped_column(DateTime, nullable=True) + # Empty container value - not persisted, but filled with cd_value on query + value: Any = None + def __repr__(self): create_time = (self.created_at.strftime(DATETIME_PRINT_FORMAT) if self.created_at is not None else None) @@ -52,7 +55,7 @@ class CustomData(ModelBase): @classmethod def query_cd(cls, key: Optional[str] = None, - trade_id: Optional[int] = None) -> Sequence['CustomData']: + trade_id: Optional[int] = None) -> Sequence['_CustomData']: """ Get all CustomData, if trade_id is not specified return will be for generic values not tied to a trade @@ -60,11 +63,11 @@ class CustomData(ModelBase): """ filters = [] if trade_id is not None: - filters.append(CustomData.ft_trade_id == trade_id) + filters.append(_CustomData.ft_trade_id == trade_id) if key is not None: - filters.append(CustomData.cd_key.ilike(key)) + filters.append(_CustomData.cd_key.ilike(key)) - return CustomData.session.scalars(select(CustomData).filter(*filters)).all() + return _CustomData.session.scalars(select(_CustomData).filter(*filters)).all() class CustomDataWrapper: @@ -75,9 +78,15 @@ class CustomDataWrapper: """ use_db = True - custom_data: List[CustomData] = [] + custom_data: List[_CustomData] = [] unserialized_types = ['bool', 'float', 'int', 'str'] + @staticmethod + def _convert_custom_data(data: _CustomData) -> _CustomData: + if data.cd_type not in CustomDataWrapper.unserialized_types: + data.value = json.loads(data.cd_value) + return data + @staticmethod def reset_custom_data() -> None: """ @@ -88,17 +97,15 @@ class CustomDataWrapper: @staticmethod def get_custom_data(key: Optional[str] = None, - trade_id: Optional[int] = None) -> CustomData: + trade_id: Optional[int] = None) -> List[_CustomData]: if trade_id is None: trade_id = 0 if CustomDataWrapper.use_db: - filtered_custom_data = [] - for data_entry in CustomData.query_cd(trade_id=trade_id, key=key): - if data_entry.cd_type not in CustomDataWrapper.unserialized_types: - data_entry.cd_value = json.loads(data_entry.cd_value) - filtered_custom_data.append(data_entry) - return filtered_custom_data + filtered_custom_data = _CustomData.session.scalars(select(_CustomData).filter( + _CustomData.ft_trade_id == trade_id, + _CustomData.cd_key.ilike(key))).all() + else: filtered_custom_data = [ data_entry for data_entry in CustomDataWrapper.custom_data @@ -109,19 +116,19 @@ class CustomDataWrapper: data_entry for data_entry in filtered_custom_data if (data_entry.cd_key.casefold() == key.casefold()) ] - return filtered_custom_data + return [CustomDataWrapper._convert_custom_data(d) for d in filtered_custom_data] @staticmethod def set_custom_data(key: str, value: Any, trade_id: Optional[int] = None) -> None: value_type = type(value).__name__ - value_db = None if value_type not in CustomDataWrapper.unserialized_types: try: value_db = json.dumps(value) except TypeError as e: logger.warning(f"could not serialize {key} value due to {e}") + return else: value_db = str(value) @@ -134,19 +141,19 @@ class CustomDataWrapper: data_entry.cd_value = value_db data_entry.updated_at = dt_now() else: - data_entry = CustomData( + data_entry = _CustomData( ft_trade_id=trade_id, cd_key=key, cd_type=value_type, cd_value=value_db, - created_at=dt_now() + created_at=dt_now(), ) + data_entry.value = value if CustomDataWrapper.use_db and value_db is not None: - data_entry.cd_value = value_db - CustomData.session.add(data_entry) - CustomData.session.commit() - elif not CustomDataWrapper.use_db: + _CustomData.session.add(data_entry) + _CustomData.session.commit() + else: cd_index = -1 for index, data_entry in enumerate(CustomDataWrapper.custom_data): if data_entry.ft_trade_id == trade_id and data_entry.cd_key == key: @@ -161,11 +168,3 @@ class CustomDataWrapper: CustomDataWrapper.custom_data[cd_index] = data_entry else: CustomDataWrapper.custom_data.append(data_entry) - - @staticmethod - def get_all_custom_data() -> List[CustomData]: - - if CustomDataWrapper.use_db: - return list(CustomData.query_cd()) - else: - return CustomDataWrapper.custom_data diff --git a/freqtrade/persistence/models.py b/freqtrade/persistence/models.py index 189b80fa6..1a69b271c 100644 --- a/freqtrade/persistence/models.py +++ b/freqtrade/persistence/models.py @@ -13,7 +13,7 @@ from sqlalchemy.pool import StaticPool from freqtrade.exceptions import OperationalException from freqtrade.persistence.base import ModelBase -from freqtrade.persistence.custom_data import CustomData +from freqtrade.persistence.custom_data import _CustomData from freqtrade.persistence.key_value_store import _KeyValueStoreModel from freqtrade.persistence.migrations import check_migrate from freqtrade.persistence.pairlock import PairLock @@ -79,8 +79,8 @@ def init_db(db_url: str) -> None: Order.session = Trade.session PairLock.session = Trade.session _KeyValueStoreModel.session = Trade.session - CustomData.session = scoped_session(sessionmaker(bind=engine, autoflush=True), - scopefunc=get_request_or_thread_id) + _CustomData.session = scoped_session(sessionmaker(bind=engine, autoflush=True), + scopefunc=get_request_or_thread_id) previous_tables = inspect(engine).get_table_names() ModelBase.metadata.create_all(engine) diff --git a/freqtrade/persistence/trade_model.py b/freqtrade/persistence/trade_model.py index 7487c72b3..55a075cc9 100644 --- a/freqtrade/persistence/trade_model.py +++ b/freqtrade/persistence/trade_model.py @@ -23,7 +23,7 @@ from freqtrade.exchange import (ROUND_DOWN, ROUND_UP, amount_to_contract_precisi from freqtrade.leverage import interest from freqtrade.misc import safe_value_fallback from freqtrade.persistence.base import ModelBase, SessionType -from freqtrade.persistence.custom_data import CustomData, CustomDataWrapper +from freqtrade.persistence.custom_data import CustomDataWrapper, _CustomData from freqtrade.util import FtPrecise, dt_from_ts, dt_now, dt_ts @@ -1206,10 +1206,28 @@ class LocalTrade: ] def set_custom_data(self, key: str, value: Any) -> None: + """ + Set custom data for this trade + :param key: key of the custom data + :param value: value of the custom data (must be JSON serializable) + """ CustomDataWrapper.set_custom_data(key=key, value=value, trade_id=self.id) - def get_custom_data(self, key: Optional[str]) -> List[_CustomData]: - return CustomDataWrapper.get_custom_data(key=key, trade_id=self.id) + def get_custom_data(self, key: str) -> Optional[_CustomData]: + """ + Get custom data for this trade + :param key: key of the custom data + """ + data = CustomDataWrapper.get_custom_data(key=key, trade_id=self.id) + if data: + return data[0] + return None + + def get_all_custom_data(self) -> List[_CustomData]: + """ + Get all custom data for this trade + """ + return CustomDataWrapper.get_custom_data(trade_id=self.id) @property def nr_of_successful_entries(self) -> int: From 790c7e386a3f2146cbee39e812c7f78fd9a551ed Mon Sep 17 00:00:00 2001 From: Matthias Date: Mon, 12 Feb 2024 20:27:56 +0100 Subject: [PATCH 28/46] re-adjust logic for custom_data in rpc module --- freqtrade/rpc/rpc.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/freqtrade/rpc/rpc.py b/freqtrade/rpc/rpc.py index 0c34c19ec..34d33ecde 100644 --- a/freqtrade/rpc/rpc.py +++ b/freqtrade/rpc/rpc.py @@ -1005,7 +1005,13 @@ class RPC: if trade is None: return [] # Query custom_data - custom_data = trade.get_custom_data(key=key) + custom_data = [] + if key: + data = trade.get_custom_data(key=key) + if data: + custom_data = [data] + else: + custom_data = trade.get_all_custom_data() return [ { 'id': data_entry.id, From 8364a704d6c14fc0b468c3dfce7245aeb994a7e0 Mon Sep 17 00:00:00 2001 From: Matthias Date: Mon, 12 Feb 2024 20:31:44 +0100 Subject: [PATCH 29/46] Fix a few sql gotchas --- freqtrade/persistence/custom_data.py | 1 + freqtrade/persistence/trade_model.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/freqtrade/persistence/custom_data.py b/freqtrade/persistence/custom_data.py index e8fa0d960..bf72510be 100644 --- a/freqtrade/persistence/custom_data.py +++ b/freqtrade/persistence/custom_data.py @@ -24,6 +24,7 @@ class _CustomData(ModelBase): - One metadata entry can only be associated with one Trade """ __tablename__ = 'trade_custom_data' + __allow_unmapped__ = True session: ClassVar[SessionType] # Uniqueness should be ensured over pair, order_id diff --git a/freqtrade/persistence/trade_model.py b/freqtrade/persistence/trade_model.py index 55a075cc9..1aa7cb607 100644 --- a/freqtrade/persistence/trade_model.py +++ b/freqtrade/persistence/trade_model.py @@ -1485,7 +1485,7 @@ class Trade(ModelBase, LocalTrade): "Order", order_by="Order.id", cascade="all, delete-orphan", lazy="selectin", innerjoin=True) # type: ignore custom_data: Mapped[List[_CustomData]] = relationship( - "CustomData", order_by="CustomData.id", cascade="all, delete-orphan", + "_CustomData", cascade="all, delete-orphan", lazy="raise") # type: ignore exchange: Mapped[str] = mapped_column(String(25), nullable=False) # type: ignore From 67b910835e05b76c170f406895a0c66a784742c8 Mon Sep 17 00:00:00 2001 From: Matthias Date: Mon, 12 Feb 2024 20:39:29 +0100 Subject: [PATCH 30/46] Simplify access to custom_data - users will usually only care about the value, not about the metadata. --- docs/strategy-advanced.md | 17 +++++++++++++---- freqtrade/persistence/trade_model.py | 12 +++++++++++- 2 files changed, 24 insertions(+), 5 deletions(-) diff --git a/docs/strategy-advanced.md b/docs/strategy-advanced.md index 69d4ced34..9f0b3c112 100644 --- a/docs/strategy-advanced.md +++ b/docs/strategy-advanced.md @@ -65,7 +65,7 @@ class AwesomeStrategy(IStrategy): for trade in Trade.get_open_order_trades(): fills = trade.select_filled_orders(trade.entry_side) if trade.pair == 'ETH/USDT': - trade_entry_type = trade.get_custom_data(key='entry_type').kv_value + trade_entry_type = trade.get_custom_data(key='entry_type') if trade_entry_type is None: trade_entry_type = 'breakout' if 'entry_1' in trade.enter_tag else 'dip' elif fills > 1: @@ -87,7 +87,7 @@ class AwesomeStrategy(IStrategy): dataframe, _ = self.dp.get_analyzed_dataframe(pair=pair, timeframe=self.timeframe) current_candle = dataframe.iloc[-1].squeeze() # store information about entry adjustment - existing_count = trade.get_custom_data(key='num_entry_adjustments').kv_value + existing_count = trade.get_custom_data('num_entry_adjustments', default=0) if not existing_count: existing_count = 1 else: @@ -102,8 +102,8 @@ class AwesomeStrategy(IStrategy): def custom_exit(self, pair: str, trade: Trade, current_time: datetime, current_rate: float, current_profit: float, **kwargs): - entry_adjustment_count = trade.get_custom_data(key='num_entry_adjustments').kv_value - trade_entry_type = trade.get_custom_data(key='entry_type').kv_value + entry_adjustment_count = trade.get_custom_data(key='num_entry_adjustments') + trade_entry_type = trade.get_custom_data(key='entry_type') if entry_adjustment_count is None: if current_profit > 0.01 and (current_time - timedelta(minutes=100) > trade.open_date_utc): return True, 'exit_1' @@ -123,6 +123,15 @@ class AwesomeStrategy(IStrategy): !!! Warning "Non-serializable data" If supplied data cannot be serialized a warning is logged and the entry for the specified `key` will contain `None` as data. +??? Note "All attributes" + custom-data has the following accessors through the Trade object (assumed as `trade` below): + + * `trade.get_custom_data(key='something', default=0)` - Returns the actual value given in the type provided. + * `trade.get_custom_data_entry(key='something')` - Returns the entry - including metadata. The value is accessible via `.value` property. + * `trade.set_custom_data(key='something', value={'some': 'value'})` - set or update the corresponding key for this trade. Value must be serializable - and we recommend to keep the stored data relatively small. + + "value" can be any type (both in setting and receiving) - but must be json serializable. + ## Dataframe access You may access dataframe in various strategy functions by querying it from dataprovider. diff --git a/freqtrade/persistence/trade_model.py b/freqtrade/persistence/trade_model.py index 1aa7cb607..abbc69f75 100644 --- a/freqtrade/persistence/trade_model.py +++ b/freqtrade/persistence/trade_model.py @@ -1213,7 +1213,17 @@ class LocalTrade: """ CustomDataWrapper.set_custom_data(key=key, value=value, trade_id=self.id) - def get_custom_data(self, key: str) -> Optional[_CustomData]: + def get_custom_data(self, key: str, default: Any = None) -> Any: + """ + Get custom data for this trade + :param key: key of the custom data + """ + data = CustomDataWrapper.get_custom_data(key=key, trade_id=self.id) + if data: + return data[0] + return default + + def get_custom_data_entry(self, key: str) -> Optional[_CustomData]: """ Get custom data for this trade :param key: key of the custom data From 6a6e3aacf33da282006d56d775e301ee33d8274b Mon Sep 17 00:00:00 2001 From: Matthias Date: Mon, 12 Feb 2024 20:51:44 +0100 Subject: [PATCH 31/46] Fix broken deletion --- freqtrade/persistence/custom_data.py | 4 ++++ freqtrade/persistence/trade_model.py | 3 +-- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/freqtrade/persistence/custom_data.py b/freqtrade/persistence/custom_data.py index bf72510be..0eb14738c 100644 --- a/freqtrade/persistence/custom_data.py +++ b/freqtrade/persistence/custom_data.py @@ -96,6 +96,10 @@ class CustomDataWrapper: if not CustomDataWrapper.use_db: CustomDataWrapper.custom_data = [] + @staticmethod + def delete_custom_data(trade_id: int) -> None: + _CustomData.session.query(_CustomData).filter(_CustomData.ft_trade_id == trade_id).delete() + @staticmethod def get_custom_data(key: Optional[str] = None, trade_id: Optional[int] = None) -> List[_CustomData]: diff --git a/freqtrade/persistence/trade_model.py b/freqtrade/persistence/trade_model.py index abbc69f75..a12a842c3 100644 --- a/freqtrade/persistence/trade_model.py +++ b/freqtrade/persistence/trade_model.py @@ -1600,8 +1600,7 @@ class Trade(ModelBase, LocalTrade): for order in self.orders: Order.session.delete(order) - for entry in self.custom_data: - _CustomData.session.delete(entry) + CustomDataWrapper.delete_custom_data(trade_id=self.id) _CustomData.session.commit() Trade.session.delete(self) From 304f52ab79c899a9b6f6cfeed95f125efcac0c89 Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 13 Feb 2024 07:06:32 +0100 Subject: [PATCH 32/46] Fix some minor custom-data bugs --- freqtrade/persistence/custom_data.py | 27 +++++++++++++++++++-------- freqtrade/persistence/trade_model.py | 8 ++++---- 2 files changed, 23 insertions(+), 12 deletions(-) diff --git a/freqtrade/persistence/custom_data.py b/freqtrade/persistence/custom_data.py index 0eb14738c..f5d6587bf 100644 --- a/freqtrade/persistence/custom_data.py +++ b/freqtrade/persistence/custom_data.py @@ -32,7 +32,7 @@ class _CustomData(ModelBase): __table_args__ = (UniqueConstraint('ft_trade_id', 'cd_key', name="_trade_id_cd_key"),) id = mapped_column(Integer, primary_key=True) - ft_trade_id = mapped_column(Integer, ForeignKey('trades.id'), index=True, default=0) + ft_trade_id = mapped_column(Integer, ForeignKey('trades.id'), index=True) trade = relationship("Trade", back_populates="custom_data") @@ -84,7 +84,15 @@ class CustomDataWrapper: @staticmethod def _convert_custom_data(data: _CustomData) -> _CustomData: - if data.cd_type not in CustomDataWrapper.unserialized_types: + if data.cd_type in CustomDataWrapper.unserialized_types: + data.value = data.cd_value + if data.cd_type == 'bool': + data.value = data.cd_value.lower() == 'true' + elif data.cd_type == 'int': + data.value = int(data.cd_value) + elif data.cd_type == 'float': + data.value = float(data.cd_value) + else: data.value = json.loads(data.cd_value) return data @@ -101,15 +109,18 @@ class CustomDataWrapper: _CustomData.session.query(_CustomData).filter(_CustomData.ft_trade_id == trade_id).delete() @staticmethod - def get_custom_data(key: Optional[str] = None, - trade_id: Optional[int] = None) -> List[_CustomData]: + def get_custom_data(*, trade_id: int, key: Optional[str] = None) -> List[_CustomData]: if trade_id is None: trade_id = 0 if CustomDataWrapper.use_db: - filtered_custom_data = _CustomData.session.scalars(select(_CustomData).filter( + filters = [ _CustomData.ft_trade_id == trade_id, - _CustomData.cd_key.ilike(key))).all() + ] + if key is not None: + filters.append(_CustomData.cd_key.ilike(key)) + filtered_custom_data = _CustomData.session.scalars(select(_CustomData).filter( + *filters)).all() else: filtered_custom_data = [ @@ -124,7 +135,7 @@ class CustomDataWrapper: return [CustomDataWrapper._convert_custom_data(d) for d in filtered_custom_data] @staticmethod - def set_custom_data(key: str, value: Any, trade_id: Optional[int] = None) -> None: + def set_custom_data(trade_id: int, key: str, value: Any) -> None: value_type = type(value).__name__ @@ -140,7 +151,7 @@ class CustomDataWrapper: if trade_id is None: trade_id = 0 - custom_data = CustomDataWrapper.get_custom_data(key=key, trade_id=trade_id) + custom_data = CustomDataWrapper.get_custom_data(trade_id=trade_id, key=key) if custom_data: data_entry = custom_data[0] data_entry.cd_value = value_db diff --git a/freqtrade/persistence/trade_model.py b/freqtrade/persistence/trade_model.py index a12a842c3..d2a92a554 100644 --- a/freqtrade/persistence/trade_model.py +++ b/freqtrade/persistence/trade_model.py @@ -1211,16 +1211,16 @@ class LocalTrade: :param key: key of the custom data :param value: value of the custom data (must be JSON serializable) """ - CustomDataWrapper.set_custom_data(key=key, value=value, trade_id=self.id) + CustomDataWrapper.set_custom_data(trade_id=self.id, key=key, value=value) def get_custom_data(self, key: str, default: Any = None) -> Any: """ Get custom data for this trade :param key: key of the custom data """ - data = CustomDataWrapper.get_custom_data(key=key, trade_id=self.id) + data = CustomDataWrapper.get_custom_data(trade_id=self.id, key=key) if data: - return data[0] + return data[0].value return default def get_custom_data_entry(self, key: str) -> Optional[_CustomData]: @@ -1228,7 +1228,7 @@ class LocalTrade: Get custom data for this trade :param key: key of the custom data """ - data = CustomDataWrapper.get_custom_data(key=key, trade_id=self.id) + data = CustomDataWrapper.get_custom_data(trade_id=self.id, key=key) if data: return data[0] return None From 9be7759e423f9e4d5382c3176225aae8b20a1d8f Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 13 Feb 2024 07:10:11 +0100 Subject: [PATCH 33/46] Add initial test for custom_data --- tests/persistence/test_trade_custom_data.py | 34 +++++++++++++++++++++ 1 file changed, 34 insertions(+) create mode 100644 tests/persistence/test_trade_custom_data.py diff --git a/tests/persistence/test_trade_custom_data.py b/tests/persistence/test_trade_custom_data.py new file mode 100644 index 000000000..d411a898f --- /dev/null +++ b/tests/persistence/test_trade_custom_data.py @@ -0,0 +1,34 @@ +import pytest +from sqlalchemy import select + +from freqtrade.persistence import Trade +from tests.conftest import create_mock_trades_usdt + + +@pytest.mark.usefixtures("init_persistence") +def test_trade_custom_data(fee): + create_mock_trades_usdt(fee) + + trade1 = Trade.session.scalars(select(Trade)).first() + + assert trade1.get_all_custom_data() == [] + trade1.set_custom_data('test_str', 'test_value') + trade1.set_custom_data('test_int', 1) + trade1.set_custom_data('test_float', 1.55) + trade1.set_custom_data('test_bool', True) + trade1.set_custom_data('test_dict', {'test': 'dict'}) + + assert trade1.get_custom_data('test_str') == 'test_value' + + assert trade1.get_custom_data('test_int') == 1 + assert isinstance(trade1.get_custom_data('test_int'), int) + + assert trade1.get_custom_data('test_float') == 1.55 + assert isinstance(trade1.get_custom_data('test_float'), float) + + assert trade1.get_custom_data('test_bool') is True + assert isinstance(trade1.get_custom_data('test_bool'), bool) + + assert trade1.get_custom_data('test_dict') == {'test': 'dict'} + assert isinstance(trade1.get_custom_data('test_dict'), dict) + assert len(trade1.get_all_custom_data()) == 5 From d5b21f2a32e9acc8460ea32b9030b2b67288a4bd Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 13 Feb 2024 07:16:09 +0100 Subject: [PATCH 34/46] Fix bug in backtest mode --- freqtrade/persistence/custom_data.py | 17 ++--------------- 1 file changed, 2 insertions(+), 15 deletions(-) diff --git a/freqtrade/persistence/custom_data.py b/freqtrade/persistence/custom_data.py index f5d6587bf..3ebcd0f48 100644 --- a/freqtrade/persistence/custom_data.py +++ b/freqtrade/persistence/custom_data.py @@ -110,8 +110,6 @@ class CustomDataWrapper: @staticmethod def get_custom_data(*, trade_id: int, key: Optional[str] = None) -> List[_CustomData]: - if trade_id is None: - trade_id = 0 if CustomDataWrapper.use_db: filters = [ @@ -170,17 +168,6 @@ class CustomDataWrapper: _CustomData.session.add(data_entry) _CustomData.session.commit() else: - cd_index = -1 - for index, data_entry in enumerate(CustomDataWrapper.custom_data): - if data_entry.ft_trade_id == trade_id and data_entry.cd_key == key: - cd_index = index - break - - if cd_index >= 0: - data_entry.cd_type = value_type - data_entry.cd_value = value_db - data_entry.updated_at = dt_now() - - CustomDataWrapper.custom_data[cd_index] = data_entry - else: + if not custom_data: CustomDataWrapper.custom_data.append(data_entry) + # Existing data will have updated interactively. From ab062d7bb145f163b78739ee69f2f1228879e3ee Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 13 Feb 2024 07:16:20 +0100 Subject: [PATCH 35/46] Add test to run in backtest mode --- tests/persistence/test_trade_custom_data.py | 23 +++++++++++++++------ 1 file changed, 17 insertions(+), 6 deletions(-) diff --git a/tests/persistence/test_trade_custom_data.py b/tests/persistence/test_trade_custom_data.py index d411a898f..12767b811 100644 --- a/tests/persistence/test_trade_custom_data.py +++ b/tests/persistence/test_trade_custom_data.py @@ -1,15 +1,23 @@ import pytest -from sqlalchemy import select -from freqtrade.persistence import Trade +from freqtrade.persistence import Trade, disable_database_use, enable_database_use +from freqtrade.persistence.custom_data import CustomDataWrapper from tests.conftest import create_mock_trades_usdt @pytest.mark.usefixtures("init_persistence") -def test_trade_custom_data(fee): - create_mock_trades_usdt(fee) +@pytest.mark.parametrize("use_db", [True, False]) +def test_trade_custom_data(fee, use_db): + if not use_db: + disable_database_use('5m') + Trade.reset_trades() + CustomDataWrapper.reset_custom_data() - trade1 = Trade.session.scalars(select(Trade)).first() + create_mock_trades_usdt(fee, use_db=use_db) + + trade1 = Trade.get_trades_proxy()[0] + if not use_db: + trade1.id = 1 assert trade1.get_all_custom_data() == [] trade1.set_custom_data('test_str', 'test_value') @@ -18,7 +26,10 @@ def test_trade_custom_data(fee): trade1.set_custom_data('test_bool', True) trade1.set_custom_data('test_dict', {'test': 'dict'}) + assert len(trade1.get_all_custom_data()) == 5 assert trade1.get_custom_data('test_str') == 'test_value' + trade1.set_custom_data('test_str', 'test_value_updated') + assert trade1.get_custom_data('test_str') == 'test_value_updated' assert trade1.get_custom_data('test_int') == 1 assert isinstance(trade1.get_custom_data('test_int'), int) @@ -31,4 +42,4 @@ def test_trade_custom_data(fee): assert trade1.get_custom_data('test_dict') == {'test': 'dict'} assert isinstance(trade1.get_custom_data('test_dict'), dict) - assert len(trade1.get_all_custom_data()) == 5 + enable_database_use() From e8ca9ce39b23172baa4ecc8e954b1a5858eed25d Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 20 Feb 2024 20:00:45 +0100 Subject: [PATCH 36/46] Add testconfirming correct functioning --- tests/persistence/test_trade_custom_data.py | 40 ++++++++++++++++++++- 1 file changed, 39 insertions(+), 1 deletion(-) diff --git a/tests/persistence/test_trade_custom_data.py b/tests/persistence/test_trade_custom_data.py index 12767b811..cc86342d2 100644 --- a/tests/persistence/test_trade_custom_data.py +++ b/tests/persistence/test_trade_custom_data.py @@ -2,7 +2,7 @@ import pytest from freqtrade.persistence import Trade, disable_database_use, enable_database_use from freqtrade.persistence.custom_data import CustomDataWrapper -from tests.conftest import create_mock_trades_usdt +from tests.conftest import EXMS, create_mock_trades_usdt, get_patched_freqtradebot @pytest.mark.usefixtures("init_persistence") @@ -43,3 +43,41 @@ def test_trade_custom_data(fee, use_db): assert trade1.get_custom_data('test_dict') == {'test': 'dict'} assert isinstance(trade1.get_custom_data('test_dict'), dict) enable_database_use() + + +def test_trade_custom_data_strategy_compat(mocker, default_conf_usdt, fee): + + mocker.patch(f'{EXMS}.get_rate', return_value=0.50) + mocker.patch('freqtrade.freqtradebot.FreqtradeBot.get_real_amount', return_value=None) + default_conf_usdt["minimal_roi"] = { + "0": 100 + } + + freqtrade = get_patched_freqtradebot(mocker, default_conf_usdt) + create_mock_trades_usdt(fee) + + trade1 = Trade.get_trades_proxy(pair='ADA/USDT')[0] + trade1.set_custom_data('test_str', 'test_value') + trade1.set_custom_data('test_int', 1) + + def custom_exit(pair, trade, **kwargs): + + if pair == 'ADA/USDT': + custom_val = trade.get_custom_data('test_str') + custom_val_i = trade.get_custom_data('test_int') + + return f"{custom_val}_{custom_val_i}" + + freqtrade.strategy.custom_exit = custom_exit + ff_spy = mocker.spy(freqtrade.strategy, 'custom_exit') + trades = Trade.get_open_trades() + freqtrade.exit_positions(trades) + Trade.commit() + + trade_after = Trade.get_trades_proxy(pair='ADA/USDT')[0] + assert trade_after.get_custom_data('test_str') == 'test_value' + assert trade_after.get_custom_data('test_int') == 1 + # 2 open pairs eligible for exit + assert ff_spy.call_count == 2 + + assert trade_after.exit_reason == 'test_value_1' From c511d65d2e10cc858591c36c2a1fc99673538eec Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 20 Feb 2024 20:25:12 +0100 Subject: [PATCH 37/46] Add backtesting test --- tests/persistence/test_trade_custom_data.py | 83 ++++++++++++++++++++- 1 file changed, 79 insertions(+), 4 deletions(-) diff --git a/tests/persistence/test_trade_custom_data.py b/tests/persistence/test_trade_custom_data.py index cc86342d2..fe73f5f6b 100644 --- a/tests/persistence/test_trade_custom_data.py +++ b/tests/persistence/test_trade_custom_data.py @@ -1,8 +1,14 @@ +from copy import deepcopy +from unittest.mock import MagicMock + import pytest +from freqtrade.data.history.history_utils import get_timerange +from freqtrade.optimize.backtesting import Backtesting from freqtrade.persistence import Trade, disable_database_use, enable_database_use from freqtrade.persistence.custom_data import CustomDataWrapper -from tests.conftest import EXMS, create_mock_trades_usdt, get_patched_freqtradebot +from tests.conftest import (EXMS, create_mock_trades_usdt, generate_test_data, + get_patched_freqtradebot, patch_exchange) @pytest.mark.usefixtures("init_persistence") @@ -49,9 +55,7 @@ def test_trade_custom_data_strategy_compat(mocker, default_conf_usdt, fee): mocker.patch(f'{EXMS}.get_rate', return_value=0.50) mocker.patch('freqtrade.freqtradebot.FreqtradeBot.get_real_amount', return_value=None) - default_conf_usdt["minimal_roi"] = { - "0": 100 - } + default_conf_usdt["minimal_roi"] = {"0": 100} freqtrade = get_patched_freqtradebot(mocker, default_conf_usdt) create_mock_trades_usdt(fee) @@ -81,3 +85,74 @@ def test_trade_custom_data_strategy_compat(mocker, default_conf_usdt, fee): assert ff_spy.call_count == 2 assert trade_after.exit_reason == 'test_value_1' + + +def test_trade_custom_data_strategy_backtest_compat(mocker, default_conf_usdt, fee): + + mocker.patch(f'{EXMS}.get_fee', fee) + mocker.patch(f"{EXMS}.get_min_pair_stake_amount", return_value=10) + mocker.patch(f"{EXMS}.get_max_pair_stake_amount", return_value=float('inf')) + mocker.patch(f"{EXMS}.get_max_leverage", return_value=10) + mocker.patch(f"{EXMS}.get_maintenance_ratio_and_amt", return_value=(0.1, 0.1)) + mocker.patch('freqtrade.optimize.backtesting.Backtesting._run_funding_fees') + + patch_exchange(mocker) + default_conf_usdt.update({ + "stake_amount": 100.0, + "max_open_trades": 2, + "dry_run_wallet": 1000.0, + "strategy": "StrategyTestV3", + "trading_mode": "futures", + "margin_mode": "isolated", + "stoploss": -2, + "minimal_roi": {"0": 100}, + }) + default_conf_usdt['pairlists'] = [{'method': 'StaticPairList', 'allow_inactive': True}] + backtesting = Backtesting(default_conf_usdt) + + df = generate_test_data(default_conf_usdt['timeframe'], 100, '2022-01-01 00:00:00+00:00') + + pair_exp = 'XRP/USDT:USDT' + + def custom_exit(pair, trade, **kwargs): + custom_val = trade.get_custom_data('test_str') + custom_val_i = trade.get_custom_data('test_int', 0) + + if pair == pair_exp: + trade.set_custom_data('test_str', 'test_value') + trade.set_custom_data('test_int', custom_val_i + 1) + + if custom_val_i >= 2: + return f"{custom_val}_{custom_val_i}" + + backtesting._set_strategy(backtesting.strategylist[0]) + processed = backtesting.strategy.advise_all_indicators({ + pair_exp: df, + 'BTC/USDT:USDT': df, + }) + + def fun(dataframe, *args, **kwargs): + dataframe.loc[dataframe.index == 50, 'enter_long'] = 1 + return dataframe + + backtesting.strategy.advise_entry = fun + backtesting.strategy.leverage = MagicMock(return_value=1) + backtesting.strategy.custom_exit = custom_exit + ff_spy = mocker.spy(backtesting.strategy, 'custom_exit') + + min_date, max_date = get_timerange(processed) + + result = backtesting.backtest( + processed=deepcopy(processed), + start_date=min_date, + end_date=max_date, + ) + results = result['results'] + assert not results.empty + assert len(results) == 2 + assert results['pair'][0] == pair_exp + assert results['pair'][1] == 'BTC/USDT:USDT' + assert results['exit_reason'][0] == 'test_value_2' + assert results['exit_reason'][1] == 'exit_signal' + + assert ff_spy.call_count == 7 From 4bbb3174b21771f2336f64ccf73c54dec626460d Mon Sep 17 00:00:00 2001 From: Matthias Date: Wed, 21 Feb 2024 06:52:22 +0100 Subject: [PATCH 38/46] re-enable use_database after bt test --- tests/persistence/test_trade_custom_data.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/persistence/test_trade_custom_data.py b/tests/persistence/test_trade_custom_data.py index fe73f5f6b..bf1d73e1d 100644 --- a/tests/persistence/test_trade_custom_data.py +++ b/tests/persistence/test_trade_custom_data.py @@ -156,3 +156,4 @@ def test_trade_custom_data_strategy_backtest_compat(mocker, default_conf_usdt, f assert results['exit_reason'][1] == 'exit_signal' assert ff_spy.call_count == 7 + enable_database_use() From c7fff45bef14cb8c3009378aa141cf2570334e3f Mon Sep 17 00:00:00 2001 From: Matthias Date: Thu, 22 Feb 2024 19:01:33 +0100 Subject: [PATCH 39/46] Fix test leakage --- tests/persistence/test_trade_custom_data.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tests/persistence/test_trade_custom_data.py b/tests/persistence/test_trade_custom_data.py index bf1d73e1d..15241aa93 100644 --- a/tests/persistence/test_trade_custom_data.py +++ b/tests/persistence/test_trade_custom_data.py @@ -48,7 +48,8 @@ def test_trade_custom_data(fee, use_db): assert trade1.get_custom_data('test_dict') == {'test': 'dict'} assert isinstance(trade1.get_custom_data('test_dict'), dict) - enable_database_use() + if not use_db: + enable_database_use() def test_trade_custom_data_strategy_compat(mocker, default_conf_usdt, fee): @@ -156,4 +157,4 @@ def test_trade_custom_data_strategy_backtest_compat(mocker, default_conf_usdt, f assert results['exit_reason'][1] == 'exit_signal' assert ff_spy.call_count == 7 - enable_database_use() + Backtesting.cleanup() From 6307e1630498e015e8df03fdc21a55e583160414 Mon Sep 17 00:00:00 2001 From: Matthias Date: Fri, 23 Feb 2024 06:45:28 +0100 Subject: [PATCH 40/46] Properly format notification date --- freqtrade/rpc/telegram.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/freqtrade/rpc/telegram.py b/freqtrade/rpc/telegram.py index d0e12cc4a..4f4ea17d3 100644 --- a/freqtrade/rpc/telegram.py +++ b/freqtrade/rpc/telegram.py @@ -33,7 +33,7 @@ from freqtrade.misc import chunks, plural from freqtrade.persistence import Trade from freqtrade.rpc import RPC, RPCException, RPCHandler from freqtrade.rpc.rpc_types import RPCEntryMsg, RPCExitMsg, RPCOrderMsg, RPCSendMsg -from freqtrade.util import dt_humanize, fmt_coin, round_value +from freqtrade.util import dt_humanize, fmt_coin, format_date, round_value MAX_MESSAGE_LENGTH = MessageLimit.MAX_TEXT_LENGTH @@ -1797,8 +1797,8 @@ class Telegram(RPCHandler): f"*Trade ID:* `{result['ft_trade_id']}`", f"*Type:* `{result['cd_type']}`", f"*Value:* `{result['cd_value']}`", - f"*Create Date:* `{result['created_at']}`", - f"*Update Date:* `{result['updated_at']}`" + f"*Create Date:* `{format_date(result['created_at'])}`", + f"*Update Date:* `{format_date(result['updated_at'])}`" ] # Filter empty lines using list-comprehension messages.append("\n".join([line for line in lines if line])) From 1176c16b93bd8b4d8375236b7a9011880b9f306c Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 3 Mar 2024 12:41:51 +0100 Subject: [PATCH 41/46] REmove unnecessary assignment --- freqtrade/optimize/backtesting.py | 1 - 1 file changed, 1 deletion(-) diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index 4b217bcf6..7147ad14f 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -337,7 +337,6 @@ class Backtesting: self.disable_database_use() PairLocks.reset_locks() Trade.reset_trades() - CustomDataWrapper.use_db = False CustomDataWrapper.reset_custom_data() self.rejected_trades = 0 self.timedout_entry_orders = 0 From 30b4f271522f233fe50ecd9e8b67e6120446d581 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 3 Mar 2024 12:46:46 +0100 Subject: [PATCH 42/46] Cleanup some nitpicks --- freqtrade/persistence/custom_data.py | 1 + freqtrade/persistence/trade_model.py | 4 +--- tests/persistence/test_persistence.py | 1 + 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/freqtrade/persistence/custom_data.py b/freqtrade/persistence/custom_data.py index 3ebcd0f48..81a9e7ad6 100644 --- a/freqtrade/persistence/custom_data.py +++ b/freqtrade/persistence/custom_data.py @@ -107,6 +107,7 @@ class CustomDataWrapper: @staticmethod def delete_custom_data(trade_id: int) -> None: _CustomData.session.query(_CustomData).filter(_CustomData.ft_trade_id == trade_id).delete() + _CustomData.session.commit() @staticmethod def get_custom_data(*, trade_id: int, key: Optional[str] = None) -> List[_CustomData]: diff --git a/freqtrade/persistence/trade_model.py b/freqtrade/persistence/trade_model.py index 06a6e818d..e74bc1f48 100644 --- a/freqtrade/persistence/trade_model.py +++ b/freqtrade/persistence/trade_model.py @@ -343,7 +343,6 @@ class LocalTrade: id: int = 0 orders: List[Order] = [] - custom_data: List[_CustomData] = [] exchange: str = '' pair: str = '' @@ -1507,7 +1506,7 @@ class Trade(ModelBase, LocalTrade): innerjoin=True) # type: ignore custom_data: Mapped[List[_CustomData]] = relationship( "_CustomData", cascade="all, delete-orphan", - lazy="raise") # type: ignore + lazy="raise") exchange: Mapped[str] = mapped_column(String(25), nullable=False) # type: ignore pair: Mapped[str] = mapped_column(String(25), nullable=False, index=True) # type: ignore @@ -1613,7 +1612,6 @@ class Trade(ModelBase, LocalTrade): CustomDataWrapper.delete_custom_data(trade_id=self.id) - _CustomData.session.commit() Trade.session.delete(self) Trade.commit() diff --git a/tests/persistence/test_persistence.py b/tests/persistence/test_persistence.py index 0e0e70ee8..18f28da2b 100644 --- a/tests/persistence/test_persistence.py +++ b/tests/persistence/test_persistence.py @@ -2099,6 +2099,7 @@ def test_Trade_object_idem(): 'get_mix_tag_performance', 'get_trading_volume', 'validate_string_len', + 'custom_data' ) EXCLUDES2 = ('trades', 'trades_open', 'bt_trades_open_pp', 'bt_open_open_trade_count', 'total_profit', 'from_json',) From c1ae110080e4b966efe1c5adf60b013b1b6a2e5b Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 3 Mar 2024 12:56:42 +0100 Subject: [PATCH 43/46] Improve documentation --- docs/strategy-advanced.md | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/docs/strategy-advanced.md b/docs/strategy-advanced.md index 9f0b3c112..36185676c 100644 --- a/docs/strategy-advanced.md +++ b/docs/strategy-advanced.md @@ -14,10 +14,10 @@ The call sequence of the methods described here is covered under [bot execution ## Storing information (Non-Persistent) !!! Warning "Deprecated" - This method of storing information is deprecated, and we do advise against using non-persistent storage. + This method of storing information is deprecated and we do advise against using non-persistent storage. Please use the below [Persistent Storing Information Section](#storing-information-persistent) instead. - It's content has therefore be collapsed. + It's content has therefore been collapsed. ??? Abstract "Storing information" Storing information can be accomplished by creating a new dictionary within the strategy class. @@ -49,11 +49,12 @@ The call sequence of the methods described here is covered under [bot execution ## Storing information (Persistent) -Storing information can also be performed in a persistent manner. Freqtrade allows storing/retrieving user custom information associated with a specific trade. +Freqtrade allows storing/retrieving user custom information associated with a specific trade in the database. -Using a trade object, information can be stored using `trade_obj.set_custom_data(key='my_key', value=my_value)` and retrieved using `trade_obj.get_custom_data(key='my_key')`. Each data entry is associated with a trade and a user supplied key (of type `string`). This means that this can only be used in callbacks that also provide a trade object. +Using a trade object, information can be stored using `trade.set_custom_data(key='my_key', value=my_value)` and retrieved using `trade.get_custom_data(key='my_key')`. Each data entry is associated with a trade and a user supplied key (of type `string`). This means that this can only be used in callbacks that also provide a trade object. -For the data to be able to be stored within the database it must be serialized. This is done by converting it to a JSON formatted string. +For the data to be able to be stored within the database, freqtrade must serialized the data. This is done by converting the data to a JSON formatted string. +Freqtrade will attempt to reverse this action on retrieval, so from a strategy perspective, this should not be relevant. ```python from freqtrade.persistence import Trade @@ -116,9 +117,11 @@ class AwesomeStrategy(IStrategy): return False, None ``` +The above is a simple example - there are simpler ways to retrieve trade data like entry-adjustments. + !!! Note It is recommended that simple data types are used `[bool, int, float, str]` to ensure no issues when serializing the data that needs to be stored. - Storing big junks of data may lead to unintended side-effects, like a database becoming big pretty fast (and as a consequence, also slow). + Storing big junks of data may lead to unintended side-effects, like a database becoming big (and as a consequence, also slow). !!! Warning "Non-serializable data" If supplied data cannot be serialized a warning is logged and the entry for the specified `key` will contain `None` as data. From ceb461a25285d9a0ce7fc44c6ceac0b1cccb3552 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 3 Mar 2024 12:58:10 +0100 Subject: [PATCH 44/46] Switch sequence of information documentation --- docs/strategy-advanced.md | 72 +++++++++++++++++++-------------------- 1 file changed, 36 insertions(+), 36 deletions(-) diff --git a/docs/strategy-advanced.md b/docs/strategy-advanced.md index 36185676c..debd5bc1b 100644 --- a/docs/strategy-advanced.md +++ b/docs/strategy-advanced.md @@ -11,42 +11,6 @@ The call sequence of the methods described here is covered under [bot execution !!! Tip Start off with a strategy template containing all available callback methods by running `freqtrade new-strategy --strategy MyAwesomeStrategy --template advanced` -## Storing information (Non-Persistent) - -!!! Warning "Deprecated" - This method of storing information is deprecated and we do advise against using non-persistent storage. - Please use the below [Persistent Storing Information Section](#storing-information-persistent) instead. - - It's content has therefore been collapsed. - -??? Abstract "Storing information" - Storing information can be accomplished by creating a new dictionary within the strategy class. - - The name of the variable can be chosen at will, but should be prefixed with `custom_` to avoid naming collisions with predefined strategy variables. - - ```python - class AwesomeStrategy(IStrategy): - # Create custom dictionary - custom_info = {} - - def populate_indicators(self, dataframe: DataFrame, metadata: dict) -> DataFrame: - # Check if the entry already exists - if not metadata["pair"] in self.custom_info: - # Create empty entry for this pair - self.custom_info[metadata["pair"]] = {} - - if "crosstime" in self.custom_info[metadata["pair"]]: - self.custom_info[metadata["pair"]]["crosstime"] += 1 - else: - self.custom_info[metadata["pair"]]["crosstime"] = 1 - ``` - - !!! Warning - The data is not persisted after a bot-restart (or config-reload). Also, the amount of data should be kept smallish (no DataFrames and such), otherwise the bot will start to consume a lot of memory and eventually run out of memory and crash. - - !!! Note - If the data is pair-specific, make sure to use pair as one of the keys in the dictionary. - ## Storing information (Persistent) Freqtrade allows storing/retrieving user custom information associated with a specific trade in the database. @@ -135,6 +99,42 @@ The above is a simple example - there are simpler ways to retrieve trade data li "value" can be any type (both in setting and receiving) - but must be json serializable. +## Storing information (Non-Persistent) + +!!! Warning "Deprecated" + This method of storing information is deprecated and we do advise against using non-persistent storage. + Please use [Persistent Storage](#storing-information-persistent) instead. + + It's content has therefore been collapsed. + +??? Abstract "Storing information" + Storing information can be accomplished by creating a new dictionary within the strategy class. + + The name of the variable can be chosen at will, but should be prefixed with `custom_` to avoid naming collisions with predefined strategy variables. + + ```python + class AwesomeStrategy(IStrategy): + # Create custom dictionary + custom_info = {} + + def populate_indicators(self, dataframe: DataFrame, metadata: dict) -> DataFrame: + # Check if the entry already exists + if not metadata["pair"] in self.custom_info: + # Create empty entry for this pair + self.custom_info[metadata["pair"]] = {} + + if "crosstime" in self.custom_info[metadata["pair"]]: + self.custom_info[metadata["pair"]]["crosstime"] += 1 + else: + self.custom_info[metadata["pair"]]["crosstime"] = 1 + ``` + + !!! Warning + The data is not persisted after a bot-restart (or config-reload). Also, the amount of data should be kept smallish (no DataFrames and such), otherwise the bot will start to consume a lot of memory and eventually run out of memory and crash. + + !!! Note + If the data is pair-specific, make sure to use pair as one of the keys in the dictionary. + ## Dataframe access You may access dataframe in various strategy functions by querying it from dataprovider. From 265a7123dad8a68a8df6d0e258fc41bdc9d81211 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 3 Mar 2024 13:10:50 +0100 Subject: [PATCH 45/46] Add explicit test for telegram functionality of list-custom-data --- tests/rpc/test_rpc_telegram.py | 46 ++++++++++++++++++++++++++++++++++ 1 file changed, 46 insertions(+) diff --git a/tests/rpc/test_rpc_telegram.py b/tests/rpc/test_rpc_telegram.py index 29a2b2723..3bd372b19 100644 --- a/tests/rpc/test_rpc_telegram.py +++ b/tests/rpc/test_rpc_telegram.py @@ -2657,3 +2657,49 @@ async def test_change_market_direction(default_conf, mocker, update) -> None: context.args = ["invalid"] await telegram._changemarketdir(update, context) assert telegram._rpc._freqtrade.strategy.market_direction == MarketDirection.LONG + + +async def test_telegram_list_custom_data(default_conf_usdt, update, ticker, fee, mocker) -> None: + + mocker.patch.multiple( + EXMS, + fetch_ticker=ticker, + get_fee=fee, + ) + telegram, _freqtradebot, msg_mock = get_telegram_testobject(mocker, default_conf_usdt) + + # Create some test data + create_mock_trades_usdt(fee) + # No trade id + context = MagicMock() + await telegram._list_custom_data(update=update, context=context) + assert msg_mock.call_count == 1 + assert 'Trade-id not set.' in msg_mock.call_args_list[0][0][0] + msg_mock.reset_mock() + + # + context.args = ['1'] + await telegram._list_custom_data(update=update, context=context) + assert msg_mock.call_count == 1 + assert ( + "Didn't find any custom-data entries for Trade ID: `1`" in msg_mock.call_args_list[0][0][0] + ) + msg_mock.reset_mock() + + # Add some custom data + trade1 = Trade.get_trades_proxy()[0] + trade1.set_custom_data('test_int', 1) + trade1.set_custom_data('test_dict', {'test': 'dict'}) + Trade.commit() + context.args = [f"{trade1.id}"] + await telegram._list_custom_data(update=update, context=context) + assert msg_mock.call_count == 3 + assert "Found custom-data entries: " in msg_mock.call_args_list[0][0][0] + assert ( + "*Key:* `test_int`\n*ID:* `1`\n*Trade ID:* `1`\n*Type:* `int`\n" + "*Value:* `1`\n*Create Date:*") in msg_mock.call_args_list[1][0][0] + assert ( + '*Key:* `test_dict`\n*ID:* `2`\n*Trade ID:* `1`\n*Type:* `dict`\n' + '*Value:* `{"test": "dict"}`\n*Create Date:* `') in msg_mock.call_args_list[2][0][0] + + msg_mock.reset_mock() From 255ea88638957d19fcd439987ae32d383243f810 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 3 Mar 2024 15:24:26 +0100 Subject: [PATCH 46/46] Add to telegram documentation --- docs/telegram-usage.md | 1 + freqtrade/rpc/telegram.py | 4 ++-- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/docs/telegram-usage.md b/docs/telegram-usage.md index e4dc02c76..2709baf9a 100644 --- a/docs/telegram-usage.md +++ b/docs/telegram-usage.md @@ -181,6 +181,7 @@ official commands. You can ask at any moment for help with `/help`. | `/locks` | Show currently locked pairs. | `/unlock ` | Remove the lock for this pair (or for this lock id). | `/marketdir [long | short | even | none]` | Updates the user managed variable that represents the current market direction. If no direction is provided, the currently set direction will be displayed. +| `/list_custom_data [key]` | List custom_data for Trade ID & Key combination. If no Key is supplied it will list all key-value pairs found for that Trade ID. | **Modify Trade states** | | `/forceexit | /fx ` | Instantly exits the given trade (Ignoring `minimum_roi`). | `/forceexit all | /fx all` | Instantly exits all open trades (Ignoring `minimum_roi`). diff --git a/freqtrade/rpc/telegram.py b/freqtrade/rpc/telegram.py index 4f4ea17d3..f7e7362ef 100644 --- a/freqtrade/rpc/telegram.py +++ b/freqtrade/rpc/telegram.py @@ -1668,6 +1668,8 @@ class Telegram(RPCHandler): "*/marketdir [long | short | even | none]:* `Updates the user managed variable " "that represents the current market direction. If no direction is provided `" "`the currently set market direction will be output.` \n" + "*/list_custom_data :* `List custom_data for Trade ID & Key combo.`\n" + "`If no Key is supplied it will list all key-value pairs found for that Trade ID.`" "_Statistics_\n" "------------\n" @@ -1691,8 +1693,6 @@ class Telegram(RPCHandler): "Avg. holding durations for buys and sells.`\n" "*/help:* `This help message`\n" "*/version:* `Show version`\n" - "*/list_custom_data :* `List custom_data for Trade ID & Key combo.`\n" - "`If no Key is supplied it will list all key-value pairs found for that Trade ID.`" ) await self._send_msg(message, parse_mode=ParseMode.MARKDOWN)