diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d187e650b..e18090f08 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -25,10 +25,10 @@ jobs: strategy: matrix: os: [ ubuntu-20.04, ubuntu-22.04 ] - python-version: ["3.8", "3.9", "3.10", "3.11"] + python-version: ["3.9", "3.10", "3.11"] steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Set up Python uses: actions/setup-python@v4 @@ -127,10 +127,10 @@ jobs: strategy: matrix: os: [ macos-latest ] - python-version: ["3.8", "3.9", "3.10", "3.11"] + python-version: ["3.9", "3.10", "3.11"] steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Set up Python uses: actions/setup-python@v4 @@ -237,10 +237,10 @@ jobs: strategy: matrix: os: [ windows-latest ] - python-version: ["3.8", "3.9", "3.10", "3.11"] + python-version: ["3.9", "3.10", "3.11"] steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Set up Python uses: actions/setup-python@v4 @@ -304,7 +304,7 @@ jobs: mypy_version_check: runs-on: ubuntu-22.04 steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Set up Python uses: actions/setup-python@v4 @@ -319,7 +319,7 @@ jobs: pre-commit: runs-on: ubuntu-22.04 steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - uses: actions/setup-python@v4 with: @@ -329,7 +329,7 @@ jobs: docs_check: runs-on: ubuntu-22.04 steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Documentation syntax run: | @@ -359,7 +359,7 @@ jobs: # Run pytest with "live" checks runs-on: ubuntu-22.04 steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Set up Python uses: actions/setup-python@v4 @@ -443,12 +443,12 @@ jobs: if: (github.event_name == 'push' || github.event_name == 'schedule' || github.event_name == 'release') && github.repository == 'freqtrade/freqtrade' steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Set up Python uses: actions/setup-python@v4 with: - python-version: "3.9" + python-version: "3.11" - name: Extract branch name shell: bash @@ -515,7 +515,7 @@ jobs: if: (github.event_name == 'push' || github.event_name == 'schedule' || github.event_name == 'release') && github.repository == 'freqtrade/freqtrade' steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Extract branch name shell: bash diff --git a/.github/workflows/docker_update_readme.yml b/.github/workflows/docker_update_readme.yml index 4587626f6..9d2f8b768 100644 --- a/.github/workflows/docker_update_readme.yml +++ b/.github/workflows/docker_update_readme.yml @@ -8,7 +8,7 @@ jobs: dockerHubDescription: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Docker Hub Description uses: peter-evans/dockerhub-description@v3 env: diff --git a/README.md b/README.md index 57c4e3a52..0cacfe703 100644 --- a/README.md +++ b/README.md @@ -59,7 +59,7 @@ Please find the complete documentation on the [freqtrade website](https://www.fr ## Features -- [x] **Based on Python 3.8+**: For botting on any operating system - Windows, macOS and Linux. +- [x] **Based on Python 3.9+**: For botting on any operating system - Windows, macOS and Linux. - [x] **Persistence**: Persistence is achieved through sqlite. - [x] **Dry-run**: Run the bot without paying money. - [x] **Backtesting**: Run a simulation of your buy/sell strategy. @@ -207,7 +207,7 @@ To run this bot we recommend you a cloud instance with a minimum of: ### Software requirements -- [Python >= 3.8](http://docs.python-guide.org/en/latest/starting/installation/) +- [Python >= 3.9](http://docs.python-guide.org/en/latest/starting/installation/) - [pip](https://pip.pypa.io/en/stable/installing/) - [git](https://git-scm.com/book/en/v2/Getting-Started-Installing-Git) - [TA-Lib](https://ta-lib.github.io/ta-lib-python/) diff --git a/build_helpers/TA_Lib-0.4.28-cp38-cp38-win_amd64.whl b/build_helpers/TA_Lib-0.4.28-cp38-cp38-win_amd64.whl deleted file mode 100644 index 34fecd677..000000000 Binary files a/build_helpers/TA_Lib-0.4.28-cp38-cp38-win_amd64.whl and /dev/null differ diff --git a/docs/configuration.md b/docs/configuration.md index d2519d58a..7303f78c7 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -177,7 +177,7 @@ Mandatory parameters are marked as **Required**, which means that they are requi | `exit_pricing.order_book_top` | Bot will use the top N rate in Order Book "price_side" to exit. I.e. a value of 2 will allow the bot to pick the 2nd ask rate in [Order Book Exit](#exit-price-with-orderbook-enabled)
*Defaults to `1`.*
**Datatype:** Positive Integer | `custom_price_max_distance_ratio` | Configure maximum distance ratio between current and custom entry or exit price.
*Defaults to `0.02` 2%).*
**Datatype:** Positive float | | **TODO** -| `use_exit_signal` | Use exit signals produced by the strategy in addition to the `minimal_roi`. [Strategy Override](#parameters-in-the-strategy).
*Defaults to `true`.*
**Datatype:** Boolean +| `use_exit_signal` | Use exit signals produced by the strategy in addition to the `minimal_roi`.
Setting this to false disables the usage of `"exit_long"` and `"exit_short"` columns. Has no influence on other exit methods (Stoploss, ROI, callbacks). [Strategy Override](#parameters-in-the-strategy).
*Defaults to `true`.*
**Datatype:** Boolean | `exit_profit_only` | Wait until the bot reaches `exit_profit_offset` before taking an exit decision. [Strategy Override](#parameters-in-the-strategy).
*Defaults to `false`.*
**Datatype:** Boolean | `exit_profit_offset` | Exit-signal is only active above this value. Only active in combination with `exit_profit_only=True`. [Strategy Override](#parameters-in-the-strategy).
*Defaults to `0.0`.*
**Datatype:** Float (as ratio) | `ignore_roi_if_entry_signal` | Do not exit if the entry signal is still active. This setting takes preference over `minimal_roi` and `use_exit_signal`. [Strategy Override](#parameters-in-the-strategy).
*Defaults to `false`.*
**Datatype:** Boolean diff --git a/docs/freqai-reinforcement-learning.md b/docs/freqai-reinforcement-learning.md index df4508c86..c5cda3bc3 100644 --- a/docs/freqai-reinforcement-learning.md +++ b/docs/freqai-reinforcement-learning.md @@ -237,11 +237,10 @@ class MyCoolRLModel(ReinforcementLearner): Reinforcement Learning models benefit from tracking training metrics. FreqAI has integrated Tensorboard to allow users to track training and evaluation performance across all coins and across all retrainings. Tensorboard is activated via the following command: ```bash -cd freqtrade tensorboard --logdir user_data/models/unique-id ``` -where `unique-id` is the `identifier` set in the `freqai` configuration file. This command must be run in a separate shell to view the output in their browser at 127.0.0.1:6006 (6006 is the default port used by Tensorboard). +where `unique-id` is the `identifier` set in the `freqai` configuration file. This command must be run in a separate shell to view the output in the browser at 127.0.0.1:6006 (6006 is the default port used by Tensorboard). ![tensorboard](assets/tensorboard.jpg) diff --git a/docs/index.md b/docs/index.md index 77542ae78..190e7e3c3 100644 --- a/docs/index.md +++ b/docs/index.md @@ -83,7 +83,7 @@ To run this bot we recommend you a linux cloud instance with a minimum of: Alternatively -- Python 3.8+ +- Python 3.9+ - pip (pip3) - git - TA-Lib diff --git a/docs/installation.md b/docs/installation.md index eab0171c5..a87a3ff4e 100644 --- a/docs/installation.md +++ b/docs/installation.md @@ -24,7 +24,7 @@ The easiest way to install and run Freqtrade is to clone the bot Github reposito The `stable` branch contains the code of the last release (done usually once per month on an approximately one week old snapshot of the `develop` branch to prevent packaging bugs, so potentially it's more stable). !!! Note - Python3.8 or higher and the corresponding `pip` are assumed to be available. The install-script will warn you and stop if that's not the case. `git` is also needed to clone the Freqtrade repository. + Python3.9 or higher and the corresponding `pip` are assumed to be available. The install-script will warn you and stop if that's not the case. `git` is also needed to clone the Freqtrade repository. Also, python headers (`python-dev` / `python-devel`) must be available for the installation to complete successfully. !!! Warning "Up-to-date clock" @@ -42,7 +42,7 @@ These requirements apply to both [Script Installation](#script-installation) and ### Install guide -* [Python >= 3.8.x](http://docs.python-guide.org/en/latest/starting/installation/) +* [Python >= 3.9](http://docs.python-guide.org/en/latest/starting/installation/) * [pip](https://pip.pypa.io/en/stable/installing/) * [git](https://git-scm.com/book/en/v2/Getting-Started-Installing-Git) * [virtualenv](https://virtualenv.pypa.io/en/stable/installation.html) (Recommended) @@ -54,7 +54,7 @@ We've included/collected install instructions for Ubuntu, MacOS, and Windows. Th OS Specific steps are listed first, the [Common](#common) section below is necessary for all systems. !!! Note - Python3.8 or higher and the corresponding pip are assumed to be available. + Python3.9 or higher and the corresponding pip are assumed to be available. === "Debian/Ubuntu" #### Install necessary dependencies @@ -169,7 +169,7 @@ You can as well update, configure and reset the codebase of your bot with `./scr ** --install ** With this option, the script will install the bot and most dependencies: -You will need to have git and python3.8+ installed beforehand for this to work. +You will need to have git and python3.9+ installed beforehand for this to work. * Mandatory software as: `ta-lib` * Setup your virtualenv under `.venv/` diff --git a/docs/requirements-docs.txt b/docs/requirements-docs.txt index 052dd9e5e..e2292e640 100644 --- a/docs/requirements-docs.txt +++ b/docs/requirements-docs.txt @@ -1,6 +1,6 @@ markdown==3.4.4 mkdocs==1.5.2 -mkdocs-material==9.2.5 +mkdocs-material==9.2.8 mdx_truly_sane_lists==1.3 pymdown-extensions==10.3 jinja2==3.1.2 diff --git a/docs/strategy-customization.md b/docs/strategy-customization.md index b8ce22843..005127715 100644 --- a/docs/strategy-customization.md +++ b/docs/strategy-customization.md @@ -264,7 +264,7 @@ def populate_entry_trend(self, dataframe: DataFrame, metadata: dict) -> DataFram ### Exit signal rules Edit the method `populate_exit_trend()` into your strategy file to update your exit strategy. -The exit-signal is only used for exits if `use_exit_signal` is set to true in the configuration. +The exit-signal can be suppressed by setting `use_exit_signal` to false in the configuration or strategy. `use_exit_signal` will not influence [signal collision rules](#colliding-signals) - which will still apply and can prevent entries. It's important to always return the dataframe without removing/modifying the columns `"open", "high", "low", "close", "volume"`, otherwise these fields would contain something unexpected. diff --git a/docs/strategy_analysis_example.md b/docs/strategy_analysis_example.md index 06dd33bc2..846c53238 100644 --- a/docs/strategy_analysis_example.md +++ b/docs/strategy_analysis_example.md @@ -167,7 +167,7 @@ trades.groupby("pair")["exit_reason"].value_counts() # Plotting equity line (starting with 0 on day 1 and adding daily profit for each backtested day) from freqtrade.configuration import Configuration -from freqtrade.data.btanalysis import load_backtest_data, load_backtest_stats +from freqtrade.data.btanalysis import load_backtest_stats import plotly.express as px import pandas as pd @@ -178,20 +178,8 @@ import pandas as pd stats = load_backtest_stats(backtest_dir) strategy_stats = stats['strategy'][strategy] -dates = [] -profits = [] -for date_profit in strategy_stats['daily_profit']: - dates.append(date_profit[0]) - profits.append(date_profit[1]) - -equity = 0 -equity_daily = [] -for daily_profit in profits: - equity_daily.append(equity) - equity += float(daily_profit) - - -df = pd.DataFrame({'dates': dates,'equity_daily': equity_daily}) +df = pd.DataFrame(columns=['dates','equity'], data=strategy_stats['daily_profit']) +df['equity_daily'] = df['equity'].cumsum() fig = px.line(df, x="dates", y="equity_daily") fig.show() diff --git a/docs/windows_installation.md b/docs/windows_installation.md index db785a1fc..5ac3d5e3d 100644 --- a/docs/windows_installation.md +++ b/docs/windows_installation.md @@ -24,7 +24,7 @@ git clone https://github.com/freqtrade/freqtrade.git Install ta-lib according to the [ta-lib documentation](https://github.com/mrjbq7/ta-lib#windows). -As compiling from source on windows has heavy dependencies (requires a partial visual studio installation), Freqtrade provides these dependencies (in the binary wheel format) for the latest 3 Python versions (3.8, 3.9, 3.10 and 3.11) and for 64bit Windows. +As compiling from source on windows has heavy dependencies (requires a partial visual studio installation), Freqtrade provides these dependencies (in the binary wheel format) for the latest 3 Python versions (3.9, 3.10 and 3.11) and for 64bit Windows. These Wheels are also used by CI running on windows, and are therefore tested together with freqtrade. Other versions must be downloaded from the above link. diff --git a/freqtrade/__main__.py b/freqtrade/__main__.py index fc45bdf61..ed950fa01 100755 --- a/freqtrade/__main__.py +++ b/freqtrade/__main__.py @@ -3,7 +3,7 @@ __main__.py for Freqtrade To launch Freqtrade as a module -> python -m freqtrade (with Python >= 3.8) +> python -m freqtrade (with Python >= 3.9) """ from freqtrade import main diff --git a/freqtrade/data/converter.py b/freqtrade/data/converter.py index 324ce5def..7d9014e87 100644 --- a/freqtrade/data/converter.py +++ b/freqtrade/data/converter.py @@ -105,7 +105,7 @@ def ohlcv_fill_up_missing_data(dataframe: DataFrame, timeframe: str, pair: str) df = dataframe.resample(resample_interval, on='date').agg(ohlcv_dict) # Forwardfill close for missing columns - df['close'] = df['close'].fillna(method='ffill') + df['close'] = df['close'].ffill() # Use close for "open, high, low" df.loc[:, ['open', 'high', 'low']] = df[['open', 'high', 'low']].fillna( value={'open': df['close'], diff --git a/freqtrade/exchange/bybit.py b/freqtrade/exchange/bybit.py index 41e667fb5..d9b7c817f 100644 --- a/freqtrade/exchange/bybit.py +++ b/freqtrade/exchange/bybit.py @@ -6,8 +6,7 @@ from typing import Any, Dict, List, Optional, Tuple import ccxt from freqtrade.constants import BuySell -from freqtrade.enums import MarginMode, PriceType, TradingMode -from freqtrade.enums.candletype import CandleType +from freqtrade.enums import CandleType, MarginMode, PriceType, TradingMode from freqtrade.exceptions import DDosProtection, OperationalException, TemporaryError from freqtrade.exchange import Exchange from freqtrade.exchange.common import retrier diff --git a/freqtrade/exchange/exchange.py b/freqtrade/exchange/exchange.py index 163e9a2a2..ddb00ecef 100644 --- a/freqtrade/exchange/exchange.py +++ b/freqtrade/exchange/exchange.py @@ -23,8 +23,7 @@ from freqtrade.constants import (DEFAULT_AMOUNT_RESERVE_PERCENT, NON_OPEN_EXCHAN BuySell, Config, EntryExit, ExchangeConfig, ListPairsWithTimeframes, MakerTaker, OBLiteral, PairWithTimeframe) from freqtrade.data.converter import clean_ohlcv_dataframe, ohlcv_to_dataframe, trades_dict_to_list -from freqtrade.enums import OPTIMIZE_MODES, CandleType, MarginMode, TradingMode -from freqtrade.enums.pricetype import PriceType +from freqtrade.enums import OPTIMIZE_MODES, CandleType, MarginMode, PriceType, TradingMode from freqtrade.exceptions import (DDosProtection, ExchangeError, InsufficientFundsError, InvalidOrderException, OperationalException, PricingError, RetryableOrderError, TemporaryError) diff --git a/freqtrade/exchange/okx.py b/freqtrade/exchange/okx.py index 7b1c90515..c0629240d 100644 --- a/freqtrade/exchange/okx.py +++ b/freqtrade/exchange/okx.py @@ -4,8 +4,7 @@ from typing import Any, Dict, List, Optional, Tuple import ccxt from freqtrade.constants import BuySell -from freqtrade.enums import CandleType, MarginMode, TradingMode -from freqtrade.enums.pricetype import PriceType +from freqtrade.enums import CandleType, MarginMode, PriceType, TradingMode from freqtrade.exceptions import (DDosProtection, OperationalException, RetryableOrderError, TemporaryError) from freqtrade.exchange import Exchange, date_minus_candles diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index 58cc8f862..1a24009b3 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -21,9 +21,8 @@ from freqtrade.enums import (ExitCheckTuple, ExitType, RPCMessageType, RunMode, State, TradingMode) from freqtrade.exceptions import (DependencyException, ExchangeError, InsufficientFundsError, InvalidOrderException, PricingError) -from freqtrade.exchange import (ROUND_DOWN, ROUND_UP, timeframe_to_minutes, timeframe_to_next_date, - timeframe_to_seconds) -from freqtrade.exchange.common import remove_exchange_credentials +from freqtrade.exchange import (ROUND_DOWN, ROUND_UP, remove_exchange_credentials, + timeframe_to_minutes, timeframe_to_next_date, timeframe_to_seconds) from freqtrade.misc import safe_value_fallback, safe_value_fallback2 from freqtrade.mixins import LoggingMixin from freqtrade.persistence import Order, PairLocks, Trade, init_db @@ -373,7 +372,10 @@ class FreqtradeBot(LoggingMixin): "Order is older than 5 days. Assuming order was fully cancelled.") fo = order.to_ccxt_object() fo['status'] = 'canceled' - self.handle_cancel_order(fo, order.trade, constants.CANCEL_REASON['TIMEOUT']) + self.handle_cancel_order( + fo, order.order_id, order.trade, + constants.CANCEL_REASON['TIMEOUT'] + ) except ExchangeError as e: @@ -440,13 +442,6 @@ class FreqtradeBot(LoggingMixin): if fo and fo['status'] == 'open': # Assume this as the open stoploss order trade.stoploss_order_id = order.order_id - elif order.ft_order_side == trade.exit_side: - if fo and fo['status'] == 'open': - # Assume this as the open order - trade.open_order_id = order.order_id - elif order.ft_order_side == trade.entry_side: - if fo and fo['status'] == 'open': - trade.open_order_id = order.order_id if fo: logger.info(f"Found {order} for trade {trade}.") self.update_trade_state(trade, order.order_id, fo, @@ -473,8 +468,6 @@ class FreqtradeBot(LoggingMixin): safe_value_fallback(order, 'lastTradeTimestamp', 'timestamp') // 1000, tz=timezone.utc) trade.orders.append(order_obj) - # TODO: how do we handle open_order_id ... - Trade.commit() prev_exit_reason = trade.exit_reason trade.exit_reason = ExitType.SOLD_ON_EXCHANGE.value self.update_trade_state(trade, order['id'], order) @@ -490,7 +483,10 @@ class FreqtradeBot(LoggingMixin): Trade.commit() except ExchangeError: - logger.warning("Error finding onexchange order") + logger.warning("Error finding onexchange order.") + except Exception: + # catching https://github.com/freqtrade/freqtrade/issues/9025 + logger.warning("Error finding onexchange order", exc_info=True) # # BUY / enter positions / open trades logic and methods # @@ -612,7 +608,8 @@ class FreqtradeBot(LoggingMixin): # Walk through each pair and check if it needs changes for trade in Trade.get_open_trades(): # If there is any open orders, wait for them to finish. - if trade.open_order_id is None: + # TODO Remove to allow mul open orders + if not trade.has_open_orders: # Do a wallets update (will be ratelimited to once per hour) self.wallets.update(False) try: @@ -846,7 +843,6 @@ class FreqtradeBot(LoggingMixin): open_rate_requested=enter_limit_requested, open_date=open_date, exchange=self.exchange.id, - open_order_id=order_id, strategy=self.strategy.get_strategy_name(), enter_tag=enter_tag, timeframe=timeframe_to_minutes(self.config['timeframe']), @@ -867,7 +863,6 @@ class FreqtradeBot(LoggingMixin): trade.is_open = True trade.fee_open_currency = None trade.open_rate_requested = enter_limit_requested - trade.open_order_id = order_id trade.orders.append(order_obj) trade.recalc_trade_from_orders() @@ -1077,7 +1072,7 @@ class FreqtradeBot(LoggingMixin): trades_closed = 0 for trade in trades: - if trade.open_order_id is None and not self.wallets.check_exit_amount(trade): + if not trade.has_open_orders and not self.wallets.check_exit_amount(trade): logger.warning( f'Not enough {trade.safe_base_currency} in wallet to exit {trade}. ' 'Trying to recover.') @@ -1095,7 +1090,7 @@ class FreqtradeBot(LoggingMixin): logger.warning( f'Unable to handle stoploss on exchange for {trade.pair}: {exception}') # Check if we can sell our current pair - if trade.open_order_id is None and trade.is_open and self.handle_trade(trade): + if not trade.has_open_orders and trade.is_open and self.handle_trade(trade): trades_closed += 1 except DependencyException as exception: @@ -1214,7 +1209,6 @@ class FreqtradeBot(LoggingMixin): """ logger.debug('Handling stoploss on exchange %s ...', trade) - stoploss_order = None try: @@ -1237,7 +1231,7 @@ class FreqtradeBot(LoggingMixin): self.handle_protections(trade.pair, trade.trade_direction) return True - if trade.open_order_id or not trade.is_open: + if trade.has_open_orders or not trade.is_open: # Trade has an open Buy or Sell order, Stoploss-handling can't happen in this case # as the Amount on the exchange is tied up in another trade. # The trade can be closed already (sell-order fill confirmation came in this iteration) @@ -1321,27 +1315,33 @@ class FreqtradeBot(LoggingMixin): Timeout setting takes priority over limit order adjustment request. :return: None """ - for trade in Trade.get_open_order_trades(): - try: - if not trade.open_order_id: + for trade in Trade.get_open_trades(): + for open_order in trade.open_orders: + try: + order = self.exchange.fetch_order(open_order.order_id, trade.pair) + + except (ExchangeError): + logger.info( + 'Cannot query order for %s due to %s', trade, traceback.format_exc() + ) continue - order = self.exchange.fetch_order(trade.open_order_id, trade.pair) - except (ExchangeError): - logger.info('Cannot query order for %s due to %s', trade, traceback.format_exc()) - continue - fully_cancelled = self.update_trade_state(trade, trade.open_order_id, order) - not_closed = order['status'] == 'open' or fully_cancelled - order_obj = trade.select_order_by_order_id(trade.open_order_id) + fully_cancelled = self.update_trade_state(trade, open_order.order_id, order) + not_closed = order['status'] == 'open' or fully_cancelled - if not_closed: - if fully_cancelled or (order_obj and self.strategy.ft_check_timed_out( - trade, order_obj, datetime.now(timezone.utc))): - self.handle_cancel_order(order, trade, constants.CANCEL_REASON['TIMEOUT']) - else: - self.replace_order(order, order_obj, trade) + if not_closed: + if fully_cancelled or ( + open_order and self.strategy.ft_check_timed_out( + trade, open_order, datetime.now(timezone.utc) + ) + ): + self.handle_cancel_order( + order, open_order.order_id, trade, constants.CANCEL_REASON['TIMEOUT'] + ) + else: + self.replace_order(order, open_order, trade) - def handle_cancel_order(self, order: Dict, trade: Trade, reason: str) -> None: + def handle_cancel_order(self, order: Dict, order_id: str, trade: Trade, reason: str) -> None: """ Check if current analyzed order timed out and cancel if necessary. :param order: Order dict grabbed with exchange.fetch_order() @@ -1349,9 +1349,9 @@ class FreqtradeBot(LoggingMixin): :return: None """ if order['side'] == trade.entry_side: - self.handle_cancel_enter(trade, order, reason) + self.handle_cancel_enter(trade, order, order_id, reason) else: - canceled = self.handle_cancel_exit(trade, order, reason) + canceled = self.handle_cancel_exit(trade, order, order_id, reason) canceled_count = trade.get_exit_order_count() max_timeouts = self.config.get('unfilledtimeout', {}).get('exit_timeout_count', 0) if canceled and max_timeouts > 0 and canceled_count >= max_timeouts: @@ -1406,7 +1406,7 @@ class FreqtradeBot(LoggingMixin): cancel_reason = constants.CANCEL_REASON['USER_CANCEL'] if order_obj.price != adjusted_entry_price: # cancel existing order if new price is supplied or None - self.handle_cancel_enter(trade, order, cancel_reason, + self.handle_cancel_enter(trade, order, order_obj.order_id, cancel_reason, replacing=replacing) if adjusted_entry_price: # place new order only if new price is supplied @@ -1434,25 +1434,28 @@ class FreqtradeBot(LoggingMixin): :return: None """ - for trade in Trade.get_open_order_trades(): - if not trade.open_order_id: - continue - try: - order = self.exchange.fetch_order(trade.open_order_id, trade.pair) - except (ExchangeError): - logger.info('Cannot query order for %s due to %s', trade, traceback.format_exc()) - continue + for trade in Trade.get_open_trades(): + for open_order in trade.open_orders: + try: + order = self.exchange.fetch_order(open_order.order_id, trade.pair) + except (ExchangeError): + logger.info("Can't query order for %s due to %s", trade, traceback.format_exc()) + continue - if order['side'] == trade.entry_side: - self.handle_cancel_enter(trade, order, constants.CANCEL_REASON['ALL_CANCELLED']) + if order['side'] == trade.entry_side: + self.handle_cancel_enter( + trade, order, open_order.order_id, constants.CANCEL_REASON['ALL_CANCELLED'] + ) - elif order['side'] == trade.exit_side: - self.handle_cancel_exit(trade, order, constants.CANCEL_REASON['ALL_CANCELLED']) + elif order['side'] == trade.exit_side: + self.handle_cancel_exit( + trade, order, open_order.order_id, constants.CANCEL_REASON['ALL_CANCELLED'] + ) Trade.commit() def handle_cancel_enter( - self, trade: Trade, order: Dict, reason: str, - replacing: Optional[bool] = False + self, trade: Trade, order: Dict, order_id: str, + reason: str, replacing: Optional[bool] = False ) -> bool: """ entry cancel - cancel order @@ -1461,7 +1464,7 @@ class FreqtradeBot(LoggingMixin): """ was_trade_fully_canceled = False side = trade.entry_side.capitalize() - if not trade.open_order_id: + if not trade.has_open_orders: logger.warning(f"No open order for {trade}.") return False @@ -1474,16 +1477,16 @@ class FreqtradeBot(LoggingMixin): if filled_val > 0 and minstake and filled_stake < minstake: logger.warning( - f"Order {trade.open_order_id} for {trade.pair} not cancelled, " + f"Order {order_id} for {trade.pair} not cancelled, " f"as the filled amount of {filled_val} would result in an unexitable trade.") return False - corder = self.exchange.cancel_order_with_result(trade.open_order_id, trade.pair, + corder = self.exchange.cancel_order_with_result(order_id, trade.pair, trade.amount) # Avoid race condition where the order could not be cancelled coz its already filled. # Simply bailing here is the only safe way - as this order will then be # handled in the next iteration. if corder.get('status') not in constants.NON_OPEN_EXCHANGE_STATES: - logger.warning(f"Order {trade.open_order_id} for {trade.pair} not cancelled.") + logger.warning(f"Order {order_id} for {trade.pair} not cancelled.") return False else: # Order was cancelled already, so we can reuse the existing dict @@ -1503,14 +1506,12 @@ class FreqtradeBot(LoggingMixin): was_trade_fully_canceled = True reason += f", {constants.CANCEL_REASON['FULLY_CANCELLED']}" else: - self.update_trade_state(trade, trade.open_order_id, corder) - trade.open_order_id = None + self.update_trade_state(trade, order_id, corder) logger.info(f'{side} Order timeout for {trade}.') else: # update_trade_state (and subsequently recalc_trade_from_orders) will handle updates # to the trade object - self.update_trade_state(trade, trade.open_order_id, corder) - trade.open_order_id = None + self.update_trade_state(trade, order_id, corder) logger.info(f'Partial {trade.entry_side} order timeout for {trade}.') reason += f", {constants.CANCEL_REASON['PARTIALLY_FILLED']}" @@ -1520,7 +1521,10 @@ class FreqtradeBot(LoggingMixin): reason=reason) return was_trade_fully_canceled - def handle_cancel_exit(self, trade: Trade, order: Dict, reason: str) -> bool: + def handle_cancel_exit( + self, trade: Trade, order: Dict, order_id: str, + reason: str + ) -> bool: """ exit order cancel - cancel order and update trade :return: True if exit order was cancelled, false otherwise @@ -1528,17 +1532,18 @@ class FreqtradeBot(LoggingMixin): cancelled = False # Cancelled orders may have the status of 'canceled' or 'closed' if order['status'] not in constants.NON_OPEN_EXCHANGE_STATES: - filled_val: float = order.get('filled', 0.0) or 0.0 - filled_rem_stake = trade.stake_amount - filled_val * trade.open_rate + filled_amt: float = order.get('filled', 0.0) or 0.0 + # Filled val is in quote currency (after leverage) + filled_rem_stake = trade.stake_amount - (filled_amt * trade.open_rate / trade.leverage) minstake = self.exchange.get_min_pair_stake_amount( trade.pair, trade.open_rate, self.strategy.stoploss) # Double-check remaining amount - if filled_val > 0: + if filled_amt > 0: reason = constants.CANCEL_REASON['PARTIALLY_FILLED'] if minstake and filled_rem_stake < minstake: logger.warning( - f"Order {trade.open_order_id} for {trade.pair} not cancelled, as " - f"the filled amount of {filled_val} would result in an unexitable trade.") + f"Order {order_id} for {trade.pair} not cancelled, as " + f"the filled amount of {filled_amt} would result in an unexitable trade.") reason = constants.CANCEL_REASON['PARTIALLY_FILLED_KEEP_OPEN'] self._notify_exit_cancel( @@ -1554,7 +1559,7 @@ class FreqtradeBot(LoggingMixin): order['id'], trade.pair, trade.amount) except InvalidOrderException: logger.exception( - f"Could not cancel {trade.exit_side} order {trade.open_order_id}") + f"Could not cancel {trade.exit_side} order {order_id}") return False # Set exit_reason for fill message @@ -1563,14 +1568,12 @@ class FreqtradeBot(LoggingMixin): # Order might be filled above in odd timing issues. if order.get('status') in ('canceled', 'cancelled'): trade.exit_reason = None - trade.open_order_id = None else: trade.exit_reason = exit_reason_prev cancelled = True else: reason = constants.CANCEL_REASON['CANCELLED_ON_EXCHANGE'] trade.exit_reason = None - trade.open_order_id = None self.update_trade_state(trade, order['id'], order) @@ -1704,7 +1707,6 @@ class FreqtradeBot(LoggingMixin): order_obj = Order.parse_from_ccxt_object(order, trade.pair, trade.exit_side, amount, limit) trade.orders.append(order_obj) - trade.open_order_id = order['id'] trade.exit_order_status = '' trade.close_rate_requested = limit trade.exit_reason = exit_reason @@ -1712,7 +1714,7 @@ class FreqtradeBot(LoggingMixin): self._notify_exit(trade, order_type, sub_trade=bool(sub_trade_amt), order=order_obj) # In case of market sell orders the order can be closed immediately if order.get('status', 'unknown') in ('closed', 'expired'): - self.update_trade_state(trade, trade.open_order_id, order) + self.update_trade_state(trade, order_obj.order_id, order) Trade.commit() return True @@ -1731,14 +1733,12 @@ class FreqtradeBot(LoggingMixin): amount = order.safe_filled if fill else order.safe_amount order_rate: float = order.safe_price - profit = trade.calc_profit(rate=order_rate, amount=amount, open_rate=trade.open_rate) - profit_ratio = trade.calc_profit_ratio(order_rate, amount, trade.open_rate) + profit = trade.calculate_profit(order_rate, amount, trade.open_rate) else: order_rate = trade.safe_close_rate - profit = trade.calc_profit(rate=order_rate) + (0.0 if fill else trade.realized_profit) - profit_ratio = trade.calc_profit_ratio(order_rate) + profit = trade.calculate_profit(rate=order_rate) amount = trade.amount - gain = "profit" if profit_ratio > 0 else "loss" + gain = "profit" if profit.profit_ratio > 0 else "loss" msg: RPCSellMsg = { 'type': (RPCMessageType.EXIT_FILL if fill @@ -1756,8 +1756,8 @@ class FreqtradeBot(LoggingMixin): 'open_rate': trade.open_rate, 'close_rate': order_rate, 'current_rate': current_rate, - 'profit_amount': profit, - 'profit_ratio': profit_ratio, + 'profit_amount': profit.profit_abs if fill else profit.total_profit, + 'profit_ratio': profit.profit_ratio, 'buy_tag': trade.enter_tag, 'enter_tag': trade.enter_tag, 'sell_reason': trade.exit_reason, # Deprecated @@ -1789,11 +1789,10 @@ class FreqtradeBot(LoggingMixin): order = self.order_obj_or_raise(order_id, order_or_none) profit_rate: float = trade.safe_close_rate - profit_trade = trade.calc_profit(rate=profit_rate) + profit = trade.calculate_profit(rate=profit_rate) current_rate = self.exchange.get_rate( trade.pair, side='exit', is_short=trade.is_short, refresh=False) - profit_ratio = trade.calc_profit_ratio(profit_rate) - gain = "profit" if profit_ratio > 0 else "loss" + gain = "profit" if profit.profit_ratio > 0 else "loss" msg: RPCSellCancelMsg = { 'type': RPCMessageType.EXIT_CANCEL, @@ -1808,8 +1807,8 @@ class FreqtradeBot(LoggingMixin): 'amount': order.safe_amount_after_fee, 'open_rate': trade.open_rate, 'current_rate': current_rate, - 'profit_amount': profit_trade, - 'profit_ratio': profit_ratio, + 'profit_amount': profit.profit_abs, + 'profit_ratio': profit.profit_ratio, 'buy_tag': trade.enter_tag, 'enter_tag': trade.enter_tag, 'sell_reason': trade.exit_reason, # Deprecated @@ -1929,11 +1928,11 @@ class FreqtradeBot(LoggingMixin): trade.amount, abs_tol=constants.MATH_CLOSE_PREC) if order.ft_order_side == trade.exit_side: # Exit notification - if send_msg and not stoploss_order and not trade.open_order_id: + if send_msg and not stoploss_order and order.order_id not in trade.open_orders_ids: self._notify_exit(trade, '', fill=True, sub_trade=sub_trade, order=order) if not trade.is_open: self.handle_protections(trade.pair, trade.trade_direction) - elif send_msg and not trade.open_order_id and not stoploss_order: + elif send_msg and order.order_id not in trade.open_orders_ids and not stoploss_order: # Enter fill self._notify_enter(trade, order, order.order_type, fill=True, sub_trade=sub_trade) diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index 5be3b4531..4b267b315 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -593,7 +593,6 @@ class Backtesting: """ if order and self._get_order_filled(order.ft_price, row): order.close_bt_order(current_date, trade) - trade.open_order_id = None if not (order.ft_order_side == trade.exit_side and order.safe_amount == trade.amount): self._call_adjust_stop(current_date, trade, order.ft_price) # pass @@ -862,7 +861,6 @@ class Backtesting: self.trade_id_counter += 1 trade = LocalTrade( id=self.trade_id_counter, - open_order_id=self.order_id_counter, pair=pair, base_currency=base_currency, stake_currency=self.config['stake_currency'], @@ -924,8 +922,7 @@ class Backtesting: ) order._trade_bt = trade trade.orders.append(order) - if not self._try_close_open_order(order, trade, current_time, row): - trade.open_order_id = str(self.order_id_counter) + self._try_close_open_order(order, trade, current_time, row) trade.recalc_trade_from_orders() return trade @@ -937,7 +934,7 @@ class Backtesting: """ for pair in open_trades.keys(): for trade in list(open_trades[pair]): - if trade.open_order_id and trade.nr_of_successful_entries == 0: + if trade.has_open_orders and trade.nr_of_successful_entries == 0: # Ignore trade if entry-order did not fill yet continue exit_row = data[pair][-1] @@ -1014,13 +1011,11 @@ class Backtesting: else: # Close additional entry order del trade.orders[trade.orders.index(order)] - trade.open_order_id = None return False if order.side == trade.exit_side: self.timedout_exit_orders += 1 # Close exit order and retry exiting on next signal. del trade.orders[trade.orders.index(order)] - trade.open_order_id = None return False return None @@ -1048,7 +1043,6 @@ class Backtesting: return False else: del trade.orders[trade.orders.index(order)] - trade.open_order_id = None self.canceled_entry_orders += 1 # place new order if result was not None @@ -1059,7 +1053,7 @@ class Backtesting: order.safe_remaining * order.ft_price / trade.leverage), direction='short' if trade.is_short else 'long') # Delete trade if no successful entries happened (if placing the new order failed) - if trade.open_order_id is None and trade.nr_of_successful_entries == 0: + if not trade.has_open_orders and trade.nr_of_successful_entries == 0: return True self.replaced_entry_orders += 1 else: @@ -1144,7 +1138,7 @@ class Backtesting: self.wallets.update() # 4. Create exit orders (if any) - if not trade.open_order_id: + if not trade.has_open_orders: self._check_trade_exit(trade, row) # Place exit order if necessary # 5. Process exit orders. diff --git a/freqtrade/optimize/hyperopt_loss/hyperopt_loss_sortino_daily.py b/freqtrade/optimize/hyperopt_loss/hyperopt_loss_sortino_daily.py index f5fe4590e..5beacc6fc 100644 --- a/freqtrade/optimize/hyperopt_loss/hyperopt_loss_sortino_daily.py +++ b/freqtrade/optimize/hyperopt_loss/hyperopt_loss_sortino_daily.py @@ -52,7 +52,7 @@ class SortinoHyperOptLossDaily(IHyperOptLoss): total_profit = sum_daily["profit_ratio_after_slippage"] - minimum_acceptable_return expected_returns_mean = total_profit.mean() - sum_daily['downside_returns'] = 0 + sum_daily['downside_returns'] = 0.0 sum_daily.loc[total_profit < 0, 'downside_returns'] = total_profit total_downside = sum_daily['downside_returns'] # Here total_downside contains min(0, P - MAR) values, diff --git a/freqtrade/persistence/migrations.py b/freqtrade/persistence/migrations.py index 1270b85ac..69d37530f 100644 --- a/freqtrade/persistence/migrations.py +++ b/freqtrade/persistence/migrations.py @@ -157,7 +157,7 @@ def migrate_trades_and_orders_table( fee_open, fee_open_cost, fee_open_currency, fee_close, fee_close_cost, fee_close_currency, open_rate, open_rate_requested, close_rate, close_rate_requested, close_profit, - stake_amount, amount, amount_requested, open_date, close_date, open_order_id, + stake_amount, amount, amount_requested, open_date, close_date, stop_loss, stop_loss_pct, initial_stop_loss, initial_stop_loss_pct, is_stop_loss_trailing, stoploss_order_id, stoploss_last_update, max_rate, min_rate, exit_reason, exit_order_status, strategy, enter_tag, @@ -174,7 +174,7 @@ def migrate_trades_and_orders_table( {fee_close_cost} fee_close_cost, {fee_close_currency} fee_close_currency, open_rate, {open_rate_requested} open_rate_requested, close_rate, {close_rate_requested} close_rate_requested, close_profit, - stake_amount, amount, {amount_requested}, open_date, close_date, open_order_id, + stake_amount, amount, {amount_requested}, open_date, close_date, {stop_loss} stop_loss, {stop_loss_pct} stop_loss_pct, {initial_stop_loss} initial_stop_loss, {initial_stop_loss_pct} initial_stop_loss_pct, @@ -272,6 +272,13 @@ def set_sqlite_to_wal(engine): def fix_old_dry_orders(engine): with engine.begin() as connection: + + # Update current dry-run Orders where + # - current Order is open + # - current Trade is closed + # - current Order trade_id not equal to current Trade.id + # - current Order not stoploss + stmt = update(Order).where( Order.ft_is_open.is_(True), tuple_(Order.ft_trade_id, Order.order_id).not_in( @@ -285,12 +292,13 @@ def fix_old_dry_orders(engine): ).values(ft_is_open=False) connection.execute(stmt) + # Close dry-run orders for closed trades. stmt = update(Order).where( Order.ft_is_open.is_(True), - tuple_(Order.ft_trade_id, Order.order_id).not_in( + Order.ft_trade_id.not_in( select( - Trade.id, Trade.open_order_id - ).where(Trade.open_order_id.is_not(None)) + Trade.id + ).where(Trade.is_open.is_(True)) ), Order.ft_order_side != 'stoploss', Order.order_id.like('dry%') diff --git a/freqtrade/persistence/trade_model.py b/freqtrade/persistence/trade_model.py index 6f31444d4..22444bb49 100644 --- a/freqtrade/persistence/trade_model.py +++ b/freqtrade/persistence/trade_model.py @@ -3,6 +3,7 @@ This module contains the class to persist trades into SQLite """ import logging from collections import defaultdict +from dataclasses import dataclass from datetime import datetime, timedelta, timezone from math import isclose from typing import Any, ClassVar, Dict, List, Optional, Sequence, cast @@ -26,6 +27,14 @@ from freqtrade.util import FtPrecise, dt_now logger = logging.getLogger(__name__) +@dataclass +class ProfitStruct: + profit_abs: float + profit_ratio: float + total_profit: float + total_profit_ratio: float + + class Order(ModelBase): """ Order database model @@ -343,7 +352,6 @@ class LocalTrade: amount_requested: Optional[float] = None open_date: datetime close_date: Optional[datetime] = None - open_order_id: Optional[str] = None # absolute value of the stop loss stop_loss: float = 0.0 # percentage value of the stop loss @@ -485,6 +493,32 @@ class LocalTrade: except IndexError: return '' + @property + def open_orders(self) -> List[Order]: + """ + All open orders for this trade excluding stoploss orders + """ + return [o for o in self.orders if o.ft_is_open and o.ft_order_side != 'stoploss'] + + @property + def has_open_orders(self) -> int: + """ + True if there are open orders for this trade excluding stoploss orders + """ + open_orders_wo_sl = [ + o for o in self.orders + if o.ft_order_side not in ['stoploss'] and o.ft_is_open + ] + return len(open_orders_wo_sl) > 0 + + @property + def open_orders_ids(self) -> List[str]: + open_orders_ids_wo_sl = [ + oo.order_id for oo in self.open_orders + if oo.ft_order_side not in ['stoploss'] + ] + return open_orders_ids_wo_sl + def __init__(self, **kwargs): for key in kwargs: setattr(self, key, kwargs[key]) @@ -503,8 +537,8 @@ class LocalTrade: ) def to_json(self, minified: bool = False) -> Dict[str, Any]: - filled_orders = self.select_filled_or_open_orders() - orders = [order.to_json(self.entry_side, minified) for order in filled_orders] + filled_or_open_orders = self.select_filled_or_open_orders() + orders_json = [order.to_json(self.entry_side, minified) for order in filled_or_open_orders] return { 'trade_id': self.id, @@ -580,11 +614,11 @@ class LocalTrade: 'is_short': self.is_short, 'trading_mode': self.trading_mode, 'funding_fees': self.funding_fees, - 'open_order_id': self.open_order_id, 'amount_precision': self.amount_precision, 'price_precision': self.price_precision, 'precision_mode': self.precision_mode, - 'orders': orders, + 'orders': orders_json, + 'has_open_orders': self.has_open_orders, } @staticmethod @@ -702,24 +736,13 @@ class LocalTrade: if self.is_open: payment = "SELL" if self.is_short else "BUY" logger.info(f'{order.order_type.upper()}_{payment} has been fulfilled for {self}.') - # condition to avoid reset value when updating fees - if self.open_order_id == order.order_id: - self.open_order_id = None - else: - logger.warning( - f'Got different open_order_id {self.open_order_id} != {order.order_id}') + self.recalc_trade_from_orders() elif order.ft_order_side == self.exit_side: if self.is_open: payment = "BUY" if self.is_short else "SELL" # * On margin shorts, you buy a little bit more than the amount (amount + interest) logger.info(f'{order.order_type.upper()}_{payment} has been fulfilled for {self}.') - # condition to avoid reset value when updating fees - if self.open_order_id == order.order_id: - self.open_order_id = None - else: - logger.warning( - f'Got different open_order_id {self.open_order_id} != {order.order_id}') elif order.ft_order_side == 'stoploss' and order.status not in ('open', ): self.stoploss_order_id = None @@ -752,7 +775,6 @@ class LocalTrade: self.close_date = self.close_date or datetime.utcnow() self.is_open = False self.exit_order_status = 'closed' - self.open_order_id = None self.recalc_trade_from_orders(is_closing=True) if show_msg: logger.info(f"Marking {self} as closed as the trade is fulfilled " @@ -888,11 +910,26 @@ class LocalTrade: open_rate: Optional[float] = None) -> float: """ Calculate the absolute profit in stake currency between Close and Open trade + Deprecated - only available for backwards compatibility :param rate: close rate to compare with. :param amount: Amount to use for the calculation. Falls back to trade.amount if not set. :param open_rate: open_rate to use. Defaults to self.open_rate if not provided. :return: profit in stake currency as float """ + prof = self.calculate_profit(rate, amount, open_rate) + return prof.profit_abs + + def calculate_profit(self, rate: float, amount: Optional[float] = None, + open_rate: Optional[float] = None) -> ProfitStruct: + """ + Calculate profit metrics (absolute, ratio, total, total ratio). + All calculations include fees. + :param rate: close rate to compare with. + :param amount: Amount to use for the calculation. Falls back to trade.amount if not set. + :param open_rate: open_rate to use. Defaults to self.open_rate if not provided. + :return: Profit structure, containing absolute and relative profits. + """ + close_trade_value = self.calc_close_trade_value(rate, amount) if amount is None or open_rate is None: open_trade_value = self.open_trade_value @@ -900,10 +937,33 @@ class LocalTrade: open_trade_value = self._calc_open_trade_value(amount, open_rate) if self.is_short: - profit = open_trade_value - close_trade_value + profit_abs = open_trade_value - close_trade_value else: - profit = close_trade_value - open_trade_value - return float(f"{profit:.8f}") + profit_abs = close_trade_value - open_trade_value + + try: + if self.is_short: + profit_ratio = (1 - (close_trade_value / open_trade_value)) * self.leverage + else: + profit_ratio = ((close_trade_value / open_trade_value) - 1) * self.leverage + profit_ratio = float(f"{profit_ratio:.8f}") + except ZeroDivisionError: + profit_ratio = 0.0 + + total_profit_abs = profit_abs + self.realized_profit + total_profit_ratio = ( + (total_profit_abs / self.max_stake_amount) * self.leverage + if self.max_stake_amount else 0.0 + ) + total_profit_ratio = float(f"{total_profit_ratio:.8f}") + profit_abs = float(f"{profit_abs:.8f}") + + return ProfitStruct( + profit_abs=profit_abs, + profit_ratio=profit_ratio, + total_profit=profit_abs + self.realized_profit, + total_profit_ratio=total_profit_ratio, + ) def calc_profit_ratio( self, rate: float, amount: Optional[float] = None, @@ -944,7 +1004,6 @@ class LocalTrade: avg_price = FtPrecise(0.0) close_profit = 0.0 close_profit_abs = 0.0 - profit = None # Reset funding fees self.funding_fees = 0.0 funding_fees = 0.0 @@ -974,11 +1033,9 @@ class LocalTrade: exit_rate = o.safe_price exit_amount = o.safe_amount_after_fee - profit = self.calc_profit(rate=exit_rate, amount=exit_amount, - open_rate=float(avg_price)) - close_profit_abs += profit - close_profit = self.calc_profit_ratio( - exit_rate, amount=exit_amount, open_rate=avg_price) + prof = self.calculate_profit(exit_rate, exit_amount, float(avg_price)) + close_profit_abs += prof.profit_abs + close_profit = prof.profit_ratio else: total_stake = total_stake + self._calc_open_trade_value(tmp_amount, price) max_stake_amount += (tmp_amount * price) @@ -988,7 +1045,7 @@ class LocalTrade: if close_profit: self.close_profit = close_profit self.realized_profit = close_profit_abs - self.close_profit_abs = profit + self.close_profit_abs = prof.profit_abs current_amount_tr = amount_to_contract_precision( float(current_amount), self.amount_precision, self.precision_mode, self.contract_size) @@ -1265,7 +1322,6 @@ class Trade(ModelBase, LocalTrade): open_date: Mapped[datetime] = mapped_column( nullable=False, default=datetime.utcnow) # type: ignore close_date: Mapped[Optional[datetime]] = mapped_column() # type: ignore - open_order_id: Mapped[Optional[str]] = mapped_column(String(255), nullable=True) # type: ignore # absolute value of the stop loss stop_loss: Mapped[float] = mapped_column(Float(), nullable=True, default=0.0) # type: ignore # percentage value of the stop loss @@ -1422,14 +1478,6 @@ class Trade(ModelBase, LocalTrade): # raise an exception. return Trade.session.scalars(query) - @staticmethod - def get_open_order_trades() -> List['Trade']: - """ - Returns all open trades - NOTE: Not supported in Backtesting. - """ - return cast(List[Trade], Trade.get_trades(Trade.open_order_id.isnot(None)).all()) - @staticmethod def get_open_trades_without_assigned_fees(): """ @@ -1728,7 +1776,6 @@ class Trade(ModelBase, LocalTrade): is_short=data["is_short"], trading_mode=data["trading_mode"], funding_fees=data["funding_fees"], - open_order_id=data["open_order_id"], ) for order in data["orders"]: diff --git a/freqtrade/plugins/pairlist/VolumePairList.py b/freqtrade/plugins/pairlist/VolumePairList.py index 9e4a4fca9..c2f102777 100644 --- a/freqtrade/plugins/pairlist/VolumePairList.py +++ b/freqtrade/plugins/pairlist/VolumePairList.py @@ -260,6 +260,7 @@ class VolumePairList(IPairList): quoteVolume = (pair_candles['quoteVolume'] .rolling(self._lookback_period) .sum() + .fillna(0) .iloc[-1]) # replace quoteVolume with range quoteVolume sum calculated above diff --git a/freqtrade/rpc/api_server/api_schemas.py b/freqtrade/rpc/api_server/api_schemas.py index 422e5ac3a..97f6251bc 100644 --- a/freqtrade/rpc/api_server/api_schemas.py +++ b/freqtrade/rpc/api_server/api_schemas.py @@ -141,6 +141,10 @@ class Profit(BaseModel): expectancy_ratio: float max_drawdown: float max_drawdown_abs: float + max_drawdown_start: str + max_drawdown_start_timestamp: int + max_drawdown_end: str + max_drawdown_end_timestamp: int trading_volume: Optional[float] = None bot_start_timestamp: int bot_start_date: str @@ -304,7 +308,7 @@ class TradeSchema(BaseModel): min_rate: Optional[float] = None max_rate: Optional[float] = None - open_order_id: Optional[str] = None + has_open_orders: bool orders: List[OrderSchema] leverage: Optional[float] = None @@ -329,8 +333,6 @@ class OpenTradeSchema(TradeSchema): total_profit_fiat: Optional[float] = None total_profit_ratio: Optional[float] = None - open_order: Optional[str] = None - class TradeResponse(BaseModel): trades: List[TradeSchema] diff --git a/freqtrade/rpc/api_server/ws_schemas.py b/freqtrade/rpc/api_server/ws_schemas.py index 34eaf0245..970ea8cf8 100644 --- a/freqtrade/rpc/api_server/ws_schemas.py +++ b/freqtrade/rpc/api_server/ws_schemas.py @@ -5,7 +5,7 @@ from pandas import DataFrame from pydantic import BaseModel, ConfigDict from freqtrade.constants import PairWithTimeframe -from freqtrade.enums.rpcmessagetype import RPCMessageType, RPCRequestType +from freqtrade.enums import RPCMessageType, RPCRequestType class BaseArbitraryModel(BaseModel): diff --git a/freqtrade/rpc/rpc.py b/freqtrade/rpc/rpc.py index 497bc1c82..0abac3975 100644 --- a/freqtrade/rpc/rpc.py +++ b/freqtrade/rpc/rpc.py @@ -16,7 +16,7 @@ from sqlalchemy import func, select from freqtrade import __version__ from freqtrade.configuration.timerange import TimeRange -from freqtrade.constants import CANCEL_REASON, DATETIME_PRINT_FORMAT, Config +from freqtrade.constants import CANCEL_REASON, Config from freqtrade.data.history import load_data from freqtrade.data.metrics import calculate_expectancy, calculate_max_drawdown from freqtrade.enums import (CandleType, ExitCheckTuple, ExitType, MarketDirection, SignalDirection, @@ -26,12 +26,12 @@ from freqtrade.exchange import timeframe_to_minutes, timeframe_to_msecs from freqtrade.exchange.types import Tickers from freqtrade.loggers import bufferHandler from freqtrade.misc import decimals_per_coin -from freqtrade.persistence import KeyStoreKeys, KeyValueStore, Order, PairLocks, Trade +from freqtrade.persistence import KeyStoreKeys, KeyValueStore, PairLocks, Trade from freqtrade.persistence.models import PairLock from freqtrade.plugins.pairlist.pairlist_helpers import expand_pairlist from freqtrade.rpc.fiat_convert import CryptoToFiatConverter from freqtrade.rpc.rpc_types import RPCSendMsg -from freqtrade.util import dt_humanize, dt_now, shorten_date +from freqtrade.util import dt_humanize, dt_now, dt_ts_def, format_date, shorten_date from freqtrade.wallets import PositionWallet, Wallet @@ -171,11 +171,20 @@ class RPC: else: results = [] for trade in trades: - order: Optional[Order] = None current_profit_fiat: Optional[float] = None total_profit_fiat: Optional[float] = None - if trade.open_order_id: - order = trade.select_order_by_order_id(trade.open_order_id) + + # prepare open orders details + oo_details: Optional[str] = "" + oo_details_lst = [ + f'({oo.order_type} {oo.side} rem={oo.safe_remaining:.8f})' + for oo in trade.open_orders + if oo.ft_order_side not in ['stoploss'] + ] + oo_details = ', '.join(oo_details_lst) + + total_profit_abs = 0.0 + total_profit_ratio: Optional[float] = None # calculate profit and send message to user if trade.is_open: try: @@ -184,23 +193,22 @@ class RPC: except (ExchangeError, PricingError): current_rate = NAN if len(trade.select_filled_orders(trade.entry_side)) > 0: - current_profit = trade.calc_profit_ratio( - current_rate) if not isnan(current_rate) else NAN - current_profit_abs = trade.calc_profit( - current_rate) if not isnan(current_rate) else NAN + + current_profit = current_profit_abs = current_profit_fiat = NAN + if not isnan(current_rate): + prof = trade.calculate_profit(current_rate) + current_profit = prof.profit_ratio + current_profit_abs = prof.profit_abs + total_profit_abs = prof.total_profit + total_profit_ratio = prof.total_profit_ratio else: current_profit = current_profit_abs = current_profit_fiat = 0.0 + else: # Closed trade ... current_rate = trade.close_rate current_profit = trade.close_profit or 0.0 current_profit_abs = trade.close_profit_abs or 0.0 - total_profit_abs = trade.realized_profit + current_profit_abs - total_profit_ratio: Optional[float] = None - if trade.max_stake_amount: - total_profit_ratio = ( - (total_profit_abs / trade.max_stake_amount) * trade.leverage - ) # Calculate fiat profit if not isnan(current_profit_abs) and self._fiat_converter: @@ -216,8 +224,11 @@ class RPC: ) # Calculate guaranteed profit (in case of trailing stop) - stoploss_entry_dist = trade.calc_profit(trade.stop_loss) - stoploss_entry_dist_ratio = trade.calc_profit_ratio(trade.stop_loss) + stop_entry = trade.calculate_profit(trade.stop_loss) + + stoploss_entry_dist = stop_entry.profit_abs + stoploss_entry_dist_ratio = stop_entry.profit_ratio + # calculate distance to stoploss stoploss_current_dist = trade.stop_loss - current_rate stoploss_current_dist_ratio = stoploss_current_dist / current_rate @@ -230,7 +241,6 @@ class RPC: profit_pct=round(current_profit * 100, 2), profit_abs=current_profit_abs, profit_fiat=current_profit_fiat, - total_profit_abs=total_profit_abs, total_profit_fiat=total_profit_fiat, total_profit_ratio=total_profit_ratio, @@ -239,10 +249,7 @@ class RPC: stoploss_current_dist_pct=round(stoploss_current_dist_ratio * 100, 2), stoploss_entry_dist=stoploss_entry_dist, stoploss_entry_dist_ratio=round(stoploss_entry_dist_ratio, 8), - open_order=( - f'({order.order_type} {order.side} rem={order.safe_remaining:.8f})' if - order else None - ), + open_orders=oo_details )) results.append(trade_dict) return results @@ -267,8 +274,9 @@ class RPC: profit_str = f'{NAN:.2%}' else: if trade.nr_of_successful_entries > 0: - trade_profit = trade.calc_profit(current_rate) - profit_str = f'{trade.calc_profit_ratio(current_rate):.2%}' + profit = trade.calculate_profit(current_rate) + trade_profit = profit.profit_abs + profit_str = f'{profit.profit_ratio:.2%}' else: trade_profit = 0.0 profit_str = f'{0.0:.2f}' @@ -283,18 +291,22 @@ class RPC: profit_str += f" ({fiat_profit:.2f})" fiat_profit_sum = fiat_profit if isnan(fiat_profit_sum) \ else fiat_profit_sum + fiat_profit - open_order = (trade.select_order_by_order_id( - trade.open_order_id) if trade.open_order_id else None) + + active_attempt_side_symbols = [ + '*' if (oo and oo.ft_order_side == trade.entry_side) else '**' + for oo in trade.open_orders + ] + + # exemple: '*.**.**' trying to enter, exit and exit with 3 different orders + active_attempt_side_symbols_str = '.'.join(active_attempt_side_symbols) detail_trade = [ f'{trade.id} {direction_str}', - trade.pair + ('*' if (open_order - and open_order.ft_order_side == trade.entry_side) else '') - + ('**' if (open_order and - open_order.ft_order_side == trade.exit_side is not None) else ''), + trade.pair + active_attempt_side_symbols_str, shorten_date(dt_humanize(trade.open_date, only_distance=True)), profit_str ] + if self._config.get('position_adjustment_enable', False): max_entry_str = '' if self._config.get('max_entry_position_adjustment', -1) > 0: @@ -487,9 +499,10 @@ class RPC: profit_ratio = NAN profit_abs = NAN else: - profit_ratio = trade.calc_profit_ratio(rate=current_rate) - profit_abs = trade.calc_profit( - rate=trade.close_rate or current_rate) + trade.realized_profit + profit = trade.calculate_profit(trade.close_rate or current_rate) + + profit_ratio = profit.profit_ratio + profit_abs = profit.total_profit profit_all_coin.append(profit_abs) profit_all_ratio.append(profit_ratio) @@ -525,7 +538,8 @@ class RPC: winrate = (winning_trades / closed_trade_count) if closed_trade_count > 0 else 0 - trades_df = DataFrame([{'close_date': trade.close_date.strftime(DATETIME_PRINT_FORMAT), + trades_df = DataFrame([{'close_date': format_date(trade.close_date), + 'close_date_dt': trade.close_date, 'profit_abs': trade.close_profit_abs} for trade in trades if not trade.is_open and trade.close_date]) @@ -533,10 +547,15 @@ class RPC: max_drawdown_abs = 0.0 max_drawdown = 0.0 + drawdown_start: Optional[datetime] = None + drawdown_end: Optional[datetime] = None + dd_high_val = dd_low_val = 0.0 if len(trades_df) > 0: try: - (max_drawdown_abs, _, _, _, _, max_drawdown) = calculate_max_drawdown( - trades_df, value_col='profit_abs', starting_balance=starting_balance) + (max_drawdown_abs, drawdown_start, drawdown_end, dd_high_val, dd_low_val, + max_drawdown) = calculate_max_drawdown( + trades_df, value_col='profit_abs', date_col='close_date_dt', + starting_balance=starting_balance) except ValueError: # ValueError if no losing trade. pass @@ -570,12 +589,12 @@ class RPC: 'profit_all_fiat': profit_all_fiat, 'trade_count': len(trades), 'closed_trade_count': closed_trade_count, - 'first_trade_date': first_date.strftime(DATETIME_PRINT_FORMAT) if first_date else '', + 'first_trade_date': format_date(first_date), 'first_trade_humanized': dt_humanize(first_date) if first_date else '', - 'first_trade_timestamp': int(first_date.timestamp() * 1000) if first_date else 0, - 'latest_trade_date': last_date.strftime(DATETIME_PRINT_FORMAT) if last_date else '', + 'first_trade_timestamp': dt_ts_def(first_date, 0), + 'latest_trade_date': format_date(last_date), 'latest_trade_humanized': dt_humanize(last_date) if last_date else '', - 'latest_trade_timestamp': int(last_date.timestamp() * 1000) if last_date else 0, + 'latest_trade_timestamp': dt_ts_def(last_date, 0), 'avg_duration': str(timedelta(seconds=sum(durations) / num)).split('.')[0], 'best_pair': best_pair[0] if best_pair else '', 'best_rate': round(best_pair[1] * 100, 2) if best_pair else 0, # Deprecated @@ -588,9 +607,15 @@ class RPC: 'expectancy_ratio': expectancy_ratio, 'max_drawdown': max_drawdown, 'max_drawdown_abs': max_drawdown_abs, + 'max_drawdown_start': format_date(drawdown_start), + 'max_drawdown_start_timestamp': dt_ts_def(drawdown_start), + 'max_drawdown_end': format_date(drawdown_end), + 'max_drawdown_end_timestamp': dt_ts_def(drawdown_end), + 'drawdown_high': dd_high_val, + 'drawdown_low': dd_low_val, 'trading_volume': trading_volume, - 'bot_start_timestamp': int(bot_start.timestamp() * 1000) if bot_start else 0, - 'bot_start_date': bot_start.strftime(DATETIME_PRINT_FORMAT) if bot_start else '', + 'bot_start_timestamp': dt_ts_def(bot_start, 0), + 'bot_start_date': format_date(bot_start), } def __balance_get_est_stake( @@ -762,21 +787,25 @@ class RPC: def __exec_force_exit(self, trade: Trade, ordertype: Optional[str], amount: Optional[float] = None) -> bool: - # Check if there is there is an open order - fully_canceled = False - if trade.open_order_id: - order = self._freqtrade.exchange.fetch_order(trade.open_order_id, trade.pair) + # Check if there is there are open orders + trade_entry_cancelation_registry = [] + for oo in trade.open_orders: + trade_entry_cancelation_res = {'order_id': oo.order_id, 'cancel_state': False} + order = self._freqtrade.exchange.fetch_order(oo.order_id, trade.pair) if order['side'] == trade.entry_side: fully_canceled = self._freqtrade.handle_cancel_enter( - trade, order, CANCEL_REASON['FORCE_EXIT']) + trade, order, oo.order_id, CANCEL_REASON['FORCE_EXIT']) + trade_entry_cancelation_res['cancel_state'] = fully_canceled + trade_entry_cancelation_registry.append(trade_entry_cancelation_res) if order['side'] == trade.exit_side: # Cancel order - so it is placed anew with a fresh price. - self._freqtrade.handle_cancel_exit(trade, order, CANCEL_REASON['FORCE_EXIT']) + self._freqtrade.handle_cancel_exit( + trade, order, oo.order_id, CANCEL_REASON['FORCE_EXIT']) - if not fully_canceled: - if trade.open_order_id is not None: + if all(tocr['cancel_state'] is False for tocr in trade_entry_cancelation_registry): + if trade.has_open_orders: # Order cancellation failed, so we can't exit. return False # Get current rate and execute sell @@ -875,10 +904,10 @@ class RPC: if trade: is_short = trade.is_short if not self._freqtrade.strategy.position_adjustment_enable: - raise RPCException(f'position for {pair} already open - id: {trade.id}') - if trade.open_order_id is not None: - raise RPCException(f'position for {pair} already open - id: {trade.id} ' - f'and has open order {trade.open_order_id}') + raise RPCException(f"position for {pair} already open - id: {trade.id}") + if trade.has_open_orders: + raise RPCException(f"position for {pair} already open - id: {trade.id} " + f"and has open order {','.join(trade.open_orders_ids)}") else: if Trade.get_open_trade_count() >= self._config['max_open_trades']: raise RPCException("Maximum number of trades is reached.") @@ -915,16 +944,18 @@ class RPC: if not trade: logger.warning('cancel_open_order: Invalid trade_id received.') raise RPCException('Invalid trade_id.') - if not trade.open_order_id: + if not trade.has_open_orders: logger.warning('cancel_open_order: No open order for trade_id.') raise RPCException('No open order for trade_id.') - try: - order = self._freqtrade.exchange.fetch_order(trade.open_order_id, trade.pair) - except ExchangeError as e: - logger.info(f"Cannot query order for {trade} due to {e}.", exc_info=True) - raise RPCException("Order not found.") - self._freqtrade.handle_cancel_order(order, trade, CANCEL_REASON['USER_CANCEL']) + for open_order in trade.open_orders: + try: + order = self._freqtrade.exchange.fetch_order(open_order.order_id, trade.pair) + except ExchangeError as e: + logger.info(f"Cannot query order for {trade} due to {e}.", exc_info=True) + raise RPCException("Order not found.") + self._freqtrade.handle_cancel_order( + order, open_order.order_id, trade, CANCEL_REASON['USER_CANCEL']) Trade.commit() def _rpc_delete(self, trade_id: int) -> Dict[str, Union[str, int]]: @@ -940,9 +971,9 @@ class RPC: raise RPCException('invalid argument') # Try cancelling regular order if that exists - if trade.open_order_id: + for open_order in trade.open_orders: try: - self._freqtrade.exchange.cancel_order(trade.open_order_id, trade.pair) + self._freqtrade.exchange.cancel_order(open_order.order_id, trade.pair) c_count += 1 except (ExchangeError): pass @@ -1092,7 +1123,7 @@ class RPC: buffer = bufferHandler.buffer[-limit:] else: buffer = bufferHandler.buffer - records = [[datetime.fromtimestamp(r.created).strftime(DATETIME_PRINT_FORMAT), + records = [[format_date(datetime.fromtimestamp(r.created)), r.created * 1000, r.name, r.levelname, r.message + ('\n' + r.exc_text if r.exc_text else '')] for r in buffer] @@ -1309,7 +1340,7 @@ class RPC: return { "last_process": str(last_p), - "last_process_loc": last_p.astimezone(tzlocal()).strftime(DATETIME_PRINT_FORMAT), + "last_process_loc": format_date(last_p.astimezone(tzlocal())), "last_process_ts": int(last_p.timestamp()), } diff --git a/freqtrade/rpc/telegram.py b/freqtrade/rpc/telegram.py index 8972f4a16..be27c38f4 100644 --- a/freqtrade/rpc/telegram.py +++ b/freqtrade/rpc/telegram.py @@ -532,40 +532,24 @@ class Telegram(RPCHandler): cur_entry_amount = order["filled"] or order["amount"] cur_entry_average = order["safe_price"] lines.append(" ") + lines.append(f"*{wording} #{order_nr}:*") if order_nr == 1: - lines.append(f"*{wording} #{order_nr}:*") lines.append( f"*Amount:* {cur_entry_amount:.8g} " f"({round_coin_value(order['cost'], quote_currency)})" ) lines.append(f"*Average Price:* {cur_entry_average:.8g}") else: - sum_stake = 0 - sum_amount = 0 - for y in range(order_nr): - loc_order = filled_orders[y] - if loc_order['is_open'] is True: - # Skip open orders (e.g. stop orders) - continue - amount = loc_order["filled"] or loc_order["amount"] - sum_stake += amount * loc_order["safe_price"] - sum_amount += amount - prev_avg_price = sum_stake / sum_amount # TODO: This calculation ignores fees. price_to_1st_entry = ((cur_entry_average - first_avg) / first_avg) - minus_on_entry = 0 - if prev_avg_price: - minus_on_entry = (cur_entry_average - prev_avg_price) / prev_avg_price - - lines.append(f"*{wording} #{order_nr}:* at {minus_on_entry:.2%} avg Profit") if is_open: lines.append("({})".format(dt_humanize(order["order_filled_date"], granularity=["day", "hour", "minute"]))) lines.append(f"*Amount:* {cur_entry_amount:.8g} " f"({round_coin_value(order['cost'], quote_currency)})") lines.append(f"*Average {wording} Price:* {cur_entry_average:.8g} " - f"({price_to_1st_entry:.2%} from 1st entry Rate)") - lines.append(f"*Order filled:* {order['order_filled_date']}") + f"({price_to_1st_entry:.2%} from 1st entry rate)") + lines.append(f"*Order Filled:* {order['order_filled_date']}") lines_detail.append("\n".join(lines)) @@ -663,10 +647,10 @@ class Telegram(RPCHandler): ("`({stop_loss_ratio:.2%})`" if r['stop_loss_ratio'] else "")) lines.append("*Stoploss distance:* `{stoploss_current_dist:.8g}` " "`({stoploss_current_dist_ratio:.2%})`") - if r['open_order']: + if r.get('open_orders'): lines.append( - "*Open Order:* `{open_order}`" - + "- `{exit_order_status}`" if r['exit_order_status'] else "") + "*Open Order:* `{open_orders}`" + + ("- `{exit_order_status}`" if r['exit_order_status'] else "")) lines_detail = self._prepare_order_details( r['orders'], r['quote_currency'], r['is_open']) @@ -889,7 +873,11 @@ class Telegram(RPCHandler): f"*Trading volume:* `{round_coin_value(stats['trading_volume'], stake_cur)}`\n" f"*Profit factor:* `{stats['profit_factor']:.2f}`\n" f"*Max Drawdown:* `{stats['max_drawdown']:.2%} " - f"({round_coin_value(stats['max_drawdown_abs'], stake_cur)})`" + f"({round_coin_value(stats['max_drawdown_abs'], stake_cur)})`\n" + f" from `{stats['max_drawdown_start']} " + f"({round_coin_value(stats['drawdown_high'], stake_cur)})`\n" + f" to `{stats['max_drawdown_end']} " + f"({round_coin_value(stats['drawdown_low'], stake_cur)})`\n" ) await self._send_msg(markdown_msg, reload_able=True, callback_path="update_profit", query=update.callback_query) diff --git a/freqtrade/strategy/strategy_helper.py b/freqtrade/strategy/strategy_helper.py index f47e1db45..7654a383f 100644 --- a/freqtrade/strategy/strategy_helper.py +++ b/freqtrade/strategy/strategy_helper.py @@ -45,10 +45,13 @@ def merge_informative_pair(dataframe: pd.DataFrame, informative: pd.DataFrame, elif minutes < minutes_inf: # Subtract "small" timeframe so merging is not delayed by 1 small candle # Detailed explanation in https://github.com/freqtrade/freqtrade/issues/4073 - informative['date_merge'] = ( - informative[date_column] + pd.to_timedelta(minutes_inf, 'm') - - pd.to_timedelta(minutes, 'm') - ) + if not informative.empty: + informative['date_merge'] = ( + informative[date_column] + pd.to_timedelta(minutes_inf, 'm') - + pd.to_timedelta(minutes, 'm') + ) + else: + informative['date_merge'] = informative[date_column] else: raise ValueError("Tried to merge a faster timeframe to a slower timeframe." "This would create new rows, and can throw off your regular indicators.") diff --git a/freqtrade/templates/strategy_analysis_example.ipynb b/freqtrade/templates/strategy_analysis_example.ipynb index 2bfa4155d..0b30dbd54 100644 --- a/freqtrade/templates/strategy_analysis_example.ipynb +++ b/freqtrade/templates/strategy_analysis_example.ipynb @@ -243,7 +243,7 @@ "# Plotting equity line (starting with 0 on day 1 and adding daily profit for each backtested day)\n", "\n", "from freqtrade.configuration import Configuration\n", - "from freqtrade.data.btanalysis import load_backtest_data, load_backtest_stats\n", + "from freqtrade.data.btanalysis import load_backtest_stats\n", "import plotly.express as px\n", "import pandas as pd\n", "\n", @@ -254,20 +254,8 @@ "stats = load_backtest_stats(backtest_dir)\n", "strategy_stats = stats['strategy'][strategy]\n", "\n", - "dates = []\n", - "profits = []\n", - "for date_profit in strategy_stats['daily_profit']:\n", - " dates.append(date_profit[0])\n", - " profits.append(date_profit[1])\n", - "\n", - "equity = 0\n", - "equity_daily = []\n", - "for daily_profit in profits:\n", - " equity_daily.append(equity)\n", - " equity += float(daily_profit)\n", - "\n", - "\n", - "df = pd.DataFrame({'dates': dates,'equity_daily': equity_daily})\n", + "df = pd.DataFrame(columns=['dates','equity'], data=strategy_stats['daily_profit'])\n", + "df['equity_daily'] = df['equity'].cumsum()\n", "\n", "fig = px.line(df, x=\"dates\", y=\"equity_daily\")\n", "fig.show()\n" @@ -414,7 +402,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.9.7" + "version": "3.11.4" }, "mimetype": "text/x-python", "name": "python", diff --git a/freqtrade/util/__init__.py b/freqtrade/util/__init__.py index af09624ac..fc7cf5f6a 100644 --- a/freqtrade/util/__init__.py +++ b/freqtrade/util/__init__.py @@ -1,5 +1,6 @@ from freqtrade.util.datetime_helpers import (dt_floor_day, dt_from_ts, dt_humanize, dt_now, dt_ts, - dt_utc, format_ms_time, shorten_date) + dt_ts_def, dt_utc, format_date, format_ms_time, + shorten_date) from freqtrade.util.ft_precise import FtPrecise from freqtrade.util.periodic_cache import PeriodicCache from freqtrade.util.template_renderer import render_template, render_template_with_fallback # noqa @@ -11,7 +12,9 @@ __all__ = [ 'dt_humanize', 'dt_now', 'dt_ts', + 'dt_ts_def', 'dt_utc', + 'format_date', 'format_ms_time', 'FtPrecise', 'PeriodicCache', diff --git a/freqtrade/util/binance_mig.py b/freqtrade/util/binance_mig.py index 7043459f2..cc9c451b2 100644 --- a/freqtrade/util/binance_mig.py +++ b/freqtrade/util/binance_mig.py @@ -4,7 +4,7 @@ from packaging import version from sqlalchemy import select from freqtrade.constants import DOCS_LINK, Config -from freqtrade.enums.tradingmode import TradingMode +from freqtrade.enums import TradingMode from freqtrade.exceptions import OperationalException from freqtrade.persistence.pairlock import PairLock from freqtrade.persistence.trade_model import Trade diff --git a/freqtrade/util/datetime_helpers.py b/freqtrade/util/datetime_helpers.py index 7f44cbdb0..102c83143 100644 --- a/freqtrade/util/datetime_helpers.py +++ b/freqtrade/util/datetime_helpers.py @@ -4,6 +4,8 @@ from typing import Optional import arrow +from freqtrade.constants import DATETIME_PRINT_FORMAT + def dt_now() -> datetime: """Return the current datetime in UTC.""" @@ -26,6 +28,16 @@ def dt_ts(dt: Optional[datetime] = None) -> int: return int(dt_now().timestamp() * 1000) +def dt_ts_def(dt: Optional[datetime], default: int = 0) -> int: + """ + Return dt in ms as a timestamp in UTC. + If dt is None, return the current datetime in UTC. + """ + if dt: + return int(dt.timestamp() * 1000) + return default + + def dt_floor_day(dt: datetime) -> datetime: """Return the floor of the day for the given datetime.""" return dt.replace(hour=0, minute=0, second=0, microsecond=0) @@ -63,6 +75,17 @@ def dt_humanize(dt: datetime, **kwargs) -> str: return arrow.get(dt).humanize(**kwargs) +def format_date(date: Optional[datetime]) -> str: + """ + Return a formatted date string. + Returns an empty string if date is None. + :param date: datetime to format + """ + if date: + return date.strftime(DATETIME_PRINT_FORMAT) + return '' + + def format_ms_time(date: int) -> str: """ convert MS date to readable format. diff --git a/requirements-dev.txt b/requirements-dev.txt index 4ee9d6912..e779429a0 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -10,7 +10,7 @@ coveralls==3.3.1 ruff==0.0.287 mypy==1.5.1 pre-commit==3.4.0 -pytest==7.4.0 +pytest==7.4.2 pytest-asyncio==0.21.1 pytest-cov==4.1.0 pytest-mock==3.11.1 diff --git a/requirements-freqai.txt b/requirements-freqai.txt index d8421e968..8f690f957 100644 --- a/requirements-freqai.txt +++ b/requirements-freqai.txt @@ -5,7 +5,7 @@ # Required for freqai scikit-learn==1.1.3 joblib==1.3.2 -catboost==1.2; 'arm' not in platform_machine +catboost==1.2.1; 'arm' not in platform_machine lightgbm==4.0.0 xgboost==1.7.6 tensorboard==2.14.0 diff --git a/requirements-hyperopt.txt b/requirements-hyperopt.txt index e5a0b21cc..4cc7cc4c7 100644 --- a/requirements-hyperopt.txt +++ b/requirements-hyperopt.txt @@ -2,8 +2,7 @@ -r requirements.txt # Required for hyperopt -scipy==1.11.2; python_version >= '3.9' -scipy==1.10.1; python_version < '3.9' +scipy==1.11.2 scikit-learn==1.1.3 scikit-optimize==0.9.0 filelock==3.12.3 diff --git a/requirements.txt b/requirements.txt index e709d562f..f91a8d2b5 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,9 +1,8 @@ -numpy==1.25.2; python_version > '3.8' -numpy==1.24.3; python_version <= '3.8' +numpy==1.25.2 pandas==2.0.3 pandas-ta==0.3.14b -ccxt==4.0.81 +ccxt==4.0.88 cryptography==41.0.3; platform_machine != 'armv7l' cryptography==40.0.1; platform_machine == 'armv7l' aiohttp==3.8.5 @@ -33,7 +32,7 @@ py_find_1st==1.1.5 # Load ticker files 30% faster python-rapidjson==1.10 # Properly format api responses -orjson==3.9.5 +orjson==3.9.7 # Notify systemd sdnotify==0.3.2 @@ -49,7 +48,7 @@ psutil==5.9.5 # Support for colorized terminal output colorama==0.4.6 # Building config files interactively -questionary==2.0.0 +questionary==2.0.1 prompt-toolkit==3.0.36 # Extensions to datetime library python-dateutil==2.8.2 diff --git a/setup.cfg b/setup.cfg index 3b06eaa7d..d4d70bc34 100644 --- a/setup.cfg +++ b/setup.cfg @@ -14,7 +14,6 @@ classifiers = Environment :: Console Intended Audience :: Science/Research License :: OSI Approved :: GNU General Public License v3 (GPLv3) - Programming Language :: Python :: 3.8 Programming Language :: Python :: 3.9 Programming Language :: Python :: 3.10 Programming Language :: Python :: 3.11 @@ -33,7 +32,7 @@ tests_require = pytest-mock packages = find: -python_requires = >=3.8 +python_requires = >=3.9 [options.entry_points] console_scripts = diff --git a/setup.sh b/setup.sh index 8ea5faab5..6bf85edab 100755 --- a/setup.sh +++ b/setup.sh @@ -25,7 +25,7 @@ function check_installed_python() { exit 2 fi - for v in 11 10 9 8 + for v in 11 10 9 do PYTHON="python3.${v}" which $PYTHON @@ -36,7 +36,7 @@ function check_installed_python() { fi done - echo "No usable python found. Please make sure to have python3.8 or newer installed." + echo "No usable python found. Please make sure to have python3.9 or newer installed." exit 1 } @@ -277,7 +277,7 @@ function install() { install_redhat else echo "This script does not support your OS." - echo "If you have Python version 3.8 - 3.11, pip, virtualenv, ta-lib you can continue." + echo "If you have Python version 3.9 - 3.11, pip, virtualenv, ta-lib you can continue." echo "Wait 10 seconds to continue the next install steps or use ctrl+c to interrupt this shell." sleep 10 fi @@ -304,7 +304,7 @@ function help() { echo " -p,--plot Install dependencies for Plotting scripts." } -# Verify if 3.8+ is installed +# Verify if 3.9+ is installed check_installed_python case $* in diff --git a/tests/conftest.py b/tests/conftest.py index 732fffd8f..4372534ce 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -2601,7 +2601,6 @@ def open_trade(): pair='ETH/BTC', open_rate=0.00001099, exchange='binance', - open_order_id='123456789', amount=90.99181073, fee_open=0.0, fee_close=0.0, @@ -2613,7 +2612,7 @@ def open_trade(): Order( ft_order_side='buy', ft_pair=trade.pair, - ft_is_open=False, + ft_is_open=True, ft_amount=trade.amount, ft_price=trade.open_rate, order_id='123456789', @@ -2639,7 +2638,6 @@ def open_trade_usdt(): pair='ADA/USDT', open_rate=2.0, exchange='binance', - open_order_id='123456789_exit', amount=30.0, fee_open=0.0, fee_close=0.0, diff --git a/tests/conftest_trades.py b/tests/conftest_trades.py index e50ed0e59..a2276ae16 100644 --- a/tests/conftest_trades.py +++ b/tests/conftest_trades.py @@ -46,7 +46,6 @@ def mock_trade_1(fee, is_short: bool): open_date=datetime.now(tz=timezone.utc) - timedelta(minutes=17), open_rate=0.123, exchange='binance', - open_order_id=f'dry_run_buy_{direc(is_short)}_12345', strategy='StrategyTestV3', timeframe=5, is_short=is_short @@ -210,7 +209,6 @@ def mock_trade_4(fee, is_short: bool): is_open=True, open_rate=0.123, exchange='binance', - open_order_id=f'prod_buy_{direc(is_short)}_12345', strategy='StrategyTestV3', timeframe=5, is_short=is_short, @@ -327,7 +325,6 @@ def mock_trade_6(fee, is_short: bool): exchange='binance', strategy='SampleStrategy', enter_tag='TEST2', - open_order_id=f"prod_sell_{direc(is_short)}_6", timeframe=5, is_short=is_short ) @@ -411,7 +408,6 @@ def short_trade(fee): # close_profit_abs=-0.6925113200000013, exchange='binance', is_open=True, - open_order_id=None, strategy='DefaultStrategy', timeframe=5, exit_reason='sell_signal', @@ -502,7 +498,6 @@ def leverage_trade(fee): close_profit_abs=2.5983135000000175, exchange='kraken', is_open=False, - open_order_id='dry_run_leverage_buy_12368', strategy='DefaultStrategy', timeframe=5, exit_reason='sell_signal', diff --git a/tests/conftest_trades_usdt.py b/tests/conftest_trades_usdt.py index fc4776148..d73a53605 100644 --- a/tests/conftest_trades_usdt.py +++ b/tests/conftest_trades_usdt.py @@ -66,7 +66,6 @@ def mock_trade_usdt_1(fee, is_short: bool): close_profit_abs=-4.09, exchange='binance', strategy='SampleStrategy', - open_order_id=f'prod_exit_1_{direc(is_short)}', timeframe=5, is_short=is_short, ) @@ -123,7 +122,6 @@ def mock_trade_usdt_2(fee, is_short: bool): close_profit_abs=3.9875, exchange='binance', is_open=False, - open_order_id=f'12366_{direc(is_short)}', strategy='StrategyTestV2', timeframe=5, enter_tag='TEST1', @@ -231,7 +229,6 @@ def mock_trade_usdt_4(fee, is_short: bool): is_open=True, open_rate=2.0, exchange='binance', - open_order_id=f'prod_buy_12345_{direc(is_short)}', strategy='StrategyTestV2', timeframe=5, is_short=is_short, @@ -340,7 +337,6 @@ def mock_trade_usdt_6(fee, is_short: bool): open_rate=10.0, exchange='binance', strategy='SampleStrategy', - open_order_id=f'prod_exit_6_{direc(is_short)}', timeframe=5, is_short=is_short, ) @@ -378,7 +374,6 @@ def mock_trade_usdt_7(fee, is_short: bool): open_date=datetime.now(tz=timezone.utc) - timedelta(minutes=17), open_rate=2.0, exchange='binance', - open_order_id=None, strategy='StrategyTestV2', timeframe=5, is_short=is_short, diff --git a/tests/persistence/test_migrations.py b/tests/persistence/test_migrations.py index 13b3f89bf..1e87d3940 100644 --- a/tests/persistence/test_migrations.py +++ b/tests/persistence/test_migrations.py @@ -15,6 +15,7 @@ from freqtrade.persistence import Trade, init_db from freqtrade.persistence.base import ModelBase from freqtrade.persistence.migrations import get_last_sequence_ids, set_sequence_ids from freqtrade.persistence.models import PairLock +from freqtrade.persistence.trade_model import Order from tests.conftest import log_has @@ -217,6 +218,23 @@ def test_migrate_new(mocker, default_conf, fee, caplog): {amount}, 0, {amount * 0.00258580} + ), + ( + -- Order without reference trade + 2, + 'buy', + 'ETC/BTC', + 1, + 'dry_buy_order55', + 'canceled', + 'ETC/BTC', + 'limit', + 'buy', + 0.00258580, + {amount}, + {amount}, + 0, + {amount * 0.00258580} ) """ engine = create_engine('sqlite://') @@ -238,9 +256,10 @@ def test_migrate_new(mocker, default_conf, fee, caplog): # Run init to test migration init_db(default_conf['db_url']) - trades = Trade.session.scalars(select(Trade).filter(Trade.id == 1)).all() + trades = Trade.session.scalars(select(Trade)).all() assert len(trades) == 1 trade = trades[0] + assert trade.id == 1 assert trade.fee_open == fee.return_value assert trade.fee_close == fee.return_value assert trade.open_rate_requested is None @@ -281,12 +300,18 @@ def test_migrate_new(mocker, default_conf, fee, caplog): assert orders[1].order_id == 'dry_buy_order22' assert orders[1].ft_order_side == 'buy' - assert orders[1].ft_is_open is False + assert orders[1].ft_is_open is True assert orders[2].order_id == 'dry_stop_order_id11X' assert orders[2].ft_order_side == 'stoploss' assert orders[2].ft_is_open is False + orders1 = Order.session.scalars(select(Order)).all() + assert len(orders1) == 5 + order = orders1[4] + assert order.ft_trade_id == 2 + assert order.ft_is_open is False + def test_migrate_too_old(mocker, default_conf, fee, caplog): """ diff --git a/tests/persistence/test_persistence.py b/tests/persistence/test_persistence.py index e4802da87..0f0057d1e 100644 --- a/tests/persistence/test_persistence.py +++ b/tests/persistence/test_persistence.py @@ -10,7 +10,8 @@ from freqtrade.enums import TradingMode from freqtrade.exceptions import DependencyException from freqtrade.persistence import LocalTrade, Order, Trade, init_db from freqtrade.util import dt_now -from tests.conftest import create_mock_trades, create_mock_trades_with_leverage, log_has, log_has_re +from tests.conftest import (create_mock_trades, create_mock_trades_usdt, + create_mock_trades_with_leverage, log_has, log_has_re) spot, margin, futures = TradingMode.SPOT, TradingMode.MARGIN, TradingMode.FUTURES @@ -457,15 +458,14 @@ def test_update_limit_order(fee, caplog, limit_buy_order_usdt, limit_sell_order_ leverage=lev, trading_mode=trading_mode ) - assert trade.open_order_id is None + assert not trade.has_open_orders assert trade.close_profit is None assert trade.close_date is None - trade.open_order_id = enter_order['id'] oobj = Order.parse_from_ccxt_object(enter_order, 'ADA/USDT', entry_side) trade.orders.append(oobj) trade.update_trade(oobj) - assert trade.open_order_id is None + assert not trade.has_open_orders assert trade.open_rate == open_rate assert trade.close_profit is None assert trade.close_date is None @@ -476,13 +476,12 @@ def test_update_limit_order(fee, caplog, limit_buy_order_usdt, limit_sell_order_ caplog) caplog.clear() - trade.open_order_id = enter_order['id'] time_machine.move_to("2022-03-31 21:45:05 +00:00") oobj = Order.parse_from_ccxt_object(exit_order, 'ADA/USDT', exit_side) trade.orders.append(oobj) trade.update_trade(oobj) - assert trade.open_order_id is None + assert not trade.has_open_orders assert trade.close_rate == close_rate assert pytest.approx(trade.close_profit) == profit assert trade.close_date is not None @@ -511,11 +510,10 @@ def test_update_market_order(market_buy_order_usdt, market_sell_order_usdt, fee, leverage=1.0, ) - trade.open_order_id = 'mocked_market_buy' oobj = Order.parse_from_ccxt_object(market_buy_order_usdt, 'ADA/USDT', 'buy') trade.orders.append(oobj) trade.update_trade(oobj) - assert trade.open_order_id is None + assert not trade.has_open_orders assert trade.open_rate == 2.0 assert trade.close_profit is None assert trade.close_date is None @@ -526,11 +524,10 @@ def test_update_market_order(market_buy_order_usdt, market_sell_order_usdt, fee, caplog.clear() trade.is_open = True - trade.open_order_id = 'mocked_market_sell' oobj = Order.parse_from_ccxt_object(market_sell_order_usdt, 'ADA/USDT', 'sell') trade.orders.append(oobj) trade.update_trade(oobj) - assert trade.open_order_id is None + assert not trade.has_open_orders assert trade.close_rate == 2.2 assert pytest.approx(trade.close_profit) == 0.094513715710723 assert trade.close_date is not None @@ -580,7 +577,6 @@ def test_calc_open_close_trade_price( ) entry_order = limit_order[trade.entry_side] exit_order = limit_order[trade.exit_side] - trade.open_order_id = f'something-{is_short}-{lev}-{exchange}' oobj = Order.parse_from_ccxt_object(entry_order, 'ADA/USDT', trade.entry_side) oobj._trade_live = trade @@ -678,7 +674,6 @@ def test_calc_close_trade_price_exception(limit_buy_order_usdt, fee): leverage=1.0, ) - trade.open_order_id = 'something' oobj = Order.parse_from_ccxt_object(limit_buy_order_usdt, 'ADA/USDT', 'buy') trade.update_trade(oobj) assert trade.calc_close_trade_value(trade.close_rate) == 0.0 @@ -697,7 +692,7 @@ def test_update_open_order(limit_buy_order_usdt): trading_mode=margin ) - assert trade.open_order_id is None + assert not trade.has_open_orders assert trade.close_profit is None assert trade.close_date is None @@ -705,7 +700,7 @@ def test_update_open_order(limit_buy_order_usdt): oobj = Order.parse_from_ccxt_object(limit_buy_order_usdt, 'ADA/USDT', 'buy') trade.update_trade(oobj) - assert trade.open_order_id is None + assert not trade.has_open_orders assert trade.close_profit is None assert trade.close_date is None @@ -778,7 +773,6 @@ def test_calc_open_trade_value( is_short=is_short, trading_mode=trading_mode ) - trade.open_order_id = 'open_trade' oobj = Order.parse_from_ccxt_object( limit_buy_order_usdt, 'ADA/USDT', 'sell' if is_short else 'buy') trade.update_trade(oobj) # Buy @ 2.0 @@ -833,7 +827,6 @@ def test_calc_close_trade_price( trading_mode=trading_mode, funding_fees=funding_fees ) - trade.open_order_id = 'close_trade' assert round(trade.calc_close_trade_value(rate=close_rate), 8) == result @@ -1152,14 +1145,30 @@ def test_calc_profit( leverage=lev, fee_open=0.0025, fee_close=fee_close, + max_stake_amount=60.0, trading_mode=trading_mode, funding_fees=funding_fees ) - trade.open_order_id = 'something' + + profit_res = trade.calculate_profit(close_rate) + assert pytest.approx(profit_res.profit_abs) == round(profit, 8) + assert pytest.approx(profit_res.profit_ratio) == round(profit_ratio, 8) + val = trade.open_trade_value * (profit_res.profit_ratio) / lev + assert pytest.approx(val) == profit_res.profit_abs + + assert pytest.approx(profit_res.total_profit) == round(profit, 8) + # assert pytest.approx(profit_res.total_profit_ratio) == round(profit_ratio, 8) assert pytest.approx(trade.calc_profit(rate=close_rate)) == round(profit, 8) assert pytest.approx(trade.calc_profit_ratio(rate=close_rate)) == round(profit_ratio, 8) + profit_res2 = trade.calculate_profit(close_rate, trade.amount, trade.open_rate) + assert pytest.approx(profit_res2.profit_abs) == round(profit, 8) + assert pytest.approx(profit_res2.profit_ratio) == round(profit_ratio, 8) + + assert pytest.approx(profit_res2.total_profit) == round(profit, 8) + # assert pytest.approx(profit_res2.total_profit_ratio) == round(profit_ratio, 8) + assert pytest.approx(trade.calc_profit(close_rate, trade.amount, trade.open_rate)) == round(profit, 8) assert pytest.approx(trade.calc_profit_ratio(close_rate, trade.amount, @@ -1335,6 +1344,24 @@ def test_get_open_lev(fee, use_db): Trade.use_db = True +@pytest.mark.parametrize('is_short', [True, False]) +@pytest.mark.parametrize('use_db', [True, False]) +@pytest.mark.usefixtures("init_persistence") +def test_get_open_orders(fee, is_short, use_db): + Trade.use_db = use_db + Trade.reset_trades() + + create_mock_trades_usdt(fee, is_short, use_db) + # Trade.commit() + trade = Trade.get_trades_proxy(pair="XRP/USDT")[0] + # assert trade.id == 3 + assert len(trade.orders) == 2 + assert len(trade.open_orders) == 0 + assert not trade.has_open_orders + + Trade.use_db = True + + @pytest.mark.usefixtures("init_persistence") def test_to_json(fee): @@ -1350,7 +1377,6 @@ def test_to_json(fee): open_rate=0.123, exchange='binance', enter_tag=None, - open_order_id='dry_run_buy_12345', precision_mode=1, amount_precision=8.0, price_precision=7.0, @@ -1366,7 +1392,6 @@ def test_to_json(fee): 'is_open': None, 'open_date': trade.open_date.strftime(DATETIME_PRINT_FORMAT), 'open_timestamp': int(trade.open_date.timestamp() * 1000), - 'open_order_id': 'dry_run_buy_12345', 'close_date': None, 'close_timestamp': None, 'open_rate': 0.123, @@ -1421,6 +1446,7 @@ def test_to_json(fee): 'price_precision': 7.0, 'precision_mode': 1, 'orders': [], + 'has_open_orders': False, } # Simulate dry_run entries @@ -1488,7 +1514,6 @@ def test_to_json(fee): 'is_open': None, 'max_rate': None, 'min_rate': None, - 'open_order_id': None, 'open_rate_requested': None, 'open_trade_value': 12.33075, 'exit_reason': None, @@ -1507,6 +1532,7 @@ def test_to_json(fee): 'price_precision': 8.0, 'precision_mode': 2, 'orders': [], + 'has_open_orders': False, } @@ -2049,7 +2075,6 @@ def test_Trade_object_idem(): 'total_open_trades_stakes', 'get_closed_trades_without_assigned_fees', 'get_open_trades_without_assigned_fees', - 'get_open_order_trades', 'get_trades', 'get_trades_query', 'get_exit_reason_performance', @@ -2659,7 +2684,7 @@ def test_recalc_trade_from_orders_dca(data) -> None: assert len(trade.orders) == idx + 1 if idx < len(data) - 1: assert trade.is_open is True - assert trade.open_order_id is None + assert not trade.has_open_orders assert trade.amount == result[0] assert trade.open_rate == result[1] assert trade.stake_amount == result[2] @@ -2673,4 +2698,4 @@ def test_recalc_trade_from_orders_dca(data) -> None: assert not trade.is_open trade = Trade.session.scalars(select(Trade)).first() assert trade - assert trade.open_order_id is None + assert not trade.has_open_orders diff --git a/tests/plugins/test_pairlist.py b/tests/plugins/test_pairlist.py index f930bc066..8319ef4c4 100644 --- a/tests/plugins/test_pairlist.py +++ b/tests/plugins/test_pairlist.py @@ -616,6 +616,10 @@ def test_VolumePairList_whitelist_gen(mocker, whitelist_conf, shitcoinmarkets, t ([{"method": "VolumePairList", "number_assets": 5, "sort_key": "quoteVolume", "lookback_timeframe": "1h", "lookback_period": 2, "refresh_period": 3600}], "BTC", "binance", ['ETH/BTC', 'LTC/BTC', 'NEO/BTC', 'TKN/BTC', 'XRP/BTC']), + # TKN/BTC is removed because it doesn't have enough candles + ([{"method": "VolumePairList", "number_assets": 5, "sort_key": "quoteVolume", + "lookback_timeframe": "1d", "lookback_period": 6, "refresh_period": 86400}], + "BTC", "binance", ['LTC/BTC', 'XRP/BTC', 'ETH/BTC', 'HOT/BTC', 'NEO/BTC']), # ftx data is already in Quote currency, therefore won't require conversion # ([{"method": "VolumePairList", "number_assets": 5, "sort_key": "quoteVolume", # "lookback_timeframe": "1d", "lookback_period": 1, "refresh_period": 86400}], @@ -626,23 +630,25 @@ def test_VolumePairList_range(mocker, whitelist_conf, shitcoinmarkets, tickers, whitelist_conf['pairlists'] = pairlists whitelist_conf['stake_currency'] = base_currency whitelist_conf['exchange']['name'] = exchange + # Ensure we have 6 candles + ohlcv_history_long = pd.concat([ohlcv_history, ohlcv_history]) - ohlcv_history_high_vola = ohlcv_history.copy() + ohlcv_history_high_vola = ohlcv_history_long.copy() ohlcv_history_high_vola.loc[ohlcv_history_high_vola.index == 1, 'close'] = 0.00090 # create candles for medium overall volume with last candle high volume - ohlcv_history_medium_volume = ohlcv_history.copy() + ohlcv_history_medium_volume = ohlcv_history_long.copy() ohlcv_history_medium_volume.loc[ohlcv_history_medium_volume.index == 2, 'volume'] = 5 # create candles for high volume with all candles high volume, but very low price. - ohlcv_history_high_volume = ohlcv_history.copy() + ohlcv_history_high_volume = ohlcv_history_long.copy() ohlcv_history_high_volume['volume'] = 10 ohlcv_history_high_volume['low'] = ohlcv_history_high_volume.loc[:, 'low'] * 0.01 ohlcv_history_high_volume['high'] = ohlcv_history_high_volume.loc[:, 'high'] * 0.01 ohlcv_history_high_volume['close'] = ohlcv_history_high_volume.loc[:, 'close'] * 0.01 ohlcv_data = { - ('ETH/BTC', '1d', CandleType.SPOT): ohlcv_history, + ('ETH/BTC', '1d', CandleType.SPOT): ohlcv_history_long, ('TKN/BTC', '1d', CandleType.SPOT): ohlcv_history, ('LTC/BTC', '1d', CandleType.SPOT): ohlcv_history_medium_volume, ('XRP/BTC', '1d', CandleType.SPOT): ohlcv_history_high_vola, diff --git a/tests/rpc/test_rpc.py b/tests/rpc/test_rpc.py index d97222adc..49700b7f4 100644 --- a/tests/rpc/test_rpc.py +++ b/tests/rpc/test_rpc.py @@ -42,7 +42,6 @@ def test_rpc_trade_status(default_conf, ticker, fee, mocker) -> None: 'strategy': ANY, 'enter_tag': ANY, 'timeframe': 5, - 'open_order_id': ANY, 'close_date': None, 'close_timestamp': None, 'open_rate': 1.098e-05, @@ -75,7 +74,7 @@ def test_rpc_trade_status(default_conf, ticker, fee, mocker) -> None: 'stoploss_current_dist_pct': -10.01, 'stoploss_entry_dist': -0.00010402, 'stoploss_entry_dist_ratio': -0.10376381, - 'open_order': None, + 'open_orders': '', 'realized_profit': 0.0, 'realized_profit_ratio': None, 'total_profit_abs': -4.09e-06, @@ -91,6 +90,7 @@ def test_rpc_trade_status(default_conf, ticker, fee, mocker) -> None: 'amount_precision': 8.0, 'price_precision': 8.0, 'precision_mode': 2, + 'has_open_orders': False, 'orders': [{ 'amount': 91.07468123, 'average': 1.098e-05, 'safe_price': 1.098e-05, 'cost': 0.0009999999999054, 'filled': 91.07468123, 'ft_order_side': 'buy', @@ -128,7 +128,8 @@ def test_rpc_trade_status(default_conf, ticker, fee, mocker) -> None: 'profit_pct': 0.0, 'profit_abs': 0.0, 'total_profit_abs': 0.0, - 'open_order': '(limit buy rem=91.07468123)', + 'open_orders': '(limit buy rem=91.07468123)', + 'has_open_orders': True, }) response_unfilled['orders'][0].update({ 'is_open': True, @@ -146,7 +147,7 @@ def test_rpc_trade_status(default_conf, ticker, fee, mocker) -> None: results = rpc._rpc_trade_status() # Reuse above object, only remaining changed. response_unfilled['orders'][0].update({ - 'remaining': None + 'remaining': None, }) assert results[0] == response_unfilled @@ -164,7 +165,8 @@ def test_rpc_trade_status(default_conf, ticker, fee, mocker) -> None: response = deepcopy(gen_response) response.update({ 'max_stake_amount': 0.001, - 'total_profit_ratio': pytest.approx(-0.00409), + 'total_profit_ratio': pytest.approx(-0.00409153), + 'has_open_orders': False, }) assert results[0] == response @@ -779,7 +781,7 @@ def test_rpc_force_exit(default_conf, ticker, fee, mocker) -> None: 'amount': amount, 'remaining': amount, 'filled': 0.0, - 'id': trade.orders[0].order_id, + 'id': trade.orders[-1].order_id, } ) cancel_order_3 = mocker.patch( @@ -791,7 +793,7 @@ def test_rpc_force_exit(default_conf, ticker, fee, mocker) -> None: 'amount': amount, 'remaining': amount, 'filled': 0.0, - 'id': trade.orders[0].order_id, + 'id': trade.orders[-1].order_id, } ) msg = rpc._rpc_force_exit('3') @@ -800,7 +802,7 @@ def test_rpc_force_exit(default_conf, ticker, fee, mocker) -> None: assert cancel_order_3.call_count == 1 assert cancel_order_mock.call_count == 0 - trade = Trade.session.scalars(select(Trade).filter(Trade.id == '2')).first() + trade = Trade.session.scalars(select(Trade).filter(Trade.id == '4')).first() amount = trade.amount # make an limit-buy open trade, if there is no 'filled', don't sell it mocker.patch( @@ -829,7 +831,7 @@ def test_rpc_force_exit(default_conf, ticker, fee, mocker) -> None: assert msg == {'result': 'Created exit order for trade 4.'} assert cancel_order_4.call_count == 1 assert cancel_order_mock.call_count == 0 - assert trade.amount == amount + assert pytest.approx(trade.amount) == amount def test_performance_handle(default_conf_usdt, ticker, fee, mocker) -> None: @@ -1097,7 +1099,8 @@ def test_rpc_force_entry(mocker, default_conf, ticker, fee, limit_buy_order_open trade = rpc._rpc_force_entry(pair, 0.0001, order_type='limit', stake_amount=0.05) assert trade.stake_amount == 0.05 assert trade.buy_tag == 'force_entry' - assert trade.open_order_id == 'mocked_limit_buy' + + assert trade.open_orders_ids[-1] == 'mocked_limit_buy' freqtradebot.strategy.position_adjustment_enable = True with pytest.raises(RPCException, match=r'position for LTC/BTC already open.*open order.*'): diff --git a/tests/rpc/test_rpc_apiserver.py b/tests/rpc/test_rpc_apiserver.py index 856d10dc5..89cb47830 100644 --- a/tests/rpc/test_rpc_apiserver.py +++ b/tests/rpc/test_rpc_apiserver.py @@ -977,6 +977,10 @@ def test_api_profit(botclient, mocker, ticker, fee, markets, is_short, expected) 'expectancy_ratio': expected['expectancy_ratio'], 'max_drawdown': ANY, 'max_drawdown_abs': ANY, + 'max_drawdown_start': ANY, + 'max_drawdown_start_timestamp': ANY, + 'max_drawdown_end': ANY, + 'max_drawdown_end_timestamp': ANY, 'trading_volume': expected['trading_volume'], 'bot_start_timestamp': 0, 'bot_start_date': '', @@ -1022,7 +1026,6 @@ def test_api_performance(botclient, fee): exchange='binance', stake_amount=1, open_rate=0.245441, - open_order_id="123456", is_open=False, fee_close=fee.return_value, fee_open=fee.return_value, @@ -1039,7 +1042,6 @@ def test_api_performance(botclient, fee): stake_amount=1, exchange='binance', open_rate=0.412, - open_order_id="123456", is_open=False, fee_close=fee.return_value, fee_open=fee.return_value, @@ -1062,11 +1064,11 @@ def test_api_performance(botclient, fee): @pytest.mark.parametrize( - 'is_short,current_rate,open_order_id,open_trade_value', - [(True, 1.098e-05, 'dry_run_buy_short_12345', 15.0911775), - (False, 1.099e-05, 'dry_run_buy_long_12345', 15.1668225)]) + 'is_short,current_rate,open_trade_value', + [(True, 1.098e-05, 15.0911775), + (False, 1.099e-05, 15.1668225)]) def test_api_status(botclient, mocker, ticker, fee, markets, is_short, - current_rate, open_order_id, open_trade_value): + current_rate, open_trade_value): ftbot, client = botclient patch_get_signal(ftbot) mocker.patch.multiple( @@ -1107,7 +1109,6 @@ def test_api_status(botclient, mocker, ticker, fee, markets, is_short, 'current_rate': current_rate, 'open_date': ANY, 'open_timestamp': ANY, - 'open_order': None, 'open_rate': 0.123, 'pair': 'ETH/BTC', 'base_currency': 'ETH', @@ -1140,7 +1141,6 @@ def test_api_status(botclient, mocker, ticker, fee, markets, is_short, "is_short": is_short, 'max_rate': ANY, 'min_rate': ANY, - 'open_order_id': open_order_id, 'open_rate_requested': ANY, 'open_trade_value': open_trade_value, 'exit_reason': None, @@ -1158,6 +1158,7 @@ def test_api_status(botclient, mocker, ticker, fee, markets, is_short, 'price_precision': None, 'precision_mode': None, 'orders': [ANY], + 'has_open_orders': True, } mocker.patch(f'{EXMS}.get_rate', @@ -1287,7 +1288,6 @@ def test_api_force_entry(botclient, mocker, fee, endpoint): exchange='binance', stake_amount=1, open_rate=0.245441, - open_order_id="123456", open_date=datetime.now(timezone.utc), is_open=False, is_short=False, @@ -1348,7 +1348,6 @@ def test_api_force_entry(botclient, mocker, fee, endpoint): 'is_short': False, 'max_rate': None, 'min_rate': None, - 'open_order_id': '123456', 'open_rate_requested': None, 'open_trade_value': 0.24605460, 'exit_reason': None, @@ -1365,6 +1364,7 @@ def test_api_force_entry(botclient, mocker, fee, endpoint): 'amount_precision': None, 'price_precision': None, 'precision_mode': None, + 'has_open_orders': False, 'orders': [], } diff --git a/tests/rpc/test_rpc_telegram.py b/tests/rpc/test_rpc_telegram.py index 899bef20e..41c24cc45 100644 --- a/tests/rpc/test_rpc_telegram.py +++ b/tests/rpc/test_rpc_telegram.py @@ -347,8 +347,8 @@ async def test_telegram_status_multi_entry(default_conf, update, mocker, fee) -> msg = msg_mock.call_args_list[3][0][0] assert re.search(r'Number of Entries.*2', msg) assert re.search(r'Number of Exits.*1', msg) - assert re.search(r'Average Entry Price', msg) - assert re.search(r'Order filled', msg) + assert re.search(r'from 1st entry rate', msg) + assert re.search(r'Order Filled', msg) assert re.search(r'Close Date:', msg) is None assert re.search(r'Close Profit:', msg) is None diff --git a/tests/strategy/test_strategy_helpers.py b/tests/strategy/test_strategy_helpers.py index d6aebd982..535b3dbd6 100644 --- a/tests/strategy/test_strategy_helpers.py +++ b/tests/strategy/test_strategy_helpers.py @@ -96,6 +96,32 @@ def test_merge_informative_pair_lower(): merge_informative_pair(data, informative, '1h', '15m', ffill=True) +def test_merge_informative_pair_empty(): + data = generate_test_data('1h', 40) + informative = pd.DataFrame(columns=data.columns) + + result = merge_informative_pair(data, informative, '1h', '2h', ffill=True) + assert result['date'].equals(data['date']) + + assert list(result.columns) == [ + 'date', + 'open', + 'high', + 'low', + 'close', + 'volume', + 'date_2h', + 'open_2h', + 'high_2h', + 'low_2h', + 'close_2h', + 'volume_2h' + ] + # We merge an empty dataframe, so all values should be NaN + for col in ['date_2h', 'open_2h', 'high_2h', 'low_2h', 'close_2h', 'volume_2h']: + assert result[col].isnull().all() + + def test_merge_informative_pair_suffix(): data = generate_test_data('15m', 20) informative = generate_test_data('1h', 20) @@ -110,6 +136,21 @@ def test_merge_informative_pair_suffix(): assert 'open_suf' in result.columns assert 'open_1h' not in result.columns + assert list(result.columns) == [ + 'date', + 'open', + 'high', + 'low', + 'close', + 'volume', + 'date_suf', + 'open_suf', + 'high_suf', + 'low_suf', + 'close_suf', + 'volume_suf' + ] + def test_merge_informative_pair_suffix_append_timeframe(): data = generate_test_data('15m', 20) diff --git a/tests/test_freqtradebot.py b/tests/test_freqtradebot.py index b1f72f2f1..886012535 100644 --- a/tests/test_freqtradebot.py +++ b/tests/test_freqtradebot.py @@ -872,7 +872,8 @@ def test_execute_entry(mocker, default_conf_usdt, fee, limit_order, trade.is_short = is_short assert trade assert trade.is_open is True - assert trade.open_order_id == '22' + assert trade.has_open_orders + assert '22' in trade.open_orders_ids # Test calling with price open_order['id'] = '33' @@ -898,7 +899,7 @@ def test_execute_entry(mocker, default_conf_usdt, fee, limit_order, trade = Trade.session.scalars(select(Trade)).all()[2] trade.is_short = is_short assert trade - assert trade.open_order_id is None + assert not trade.has_open_orders assert trade.open_rate == 10 assert trade.stake_amount == round(order['average'] * order['filled'] / leverage, 8) assert pytest.approx(trade.liquidation_price) == liq_price @@ -916,7 +917,7 @@ def test_execute_entry(mocker, default_conf_usdt, fee, limit_order, trade = Trade.session.scalars(select(Trade)).all()[3] trade.is_short = is_short assert trade - assert trade.open_order_id is None + assert not trade.has_open_orders assert trade.open_rate == 0.5 assert trade.stake_amount == round(order['average'] * order['filled'] / leverage, 8) @@ -1118,7 +1119,6 @@ def test_add_stoploss_on_exchange(mocker, default_conf_usdt, limit_order, is_sho freqtrade.enter_positions() trade = Trade.session.scalars(select(Trade)).first() trade.is_short = is_short - trade.open_order_id = None trade.stoploss_order_id = None trade.is_open = True trades = [trade] @@ -1163,7 +1163,6 @@ def test_handle_stoploss_on_exchange(mocker, default_conf_usdt, fee, caplog, is_ trade = Trade.session.scalars(select(Trade)).first() trade.is_short = is_short trade.is_open = True - trade.open_order_id = None trade.stoploss_order_id = None assert freqtrade.handle_stoploss_on_exchange(trade) is False @@ -1174,7 +1173,6 @@ def test_handle_stoploss_on_exchange(mocker, default_conf_usdt, fee, caplog, is_ # should do nothing and return false stop_order_dict.update({'id': "102"}) trade.is_open = True - trade.open_order_id = None trade.stoploss_order_id = "102" trade.orders.append( Order( @@ -1198,7 +1196,6 @@ def test_handle_stoploss_on_exchange(mocker, default_conf_usdt, fee, caplog, is_ # should set a stoploss immediately and return False caplog.clear() trade.is_open = True - trade.open_order_id = None trade.stoploss_order_id = "102" canceled_stoploss_order = MagicMock(return_value={'id': '103_1', 'status': 'canceled'}) @@ -1223,7 +1220,6 @@ def test_handle_stoploss_on_exchange(mocker, default_conf_usdt, fee, caplog, is_ trade = Trade.session.scalars(select(Trade)).first() trade.is_short = is_short trade.is_open = True - trade.open_order_id = None trade.stoploss_order_id = "104" trade.orders.append(Order( ft_order_side='stoploss', @@ -1351,7 +1347,6 @@ def test_handle_stoploss_on_exchange_partial( trade = Trade.session.scalars(select(Trade)).first() trade.is_short = is_short trade.is_open = True - trade.open_order_id = None trade.stoploss_order_id = None assert freqtrade.handle_stoploss_on_exchange(trade) is False @@ -1410,7 +1405,6 @@ def test_handle_stoploss_on_exchange_partial_cancel_here( trade = Trade.session.scalars(select(Trade)).first() trade.is_short = is_short trade.is_open = True - trade.open_order_id = None trade.stoploss_order_id = None assert freqtrade.handle_stoploss_on_exchange(trade) is False @@ -1468,8 +1462,8 @@ def test_handle_sle_cancel_cant_recreate(mocker, default_conf_usdt, fee, caplog, 'last': 1.9 }), create_order=MagicMock(side_effect=[ - {'id': enter_order['id']}, - {'id': exit_order['id']}, + enter_order, + exit_order, ]), get_fee=fee, ) @@ -1485,7 +1479,6 @@ def test_handle_sle_cancel_cant_recreate(mocker, default_conf_usdt, fee, caplog, trade = Trade.session.scalars(select(Trade)).first() assert trade.is_short == is_short trade.is_open = True - trade.open_order_id = None trade.stoploss_order_id = "100" trade.orders.append( Order( @@ -1663,7 +1656,6 @@ def test_handle_stoploss_on_exchange_trailing( trade = Trade.session.scalars(select(Trade)).first() trade.is_short = is_short trade.is_open = True - trade.open_order_id = None trade.stoploss_order_id = '100' trade.stoploss_last_update = dt_now() - timedelta(minutes=20) trade.orders.append( @@ -1793,7 +1785,6 @@ def test_handle_stoploss_on_exchange_trailing_error( trade = Trade.session.scalars(select(Trade)).first() trade.is_short = is_short trade.is_open = True - trade.open_order_id = None trade.stoploss_order_id = "abcd" trade.stop_loss = 0.2 trade.stoploss_last_update = (dt_now() - timedelta(minutes=601)).replace(tzinfo=None) @@ -1873,8 +1864,8 @@ def test_handle_stoploss_on_exchange_custom_stop( 'last': 1.9 }), create_order=MagicMock(side_effect=[ - {'id': enter_order['id']}, - {'id': exit_order['id']}, + enter_order, + exit_order, ]), get_fee=fee, ) @@ -1907,7 +1898,6 @@ def test_handle_stoploss_on_exchange_custom_stop( trade = Trade.session.scalars(select(Trade)).first() trade.is_short = is_short trade.is_open = True - trade.open_order_id = None trade.stoploss_order_id = '100' trade.stoploss_last_update = dt_now() - timedelta(minutes=601) trade.orders.append( @@ -2045,7 +2035,6 @@ def test_tsl_on_exchange_compatible_with_edge(mocker, edge_conf, fee, limit_orde freqtrade.enter_positions() trade = Trade.session.scalars(select(Trade)).first() trade.is_open = True - trade.open_order_id = None trade.stoploss_order_id = '100' trade.stoploss_last_update = dt_now() trade.orders.append( @@ -2152,7 +2141,6 @@ def test_exit_positions(mocker, default_conf_usdt, limit_order, is_short, caplog order_id = '123' trade = Trade( - open_order_id=order_id, pair='ETH/USDT', fee_open=0.001, fee_close=0.001, @@ -2198,7 +2186,6 @@ def test_exit_positions_exception(mocker, default_conf_usdt, limit_order, caplog order_id = '123' trade = Trade( - open_order_id=order_id, pair='ETH/USDT', fee_open=0.001, fee_close=0.001, @@ -2217,9 +2204,9 @@ def test_exit_positions_exception(mocker, default_conf_usdt, limit_order, caplog ft_amount=trade.amount, ft_price=trade.open_rate, order_id=order_id, + ft_is_open=False, )) - trade.open_order_id = None Trade.session.add(trade) Trade.commit() freqtrade.wallets.update() @@ -2248,7 +2235,6 @@ def test_update_trade_state(mocker, default_conf_usdt, limit_order, is_short, ca order_id = order['id'] trade = Trade( - open_order_id=order_id, fee_open=0.001, fee_close=0.001, open_rate=0.01, @@ -2272,19 +2258,17 @@ def test_update_trade_state(mocker, default_conf_usdt, limit_order, is_short, ca # Test amount not modified by fee-logic assert not log_has_re(r'Applying fee to .*', caplog) caplog.clear() - assert trade.open_order_id is None + assert not trade.has_open_orders assert trade.amount == order['amount'] - trade.open_order_id = order_id mocker.patch('freqtrade.freqtradebot.FreqtradeBot.get_real_amount', return_value=0.01) assert trade.amount == 30.0 # test amount modified by fee-logic freqtrade.update_trade_state(trade, order_id) assert trade.amount == 29.99 - assert trade.open_order_id is None + assert not trade.has_open_orders trade.is_open = True - trade.open_order_id = None # Assert we call handle_trade() if trade is feasible for execution freqtrade.update_trade_state(trade, order_id) @@ -2328,7 +2312,6 @@ def test_update_trade_state_withorderdict( open_date=dt_now(), fee_open=fee.return_value, fee_close=fee.return_value, - open_order_id=order_id, is_open=True, leverage=1, is_short=is_short, @@ -2361,15 +2344,15 @@ def test_update_trade_state_exception(mocker, default_conf_usdt, is_short, limit # TODO: should not be magicmock trade = MagicMock() - trade.open_order_id = '123' trade.amount = 123 + open_order_id = '123' # Test raise of OperationalException exception mocker.patch( 'freqtrade.freqtradebot.FreqtradeBot.get_real_amount', side_effect=DependencyException() ) - freqtrade.update_trade_state(trade, trade.open_order_id) + freqtrade.update_trade_state(trade, open_order_id) assert log_has('Could not update trade amount: ', caplog) @@ -2379,13 +2362,13 @@ def test_update_trade_state_orderexception(mocker, default_conf_usdt, caplog) -> # TODO: should not be magicmock trade = MagicMock() - trade.open_order_id = '123' + open_order_id = '123' # Test raise of OperationalException exception grm_mock = mocker.patch("freqtrade.freqtradebot.FreqtradeBot.get_real_amount", MagicMock()) - freqtrade.update_trade_state(trade, trade.open_order_id) + freqtrade.update_trade_state(trade, open_order_id) assert grm_mock.call_count == 0 - assert log_has(f'Unable to fetch order {trade.open_order_id}: ', caplog) + assert log_has(f'Unable to fetch order {open_order_id}: ', caplog) @pytest.mark.parametrize("is_short", [False, True]) @@ -2413,7 +2396,6 @@ def test_update_trade_state_sell( fee_open=0.0025, fee_close=0.0025, open_date=dt_now(), - open_order_id=open_order['id'], is_open=True, interest_rate=0.0005, leverage=1, @@ -2425,7 +2407,7 @@ def test_update_trade_state_sell( order = Order.parse_from_ccxt_object(open_order, 'LTC/ETH', exit_side(is_short)) trade.orders.append(order) assert order.status == 'open' - freqtrade.update_trade_state(trade, trade.open_order_id, l_order) + freqtrade.update_trade_state(trade, trade.open_orders_ids[-1], l_order) assert trade.amount == l_order['amount'] # Wallet needs to be updated after closing a limit-sell order to reenable buying assert wallet_mock.call_count == 1 @@ -2475,7 +2457,7 @@ def test_handle_trade( patch_get_signal(freqtrade, enter_long=False, exit_short=is_short, exit_long=not is_short, exit_tag='sell_signal1') assert freqtrade.handle_trade(trade) is True - assert trade.open_order_id == exit_order['id'] + assert trade.open_orders_ids[-1] == exit_order['id'] # Simulate fulfilled LIMIT_SELL order for trade trade.orders[-1].ft_is_open = False @@ -2706,7 +2688,7 @@ def test_manage_open_orders_entry_usercustom( ) -> None: old_order = limit_sell_order_old if is_short else limit_buy_order_old - old_order['id'] = open_trade.open_order_id + old_order['id'] = open_trade.open_orders_ids[0] default_conf_usdt["unfilledtimeout"] = {"entry": 1400, "exit": 30} @@ -2741,7 +2723,11 @@ def test_manage_open_orders_entry_usercustom( freqtrade.manage_open_orders() assert cancel_order_mock.call_count == 0 trades = Trade.session.scalars( - select(Trade).filter(Trade.open_order_id.is_(open_trade.open_order_id))).all() + select(Trade) + .where(Order.ft_is_open.is_(True)) + .where(Order.ft_order_side != "stoploss") + .where(Order.ft_trade_id == Trade.id) + ).all() nb_trades = len(trades) assert nb_trades == 1 assert freqtrade.strategy.check_entry_timeout.call_count == 1 @@ -2750,7 +2736,11 @@ def test_manage_open_orders_entry_usercustom( freqtrade.manage_open_orders() assert cancel_order_mock.call_count == 0 trades = Trade.session.scalars( - select(Trade).filter(Trade.open_order_id.is_(open_trade.open_order_id))).all() + select(Trade) + .where(Order.ft_is_open.is_(True)) + .where(Order.ft_order_side != "stoploss") + .where(Order.ft_trade_id == Trade.id) + ).all() nb_trades = len(trades) assert nb_trades == 1 assert freqtrade.strategy.check_entry_timeout.call_count == 1 @@ -2761,7 +2751,11 @@ def test_manage_open_orders_entry_usercustom( assert cancel_order_wr_mock.call_count == 1 assert rpc_mock.call_count == 2 trades = Trade.session.scalars( - select(Trade).filter(Trade.open_order_id.is_(open_trade.open_order_id))).all() + select(Trade) + .where(Order.ft_is_open.is_(True)) + .where(Order.ft_order_side != "stoploss") + .where(Order.ft_trade_id == Trade.id) + ).all() nb_trades = len(trades) assert nb_trades == 0 assert freqtrade.strategy.check_entry_timeout.call_count == 1 @@ -2774,7 +2768,7 @@ def test_manage_open_orders_entry( ) -> None: old_order = limit_sell_order_old if is_short else limit_buy_order_old rpc_mock = patch_RPCManager(mocker) - open_trade.open_order_id = old_order['id'] + order = Order.parse_from_ccxt_object(old_order, 'mocked', 'buy') open_trade.orders[0] = order limit_buy_cancel = deepcopy(old_order) @@ -2801,7 +2795,11 @@ def test_manage_open_orders_entry( assert cancel_order_mock.call_count == 1 assert rpc_mock.call_count == 2 trades = Trade.session.scalars( - select(Trade).filter(Trade.open_order_id.is_(open_trade.open_order_id))).all() + select(Trade) + .where(Order.ft_is_open.is_(True)) + .where(Order.ft_order_side != "stoploss") + .where(Order.ft_trade_id == Trade.id) + ).all() nb_trades = len(trades) assert nb_trades == 0 # Custom user entry-timeout is never called @@ -2817,7 +2815,7 @@ def test_adjust_entry_cancel( ) -> None: freqtrade = get_patched_freqtradebot(mocker, default_conf_usdt) old_order = limit_sell_order_old if is_short else limit_buy_order_old - old_order['id'] = open_trade.open_order_id + old_order['id'] = open_trade.open_orders[0].order_id limit_buy_cancel = deepcopy(old_order) limit_buy_cancel['status'] = 'canceled' cancel_order_mock = MagicMock(return_value=limit_buy_cancel) @@ -2840,7 +2838,10 @@ def test_adjust_entry_cancel( freqtrade.strategy.adjust_entry_price = MagicMock(return_value=None) freqtrade.manage_open_orders() trades = Trade.session.scalars( - select(Trade).filter(Trade.open_order_id.is_(open_trade.open_order_id))).all() + select(Trade) + .where(Order.ft_trade_id == Trade.id) + ).all() + assert len(trades) == 0 assert len(Order.session.scalars(select(Order)).all()) == 0 assert log_has_re( @@ -2859,7 +2860,7 @@ def test_adjust_entry_maintain_replace( ) -> None: freqtrade = get_patched_freqtradebot(mocker, default_conf_usdt) old_order = limit_sell_order_old if is_short else limit_buy_order_old - old_order['id'] = open_trade.open_order_id + old_order['id'] = open_trade.open_orders_ids[0] limit_buy_cancel = deepcopy(old_order) limit_buy_cancel['status'] = 'canceled' cancel_order_mock = MagicMock(return_value=limit_buy_cancel) @@ -2868,7 +2869,8 @@ def test_adjust_entry_maintain_replace( fetch_ticker=ticker_usdt, fetch_order=MagicMock(return_value=old_order), cancel_order_with_result=cancel_order_mock, - get_fee=fee + get_fee=fee, + _dry_is_price_crossed=MagicMock(return_value=False), ) open_trade.is_short = is_short @@ -2882,7 +2884,10 @@ def test_adjust_entry_maintain_replace( freqtrade.strategy.adjust_entry_price = MagicMock(return_value=old_order['price']) freqtrade.manage_open_orders() trades = Trade.session.scalars( - select(Trade).filter(Trade.open_order_id.is_(open_trade.open_order_id))).all() + select(Trade) + .where(Order.ft_is_open.is_(True)) + .where(Order.ft_trade_id == Trade.id) + ).all() assert len(trades) == 1 assert len(Order.get_open_orders()) == 1 # Entry adjustment is called @@ -2891,9 +2896,16 @@ def test_adjust_entry_maintain_replace( # Check that order is replaced freqtrade.get_valid_enter_price_and_stake = MagicMock(return_value={100, 10, 1}) freqtrade.strategy.adjust_entry_price = MagicMock(return_value=1234) + freqtrade.manage_open_orders() + + assert freqtrade.strategy.adjust_entry_price.call_count == 1 + trades = Trade.session.scalars( - select(Trade).filter(Trade.open_order_id.is_(open_trade.open_order_id))).all() + select(Trade) + .where(Order.ft_is_open.is_(True)) + .where(Order.ft_trade_id == Trade.id) + ).all() assert len(trades) == 1 nb_all_orders = len(Order.session.scalars(select(Order)).all()) assert nb_all_orders == 2 @@ -2917,6 +2929,8 @@ def test_check_handle_cancelled_buy( cancel_order_mock = MagicMock() patch_exchange(mocker) old_order.update({"status": "canceled", 'filled': 0.0}) + old_order['side'] = 'buy' if is_short else 'sell' + old_order['id'] = open_trade.open_orders[0].order_id mocker.patch.multiple( EXMS, fetch_ticker=ticker_usdt, @@ -2925,7 +2939,6 @@ def test_check_handle_cancelled_buy( get_fee=fee ) freqtrade = FreqtradeBot(default_conf_usdt) - open_trade.orders = [] open_trade.is_short = is_short Trade.session.add(open_trade) Trade.commit() @@ -2935,10 +2948,13 @@ def test_check_handle_cancelled_buy( assert cancel_order_mock.call_count == 0 assert rpc_mock.call_count == 2 trades = Trade.session.scalars( - select(Trade).filter(Trade.open_order_id.is_(open_trade.open_order_id))).all() + select(Trade) + .where(Order.ft_is_open.is_(True)) + .where(Order.ft_trade_id == Trade.id) + ).all() assert len(trades) == 0 - assert log_has_re( - f"{'Sell' if is_short else 'Buy'} order cancelled on exchange for Trade.*", caplog) + exit_name = 'Buy' if is_short else 'Sell' + assert log_has_re(f"{exit_name} order cancelled on exchange for Trade.*", caplog) @pytest.mark.parametrize("is_short", [False, True]) @@ -2966,10 +2982,7 @@ def test_manage_open_orders_buy_exception( freqtrade.manage_open_orders() assert cancel_order_mock.call_count == 0 assert rpc_mock.call_count == 1 - trades = Trade.session.scalars( - select(Trade).filter(Trade.open_order_id.is_(open_trade.open_order_id))).all() - nb_trades = len(trades) - assert nb_trades == 1 + assert len(open_trade.open_orders) == 1 @pytest.mark.parametrize("is_short", [False, True]) @@ -2978,7 +2991,7 @@ def test_manage_open_orders_exit_usercustom( is_short, open_trade_usdt, caplog ) -> None: default_conf_usdt["unfilledtimeout"] = {"entry": 1440, "exit": 1440, "exit_timeout_count": 1} - open_trade_usdt.open_order_id = limit_sell_order_old['id'] + if is_short: limit_sell_order_old['side'] = 'buy' open_trade_usdt.is_short = is_short @@ -3035,13 +3048,10 @@ def test_manage_open_orders_exit_usercustom( assert rpc_mock.call_count == 2 assert freqtrade.strategy.check_exit_timeout.call_count == 1 assert freqtrade.strategy.check_entry_timeout.call_count == 0 - trade = Trade.session.scalars(select(Trade)).first() - # cancelling didn't succeed - order-id remains open. - assert trade.open_order_id is not None # 2nd canceled trade - Fail execute exit caplog.clear() - open_trade_usdt.open_order_id = limit_sell_order_old['id'] + mocker.patch('freqtrade.persistence.Trade.get_exit_order_count', return_value=1) mocker.patch('freqtrade.freqtradebot.FreqtradeBot.execute_trade_exit', side_effect=DependencyException) @@ -3051,7 +3061,6 @@ def test_manage_open_orders_exit_usercustom( et_mock = mocker.patch('freqtrade.freqtradebot.FreqtradeBot.execute_trade_exit') caplog.clear() # 2nd canceled trade ... - open_trade_usdt.open_order_id = limit_sell_order_old['id'] # If cancelling fails - no emergency exit! with patch('freqtrade.freqtradebot.FreqtradeBot.handle_cancel_exit', return_value=False): @@ -3069,7 +3078,7 @@ def test_manage_open_orders_exit( ) -> None: rpc_mock = patch_RPCManager(mocker) cancel_order_mock = MagicMock() - limit_sell_order_old['id'] = open_trade_usdt.open_order_id + limit_sell_order_old['id'] = '123456789_exit' limit_sell_order_old['side'] = 'buy' if is_short else 'sell' patch_exchange(mocker) mocker.patch.multiple( @@ -3111,7 +3120,7 @@ def test_check_handle_cancelled_exit( cancel_order_mock = MagicMock() limit_sell_order_old.update({"status": "canceled", 'filled': 0.0}) limit_sell_order_old['side'] = 'buy' if is_short else 'sell' - limit_sell_order_old['id'] = open_trade_usdt.open_order_id + limit_sell_order_old['id'] = open_trade_usdt.open_orders[0].order_id patch_exchange(mocker) mocker.patch.multiple( @@ -3148,7 +3157,8 @@ def test_manage_open_orders_partial( open_trade.is_short = is_short open_trade.leverage = leverage open_trade.orders[0].ft_order_side = 'sell' if is_short else 'buy' - limit_buy_order_old_partial['id'] = open_trade.open_order_id + + limit_buy_order_old_partial['id'] = open_trade.orders[0].order_id limit_buy_order_old_partial['side'] = 'sell' if is_short else 'buy' limit_buy_canceled = deepcopy(limit_buy_order_old_partial) limit_buy_canceled['status'] = 'canceled' @@ -3172,11 +3182,13 @@ def test_manage_open_orders_partial( assert cancel_order_mock.call_count == 1 assert rpc_mock.call_count == 3 trades = Trade.session.scalars( - select(Trade).filter(Trade.open_order_id.is_(open_trade.open_order_id))).all() + select(Trade) + ).all() assert len(trades) == 1 assert trades[0].amount == 23.0 assert trades[0].stake_amount == open_trade.open_rate * trades[0].amount / leverage assert trades[0].stake_amount != prior_stake + assert not trades[0].has_open_orders @pytest.mark.parametrize("is_short", [False, True]) @@ -3188,8 +3200,8 @@ def test_manage_open_orders_partial_fee( open_trade.is_short = is_short open_trade.orders[0].ft_order_side = 'sell' if is_short else 'buy' rpc_mock = patch_RPCManager(mocker) - limit_buy_order_old_partial['id'] = open_trade.open_order_id - limit_buy_order_old_partial_canceled['id'] = open_trade.open_order_id + limit_buy_order_old_partial['id'] = open_trade.orders[0].order_id + limit_buy_order_old_partial_canceled['id'] = open_trade.open_orders_ids[0] limit_buy_order_old_partial['side'] = 'sell' if is_short else 'buy' limit_buy_order_old_partial_canceled['side'] = 'sell' if is_short else 'buy' @@ -3220,12 +3232,14 @@ def test_manage_open_orders_partial_fee( assert cancel_order_mock.call_count == 1 assert rpc_mock.call_count == 3 trades = Trade.session.scalars( - select(Trade).filter(Trade.open_order_id.is_(open_trade.open_order_id))).all() + select(Trade) + .where(Order.ft_trade_id == Trade.id) + ).all() assert len(trades) == 1 # Verify that trade has been updated assert trades[0].amount == (limit_buy_order_old_partial['amount'] - limit_buy_order_old_partial['remaining']) - 0.023 - assert trades[0].open_order_id is None + assert not trades[0].has_open_orders assert trades[0].fee_updated(open_trade.entry_side) assert pytest.approx(trades[0].fee_open) == 0.001 @@ -3239,8 +3253,8 @@ def test_manage_open_orders_partial_except( open_trade.is_short = is_short open_trade.orders[0].ft_order_side = 'sell' if is_short else 'buy' rpc_mock = patch_RPCManager(mocker) - limit_buy_order_old_partial_canceled['id'] = open_trade.open_order_id - limit_buy_order_old_partial['id'] = open_trade.open_order_id + limit_buy_order_old_partial_canceled['id'] = open_trade.open_orders_ids[0] + limit_buy_order_old_partial['id'] = open_trade.open_orders_ids[0] if is_short: limit_buy_order_old_partial['side'] = 'sell' cancel_order_mock = MagicMock(return_value=limit_buy_order_old_partial_canceled) @@ -3271,13 +3285,14 @@ def test_manage_open_orders_partial_except( assert cancel_order_mock.call_count == 1 assert rpc_mock.call_count == 3 trades = Trade.session.scalars( - select(Trade).filter(Trade.open_order_id.is_(open_trade.open_order_id))).all() + select(Trade) + ).all() assert len(trades) == 1 # Verify that trade has been updated assert trades[0].amount == (limit_buy_order_old_partial['amount'] - limit_buy_order_old_partial['remaining']) - assert trades[0].open_order_id is None + assert not trades[0].has_open_orders assert trades[0].fee_open == fee() @@ -3335,32 +3350,34 @@ def test_handle_cancel_enter(mocker, caplog, default_conf_usdt, limit_order, is_ l_order['filled'] = 0.0 l_order['status'] = 'open' reason = CANCEL_REASON['TIMEOUT'] - assert freqtrade.handle_cancel_enter(trade, l_order, reason) + assert freqtrade.handle_cancel_enter(trade, l_order, trade.open_orders_ids[0], reason) assert cancel_order_mock.call_count == 1 cancel_order_mock.reset_mock() caplog.clear() l_order['filled'] = 0.01 - assert not freqtrade.handle_cancel_enter(trade, l_order, reason) + assert not freqtrade.handle_cancel_enter(trade, l_order, trade.open_orders_ids[0], reason) assert cancel_order_mock.call_count == 0 assert log_has_re("Order .* for .* not cancelled, as the filled amount.* unexitable.*", caplog) caplog.clear() cancel_order_mock.reset_mock() l_order['filled'] = 2 - assert not freqtrade.handle_cancel_enter(trade, l_order, reason) + assert not freqtrade.handle_cancel_enter(trade, l_order, trade.open_orders_ids[0], reason) assert cancel_order_mock.call_count == 1 # Order remained open for some reason (cancel failed) cancel_buy_order['status'] = 'open' cancel_order_mock = MagicMock(return_value=cancel_buy_order) - trade.open_order_id = 'some_open_order' + mocker.patch(f'{EXMS}.cancel_order_with_result', cancel_order_mock) - assert not freqtrade.handle_cancel_enter(trade, l_order, reason) + assert not freqtrade.handle_cancel_enter(trade, l_order, trade.open_orders_ids[0], reason) assert log_has_re(r"Order .* for .* not cancelled.", caplog) # min_pair_stake empty should not crash mocker.patch(f'{EXMS}.get_min_pair_stake_amount', return_value=None) - assert not freqtrade.handle_cancel_enter(trade, limit_order[entry_side(is_short)], reason) + assert not freqtrade.handle_cancel_enter( + trade, limit_order[entry_side(is_short)], trade.open_orders_ids[0], reason + ) @pytest.mark.parametrize("is_short", [False, True]) @@ -3381,7 +3398,9 @@ def test_handle_cancel_enter_exchanges(mocker, caplog, default_conf_usdt, is_sho trade = mock_trade_usdt_4(fee, is_short) Trade.session.add(trade) Trade.commit() - assert freqtrade.handle_cancel_enter(trade, limit_buy_order_canceled_empty, reason) + assert freqtrade.handle_cancel_enter( + trade, limit_buy_order_canceled_empty, trade.open_orders_ids[0], reason + ) assert cancel_order_mock.call_count == 0 assert log_has_re( f'{trade.entry_side.capitalize()} order fully cancelled. ' @@ -3418,7 +3437,7 @@ def test_handle_cancel_enter_corder_empty(mocker, default_conf_usdt, limit_order l_order['filled'] = 0.0 l_order['status'] = 'open' reason = CANCEL_REASON['TIMEOUT'] - assert freqtrade.handle_cancel_enter(trade, l_order, reason) + assert freqtrade.handle_cancel_enter(trade, l_order, trade.open_orders_ids[0], reason) assert cancel_order_mock.call_count == 1 cancel_order_mock.reset_mock() @@ -3426,11 +3445,15 @@ def test_handle_cancel_enter_corder_empty(mocker, default_conf_usdt, limit_order order = deepcopy(l_order) order['status'] = 'canceled' mocker.patch(f'{EXMS}.fetch_order', return_value=order) - assert not freqtrade.handle_cancel_enter(trade, l_order, reason) + assert not freqtrade.handle_cancel_enter(trade, l_order, trade.open_orders_ids[0], reason) assert cancel_order_mock.call_count == 1 -def test_handle_cancel_exit_limit(mocker, default_conf_usdt, fee) -> None: +@pytest.mark.parametrize('is_short', [True, False]) +@pytest.mark.parametrize('leverage', [1, 5]) +@pytest.mark.parametrize('amount', [2, 50]) +def test_handle_cancel_exit_limit(mocker, default_conf_usdt, fee, is_short, + leverage, amount) -> None: send_msg_mock = patch_RPCManager(mocker) patch_exchange(mocker) cancel_order_mock = MagicMock() @@ -3438,7 +3461,9 @@ def test_handle_cancel_exit_limit(mocker, default_conf_usdt, fee) -> None: EXMS, cancel_order=cancel_order_mock, ) - mocker.patch(f'{EXMS}.get_rate', return_value=0.245441) + entry_price = 0.245441 + + mocker.patch(f'{EXMS}.get_rate', return_value=entry_price) mocker.patch(f'{EXMS}.get_min_pair_stake_amount', return_value=0.2) mocker.patch('freqtrade.freqtradebot.FreqtradeBot.handle_order_fee') @@ -3446,30 +3471,30 @@ def test_handle_cancel_exit_limit(mocker, default_conf_usdt, fee) -> None: freqtrade = FreqtradeBot(default_conf_usdt) trade = Trade( - pair='LTC/ETH', - amount=2, + pair='LTC/USDT', + amount=amount * leverage, exchange='binance', - open_rate=0.245441, - open_order_id="sell_123456", + open_rate=entry_price, open_date=dt_now() - timedelta(days=2), fee_open=fee.return_value, fee_close=fee.return_value, close_rate=0.555, close_date=dt_now(), exit_reason="sell_reason_whatever", - stake_amount=0.245441 * 2, - leverage=1, + stake_amount=entry_price * amount, + leverage=leverage, + is_short=is_short, ) trade.orders = [ Order( - ft_order_side='buy', + ft_order_side=entry_side(is_short), ft_pair=trade.pair, ft_is_open=False, order_id='buy_123456', status="closed", symbol=trade.pair, order_type="market", - side="buy", + side=entry_side(is_short), price=trade.open_rate, average=trade.open_rate, filled=trade.amount, @@ -3479,14 +3504,14 @@ def test_handle_cancel_exit_limit(mocker, default_conf_usdt, fee) -> None: order_filled_date=trade.open_date, ), Order( - ft_order_side='sell', + ft_order_side=exit_side(is_short), ft_pair=trade.pair, ft_is_open=True, order_id='sell_123456', status="open", symbol=trade.pair, order_type="limit", - side="sell", + side=exit_side(is_short), price=trade.open_rate, average=trade.open_rate, filled=0.0, @@ -3502,26 +3527,26 @@ def test_handle_cancel_exit_limit(mocker, default_conf_usdt, fee) -> None: 'status': "open"} reason = CANCEL_REASON['TIMEOUT'] send_msg_mock.reset_mock() - assert freqtrade.handle_cancel_exit(trade, order, reason) + assert freqtrade.handle_cancel_exit(trade, order, order['id'], reason) assert cancel_order_mock.call_count == 1 assert send_msg_mock.call_count == 1 assert trade.close_rate is None assert trade.exit_reason is None - assert trade.open_order_id is None + assert not trade.has_open_orders send_msg_mock.reset_mock() # Partial exit - below exit threshold - order['amount'] = 2 - order['filled'] = 1.9 - assert not freqtrade.handle_cancel_exit(trade, order, reason) + order['amount'] = amount * leverage + order['filled'] = amount * 0.99 * leverage + assert not freqtrade.handle_cancel_exit(trade, order, order['id'], reason) # Assert cancel_order was not called (callcount remains unchanged) assert cancel_order_mock.call_count == 1 assert send_msg_mock.call_count == 1 assert (send_msg_mock.call_args_list[0][0][0]['reason'] == CANCEL_REASON['PARTIALLY_FILLED_KEEP_OPEN']) - assert not freqtrade.handle_cancel_exit(trade, order, reason) + assert not freqtrade.handle_cancel_exit(trade, order, order['id'], reason) assert (send_msg_mock.call_args_list[0][0][0]['reason'] == CANCEL_REASON['PARTIALLY_FILLED_KEEP_OPEN']) @@ -3532,8 +3557,8 @@ def test_handle_cancel_exit_limit(mocker, default_conf_usdt, fee) -> None: send_msg_mock.reset_mock() - order['filled'] = 1 - assert freqtrade.handle_cancel_exit(trade, order, reason) + order['filled'] = amount * 0.5 * leverage + assert freqtrade.handle_cancel_exit(trade, order, order['id'], reason) assert send_msg_mock.call_count == 1 assert (send_msg_mock.call_args_list[0][0][0]['reason'] == CANCEL_REASON['PARTIALLY_FILLED']) @@ -3549,17 +3574,16 @@ def test_handle_cancel_exit_cancel_exception(mocker, default_conf_usdt) -> None: # TODO: should not be magicmock trade = MagicMock() - trade.open_order_id = '125' + order_id = '125' reason = CANCEL_REASON['TIMEOUT'] order = {'remaining': 1, 'id': '125', 'amount': 1, 'status': "open"} - assert not freqtrade.handle_cancel_exit(trade, order, reason) + assert not freqtrade.handle_cancel_exit(trade, order, order_id, reason) # mocker.patch(f'{EXMS}.cancel_order_with_result', return_value=order) # assert not freqtrade.handle_cancel_exit(trade, order, reason) - # assert trade.open_order_id == '125' @pytest.mark.parametrize("is_short, open_rate, amt", [ @@ -4006,7 +4030,7 @@ def test_may_execute_trade_exit_after_stoploss_on_exchange_hit( freqtrade.exit_positions(trades) assert trade assert trade.stoploss_order_id == '123' - assert trade.open_order_id is None + assert not trade.has_open_orders # Assuming stoploss on exchange is hit # stoploss_order_id should become None @@ -4301,7 +4325,6 @@ def test__safe_exit_amount(default_conf_usdt, fee, caplog, mocker, amount_wallet amount=amount, exchange='binance', open_rate=0.245441, - open_order_id="123456", fee_open=fee.return_value, fee_close=fee.return_value, ) @@ -4644,7 +4667,6 @@ def test_get_real_amount_quote(default_conf_usdt, trades_for_order, buy_order_fe open_rate=0.245441, fee_open=fee.return_value, fee_close=fee.return_value, - open_order_id="123456" ) freqtrade = get_patched_freqtradebot(mocker, default_conf_usdt) @@ -4672,7 +4694,6 @@ def test_get_real_amount_quote_dust(default_conf_usdt, trades_for_order, buy_ord open_rate=0.245441, fee_open=fee.return_value, fee_close=fee.return_value, - open_order_id="123456" ) freqtrade = get_patched_freqtradebot(mocker, default_conf_usdt) @@ -4696,7 +4717,6 @@ def test_get_real_amount_no_trade(default_conf_usdt, buy_order_fee, caplog, mock open_rate=0.245441, fee_open=fee.return_value, fee_close=fee.return_value, - open_order_id="123456" ) freqtrade = get_patched_freqtradebot(mocker, default_conf_usdt) @@ -4749,7 +4769,6 @@ def test_get_real_amount( fee_open=fee.return_value, fee_close=fee.return_value, open_rate=0.245441, - open_order_id="123456" ) freqtrade = get_patched_freqtradebot(mocker, default_conf_usdt) @@ -4796,8 +4815,7 @@ def test_get_real_amount_multi( exchange='binance', fee_open=fee.return_value, fee_close=fee.return_value, - open_rate=0.245441, - open_order_id="123456" + open_rate=0.245441 ) # Fake markets entry to enable fee parsing @@ -4842,7 +4860,6 @@ def test_get_real_amount_invalid_order(default_conf_usdt, trades_for_order, buy_ fee_open=fee.return_value, fee_close=fee.return_value, open_rate=0.245441, - open_order_id="123456" ) freqtrade = get_patched_freqtradebot(mocker, default_conf_usdt) @@ -4864,7 +4881,6 @@ def test_get_real_amount_fees_order(default_conf_usdt, market_buy_order_usdt_dou fee_open=fee.return_value, fee_close=fee.return_value, open_rate=0.245441, - open_order_id="123456" ) freqtrade = get_patched_freqtradebot(mocker, default_conf_usdt) @@ -4891,7 +4907,6 @@ def test_get_real_amount_wrong_amount(default_conf_usdt, trades_for_order, buy_o open_rate=0.245441, fee_open=fee.return_value, fee_close=fee.return_value, - open_order_id="123456" ) freqtrade = get_patched_freqtradebot(mocker, default_conf_usdt) @@ -4916,7 +4931,6 @@ def test_get_real_amount_wrong_amount_rounding(default_conf_usdt, trades_for_ord fee_open=fee.return_value, fee_close=fee.return_value, open_rate=0.245441, - open_order_id="123456" ) freqtrade = get_patched_freqtradebot(mocker, default_conf_usdt) @@ -4935,7 +4949,6 @@ def test_get_real_amount_open_trade_usdt(default_conf_usdt, fee, mocker): open_rate=0.245441, fee_open=fee.return_value, fee_close=fee.return_value, - open_order_id="123456" ) order = { 'id': 'mocked_order', @@ -4995,8 +5008,7 @@ def test_get_real_amount_in_point(default_conf_usdt, buy_order_fee, fee, mocker, exchange='binance', fee_open=fee.return_value, fee_close=fee.return_value, - open_rate=0.245441, - open_order_id="123456" + open_rate=0.245441 ) limit_buy_order_usdt['amount'] = amount freqtrade = get_patched_freqtradebot(mocker, default_conf_usdt) @@ -5038,7 +5050,6 @@ def test_apply_fee_conditional(default_conf_usdt, fee, mocker, caplog, open_rate=0.245441, fee_open=fee.return_value, fee_close=fee.return_value, - open_order_id="123456" ) order = Order( ft_order_side='buy', @@ -5076,8 +5087,7 @@ def test_apply_fee_conditional_multibuy(default_conf_usdt, fee, mocker, caplog, exchange='binance', open_rate=0.245441, fee_open=fee.return_value, - fee_close=fee.return_value, - open_order_id="123456" + fee_close=fee.return_value ) # One closed order order = Order( @@ -5551,7 +5561,7 @@ def test_handle_insufficient_funds(mocker, default_conf_usdt, fee, is_short, cap return_value={'status': 'open'}) def reset_open_orders(trade): - trade.open_order_id = None + trade.stoploss_order_id = None trade.is_short = is_short @@ -5563,7 +5573,7 @@ def test_handle_insufficient_funds(mocker, default_conf_usdt, fee, is_short, cap # No open order trade = trades[1] reset_open_orders(trade) - assert trade.open_order_id is None + assert not trade.has_open_orders assert trade.stoploss_order_id is None freqtrade.handle_insufficient_funds(trade) @@ -5573,7 +5583,7 @@ def test_handle_insufficient_funds(mocker, default_conf_usdt, fee, is_short, cap assert mock_fo.call_count == 0 assert mock_uts.call_count == 0 # No change to orderid - as update_trade_state is mocked - assert trade.open_order_id is None + assert not trade.has_open_orders assert trade.stoploss_order_id is None caplog.clear() @@ -5582,7 +5592,9 @@ def test_handle_insufficient_funds(mocker, default_conf_usdt, fee, is_short, cap # Open buy order trade = trades[3] reset_open_orders(trade) - assert trade.open_order_id is None + + # This part in not relevant anymore + # assert not trade.has_open_orders assert trade.stoploss_order_id is None freqtrade.handle_insufficient_funds(trade) @@ -5591,7 +5603,7 @@ def test_handle_insufficient_funds(mocker, default_conf_usdt, fee, is_short, cap assert mock_fo.call_count == 1 assert mock_uts.call_count == 1 # Found open buy order - assert trade.open_order_id is not None + assert trade.has_open_orders assert trade.stoploss_order_id is None caplog.clear() @@ -5600,7 +5612,7 @@ def test_handle_insufficient_funds(mocker, default_conf_usdt, fee, is_short, cap # Open stoploss order trade = trades[4] reset_open_orders(trade) - assert trade.open_order_id is None + assert not trade.has_open_orders assert trade.stoploss_order_id is None freqtrade.handle_insufficient_funds(trade) @@ -5609,7 +5621,7 @@ def test_handle_insufficient_funds(mocker, default_conf_usdt, fee, is_short, cap assert mock_fo.call_count == 1 assert mock_uts.call_count == 2 # stoploss_order_id is "refound" and added to the trade - assert trade.open_order_id is None + assert not trade.has_open_orders assert trade.stoploss_order_id is not None caplog.clear() @@ -5619,7 +5631,8 @@ def test_handle_insufficient_funds(mocker, default_conf_usdt, fee, is_short, cap # Open sell order trade = trades[5] reset_open_orders(trade) - assert trade.open_order_id is None + # This part in not relevant anymore + # assert not trade.has_open_orders assert trade.stoploss_order_id is None freqtrade.handle_insufficient_funds(trade) @@ -5628,7 +5641,7 @@ def test_handle_insufficient_funds(mocker, default_conf_usdt, fee, is_short, cap assert mock_fo.call_count == 1 assert mock_uts.call_count == 1 # sell-orderid is "refound" and added to the trade - assert trade.open_order_id == order['id'] + assert trade.open_orders_ids[0] == order['id'] assert trade.stoploss_order_id is None caplog.clear() @@ -5655,10 +5668,7 @@ def test_handle_onexchange_order(mocker, default_conf_usdt, limit_order, is_shor exit_order, ]) - order_id = entry_order['id'] - trade = Trade( - open_order_id=order_id, pair='ETH/USDT', fee_open=0.001, fee_close=0.001, @@ -5982,7 +5992,7 @@ def test_position_adjust(mocker, default_conf_usdt, fee) -> None: trade = Trade.session.scalars(select(Trade)).first() assert trade assert trade.is_open is True - assert trade.open_order_id is None + assert not trade.has_open_orders assert trade.open_rate == 11 assert trade.stake_amount == 110 @@ -5992,7 +6002,7 @@ def test_position_adjust(mocker, default_conf_usdt, fee) -> None: trade = Trade.session.scalars(select(Trade)).first() assert trade assert trade.is_open is True - assert trade.open_order_id is None + assert not trade.has_open_orders assert trade.open_rate == 11 assert trade.stake_amount == 110 assert not trade.fee_updated('buy') @@ -6002,7 +6012,7 @@ def test_position_adjust(mocker, default_conf_usdt, fee) -> None: trade = Trade.session.scalars(select(Trade)).first() assert trade assert trade.is_open is True - assert trade.open_order_id is None + assert not trade.has_open_orders assert trade.open_rate == 11 assert trade.stake_amount == 110 assert not trade.fee_updated('buy') @@ -6030,7 +6040,7 @@ def test_position_adjust(mocker, default_conf_usdt, fee) -> None: assert len(orders) == 2 trade = Trade.session.scalars(select(Trade)).first() assert trade - assert trade.open_order_id == '651' + assert '651' in trade.open_orders_ids assert trade.open_rate == 11 assert trade.amount == 10 assert trade.stake_amount == 110 @@ -6067,7 +6077,7 @@ def test_position_adjust(mocker, default_conf_usdt, fee) -> None: # Assert trade is as expected trade = Trade.session.scalars(select(Trade)).first() assert trade - assert trade.open_order_id == '651' + assert '651' in trade.open_orders_ids assert trade.open_rate == 11 assert trade.amount == 10 assert trade.stake_amount == 110 @@ -6104,7 +6114,7 @@ def test_position_adjust(mocker, default_conf_usdt, fee) -> None: # Assert trade is as expected (averaged dca) trade = Trade.session.scalars(select(Trade)).first() assert trade - assert trade.open_order_id is None + assert not trade.has_open_orders assert pytest.approx(trade.open_rate) == 9.90909090909 assert trade.amount == 22 assert pytest.approx(trade.stake_amount) == 218 @@ -6146,7 +6156,7 @@ def test_position_adjust(mocker, default_conf_usdt, fee) -> None: # Assert trade is as expected (averaged dca) trade = Trade.session.scalars(select(Trade)).first() assert trade - assert trade.open_order_id is None + assert not trade.has_open_orders assert pytest.approx(trade.open_rate) == 8.729729729729 assert trade.amount == 37 assert trade.stake_amount == 323 @@ -6184,7 +6194,7 @@ def test_position_adjust(mocker, default_conf_usdt, fee) -> None: # Assert trade is as expected (averaged dca) trade = Trade.session.scalars(select(Trade)).first() assert trade - assert trade.open_order_id is None + assert not trade.has_open_orders assert trade.is_open assert trade.amount == 22 assert trade.stake_amount == 192.05405405405406 @@ -6261,7 +6271,7 @@ def test_position_adjust2(mocker, default_conf_usdt, fee) -> None: trade = Trade.session.scalars(select(Trade)).first() assert trade assert trade.is_open is True - assert trade.open_order_id is None + assert not trade.has_open_orders assert trade.open_rate == bid assert trade.stake_amount == bid * amount @@ -6271,7 +6281,7 @@ def test_position_adjust2(mocker, default_conf_usdt, fee) -> None: trade = Trade.session.scalars(select(Trade)).first() assert trade assert trade.is_open is True - assert trade.open_order_id is None + assert not trade.has_open_orders assert trade.open_rate == bid assert trade.stake_amount == bid * amount assert not trade.fee_updated(trade.entry_side) @@ -6281,7 +6291,7 @@ def test_position_adjust2(mocker, default_conf_usdt, fee) -> None: trade = Trade.session.scalars(select(Trade)).first() assert trade assert trade.is_open is True - assert trade.open_order_id is None + assert not trade.has_open_orders assert trade.open_rate == bid assert trade.stake_amount == bid * amount assert not trade.fee_updated(trade.entry_side) @@ -6316,7 +6326,7 @@ def test_position_adjust2(mocker, default_conf_usdt, fee) -> None: trade = Trade.session.scalars(select(Trade)).first() assert trade - assert trade.open_order_id is None + assert not trade.has_open_orders assert trade.amount == 50 assert trade.open_rate == 11 assert trade.stake_amount == 550 @@ -6358,7 +6368,7 @@ def test_position_adjust2(mocker, default_conf_usdt, fee) -> None: trade = Trade.session.scalars(select(Trade)).first() assert trade - assert trade.open_order_id is None + assert not trade.has_open_orders assert trade.amount == 50 assert trade.open_rate == 11 assert trade.stake_amount == 550 @@ -6458,7 +6468,7 @@ def test_position_adjust3(mocker, default_conf_usdt, fee, data) -> None: assert trade if idx < len(data) - 1: assert trade.is_open is True - assert trade.open_order_id is None + assert not trade.has_open_orders assert trade.amount == result[0] assert trade.open_rate == result[1] assert trade.stake_amount == result[2] @@ -6471,7 +6481,7 @@ def test_position_adjust3(mocker, default_conf_usdt, fee, data) -> None: trade = Trade.session.scalars(select(Trade)).first() assert trade - assert trade.open_order_id is None + assert not trade.has_open_orders assert trade.is_open is False diff --git a/tests/test_integration.py b/tests/test_integration.py index 520cb4244..ee1d4bbb3 100644 --- a/tests/test_integration.py +++ b/tests/test_integration.py @@ -103,7 +103,6 @@ def test_may_execute_exit_stoploss_on_exchange_multi(default_conf, ticker, fee, trade.orders.append(oobj) trade.stoploss_order_id = f"stop{idx}" - trade.open_order_id = None n = freqtrade.exit_positions(trades) assert n == 2 @@ -194,8 +193,7 @@ def test_forcebuy_last_unlimited(default_conf, ticker, fee, mocker, balance_rati for trade in trades: assert pytest.approx(trade.stake_amount) == result1 - # Reset trade open order id's - trade.open_order_id = None + trades = Trade.get_open_trades() assert len(trades) == 5 bals = freqtrade.wallets.get_all_balances() @@ -386,7 +384,7 @@ def test_dca_order_adjust(default_conf_usdt, ticker_usdt, leverage, fee, mocker) assert len(Trade.get_trades().all()) == 1 trade: Trade = Trade.get_trades().first() assert len(trade.orders) == 1 - assert trade.open_order_id is not None + assert trade.has_open_orders assert pytest.approx(trade.stake_amount) == 60 assert trade.open_rate == 1.96 assert trade.stop_loss_pct == -0.1 @@ -399,7 +397,7 @@ def test_dca_order_adjust(default_conf_usdt, ticker_usdt, leverage, fee, mocker) freqtrade.process() trade = Trade.get_trades().first() assert len(trade.orders) == 1 - assert trade.open_order_id is not None + assert trade.has_open_orders assert pytest.approx(trade.stake_amount) == 60 # Cancel order and place new one @@ -407,7 +405,7 @@ def test_dca_order_adjust(default_conf_usdt, ticker_usdt, leverage, fee, mocker) freqtrade.process() trade = Trade.get_trades().first() assert len(trade.orders) == 2 - assert trade.open_order_id is not None + assert trade.has_open_orders # Open rate is not adjusted yet assert trade.open_rate == 1.96 assert trade.stop_loss_pct == -0.1 @@ -421,7 +419,7 @@ def test_dca_order_adjust(default_conf_usdt, ticker_usdt, leverage, fee, mocker) freqtrade.process() trade = Trade.get_trades().first() assert len(trade.orders) == 2 - assert trade.open_order_id is None + assert not trade.has_open_orders # Open rate is not adjusted yet assert trade.open_rate == 1.99 assert pytest.approx(trade.stake_amount) == 60 @@ -438,7 +436,7 @@ def test_dca_order_adjust(default_conf_usdt, ticker_usdt, leverage, fee, mocker) freqtrade.process() trade = Trade.get_trades().first() assert len(trade.orders) == 3 - assert trade.open_order_id is not None + assert trade.has_open_orders assert trade.open_rate == 1.99 assert trade.orders[-1].price == 1.96 assert trade.orders[-1].cost == 120 * leverage @@ -449,7 +447,7 @@ def test_dca_order_adjust(default_conf_usdt, ticker_usdt, leverage, fee, mocker) freqtrade.process() trade = Trade.get_trades().first() assert len(trade.orders) == 4 - assert trade.open_order_id is not None + assert trade.has_open_orders assert trade.open_rate == 1.99 assert pytest.approx(trade.stake_amount) == 60 assert trade.orders[-1].price == 1.95 @@ -463,7 +461,7 @@ def test_dca_order_adjust(default_conf_usdt, ticker_usdt, leverage, fee, mocker) freqtrade.process() trade = Trade.get_trades().first() assert len(trade.orders) == 4 - assert trade.open_order_id is None + assert not trade.has_open_orders assert pytest.approx(trade.open_rate) == 1.963153456 assert trade.orders[-1].price == 1.95 assert pytest.approx(trade.orders[-1].cost) == 120 * leverage @@ -522,7 +520,11 @@ def test_dca_order_adjust_entry_replace_fails( freqtrade.enter_positions() trades = Trade.session.scalars( - select(Trade).filter(Trade.open_order_id.is_not(None))).all() + select(Trade) + .where(Order.ft_is_open.is_(True)) + .where(Order.ft_order_side != "stoploss") + .where(Order.ft_trade_id == Trade.id) + ).all() assert len(trades) == 1 mocker.patch(f'{EXMS}._dry_is_price_crossed', return_value=False) @@ -539,14 +541,22 @@ def test_dca_order_adjust_entry_replace_fails( assert freqtrade.strategy.adjust_trade_position.call_count == 1 trades = Trade.session.scalars( - select(Trade).filter(Trade.open_order_id.is_not(None))).all() + select(Trade) + .where(Order.ft_is_open.is_(True)) + .where(Order.ft_order_side != "stoploss") + .where(Order.ft_trade_id == Trade.id) + ).all() assert len(trades) == 2 # We now have 2 orders open freqtrade.strategy.adjust_entry_price = MagicMock(return_value=2.05) freqtrade.manage_open_orders() trades = Trade.session.scalars( - select(Trade).filter(Trade.open_order_id.is_not(None))).all() + select(Trade) + .where(Order.ft_is_open.is_(True)) + .where(Order.ft_order_side != "stoploss") + .where(Order.ft_trade_id == Trade.id) + ).all() assert len(trades) == 2 assert len(Order.get_open_orders()) == 2 # Entry adjustment is called diff --git a/tests/utils/test_datetime_helpers.py b/tests/utils/test_datetime_helpers.py index 6ce975732..b70065645 100644 --- a/tests/utils/test_datetime_helpers.py +++ b/tests/utils/test_datetime_helpers.py @@ -3,8 +3,8 @@ from datetime import datetime, timedelta, timezone import pytest import time_machine -from freqtrade.util import (dt_floor_day, dt_from_ts, dt_humanize, dt_now, dt_ts, dt_utc, - format_ms_time, shorten_date) +from freqtrade.util import (dt_floor_day, dt_from_ts, dt_humanize, dt_now, dt_ts, dt_ts_def, dt_utc, + format_date, format_ms_time, shorten_date) def test_dt_now(): @@ -22,6 +22,13 @@ def test_dt_now(): assert dt_ts(now) == int(now.timestamp() * 1000) +def test_dt_ts_def(): + assert dt_ts_def(None) == 0 + assert dt_ts_def(None, 123) == 123 + assert dt_ts_def(datetime(2023, 5, 5, tzinfo=timezone.utc)) == 1683244800000 + assert dt_ts_def(datetime(2023, 5, 5, tzinfo=timezone.utc), 123) == 1683244800000 + + def test_dt_utc(): assert dt_utc(2023, 5, 5) == datetime(2023, 5, 5, tzinfo=timezone.utc) assert dt_utc(2023, 5, 5, 0, 0, 0, 555500) == datetime(2023, 5, 5, 0, 0, 0, 555500, @@ -70,3 +77,14 @@ def test_format_ms_time() -> None: # Date 2017-12-13 08:02:01 date_in_epoch_ms = 1513152121000 assert format_ms_time(date_in_epoch_ms) == res.astimezone(None).strftime('%Y-%m-%dT%H:%M:%S') + + +def test_format_date() -> None: + + date = datetime(2023, 9, 1, 5, 2, 3, 455555, tzinfo=timezone.utc) + assert format_date(date) == '2023-09-01 05:02:03' + assert format_date(None) == '' + + date = datetime(2021, 9, 30, 22, 59, 3, 455555, tzinfo=timezone.utc) + assert format_date(date) == '2021-09-30 22:59:03' + assert format_date(None) == ''