diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index a85c84eb8..4be298d7b 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -18,7 +18,7 @@ repos: - types-requests==2.30.0.0 - types-tabulate==0.9.0.2 - types-python-dateutil==2.8.19.13 - - SQLAlchemy==2.0.13 + - SQLAlchemy==2.0.15 # stages: [push] - repo: https://github.com/pycqa/isort diff --git a/docs/developer.md b/docs/developer.md index 1bc75551f..2782f0117 100644 --- a/docs/developer.md +++ b/docs/developer.md @@ -327,18 +327,18 @@ To check how the new exchange behaves, you can use the following snippet: ``` python import ccxt -from datetime import datetime +from datetime import datetime, timezone from freqtrade.data.converter import ohlcv_to_dataframe -ct = ccxt.binance() +ct = ccxt.binance() # Use the exchange you're testing timeframe = "1d" -pair = "XLM/BTC" # Make sure to use a pair that exists on that exchange! +pair = "BTC/USDT" # Make sure to use a pair that exists on that exchange! raw = ct.fetch_ohlcv(pair, timeframe=timeframe) # convert to dataframe df1 = ohlcv_to_dataframe(raw, timeframe, pair=pair, drop_incomplete=False) print(df1.tail(1)) -print(datetime.utcnow()) +print(datetime.now(timezone.utc)) ``` ``` output diff --git a/docs/requirements-docs.txt b/docs/requirements-docs.txt index f7c0aebe9..c5e478c78 100644 --- a/docs/requirements-docs.txt +++ b/docs/requirements-docs.txt @@ -1,6 +1,6 @@ markdown==3.3.7 mkdocs==1.4.3 -mkdocs-material==9.1.12 +mkdocs-material==9.1.14 mdx_truly_sane_lists==1.3 pymdown-extensions==10.0.1 jinja2==3.1.2 diff --git a/freqtrade/configuration/timerange.py b/freqtrade/configuration/timerange.py index 0c2f0d1b8..cff35db7e 100644 --- a/freqtrade/configuration/timerange.py +++ b/freqtrade/configuration/timerange.py @@ -6,8 +6,6 @@ import re from datetime import datetime, timezone from typing import Optional -import arrow - from freqtrade.constants import DATETIME_PRINT_FORMAT from freqtrade.exceptions import OperationalException @@ -139,7 +137,8 @@ class TimeRange: if stype[0]: starts = rvals[index] if stype[0] == 'date' and len(starts) == 8: - start = arrow.get(starts, 'YYYYMMDD').int_timestamp + start = int(datetime.strptime(starts, '%Y%m%d').replace( + tzinfo=timezone.utc).timestamp()) elif len(starts) == 13: start = int(starts) // 1000 else: @@ -148,7 +147,8 @@ class TimeRange: if stype[1]: stops = rvals[index] if stype[1] == 'date' and len(stops) == 8: - stop = arrow.get(stops, 'YYYYMMDD').int_timestamp + stop = int(datetime.strptime(stops, '%Y%m%d').replace( + tzinfo=timezone.utc).timestamp()) elif len(stops) == 13: stop = int(stops) // 1000 else: diff --git a/freqtrade/data/history/history_utils.py b/freqtrade/data/history/history_utils.py index b567b58bf..dc3c7c1e6 100644 --- a/freqtrade/data/history/history_utils.py +++ b/freqtrade/data/history/history_utils.py @@ -1,10 +1,9 @@ import logging import operator -from datetime import datetime +from datetime import datetime, timedelta from pathlib import Path from typing import Dict, List, Optional, Tuple -import arrow from pandas import DataFrame, concat from freqtrade.configuration import TimeRange @@ -236,8 +235,8 @@ def _download_pair_history(pair: str, *, new_data = exchange.get_historic_ohlcv(pair=pair, timeframe=timeframe, since_ms=since_ms if since_ms else - arrow.utcnow().shift( - days=-new_pairs_days).int_timestamp * 1000, + int((datetime.now() - timedelta(days=new_pairs_days) + ).timestamp()) * 1000, is_new_pair=data.empty, candle_type=candle_type, until_ms=until_ms if until_ms else None @@ -349,7 +348,7 @@ def _download_trades_history(exchange: Exchange, trades = [] if not since: - since = arrow.utcnow().shift(days=-new_pairs_days).int_timestamp * 1000 + since = int((datetime.now() - timedelta(days=-new_pairs_days)).timestamp()) * 1000 from_id = trades[-1][1] if trades else None if trades and since < trades[-1][0]: diff --git a/freqtrade/edge/edge_positioning.py b/freqtrade/edge/edge_positioning.py index 73820ecbe..f2df0d3f2 100644 --- a/freqtrade/edge/edge_positioning.py +++ b/freqtrade/edge/edge_positioning.py @@ -3,9 +3,9 @@ import logging from collections import defaultdict from copy import deepcopy +from datetime import timedelta from typing import Any, Dict, List, NamedTuple -import arrow import numpy as np import utils_find_1st as utf1st from pandas import DataFrame @@ -18,6 +18,7 @@ from freqtrade.exceptions import OperationalException from freqtrade.exchange import timeframe_to_seconds from freqtrade.plugins.pairlist.pairlist_helpers import expand_pairlist from freqtrade.strategy.interface import IStrategy +from freqtrade.util import dt_now logger = logging.getLogger(__name__) @@ -79,8 +80,8 @@ class Edge: self._stoploss_range_step ) - self._timerange: TimeRange = TimeRange.parse_timerange("%s-" % arrow.now().shift( - days=-1 * self._since_number_of_days).format('YYYYMMDD')) + self._timerange: TimeRange = TimeRange.parse_timerange( + f"{(dt_now() - timedelta(days=self._since_number_of_days)).strftime('%Y%m%d')}-") if config.get('fee'): self.fee = config['fee'] else: @@ -97,7 +98,7 @@ class Edge: heartbeat = self.edge_config.get('process_throttle_secs') if (self._last_updated > 0) and ( - self._last_updated + heartbeat > arrow.utcnow().int_timestamp): + self._last_updated + heartbeat > int(dt_now().timestamp())): return False data: Dict[str, Any] = {} @@ -189,7 +190,7 @@ class Edge: # Fill missing, calculable columns, profit, duration , abs etc. trades_df = self._fill_calculable_fields(DataFrame(trades)) self._cached_pairs = self._process_expectancy(trades_df) - self._last_updated = arrow.utcnow().int_timestamp + self._last_updated = int(dt_now().timestamp()) return True diff --git a/freqtrade/exchange/binance.py b/freqtrade/exchange/binance.py index 7ac496f62..8075d775a 100644 --- a/freqtrade/exchange/binance.py +++ b/freqtrade/exchange/binance.py @@ -1,10 +1,9 @@ """ Binance exchange subclass """ import logging -from datetime import datetime +from datetime import datetime, timezone from pathlib import Path from typing import Dict, List, Optional, Tuple -import arrow import ccxt from freqtrade.enums import CandleType, MarginMode, PriceType, TradingMode @@ -66,7 +65,7 @@ class Binance(Exchange): """ try: if self.trading_mode == TradingMode.FUTURES and not self._config['dry_run']: - position_side = self._api.fapiPrivateGetPositionsideDual() + position_side = self._api.fapiPrivateGetPositionSideDual() self._log_exchange_response('position_side_setting', position_side) assets_margin = self._api.fapiPrivateGetMultiAssetsMargin() self._log_exchange_response('multi_asset_margin', assets_margin) @@ -105,8 +104,9 @@ class Binance(Exchange): if x and x[3] and x[3][0] and x[3][0][0] > since_ms: # Set starting date to first available candle. since_ms = x[3][0][0] - logger.info(f"Candle-data for {pair} available starting with " - f"{arrow.get(since_ms // 1000).isoformat()}.") + logger.info( + f"Candle-data for {pair} available starting with " + f"{datetime.fromtimestamp(since_ms // 1000, tz=timezone.utc).isoformat()}.") return await super()._async_get_historic_ohlcv( pair=pair, diff --git a/freqtrade/exchange/exchange.py b/freqtrade/exchange/exchange.py index 7c68eaa99..3b1466c69 100644 --- a/freqtrade/exchange/exchange.py +++ b/freqtrade/exchange/exchange.py @@ -11,7 +11,6 @@ from math import floor from threading import Lock from typing import Any, Coroutine, Dict, List, Literal, Optional, Tuple, Union -import arrow import ccxt import ccxt.async_support as ccxt_async from cachetools import TTLCache @@ -42,6 +41,8 @@ from freqtrade.exchange.types import OHLCVResponse, OrderBook, Ticker, Tickers from freqtrade.misc import (chunks, deep_merge_dicts, file_dump_json, file_load_json, safe_value_fallback2) from freqtrade.plugins.pairlist.pairlist_helpers import expand_pairlist +from freqtrade.util import dt_from_ts, dt_now +from freqtrade.util.datetime_helpers import dt_humanize, dt_ts logger = logging.getLogger(__name__) @@ -490,7 +491,7 @@ class Exchange: try: self._markets = self._api.load_markets(params={}) self._load_async_markets() - self._last_markets_refresh = arrow.utcnow().int_timestamp + self._last_markets_refresh = dt_ts() if self._ft_has['needs_trading_fees']: self._trading_fees = self.fetch_trading_fees() @@ -501,15 +502,14 @@ class Exchange: """Reload markets both sync and async if refresh interval has passed """ # Check whether markets have to be reloaded if (self._last_markets_refresh > 0) and ( - self._last_markets_refresh + self.markets_refresh_interval - > arrow.utcnow().int_timestamp): + self._last_markets_refresh + self.markets_refresh_interval > dt_ts()): return None logger.debug("Performing scheduled market reload..") try: self._markets = self._api.load_markets(reload=True, params={}) # Also reload async markets to avoid issues with newly listed pairs self._load_async_markets(reload=True) - self._last_markets_refresh = arrow.utcnow().int_timestamp + self._last_markets_refresh = dt_ts() self.fill_leverage_tiers() except ccxt.BaseError: logger.exception("Could not reload markets.") @@ -843,7 +843,8 @@ class Exchange: def create_dry_run_order(self, pair: str, ordertype: str, side: str, amount: float, rate: float, leverage: float, params: Dict = {}, stop_loss: bool = False) -> Dict[str, Any]: - order_id = f'dry_run_{side}_{datetime.now().timestamp()}' + now = dt_now() + order_id = f'dry_run_{side}_{now.timestamp()}' # Rounding here must respect to contract sizes _amount = self._contracts_to_amount( pair, self.amount_to_precision(pair, self._amount_to_contracts(pair, amount))) @@ -858,8 +859,8 @@ class Exchange: 'side': side, 'filled': 0, 'remaining': _amount, - 'datetime': arrow.utcnow().strftime('%Y-%m-%dT%H:%M:%S.%fZ'), - 'timestamp': arrow.utcnow().int_timestamp * 1000, + 'datetime': now.strftime('%Y-%m-%dT%H:%M:%S.%fZ'), + 'timestamp': dt_ts(now), 'status': "open", 'fee': None, 'info': {}, @@ -1930,11 +1931,11 @@ class Exchange: logger.debug( "one_call: %s msecs (%s)", one_call, - arrow.utcnow().shift(seconds=one_call // 1000).humanize(only_distance=True) + dt_humanize(dt_now() - timedelta(milliseconds=one_call), only_distance=True) ) input_coroutines = [self._async_get_candle_history( pair, timeframe, candle_type, since) for since in - range(since_ms, until_ms or (arrow.utcnow().int_timestamp * 1000), one_call)] + range(since_ms, until_ms or dt_ts(), one_call)] data: List = [] # Chunk requests into batches of 100 to avoid overwelming ccxt Throttling @@ -2117,7 +2118,7 @@ class Exchange: """ try: # Fetch OHLCV asynchronously - s = '(' + arrow.get(since_ms // 1000).isoformat() + ') ' if since_ms is not None else '' + s = '(' + dt_from_ts(since_ms).isoformat() + ') ' if since_ms is not None else '' logger.debug( "Fetching pair %s, %s, interval %s, since %s %s...", pair, candle_type, timeframe, since_ms, s @@ -2207,7 +2208,7 @@ class Exchange: logger.debug( "Fetching trades for pair %s, since %s %s...", pair, since, - '(' + arrow.get(since // 1000).isoformat() + ') ' if since is not None else '' + '(' + dt_from_ts(since).isoformat() + ') ' if since is not None else '' ) trades = await self._api_async.fetch_trades(pair, since=since, limit=1000) trades = self._trades_contracts_to_amount(trades) diff --git a/freqtrade/exchange/exchange_utils.py b/freqtrade/exchange/exchange_utils.py index 83d2a214d..c6c2d5a24 100644 --- a/freqtrade/exchange/exchange_utils.py +++ b/freqtrade/exchange/exchange_utils.py @@ -11,6 +11,7 @@ from ccxt import (DECIMAL_PLACES, ROUND, ROUND_DOWN, ROUND_UP, SIGNIFICANT_DIGIT from freqtrade.exchange.common import BAD_EXCHANGES, EXCHANGE_HAS_OPTIONAL, EXCHANGE_HAS_REQUIRED from freqtrade.util import FtPrecise +from freqtrade.util.datetime_helpers import dt_from_ts, dt_ts CcxtModuleType = Any @@ -99,9 +100,8 @@ def timeframe_to_prev_date(timeframe: str, date: Optional[datetime] = None) -> d if not date: date = datetime.now(timezone.utc) - new_timestamp = ccxt.Exchange.round_timeframe(timeframe, date.timestamp() * 1000, - ROUND_DOWN) // 1000 - return datetime.fromtimestamp(new_timestamp, tz=timezone.utc) + new_timestamp = ccxt.Exchange.round_timeframe(timeframe, dt_ts(date), ROUND_DOWN) // 1000 + return dt_from_ts(new_timestamp) def timeframe_to_next_date(timeframe: str, date: Optional[datetime] = None) -> datetime: @@ -113,9 +113,8 @@ def timeframe_to_next_date(timeframe: str, date: Optional[datetime] = None) -> d """ if not date: date = datetime.now(timezone.utc) - new_timestamp = ccxt.Exchange.round_timeframe(timeframe, date.timestamp() * 1000, - ROUND_UP) // 1000 - return datetime.fromtimestamp(new_timestamp, tz=timezone.utc) + new_timestamp = ccxt.Exchange.round_timeframe(timeframe, dt_ts(date), ROUND_UP) // 1000 + return dt_from_ts(new_timestamp) def date_minus_candles( diff --git a/freqtrade/exchange/okx.py b/freqtrade/exchange/okx.py index 84b7deb7a..af889897c 100644 --- a/freqtrade/exchange/okx.py +++ b/freqtrade/exchange/okx.py @@ -169,6 +169,22 @@ class Okx(Exchange): params['posSide'] = self._get_posSide(side, True) return params + def _convert_stop_order(self, pair: str, order_id: str, order: Dict) -> Dict: + if ( + order['status'] == 'closed' + and (real_order_id := order.get('info', {}).get('ordId')) is not None + ): + # Once a order triggered, we fetch the regular followup order. + order_reg = self.fetch_order(real_order_id, pair) + self._log_exchange_response('fetch_stoploss_order1', order_reg) + order_reg['id_stop'] = order_reg['id'] + order_reg['id'] = order_id + order_reg['type'] = 'stoploss' + order_reg['status_stop'] = 'triggered' + return order_reg + order['type'] = 'stoploss' + return order + def fetch_stoploss_order(self, order_id: str, pair: str, params: Dict = {}) -> Dict: if self._config['dry_run']: return self.fetch_dry_run_order(order_id) @@ -177,7 +193,7 @@ class Okx(Exchange): params1 = {'stop': True} order_reg = self._api.fetch_order(order_id, pair, params=params1) self._log_exchange_response('fetch_stoploss_order', order_reg) - return order_reg + return self._convert_stop_order(pair, order_id, order_reg) except ccxt.OrderNotFound: pass params2 = {'stop': True, 'ordType': 'conditional'} @@ -188,18 +204,7 @@ class Okx(Exchange): orders_f = [order for order in orders if order['id'] == order_id] if orders_f: order = orders_f[0] - if (order['status'] == 'closed' - and (real_order_id := order.get('info', {}).get('ordId')) is not None): - # Once a order triggered, we fetch the regular followup order. - order_reg = self.fetch_order(real_order_id, pair) - self._log_exchange_response('fetch_stoploss_order1', order_reg) - order_reg['id_stop'] = order_reg['id'] - order_reg['id'] = order_id - order_reg['type'] = 'stoploss' - order_reg['status_stop'] = 'triggered' - return order_reg - order['type'] = 'stoploss' - return order + return self._convert_stop_order(pair, order_id, order) except ccxt.BaseError: pass raise RetryableOrderError( diff --git a/freqtrade/freqai/RL/BaseEnvironment.py b/freqtrade/freqai/RL/BaseEnvironment.py index 7c83a7e42..42e644f0a 100644 --- a/freqtrade/freqai/RL/BaseEnvironment.py +++ b/freqtrade/freqai/RL/BaseEnvironment.py @@ -180,7 +180,7 @@ class BaseEnvironment(gym.Env): def reset_tensorboard_log(self): self.tensorboard_metrics = {} - def reset(self): + def reset(self, seed=None): """ Reset is called at the beginning of every episode """ diff --git a/freqtrade/freqai/tensorboard/base_tensorboard.py b/freqtrade/freqai/tensorboard/base_tensorboard.py index c2d47137e..72f47111c 100644 --- a/freqtrade/freqai/tensorboard/base_tensorboard.py +++ b/freqtrade/freqai/tensorboard/base_tensorboard.py @@ -10,8 +10,7 @@ logger = logging.getLogger(__name__) class BaseTensorboardLogger: def __init__(self, logdir: Path, activate: bool = True): - logger.warning("Tensorboard is not installed, no logs will be written." - "Ensure torch is installed, or use the torch/RL docker images") + pass def log_scalar(self, tag: str, scalar_value: Any, step: int): return @@ -23,8 +22,7 @@ class BaseTensorboardLogger: class BaseTensorBoardCallback(TrainingCallback): def __init__(self, logdir: Path, activate: bool = True): - logger.warning("Tensorboard is not installed, no logs will be written." - "Ensure torch is installed, or use the torch/RL docker images") + pass def after_iteration( self, model, epoch: int, evals_log: TrainingCallback.EvalsLog diff --git a/freqtrade/freqai/torch/PyTorchTransformerModel.py b/freqtrade/freqai/torch/PyTorchTransformerModel.py index 702a7a08b..162459776 100644 --- a/freqtrade/freqai/torch/PyTorchTransformerModel.py +++ b/freqtrade/freqai/torch/PyTorchTransformerModel.py @@ -1,7 +1,7 @@ import math import torch -import torch.nn as nn +from torch import nn """ @@ -68,7 +68,7 @@ class PyTorchTransformerModel(nn.Module): return x -class PositionalEncoding(torch.nn.Module): +class PositionalEncoding(nn.Module): def __init__(self, d_model, max_len=5000): """ Args diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index 21426623f..fc4c65caf 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -1075,7 +1075,7 @@ class FreqtradeBot(LoggingMixin): trades_closed = 0 for trade in trades: - if not self.wallets.check_exit_amount(trade): + if trade.open_order_id is None and not self.wallets.check_exit_amount(trade): logger.warning( f'Not enough {trade.safe_base_currency} in wallet to exit {trade}. ' 'Trying to recover.') diff --git a/freqtrade/misc.py b/freqtrade/misc.py index 0cd5c6ffd..1e84bba87 100644 --- a/freqtrade/misc.py +++ b/freqtrade/misc.py @@ -3,13 +3,11 @@ Various tool function for Freqtrade and scripts """ import gzip import logging -import re from datetime import datetime from pathlib import Path from typing import Any, Dict, Iterator, List, Mapping, Optional, TextIO, Union from urllib.parse import urlparse -import orjson import pandas as pd import rapidjson @@ -48,18 +46,6 @@ def round_coin_value( return val -def shorten_date(_date: str) -> str: - """ - Trim the date so it fits on small screens - """ - new_date = re.sub('seconds?', 'sec', _date) - new_date = re.sub('minutes?', 'min', new_date) - new_date = re.sub('hours?', 'h', new_date) - new_date = re.sub('days?', 'd', new_date) - new_date = re.sub('^an?', '1', new_date) - return new_date - - def file_dump_json(filename: Path, data: Any, is_zip: bool = False, log: bool = True) -> None: """ Dump JSON data into a file @@ -262,17 +248,7 @@ def dataframe_to_json(dataframe: pd.DataFrame) -> str: :param dataframe: A pandas DataFrame :returns: A JSON string of the pandas DataFrame """ - # https://github.com/pandas-dev/pandas/issues/24889 - # https://github.com/pandas-dev/pandas/issues/40443 - # We need to convert to a dict to avoid mem leak - def default(z): - if isinstance(z, pd.Timestamp): - return z.timestamp() * 1e3 - if z is pd.NaT: - return 'NaT' - raise TypeError - - return str(orjson.dumps(dataframe.to_dict(orient='split'), default=default), 'utf-8') + return dataframe.to_json(orient='split') def json_to_dataframe(data: str) -> pd.DataFrame: diff --git a/freqtrade/persistence/trade_model.py b/freqtrade/persistence/trade_model.py index cc72e2bf0..5d8aada6b 100644 --- a/freqtrade/persistence/trade_model.py +++ b/freqtrade/persistence/trade_model.py @@ -19,7 +19,7 @@ from freqtrade.exchange import (ROUND_DOWN, ROUND_UP, amount_to_contract_precisi price_to_precision) from freqtrade.leverage import interest from freqtrade.persistence.base import ModelBase, SessionType -from freqtrade.util import FtPrecise +from freqtrade.util import FtPrecise, dt_now logger = logging.getLogger(__name__) @@ -68,7 +68,7 @@ class Order(ModelBase): remaining: Mapped[Optional[float]] = mapped_column(Float(), nullable=True) cost: Mapped[Optional[float]] = mapped_column(Float(), nullable=True) stop_price: Mapped[Optional[float]] = mapped_column(Float(), nullable=True) - order_date: Mapped[datetime] = mapped_column(nullable=True, default=datetime.utcnow) + order_date: Mapped[datetime] = mapped_column(nullable=True, default=dt_now) order_filled_date: Mapped[Optional[datetime]] = mapped_column(nullable=True) order_update_date: Mapped[Optional[datetime]] = mapped_column(nullable=True) funding_fee: Mapped[Optional[float]] = mapped_column(Float(), nullable=True) diff --git a/freqtrade/plugins/pairlist/AgeFilter.py b/freqtrade/plugins/pairlist/AgeFilter.py index f9c02e250..2af86592f 100644 --- a/freqtrade/plugins/pairlist/AgeFilter.py +++ b/freqtrade/plugins/pairlist/AgeFilter.py @@ -3,9 +3,9 @@ Minimum age (days listed) pair list filter """ import logging from copy import deepcopy +from datetime import timedelta from typing import Any, Dict, List, Optional -import arrow from pandas import DataFrame from freqtrade.constants import Config, ListPairsWithTimeframes @@ -13,7 +13,7 @@ from freqtrade.exceptions import OperationalException from freqtrade.exchange.types import Tickers from freqtrade.misc import plural from freqtrade.plugins.pairlist.IPairList import IPairList -from freqtrade.util import PeriodicCache +from freqtrade.util import PeriodicCache, dt_floor_day, dt_now, dt_ts logger = logging.getLogger(__name__) @@ -84,10 +84,7 @@ class AgeFilter(IPairList): since_days = -( self._max_days_listed if self._max_days_listed else self._min_days_listed ) - 1 - since_ms = int(arrow.utcnow() - .floor('day') - .shift(days=since_days) - .float_timestamp) * 1000 + since_ms = dt_ts(dt_floor_day(dt_now()) + timedelta(days=since_days)) candles = self._exchange.refresh_latest_ohlcv(needed_pairs, since_ms=since_ms, cache=False) if self._enabled: for p in deepcopy(pairlist): @@ -116,7 +113,7 @@ class AgeFilter(IPairList): ): # We have fetched at least the minimum required number of daily candles # Add to cache, store the time we last checked this symbol - self._symbolsChecked[pair] = arrow.utcnow().int_timestamp * 1000 + self._symbolsChecked[pair] = dt_ts() return True else: self.log_once(( @@ -127,6 +124,6 @@ class AgeFilter(IPairList): " or more than " f"{self._max_days_listed} {plural(self._max_days_listed, 'day')}" ) if self._max_days_listed else ''), logger.info) - self._symbolsCheckFailed[pair] = arrow.utcnow().int_timestamp * 1000 + self._symbolsCheckFailed[pair] = dt_ts() return False return False diff --git a/freqtrade/plugins/pairlist/VolatilityFilter.py b/freqtrade/plugins/pairlist/VolatilityFilter.py index 401a2e86c..9196026bb 100644 --- a/freqtrade/plugins/pairlist/VolatilityFilter.py +++ b/freqtrade/plugins/pairlist/VolatilityFilter.py @@ -4,9 +4,9 @@ Volatility pairlist filter import logging import sys from copy import deepcopy +from datetime import timedelta from typing import Any, Dict, List, Optional -import arrow import numpy as np from cachetools import TTLCache from pandas import DataFrame @@ -16,6 +16,7 @@ from freqtrade.exceptions import OperationalException from freqtrade.exchange.types import Tickers from freqtrade.misc import plural from freqtrade.plugins.pairlist.IPairList import IPairList +from freqtrade.util import dt_floor_day, dt_now, dt_ts logger = logging.getLogger(__name__) @@ -73,10 +74,7 @@ class VolatilityFilter(IPairList): needed_pairs: ListPairsWithTimeframes = [ (p, '1d', self._def_candletype) for p in pairlist if p not in self._pair_cache] - since_ms = (arrow.utcnow() - .floor('day') - .shift(days=-self._days - 1) - .int_timestamp) * 1000 + since_ms = dt_ts(dt_floor_day(dt_now()) - timedelta(days=self._days - 1)) # Get all candles candles = {} if needed_pairs: diff --git a/freqtrade/plugins/pairlist/VolumePairList.py b/freqtrade/plugins/pairlist/VolumePairList.py index 2649a8425..b9c312f87 100644 --- a/freqtrade/plugins/pairlist/VolumePairList.py +++ b/freqtrade/plugins/pairlist/VolumePairList.py @@ -4,7 +4,7 @@ Volume PairList provider Provides dynamic pair list based on trade volumes """ import logging -from datetime import datetime, timedelta, timezone +from datetime import timedelta from typing import Any, Dict, List, Literal from cachetools import TTLCache @@ -15,6 +15,7 @@ from freqtrade.exchange import timeframe_to_minutes, timeframe_to_prev_date from freqtrade.exchange.types import Tickers from freqtrade.misc import format_ms_time from freqtrade.plugins.pairlist.IPairList import IPairList +from freqtrade.util import dt_now logger = logging.getLogger(__name__) @@ -161,13 +162,13 @@ class VolumePairList(IPairList): # get lookback period in ms, for exchange ohlcv fetch since_ms = int(timeframe_to_prev_date( self._lookback_timeframe, - datetime.now(timezone.utc) + timedelta( + dt_now() + timedelta( minutes=-(self._lookback_period * self._tf_in_min) - self._tf_in_min) ).timestamp()) * 1000 to_ms = int(timeframe_to_prev_date( self._lookback_timeframe, - datetime.now(timezone.utc) - timedelta(minutes=self._tf_in_min) + dt_now() - timedelta(minutes=self._tf_in_min) ).timestamp()) * 1000 # todo: utc date output for starting date diff --git a/freqtrade/plugins/pairlist/rangestabilityfilter.py b/freqtrade/plugins/pairlist/rangestabilityfilter.py index 546b026cb..1181b2812 100644 --- a/freqtrade/plugins/pairlist/rangestabilityfilter.py +++ b/freqtrade/plugins/pairlist/rangestabilityfilter.py @@ -3,9 +3,9 @@ Rate of change pairlist filter """ import logging from copy import deepcopy +from datetime import timedelta from typing import Any, Dict, List, Optional -import arrow from cachetools import TTLCache from pandas import DataFrame @@ -14,6 +14,7 @@ from freqtrade.exceptions import OperationalException from freqtrade.exchange.types import Tickers from freqtrade.misc import plural from freqtrade.plugins.pairlist.IPairList import IPairList +from freqtrade.util import dt_floor_day, dt_now, dt_ts logger = logging.getLogger(__name__) @@ -71,10 +72,7 @@ class RangeStabilityFilter(IPairList): needed_pairs: ListPairsWithTimeframes = [ (p, '1d', self._def_candletype) for p in pairlist if p not in self._pair_cache] - since_ms = (arrow.utcnow() - .floor('day') - .shift(days=-self._days - 1) - .int_timestamp) * 1000 + since_ms = dt_ts(dt_floor_day(dt_now()) - timedelta(days=self._days - 1)) # Get all candles candles = {} if needed_pairs: diff --git a/freqtrade/rpc/api_server/api_backtest.py b/freqtrade/rpc/api_server/api_backtest.py index b168affc3..8fa1a87b8 100644 --- a/freqtrade/rpc/api_server/api_backtest.py +++ b/freqtrade/rpc/api_server/api_backtest.py @@ -8,6 +8,7 @@ from fastapi import APIRouter, BackgroundTasks, Depends from fastapi.exceptions import HTTPException from freqtrade.configuration.config_validation import validate_config_consistency +from freqtrade.constants import Config from freqtrade.data.btanalysis import get_backtest_resultlist, load_and_merge_backtest_result from freqtrade.enums import BacktestState from freqtrade.exceptions import DependencyException, OperationalException @@ -16,7 +17,7 @@ from freqtrade.misc import deep_merge_dicts from freqtrade.rpc.api_server.api_schemas import (BacktestHistoryEntry, BacktestRequest, BacktestResponse) from freqtrade.rpc.api_server.deps import get_config, is_webserver_mode -from freqtrade.rpc.api_server.webserver import ApiServer +from freqtrade.rpc.api_server.webserver_bgwork import ApiBG from freqtrade.rpc.rpc import RPCException @@ -26,13 +27,85 @@ logger = logging.getLogger(__name__) router = APIRouter() +def __run_backtest_bg(btconfig: Config): + from freqtrade.optimize.optimize_reports import generate_backtest_stats, store_backtest_stats + from freqtrade.resolvers import StrategyResolver + asyncio.set_event_loop(asyncio.new_event_loop()) + try: + # Reload strategy + lastconfig = ApiBG.bt['last_config'] + strat = StrategyResolver.load_strategy(btconfig) + validate_config_consistency(btconfig) + + if ( + not ApiBG.bt['bt'] + or lastconfig.get('timeframe') != strat.timeframe + or lastconfig.get('timeframe_detail') != btconfig.get('timeframe_detail') + or lastconfig.get('timerange') != btconfig['timerange'] + ): + from freqtrade.optimize.backtesting import Backtesting + ApiBG.bt['bt'] = Backtesting(btconfig) + ApiBG.bt['bt'].load_bt_data_detail() + else: + ApiBG.bt['bt'].config = btconfig + ApiBG.bt['bt'].init_backtest() + # Only reload data if timeframe changed. + if ( + not ApiBG.bt['data'] + or not ApiBG.bt['timerange'] + or lastconfig.get('timeframe') != strat.timeframe + or lastconfig.get('timerange') != btconfig['timerange'] + ): + ApiBG.bt['data'], ApiBG.bt['timerange'] = ApiBG.bt[ + 'bt'].load_bt_data() + + lastconfig['timerange'] = btconfig['timerange'] + lastconfig['timeframe'] = strat.timeframe + lastconfig['protections'] = btconfig.get('protections', []) + lastconfig['enable_protections'] = btconfig.get('enable_protections') + lastconfig['dry_run_wallet'] = btconfig.get('dry_run_wallet') + + ApiBG.bt['bt'].enable_protections = btconfig.get('enable_protections', False) + ApiBG.bt['bt'].strategylist = [strat] + ApiBG.bt['bt'].results = {} + ApiBG.bt['bt'].load_prior_backtest() + + ApiBG.bt['bt'].abort = False + if (ApiBG.bt['bt'].results and + strat.get_strategy_name() in ApiBG.bt['bt'].results['strategy']): + # When previous result hash matches - reuse that result and skip backtesting. + logger.info(f'Reusing result of previous backtest for {strat.get_strategy_name()}') + else: + min_date, max_date = ApiBG.bt['bt'].backtest_one_strategy( + strat, ApiBG.bt['data'], ApiBG.bt['timerange']) + + ApiBG.bt['bt'].results = generate_backtest_stats( + ApiBG.bt['data'], ApiBG.bt['bt'].all_results, + min_date=min_date, max_date=max_date) + + if btconfig.get('export', 'none') == 'trades': + store_backtest_stats( + btconfig['exportfilename'], ApiBG.bt['bt'].results, + datetime.now().strftime("%Y-%m-%d_%H-%M-%S") + ) + + logger.info("Backtest finished.") + + except (Exception, OperationalException, DependencyException) as e: + logger.exception(f"Backtesting caused an error: {e}") + ApiBG.bt['bt_error'] = str(e) + pass + finally: + ApiBG.bgtask_running = False + + @router.post('/backtest', response_model=BacktestResponse, tags=['webserver', 'backtest']) -async def api_start_backtest( # noqa: C901 +async def api_start_backtest( bt_settings: BacktestRequest, background_tasks: BackgroundTasks, config=Depends(get_config), ws_mode=Depends(is_webserver_mode)): - ApiServer._bt['bt_error'] = None + ApiBG.bt['bt_error'] = None """Start backtesting if not done so already""" - if ApiServer._bgtask_running: + if ApiBG.bgtask_running: raise RPCException('Bot Background task already running') if ':' in bt_settings.strategy: @@ -56,80 +129,9 @@ async def api_start_backtest( # noqa: C901 # Start backtesting # Initialize backtesting object - def run_backtest(): - from freqtrade.optimize.optimize_reports import (generate_backtest_stats, - store_backtest_stats) - from freqtrade.resolvers import StrategyResolver - asyncio.set_event_loop(asyncio.new_event_loop()) - try: - # Reload strategy - lastconfig = ApiServer._bt['last_config'] - strat = StrategyResolver.load_strategy(btconfig) - validate_config_consistency(btconfig) - if ( - not ApiServer._bt['bt'] - or lastconfig.get('timeframe') != strat.timeframe - or lastconfig.get('timeframe_detail') != btconfig.get('timeframe_detail') - or lastconfig.get('timerange') != btconfig['timerange'] - ): - from freqtrade.optimize.backtesting import Backtesting - ApiServer._bt['bt'] = Backtesting(btconfig) - ApiServer._bt['bt'].load_bt_data_detail() - else: - ApiServer._bt['bt'].config = btconfig - ApiServer._bt['bt'].init_backtest() - # Only reload data if timeframe changed. - if ( - not ApiServer._bt['data'] - or not ApiServer._bt['timerange'] - or lastconfig.get('timeframe') != strat.timeframe - or lastconfig.get('timerange') != btconfig['timerange'] - ): - ApiServer._bt['data'], ApiServer._bt['timerange'] = ApiServer._bt[ - 'bt'].load_bt_data() - - lastconfig['timerange'] = btconfig['timerange'] - lastconfig['timeframe'] = strat.timeframe - lastconfig['protections'] = btconfig.get('protections', []) - lastconfig['enable_protections'] = btconfig.get('enable_protections') - lastconfig['dry_run_wallet'] = btconfig.get('dry_run_wallet') - - ApiServer._bt['bt'].enable_protections = btconfig.get('enable_protections', False) - ApiServer._bt['bt'].strategylist = [strat] - ApiServer._bt['bt'].results = {} - ApiServer._bt['bt'].load_prior_backtest() - - ApiServer._bt['bt'].abort = False - if (ApiServer._bt['bt'].results and - strat.get_strategy_name() in ApiServer._bt['bt'].results['strategy']): - # When previous result hash matches - reuse that result and skip backtesting. - logger.info(f'Reusing result of previous backtest for {strat.get_strategy_name()}') - else: - min_date, max_date = ApiServer._bt['bt'].backtest_one_strategy( - strat, ApiServer._bt['data'], ApiServer._bt['timerange']) - - ApiServer._bt['bt'].results = generate_backtest_stats( - ApiServer._bt['data'], ApiServer._bt['bt'].all_results, - min_date=min_date, max_date=max_date) - - if btconfig.get('export', 'none') == 'trades': - store_backtest_stats( - btconfig['exportfilename'], ApiServer._bt['bt'].results, - datetime.now().strftime("%Y-%m-%d_%H-%M-%S") - ) - - logger.info("Backtest finished.") - - except (Exception, OperationalException, DependencyException) as e: - logger.exception(f"Backtesting caused an error: {e}") - ApiServer._bt['bt_error'] = str(e) - pass - finally: - ApiServer._bgtask_running = False - - background_tasks.add_task(run_backtest) - ApiServer._bgtask_running = True + background_tasks.add_task(__run_backtest_bg, btconfig=btconfig) + ApiBG.bgtask_running = True return { "status": "running", @@ -147,18 +149,18 @@ def api_get_backtest(ws_mode=Depends(is_webserver_mode)): Returns Result after backtesting has been ran. """ from freqtrade.persistence import LocalTrade - if ApiServer._bgtask_running: + if ApiBG.bgtask_running: return { "status": "running", "running": True, - "step": (ApiServer._bt['bt'].progress.action if ApiServer._bt['bt'] + "step": (ApiBG.bt['bt'].progress.action if ApiBG.bt['bt'] else str(BacktestState.STARTUP)), - "progress": ApiServer._bt['bt'].progress.progress if ApiServer._bt['bt'] else 0, + "progress": ApiBG.bt['bt'].progress.progress if ApiBG.bt['bt'] else 0, "trade_count": len(LocalTrade.trades), "status_msg": "Backtest running", } - if not ApiServer._bt['bt']: + if not ApiBG.bt['bt']: return { "status": "not_started", "running": False, @@ -166,13 +168,13 @@ def api_get_backtest(ws_mode=Depends(is_webserver_mode)): "progress": 0, "status_msg": "Backtest not yet executed" } - if ApiServer._bt['bt_error']: + if ApiBG.bt['bt_error']: return { "status": "error", "running": False, "step": "", "progress": 0, - "status_msg": f"Backtest failed with {ApiServer._bt['bt_error']}" + "status_msg": f"Backtest failed with {ApiBG.bt['bt_error']}" } return { @@ -181,14 +183,14 @@ def api_get_backtest(ws_mode=Depends(is_webserver_mode)): "status_msg": "Backtest ended", "step": "finished", "progress": 1, - "backtest_result": ApiServer._bt['bt'].results, + "backtest_result": ApiBG.bt['bt'].results, } @router.delete('/backtest', response_model=BacktestResponse, tags=['webserver', 'backtest']) def api_delete_backtest(ws_mode=Depends(is_webserver_mode)): """Reset backtesting""" - if ApiServer._bgtask_running: + if ApiBG.bgtask_running: return { "status": "running", "running": True, @@ -196,12 +198,12 @@ def api_delete_backtest(ws_mode=Depends(is_webserver_mode)): "progress": 0, "status_msg": "Backtest running", } - if ApiServer._bt['bt']: - ApiServer._bt['bt'].cleanup() - del ApiServer._bt['bt'] - ApiServer._bt['bt'] = None - del ApiServer._bt['data'] - ApiServer._bt['data'] = None + if ApiBG.bt['bt']: + ApiBG.bt['bt'].cleanup() + del ApiBG.bt['bt'] + ApiBG.bt['bt'] = None + del ApiBG.bt['data'] + ApiBG.bt['data'] = None logger.info("Backtesting reset") return { "status": "reset", @@ -214,7 +216,7 @@ def api_delete_backtest(ws_mode=Depends(is_webserver_mode)): @router.get('/backtest/abort', response_model=BacktestResponse, tags=['webserver', 'backtest']) def api_backtest_abort(ws_mode=Depends(is_webserver_mode)): - if not ApiServer._bgtask_running: + if not ApiBG.bgtask_running: return { "status": "not_running", "running": False, @@ -222,7 +224,7 @@ def api_backtest_abort(ws_mode=Depends(is_webserver_mode)): "progress": 0, "status_msg": "Backtest ended", } - ApiServer._bt['bt'].abort = True + ApiBG.bt['bt'].abort = True return { "status": "stopping", "running": False, diff --git a/freqtrade/rpc/api_server/deps.py b/freqtrade/rpc/api_server/deps.py index bfc1e698c..8fd105d3e 100644 --- a/freqtrade/rpc/api_server/deps.py +++ b/freqtrade/rpc/api_server/deps.py @@ -6,6 +6,7 @@ from fastapi import Depends from freqtrade.enums import RunMode from freqtrade.persistence import Trade from freqtrade.persistence.models import _request_id_ctx_var +from freqtrade.rpc.api_server.webserver_bgwork import ApiBG from freqtrade.rpc.rpc import RPC, RPCException from .webserver import ApiServer @@ -43,11 +44,11 @@ def get_api_config() -> Dict[str, Any]: def get_exchange(config=Depends(get_config)): - if not ApiServer._exchange: + if not ApiBG.exchange: from freqtrade.resolvers import ExchangeResolver - ApiServer._exchange = ExchangeResolver.load_exchange( + ApiBG.exchange = ExchangeResolver.load_exchange( config, load_leverage_tiers=False) - return ApiServer._exchange + return ApiBG.exchange def get_message_stream(): diff --git a/freqtrade/rpc/api_server/webserver.py b/freqtrade/rpc/api_server/webserver.py index 8030e303b..165849a7f 100644 --- a/freqtrade/rpc/api_server/webserver.py +++ b/freqtrade/rpc/api_server/webserver.py @@ -1,6 +1,6 @@ import logging from ipaddress import IPv4Address -from typing import Any, Dict, Optional +from typing import Any, Optional import orjson import uvicorn @@ -36,19 +36,8 @@ class ApiServer(RPCHandler): __initialized = False _rpc: RPC - # Backtesting type: Backtesting - _bt: Dict[str, Any] = { - 'bt': None, - 'data': None, - 'timerange': None, - 'last_config': {}, - 'bt_error': None, - } _has_rpc: bool = False - _bgtask_running: bool = False _config: Config = {} - # Exchange - only available in webserver mode. - _exchange = None # websocket message stuff _message_stream: Optional[MessageStream] = None @@ -85,7 +74,7 @@ class ApiServer(RPCHandler): """ Attach rpc handler """ - if not self._has_rpc: + if not ApiServer._has_rpc: ApiServer._rpc = rpc ApiServer._has_rpc = True else: diff --git a/freqtrade/rpc/api_server/webserver_bgwork.py b/freqtrade/rpc/api_server/webserver_bgwork.py new file mode 100644 index 000000000..925f34de3 --- /dev/null +++ b/freqtrade/rpc/api_server/webserver_bgwork.py @@ -0,0 +1,16 @@ + +from typing import Any, Dict + + +class ApiBG(): + # Backtesting type: Backtesting + bt: Dict[str, Any] = { + 'bt': None, + 'data': None, + 'timerange': None, + 'last_config': {}, + 'bt_error': None, + } + bgtask_running: bool = False + # Exchange - only available in webserver mode. + exchange = None diff --git a/freqtrade/rpc/rpc.py b/freqtrade/rpc/rpc.py index 2e256ee98..dedb35503 100644 --- a/freqtrade/rpc/rpc.py +++ b/freqtrade/rpc/rpc.py @@ -7,7 +7,6 @@ from datetime import date, datetime, timedelta, timezone from math import isnan from typing import Any, Dict, Generator, List, Optional, Sequence, Tuple, Union -import arrow import psutil from dateutil.relativedelta import relativedelta from dateutil.tz import tzlocal @@ -26,12 +25,13 @@ from freqtrade.exceptions import ExchangeError, PricingError from freqtrade.exchange import timeframe_to_minutes, timeframe_to_msecs from freqtrade.exchange.types import Tickers from freqtrade.loggers import bufferHandler -from freqtrade.misc import decimals_per_coin, shorten_date +from freqtrade.misc import decimals_per_coin from freqtrade.persistence import KeyStoreKeys, KeyValueStore, Order, PairLocks, Trade from freqtrade.persistence.models import PairLock from freqtrade.plugins.pairlist.pairlist_helpers import expand_pairlist from freqtrade.rpc.fiat_convert import CryptoToFiatConverter from freqtrade.rpc.rpc_types import RPCSendMsg +from freqtrade.util import dt_humanize, dt_now, shorten_date from freqtrade.wallets import PositionWallet, Wallet @@ -292,7 +292,7 @@ class RPC: and open_order.ft_order_side == trade.entry_side) else '') + ('**' if (open_order and open_order.ft_order_side == trade.exit_side is not None) else ''), - shorten_date(arrow.get(trade.open_date).humanize(only_distance=True)), + shorten_date(dt_humanize(trade.open_date, only_distance=True)), profit_str ] if self._config.get('position_adjustment_enable', False): @@ -564,10 +564,10 @@ class RPC: 'trade_count': len(trades), 'closed_trade_count': len([t for t in trades if not t.is_open]), 'first_trade_date': first_date.strftime(DATETIME_PRINT_FORMAT) if first_date else '', - 'first_trade_humanized': arrow.get(first_date).humanize() if first_date else '', + 'first_trade_humanized': dt_humanize(first_date) if first_date else '', 'first_trade_timestamp': int(first_date.timestamp() * 1000) if first_date else 0, 'latest_trade_date': last_date.strftime(DATETIME_PRINT_FORMAT) if last_date else '', - 'latest_trade_humanized': arrow.get(last_date).humanize() if last_date else '', + 'latest_trade_humanized': dt_humanize(last_date) if last_date else '', 'latest_trade_timestamp': int(last_date.timestamp() * 1000) if last_date else 0, 'avg_duration': str(timedelta(seconds=sum(durations) / num)).split('.')[0], 'best_pair': best_pair[0] if best_pair else '', @@ -1252,7 +1252,7 @@ class RPC: df_analyzed = strategy.analyze_ticker(_data[pair], {'pair': pair}) return RPC._convert_dataframe_to_dict(strategy.get_strategy_name(), pair, timeframe, - df_analyzed, arrow.Arrow.utcnow().datetime) + df_analyzed, dt_now()) def _rpc_plot_config(self) -> Dict[str, Any]: if (self._freqtrade.strategy.plot_config and diff --git a/freqtrade/rpc/telegram.py b/freqtrade/rpc/telegram.py index 9ecc2b677..d082299cb 100644 --- a/freqtrade/rpc/telegram.py +++ b/freqtrade/rpc/telegram.py @@ -17,7 +17,6 @@ from math import isnan from threading import Thread from typing import Any, Callable, Coroutine, Dict, List, Optional, Union -import arrow from tabulate import tabulate from telegram import (CallbackQuery, InlineKeyboardButton, InlineKeyboardMarkup, KeyboardButton, ReplyKeyboardMarkup, Update) @@ -34,6 +33,7 @@ from freqtrade.misc import chunks, plural, round_coin_value from freqtrade.persistence import Trade from freqtrade.rpc import RPC, RPCException, RPCHandler from freqtrade.rpc.rpc_types import RPCSendMsg +from freqtrade.util import dt_humanize MAX_MESSAGE_LENGTH = MessageLimit.MAX_TEXT_LENGTH @@ -528,7 +528,6 @@ class Telegram(RPCHandler): order_nr += 1 wording = 'Entry' if order['ft_is_entry'] else 'Exit' - cur_entry_datetime = arrow.get(order["order_filled_date"]) cur_entry_amount = order["filled"] or order["amount"] cur_entry_average = order["safe_price"] lines.append(" ") @@ -559,22 +558,14 @@ class Telegram(RPCHandler): lines.append(f"*{wording} #{order_nr}:* at {minus_on_entry:.2%} avg Profit") if is_open: - lines.append("({})".format(cur_entry_datetime - .humanize(granularity=["day", "hour", "minute"]))) + lines.append("({})".format(dt_humanize(order["order_filled_date"], + granularity=["day", "hour", "minute"]))) lines.append(f"*Amount:* {cur_entry_amount} " f"({round_coin_value(order['cost'], quote_currency)})") lines.append(f"*Average {wording} Price:* {cur_entry_average} " f"({price_to_1st_entry:.2%} from 1st entry Rate)") lines.append(f"*Order filled:* {order['order_filled_date']}") - # TODO: is this really useful? - # dur_entry = cur_entry_datetime - arrow.get( - # filled_orders[x - 1]["order_filled_date"]) - # days = dur_entry.days - # hours, remainder = divmod(dur_entry.seconds, 3600) - # minutes, seconds = divmod(remainder, 60) - # lines.append( - # f"({days}d {hours}h {minutes}m {seconds}s from previous {wording.lower()})") lines_detail.append("\n".join(lines)) return lines_detail @@ -610,7 +601,7 @@ class Telegram(RPCHandler): position_adjust = self._config.get('position_adjustment_enable', False) max_entries = self._config.get('max_entry_position_adjustment', -1) for r in results: - r['open_date_hum'] = arrow.get(r['open_date']).humanize() + r['open_date_hum'] = dt_humanize(r['open_date']) r['num_entries'] = len([o for o in r['orders'] if o['ft_is_entry']]) r['num_exits'] = len([o for o in r['orders'] if not o['ft_is_entry'] and not o['ft_order_side'] == 'stoploss']) @@ -1219,7 +1210,7 @@ class Telegram(RPCHandler): nrecent ) trades_tab = tabulate( - [[arrow.get(trade['close_date']).humanize(), + [[dt_humanize(trade['close_date']), trade['pair'] + " (#" + str(trade['trade_id']) + ")", f"{(trade['close_profit']):.2%} ({trade['close_profit_abs']})"] for trade in trades['trades']], diff --git a/freqtrade/strategy/interface.py b/freqtrade/strategy/interface.py index 7adb7a154..382f38c9a 100644 --- a/freqtrade/strategy/interface.py +++ b/freqtrade/strategy/interface.py @@ -7,7 +7,6 @@ from abc import ABC, abstractmethod from datetime import datetime, timedelta, timezone from typing import Dict, List, Optional, Tuple, Union -import arrow from pandas import DataFrame from freqtrade.constants import CUSTOM_TAG_MAX_LENGTH, Config, IntOrInf, ListPairsWithTimeframes @@ -23,6 +22,7 @@ from freqtrade.strategy.informative_decorator import (InformativeData, PopulateI _create_and_merge_informative_pair, _format_pair_name) from freqtrade.strategy.strategy_wrapper import strategy_safe_wrapper +from freqtrade.util import dt_now from freqtrade.wallets import Wallets @@ -938,7 +938,7 @@ class IStrategy(ABC, HyperStrategyMixin): pair: str, timeframe: str, dataframe: DataFrame, - ) -> Tuple[Optional[DataFrame], Optional[arrow.Arrow]]: + ) -> Tuple[Optional[DataFrame], Optional[datetime]]: """ Calculates current signal based based on the entry order or exit order columns of the dataframe. @@ -954,16 +954,16 @@ class IStrategy(ABC, HyperStrategyMixin): latest_date = dataframe['date'].max() latest = dataframe.loc[dataframe['date'] == latest_date].iloc[-1] - # Explicitly convert to arrow object to ensure the below comparison does not fail - latest_date = arrow.get(latest_date) + # Explicitly convert to datetime object to ensure the below comparison does not fail + latest_date = latest_date.to_pydatetime() # Check if dataframe is out of date timeframe_minutes = timeframe_to_minutes(timeframe) offset = self.config.get('exchange', {}).get('outdated_offset', 5) - if latest_date < (arrow.utcnow().shift(minutes=-(timeframe_minutes * 2 + offset))): + if latest_date < (dt_now() - timedelta(minutes=timeframe_minutes * 2 + offset)): logger.warning( 'Outdated history for pair %s. Last tick is %s minutes old', - pair, int((arrow.utcnow() - latest_date).total_seconds() // 60) + pair, int((dt_now() - latest_date).total_seconds() // 60) ) return None, None return latest, latest_date @@ -1046,8 +1046,8 @@ class IStrategy(ABC, HyperStrategyMixin): timeframe_seconds = timeframe_to_seconds(timeframe) if self.ignore_expired_candle( - latest_date=latest_date.datetime, - current_time=datetime.now(timezone.utc), + latest_date=latest_date, + current_time=dt_now(), timeframe_seconds=timeframe_seconds, enter=bool(enter_signal) ): diff --git a/freqtrade/util/__init__.py b/freqtrade/util/__init__.py index 3c3c034c1..bed65a54b 100644 --- a/freqtrade/util/__init__.py +++ b/freqtrade/util/__init__.py @@ -1,2 +1,17 @@ -from freqtrade.util.ft_precise import FtPrecise # noqa: F401 -from freqtrade.util.periodic_cache import PeriodicCache # noqa: F401 +from freqtrade.util.datetime_helpers import (dt_floor_day, dt_from_ts, dt_humanize, dt_now, dt_ts, + dt_utc, shorten_date) +from freqtrade.util.ft_precise import FtPrecise +from freqtrade.util.periodic_cache import PeriodicCache + + +__all__ = [ + 'dt_floor_day', + 'dt_from_ts', + 'dt_now', + 'dt_ts', + 'dt_utc', + 'dt_humanize', + 'shorten_date', + 'FtPrecise', + 'PeriodicCache', +] diff --git a/freqtrade/util/datetime_helpers.py b/freqtrade/util/datetime_helpers.py new file mode 100644 index 000000000..39d134e11 --- /dev/null +++ b/freqtrade/util/datetime_helpers.py @@ -0,0 +1,63 @@ +import re +from datetime import datetime, timezone +from typing import Optional + +import arrow + + +def dt_now() -> datetime: + """Return the current datetime in UTC.""" + return datetime.now(timezone.utc) + + +def dt_utc(year: int, month: int, day: int, hour: int = 0, minute: int = 0, second: int = 0, + microsecond: int = 0) -> datetime: + """Return a datetime in UTC.""" + return datetime(year, month, day, hour, minute, second, microsecond, tzinfo=timezone.utc) + + +def dt_ts(dt: Optional[datetime] = None) -> int: + """ + Return dt in ms as a timestamp in UTC. + If dt is None, return the current datetime in UTC. + """ + if dt: + return int(dt.timestamp() * 1000) + return int(dt_now().timestamp() * 1000) + + +def dt_floor_day(dt: datetime) -> datetime: + """Return the floor of the day for the given datetime.""" + return dt.replace(hour=0, minute=0, second=0, microsecond=0) + + +def dt_from_ts(timestamp: float) -> datetime: + """ + Return a datetime from a timestamp. + :param timestamp: timestamp in seconds or milliseconds + """ + if timestamp > 1e10: + # Timezone in ms - convert to seconds + timestamp /= 1000 + return datetime.fromtimestamp(timestamp, tz=timezone.utc) + + +def shorten_date(_date: str) -> str: + """ + Trim the date so it fits on small screens + """ + new_date = re.sub('seconds?', 'sec', _date) + new_date = re.sub('minutes?', 'min', new_date) + new_date = re.sub('hours?', 'h', new_date) + new_date = re.sub('days?', 'd', new_date) + new_date = re.sub('^an?', '1', new_date) + return new_date + + +def dt_humanize(dt: datetime, **kwargs) -> str: + """ + Return a humanized string for the given datetime. + :param dt: datetime to humanize + :param kwargs: kwargs to pass to arrow's humanize() + """ + return arrow.get(dt).humanize(**kwargs) diff --git a/freqtrade/wallets.py b/freqtrade/wallets.py index 9a33d1fb1..da64515a4 100644 --- a/freqtrade/wallets.py +++ b/freqtrade/wallets.py @@ -3,16 +3,16 @@ import logging from copy import deepcopy +from datetime import datetime, timedelta from typing import Dict, NamedTuple, Optional -import arrow - from freqtrade.constants import UNLIMITED_STAKE_AMOUNT, Config from freqtrade.enums import RunMode, TradingMode from freqtrade.exceptions import DependencyException from freqtrade.exchange import Exchange from freqtrade.misc import safe_value_fallback from freqtrade.persistence import LocalTrade, Trade +from freqtrade.util.datetime_helpers import dt_now logger = logging.getLogger(__name__) @@ -43,7 +43,7 @@ class Wallets: self._wallets: Dict[str, Wallet] = {} self._positions: Dict[str, PositionWallet] = {} self.start_cap = config['dry_run_wallet'] - self._last_wallet_refresh = 0 + self._last_wallet_refresh: Optional[datetime] = None self.update() def get_free(self, currency: str) -> float: @@ -166,14 +166,19 @@ class Wallets: for trading operations, the latest balance is needed. :param require_update: Allow skipping an update if balances were recently refreshed """ - if (require_update or (self._last_wallet_refresh + 3600 < arrow.utcnow().int_timestamp)): + now = dt_now() + if ( + require_update + or self._last_wallet_refresh is None + or (self._last_wallet_refresh + timedelta(seconds=3600) < now) + ): if (not self._config['dry_run'] or self._config.get('runmode') == RunMode.LIVE): self._update_live() else: self._update_dry() if self._log: logger.info('Wallets synced.') - self._last_wallet_refresh = arrow.utcnow().int_timestamp + self._last_wallet_refresh = dt_now() def get_all_balances(self) -> Dict[str, Wallet]: return self._wallets diff --git a/requirements-dev.txt b/requirements-dev.txt index b1ba32b43..cc3463174 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -7,9 +7,9 @@ -r docs/requirements-docs.txt coveralls==3.3.1 -ruff==0.0.267 +ruff==0.0.269 mypy==1.3.0 -pre-commit==3.3.1 +pre-commit==3.3.2 pytest==7.3.1 pytest-asyncio==0.21.0 pytest-cov==4.0.0 diff --git a/requirements-freqai-rl.txt b/requirements-freqai-rl.txt index 535d10f4b..de48a1da4 100644 --- a/requirements-freqai-rl.txt +++ b/requirements-freqai-rl.txt @@ -5,7 +5,7 @@ torch==2.0.1 #until these branches will be released we can use this gymnasium==0.28.1 -stable_baselines3==2.0.0a5 -sb3_contrib>=2.0.0a4 +stable_baselines3==2.0.0a10 +sb3_contrib>=2.0.0a9 # Progress bar for stable-baselines3 and sb3-contrib tqdm==4.65.0 diff --git a/requirements.txt b/requirements.txt index 5d2f4147c..cff54ca1c 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,17 +2,17 @@ numpy==1.24.3 pandas==2.0.1 pandas-ta==0.3.14b -ccxt==3.0.103 +ccxt==3.1.5 cryptography==40.0.2; platform_machine != 'armv7l' cryptography==40.0.1; platform_machine == 'armv7l' aiohttp==3.8.4 -SQLAlchemy==2.0.13 +SQLAlchemy==2.0.15 python-telegram-bot==20.3 # can't be hard-pinned due to telegram-bot pinning httpx with ~ httpx>=0.23.3 arrow==1.2.3 -cachetools==4.2.2 -requests==2.30.0 +cachetools==5.3.0 +requests==2.31.0 urllib3==2.0.2 jsonschema==4.17.3 TA-Lib==0.4.26 @@ -38,7 +38,7 @@ orjson==3.8.12 sdnotify==0.3.2 # API Server -fastapi==0.95.1 +fastapi==0.95.2 pydantic==1.10.7 uvicorn==0.22.0 pyjwt==2.7.0 diff --git a/scripts/rest_client.py b/scripts/rest_client.py index ccffe7f5f..0772af269 100755 --- a/scripts/rest_client.py +++ b/scripts/rest_client.py @@ -279,8 +279,9 @@ class FtRestClient(): """ data = {"pair": pair, "side": side, - "price": price, } + if price: + data['price'] = price return self._post("forceenter", data=data) def forceexit(self, tradeid, ordertype=None, amount=None): diff --git a/setup.py b/setup.py index b59e98ae8..f8b8b515c 100644 --- a/setup.py +++ b/setup.py @@ -72,7 +72,7 @@ setup( 'ccxt>=3.0.0', 'SQLAlchemy>=2.0.6', 'python-telegram-bot>=20.1', - 'arrow>=0.17.0', + 'arrow>=1.0.0', 'cachetools', 'requests', 'urllib3', diff --git a/tests/commands/test_commands.py b/tests/commands/test_commands.py index 318590b32..fe847e94b 100644 --- a/tests/commands/test_commands.py +++ b/tests/commands/test_commands.py @@ -1,12 +1,11 @@ import json import re -from datetime import datetime +from datetime import datetime, timedelta from io import BytesIO from pathlib import Path from unittest.mock import MagicMock, PropertyMock from zipfile import ZipFile -import arrow import pytest from freqtrade.commands import (start_backtesting_show, start_convert_data, start_convert_trades, @@ -25,6 +24,7 @@ from freqtrade.enums import RunMode from freqtrade.exceptions import OperationalException from freqtrade.persistence.models import init_db from freqtrade.persistence.pairlock_middleware import PairLocks +from freqtrade.util import dt_floor_day, dt_now, dt_utc from tests.conftest import (CURRENT_TEST_STRATEGY, EXMS, create_mock_trades, get_args, log_has, log_has_re, patch_exchange, patched_configuration_load_config_file) from tests.conftest_trades import MOCK_TRADE_COUNT @@ -689,7 +689,7 @@ def test_download_data_timerange(mocker, markets): start_download_data(pargs) assert dl_mock.call_count == 1 # 20days ago - days_ago = arrow.get(arrow.now().shift(days=-20).date()).int_timestamp + days_ago = dt_floor_day(dt_now() - timedelta(days=20)).timestamp() assert dl_mock.call_args_list[0][1]['timerange'].startts == days_ago dl_mock.reset_mock() @@ -704,8 +704,7 @@ def test_download_data_timerange(mocker, markets): start_download_data(pargs) assert dl_mock.call_count == 1 - assert dl_mock.call_args_list[0][1]['timerange'].startts == arrow.Arrow( - 2020, 1, 1).int_timestamp + assert dl_mock.call_args_list[0][1]['timerange'].startts == int(dt_utc(2020, 1, 1).timestamp()) def test_download_data_no_markets(mocker, caplog): diff --git a/tests/conftest.py b/tests/conftest.py index 70d15c6df..66f331cae 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -8,7 +8,6 @@ from pathlib import Path from typing import Optional from unittest.mock import MagicMock, Mock, PropertyMock -import arrow import numpy as np import pandas as pd import pytest @@ -23,6 +22,8 @@ from freqtrade.exchange.exchange import timeframe_to_minutes from freqtrade.freqtradebot import FreqtradeBot from freqtrade.persistence import LocalTrade, Order, Trade, init_db from freqtrade.resolvers import ExchangeResolver +from freqtrade.util import dt_ts +from freqtrade.util.datetime_helpers import dt_now from freqtrade.worker import Worker from tests.conftest_trades import (leverage_trade, mock_trade_1, mock_trade_2, mock_trade_3, mock_trade_4, mock_trade_5, mock_trade_6, short_trade) @@ -1663,8 +1664,8 @@ def limit_buy_order_open(): 'type': 'limit', 'side': 'buy', 'symbol': 'mocked', - 'timestamp': arrow.utcnow().int_timestamp * 1000, - 'datetime': arrow.utcnow().isoformat(), + 'timestamp': dt_ts(), + 'datetime': dt_now().isoformat(), 'price': 0.00001099, 'average': 0.00001099, 'amount': 90.99181073, @@ -1691,8 +1692,8 @@ def limit_buy_order_old(): 'type': 'limit', 'side': 'buy', 'symbol': 'mocked', - 'datetime': arrow.utcnow().shift(minutes=-601).isoformat(), - 'timestamp': arrow.utcnow().shift(minutes=-601).int_timestamp * 1000, + 'datetime': (dt_now() - timedelta(minutes=601)).isoformat(), + 'timestamp': dt_ts(dt_now() - timedelta(minutes=601)), 'price': 0.00001099, 'amount': 90.99181073, 'filled': 0.0, @@ -1708,8 +1709,8 @@ def limit_sell_order_old(): 'type': 'limit', 'side': 'sell', 'symbol': 'ETH/BTC', - 'timestamp': arrow.utcnow().shift(minutes=-601).int_timestamp * 1000, - 'datetime': arrow.utcnow().shift(minutes=-601).isoformat(), + 'timestamp': dt_ts(dt_now() - timedelta(minutes=601)), + 'datetime': (dt_now() - timedelta(minutes=601)).isoformat(), 'price': 0.00001099, 'amount': 90.99181073, 'filled': 0.0, @@ -1725,8 +1726,8 @@ def limit_buy_order_old_partial(): 'type': 'limit', 'side': 'buy', 'symbol': 'ETH/BTC', - 'timestamp': arrow.utcnow().shift(minutes=-601).int_timestamp * 1000, - 'datetime': arrow.utcnow().shift(minutes=-601).isoformat(), + 'timestamp': dt_ts(dt_now() - timedelta(minutes=601)), + 'datetime': (dt_now() - timedelta(minutes=601)).isoformat(), 'price': 0.00001099, 'amount': 90.99181073, 'filled': 23.0, @@ -1756,8 +1757,8 @@ def limit_buy_order_canceled_empty(request): 'info': {}, 'id': 'AZNPFF-4AC4N-7MKTAT', 'clientOrderId': None, - 'timestamp': arrow.utcnow().shift(minutes=-601).int_timestamp * 1000, - 'datetime': arrow.utcnow().shift(minutes=-601).isoformat(), + 'timestamp': dt_ts(dt_now() - timedelta(minutes=601)), + 'datetime': (dt_now() - timedelta(minutes=601)).isoformat(), 'lastTradeTimestamp': None, 'status': 'canceled', 'symbol': 'LTC/USDT', @@ -1777,8 +1778,8 @@ def limit_buy_order_canceled_empty(request): 'info': {}, 'id': '1234512345', 'clientOrderId': 'alb1234123', - 'timestamp': arrow.utcnow().shift(minutes=-601).int_timestamp * 1000, - 'datetime': arrow.utcnow().shift(minutes=-601).isoformat(), + 'timestamp': dt_ts(dt_now() - timedelta(minutes=601)), + 'datetime': (dt_now() - timedelta(minutes=601)).isoformat(), 'lastTradeTimestamp': None, 'symbol': 'LTC/USDT', 'type': 'limit', @@ -1798,8 +1799,8 @@ def limit_buy_order_canceled_empty(request): 'info': {}, 'id': '1234512345', 'clientOrderId': 'alb1234123', - 'timestamp': arrow.utcnow().shift(minutes=-601).int_timestamp * 1000, - 'datetime': arrow.utcnow().shift(minutes=-601).isoformat(), + 'timestamp': dt_ts(dt_now() - timedelta(minutes=601)), + 'datetime': (dt_now() - timedelta(minutes=601)).isoformat(), 'lastTradeTimestamp': None, 'symbol': 'LTC/USDT', 'type': 'limit', @@ -1823,8 +1824,8 @@ def limit_sell_order_open(): 'type': 'limit', 'side': 'sell', 'symbol': 'mocked', - 'datetime': arrow.utcnow().isoformat(), - 'timestamp': arrow.utcnow().int_timestamp * 1000, + 'datetime': dt_now().isoformat(), + 'timestamp': dt_ts(), 'price': 0.00001173, 'amount': 90.99181073, 'filled': 0.0, @@ -2486,8 +2487,8 @@ def buy_order_fee(): 'type': 'limit', 'side': 'buy', 'symbol': 'mocked', - 'timestamp': arrow.utcnow().shift(minutes=-601).int_timestamp * 1000, - 'datetime': arrow.utcnow().shift(minutes=-601).isoformat(), + 'timestamp': dt_ts(dt_now() - timedelta(minutes=601)), + 'datetime': (dt_now() - timedelta(minutes=601)).isoformat(), 'price': 0.245441, 'amount': 8.0, 'cost': 1.963528, @@ -2596,7 +2597,7 @@ def open_trade(): fee_open=0.0, fee_close=0.0, stake_amount=1, - open_date=arrow.utcnow().shift(minutes=-601).datetime, + open_date=dt_now() - timedelta(minutes=601), is_open=True ) trade.orders = [ @@ -2634,7 +2635,7 @@ def open_trade_usdt(): fee_open=0.0, fee_close=0.0, stake_amount=60.0, - open_date=arrow.utcnow().shift(minutes=-601).datetime, + open_date=dt_now() - timedelta(minutes=601), is_open=True ) trade.orders = [ @@ -2838,8 +2839,8 @@ def limit_buy_order_usdt_open(): 'type': 'limit', 'side': 'buy', 'symbol': 'mocked', - 'datetime': arrow.utcnow().isoformat(), - 'timestamp': arrow.utcnow().int_timestamp * 1000, + 'datetime': dt_now().isoformat(), + 'timestamp': dt_ts(), 'price': 2.00, 'average': 2.00, 'amount': 30.0, @@ -2866,8 +2867,8 @@ def limit_sell_order_usdt_open(): 'type': 'limit', 'side': 'sell', 'symbol': 'mocked', - 'datetime': arrow.utcnow().isoformat(), - 'timestamp': arrow.utcnow().int_timestamp * 1000, + 'datetime': dt_now().isoformat(), + 'timestamp': dt_ts(), 'price': 2.20, 'amount': 30.0, 'cost': 66.0, @@ -2893,8 +2894,8 @@ def market_buy_order_usdt(): 'type': 'market', 'side': 'buy', 'symbol': 'mocked', - 'timestamp': arrow.utcnow().int_timestamp * 1000, - 'datetime': arrow.utcnow().isoformat(), + 'timestamp': dt_ts(), + 'datetime': dt_now().isoformat(), 'price': 2.00, 'amount': 30.0, 'filled': 30.0, @@ -2950,8 +2951,8 @@ def market_sell_order_usdt(): 'type': 'market', 'side': 'sell', 'symbol': 'mocked', - 'timestamp': arrow.utcnow().int_timestamp * 1000, - 'datetime': arrow.utcnow().isoformat(), + 'timestamp': dt_ts(), + 'datetime': dt_now().isoformat(), 'price': 2.20, 'amount': 30.0, 'filled': 30.0, diff --git a/tests/data/test_btanalysis.py b/tests/data/test_btanalysis.py index 2c5515f7c..5e377f851 100644 --- a/tests/data/test_btanalysis.py +++ b/tests/data/test_btanalysis.py @@ -1,8 +1,8 @@ +from datetime import datetime, timedelta, timezone from pathlib import Path from unittest.mock import MagicMock import pytest -from arrow import Arrow from pandas import DataFrame, DateOffset, Timestamp, to_datetime from freqtrade.configuration import TimeRange @@ -18,6 +18,7 @@ from freqtrade.data.metrics import (calculate_cagr, calculate_calmar, calculate_ calculate_underwater, combine_dataframes_with_mean, create_cum_profit) from freqtrade.exceptions import OperationalException +from freqtrade.util import dt_utc from tests.conftest import CURRENT_TEST_STRATEGY, create_mock_trades from tests.conftest_trades import MOCK_TRADE_COUNT @@ -162,25 +163,25 @@ def test_extract_trades_of_period(testdatadir): {'pair': [pair, pair, pair, pair], 'profit_ratio': [0.0, 0.1, -0.2, -0.5], 'profit_abs': [0.0, 1, -2, -5], - 'open_date': to_datetime([Arrow(2017, 11, 13, 15, 40, 0).datetime, - Arrow(2017, 11, 14, 9, 41, 0).datetime, - Arrow(2017, 11, 14, 14, 20, 0).datetime, - Arrow(2017, 11, 15, 3, 40, 0).datetime, + 'open_date': to_datetime([datetime(2017, 11, 13, 15, 40, 0, tzinfo=timezone.utc), + datetime(2017, 11, 14, 9, 41, 0, tzinfo=timezone.utc), + datetime(2017, 11, 14, 14, 20, 0, tzinfo=timezone.utc), + datetime(2017, 11, 15, 3, 40, 0, tzinfo=timezone.utc), ], utc=True ), - 'close_date': to_datetime([Arrow(2017, 11, 13, 16, 40, 0).datetime, - Arrow(2017, 11, 14, 10, 41, 0).datetime, - Arrow(2017, 11, 14, 15, 25, 0).datetime, - Arrow(2017, 11, 15, 3, 55, 0).datetime, + 'close_date': to_datetime([datetime(2017, 11, 13, 16, 40, 0, tzinfo=timezone.utc), + datetime(2017, 11, 14, 10, 41, 0, tzinfo=timezone.utc), + datetime(2017, 11, 14, 15, 25, 0, tzinfo=timezone.utc), + datetime(2017, 11, 15, 3, 55, 0, tzinfo=timezone.utc), ], utc=True) }) trades1 = extract_trades_of_period(data, trades) # First and last trade are dropped as they are out of range assert len(trades1) == 2 - assert trades1.iloc[0].open_date == Arrow(2017, 11, 14, 9, 41, 0).datetime - assert trades1.iloc[0].close_date == Arrow(2017, 11, 14, 10, 41, 0).datetime - assert trades1.iloc[-1].open_date == Arrow(2017, 11, 14, 14, 20, 0).datetime - assert trades1.iloc[-1].close_date == Arrow(2017, 11, 14, 15, 25, 0).datetime + assert trades1.iloc[0].open_date == datetime(2017, 11, 14, 9, 41, 0, tzinfo=timezone.utc) + assert trades1.iloc[0].close_date == datetime(2017, 11, 14, 10, 41, 0, tzinfo=timezone.utc) + assert trades1.iloc[-1].open_date == datetime(2017, 11, 14, 14, 20, 0, tzinfo=timezone.utc) + assert trades1.iloc[-1].close_date == datetime(2017, 11, 14, 15, 25, 0, tzinfo=timezone.utc) def test_analyze_trade_parallelism(testdatadir): @@ -420,7 +421,7 @@ def test_calculate_max_drawdown2(): -0.025782, 0.010400, 0.012374, 0.012467, 0.114741, 0.010303, 0.010088, -0.033961, 0.010680, 0.010886, -0.029274, 0.011178, 0.010693, 0.010711] - dates = [Arrow(2020, 1, 1).shift(days=i) for i in range(len(values))] + dates = [dt_utc(2020, 1, 1) + timedelta(days=i) for i in range(len(values))] df = DataFrame(zip(values, dates), columns=['profit', 'open_date']) # sort by profit and reset index df = df.sort_values('profit').reset_index(drop=True) @@ -454,8 +455,8 @@ def test_calculate_max_drawdown_abs(profits, relative, highd, lowd, result, resu [1000, 500, 1000, 11000, 10000] # absolute results [1000, 50%, 0%, 0%, ~9%] # Relative drawdowns """ - init_date = Arrow(2020, 1, 1) - dates = [init_date.shift(days=i) for i in range(len(profits))] + init_date = datetime(2020, 1, 1, tzinfo=timezone.utc) + dates = [init_date + timedelta(days=i) for i in range(len(profits))] df = DataFrame(zip(profits, dates), columns=['profit_abs', 'open_date']) # sort by profit and reset index df = df.sort_values('profit_abs').reset_index(drop=True) @@ -467,8 +468,8 @@ def test_calculate_max_drawdown_abs(profits, relative, highd, lowd, result, resu assert isinstance(drawdown, float) assert isinstance(drawdown_rel, float) - assert hdate == init_date.shift(days=highd) - assert ldate == init_date.shift(days=lowd) + assert hdate == init_date + timedelta(days=highd) + assert ldate == init_date + timedelta(days=lowd) # High must be before low assert hdate < ldate diff --git a/tests/data/test_history.py b/tests/data/test_history.py index 24ad8bcc9..e397c97c1 100644 --- a/tests/data/test_history.py +++ b/tests/data/test_history.py @@ -6,7 +6,6 @@ from pathlib import Path from shutil import copyfile from unittest.mock import MagicMock, PropertyMock -import arrow import pytest from pandas import DataFrame from pandas.testing import assert_frame_equal @@ -26,6 +25,7 @@ from freqtrade.enums import CandleType from freqtrade.exchange import timeframe_to_minutes from freqtrade.misc import file_dump_json from freqtrade.resolvers import StrategyResolver +from freqtrade.util import dt_utc from tests.conftest import (CURRENT_TEST_STRATEGY, EXMS, get_patched_exchange, log_has, log_has_re, patch_exchange) @@ -198,7 +198,6 @@ def test_load_cached_data_for_updating(mocker, testdatadir) -> None: fill_missing=False, drop_incomplete=False) # now = last cached item + 1 hour now_ts = test_data[-1][0] / 1000 + 60 * 60 - mocker.patch('arrow.utcnow', return_value=arrow.get(now_ts)) # timeframe starts earlier than the cached data # should fully update data @@ -353,10 +352,10 @@ def test_download_backtesting_data_exception(mocker, caplog, default_conf, tmpdi def test_load_partial_missing(testdatadir, caplog) -> None: # Make sure we start fresh - test missing data at start - start = arrow.get('2018-01-01T00:00:00') - end = arrow.get('2018-01-11T00:00:00') + start = dt_utc(2018, 1, 1) + end = dt_utc(2018, 1, 11) data = load_data(testdatadir, '5m', ['UNITTEST/BTC'], startup_candles=20, - timerange=TimeRange('date', 'date', start.int_timestamp, end.int_timestamp)) + timerange=TimeRange('date', 'date', start.timestamp(), end.timestamp())) assert log_has( 'Using indicator startup period: 20 ...', caplog ) @@ -369,16 +368,16 @@ def test_load_partial_missing(testdatadir, caplog) -> None: caplog) # Make sure we start fresh - test missing data at end caplog.clear() - start = arrow.get('2018-01-10T00:00:00') - end = arrow.get('2018-02-20T00:00:00') + start = dt_utc(2018, 1, 10) + end = dt_utc(2018, 2, 20) data = load_data(datadir=testdatadir, timeframe='5m', pairs=['UNITTEST/BTC'], - timerange=TimeRange('date', 'date', start.int_timestamp, end.int_timestamp)) + timerange=TimeRange('date', 'date', start.timestamp(), end.timestamp())) # timedifference in 5 minutes td = ((end - start).total_seconds() // 60 // 5) + 1 assert td != len(data['UNITTEST/BTC']) # Shift endtime with +5 - end_real = arrow.get(data['UNITTEST/BTC'].iloc[-1, 0]) + end_real = data['UNITTEST/BTC'].iloc[-1, 0].to_pydatetime() assert log_has(f'UNITTEST/BTC, spot, 5m, ' f'data ends at {end_real.strftime(DATETIME_PRINT_FORMAT)}', caplog) diff --git a/tests/edge/test_edge.py b/tests/edge/test_edge.py index be0346b78..4829dd035 100644 --- a/tests/edge/test_edge.py +++ b/tests/edge/test_edge.py @@ -3,9 +3,9 @@ import logging import math +from datetime import timedelta from unittest.mock import MagicMock -import arrow import numpy as np import pytest from pandas import DataFrame @@ -14,6 +14,7 @@ from freqtrade.data.converter import ohlcv_to_dataframe from freqtrade.edge import Edge, PairInfo from freqtrade.enums import ExitType from freqtrade.exceptions import OperationalException +from freqtrade.util.datetime_helpers import dt_ts, dt_utc from tests.conftest import EXMS, get_patched_freqtradebot, log_has from tests.optimize import (BTContainer, BTrade, _build_backtest_dataframe, _get_frame_time_from_offset) @@ -27,7 +28,7 @@ from tests.optimize import (BTContainer, BTrade, _build_backtest_dataframe, # 5) Stoploss and sell are hit. should sell on stoploss #################################################################### -tests_start_time = arrow.get(2018, 10, 3) +tests_start_time = dt_utc(2018, 10, 3) timeframe_in_minute = 60 # End helper functions @@ -220,7 +221,7 @@ def test_edge_heartbeat_calculate(mocker, edge_conf): heartbeat = edge_conf['edge']['process_throttle_secs'] # should not recalculate if heartbeat not reached - edge._last_updated = arrow.utcnow().int_timestamp - heartbeat + 1 + edge._last_updated = dt_ts() - heartbeat + 1 assert edge.calculate(edge_conf['exchange']['pair_whitelist']) is False @@ -232,7 +233,7 @@ def mocked_load_data(datadir, pairs=[], timeframe='0m', NEOBTC = [ [ - tests_start_time.shift(minutes=(x * timeframe_in_minute)).int_timestamp * 1000, + dt_ts(tests_start_time + timedelta(minutes=(x * timeframe_in_minute))), math.sin(x * hz) / 1000 + base, math.sin(x * hz) / 1000 + base + 0.0001, math.sin(x * hz) / 1000 + base - 0.0001, @@ -244,7 +245,7 @@ def mocked_load_data(datadir, pairs=[], timeframe='0m', base = 0.002 LTCBTC = [ [ - tests_start_time.shift(minutes=(x * timeframe_in_minute)).int_timestamp * 1000, + dt_ts(tests_start_time + timedelta(minutes=(x * timeframe_in_minute))), math.sin(x * hz) / 1000 + base, math.sin(x * hz) / 1000 + base + 0.0001, math.sin(x * hz) / 1000 + base - 0.0001, @@ -268,7 +269,7 @@ def test_edge_process_downloaded_data(mocker, edge_conf): assert edge.calculate(edge_conf['exchange']['pair_whitelist']) assert len(edge._cached_pairs) == 2 - assert edge._last_updated <= arrow.utcnow().int_timestamp + 2 + assert edge._last_updated <= dt_ts() + 2 def test_edge_process_no_data(mocker, edge_conf, caplog): diff --git a/tests/exchange/test_binance.py b/tests/exchange/test_binance.py index d44dae00d..9018d2db9 100644 --- a/tests/exchange/test_binance.py +++ b/tests/exchange/test_binance.py @@ -514,7 +514,7 @@ def test_fill_leverage_tiers_binance_dryrun(default_conf, mocker, leverage_tiers def test_additional_exchange_init_binance(default_conf, mocker): api_mock = MagicMock() - api_mock.fapiPrivateGetPositionsideDual = MagicMock(return_value={"dualSidePosition": True}) + api_mock.fapiPrivateGetPositionSideDual = MagicMock(return_value={"dualSidePosition": True}) api_mock.fapiPrivateGetMultiAssetsMargin = MagicMock(return_value={"multiAssetsMargin": True}) default_conf['dry_run'] = False default_conf['trading_mode'] = TradingMode.FUTURES @@ -522,12 +522,12 @@ def test_additional_exchange_init_binance(default_conf, mocker): with pytest.raises(OperationalException, match=r"Hedge Mode is not supported.*\nMulti-Asset Mode is not supported.*"): get_patched_exchange(mocker, default_conf, id="binance", api_mock=api_mock) - api_mock.fapiPrivateGetPositionsideDual = MagicMock(return_value={"dualSidePosition": False}) + api_mock.fapiPrivateGetPositionSideDual = MagicMock(return_value={"dualSidePosition": False}) api_mock.fapiPrivateGetMultiAssetsMargin = MagicMock(return_value={"multiAssetsMargin": False}) exchange = get_patched_exchange(mocker, default_conf, id="binance", api_mock=api_mock) assert exchange ccxt_exceptionhandlers(mocker, default_conf, api_mock, 'binance', - "additional_exchange_init", "fapiPrivateGetPositionsideDual") + "additional_exchange_init", "fapiPrivateGetPositionSideDual") def test__set_leverage_binance(mocker, default_conf): diff --git a/tests/exchange/test_ccxt_compat.py b/tests/exchange/test_ccxt_compat.py index 6f5987202..404b51d10 100644 --- a/tests/exchange/test_ccxt_compat.py +++ b/tests/exchange/test_ccxt_compat.py @@ -43,6 +43,10 @@ EXCHANGES = { 'hasQuoteVolumeFutures': True, 'leverage_tiers_public': False, 'leverage_in_spot_market': False, + 'private_methods': [ + 'fapiPrivateGetPositionSideDual', + 'fapiPrivateGetMultiAssetsMargin' + ], 'sample_order': [{ "symbol": "SOLUSDT", "orderId": 3551312894, @@ -221,11 +225,13 @@ EXCHANGES = { 'hasQuoteVolumeFutures': False, 'leverage_tiers_public': True, 'leverage_in_spot_market': True, + 'private_methods': ['fetch_accounts'], }, 'bybit': { 'pair': 'BTC/USDT', 'stake_currency': 'USDT', 'hasQuoteVolume': True, + 'use_ci_proxy': True, 'timeframe': '1h', 'futures_pair': 'BTC/USDT:USDT', 'futures': True, @@ -755,3 +761,8 @@ class TestCCXTExchange(): max_stake_amount = futures.get_max_pair_stake_amount(futures_pair, 40000) assert (isinstance(max_stake_amount, float)) assert max_stake_amount >= 0.0 + + def test_private_method_presence(self, exchange: EXCHANGE_FIXTURE_TYPE): + exch, exchangename = exchange + for method in EXCHANGES[exchangename].get('private_methods', []): + assert hasattr(exch._api, method) diff --git a/tests/exchange/test_exchange.py b/tests/exchange/test_exchange.py index 4c7a7dcc8..ef70c8ba1 100644 --- a/tests/exchange/test_exchange.py +++ b/tests/exchange/test_exchange.py @@ -5,7 +5,6 @@ from datetime import datetime, timedelta, timezone from random import randint from unittest.mock import MagicMock, Mock, PropertyMock, patch -import arrow import ccxt import pytest from ccxt import DECIMAL_PLACES, ROUND, ROUND_UP, TICK_SIZE, TRUNCATE @@ -23,6 +22,7 @@ from freqtrade.exchange.common import (API_FETCH_ORDER_RETRY_COUNT, API_RETRY_CO calculate_backoff, remove_exchange_credentials) from freqtrade.exchange.exchange import amount_to_contract_precision from freqtrade.resolvers.exchange_resolver import ExchangeResolver +from freqtrade.util import dt_now, dt_ts from tests.conftest import (EXMS, generate_test_data_raw, get_mock_coro, get_patched_exchange, log_has, log_has_re, num_log_has_re) @@ -644,7 +644,7 @@ def test_reload_markets(default_conf, mocker, caplog): exchange = get_patched_exchange(mocker, default_conf, api_mock, id="binance", mock_markets=False) exchange._load_async_markets = MagicMock() - exchange._last_markets_refresh = arrow.utcnow().int_timestamp + exchange._last_markets_refresh = dt_ts() assert exchange.markets == initial_markets @@ -655,7 +655,7 @@ def test_reload_markets(default_conf, mocker, caplog): api_mock.load_markets = MagicMock(return_value=updated_markets) # more than 10 minutes have passed, reload is executed - exchange._last_markets_refresh = arrow.utcnow().int_timestamp - 15 * 60 + exchange._last_markets_refresh = dt_ts(dt_now() - timedelta(minutes=15)) exchange.reload_markets() assert exchange.markets == updated_markets assert exchange._load_async_markets.call_count == 1 @@ -2076,7 +2076,7 @@ def test_get_historic_ohlcv(default_conf, mocker, caplog, exchange_name, candle_ exchange = get_patched_exchange(mocker, default_conf, id=exchange_name) ohlcv = [ [ - arrow.utcnow().int_timestamp * 1000, # unix timestamp ms + dt_ts(), # unix timestamp ms 1, # open 2, # high 3, # low @@ -2096,7 +2096,7 @@ def test_get_historic_ohlcv(default_conf, mocker, caplog, exchange_name, candle_ ret = exchange.get_historic_ohlcv( pair, "5m", - int((arrow.utcnow().int_timestamp - since) * 1000), + dt_ts(dt_now() - timedelta(seconds=since)), candle_type=candle_type ) @@ -2114,7 +2114,7 @@ def test_get_historic_ohlcv(default_conf, mocker, caplog, exchange_name, candle_ ret = exchange.get_historic_ohlcv( pair, "5m", - int((arrow.utcnow().int_timestamp - since) * 1000), + dt_ts(dt_now() - timedelta(seconds=since)), candle_type=candle_type ) assert log_has_re(r"Async code raised an exception: .*", caplog) @@ -2166,7 +2166,7 @@ async def test__async_get_historic_ohlcv(default_conf, mocker, caplog, exchange_ def test_refresh_latest_ohlcv(mocker, default_conf, caplog, candle_type) -> None: ohlcv = [ [ - (arrow.utcnow().shift(minutes=-5).int_timestamp) * 1000, # unix timestamp ms + dt_ts(dt_now() - timedelta(minutes=5)), # unix timestamp ms 1, # open 2, # high 3, # low @@ -2174,7 +2174,7 @@ def test_refresh_latest_ohlcv(mocker, default_conf, caplog, candle_type) -> None 5, # volume (in quote currency) ], [ - arrow.utcnow().int_timestamp * 1000, # unix timestamp ms + dt_ts(), # unix timestamp ms 3, # open 1, # high 4, # low @@ -2364,7 +2364,7 @@ def test_refresh_latest_ohlcv_cache(mocker, default_conf, candle_type, time_mach async def test__async_get_candle_history(default_conf, mocker, caplog, exchange_name): ohlcv = [ [ - arrow.utcnow().int_timestamp * 1000, # unix timestamp ms + dt_ts(), # unix timestamp ms 1, # open 2, # high 3, # low @@ -2401,7 +2401,7 @@ async def test__async_get_candle_history(default_conf, mocker, caplog, exchange_ api_mock.fetch_ohlcv = MagicMock(side_effect=ccxt.BaseError("Unknown error")) exchange = get_patched_exchange(mocker, default_conf, api_mock, id=exchange_name) await exchange._async_get_candle_history(pair, "5m", CandleType.SPOT, - (arrow.utcnow().int_timestamp - 2000) * 1000) + dt_ts(dt_now() - timedelta(seconds=2000))) exchange.close() @@ -2410,7 +2410,7 @@ async def test__async_get_candle_history(default_conf, mocker, caplog, exchange_ api_mock.fetch_ohlcv = MagicMock(side_effect=ccxt.NotSupported("Not supported")) exchange = get_patched_exchange(mocker, default_conf, api_mock, id=exchange_name) await exchange._async_get_candle_history(pair, "5m", CandleType.SPOT, - (arrow.utcnow().int_timestamp - 2000) * 1000) + dt_ts(dt_now() - timedelta(seconds=2000))) exchange.close() @@ -2433,7 +2433,7 @@ async def test__async_kucoin_get_candle_history(default_conf, mocker, caplog): with pytest.raises(DDosProtection, match=r'429 Too Many Requests'): await exchange._async_get_candle_history( "ETH/BTC", "5m", CandleType.SPOT, - since_ms=(arrow.utcnow().int_timestamp - 2000) * 1000, count=3) + since_ms=dt_ts(dt_now() - timedelta(seconds=2000)), count=3) assert num_log_has_re(msg, caplog) == 3 caplog.clear() @@ -2450,7 +2450,7 @@ async def test__async_kucoin_get_candle_history(default_conf, mocker, caplog): with pytest.raises(DDosProtection, match=r'429 Too Many Requests'): await exchange._async_get_candle_history( "ETH/BTC", "5m", CandleType.SPOT, - (arrow.utcnow().int_timestamp - 2000) * 1000, count=3) + dt_ts(dt_now() - timedelta(seconds=2000)), count=3) # Expect the "returned exception" message 12 times (4 retries * 3 (loop)) assert num_log_has_re(msg, caplog) == 12 assert num_log_has_re(msg2, caplog) == 9 @@ -2908,14 +2908,14 @@ async def test__async_fetch_trades(default_conf, mocker, caplog, exchange_name, with pytest.raises(OperationalException, match=r'Could not fetch trade data*'): api_mock.fetch_trades = MagicMock(side_effect=ccxt.BaseError("Unknown error")) exchange = get_patched_exchange(mocker, default_conf, api_mock, id=exchange_name) - await exchange._async_fetch_trades(pair, since=(arrow.utcnow().int_timestamp - 2000) * 1000) + await exchange._async_fetch_trades(pair, since=dt_ts(dt_now() - timedelta(seconds=2000))) exchange.close() with pytest.raises(OperationalException, match=r'Exchange.* does not support fetching ' r'historical trade data\..*'): api_mock.fetch_trades = MagicMock(side_effect=ccxt.NotSupported("Not supported")) exchange = get_patched_exchange(mocker, default_conf, api_mock, id=exchange_name) - await exchange._async_fetch_trades(pair, since=(arrow.utcnow().int_timestamp - 2000) * 1000) + await exchange._async_fetch_trades(pair, since=dt_ts(dt_now() - timedelta(seconds=2000))) exchange.close() diff --git a/tests/optimize/__init__.py b/tests/optimize/__init__.py index a3dd59004..b95764ba5 100644 --- a/tests/optimize/__init__.py +++ b/tests/optimize/__init__.py @@ -1,13 +1,14 @@ +from datetime import timedelta from typing import Dict, List, NamedTuple, Optional -import arrow from pandas import DataFrame from freqtrade.enums import ExitType from freqtrade.exchange import timeframe_to_minutes +from freqtrade.util.datetime_helpers import dt_utc -tests_start_time = arrow.get(2018, 10, 3) +tests_start_time = dt_utc(2018, 10, 3) tests_timeframe = '1h' @@ -46,7 +47,7 @@ class BTContainer(NamedTuple): def _get_frame_time_from_offset(offset): minutes = offset * timeframe_to_minutes(tests_timeframe) - return tests_start_time.shift(minutes=minutes).datetime + return tests_start_time + timedelta(minutes=minutes) def _build_backtest_dataframe(data): diff --git a/tests/optimize/test_backtesting.py b/tests/optimize/test_backtesting.py index a9e87347c..bef942b43 100644 --- a/tests/optimize/test_backtesting.py +++ b/tests/optimize/test_backtesting.py @@ -9,7 +9,6 @@ from unittest.mock import MagicMock, PropertyMock import numpy as np import pandas as pd import pytest -from arrow import Arrow from freqtrade import constants from freqtrade.commands.optimize_commands import setup_optimize_configuration, start_backtesting @@ -26,6 +25,7 @@ from freqtrade.optimize.backtest_caching import get_strategy_run_id from freqtrade.optimize.backtesting import Backtesting from freqtrade.persistence import LocalTrade, Trade from freqtrade.resolvers import StrategyResolver +from freqtrade.util.datetime_helpers import dt_utc from tests.conftest import (CURRENT_TEST_STRATEGY, EXMS, get_args, log_has, log_has_re, patch_exchange, patched_configuration_load_config_file) @@ -346,7 +346,7 @@ def test_backtest_abort(default_conf, mocker, testdatadir) -> None: def test_backtesting_start(default_conf, mocker, caplog) -> None: def get_timerange(input1): - return Arrow(2017, 11, 14, 21, 17), Arrow(2017, 11, 14, 22, 59) + return dt_utc(2017, 11, 14, 21, 17), dt_utc(2017, 11, 14, 22, 59) mocker.patch('freqtrade.data.history.get_timerange', get_timerange) patch_exchange(mocker) @@ -385,7 +385,7 @@ def test_backtesting_start(default_conf, mocker, caplog) -> None: def test_backtesting_start_no_data(default_conf, mocker, caplog, testdatadir) -> None: def get_timerange(input1): - return Arrow(2017, 11, 14, 21, 17), Arrow(2017, 11, 14, 22, 59) + return dt_utc(2017, 11, 14, 21, 17), dt_utc(2017, 11, 14, 22, 59) mocker.patch('freqtrade.data.history.history_utils.load_pair_history', MagicMock(return_value=pd.DataFrame())) @@ -710,11 +710,11 @@ def test_backtest_one(default_conf, fee, mocker, testdatadir) -> None: 'stake_amount': [0.001, 0.001], 'max_stake_amount': [0.001, 0.001], 'amount': [0.00957442, 0.0097064], - 'open_date': pd.to_datetime([Arrow(2018, 1, 29, 18, 40, 0).datetime, - Arrow(2018, 1, 30, 3, 30, 0).datetime], utc=True + 'open_date': pd.to_datetime([dt_utc(2018, 1, 29, 18, 40, 0), + dt_utc(2018, 1, 30, 3, 30, 0)], utc=True ), - 'close_date': pd.to_datetime([Arrow(2018, 1, 29, 22, 35, 0).datetime, - Arrow(2018, 1, 30, 4, 10, 0).datetime], utc=True), + 'close_date': pd.to_datetime([dt_utc(2018, 1, 29, 22, 35, 0), + dt_utc(2018, 1, 30, 4, 10, 0)], utc=True), 'open_rate': [0.104445, 0.10302485], 'close_rate': [0.104969, 0.103541], 'fee_open': [0.0025, 0.0025], diff --git a/tests/optimize/test_backtesting_adjust_position.py b/tests/optimize/test_backtesting_adjust_position.py index 0d57ff89a..ce26e836e 100644 --- a/tests/optimize/test_backtesting_adjust_position.py +++ b/tests/optimize/test_backtesting_adjust_position.py @@ -5,13 +5,13 @@ from unittest.mock import MagicMock import pandas as pd import pytest -from arrow import Arrow from freqtrade.configuration import TimeRange from freqtrade.data import history from freqtrade.data.history import get_timerange from freqtrade.enums import ExitType, TradingMode from freqtrade.optimize.backtesting import Backtesting +from freqtrade.util.datetime_helpers import dt_utc from tests.conftest import EXMS, patch_exchange @@ -52,11 +52,11 @@ def test_backtest_position_adjustment(default_conf, fee, mocker, testdatadir) -> 'stake_amount': [500.0, 100.0], 'max_stake_amount': [500.0, 100], 'amount': [4806.87657523, 970.63960782], - 'open_date': pd.to_datetime([Arrow(2018, 1, 29, 18, 40, 0).datetime, - Arrow(2018, 1, 30, 3, 30, 0).datetime], utc=True + 'open_date': pd.to_datetime([dt_utc(2018, 1, 29, 18, 40, 0), + dt_utc(2018, 1, 30, 3, 30, 0)], utc=True ), - 'close_date': pd.to_datetime([Arrow(2018, 1, 29, 22, 00, 0).datetime, - Arrow(2018, 1, 30, 4, 10, 0).datetime], utc=True), + 'close_date': pd.to_datetime([dt_utc(2018, 1, 29, 22, 00, 0), + dt_utc(2018, 1, 30, 4, 10, 0)], utc=True), 'open_rate': [0.10401764894444211, 0.10302485], 'close_rate': [0.10453904066847439, 0.103541], 'fee_open': [0.0025, 0.0025], diff --git a/tests/optimize/test_hyperopt.py b/tests/optimize/test_hyperopt.py index 786720030..ed5eeafd6 100644 --- a/tests/optimize/test_hyperopt.py +++ b/tests/optimize/test_hyperopt.py @@ -6,7 +6,6 @@ from unittest.mock import ANY, MagicMock, PropertyMock import pandas as pd import pytest -from arrow import Arrow from filelock import Timeout from skopt.space import Integer @@ -20,6 +19,7 @@ from freqtrade.optimize.hyperopt_tools import HyperoptTools from freqtrade.optimize.optimize_reports import generate_strategy_stats from freqtrade.optimize.space import SKDecimal from freqtrade.strategy import IntParameter +from freqtrade.util import dt_utc from tests.conftest import (CURRENT_TEST_STRATEGY, EXMS, get_args, get_markets, log_has, log_has_re, patch_exchange, patched_configuration_load_config_file) @@ -349,14 +349,14 @@ def test_hyperopt_format_results(hyperopt): "UNITTEST/BTC", "UNITTEST/BTC"], "profit_ratio": [0.003312, 0.010801, 0.013803, 0.002780], "profit_abs": [0.000003, 0.000011, 0.000014, 0.000003], - "open_date": [Arrow(2017, 11, 14, 19, 32, 00).datetime, - Arrow(2017, 11, 14, 21, 36, 00).datetime, - Arrow(2017, 11, 14, 22, 12, 00).datetime, - Arrow(2017, 11, 14, 22, 44, 00).datetime], - "close_date": [Arrow(2017, 11, 14, 21, 35, 00).datetime, - Arrow(2017, 11, 14, 22, 10, 00).datetime, - Arrow(2017, 11, 14, 22, 43, 00).datetime, - Arrow(2017, 11, 14, 22, 58, 00).datetime], + "open_date": [dt_utc(2017, 11, 14, 19, 32, 00), + dt_utc(2017, 11, 14, 21, 36, 00), + dt_utc(2017, 11, 14, 22, 12, 00), + dt_utc(2017, 11, 14, 22, 44, 00)], + "close_date": [dt_utc(2017, 11, 14, 21, 35, 00), + dt_utc(2017, 11, 14, 22, 10, 00), + dt_utc(2017, 11, 14, 22, 43, 00), + dt_utc(2017, 11, 14, 22, 58, 00)], "open_rate": [0.002543, 0.003003, 0.003089, 0.003214], "close_rate": [0.002546, 0.003014, 0.003103, 0.003217], "trade_duration": [123, 34, 31, 14], @@ -379,8 +379,8 @@ def test_hyperopt_format_results(hyperopt): 'backtest_end_time': 1619718665, } results_metrics = generate_strategy_stats(['XRP/BTC'], '', bt_result, - Arrow(2017, 11, 14, 19, 32, 00), - Arrow(2017, 12, 14, 19, 32, 00), market_change=0) + dt_utc(2017, 11, 14, 19, 32, 00), + dt_utc(2017, 12, 14, 19, 32, 00), market_change=0) results_explanation = HyperoptTools.format_results_explanation_string(results_metrics, 'BTC') total_profit = results_metrics['profit_total_abs'] @@ -423,14 +423,14 @@ def test_generate_optimizer(mocker, hyperopt_conf) -> None: "UNITTEST/BTC", "UNITTEST/BTC"], "profit_ratio": [0.003312, 0.010801, 0.013803, 0.002780], "profit_abs": [0.000003, 0.000011, 0.000014, 0.000003], - "open_date": [Arrow(2017, 11, 14, 19, 32, 00).datetime, - Arrow(2017, 11, 14, 21, 36, 00).datetime, - Arrow(2017, 11, 14, 22, 12, 00).datetime, - Arrow(2017, 11, 14, 22, 44, 00).datetime], - "close_date": [Arrow(2017, 11, 14, 21, 35, 00).datetime, - Arrow(2017, 11, 14, 22, 10, 00).datetime, - Arrow(2017, 11, 14, 22, 43, 00).datetime, - Arrow(2017, 11, 14, 22, 58, 00).datetime], + "open_date": [dt_utc(2017, 11, 14, 19, 32, 00), + dt_utc(2017, 11, 14, 21, 36, 00), + dt_utc(2017, 11, 14, 22, 12, 00), + dt_utc(2017, 11, 14, 22, 44, 00)], + "close_date": [dt_utc(2017, 11, 14, 21, 35, 00), + dt_utc(2017, 11, 14, 22, 10, 00), + dt_utc(2017, 11, 14, 22, 43, 00), + dt_utc(2017, 11, 14, 22, 58, 00)], "open_rate": [0.002543, 0.003003, 0.003089, 0.003214], "close_rate": [0.002546, 0.003014, 0.003103, 0.003217], "trade_duration": [123, 34, 31, 14], @@ -453,7 +453,7 @@ def test_generate_optimizer(mocker, hyperopt_conf) -> None: mocker.patch('freqtrade.optimize.hyperopt.Backtesting.backtest', return_value=backtest_result) mocker.patch('freqtrade.optimize.hyperopt.get_timerange', - return_value=(Arrow(2017, 12, 10), Arrow(2017, 12, 13))) + return_value=(dt_utc(2017, 12, 10), dt_utc(2017, 12, 13))) patch_exchange(mocker) mocker.patch.object(Path, 'open') mocker.patch('freqtrade.configuration.config_validation.validate_config_schema') @@ -513,8 +513,8 @@ def test_generate_optimizer(mocker, hyperopt_conf) -> None: } hyperopt = Hyperopt(hyperopt_conf) - hyperopt.min_date = Arrow(2017, 12, 10) - hyperopt.max_date = Arrow(2017, 12, 13) + hyperopt.min_date = dt_utc(2017, 12, 10) + hyperopt.max_date = dt_utc(2017, 12, 13) hyperopt.init_spaces() generate_optimizer_value = hyperopt.generate_optimizer(list(optimizer_param.values())) assert generate_optimizer_value == response_expected diff --git a/tests/optimize/test_optimize_reports.py b/tests/optimize/test_optimize_reports.py index 4e3803f17..82e8a46fb 100644 --- a/tests/optimize/test_optimize_reports.py +++ b/tests/optimize/test_optimize_reports.py @@ -6,7 +6,6 @@ from shutil import copyfile import joblib import pandas as pd import pytest -from arrow import Arrow from freqtrade.configuration import TimeRange from freqtrade.constants import BACKTEST_BREAKDOWNS, DATETIME_PRINT_FORMAT, LAST_BT_RESULT_FN @@ -25,6 +24,8 @@ from freqtrade.optimize.optimize_reports import (_get_resample_from_period, gene store_backtest_stats, text_table_bt_results, text_table_exit_reason, text_table_strategy) from freqtrade.resolvers.strategy_resolver import StrategyResolver +from freqtrade.util import dt_ts +from freqtrade.util.datetime_helpers import dt_from_ts, dt_utc from tests.conftest import CURRENT_TEST_STRATEGY from tests.data.test_history import _clean_test_file @@ -80,14 +81,14 @@ def test_generate_backtest_stats(default_conf, testdatadir, tmpdir): "UNITTEST/BTC", "UNITTEST/BTC"], "profit_ratio": [0.003312, 0.010801, 0.013803, 0.002780], "profit_abs": [0.000003, 0.000011, 0.000014, 0.000003], - "open_date": [Arrow(2017, 11, 14, 19, 32, 00).datetime, - Arrow(2017, 11, 14, 21, 36, 00).datetime, - Arrow(2017, 11, 14, 22, 12, 00).datetime, - Arrow(2017, 11, 14, 22, 44, 00).datetime], - "close_date": [Arrow(2017, 11, 14, 21, 35, 00).datetime, - Arrow(2017, 11, 14, 22, 10, 00).datetime, - Arrow(2017, 11, 14, 22, 43, 00).datetime, - Arrow(2017, 11, 14, 22, 58, 00).datetime], + "open_date": [dt_utc(2017, 11, 14, 19, 32, 00), + dt_utc(2017, 11, 14, 21, 36, 00), + dt_utc(2017, 11, 14, 22, 12, 00), + dt_utc(2017, 11, 14, 22, 44, 00)], + "close_date": [dt_utc(2017, 11, 14, 21, 35, 00), + dt_utc(2017, 11, 14, 22, 10, 00), + dt_utc(2017, 11, 14, 22, 43, 00), + dt_utc(2017, 11, 14, 22, 58, 00)], "open_rate": [0.002543, 0.003003, 0.003089, 0.003214], "close_rate": [0.002546, 0.003014, 0.003103, 0.003217], "trade_duration": [123, 34, 31, 14], @@ -106,14 +107,14 @@ def test_generate_backtest_stats(default_conf, testdatadir, tmpdir): 'canceled_trade_entries': 0, 'canceled_entry_orders': 0, 'replaced_entry_orders': 0, - 'backtest_start_time': Arrow.utcnow().int_timestamp, - 'backtest_end_time': Arrow.utcnow().int_timestamp, + 'backtest_start_time': dt_ts() // 1000, + 'backtest_end_time': dt_ts() // 1000, 'run_id': '123', } } timerange = TimeRange.parse_timerange('1510688220-1510700340') - min_date = Arrow.fromtimestamp(1510688220) - max_date = Arrow.fromtimestamp(1510700340) + min_date = dt_from_ts(1510688220) + max_date = dt_from_ts(1510700340) btdata = history.load_data(testdatadir, '1m', ['UNITTEST/BTC'], timerange=timerange, fill_up_missing=True) @@ -135,14 +136,14 @@ def test_generate_backtest_stats(default_conf, testdatadir, tmpdir): {"pair": ["UNITTEST/BTC", "UNITTEST/BTC", "UNITTEST/BTC", "UNITTEST/BTC"], "profit_ratio": [0.003312, 0.010801, -0.013803, 0.002780], "profit_abs": [0.000003, 0.000011, -0.000014, 0.000003], - "open_date": [Arrow(2017, 11, 14, 19, 32, 00).datetime, - Arrow(2017, 11, 14, 21, 36, 00).datetime, - Arrow(2017, 11, 14, 22, 12, 00).datetime, - Arrow(2017, 11, 14, 22, 44, 00).datetime], - "close_date": [Arrow(2017, 11, 14, 21, 35, 00).datetime, - Arrow(2017, 11, 14, 22, 10, 00).datetime, - Arrow(2017, 11, 14, 22, 43, 00).datetime, - Arrow(2017, 11, 14, 22, 58, 00).datetime], + "open_date": [dt_utc(2017, 11, 14, 19, 32, 00), + dt_utc(2017, 11, 14, 21, 36, 00), + dt_utc(2017, 11, 14, 22, 12, 00), + dt_utc(2017, 11, 14, 22, 44, 00)], + "close_date": [dt_utc(2017, 11, 14, 21, 35, 00), + dt_utc(2017, 11, 14, 22, 10, 00), + dt_utc(2017, 11, 14, 22, 43, 00), + dt_utc(2017, 11, 14, 22, 58, 00)], "open_rate": [0.002543, 0.003003, 0.003089, 0.003214], "close_rate": [0.002546, 0.003014, 0.0032903, 0.003217], "trade_duration": [123, 34, 31, 14], @@ -161,8 +162,8 @@ def test_generate_backtest_stats(default_conf, testdatadir, tmpdir): 'canceled_trade_entries': 0, 'canceled_entry_orders': 0, 'replaced_entry_orders': 0, - 'backtest_start_time': Arrow.utcnow().int_timestamp, - 'backtest_end_time': Arrow.utcnow().int_timestamp, + 'backtest_start_time': dt_ts() // 1000, + 'backtest_end_time': dt_ts() // 1000, 'run_id': '124', } } diff --git a/tests/persistence/test_persistence.py b/tests/persistence/test_persistence.py index 6af629c75..4aa3b1e96 100644 --- a/tests/persistence/test_persistence.py +++ b/tests/persistence/test_persistence.py @@ -2,7 +2,6 @@ from datetime import datetime, timedelta, timezone from types import FunctionType -import arrow import pytest from sqlalchemy import select @@ -10,6 +9,7 @@ from freqtrade.constants import CUSTOM_TAG_MAX_LENGTH, DATETIME_PRINT_FORMAT from freqtrade.enums import TradingMode from freqtrade.exceptions import DependencyException from freqtrade.persistence import LocalTrade, Order, Trade, init_db +from freqtrade.util import dt_now from tests.conftest import create_mock_trades, create_mock_trades_with_leverage, log_has, log_has_re @@ -27,7 +27,7 @@ def test_enter_exit_side(fee, is_short): open_rate=0.01, amount=5, is_open=True, - open_date=arrow.utcnow().datetime, + open_date=dt_now(), fee_open=fee.return_value, fee_close=fee.return_value, exchange='binance', @@ -49,7 +49,7 @@ def test_set_stop_loss_liquidation(fee): open_rate=2.0, amount=30.0, is_open=True, - open_date=arrow.utcnow().datetime, + open_date=dt_now(), fee_open=fee.return_value, fee_close=fee.return_value, exchange='binance', @@ -329,7 +329,7 @@ def test_borrowed(fee, is_short, lev, borrowed, trading_mode): open_rate=2.0, amount=30.0, is_open=True, - open_date=arrow.utcnow().datetime, + open_date=dt_now(), fee_open=fee.return_value, fee_close=fee.return_value, exchange='binance', @@ -428,7 +428,7 @@ def test_update_limit_order(fee, caplog, limit_buy_order_usdt, limit_sell_order_ open_rate=open_rate, amount=30.0, is_open=True, - open_date=arrow.utcnow().datetime, + open_date=dt_now(), fee_open=fee.return_value, fee_close=fee.return_value, exchange='binance', @@ -485,7 +485,7 @@ def test_update_market_order(market_buy_order_usdt, market_sell_order_usdt, fee, is_open=True, fee_open=fee.return_value, fee_close=fee.return_value, - open_date=arrow.utcnow().datetime, + open_date=dt_now(), exchange='binance', trading_mode=margin, leverage=1.0, @@ -635,7 +635,7 @@ def test_trade_close(fee): assert pytest.approx(trade.close_profit) == 0.094513715 assert trade.close_date is not None - new_date = arrow.Arrow(2020, 2, 2, 15, 6, 1).datetime, + new_date = datetime(2020, 2, 2, 15, 6, 1), assert trade.close_date != new_date # Close should NOT update close_date if the trade has been closed already assert trade.is_open is False @@ -1326,7 +1326,7 @@ def test_to_json(fee): amount_requested=123.0, fee_open=fee.return_value, fee_close=fee.return_value, - open_date=arrow.utcnow().shift(hours=-2).datetime, + open_date=dt_now() - timedelta(hours=2), open_rate=0.123, exchange='binance', enter_tag=None, @@ -1411,8 +1411,8 @@ def test_to_json(fee): amount_requested=101.0, fee_open=fee.return_value, fee_close=fee.return_value, - open_date=arrow.utcnow().shift(hours=-2).datetime, - close_date=arrow.utcnow().shift(hours=-1).datetime, + open_date=dt_now() - timedelta(hours=2), + close_date=dt_now() - timedelta(hours=1), open_rate=0.123, close_rate=0.125, enter_tag='buys_signal_001', @@ -1496,7 +1496,7 @@ def test_stoploss_reinitialization(default_conf, fee): pair='ADA/USDT', stake_amount=30.0, fee_open=fee.return_value, - open_date=arrow.utcnow().shift(hours=-2).datetime, + open_date=dt_now() - timedelta(hours=2), amount=30.0, fee_close=fee.return_value, exchange='binance', @@ -1557,7 +1557,7 @@ def test_stoploss_reinitialization_leverage(default_conf, fee): pair='ADA/USDT', stake_amount=30.0, fee_open=fee.return_value, - open_date=arrow.utcnow().shift(hours=-2).datetime, + open_date=dt_now() - timedelta(hours=2), amount=30.0, fee_close=fee.return_value, exchange='binance', @@ -1619,7 +1619,7 @@ def test_stoploss_reinitialization_short(default_conf, fee): pair='ADA/USDT', stake_amount=0.001, fee_open=fee.return_value, - open_date=arrow.utcnow().shift(hours=-2).datetime, + open_date=dt_now() - timedelta(hours=2), amount=10, fee_close=fee.return_value, exchange='binance', @@ -1678,7 +1678,7 @@ def test_update_fee(fee): pair='ADA/USDT', stake_amount=30.0, fee_open=fee.return_value, - open_date=arrow.utcnow().shift(hours=-2).datetime, + open_date=dt_now() - timedelta(hours=2), amount=30.0, fee_close=fee.return_value, exchange='binance', @@ -1717,7 +1717,7 @@ def test_fee_updated(fee): pair='ADA/USDT', stake_amount=30.0, fee_open=fee.return_value, - open_date=arrow.utcnow().shift(hours=-2).datetime, + open_date=dt_now() - timedelta(hours=2), amount=30.0, fee_close=fee.return_value, exchange='binance', @@ -2092,7 +2092,7 @@ def test_recalc_trade_from_orders(fee): trade = Trade( pair='ADA/USDT', stake_amount=o1_cost, - open_date=arrow.utcnow().shift(hours=-2).datetime, + open_date=dt_now() - timedelta(hours=2), amount=o1_amount, fee_open=fee.return_value, fee_close=fee.return_value, @@ -2167,8 +2167,8 @@ def test_recalc_trade_from_orders(fee): filled=o2_amount, remaining=0, cost=o2_cost, - order_date=arrow.utcnow().shift(hours=-1).datetime, - order_filled_date=arrow.utcnow().shift(hours=-1).datetime, + order_date=dt_now() - timedelta(hours=1), + order_filled_date=dt_now() - timedelta(hours=1), ) trade.orders.append(order2) trade.recalc_trade_from_orders() @@ -2201,8 +2201,8 @@ def test_recalc_trade_from_orders(fee): filled=o3_amount, remaining=0, cost=o3_cost, - order_date=arrow.utcnow().shift(hours=-1).datetime, - order_filled_date=arrow.utcnow().shift(hours=-1).datetime, + order_date=dt_now() - timedelta(hours=1), + order_filled_date=dt_now() - timedelta(hours=1), ) trade.orders.append(order3) trade.recalc_trade_from_orders() @@ -2257,7 +2257,7 @@ def test_recalc_trade_from_orders_ignores_bad_orders(fee, is_short): trade = Trade( pair='ADA/USDT', stake_amount=o1_cost, - open_date=arrow.utcnow().shift(hours=-2).datetime, + open_date=dt_now() - timedelta(hours=2), amount=o1_amount, fee_open=fee.return_value, fee_close=fee.return_value, @@ -2309,8 +2309,8 @@ def test_recalc_trade_from_orders_ignores_bad_orders(fee, is_short): filled=o1_amount, remaining=0, cost=o1_cost, - order_date=arrow.utcnow().shift(hours=-1).datetime, - order_filled_date=arrow.utcnow().shift(hours=-1).datetime, + order_date=dt_now() - timedelta(hours=1), + order_filled_date=dt_now() - timedelta(hours=1), ) trade.orders.append(order2) trade.recalc_trade_from_orders() @@ -2337,8 +2337,8 @@ def test_recalc_trade_from_orders_ignores_bad_orders(fee, is_short): filled=0, remaining=4, cost=5, - order_date=arrow.utcnow().shift(hours=-1).datetime, - order_filled_date=arrow.utcnow().shift(hours=-1).datetime, + order_date=dt_now() - timedelta(hours=1), + order_filled_date=dt_now() - timedelta(hours=1), ) trade.orders.append(order3) trade.recalc_trade_from_orders() @@ -2364,8 +2364,8 @@ def test_recalc_trade_from_orders_ignores_bad_orders(fee, is_short): filled=o1_amount, remaining=0, cost=o1_cost, - order_date=arrow.utcnow().shift(hours=-1).datetime, - order_filled_date=arrow.utcnow().shift(hours=-1).datetime, + order_date=dt_now() - timedelta(hours=1), + order_filled_date=dt_now() - timedelta(hours=1), ) trade.orders.append(order4) trade.recalc_trade_from_orders() @@ -2592,7 +2592,7 @@ def test_recalc_trade_from_orders_dca(data) -> None: open_rate=data['orders'][0][0][2], amount=data['orders'][0][0][1], is_open=True, - open_date=arrow.utcnow().datetime, + open_date=dt_now(), fee_open=data['fee'], fee_close=data['fee'], exchange='binance', @@ -2622,8 +2622,8 @@ def test_recalc_trade_from_orders_dca(data) -> None: filled=amount, remaining=0, cost=amount * price, - order_date=arrow.utcnow().shift(hours=-10 + idx).datetime, - order_filled_date=arrow.utcnow().shift(hours=-10 + idx).datetime, + order_date=dt_now() - timedelta(hours=10 + idx), + order_filled_date=dt_now() - timedelta(hours=10 + idx), ) trade.orders.append(order_obj) trade.recalc_trade_from_orders() diff --git a/tests/plugins/test_pairlocks.py b/tests/plugins/test_pairlocks.py index 6b7112f98..6e209df60 100644 --- a/tests/plugins/test_pairlocks.py +++ b/tests/plugins/test_pairlocks.py @@ -1,10 +1,10 @@ from datetime import datetime, timedelta, timezone -import arrow import pytest from freqtrade.persistence import PairLocks from freqtrade.persistence.models import PairLock +from freqtrade.util import dt_now @pytest.mark.parametrize('use_db', (False, True)) @@ -20,20 +20,20 @@ def test_PairLocks(use_db): pair = 'ETH/BTC' assert not PairLocks.is_pair_locked(pair) - PairLocks.lock_pair(pair, arrow.utcnow().shift(minutes=4).datetime) + PairLocks.lock_pair(pair, dt_now() + timedelta(minutes=4)) # ETH/BTC locked for 4 minutes (on both sides) assert PairLocks.is_pair_locked(pair) assert PairLocks.is_pair_locked(pair, side='long') assert PairLocks.is_pair_locked(pair, side='short') pair = 'BNB/BTC' - PairLocks.lock_pair(pair, arrow.utcnow().shift(minutes=4).datetime, side='long') + PairLocks.lock_pair(pair, dt_now() + timedelta(minutes=4), side='long') assert not PairLocks.is_pair_locked(pair) assert PairLocks.is_pair_locked(pair, side='long') assert not PairLocks.is_pair_locked(pair, side='short') pair = 'BNB/USDT' - PairLocks.lock_pair(pair, arrow.utcnow().shift(minutes=4).datetime, side='short') + PairLocks.lock_pair(pair, dt_now() + timedelta(minutes=4), side='short') assert not PairLocks.is_pair_locked(pair) assert not PairLocks.is_pair_locked(pair, side='long') assert PairLocks.is_pair_locked(pair, side='short') @@ -44,7 +44,7 @@ def test_PairLocks(use_db): # Unlocking a pair that's not locked should not raise an error PairLocks.unlock_pair(pair) - PairLocks.lock_pair(pair, arrow.utcnow().shift(minutes=4).datetime) + PairLocks.lock_pair(pair, dt_now() + timedelta(minutes=4)) assert PairLocks.is_pair_locked(pair) # Get both locks from above @@ -113,20 +113,20 @@ def test_PairLocks_getlongestlock(use_db): pair = 'ETH/BTC' assert not PairLocks.is_pair_locked(pair) - PairLocks.lock_pair(pair, arrow.utcnow().shift(minutes=4).datetime) + PairLocks.lock_pair(pair, dt_now() + timedelta(minutes=4)) # ETH/BTC locked for 4 minutes assert PairLocks.is_pair_locked(pair) lock = PairLocks.get_pair_longest_lock(pair) - assert lock.lock_end_time.replace(tzinfo=timezone.utc) > arrow.utcnow().shift(minutes=3) - assert lock.lock_end_time.replace(tzinfo=timezone.utc) < arrow.utcnow().shift(minutes=14) + assert lock.lock_end_time.replace(tzinfo=timezone.utc) > dt_now() + timedelta(minutes=3) + assert lock.lock_end_time.replace(tzinfo=timezone.utc) < dt_now() + timedelta(minutes=14) - PairLocks.lock_pair(pair, arrow.utcnow().shift(minutes=15).datetime) + PairLocks.lock_pair(pair, dt_now() + timedelta(minutes=15)) assert PairLocks.is_pair_locked(pair) lock = PairLocks.get_pair_longest_lock(pair) # Must be longer than above - assert lock.lock_end_time.replace(tzinfo=timezone.utc) > arrow.utcnow().shift(minutes=14) + assert lock.lock_end_time.replace(tzinfo=timezone.utc) > dt_now() + timedelta(minutes=14) PairLocks.reset_locks() PairLocks.use_db = True @@ -143,8 +143,8 @@ def test_PairLocks_reason(use_db): assert PairLocks.use_db == use_db - PairLocks.lock_pair('XRP/USDT', arrow.utcnow().shift(minutes=4).datetime, 'TestLock1') - PairLocks.lock_pair('ETH/USDT', arrow.utcnow().shift(minutes=4).datetime, 'TestLock2') + PairLocks.lock_pair('XRP/USDT', dt_now() + timedelta(minutes=4), 'TestLock1') + PairLocks.lock_pair('ETH/USDT', dt_now() + timedelta(minutes=4), 'TestLock2') assert PairLocks.is_pair_locked('XRP/USDT') assert PairLocks.is_pair_locked('ETH/USDT') diff --git a/tests/rpc/test_rpc_apiserver.py b/tests/rpc/test_rpc_apiserver.py index 1e76ce557..842981ad0 100644 --- a/tests/rpc/test_rpc_apiserver.py +++ b/tests/rpc/test_rpc_apiserver.py @@ -21,11 +21,13 @@ from freqtrade.__init__ import __version__ from freqtrade.enums import CandleType, RunMode, State, TradingMode from freqtrade.exceptions import DependencyException, ExchangeError, OperationalException from freqtrade.loggers import setup_logging, setup_logging_pre +from freqtrade.optimize.backtesting import Backtesting from freqtrade.persistence import PairLocks, Trade from freqtrade.rpc import RPC from freqtrade.rpc.api_server import ApiServer from freqtrade.rpc.api_server.api_auth import create_token, get_user_from_token from freqtrade.rpc.api_server.uvicorn_threaded import UvicornServer +from freqtrade.rpc.api_server.webserver_bgwork import ApiBG from tests.conftest import (CURRENT_TEST_STRATEGY, EXMS, create_mock_trades, get_mock_coro, get_patched_freqtradebot, log_has, log_has_re, patch_get_signal) @@ -1665,137 +1667,140 @@ def test_sysinfo(botclient): def test_api_backtesting(botclient, mocker, fee, caplog, tmpdir): - ftbot, client = botclient - mocker.patch(f'{EXMS}.get_fee', fee) + try: + ftbot, client = botclient + mocker.patch(f'{EXMS}.get_fee', fee) - rc = client_get(client, f"{BASE_URI}/backtest") - # Backtest prevented in default mode - assert_response(rc, 502) + rc = client_get(client, f"{BASE_URI}/backtest") + # Backtest prevented in default mode + assert_response(rc, 502) - ftbot.config['runmode'] = RunMode.WEBSERVER - # Backtesting not started yet - rc = client_get(client, f"{BASE_URI}/backtest") - assert_response(rc) + ftbot.config['runmode'] = RunMode.WEBSERVER + # Backtesting not started yet + rc = client_get(client, f"{BASE_URI}/backtest") + assert_response(rc) - result = rc.json() - assert result['status'] == 'not_started' - assert not result['running'] - assert result['status_msg'] == 'Backtest not yet executed' - assert result['progress'] == 0 + result = rc.json() + assert result['status'] == 'not_started' + assert not result['running'] + assert result['status_msg'] == 'Backtest not yet executed' + assert result['progress'] == 0 - # Reset backtesting - rc = client_delete(client, f"{BASE_URI}/backtest") - assert_response(rc) - result = rc.json() - assert result['status'] == 'reset' - assert not result['running'] - assert result['status_msg'] == 'Backtest reset' - ftbot.config['export'] = 'trades' - ftbot.config['backtest_cache'] = 'day' - ftbot.config['user_data_dir'] = Path(tmpdir) - ftbot.config['exportfilename'] = Path(tmpdir) / "backtest_results" - ftbot.config['exportfilename'].mkdir() + # Reset backtesting + rc = client_delete(client, f"{BASE_URI}/backtest") + assert_response(rc) + result = rc.json() + assert result['status'] == 'reset' + assert not result['running'] + assert result['status_msg'] == 'Backtest reset' + ftbot.config['export'] = 'trades' + ftbot.config['backtest_cache'] = 'day' + ftbot.config['user_data_dir'] = Path(tmpdir) + ftbot.config['exportfilename'] = Path(tmpdir) / "backtest_results" + ftbot.config['exportfilename'].mkdir() - # start backtesting - data = { - "strategy": CURRENT_TEST_STRATEGY, - "timeframe": "5m", - "timerange": "20180110-20180111", - "max_open_trades": 3, - "stake_amount": 100, - "dry_run_wallet": 1000, - "enable_protections": False - } - rc = client_post(client, f"{BASE_URI}/backtest", data=data) - assert_response(rc) - result = rc.json() + # start backtesting + data = { + "strategy": CURRENT_TEST_STRATEGY, + "timeframe": "5m", + "timerange": "20180110-20180111", + "max_open_trades": 3, + "stake_amount": 100, + "dry_run_wallet": 1000, + "enable_protections": False + } + rc = client_post(client, f"{BASE_URI}/backtest", data=data) + assert_response(rc) + result = rc.json() - assert result['status'] == 'running' - assert result['progress'] == 0 - assert result['running'] - assert result['status_msg'] == 'Backtest started' + assert result['status'] == 'running' + assert result['progress'] == 0 + assert result['running'] + assert result['status_msg'] == 'Backtest started' - rc = client_get(client, f"{BASE_URI}/backtest") - assert_response(rc) + rc = client_get(client, f"{BASE_URI}/backtest") + assert_response(rc) - result = rc.json() - assert result['status'] == 'ended' - assert not result['running'] - assert result['status_msg'] == 'Backtest ended' - assert result['progress'] == 1 - assert result['backtest_result'] + result = rc.json() + assert result['status'] == 'ended' + assert not result['running'] + assert result['status_msg'] == 'Backtest ended' + assert result['progress'] == 1 + assert result['backtest_result'] - rc = client_get(client, f"{BASE_URI}/backtest/abort") - assert_response(rc) - result = rc.json() - assert result['status'] == 'not_running' - assert not result['running'] - assert result['status_msg'] == 'Backtest ended' + rc = client_get(client, f"{BASE_URI}/backtest/abort") + assert_response(rc) + result = rc.json() + assert result['status'] == 'not_running' + assert not result['running'] + assert result['status_msg'] == 'Backtest ended' - # Simulate running backtest - ApiServer._bgtask_running = True - rc = client_get(client, f"{BASE_URI}/backtest/abort") - assert_response(rc) - result = rc.json() - assert result['status'] == 'stopping' - assert not result['running'] - assert result['status_msg'] == 'Backtest ended' + # Simulate running backtest + ApiBG.bgtask_running = True + rc = client_get(client, f"{BASE_URI}/backtest/abort") + assert_response(rc) + result = rc.json() + assert result['status'] == 'stopping' + assert not result['running'] + assert result['status_msg'] == 'Backtest ended' - # Get running backtest... - rc = client_get(client, f"{BASE_URI}/backtest") - assert_response(rc) - result = rc.json() - assert result['status'] == 'running' - assert result['running'] - assert result['step'] == "backtest" - assert result['status_msg'] == "Backtest running" + # Get running backtest... + rc = client_get(client, f"{BASE_URI}/backtest") + assert_response(rc) + result = rc.json() + assert result['status'] == 'running' + assert result['running'] + assert result['step'] == "backtest" + assert result['status_msg'] == "Backtest running" - # Try delete with task still running - rc = client_delete(client, f"{BASE_URI}/backtest") - assert_response(rc) - result = rc.json() - assert result['status'] == 'running' + # Try delete with task still running + rc = client_delete(client, f"{BASE_URI}/backtest") + assert_response(rc) + result = rc.json() + assert result['status'] == 'running' - # Post to backtest that's still running - rc = client_post(client, f"{BASE_URI}/backtest", data=data) - assert_response(rc, 502) - result = rc.json() - assert 'Bot Background task already running' in result['error'] + # Post to backtest that's still running + rc = client_post(client, f"{BASE_URI}/backtest", data=data) + assert_response(rc, 502) + result = rc.json() + assert 'Bot Background task already running' in result['error'] - ApiServer._bgtask_running = False + ApiBG.bgtask_running = False - # Rerun backtest (should get previous result) - rc = client_post(client, f"{BASE_URI}/backtest", data=data) - assert_response(rc) - result = rc.json() - assert log_has_re('Reusing result of previous backtest.*', caplog) + # Rerun backtest (should get previous result) + rc = client_post(client, f"{BASE_URI}/backtest", data=data) + assert_response(rc) + result = rc.json() + assert log_has_re('Reusing result of previous backtest.*', caplog) - data['stake_amount'] = 101 + data['stake_amount'] = 101 - mocker.patch('freqtrade.optimize.backtesting.Backtesting.backtest_one_strategy', - side_effect=DependencyException('DeadBeef')) - rc = client_post(client, f"{BASE_URI}/backtest", data=data) - assert log_has("Backtesting caused an error: DeadBeef", caplog) + mocker.patch('freqtrade.optimize.backtesting.Backtesting.backtest_one_strategy', + side_effect=DependencyException('DeadBeef')) + rc = client_post(client, f"{BASE_URI}/backtest", data=data) + assert log_has("Backtesting caused an error: DeadBeef", caplog) - rc = client_get(client, f"{BASE_URI}/backtest") - assert_response(rc) - result = rc.json() - assert result['status'] == 'error' - assert 'Backtest failed' in result['status_msg'] + rc = client_get(client, f"{BASE_URI}/backtest") + assert_response(rc) + result = rc.json() + assert result['status'] == 'error' + assert 'Backtest failed' in result['status_msg'] - # Delete backtesting to avoid leakage since the backtest-object may stick around. - rc = client_delete(client, f"{BASE_URI}/backtest") - assert_response(rc) + # Delete backtesting to avoid leakage since the backtest-object may stick around. + rc = client_delete(client, f"{BASE_URI}/backtest") + assert_response(rc) - result = rc.json() - assert result['status'] == 'reset' - assert not result['running'] - assert result['status_msg'] == 'Backtest reset' + result = rc.json() + assert result['status'] == 'reset' + assert not result['running'] + assert result['status_msg'] == 'Backtest reset' - # Disallow base64 strategies - data['strategy'] = "xx:cHJpbnQoImhlbGxvIHdvcmxkIik=" - rc = client_post(client, f"{BASE_URI}/backtest", data=data) - assert_response(rc, 500) + # Disallow base64 strategies + data['strategy'] = "xx:cHJpbnQoImhlbGxvIHdvcmxkIik=" + rc = client_post(client, f"{BASE_URI}/backtest", data=data) + assert_response(rc, 500) + finally: + Backtesting.cleanup() def test_api_backtest_history(botclient, mocker, testdatadir): diff --git a/tests/rpc/test_rpc_telegram.py b/tests/rpc/test_rpc_telegram.py index 0d8c98d29..51879f5ad 100644 --- a/tests/rpc/test_rpc_telegram.py +++ b/tests/rpc/test_rpc_telegram.py @@ -12,7 +12,6 @@ from random import choice, randint from string import ascii_uppercase from unittest.mock import ANY, AsyncMock, MagicMock -import arrow import pytest import time_machine from pandas import DataFrame @@ -33,6 +32,7 @@ from freqtrade.persistence.models import Order from freqtrade.rpc import RPC from freqtrade.rpc.rpc import RPCException from freqtrade.rpc.telegram import Telegram, authorized_only +from freqtrade.util.datetime_helpers import dt_now from tests.conftest import (CURRENT_TEST_STRATEGY, EXMS, create_mock_trades, create_mock_trades_usdt, get_patched_freqtradebot, log_has, log_has_re, patch_exchange, patch_get_signal, patch_whitelist) @@ -259,7 +259,7 @@ async def test_telegram_status(default_conf, update, mocker) -> None: 'pair': 'ETH/BTC', 'base_currency': 'ETH', 'quote_currency': 'BTC', - 'open_date': arrow.utcnow(), + 'open_date': dt_now(), 'close_date': None, 'open_rate': 1.099e-05, 'close_rate': None, @@ -1518,8 +1518,8 @@ async def test_telegram_lock_handle(default_conf, update, ticker, fee, mocker) - msg_mock.reset_mock() - PairLocks.lock_pair('ETH/BTC', arrow.utcnow().shift(minutes=4).datetime, 'randreason') - PairLocks.lock_pair('XRP/BTC', arrow.utcnow().shift(minutes=20).datetime, 'deadbeef') + PairLocks.lock_pair('ETH/BTC', dt_now() + timedelta(minutes=4), 'randreason') + PairLocks.lock_pair('XRP/BTC', dt_now() + timedelta(minutes=20), 'deadbeef') await telegram._locks(update=update, context=MagicMock()) @@ -1898,7 +1898,7 @@ def test_send_msg_enter_notification(default_conf, mocker, caplog, message_type, 'current_rate': 1.099e-05, 'amount': 1333.3333333333335, 'analyzed_candle': {'open': 1.1, 'high': 2.2, 'low': 1.0, 'close': 1.5}, - 'open_date': arrow.utcnow().shift(hours=-1) + 'open_date': dt_now() + timedelta(hours=-1) } telegram, freqtradebot, msg_mock = get_telegram_testobject(mocker, default_conf) @@ -1959,7 +1959,7 @@ def test_send_msg_protection_notification(default_conf, mocker, time_machine) -> telegram, _, msg_mock = get_telegram_testobject(mocker, default_conf) time_machine.move_to("2021-09-01 05:00:00 +00:00") - lock = PairLocks.lock_pair('ETH/BTC', arrow.utcnow().shift(minutes=6).datetime, 'randreason') + lock = PairLocks.lock_pair('ETH/BTC', dt_now() + timedelta(minutes=6), 'randreason') msg = { 'type': RPCMessageType.PROTECTION_TRIGGER, } @@ -1974,7 +1974,7 @@ def test_send_msg_protection_notification(default_conf, mocker, time_machine) -> msg = { 'type': RPCMessageType.PROTECTION_TRIGGER_GLOBAL, } - lock = PairLocks.lock_pair('*', arrow.utcnow().shift(minutes=100).datetime, 'randreason') + lock = PairLocks.lock_pair('*', dt_now() + timedelta(minutes=100), 'randreason') msg.update(lock.to_json()) telegram.send_msg(msg) assert (msg_mock.call_args[0][0] == "*Protection* triggered due to randreason. " @@ -2005,7 +2005,7 @@ def test_send_msg_entry_fill_notification(default_conf, mocker, message_type, en 'fiat_currency': 'USD', 'open_rate': 1.099e-05, 'amount': 1333.3333333333335, - 'open_date': arrow.utcnow().shift(hours=-1) + 'open_date': dt_now() - timedelta(hours=1) }) leverage_text = f'*Leverage:* `{leverage}`\n' if leverage != 1.0 else '' assert msg_mock.call_args[0][0] == ( @@ -2032,7 +2032,7 @@ def test_send_msg_entry_fill_notification(default_conf, mocker, message_type, en 'fiat_currency': 'USD', 'open_rate': 1.099e-05, 'amount': 1333.3333333333335, - 'open_date': arrow.utcnow().shift(hours=-1) + 'open_date': dt_now() - timedelta(hours=1) }) assert msg_mock.call_args[0][0] == ( @@ -2071,8 +2071,8 @@ def test_send_msg_sell_notification(default_conf, mocker) -> None: 'fiat_currency': 'USD', 'enter_tag': 'buy_signal1', 'exit_reason': ExitType.STOP_LOSS.value, - 'open_date': arrow.utcnow().shift(hours=-1), - 'close_date': arrow.utcnow(), + 'open_date': dt_now() - timedelta(hours=1), + 'close_date': dt_now(), }) assert msg_mock.call_args[0][0] == ( '\N{WARNING SIGN} *Binance (dry):* Exiting KEY/ETH (#1)\n' @@ -2107,8 +2107,8 @@ def test_send_msg_sell_notification(default_conf, mocker) -> None: 'fiat_currency': 'USD', 'enter_tag': 'buy_signal1', 'exit_reason': ExitType.STOP_LOSS.value, - 'open_date': arrow.utcnow().shift(days=-1, hours=-2, minutes=-30), - 'close_date': arrow.utcnow(), + 'open_date': dt_now() - timedelta(days=1, hours=2, minutes=30), + 'close_date': dt_now(), 'stake_amount': 0.01, 'sub_trade': True, }) @@ -2144,8 +2144,8 @@ def test_send_msg_sell_notification(default_conf, mocker) -> None: 'stake_currency': 'ETH', 'enter_tag': 'buy_signal1', 'exit_reason': ExitType.STOP_LOSS.value, - 'open_date': arrow.utcnow().shift(days=-1, hours=-2, minutes=-30), - 'close_date': arrow.utcnow(), + 'open_date': dt_now() - timedelta(days=1, hours=2, minutes=30), + 'close_date': dt_now(), }) assert msg_mock.call_args[0][0] == ( '\N{WARNING SIGN} *Binance (dry):* Exiting KEY/ETH (#1)\n' @@ -2226,8 +2226,8 @@ def test_send_msg_sell_fill_notification(default_conf, mocker, direction, 'stake_currency': 'ETH', 'enter_tag': enter_signal, 'exit_reason': ExitType.STOP_LOSS.value, - 'open_date': arrow.utcnow().shift(days=-1, hours=-2, minutes=-30), - 'close_date': arrow.utcnow(), + 'open_date': dt_now() - timedelta(days=1, hours=2, minutes=30), + 'close_date': dt_now(), }) leverage_text = f'*Leverage:* `{leverage}`\n' if leverage and leverage != 1.0 else '' @@ -2317,7 +2317,7 @@ def test_send_msg_buy_notification_no_fiat( 'fiat_currency': None, 'current_rate': 1.099e-05, 'amount': 1333.3333333333335, - 'open_date': arrow.utcnow().shift(hours=-1) + 'open_date': dt_now() - timedelta(hours=1) }) leverage_text = f'*Leverage:* `{leverage}`\n' if leverage and leverage != 1.0 else '' @@ -2363,8 +2363,8 @@ def test_send_msg_sell_notification_no_fiat( 'fiat_currency': 'USD', 'enter_tag': enter_signal, 'exit_reason': ExitType.STOP_LOSS.value, - 'open_date': arrow.utcnow().shift(hours=-2, minutes=-35, seconds=-3), - 'close_date': arrow.utcnow(), + 'open_date': dt_now() - timedelta(hours=2, minutes=35, seconds=3), + 'close_date': dt_now(), }) leverage_text = f'*Leverage:* `{leverage}`\n' if leverage and leverage != 1.0 else '' diff --git a/tests/strategy/test_interface.py b/tests/strategy/test_interface.py index 204fa996d..8a609cf30 100644 --- a/tests/strategy/test_interface.py +++ b/tests/strategy/test_interface.py @@ -4,7 +4,6 @@ from datetime import datetime, timedelta, timezone from pathlib import Path from unittest.mock import MagicMock -import arrow import pytest from pandas import DataFrame @@ -22,6 +21,7 @@ from freqtrade.strategy.hyper import detect_parameters from freqtrade.strategy.parameters import (BaseParameter, BooleanParameter, CategoricalParameter, DecimalParameter, IntParameter, RealParameter) from freqtrade.strategy.strategy_wrapper import strategy_safe_wrapper +from freqtrade.util import dt_now from tests.conftest import (CURRENT_TEST_STRATEGY, TRADE_SIDES, create_mock_trades, log_has, log_has_re) @@ -34,7 +34,7 @@ _STRATEGY.dp = DataProvider({}, None, None) def test_returns_latest_signal(ohlcv_history): - ohlcv_history.loc[1, 'date'] = arrow.utcnow() + ohlcv_history.loc[1, 'date'] = dt_now() # Take a copy to correctly modify the call mocked_history = ohlcv_history.copy() mocked_history['enter_long'] = 0 @@ -159,7 +159,7 @@ def test_get_signal_exception_valueerror(mocker, caplog, ohlcv_history): def test_get_signal_old_dataframe(default_conf, mocker, caplog, ohlcv_history): # default_conf defines a 5m interval. we check interval * 2 + 5m # this is necessary as the last candle is removed (partial candles) by default - ohlcv_history.loc[1, 'date'] = arrow.utcnow().shift(minutes=-16) + ohlcv_history.loc[1, 'date'] = dt_now() - timedelta(minutes=16) # Take a copy to correctly modify the call mocked_history = ohlcv_history.copy() mocked_history['exit_long'] = 0 @@ -180,7 +180,7 @@ def test_get_signal_old_dataframe(default_conf, mocker, caplog, ohlcv_history): def test_get_signal_no_sell_column(default_conf, mocker, caplog, ohlcv_history): # default_conf defines a 5m interval. we check interval * 2 + 5m # this is necessary as the last candle is removed (partial candles) by default - ohlcv_history.loc[1, 'date'] = arrow.utcnow() + ohlcv_history.loc[1, 'date'] = dt_now() # Take a copy to correctly modify the call mocked_history = ohlcv_history.copy() # Intentionally don't set sell column @@ -224,7 +224,7 @@ def test_ignore_expired_candle(default_conf): def test_assert_df_raise(mocker, caplog, ohlcv_history): - ohlcv_history.loc[1, 'date'] = arrow.utcnow().shift(minutes=-16) + ohlcv_history.loc[1, 'date'] = dt_now() - timedelta(minutes=16) # Take a copy to correctly modify the call mocked_history = ohlcv_history.copy() mocked_history['sell'] = 0 @@ -323,21 +323,21 @@ def test_min_roi_reached(default_conf, fee) -> None: pair='ETH/BTC', stake_amount=0.001, amount=5, - open_date=arrow.utcnow().shift(hours=-1).datetime, + open_date=dt_now() - timedelta(hours=1), fee_open=fee.return_value, fee_close=fee.return_value, exchange='binance', open_rate=1, ) - assert not strategy.min_roi_reached(trade, 0.02, arrow.utcnow().shift(minutes=-56).datetime) - assert strategy.min_roi_reached(trade, 0.12, arrow.utcnow().shift(minutes=-56).datetime) + assert not strategy.min_roi_reached(trade, 0.02, dt_now() - timedelta(minutes=56)) + assert strategy.min_roi_reached(trade, 0.12, dt_now() - timedelta(minutes=56)) - assert not strategy.min_roi_reached(trade, 0.04, arrow.utcnow().shift(minutes=-39).datetime) - assert strategy.min_roi_reached(trade, 0.06, arrow.utcnow().shift(minutes=-39).datetime) + assert not strategy.min_roi_reached(trade, 0.04, dt_now() - timedelta(minutes=39)) + assert strategy.min_roi_reached(trade, 0.06, dt_now() - timedelta(minutes=39)) - assert not strategy.min_roi_reached(trade, -0.01, arrow.utcnow().shift(minutes=-1).datetime) - assert strategy.min_roi_reached(trade, 0.02, arrow.utcnow().shift(minutes=-1).datetime) + assert not strategy.min_roi_reached(trade, -0.01, dt_now() - timedelta(minutes=1)) + assert strategy.min_roi_reached(trade, 0.02, dt_now() - timedelta(minutes=1)) def test_min_roi_reached2(default_conf, fee) -> None: @@ -361,25 +361,25 @@ def test_min_roi_reached2(default_conf, fee) -> None: pair='ETH/BTC', stake_amount=0.001, amount=5, - open_date=arrow.utcnow().shift(hours=-1).datetime, + open_date=dt_now() - timedelta(hours=1), fee_open=fee.return_value, fee_close=fee.return_value, exchange='binance', open_rate=1, ) - assert not strategy.min_roi_reached(trade, 0.02, arrow.utcnow().shift(minutes=-56).datetime) - assert strategy.min_roi_reached(trade, 0.12, arrow.utcnow().shift(minutes=-56).datetime) + assert not strategy.min_roi_reached(trade, 0.02, dt_now() - timedelta(minutes=56)) + assert strategy.min_roi_reached(trade, 0.12, dt_now() - timedelta(minutes=56)) - assert not strategy.min_roi_reached(trade, 0.04, arrow.utcnow().shift(minutes=-39).datetime) - assert strategy.min_roi_reached(trade, 0.071, arrow.utcnow().shift(minutes=-39).datetime) + assert not strategy.min_roi_reached(trade, 0.04, dt_now() - timedelta(minutes=39)) + assert strategy.min_roi_reached(trade, 0.071, dt_now() - timedelta(minutes=39)) - assert not strategy.min_roi_reached(trade, 0.04, arrow.utcnow().shift(minutes=-26).datetime) - assert strategy.min_roi_reached(trade, 0.06, arrow.utcnow().shift(minutes=-26).datetime) + assert not strategy.min_roi_reached(trade, 0.04, dt_now() - timedelta(minutes=26)) + assert strategy.min_roi_reached(trade, 0.06, dt_now() - timedelta(minutes=26)) # Should not trigger with 20% profit since after 55 minutes only 30% is active. - assert not strategy.min_roi_reached(trade, 0.20, arrow.utcnow().shift(minutes=-2).datetime) - assert strategy.min_roi_reached(trade, 0.31, arrow.utcnow().shift(minutes=-2).datetime) + assert not strategy.min_roi_reached(trade, 0.20, dt_now() - timedelta(minutes=2)) + assert strategy.min_roi_reached(trade, 0.31, dt_now() - timedelta(minutes=2)) def test_min_roi_reached3(default_conf, fee) -> None: @@ -395,25 +395,25 @@ def test_min_roi_reached3(default_conf, fee) -> None: pair='ETH/BTC', stake_amount=0.001, amount=5, - open_date=arrow.utcnow().shift(hours=-1).datetime, + open_date=dt_now() - timedelta(hours=1), fee_open=fee.return_value, fee_close=fee.return_value, exchange='binance', open_rate=1, ) - assert not strategy.min_roi_reached(trade, 0.02, arrow.utcnow().shift(minutes=-56).datetime) - assert not strategy.min_roi_reached(trade, 0.12, arrow.utcnow().shift(minutes=-56).datetime) + assert not strategy.min_roi_reached(trade, 0.02, dt_now() - timedelta(minutes=56)) + assert not strategy.min_roi_reached(trade, 0.12, dt_now() - timedelta(minutes=56)) - assert not strategy.min_roi_reached(trade, 0.04, arrow.utcnow().shift(minutes=-39).datetime) - assert strategy.min_roi_reached(trade, 0.071, arrow.utcnow().shift(minutes=-39).datetime) + assert not strategy.min_roi_reached(trade, 0.04, dt_now() - timedelta(minutes=39)) + assert strategy.min_roi_reached(trade, 0.071, dt_now() - timedelta(minutes=39)) - assert not strategy.min_roi_reached(trade, 0.04, arrow.utcnow().shift(minutes=-26).datetime) - assert strategy.min_roi_reached(trade, 0.06, arrow.utcnow().shift(minutes=-26).datetime) + assert not strategy.min_roi_reached(trade, 0.04, dt_now() - timedelta(minutes=26)) + assert strategy.min_roi_reached(trade, 0.06, dt_now() - timedelta(minutes=26)) # Should not trigger with 20% profit since after 55 minutes only 30% is active. - assert not strategy.min_roi_reached(trade, 0.20, arrow.utcnow().shift(minutes=-2).datetime) - assert strategy.min_roi_reached(trade, 0.31, arrow.utcnow().shift(minutes=-2).datetime) + assert not strategy.min_roi_reached(trade, 0.20, dt_now() - timedelta(minutes=2)) + assert strategy.min_roi_reached(trade, 0.31, dt_now() - timedelta(minutes=2)) @pytest.mark.parametrize( @@ -449,7 +449,7 @@ def test_ft_stoploss_reached(default_conf, fee, profit, adjusted, expected, liq, pair='ETH/BTC', stake_amount=0.01, amount=1, - open_date=arrow.utcnow().shift(hours=-1).datetime, + open_date=dt_now() - timedelta(hours=1), fee_open=fee.return_value, fee_close=fee.return_value, exchange='binance', @@ -464,7 +464,7 @@ def test_ft_stoploss_reached(default_conf, fee, profit, adjusted, expected, liq, if custom_stop: strategy.custom_stoploss = custom_stop - now = arrow.utcnow().datetime + now = dt_now() current_rate = trade.open_rate * (1 + profit) sl_flag = strategy.ft_stoploss_reached(current_rate=current_rate, trade=trade, current_time=now, current_profit=profit, @@ -498,14 +498,14 @@ def test_custom_exit(default_conf, fee, caplog) -> None: pair='ETH/BTC', stake_amount=0.01, amount=1, - open_date=arrow.utcnow().shift(hours=-1).datetime, + open_date=dt_now() - timedelta(hours=1), fee_open=fee.return_value, fee_close=fee.return_value, exchange='binance', open_rate=1, ) - now = arrow.utcnow().datetime + now = dt_now() res = strategy.should_exit(trade, 1, now, enter=False, exit_=False, low=None, high=None) @@ -547,13 +547,13 @@ def test_should_sell(default_conf, fee) -> None: pair='ETH/BTC', stake_amount=0.01, amount=1, - open_date=arrow.utcnow().shift(hours=-1).datetime, + open_date=dt_now() - timedelta(hours=1), fee_open=fee.return_value, fee_close=fee.return_value, exchange='binance', open_rate=1, ) - now = arrow.utcnow().datetime + now = dt_now() res = strategy.should_exit(trade, 1, now, enter=False, exit_=False, low=None, high=None) @@ -728,7 +728,7 @@ def test_is_pair_locked(default_conf): pair = 'ETH/BTC' assert not strategy.is_pair_locked(pair) - strategy.lock_pair(pair, arrow.now(timezone.utc).shift(minutes=4).datetime) + strategy.lock_pair(pair, dt_now() + timedelta(minutes=4)) # ETH/BTC locked for 4 minutes assert strategy.is_pair_locked(pair) @@ -746,7 +746,7 @@ def test_is_pair_locked(default_conf): # Lock with reason reason = "TestLockR" - strategy.lock_pair(pair, arrow.now(timezone.utc).shift(minutes=4).datetime, reason) + strategy.lock_pair(pair, dt_now() + timedelta(minutes=4), reason) assert strategy.is_pair_locked(pair) strategy.unlock_reason(reason) assert not strategy.is_pair_locked(pair) diff --git a/tests/test_freqtradebot.py b/tests/test_freqtradebot.py index bd78e2fda..71f494372 100644 --- a/tests/test_freqtradebot.py +++ b/tests/test_freqtradebot.py @@ -4,10 +4,10 @@ import logging import time from copy import deepcopy +from datetime import timedelta from typing import List from unittest.mock import ANY, MagicMock, PropertyMock, patch -import arrow import pytest from pandas import DataFrame from sqlalchemy import select @@ -22,6 +22,7 @@ from freqtrade.freqtradebot import FreqtradeBot from freqtrade.persistence import Order, PairLocks, Trade from freqtrade.persistence.models import PairLock from freqtrade.plugins.protections.iprotection import ProtectionReturn +from freqtrade.util.datetime_helpers import dt_now, dt_utc from freqtrade.worker import Worker from tests.conftest import (EXMS, create_mock_trades, create_mock_trades_usdt, get_patched_freqtradebot, get_patched_worker, log_has, log_has_re, @@ -473,7 +474,7 @@ def test_enter_positions_global_pairlock(default_conf_usdt, ticker_usdt, limit_b assert not log_has_re(message, caplog) caplog.clear() - PairLocks.lock_pair('*', arrow.utcnow().shift(minutes=20).datetime, 'Just because', side='*') + PairLocks.lock_pair('*', dt_now() + timedelta(minutes=20), 'Just because', side='*') n = freqtrade.enter_positions() assert n == 0 assert log_has_re(message, caplog) @@ -494,7 +495,7 @@ def test_handle_protections(mocker, default_conf_usdt, fee, is_short): freqtrade = get_patched_freqtradebot(mocker, default_conf_usdt) freqtrade.protections._protection_handlers[1].global_stop = MagicMock( - return_value=ProtectionReturn(True, arrow.utcnow().shift(hours=1).datetime, "asdf")) + return_value=ProtectionReturn(True, dt_now() + timedelta(hours=1), "asdf")) create_mock_trades(fee, is_short) freqtrade.handle_protections('ETC/BTC', '*') send_msg_mock = freqtrade.rpc.send_msg @@ -1290,7 +1291,7 @@ def test_handle_stoploss_on_exchange(mocker, default_conf_usdt, fee, caplog, is_ }]) trade.stoploss_order_id = "107" trade.is_open = True - trade.stoploss_last_update = arrow.utcnow().shift(hours=-1).datetime + trade.stoploss_last_update = dt_now() - timedelta(hours=1) trade.stop_loss = 24 trade.exit_reason = None trade.orders.append( @@ -1439,7 +1440,7 @@ def test_handle_stoploss_on_exchange_partial_cancel_here( }) mocker.patch(f'{EXMS}.fetch_stoploss_order', stoploss_order_hit) mocker.patch(f'{EXMS}.cancel_stoploss_order_with_result', stoploss_order_cancel) - trade.stoploss_last_update = arrow.utcnow().shift(minutes=-10).datetime + trade.stoploss_last_update = dt_now() - timedelta(minutes=10) assert freqtrade.handle_stoploss_on_exchange(trade) is False # Canceled Stoploss filled partially ... @@ -1659,7 +1660,7 @@ def test_handle_stoploss_on_exchange_trailing( trade.is_open = True trade.open_order_id = None trade.stoploss_order_id = '100' - trade.stoploss_last_update = arrow.utcnow().shift(minutes=-20).datetime + trade.stoploss_last_update = dt_now() - timedelta(minutes=20) trade.orders.append( Order( ft_order_side='stoploss', @@ -1790,7 +1791,7 @@ def test_handle_stoploss_on_exchange_trailing_error( trade.open_order_id = None trade.stoploss_order_id = "abcd" trade.stop_loss = 0.2 - trade.stoploss_last_update = arrow.utcnow().shift(minutes=-601).datetime.replace(tzinfo=None) + trade.stoploss_last_update = (dt_now() - timedelta(minutes=601)).replace(tzinfo=None) trade.is_short = is_short stoploss_order_hanging = { @@ -1814,7 +1815,7 @@ def test_handle_stoploss_on_exchange_trailing_error( assert stoploss.call_count == 1 # Fail creating stoploss order - trade.stoploss_last_update = arrow.utcnow().shift(minutes=-601).datetime + trade.stoploss_last_update = dt_now() - timedelta(minutes=601) caplog.clear() cancel_mock = mocker.patch(f'{EXMS}.cancel_stoploss_order') mocker.patch(f'{EXMS}.create_stoploss', side_effect=ExchangeError()) @@ -1903,7 +1904,7 @@ def test_handle_stoploss_on_exchange_custom_stop( trade.is_open = True trade.open_order_id = None trade.stoploss_order_id = '100' - trade.stoploss_last_update = arrow.utcnow().shift(minutes=-601).datetime + trade.stoploss_last_update = dt_now() - timedelta(minutes=601) trade.orders.append( Order( ft_order_side='stoploss', @@ -2041,7 +2042,7 @@ def test_tsl_on_exchange_compatible_with_edge(mocker, edge_conf, fee, limit_orde trade.is_open = True trade.open_order_id = None trade.stoploss_order_id = '100' - trade.stoploss_last_update = arrow.utcnow().datetime + trade.stoploss_last_update = dt_now() trade.orders.append( Order( ft_order_side='stoploss', @@ -2151,7 +2152,7 @@ def test_exit_positions(mocker, default_conf_usdt, limit_order, is_short, caplog fee_open=0.001, fee_close=0.001, open_rate=0.01, - open_date=arrow.utcnow().datetime, + open_date=dt_now(), stake_amount=0.01, amount=11, exchange="binance", @@ -2197,7 +2198,7 @@ def test_exit_positions_exception(mocker, default_conf_usdt, limit_order, caplog fee_open=0.001, fee_close=0.001, open_rate=0.01, - open_date=arrow.utcnow().datetime, + open_date=dt_now(), stake_amount=0.01, amount=11, exchange="binance", @@ -2246,7 +2247,7 @@ def test_update_trade_state(mocker, default_conf_usdt, limit_order, is_short, ca fee_open=0.001, fee_close=0.001, open_rate=0.01, - open_date=arrow.utcnow().datetime, + open_date=dt_now(), amount=11, exchange="binance", is_short=is_short, @@ -2319,7 +2320,7 @@ def test_update_trade_state_withorderdict( amount=amount, exchange='binance', open_rate=2.0, - open_date=arrow.utcnow().datetime, + open_date=dt_now(), fee_open=fee.return_value, fee_close=fee.return_value, open_order_id=order_id, @@ -2406,7 +2407,7 @@ def test_update_trade_state_sell( open_rate=0.245441, fee_open=0.0025, fee_close=0.0025, - open_date=arrow.utcnow().datetime, + open_date=dt_now(), open_order_id=open_order['id'], is_open=True, interest_rate=0.0005, @@ -2992,8 +2993,8 @@ def test_manage_open_orders_exit_usercustom( ) freqtrade = FreqtradeBot(default_conf_usdt) - open_trade_usdt.open_date = arrow.utcnow().shift(hours=-5).datetime - open_trade_usdt.close_date = arrow.utcnow().shift(minutes=-601).datetime + open_trade_usdt.open_date = dt_now() - timedelta(hours=5) + open_trade_usdt.close_date = dt_now() - timedelta(minutes=601) open_trade_usdt.close_profit_abs = 0.001 Trade.session.add(open_trade_usdt) @@ -3074,8 +3075,8 @@ def test_manage_open_orders_exit( ) freqtrade = FreqtradeBot(default_conf_usdt) - open_trade_usdt.open_date = arrow.utcnow().shift(hours=-5).datetime - open_trade_usdt.close_date = arrow.utcnow().shift(minutes=-601).datetime + open_trade_usdt.open_date = dt_now() - timedelta(hours=5) + open_trade_usdt.close_date = dt_now() - timedelta(minutes=601) open_trade_usdt.close_profit_abs = 0.001 open_trade_usdt.is_short = is_short @@ -3115,8 +3116,8 @@ def test_check_handle_cancelled_exit( ) freqtrade = FreqtradeBot(default_conf_usdt) - open_trade_usdt.open_date = arrow.utcnow().shift(hours=-5).datetime - open_trade_usdt.close_date = arrow.utcnow().shift(minutes=-601).datetime + open_trade_usdt.open_date = dt_now() - timedelta(hours=5) + open_trade_usdt.close_date = dt_now() - timedelta(minutes=601) open_trade_usdt.is_short = is_short Trade.session.add(open_trade_usdt) @@ -3444,11 +3445,11 @@ def test_handle_cancel_exit_limit(mocker, default_conf_usdt, fee) -> None: exchange='binance', open_rate=0.245441, open_order_id="sell_123456", - open_date=arrow.utcnow().shift(days=-2).datetime, + open_date=dt_now() - timedelta(days=2), fee_open=fee.return_value, fee_close=fee.return_value, close_rate=0.555, - close_date=arrow.utcnow().datetime, + close_date=dt_now(), exit_reason="sell_reason_whatever", stake_amount=0.245441 * 2, ) @@ -5465,7 +5466,7 @@ def test_reupdate_enter_order_fees(mocker, default_conf_usdt, fee, caplog, is_sh stake_amount=60.0, fee_open=fee.return_value, fee_close=fee.return_value, - open_date=arrow.utcnow().datetime, + open_date=dt_now(), is_open=True, amount=30, open_rate=2.0, @@ -5601,7 +5602,7 @@ def test_handle_onexchange_order(mocker, default_conf_usdt, limit_order, is_shor fee_open=0.001, fee_close=0.001, open_rate=entry_order['price'], - open_date=arrow.utcnow().datetime, + open_date=dt_now(), stake_amount=entry_order['cost'], amount=entry_order['amount'], exchange="binance", @@ -5739,9 +5740,9 @@ def test_update_funding_fees( default_conf['trading_mode'] = 'futures' default_conf['margin_mode'] = 'isolated' - date_midnight = arrow.get('2021-09-01 00:00:00').datetime - date_eight = arrow.get('2021-09-01 08:00:00').datetime - date_sixteen = arrow.get('2021-09-01 16:00:00').datetime + date_midnight = dt_utc(2021, 9, 1) + date_eight = dt_utc(2021, 9, 1, 8) + date_sixteen = dt_utc(2021, 9, 1, 16) columns = ['date', 'open', 'high', 'low', 'close', 'volume'] # 16:00 entry is actually never used # But should be kept in the test to ensure we're filtering correctly. @@ -6030,7 +6031,7 @@ def test_position_adjust(mocker, default_conf_usdt, fee) -> None: 'ft_is_open': False, 'id': '651', 'order_id': '651', - 'datetime': arrow.utcnow().isoformat(), + 'datetime': dt_now().isoformat(), } mocker.patch(f'{EXMS}.create_order', MagicMock(return_value=closed_dca_order_1)) diff --git a/tests/test_misc.py b/tests/test_misc.py index 6b4343ab2..03a236d73 100644 --- a/tests/test_misc.py +++ b/tests/test_misc.py @@ -12,7 +12,7 @@ from freqtrade.misc import (dataframe_to_json, decimals_per_coin, deep_merge_dic file_load_json, format_ms_time, json_to_dataframe, pair_to_filename, parse_db_uri_for_logging, plural, render_template, render_template_with_fallback, round_coin_value, safe_value_fallback, - safe_value_fallback2, shorten_date) + safe_value_fallback2) def test_decimals_per_coin(): @@ -39,12 +39,6 @@ def test_round_coin_value(): assert round_coin_value(222.2, 'USDT', False, True) == '222.200' -def test_shorten_date() -> None: - str_data = '1 day, 2 hours, 3 minutes, 4 seconds ago' - str_shorten_data = '1 d, 2 h, 3 min, 4 sec ago' - assert shorten_date(str_data) == str_shorten_data - - def test_file_dump_json(mocker) -> None: file_open = mocker.patch('freqtrade.misc.Path.open', MagicMock()) json_dump = mocker.patch('rapidjson.dump', MagicMock()) diff --git a/tests/test_timerange.py b/tests/test_timerange.py index 993b24d95..d1c61704f 100644 --- a/tests/test_timerange.py +++ b/tests/test_timerange.py @@ -1,7 +1,6 @@ # pragma pylint: disable=missing-docstring, C0103 from datetime import datetime, timezone -import arrow import pytest from freqtrade.configuration import TimeRange @@ -69,7 +68,7 @@ def test_subtract_start(): def test_adjust_start_if_necessary(): - min_date = arrow.Arrow(2017, 11, 14, 21, 15, 00) + min_date = datetime(2017, 11, 14, 21, 15, 00, tzinfo=timezone.utc) x = TimeRange('date', 'date', 1510694100, 1510780500) # Adjust by 20 candles - min_date == startts diff --git a/tests/test_wallets.py b/tests/test_wallets.py index 09adf6e15..c3ff4ccd0 100644 --- a/tests/test_wallets.py +++ b/tests/test_wallets.py @@ -45,7 +45,7 @@ def test_sync_wallet_at_boot(mocker, default_conf): assert freqtrade.wallets._wallets['GAS'].total == 0.260739 assert freqtrade.wallets.get_free('BNT') == 1.0 assert 'USDT' in freqtrade.wallets._wallets - assert freqtrade.wallets._last_wallet_refresh > 0 + assert freqtrade.wallets._last_wallet_refresh is not None mocker.patch.multiple( EXMS, get_balances=MagicMock(return_value={ @@ -332,7 +332,7 @@ def test_sync_wallet_futures_live(mocker, default_conf): assert 'USDT' in freqtrade.wallets._wallets assert 'ETH/USDT:USDT' in freqtrade.wallets._positions - assert freqtrade.wallets._last_wallet_refresh > 0 + assert freqtrade.wallets._last_wallet_refresh is not None # Remove ETH/USDT:USDT position del mock_result[0] diff --git a/tests/utils/test_datetime_helpers.py b/tests/utils/test_datetime_helpers.py new file mode 100644 index 000000000..5aec0da54 --- /dev/null +++ b/tests/utils/test_datetime_helpers.py @@ -0,0 +1,59 @@ +from datetime import datetime, timedelta, timezone + +import pytest +import time_machine + +from freqtrade.util import dt_floor_day, dt_from_ts, dt_now, dt_ts, dt_utc, shorten_date +from freqtrade.util.datetime_helpers import dt_humanize + + +def test_dt_now(): + with time_machine.travel("2021-09-01 05:01:00 +00:00", tick=False) as t: + now = datetime.now(timezone.utc) + assert dt_now() == now + assert dt_ts() == int(now.timestamp() * 1000) + assert dt_ts(now) == int(now.timestamp() * 1000) + + t.shift(timedelta(hours=5)) + assert dt_now() >= now + assert dt_now() == datetime.now(timezone.utc) + assert dt_ts() == int(dt_now().timestamp() * 1000) + # Test with different time than now + assert dt_ts(now) == int(now.timestamp() * 1000) + + +def test_dt_utc(): + assert dt_utc(2023, 5, 5) == datetime(2023, 5, 5, tzinfo=timezone.utc) + assert dt_utc(2023, 5, 5, 0, 0, 0, 555500) == datetime(2023, 5, 5, 0, 0, 0, 555500, + tzinfo=timezone.utc) + + +@pytest.mark.parametrize('as_ms', [True, False]) +def test_dt_from_ts(as_ms): + multi = 1000 if as_ms else 1 + assert dt_from_ts(1683244800.0 * multi) == datetime(2023, 5, 5, tzinfo=timezone.utc) + assert dt_from_ts(1683244800.5555 * multi) == datetime(2023, 5, 5, 0, 0, 0, 555500, + tzinfo=timezone.utc) + # As int + assert dt_from_ts(1683244800 * multi) == datetime(2023, 5, 5, tzinfo=timezone.utc) + # As milliseconds + assert dt_from_ts(1683244800 * multi) == datetime(2023, 5, 5, tzinfo=timezone.utc) + assert dt_from_ts(1683242400 * multi) == datetime(2023, 5, 4, 23, 20, tzinfo=timezone.utc) + + +def test_dt_floor_day(): + now = datetime(2023, 9, 1, 5, 2, 3, 455555, tzinfo=timezone.utc) + + assert dt_floor_day(now) == datetime(2023, 9, 1, tzinfo=timezone.utc) + + +def test_shorten_date() -> None: + str_data = '1 day, 2 hours, 3 minutes, 4 seconds ago' + str_shorten_data = '1 d, 2 h, 3 min, 4 sec ago' + assert shorten_date(str_data) == str_shorten_data + + +def test_dt_humanize() -> None: + assert dt_humanize(dt_now()) == 'just now' + assert dt_humanize(dt_now(), only_distance=True) == 'instantly' + assert dt_humanize(dt_now() - timedelta(hours=16), only_distance=True) == '16 hours'