From 90e3c387577cbc6992c9e8825dd86b0619a8dbac Mon Sep 17 00:00:00 2001 From: creslinux Date: Sat, 14 Jul 2018 22:54:23 +0000 Subject: [PATCH 001/699] First cut, Bslap science project replacement for freqtrade backtest analysis - appprox 300-500x quicker to execute - fixes stop on close take close price bug in FT Bslap is configurable but by default stops are triggerd on low and pay stop price Not implimented dynamic stops or roi --- freqtrade/optimize/backtesting.py | 495 ++++++++++++++++++++++++++++-- 1 file changed, 463 insertions(+), 32 deletions(-) diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index 05bcdf4b7..14e4fe1f5 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -21,6 +21,8 @@ from freqtrade.configuration import Configuration from freqtrade.exchange import Exchange from freqtrade.misc import file_dump_json from freqtrade.persistence import Trade +from profilehooks import profile +from freqtrade.strategy.resolver import IStrategy, StrategyResolver logger = logging.getLogger(__name__) @@ -67,6 +69,8 @@ class Backtesting(object): self.exchange = Exchange(self.config) self.fee = self.exchange.get_fee() + self.stop_loss_value = self.analyze.strategy.stoploss + @staticmethod def get_timeframe(data: Dict[str, DataFrame]) -> Tuple[arrow.Arrow, arrow.Arrow]: """ @@ -186,6 +190,7 @@ class Backtesting(object): return btr return None + @profile def backtest(self, args: Dict) -> DataFrame: """ Implements backtesting functionality @@ -201,55 +206,481 @@ class Backtesting(object): realistic: do we try to simulate realistic trades? (default: True) :return: DataFrame """ - headers = ['date', 'buy', 'open', 'close', 'sell'] + headers = ['date', 'buy', 'open', 'close', 'sell', 'high', 'low'] processed = args['processed'] max_open_trades = args.get('max_open_trades', 0) realistic = args.get('realistic', False) trades = [] trade_count_lock: Dict = {} + ########################### Call out BSlap instead of using FT + bslap_results: list = [] + last_bslap_resultslist = [] for pair, pair_data in processed.items(): - pair_data['buy'], pair_data['sell'] = 0, 0 # cleanup from previous run - ticker_data = self.populate_sell_trend( - self.populate_buy_trend(pair_data))[headers].copy() + self.populate_buy_trend(pair_data))[headers].copy() - # to avoid using data from future, we buy/sell with signal from previous candle - ticker_data.loc[:, 'buy'] = ticker_data['buy'].shift(1) - ticker_data.loc[:, 'sell'] = ticker_data['sell'].shift(1) + bslap_pair_results = self.backslap_pair(ticker_data, pair) - ticker_data.drop(ticker_data.head(1).index, inplace=True) + last_bslap_results = bslap_results + bslap_results = last_bslap_results + bslap_pair_results + bslap_results_df = DataFrame(bslap_results) + print(bslap_results_df.dtypes()) - # Convert from Pandas to list for performance reasons - # (Looping Pandas is slow.) - ticker = [x for x in ticker_data.itertuples()] + return bslap_results_df + ########################### Original BT loop + # for pair, pair_data in processed.items(): + # pair_data['buy'], pair_data['sell'] = 0, 0 # cleanup from previous run + # + # ticker_data = self.populate_sell_trend( + # self.populate_buy_trend(pair_data))[headers].copy() + # + # # to avoid using data from future, we buy/sell with signal from previous candle + # ticker_data.loc[:, 'buy'] = ticker_data['buy'].shift(1) + # ticker_data.loc[:, 'sell'] = ticker_data['sell'].shift(1) + # + # ticker_data.drop(ticker_data.head(1).index, inplace=True) + # + # # Convert from Pandas to list for performance reasons + # # (Looping Pandas is slow.) + # ticker = [x for x in ticker_data.itertuples()] + # + # lock_pair_until = None + # for index, row in enumerate(ticker): + # if row.buy == 0 or row.sell == 1: + # continue # skip rows where no buy signal or that would immediately sell off + # + # if realistic: + # if lock_pair_until is not None and row.date <= lock_pair_until: + # continue + # if max_open_trades > 0: + # # Check if max_open_trades has already been reached for the given date + # if not trade_count_lock.get(row.date, 0) < max_open_trades: + # continue + # + # trade_count_lock[row.date] = trade_count_lock.get(row.date, 0) + 1 + # + # trade_entry = self._get_sell_trade_entry(pair, row, ticker[index + 1:], + # trade_count_lock, args) + # + # + # if trade_entry: + # lock_pair_until = trade_entry.close_time + # trades.append(trade_entry) + # else: + # # Set lock_pair_until to end of testing period if trade could not be closed + # # This happens only if the buy-signal was with the last candle + # lock_pair_until = ticker_data.iloc[-1].date + # + # return DataFrame.from_records(trades, columns=BacktestResult._fields) + ######################## Original BT loop end - lock_pair_until = None - for index, row in enumerate(ticker): - if row.buy == 0 or row.sell == 1: - continue # skip rows where no buy signal or that would immediately sell off + def np_get_t_open_ind(self, np_buy_arr, t_exit_ind: int): + import utils_find_1st as utf1st + """ + The purpose of this def is to return the next "buy" = 1 + after t_exit_ind. - if realistic: - if lock_pair_until is not None and row.date <= lock_pair_until: - continue - if max_open_trades > 0: - # Check if max_open_trades has already been reached for the given date - if not trade_count_lock.get(row.date, 0) < max_open_trades: - continue + t_exit_ind is the index the last trade exited on + or 0 if first time around this loop. + """ + t_open_ind: int - trade_count_lock[row.date] = trade_count_lock.get(row.date, 0) + 1 + # Create a view on our buy index starting after last trade exit + # Search for next buy + np_buy_arr_v = np_buy_arr[t_exit_ind:] + t_open_ind = utf1st.find_1st(np_buy_arr_v, 1, utf1st.cmp_equal) + t_open_ind = t_open_ind + t_exit_ind # Align numpy index + return t_open_ind - trade_entry = self._get_sell_trade_entry(pair, row, ticker[index + 1:], - trade_count_lock, args) + def backslap_pair(self, ticker_data, pair): + import pandas as pd + import numpy as np + import timeit + import utils_find_1st as utf1st - if trade_entry: - lock_pair_until = trade_entry.close_time - trades.append(trade_entry) + stop = self.stop_loss_value + p_stop = (-stop + 1) # What stop really means, e.g 0.01 is 0.99 of price + + ### backslap debug wrap + debug_2loops = False # only loop twice, for faster debug + debug_timing = False # print timing for each step + debug = False # print values, to check accuracy + if debug: + from pandas import set_option + set_option('display.max_rows', 5000) + set_option('display.max_columns', 8) + pd.set_option('display.width', 1000) + pd.set_option('max_colwidth', 40) + pd.set_option('precision', 12) + + def s(): + st = timeit.default_timer() + return st + + def f(st): + return (timeit.default_timer() - st) + #### backslap config + """ + A couple legacy Pandas vars still used for pretty debug output. + If have debug enabled the code uses these fields for dataframe output + + Ensure bto, sto, sco are aligned with Numpy values next + to align debug and actual. Options are: + buy - open - close - sell - high - low - np_stop_pri + """ + bto = buys_triggered_on = "close" + sto = stops_triggered_on = "low" ## Should be low, FT uses close + sco = stops_calculated_on = "np_stop_pri" ## should use np_stop_pri, FT uses close + ''' + Numpy arrays are used for 100x speed up + We requires setting Int values for + buy stop triggers and stop calculated on + # buy 0 - open 1 - close 2 - sell 3 - high 4 - low 5 - stop 6 + ''' + np_buy: int = 0 + np_open: int = 1 + np_close: int = 2 + np_sell: int = 3 + np_high: int = 4 + np_low: int = 5 + np_stop: int = 6 + np_bto: int = np_close # buys_triggered_on - should be close + np_bco: int = np_open # buys calculated on - open of the next candle. + np_sto: int = np_low # stops_triggered_on - Should be low, FT uses close + np_sco: int = np_stop # stops_calculated_on - Should be stop, FT uses close + # + ### End Config + + pair: str = pair + loop: int = 1 + + #ticker_data: DataFrame = ticker_dfs[t_file] + bslap: DataFrame = ticker_data + + # Build a single dimension numpy array from "buy" index for faster search + # (500x faster than pandas) + np_buy_arr = bslap['buy'].values + np_buy_arr_len: int = len(np_buy_arr) + + # use numpy array for faster searches in loop, 20x faster than pandas + # buy 0 - open 1 - close 2 - sell 3 - high 4 - low 5 + np_bslap = np.array(bslap[['buy', 'open', 'close', 'sell', 'high', 'low']]) + + loop: int = 0 # how many time around the loop + t_exit_ind = 0 # Start loop from first index + t_exit_last = 0 # To test for exit + + st = s() # Start timer for processing dataframe + if debug: + print('Processing:', pair) + + # Results will be stored in a list of dicts + bslap_pair_results: list = [] + bslap_result: dict = {} + + while t_exit_ind < np_buy_arr_len: + loop = loop + 1 + if debug or debug_timing: + print("----------------------------------------------------------- Loop", loop, pair) + if debug_2loops: + if loop == 4: + break + ''' + Dev phases + Phase 1 + 1) Manage buy, sell, stop enter/exit + a) Find first buy index + b) Discover first stop and sell hit after buy index + c) Chose first intance as trade exit + + Phase 2 + 2) Manage dynamic Stop and ROI Exit + a) Create trade slice from 1 + b) search within trade slice for dynamice stop hit + c) search within trade slice for ROI hit + ''' + + ''' + Finds index for first buy = 1 flag, use .values numpy array for speed + + Create a slice, from first buy index onwards. + Slice will be used to find exit conditions after trade open + ''' + if debug_timing: + st = s() + + t_open_ind = self.np_get_t_open_ind(np_buy_arr, t_exit_ind) + + if debug_timing: + t_t = f(st) + print("0-numpy", str.format('{0:.17f}', t_t)) + st = s() + + ''' + Calculate np_t_stop_pri (Trade Stop Price) based on the buy price + + As stop in based on buy price we are interested in buy + - Buys are Triggered On np_bto, typically the CLOSE of candle + - Buys are Calculated On np_bco, default is OPEN of the next candle. + as we only see the CLOSE after it has happened. + The assumption is we have bought at first available price, the OPEN + ''' + np_t_stop_pri = (np_bslap[t_open_ind + 1, np_bco] * p_stop) + + if debug_timing: + t_t = f(st) + print("1-numpy", str.format('{0:.17f}', t_t)) + st = s() + + """ + 1)Create a View from our open trade forward + + The view is our search space for the next Stop or Sell + We use a numpy view: + Using a numpy for speed on views, 1,000 faster than pandas + Pandas cannot assure it will always return a view, copies are + 3 orders of magnitude slower + + The view contains columns: + buy 0 - open 1 - close 2 - sell 3 - high 4 - low 5 + """ + np_t_open_v = np_bslap[t_open_ind:] + + if debug_timing: + t_t = f(st) + print("2-numpy", str.format('{0:.17f}', t_t)) + st = s() + + ''' + Find first stop index after Trade Open: + + First index in np_t_open_v (numpy view of bslap dataframe) + Using a numpy view a orders of magnitude faster + + where [np_sto] (stop tiggered on variable: "close", "low" etc) < np_t_stop_pri + ''' + np_t_stop_ind = utf1st.find_1st(np_t_open_v[:, np_sto], + np_t_stop_pri, + utf1st.cmp_smaller) \ + + t_open_ind + + if debug_timing: + t_t = f(st) + print("3-numpy", str.format('{0:.17f}', t_t)) + st = s() + + ''' + Find first sell index after trade open + + First index in t_open_slice where ['sell'] = 1 + ''' + # Use numpy array for faster search for sell + # Sell uses column 3. + # buy 0 - open 1 - close 2 - sell 3 - high 4 - low 5 + # Numpy searches 25-35x quicker than pandas on this data + + np_t_sell_ind = utf1st.find_1st(np_t_open_v[:, np_sell], + 1, utf1st.cmp_equal) \ + + t_open_ind + + if debug_timing: + t_t = f(st) + print("4-numpy", str.format('{0:.17f}', t_t)) + st = s() + + ''' + Determine which was hit first stop or sell, use as exit + + STOP takes priority over SELL as would be 'in candle' from tick data + Sell would use Open from Next candle. + So in a draw Stop would be hit first on ticker data in live + ''' + if np_t_stop_ind <= np_t_sell_ind: + t_exit_ind = np_t_stop_ind # Set Exit row index + t_exit_type = 'stop' # Set Exit type (sell|stop) + np_t_exit_pri = np_sco # The price field our STOP exit will use + else: + # move sell onto next candle, we only look back on sell + # will use the open price later. + t_exit_ind = np_t_sell_ind # Set Exit row index + t_exit_type = 'sell' # Set Exit type (sell|stop) + np_t_exit_pri = np_open # The price field our SELL exit will use + + if debug_timing: + t_t = f(st) + print("5-logic", str.format('{0:.17f}', t_t)) + st = s() + + if debug: + ''' + Print out the buys, stops, sells + Include Line before and after to for easy + Human verification + ''' + # Combine the np_t_stop_pri value to bslap dataframe to make debug + # life easy. This is the currenct stop price based on buy price_ + # Don't care about performance in debug + # (add an extra column if printing as df has date in col1 not in npy) + bslap['np_stop_pri'] = np_t_stop_pri + + # Buy + print("=================== BUY ", pair) + print("Numpy Array BUY Index is:", t_open_ind) + print("DataFrame BUY Index is:", t_open_ind + 1, "displaying DF \n") + print("HINT, BUY trade should use OPEN price from next candle, i.e ", t_open_ind + 2, "\n") + op_is = t_open_ind - 1 # Print open index start, line before + op_if = t_open_ind + 3 # Print open index finish, line after + print(bslap.iloc[op_is:op_if], "\n") + print(bslap.iloc[t_open_ind + 1]['date']) + + # Stop - Stops trigger price sto, and price received sco. (Stop Trigger|Calculated On) + print("=================== STOP ", pair) + print("Numpy Array STOP Index is:", np_t_stop_ind) + print("DataFrame STOP Index is:", np_t_stop_ind + 1, "displaying DF \n") + print("First Stop Index after Trade open in candle", np_t_stop_ind + 1, ": \n", + str.format('{0:.17f}', bslap.iloc[np_t_stop_ind][sto]), + "is less than", str.format('{0:.17f}', np_t_stop_pri)) + print("Tokens will be sold at rate:", str.format('{0:.17f}', bslap.iloc[np_t_stop_ind][sco])) + print("HINT, STOPs should close in-candle, i.e", np_t_stop_ind + 1, + ": As live STOPs are not linked to O-C times") + + st_is = np_t_stop_ind - 1 # Print stop index start, line before + st_if = np_t_stop_ind + 2 # Print stop index finish, line after + print(bslap.iloc[st_is:st_if], "\n") + + # Sell + print("=================== SELL ", pair) + print("Numpy Array SELL Index is:", np_t_sell_ind) + print("DataFrame SELL Index is:", np_t_sell_ind + 1, "displaying DF \n") + print("First Sell Index after Trade open in in candle", np_t_sell_ind + 1) + print("HINT, if exit is SELL (not stop) trade should use OPEN price from next candle", + np_t_sell_ind + 2, "\n") + sl_is = np_t_sell_ind - 1 # Print sell index start, line before + sl_if = np_t_sell_ind + 3 # Print sell index finish, line after + print(bslap.iloc[sl_is:sl_if], "\n") + + # Chosen Exit (stop or sell) + print("=================== EXIT ", pair) + print("Exit type is :", t_exit_type) + # print((bslap.iloc[t_exit_ind], "\n")) + print("trade exit price field is", np_t_exit_pri, "\n") + + ''' + Trade entry is always the next candles "open" price + We trigger on close, so cannot see that till after + its closed. + + The exception to this is a STOP which is calculated in candle + ''' + if debug_timing: + t_t = f(st) + print("6-depra", str.format('{0:.17f}', t_t)) + st = s() + + ## use numpy view "np_t_open_v" for speed. Columns are + # buy 0 - open 1 - close 2 - sell 3 - high 4 - low 5 + # exception is 6 which is use the stop value. + + np_trade_enter_price = np_bslap[t_open_ind + 1, np_open] + if t_exit_type == 'stop': + if np_t_exit_pri == 6: + np_trade_exit_price = np_t_stop_pri else: - # Set lock_pair_until to end of testing period if trade could not be closed - # This happens only if the buy-signal was with the last candle - lock_pair_until = ticker_data.iloc[-1].date + np_trade_exit_price = np_bslap[t_exit_ind, np_t_exit_pri] + if t_exit_type == 'sell': + np_trade_exit_price = np_bslap[t_exit_ind + 1, np_t_exit_pri] + + if debug_timing: + t_t = f(st) + print("7-numpy", str.format('{0:.17f}', t_t)) + st = s() + + if debug: + print("//////////////////////////////////////////////") + print("+++++++++++++++++++++++++++++++++ Trade Enter ") + print("np_trade Enterprice is ", str.format('{0:.17f}', np_trade_enter_price)) + print("--------------------------------- Trade Exit ") + print("Trade Exit Type is ", t_exit_type) + print("np_trade Exit Price is", str.format('{0:.17f}', np_trade_exit_price)) + print("//////////////////////////////////////////////") + + # Loop control - catch no closed trades. + if debug: + print("---------------------------------------- end of loop", loop - 1, + " Dataframe Exit Index is: ", t_exit_ind) + print("Exit Index Last, Exit Index Now Are: ", t_exit_last, t_exit_ind) + + if t_exit_last >= t_exit_ind: + """ + When last trade exit equals index of last exit, there is no + opportunity to close any more trades. + + Break loop and go on to next pair. + + TODO + add handing here to record none closed open trades + """ + + if debug: + print(bslap_pair_results) + + break + else: + """ + Add trade to backtest looking results list of dicts + Loop back to look for more trades. + """ + # Index will change if incandle stop or look back as close Open and Sell + if t_exit_type == 'stop': + close_index: int = t_exit_ind + 1 + elif t_exit_type == 'sell': + close_index: int = t_exit_ind + 2 + else: + close_index: int = t_exit_ind + 1 + # Munge the date / delta + start_date: str = bslap.iloc[t_open_ind + 1]['date'] + end_date: str = bslap.iloc[close_index]['date'] + + def __datetime(date_str): + return datetime.strptime(date_str, '%Y-%m-%d %H:%M:%S+00:00') + + # trade_start = __datetime(start_date) + # trade_end = __datetime(end_date) + # trade_mins = (trade_end - trade_start).total_seconds() / 60 + # build trade dictionary + bslap_result["pair"] = pair + bslap_result["profit_percent"] = ( np_trade_exit_price - np_trade_enter_price) // np_trade_enter_price * 100 + bslap_result["profit_abs"] = "" + bslap_result["open_time"] = start_date + bslap_result["close_time"] = end_date + bslap_result["open_index"] = t_open_ind + 1 + bslap_result["close_index"] = close_index + # bslap_result["trade_duration"] = trade_mins + bslap_result["open_at_end"] = False + bslap_result["open_rate"] = str.format('{0:.10f}', np_trade_enter_price) + bslap_result["close_rate"] = str.format('{0:.10f}', np_trade_exit_price) + bslap_result["exit_type"] = t_exit_type + # Add trade dictionary to list + bslap_pair_results.append(bslap_result) + if debug: + print(bslap_pair_results) + + """ + Loop back to start. t_exit_last becomes where loop + will seek to open new trades from. + Push index on 1 to not open on close + """ + t_exit_last = t_exit_ind + 1 + + if debug_timing: + t_t = f(st) + print("8", str.format('{0:.17f}', t_t)) + + # Send back List of trade dicts + return bslap_pair_results + + - return DataFrame.from_records(trades, columns=BacktestResult._fields) def start(self) -> None: """ From 07175ebc5a9d3fc2a068bbd21727f356e00c1930 Mon Sep 17 00:00:00 2001 From: creslinux Date: Sat, 14 Jul 2018 23:45:06 +0000 Subject: [PATCH 002/699] up --- freqtrade/optimize/backtesting.py | 37 ++++++++++++++++++------------- 1 file changed, 21 insertions(+), 16 deletions(-) diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index 14e4fe1f5..9110694ce 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -22,7 +22,7 @@ from freqtrade.exchange import Exchange from freqtrade.misc import file_dump_json from freqtrade.persistence import Trade from profilehooks import profile -from freqtrade.strategy.resolver import IStrategy, StrategyResolver +from collections import OrderedDict logger = logging.getLogger(__name__) @@ -190,7 +190,7 @@ class Backtesting(object): return btr return None - @profile + def backtest(self, args: Dict) -> DataFrame: """ Implements backtesting functionality @@ -218,15 +218,18 @@ class Backtesting(object): for pair, pair_data in processed.items(): ticker_data = self.populate_sell_trend( self.populate_buy_trend(pair_data))[headers].copy() + ticker_data.loc[:, 'buy'] = ticker_data['buy'].shift(1) + ticker_data.loc[:, 'sell'] = ticker_data['sell'].shift(1) bslap_pair_results = self.backslap_pair(ticker_data, pair) last_bslap_results = bslap_results bslap_results = last_bslap_results + bslap_pair_results - bslap_results_df = DataFrame(bslap_results) - print(bslap_results_df.dtypes()) + #bslap_results_df = DataFrame(bslap_results) - return bslap_results_df + res = DataFrame.from_records(bslap_results, columns=BacktestResult._fields) + print(res) + return res ########################### Original BT loop # for pair, pair_data in processed.items(): # pair_data['buy'], pair_data['sell'] = 0, 0 # cleanup from previous run @@ -297,14 +300,15 @@ class Backtesting(object): import numpy as np import timeit import utils_find_1st as utf1st + from datetime import datetime stop = self.stop_loss_value p_stop = (-stop + 1) # What stop really means, e.g 0.01 is 0.99 of price ### backslap debug wrap - debug_2loops = False # only loop twice, for faster debug + debug_2loops = True # only loop twice, for faster debug debug_timing = False # print timing for each step - debug = False # print values, to check accuracy + debug = True # print values, to check accuracy if debug: from pandas import set_option set_option('display.max_rows', 5000) @@ -638,24 +642,25 @@ class Backtesting(object): else: close_index: int = t_exit_ind + 1 # Munge the date / delta - start_date: str = bslap.iloc[t_open_ind + 1]['date'] - end_date: str = bslap.iloc[close_index]['date'] + start_date = bslap.iloc[t_open_ind + 1]['date'] + end_date = bslap.iloc[close_index]['date'] - def __datetime(date_str): - return datetime.strptime(date_str, '%Y-%m-%d %H:%M:%S+00:00') + # def __datetime(date_str): + # return datetime.strptime(date_str, '%Y-%m-%d %H:%M:%S+00:00') + + trade_start = start_date + trade_end = end_date + trade_mins = (trade_end - trade_start).total_seconds() / 60 - # trade_start = __datetime(start_date) - # trade_end = __datetime(end_date) - # trade_mins = (trade_end - trade_start).total_seconds() / 60 # build trade dictionary bslap_result["pair"] = pair - bslap_result["profit_percent"] = ( np_trade_exit_price - np_trade_enter_price) // np_trade_enter_price * 100 + bslap_result["profit_percent"] = ( np_trade_exit_price - np_trade_enter_price)/np_trade_enter_price bslap_result["profit_abs"] = "" bslap_result["open_time"] = start_date bslap_result["close_time"] = end_date bslap_result["open_index"] = t_open_ind + 1 bslap_result["close_index"] = close_index - # bslap_result["trade_duration"] = trade_mins + bslap_result["trade_duration"] = trade_mins bslap_result["open_at_end"] = False bslap_result["open_rate"] = str.format('{0:.10f}', np_trade_enter_price) bslap_result["close_rate"] = str.format('{0:.10f}', np_trade_exit_price) From 71c3106f8f94cd480c404b20d46c8b8abf555142 Mon Sep 17 00:00:00 2001 From: creslinux Date: Sun, 15 Jul 2018 09:30:01 +0000 Subject: [PATCH 003/699] Added ABS and Fees Fixed Index Alignment that was off moving from scratch to FT Fixed Stoploss, its a negative in FT, had been using positve stop -1 in scratch --- freqtrade/optimize/backtesting.py | 77 ++++++++++++++++++++----------- 1 file changed, 51 insertions(+), 26 deletions(-) diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index 9110694ce..3cfb588b9 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -214,22 +214,28 @@ class Backtesting(object): trade_count_lock: Dict = {} ########################### Call out BSlap instead of using FT bslap_results: list = [] - last_bslap_resultslist = [] + last_bslap_results: list = [] + for pair, pair_data in processed.items(): ticker_data = self.populate_sell_trend( self.populate_buy_trend(pair_data))[headers].copy() - ticker_data.loc[:, 'buy'] = ticker_data['buy'].shift(1) - ticker_data.loc[:, 'sell'] = ticker_data['sell'].shift(1) + ticker_data.drop(ticker_data.head(1).index, inplace=True) + + # #dump same DFs to disk for offline testing in scratch + # f_pair:str = pair + # csv = f_pair.replace("/", "_") + # csv="/Users/creslin/PycharmProjects/freqtrade_new/frames/" + csv + # ticker_data.to_csv(csv, sep='\t', encoding='utf-8') + + #call bslap - results are a list of dicts bslap_pair_results = self.backslap_pair(ticker_data, pair) - last_bslap_results = bslap_results bslap_results = last_bslap_results + bslap_pair_results - #bslap_results_df = DataFrame(bslap_results) - res = DataFrame.from_records(bslap_results, columns=BacktestResult._fields) - print(res) - return res + bslap_results_df = DataFrame(bslap_results, columns=BacktestResult._fields) + return bslap_results_df + ########################### Original BT loop # for pair, pair_data in processed.items(): # pair_data['buy'], pair_data['sell'] = 0, 0 # cleanup from previous run @@ -302,13 +308,28 @@ class Backtesting(object): import utils_find_1st as utf1st from datetime import datetime - stop = self.stop_loss_value - p_stop = (-stop + 1) # What stop really means, e.g 0.01 is 0.99 of price - ### backslap debug wrap debug_2loops = True # only loop twice, for faster debug debug_timing = False # print timing for each step debug = True # print values, to check accuracy + + # Read Stop Loss Values and Stake + stop = self.stop_loss_value + p_stop = (stop + 1) # What stop really means, e.g 0.01 is 0.99 of price + stake = self.config.get('stake_amount') + + # Set fees + # TODO grab these from the environment, do not hard set + # Fees + open_fee = 0.05 + close_fee = 0.05 + + if debug: + print("Stop is ", stop, "value from stragey file") + print("p_stop is", p_stop, "value used to multiply to entry price") + print("Stake is,", stake, "the sum of currency to spend per trade") + print("The open fee is", open_fee, "The close fee is", close_fee) + if debug: from pandas import set_option set_option('display.max_rows', 5000) @@ -385,9 +406,10 @@ class Backtesting(object): while t_exit_ind < np_buy_arr_len: loop = loop + 1 if debug or debug_timing: - print("----------------------------------------------------------- Loop", loop, pair) + print("-- T_exit_Ind - Numpy Index is", t_exit_ind, " ----------------------- Loop", loop, pair) if debug_2loops: - if loop == 4: + if loop == 2: + print("++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++Loop debug max met - breaking") break ''' Dev phases @@ -541,10 +563,10 @@ class Backtesting(object): print("=================== STOP ", pair) print("Numpy Array STOP Index is:", np_t_stop_ind) print("DataFrame STOP Index is:", np_t_stop_ind + 1, "displaying DF \n") - print("First Stop Index after Trade open in candle", np_t_stop_ind + 1, ": \n", + print("First Stop after Trade open in candle", t_open_ind + 1, "is ", np_t_stop_ind + 1,": \n", str.format('{0:.17f}', bslap.iloc[np_t_stop_ind][sto]), "is less than", str.format('{0:.17f}', np_t_stop_pri)) - print("Tokens will be sold at rate:", str.format('{0:.17f}', bslap.iloc[np_t_stop_ind][sco])) + print("If stop is first exit match sell rate is :", str.format('{0:.17f}', bslap.iloc[np_t_stop_ind][sco])) print("HINT, STOPs should close in-candle, i.e", np_t_stop_ind + 1, ": As live STOPs are not linked to O-C times") @@ -610,7 +632,7 @@ class Backtesting(object): # Loop control - catch no closed trades. if debug: - print("---------------------------------------- end of loop", loop - 1, + print("---------------------------------------- end of loop", loop, " Dataframe Exit Index is: ", t_exit_ind) print("Exit Index Last, Exit Index Now Are: ", t_exit_last, t_exit_ind) @@ -641,23 +663,26 @@ class Backtesting(object): close_index: int = t_exit_ind + 2 else: close_index: int = t_exit_ind + 1 - # Munge the date / delta - start_date = bslap.iloc[t_open_ind + 1]['date'] - end_date = bslap.iloc[close_index]['date'] + # Munge the date / delta (bt already date formats...just subract) + trade_start = bslap.iloc[t_open_ind + 1]['date'] + trade_end = bslap.iloc[close_index]['date'] # def __datetime(date_str): # return datetime.strptime(date_str, '%Y-%m-%d %H:%M:%S+00:00') - - trade_start = start_date - trade_end = end_date trade_mins = (trade_end - trade_start).total_seconds() / 60 + # Profit ABS. + # sumrecieved((rate * numTokens) * fee) - sumpaid ((rate * numTokens) * fee) + sumpaid: float = (np_trade_enter_price * stake) * open_fee + sumrecieved: float = (np_trade_exit_price * stake) * close_fee + profit_abs: float = sumrecieved - sumpaid + # build trade dictionary bslap_result["pair"] = pair - bslap_result["profit_percent"] = ( np_trade_exit_price - np_trade_enter_price)/np_trade_enter_price - bslap_result["profit_abs"] = "" - bslap_result["open_time"] = start_date - bslap_result["close_time"] = end_date + bslap_result["profit_percent"] = (np_trade_exit_price - np_trade_enter_price) / np_trade_enter_price + bslap_result["profit_abs"] = str.format('{0:.10f}', profit_abs) + bslap_result["open_time"] = trade_start + bslap_result["close_time"] = trade_end bslap_result["open_index"] = t_open_ind + 1 bslap_result["close_index"] = close_index bslap_result["trade_duration"] = trade_mins From 4e68362d46d6ca7c3aa96ac8a5fa090d18ca3c4f Mon Sep 17 00:00:00 2001 From: creslinux Date: Sun, 15 Jul 2018 10:33:00 +0000 Subject: [PATCH 004/699] Works with reporting output Bugs Calculating % prof ok, but abs wrong BAT/BTC DF is very broken all OHLC are the same - but exposes a buy after stop on last row "oddness" to be investigated / handled --- freqtrade/optimize/backtesting.py | 61 +++++++++++++++++-------------- 1 file changed, 34 insertions(+), 27 deletions(-) diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index 3cfb588b9..9698ef471 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -190,7 +190,7 @@ class Backtesting(object): return btr return None - + @profile def backtest(self, args: Dict) -> DataFrame: """ Implements backtesting functionality @@ -309,9 +309,9 @@ class Backtesting(object): from datetime import datetime ### backslap debug wrap - debug_2loops = True # only loop twice, for faster debug + debug_2loops = False # only loop twice, for faster debug debug_timing = False # print timing for each step - debug = True # print values, to check accuracy + debug = False # print values, to check accuracy # Read Stop Loss Values and Stake stop = self.stop_loss_value @@ -354,8 +354,10 @@ class Backtesting(object): buy - open - close - sell - high - low - np_stop_pri """ bto = buys_triggered_on = "close" - sto = stops_triggered_on = "low" ## Should be low, FT uses close - sco = stops_calculated_on = "np_stop_pri" ## should use np_stop_pri, FT uses close + # sto = stops_triggered_on = "low" ## Should be low, FT uses close + # sco = stops_calculated_on = "np_stop_pri" ## should use np_stop_pri, FT uses close + sto = stops_triggered_on = "close" ## Should be low, FT uses close + sco = stops_calculated_on = "close" ## should use np_stop_pri, FT uses close ''' Numpy arrays are used for 100x speed up We requires setting Int values for @@ -371,8 +373,10 @@ class Backtesting(object): np_stop: int = 6 np_bto: int = np_close # buys_triggered_on - should be close np_bco: int = np_open # buys calculated on - open of the next candle. - np_sto: int = np_low # stops_triggered_on - Should be low, FT uses close - np_sco: int = np_stop # stops_calculated_on - Should be stop, FT uses close + #np_sto: int = np_low # stops_triggered_on - Should be low, FT uses close + #np_sco: int = np_stop # stops_calculated_on - Should be stop, FT uses close + np_sto: int = np_close # stops_triggered_on - Should be low, FT uses close + np_sco: int = np_close # stops_calculated_on - Should be stop, FT uses close # ### End Config @@ -408,7 +412,7 @@ class Backtesting(object): if debug or debug_timing: print("-- T_exit_Ind - Numpy Index is", t_exit_ind, " ----------------------- Loop", loop, pair) if debug_2loops: - if loop == 2: + if loop == 3: print("++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++Loop debug max met - breaking") break ''' @@ -673,23 +677,25 @@ class Backtesting(object): # Profit ABS. # sumrecieved((rate * numTokens) * fee) - sumpaid ((rate * numTokens) * fee) - sumpaid: float = (np_trade_enter_price * stake) * open_fee - sumrecieved: float = (np_trade_exit_price * stake) * close_fee - profit_abs: float = sumrecieved - sumpaid + sumpaid: float = (np_trade_enter_price * stake) + sumpaid_fee: float = sumpaid * open_fee + sumrecieved: float = (np_trade_exit_price * stake) + sumrecieved_fee: float = sumrecieved * close_fee + profit_abs: float = sumrecieved - sumpaid - sumpaid_fee - sumrecieved_fee # build trade dictionary bslap_result["pair"] = pair bslap_result["profit_percent"] = (np_trade_exit_price - np_trade_enter_price) / np_trade_enter_price - bslap_result["profit_abs"] = str.format('{0:.10f}', profit_abs) + bslap_result["profit_abs"] = round(profit_abs, 15) bslap_result["open_time"] = trade_start bslap_result["close_time"] = trade_end - bslap_result["open_index"] = t_open_ind + 1 + bslap_result["open_index"] = t_open_ind + 2 # +1 between np and df, +1 as we buy on next. bslap_result["close_index"] = close_index bslap_result["trade_duration"] = trade_mins bslap_result["open_at_end"] = False - bslap_result["open_rate"] = str.format('{0:.10f}', np_trade_enter_price) - bslap_result["close_rate"] = str.format('{0:.10f}', np_trade_exit_price) - bslap_result["exit_type"] = t_exit_type + bslap_result["open_rate"] = round(np_trade_enter_price, 15) + bslap_result["close_rate"] = round(np_trade_exit_price, 15) + #bslap_result["exit_type"] = t_exit_type # Add trade dictionary to list bslap_pair_results.append(bslap_result) if debug: @@ -704,7 +710,7 @@ class Backtesting(object): if debug_timing: t_t = f(st) - print("8", str.format('{0:.17f}', t_t)) + print("8+trade", str.format('{0:.17f}', t_t)) # Send back List of trade dicts return bslap_pair_results @@ -785,16 +791,17 @@ class Backtesting(object): ) ) - logger.info( - '\n=============================================== ' - 'LEFT OPEN TRADES REPORT' - ' ===============================================\n' - '%s', - self._generate_text_table( - data, - results.loc[results.open_at_end] - ) - ) + ## TODO. Catch open trades for this report. + # logger.info( + # '\n=============================================== ' + # 'LEFT OPEN TRADES REPORT' + # ' ===============================================\n' + # '%s', + # self._generate_text_table( + # data, + # results.loc[results.open_at_end] + # ) + # ) def setup_configuration(args: Namespace) -> Dict[str, Any]: From a8b62a21ccce9bc7e79687d86167e9559b9b8962 Mon Sep 17 00:00:00 2001 From: creslinux Date: Sun, 15 Jul 2018 17:03:47 +0000 Subject: [PATCH 005/699] hmm --- freqtrade/optimize/backtesting.py | 153 +++++++++++++++++++++++------- 1 file changed, 120 insertions(+), 33 deletions(-) diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index 9698ef471..045b61a39 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -190,7 +190,6 @@ class Backtesting(object): return btr return None - @profile def backtest(self, args: Dict) -> DataFrame: """ Implements backtesting functionality @@ -233,7 +232,13 @@ class Backtesting(object): last_bslap_results = bslap_results bslap_results = last_bslap_results + bslap_pair_results + # Switch List of Trade Dicts (bslap_results) to Dataframe + # Fill missing, calculable columns, profit, duration , abs etc. bslap_results_df = DataFrame(bslap_results, columns=BacktestResult._fields) + bslap_results_df = self.vector_fill_results_table(bslap_results_df) + + print(bslap_results_df.dtypes) + return bslap_results_df ########################### Original BT loop @@ -283,6 +288,69 @@ class Backtesting(object): # return DataFrame.from_records(trades, columns=BacktestResult._fields) ######################## Original BT loop end + def vector_fill_results_table(self, bslap_results_df: DataFrame): + """ + The Results frame contains a number of columns that are calculable + from othe columns. These are left blank till all rows are added, + to be populated in single vector calls. + + Columns to be populated are: + - Profit + - trade duration + - profit abs + + :param bslap_results Dataframe + :return: bslap_results Dataframe + """ + import pandas as pd + debug = True + + # stake and fees + stake = self.config.get('stake_amount') + # TODO grab these from the environment, do not hard set + open_fee = 0.05 + close_fee = 0.05 + + if debug: + print("Stake is,", stake, "the sum of currency to spend per trade") + print("The open fee is", open_fee, "The close fee is", close_fee) + if debug: + from pandas import set_option + set_option('display.max_rows', 5000) + set_option('display.max_columns', 8) + pd.set_option('display.width', 1000) + pd.set_option('max_colwidth', 40) + pd.set_option('precision', 12) + + # Align with BT + bslap_results_df['open_time'] = pd.to_datetime(bslap_results_df['open_time']) + bslap_results_df['close_time'] = pd.to_datetime(bslap_results_df['close_time']) + + # Populate duration + bslap_results_df['trade_duration'] = bslap_results_df['close_time'] - bslap_results_df['open_time'] + if debug: + print(bslap_results_df[['open_time', 'close_time', 'trade_duration']]) + + ## Spends, Takes, Profit, Absolute Profit + # Buy Price + bslap_results_df['buy_sum'] = stake * bslap_results_df['open_rate'] + bslap_results_df['buy_fee'] = bslap_results_df['buy_sum'] * open_fee + bslap_results_df['buy_spend'] = bslap_results_df['buy_sum'] + bslap_results_df['buy_fee'] + # Sell price + bslap_results_df['sell_sum'] = stake * bslap_results_df['close_rate'] + bslap_results_df['sell_fee'] = bslap_results_df['sell_sum'] * close_fee + bslap_results_df['sell_take'] = bslap_results_df['sell_sum'] - bslap_results_df['sell_fee'] + # profit_percent + bslap_results_df['profit_percent'] = bslap_results_df['sell_take'] / bslap_results_df['buy_spend'] - 1 + # Absolute profit + bslap_results_df['profit_abs'] = bslap_results_df['sell_take'] - bslap_results_df['buy_spend'] + + if debug: + print("\n") + print(bslap_results_df[['buy_spend', 'sell_take', 'profit_percent', 'profit_abs']]) + + return bslap_results_df + def np_get_t_open_ind(self, np_buy_arr, t_exit_ind: int): import utils_find_1st as utf1st """ @@ -301,6 +369,7 @@ class Backtesting(object): t_open_ind = t_open_ind + t_exit_ind # Align numpy index return t_open_ind + @profile def backslap_pair(self, ticker_data, pair): import pandas as pd import numpy as np @@ -316,19 +385,10 @@ class Backtesting(object): # Read Stop Loss Values and Stake stop = self.stop_loss_value p_stop = (stop + 1) # What stop really means, e.g 0.01 is 0.99 of price - stake = self.config.get('stake_amount') - - # Set fees - # TODO grab these from the environment, do not hard set - # Fees - open_fee = 0.05 - close_fee = 0.05 if debug: print("Stop is ", stop, "value from stragey file") print("p_stop is", p_stop, "value used to multiply to entry price") - print("Stake is,", stake, "the sum of currency to spend per trade") - print("The open fee is", open_fee, "The close fee is", close_fee) if debug: from pandas import set_option @@ -395,6 +455,12 @@ class Backtesting(object): # buy 0 - open 1 - close 2 - sell 3 - high 4 - low 5 np_bslap = np.array(bslap[['buy', 'open', 'close', 'sell', 'high', 'low']]) + # Build a numpy list of date-times. + # We use these when building the trade + # The rationale is to address a value from a pandas cell is thousands of + # times more expensive. Processing time went X25 when trying to use any data from pandas + np_bslap_dates = bslap['date'].values + loop: int = 0 # how many time around the loop t_exit_ind = 0 # Start loop from first index t_exit_last = 0 # To test for exit @@ -657,9 +723,13 @@ class Backtesting(object): break else: """ - Add trade to backtest looking results list of dicts - Loop back to look for more trades. - """ + Add trade to backtest looking results list of dicts + Loop back to look for more trades. + """ + if debug_timing: + t_t = f(st) + print("8a-IfEls", str.format('{0:.17f}', t_t)) + st = s() # Index will change if incandle stop or look back as close Open and Sell if t_exit_type == 'stop': close_index: int = t_exit_ind + 1 @@ -668,39 +738,56 @@ class Backtesting(object): else: close_index: int = t_exit_ind + 1 - # Munge the date / delta (bt already date formats...just subract) - trade_start = bslap.iloc[t_open_ind + 1]['date'] - trade_end = bslap.iloc[close_index]['date'] - # def __datetime(date_str): - # return datetime.strptime(date_str, '%Y-%m-%d %H:%M:%S+00:00') - trade_mins = (trade_end - trade_start).total_seconds() / 60 + if debug_timing: + t_t = f(st) + print("8b-Index", str.format('{0:.17f}', t_t)) + st = s() - # Profit ABS. - # sumrecieved((rate * numTokens) * fee) - sumpaid ((rate * numTokens) * fee) - sumpaid: float = (np_trade_enter_price * stake) - sumpaid_fee: float = sumpaid * open_fee - sumrecieved: float = (np_trade_exit_price * stake) - sumrecieved_fee: float = sumrecieved * close_fee - profit_abs: float = sumrecieved - sumpaid - sumpaid_fee - sumrecieved_fee + # # Profit ABS. + # # sumrecieved((rate * numTokens) * fee) - sumpaid ((rate * numTokens) * fee) + # sumpaid: float = (np_trade_enter_price * stake) + # sumpaid_fee: float = sumpaid * open_fee + # sumrecieved: float = (np_trade_exit_price * stake) + # sumrecieved_fee: float = sumrecieved * close_fee + # profit_abs: float = sumrecieved - sumpaid - sumpaid_fee - sumrecieved_fee + + if debug_timing: + t_t = f(st) + print("8d---ABS", str.format('{0:.17f}', t_t)) + st = s() + + # Build trade dictionary + ## In general if a field can be calculated later from other fields leave blank here + ## Its X(numer of trades faster) to calc all in a single vector than 1 trade at a time - # build trade dictionary bslap_result["pair"] = pair - bslap_result["profit_percent"] = (np_trade_exit_price - np_trade_enter_price) / np_trade_enter_price - bslap_result["profit_abs"] = round(profit_abs, 15) - bslap_result["open_time"] = trade_start - bslap_result["close_time"] = trade_end + bslap_result["profit_percent"] = "1" # To be 1 vector calc across trades when loop complete + bslap_result["profit_abs"] = "1" # To be 1 vector calc across trades when loop complete + bslap_result["open_time"] = np_bslap_dates[t_open_ind + 1] # use numpy array, pandas 20x slower + bslap_result["close_time"] = np_bslap_dates[close_index] # use numpy array, pandas 20x slower bslap_result["open_index"] = t_open_ind + 2 # +1 between np and df, +1 as we buy on next. bslap_result["close_index"] = close_index - bslap_result["trade_duration"] = trade_mins + bslap_result["trade_duration"] = "1" # To be 1 vector calc across trades when loop complete bslap_result["open_at_end"] = False bslap_result["open_rate"] = round(np_trade_enter_price, 15) bslap_result["close_rate"] = round(np_trade_exit_price, 15) - #bslap_result["exit_type"] = t_exit_type + bslap_result["exit_type"] = t_exit_type + + if debug_timing: + t_t = f(st) + print("8e-trade", str.format('{0:.17f}', t_t)) + st = s() # Add trade dictionary to list bslap_pair_results.append(bslap_result) + if debug: print(bslap_pair_results) + if debug_timing: + t_t = f(st) + print("8f--list", str.format('{0:.17f}', t_t)) + st = s() + """ Loop back to start. t_exit_last becomes where loop will seek to open new trades from. From 7174f27eb85296ef1886e3ebc0f71c5c3e7ec0d0 Mon Sep 17 00:00:00 2001 From: creslinux Date: Mon, 16 Jul 2018 12:01:02 +0000 Subject: [PATCH 006/699] Rewrite to used algned numpy/dataframes updated logic added vector fill for abs/profit/duration in single hit on results. --- freqtrade/optimize/backtesting.py | 663 +++++++++++++++++------------- 1 file changed, 370 insertions(+), 293 deletions(-) diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index 045b61a39..2629ed5fa 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -10,7 +10,7 @@ from datetime import datetime from typing import Any, Dict, List, NamedTuple, Optional, Tuple import arrow -from pandas import DataFrame +from pandas import DataFrame, to_datetime from tabulate import tabulate import freqtrade.optimize as optimize @@ -23,6 +23,7 @@ from freqtrade.misc import file_dump_json from freqtrade.persistence import Trade from profilehooks import profile from collections import OrderedDict +import timeit logger = logging.getLogger(__name__) @@ -190,6 +191,7 @@ class Backtesting(object): return btr return None + @profile def backtest(self, args: Dict) -> DataFrame: """ Implements backtesting functionality @@ -234,10 +236,13 @@ class Backtesting(object): # Switch List of Trade Dicts (bslap_results) to Dataframe # Fill missing, calculable columns, profit, duration , abs etc. - bslap_results_df = DataFrame(bslap_results, columns=BacktestResult._fields) - bslap_results_df = self.vector_fill_results_table(bslap_results_df) + bslap_results_df = DataFrame(bslap_results) + bslap_results_df['open_time'] = to_datetime(bslap_results_df['open_time']) + bslap_results_df['close_time'] = to_datetime(bslap_results_df['close_time']) + ### don't use this, itll drop exit type field + # bslap_results_df = DataFrame(bslap_results, columns=BacktestResult._fields) - print(bslap_results_df.dtypes) + bslap_results_df = self.vector_fill_results_table(bslap_results_df) return bslap_results_df @@ -303,14 +308,13 @@ class Backtesting(object): :return: bslap_results Dataframe """ import pandas as pd - debug = True + debug = False # stake and fees - stake = self.config.get('stake_amount') - # TODO grab these from the environment, do not hard set - open_fee = 0.05 - close_fee = 0.05 - + stake = 0.015 + # 0.05% is 0.00,05 + open_fee = 0.0000 + close_fee = 0.0000 if debug: print("Stake is,", stake, "the sum of currency to spend per trade") print("The open fee is", open_fee, "The close fee is", close_fee) @@ -322,10 +326,6 @@ class Backtesting(object): pd.set_option('max_colwidth', 40) pd.set_option('precision', 12) - # Align with BT - bslap_results_df['open_time'] = pd.to_datetime(bslap_results_df['open_time']) - bslap_results_df['close_time'] = pd.to_datetime(bslap_results_df['close_time']) - # Populate duration bslap_results_df['trade_duration'] = bslap_results_df['close_time'] - bslap_results_df['open_time'] if debug: @@ -347,29 +347,46 @@ class Backtesting(object): if debug: print("\n") - print(bslap_results_df[['buy_spend', 'sell_take', 'profit_percent', 'profit_abs']]) + print(bslap_results_df[ + ['buy_sum', 'buy_fee', 'buy_spend', 'sell_sum', 'sell_take', 'profit_percent', 'profit_abs', + 'exit_type']]) return bslap_results_df def np_get_t_open_ind(self, np_buy_arr, t_exit_ind: int): import utils_find_1st as utf1st """ - The purpose of this def is to return the next "buy" = 1 - after t_exit_ind. + The purpose of this def is to return the next "buy" = 1 + after t_exit_ind. - t_exit_ind is the index the last trade exited on - or 0 if first time around this loop. - """ + t_exit_ind is the index the last trade exited on + or 0 if first time around this loop. + """ + # Timers, to be called if in debug + def s(): + st = timeit.default_timer() + return st + def f(st): + return (timeit.default_timer() - st) + + st = s() t_open_ind: int - # Create a view on our buy index starting after last trade exit - # Search for next buy + """ + Create a view on our buy index starting after last trade exit + Search for next buy + """ np_buy_arr_v = np_buy_arr[t_exit_ind:] t_open_ind = utf1st.find_1st(np_buy_arr_v, 1, utf1st.cmp_equal) - t_open_ind = t_open_ind + t_exit_ind # Align numpy index + + ''' + If -1 is returned no buy has been found, preserve the value + ''' + if t_open_ind != -1: # send back the -1 if no buys found. otherwise update index + t_open_ind = t_open_ind + t_exit_ind # Align numpy index + return t_open_ind - @profile def backslap_pair(self, ticker_data, pair): import pandas as pd import numpy as np @@ -397,27 +414,12 @@ class Backtesting(object): pd.set_option('display.width', 1000) pd.set_option('max_colwidth', 40) pd.set_option('precision', 12) - def s(): st = timeit.default_timer() return st - def f(st): return (timeit.default_timer() - st) #### backslap config - """ - A couple legacy Pandas vars still used for pretty debug output. - If have debug enabled the code uses these fields for dataframe output - - Ensure bto, sto, sco are aligned with Numpy values next - to align debug and actual. Options are: - buy - open - close - sell - high - low - np_stop_pri - """ - bto = buys_triggered_on = "close" - # sto = stops_triggered_on = "low" ## Should be low, FT uses close - # sco = stops_calculated_on = "np_stop_pri" ## should use np_stop_pri, FT uses close - sto = stops_triggered_on = "close" ## Should be low, FT uses close - sco = stops_calculated_on = "close" ## should use np_stop_pri, FT uses close ''' Numpy arrays are used for 100x speed up We requires setting Int values for @@ -441,7 +443,6 @@ class Backtesting(object): ### End Config pair: str = pair - loop: int = 1 #ticker_data: DataFrame = ticker_dfs[t_file] bslap: DataFrame = ticker_data @@ -482,223 +483,342 @@ class Backtesting(object): print("++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++Loop debug max met - breaking") break ''' - Dev phases - Phase 1 - 1) Manage buy, sell, stop enter/exit - a) Find first buy index - b) Discover first stop and sell hit after buy index - c) Chose first intance as trade exit + Dev phases + Phase 1 + 1) Manage buy, sell, stop enter/exit + a) Find first buy index + b) Discover first stop and sell hit after buy index + c) Chose first instance as trade exit + + Phase 2 + 2) Manage dynamic Stop and ROI Exit + a) Create trade slice from 1 + b) search within trade slice for dynamice stop hit + c) search within trade slice for ROI hit + ''' - Phase 2 - 2) Manage dynamic Stop and ROI Exit - a) Create trade slice from 1 - b) search within trade slice for dynamice stop hit - c) search within trade slice for ROI hit - ''' - - ''' - Finds index for first buy = 1 flag, use .values numpy array for speed - - Create a slice, from first buy index onwards. - Slice will be used to find exit conditions after trade open - ''' if debug_timing: st = s() - + ''' + 0 - Find next buy entry + Finds index for first (buy = 1) flag + + Requires: np_buy_arr - a 1D array of the 'buy' column. To find next "1" + Required: t_exit_ind - Either 0, first loop. Or The index we last exited on + Provides: The next "buy" index after t_exit_ind + + If -1 is returned no buy has been found in remainder of array, skip to exit loop + ''' t_open_ind = self.np_get_t_open_ind(np_buy_arr, t_exit_ind) + if debug: + print("\n(0) numpy debug \nnp_get_t_open, has returned the next valid buy index as", t_open_ind) + print("If -1 there are no valid buys in the remainder of ticker data. Skipping to end of loop") if debug_timing: t_t = f(st) print("0-numpy", str.format('{0:.17f}', t_t)) st = s() - ''' - Calculate np_t_stop_pri (Trade Stop Price) based on the buy price + if t_open_ind != -1: - As stop in based on buy price we are interested in buy - - Buys are Triggered On np_bto, typically the CLOSE of candle - - Buys are Calculated On np_bco, default is OPEN of the next candle. - as we only see the CLOSE after it has happened. - The assumption is we have bought at first available price, the OPEN - ''' - np_t_stop_pri = (np_bslap[t_open_ind + 1, np_bco] * p_stop) + """ + 1 - Create view to search within for our open trade + + The view is our search space for the next Stop or Sell + Numpy view is employed as: + 1,000 faster than pandas searches + Pandas cannot assure it will always return a view, it may make a slow copy. + + The view contains columns: + buy 0 - open 1 - close 2 - sell 3 - high 4 - low 5 + + Requires: np_bslap is our numpy array of the ticker DataFrame + Requires: t_open_ind is the index row with the buy. + Provided: np_t_open_v View of array after trade. + """ + np_t_open_v = np_bslap[t_open_ind:] - if debug_timing: - t_t = f(st) - print("1-numpy", str.format('{0:.17f}', t_t)) - st = s() + if debug: + print("\n(1) numpy debug \nNumpy view row 0 is now Ticker_Data Index", t_open_ind) + print("Numpy View: Buy - Open - Close - Sell - High - Low") + print("Row 0", np_t_open_v[0]) + print("Row 1", np_t_open_v[1], ) + if debug_timing: + t_t = f(st) + print("2-numpy", str.format('{0:.17f}', t_t)) + st = s() - """ - 1)Create a View from our open trade forward - - The view is our search space for the next Stop or Sell - We use a numpy view: - Using a numpy for speed on views, 1,000 faster than pandas - Pandas cannot assure it will always return a view, copies are - 3 orders of magnitude slower - - The view contains columns: - buy 0 - open 1 - close 2 - sell 3 - high 4 - low 5 - """ - np_t_open_v = np_bslap[t_open_ind:] - - if debug_timing: - t_t = f(st) - print("2-numpy", str.format('{0:.17f}', t_t)) - st = s() - - ''' - Find first stop index after Trade Open: - - First index in np_t_open_v (numpy view of bslap dataframe) - Using a numpy view a orders of magnitude faster - - where [np_sto] (stop tiggered on variable: "close", "low" etc) < np_t_stop_pri - ''' - np_t_stop_ind = utf1st.find_1st(np_t_open_v[:, np_sto], - np_t_stop_pri, - utf1st.cmp_smaller) \ - + t_open_ind - - if debug_timing: - t_t = f(st) - print("3-numpy", str.format('{0:.17f}', t_t)) - st = s() - - ''' - Find first sell index after trade open - - First index in t_open_slice where ['sell'] = 1 - ''' - # Use numpy array for faster search for sell - # Sell uses column 3. - # buy 0 - open 1 - close 2 - sell 3 - high 4 - low 5 - # Numpy searches 25-35x quicker than pandas on this data - - np_t_sell_ind = utf1st.find_1st(np_t_open_v[:, np_sell], - 1, utf1st.cmp_equal) \ - + t_open_ind - - if debug_timing: - t_t = f(st) - print("4-numpy", str.format('{0:.17f}', t_t)) - st = s() - - ''' - Determine which was hit first stop or sell, use as exit - - STOP takes priority over SELL as would be 'in candle' from tick data - Sell would use Open from Next candle. - So in a draw Stop would be hit first on ticker data in live - ''' - if np_t_stop_ind <= np_t_sell_ind: - t_exit_ind = np_t_stop_ind # Set Exit row index - t_exit_type = 'stop' # Set Exit type (sell|stop) - np_t_exit_pri = np_sco # The price field our STOP exit will use - else: - # move sell onto next candle, we only look back on sell - # will use the open price later. - t_exit_ind = np_t_sell_ind # Set Exit row index - t_exit_type = 'sell' # Set Exit type (sell|stop) - np_t_exit_pri = np_open # The price field our SELL exit will use - - if debug_timing: - t_t = f(st) - print("5-logic", str.format('{0:.17f}', t_t)) - st = s() - - if debug: ''' - Print out the buys, stops, sells - Include Line before and after to for easy - Human verification + 2 - Calculate our stop-loss price + + As stop is based on buy price of our trade + - (BTO)Buys are Triggered On np_bto, typically the CLOSE of candle + - (BCO)Buys are Calculated On np_bco, default is OPEN of the next candle. + This is as we only see the CLOSE after it has happened. + The back test assumption is we have bought at first available price, the OPEN + + Requires: np_bslap - is our numpy array of the ticker DataFrame + Requires: t_open_ind - is the index row with the first buy. + Requires: p_stop - is the stop rate, ie. 0.99 is -1% + Provides: np_t_stop_pri - The value stop-loss will be triggered on ''' - # Combine the np_t_stop_pri value to bslap dataframe to make debug - # life easy. This is the currenct stop price based on buy price_ - # Don't care about performance in debug - # (add an extra column if printing as df has date in col1 not in npy) - bslap['np_stop_pri'] = np_t_stop_pri + np_t_stop_pri = (np_bslap[t_open_ind + 1, np_bco] * p_stop) - # Buy - print("=================== BUY ", pair) - print("Numpy Array BUY Index is:", t_open_ind) - print("DataFrame BUY Index is:", t_open_ind + 1, "displaying DF \n") - print("HINT, BUY trade should use OPEN price from next candle, i.e ", t_open_ind + 2, "\n") - op_is = t_open_ind - 1 # Print open index start, line before - op_if = t_open_ind + 3 # Print open index finish, line after - print(bslap.iloc[op_is:op_if], "\n") - print(bslap.iloc[t_open_ind + 1]['date']) + if debug: + print("\n(2) numpy debug\nStop-Loss has been calculated at:", np_t_stop_pri) + if debug_timing: + t_t = f(st) + print("2-numpy", str.format('{0:.17f}', t_t)) + st = s() - # Stop - Stops trigger price sto, and price received sco. (Stop Trigger|Calculated On) - print("=================== STOP ", pair) - print("Numpy Array STOP Index is:", np_t_stop_ind) - print("DataFrame STOP Index is:", np_t_stop_ind + 1, "displaying DF \n") - print("First Stop after Trade open in candle", t_open_ind + 1, "is ", np_t_stop_ind + 1,": \n", - str.format('{0:.17f}', bslap.iloc[np_t_stop_ind][sto]), - "is less than", str.format('{0:.17f}', np_t_stop_pri)) - print("If stop is first exit match sell rate is :", str.format('{0:.17f}', bslap.iloc[np_t_stop_ind][sco])) - print("HINT, STOPs should close in-candle, i.e", np_t_stop_ind + 1, - ": As live STOPs are not linked to O-C times") + ''' + 3 - Find candle STO is under Stop-Loss After Trade opened. + + where [np_sto] (stop tiggered on variable: "close", "low" etc) < np_t_stop_pri + + Requires: np_t_open_v Numpy view of ticker_data after trade open + Requires: np_sto User Var(STO)StopTriggeredOn. Typically set to "low" or "close" + Requires: np_t_stop_pri The stop-loss price STO must fall under to trigger stop + Provides: np_t_stop_ind The first candle after trade open where STO is under stop-loss + ''' + np_t_stop_ind = utf1st.find_1st(np_t_open_v[:, np_sto], + np_t_stop_pri, + utf1st.cmp_smaller) - st_is = np_t_stop_ind - 1 # Print stop index start, line before - st_if = np_t_stop_ind + 2 # Print stop index finish, line after - print(bslap.iloc[st_is:st_if], "\n") + if debug: + print("\n(3) numpy debug\nNext view index with STO (stop trigger on) under Stop-Loss is", np_t_stop_ind, + ". STO is using field", np_sto, + "\nFrom key: buy 0 - open 1 - close 2 - sell 3 - high 4 - low 5\n") - # Sell - print("=================== SELL ", pair) - print("Numpy Array SELL Index is:", np_t_sell_ind) - print("DataFrame SELL Index is:", np_t_sell_ind + 1, "displaying DF \n") - print("First Sell Index after Trade open in in candle", np_t_sell_ind + 1) - print("HINT, if exit is SELL (not stop) trade should use OPEN price from next candle", - np_t_sell_ind + 2, "\n") - sl_is = np_t_sell_ind - 1 # Print sell index start, line before - sl_if = np_t_sell_ind + 3 # Print sell index finish, line after - print(bslap.iloc[sl_is:sl_if], "\n") + print("If -1 returned there is no stop found to end of view, then next two array lines are garbage") + print("Row", np_t_stop_ind, np_t_open_v[np_t_stop_ind]) + print("Row", np_t_stop_ind + 1, np_t_open_v[np_t_stop_ind + 1]) + if debug_timing: + t_t = f(st) + print("3-numpy", str.format('{0:.17f}', t_t)) + st = s() - # Chosen Exit (stop or sell) - print("=================== EXIT ", pair) - print("Exit type is :", t_exit_type) - # print((bslap.iloc[t_exit_ind], "\n")) - print("trade exit price field is", np_t_exit_pri, "\n") + ''' + 4 - Find first sell index after trade open + + First index in the view np_t_open_v where ['sell'] = 1 + + Requires: np_t_open_v - view of ticker_data from buy onwards + Requires: no_sell - integer '3', the buy column in the array + Provides: np_t_sell_ind index of view where first sell=1 after buy + ''' + # Use numpy array for faster search for sell + # Sell uses column 3. + # buy 0 - open 1 - close 2 - sell 3 - high 4 - low 5 + # Numpy searches 25-35x quicker than pandas on this data - ''' - Trade entry is always the next candles "open" price - We trigger on close, so cannot see that till after - its closed. + np_t_sell_ind = utf1st.find_1st(np_t_open_v[:, np_sell], + 1, utf1st.cmp_equal) + if debug: + print("\n(4) numpy debug\nNext view index with sell = 1 is ", np_t_sell_ind) + print("If 0 or less is returned there is no sell found to end of view, then next lines garbage") + print("Row", np_t_sell_ind, np_t_open_v[np_t_stop_ind]) + print("Row", np_t_sell_ind + 1, np_t_open_v[np_t_stop_ind + 1]) + if debug_timing: + t_t = f(st) + print("4-numpy", str.format('{0:.17f}', t_t)) + st = s() - The exception to this is a STOP which is calculated in candle - ''' - if debug_timing: - t_t = f(st) - print("6-depra", str.format('{0:.17f}', t_t)) - st = s() + ''' + 5 - Determine which was hit first a stop or sell + To then use as exit index price-field (sell on buy, stop on stop) + + STOP takes priority over SELL as would be 'in candle' from tick data + Sell would use Open from Next candle. + So in a draw Stop would be hit first on ticker data in live + + Validity of when types of trades may be executed can be summarised as: + + Tick View + index index Buy Sell open low close high Stop price + open 2am 94 -1 0 0 ----- ------ ------ ----- ----- + open 3am 95 0 1 0 ----- ------ trg buy ----- ----- + open 4am 96 1 0 1 Enter trgstop trg sel ROI out Stop out + open 5am 97 2 0 0 Exit ------ ------- ----- ----- + open 6am 98 3 0 0 ----- ------ ------- ----- ----- + + -1 means not found till end of view i.e no valid Stop found. Exclude from match. + Stop tiggering in 1, candle we bought at OPEN is valid. + + Buys and sells are triggered at candle close + Both with action their postions at the open of the next candle Index + 1 + + Stop and buy Indexes are on the view. To map to the ticker dataframe + the t_open_ind index should be summed. + + np_t_stop_ind: Stop Found index in view + t_exit_ind : Sell found in view + t_open_ind : Where view was started on ticker_data + + TODO: fix this frig for logig test,, case/switch/dictionary would be better... + more so when later testing many options, dynamic stop / roi etc + cludge - Im setting np_t_sell_ind as 9999999999 when -1 (not found) + cludge - Im setting np_t_stop_ind as 9999999999 when -1 (not found) + + ''' + if debug: + print("\n(5) numpy debug\nStop or Sell Logic Processing") - ## use numpy view "np_t_open_v" for speed. Columns are - # buy 0 - open 1 - close 2 - sell 3 - high 4 - low 5 - # exception is 6 which is use the stop value. + # cludge for logic test (-1) means it was not found, set crazy high to lose < test + np_t_sell_ind = 99999999 if np_t_sell_ind <= 0 else np_t_sell_ind + np_t_stop_ind = 99999999 if np_t_stop_ind == -1 else np_t_stop_ind - np_trade_enter_price = np_bslap[t_open_ind + 1, np_open] - if t_exit_type == 'stop': - if np_t_exit_pri == 6: - np_trade_exit_price = np_t_stop_pri + # Stoploss trigger found before a sell =1 + if np_t_stop_ind < 99999999 and np_t_stop_ind <= np_t_sell_ind: + t_exit_ind = t_open_ind + np_t_stop_ind # Set Exit row index + t_exit_type = 'stop' # Set Exit type (stop) + np_t_exit_pri = np_sco # The price field our STOP exit will use + if debug: + print("Type STOP is first exit condition. " + "At view index:", np_t_stop_ind, ". Ticker data exit index is", t_exit_ind) + + # Buy = 1 found before a stoploss triggered + elif np_t_sell_ind < 99999999 and np_t_sell_ind < np_t_stop_ind: + # move sell onto next candle, we only look back on sell + # will use the open price later. + t_exit_ind = t_open_ind + np_t_sell_ind + 1 # Set Exit row index + t_exit_type = 'sell' # Set Exit type (sell) + np_t_exit_pri = np_open # The price field our SELL exit will use + if debug: + print("Type SELL is first exit condition. " + "At view index", np_t_sell_ind, ". Ticker data exit index is", t_exit_ind) + + # No stop or buy left in view - set t_exit_last -1 to handle gracefully else: + t_exit_last: int = -1 # Signal loop to exit, no buys or sells found. + t_exit_type = "No Exit" + np_t_exit_pri = 999 # field price should be calculated on. 999 a non-existent column + if debug: + print("No valid STOP or SELL found. Signalling t_exit_last to gracefully exit") + + # TODO: fix having to cludge/uncludge this .. + # Undo cludge + np_t_sell_ind = -1 if np_t_sell_ind == 99999999 else np_t_sell_ind + np_t_stop_ind = -1 if np_t_stop_ind == 99999999 else np_t_stop_ind + + if debug_timing: + t_t = f(st) + print("5-logic", str.format('{0:.17f}', t_t)) + st = s() + + if debug: + ''' + Print out the buys, stops, sells + Include Line before and after to for easy + Human verification + ''' + # Combine the np_t_stop_pri value to bslap dataframe to make debug + # life easy. This is the current stop price based on buy price_ + # This is slow but don't care about performance in debug + # + # When referencing equiv np_column, as example np_sto, its 5 in numpy and 6 in df, so +1 + # as there is no data column in the numpy array. + bslap['np_stop_pri'] = np_t_stop_pri + + # Buy + print("\n\nDATAFRAME DEBUG =================== BUY ", pair) + print("Numpy Array BUY Index is:", 0) + print("DataFrame BUY Index is:", t_open_ind, "displaying DF \n") + print("HINT, BUY trade should use OPEN price from next candle, i.e ", t_open_ind + 1) + op_is = t_open_ind - 1 # Print open index start, line before + op_if = t_open_ind + 3 # Print open index finish, line after + print(bslap.iloc[op_is:op_if], "\n") + + # Stop - Stops trigger price np_sto (+1 for pandas column), and price received np_sco +1. (Stop Trigger|Calculated On) + if np_t_stop_ind < 0: + print("DATAFRAME DEBUG =================== STOP ", pair) + print("No STOPS were found until the end of ticker data file\n") + else: + print("DATAFRAME DEBUG =================== STOP ", pair) + print("Numpy Array STOP Index is:", np_t_stop_ind, "View starts at index", t_open_ind) + df_stop_index = (t_open_ind + np_t_stop_ind) + + print("DataFrame STOP Index is:", df_stop_index, "displaying DF \n") + print("First Stoploss trigger after Trade entered at OPEN in candle", t_open_ind + 1, "is ", + df_stop_index, ": \n", + str.format('{0:.17f}', bslap.iloc[df_stop_index][np_sto + 1]), + "is less than", str.format('{0:.17f}', np_t_stop_pri)) + + print("A stoploss exit will be calculated at rate:", + str.format('{0:.17f}', bslap.iloc[df_stop_index][np_sco + 1])) + + print("\nHINT, STOPs should exit in-candle, i.e", df_stop_index, + ": As live STOPs are not linked to O-C times") + + st_is = df_stop_index - 1 # Print stop index start, line before + st_if = df_stop_index + 2 # Print stop index finish, line after + print(bslap.iloc[st_is:st_if], "\n") + + # Sell + if np_t_sell_ind < 0: + print("DATAFRAME DEBUG =================== SELL ", pair) + print("No SELLS were found till the end of ticker data file\n") + else: + print("DATAFRAME DEBUG =================== SELL ", pair) + print("Numpy View SELL Index is:", np_t_sell_ind, "View starts at index", t_open_ind) + df_sell_index = (t_open_ind + np_t_sell_ind) + + print("DataFrame SELL Index is:", df_sell_index, "displaying DF \n") + print("First Sell Index after Trade open is in candle", df_sell_index) + print("HINT, if exit is SELL (not stop) trade should use OPEN price from next candle", + df_sell_index + 1) + sl_is = df_sell_index - 1 # Print sell index start, line before + sl_if = df_sell_index + 3 # Print sell index finish, line after + print(bslap.iloc[sl_is:sl_if], "\n") + + # Chosen Exit (stop or sell) + + print("DATAFRAME DEBUG =================== EXIT ", pair) + print("Exit type is :", t_exit_type) + print("trade exit price field is", np_t_exit_pri, "\n") + + if debug_timing: + t_t = f(st) + print("6-depra", str.format('{0:.17f}', t_t)) + st = s() + + ## use numpy view "np_t_open_v" for speed. Columns are + # buy 0 - open 1 - close 2 - sell 3 - high 4 - low 5 + # exception is 6 which is use the stop value. + + # TODO no! this is hard coded bleh fix this open + np_trade_enter_price = np_bslap[t_open_ind + 1, np_open] + if t_exit_type == 'stop': + if np_t_exit_pri == 6: + np_trade_exit_price = np_t_stop_pri + else: + np_trade_exit_price = np_bslap[t_exit_ind, np_t_exit_pri] + if t_exit_type == 'sell': np_trade_exit_price = np_bslap[t_exit_ind, np_t_exit_pri] - if t_exit_type == 'sell': - np_trade_exit_price = np_bslap[t_exit_ind + 1, np_t_exit_pri] - if debug_timing: - t_t = f(st) - print("7-numpy", str.format('{0:.17f}', t_t)) - st = s() + # Catch no exit found + if t_exit_type == "No Exit": + np_trade_exit_price = 0 - if debug: - print("//////////////////////////////////////////////") - print("+++++++++++++++++++++++++++++++++ Trade Enter ") - print("np_trade Enterprice is ", str.format('{0:.17f}', np_trade_enter_price)) - print("--------------------------------- Trade Exit ") - print("Trade Exit Type is ", t_exit_type) - print("np_trade Exit Price is", str.format('{0:.17f}', np_trade_exit_price)) - print("//////////////////////////////////////////////") + if debug_timing: + t_t = f(st) + print("7-numpy", str.format('{0:.17f}', t_t)) + st = s() + + if debug: + print("//////////////////////////////////////////////") + print("+++++++++++++++++++++++++++++++++ Trade Enter ") + print("np_trade Enter Price is ", str.format('{0:.17f}', np_trade_enter_price)) + print("--------------------------------- Trade Exit ") + print("Trade Exit Type is ", t_exit_type) + print("np_trade Exit Price is", str.format('{0:.17f}', np_trade_exit_price)) + print("//////////////////////////////////////////////") + + else: # no buys were found, step 0 returned -1 + # Gracefully exit the loop + t_exit_last == -1 + if debug: + print("\n(E) No buys were found in remaining ticker file. Exiting", pair) # Loop control - catch no closed trades. if debug: @@ -706,87 +826,47 @@ class Backtesting(object): " Dataframe Exit Index is: ", t_exit_ind) print("Exit Index Last, Exit Index Now Are: ", t_exit_last, t_exit_ind) - if t_exit_last >= t_exit_ind: + if t_exit_last >= t_exit_ind or t_exit_last == -1: """ + Break loop and go on to next pair. + When last trade exit equals index of last exit, there is no opportunity to close any more trades. - - Break loop and go on to next pair. - - TODO - add handing here to record none closed open trades """ - + # TODO :add handing here to record none closed open trades if debug: print(bslap_pair_results) - break else: """ - Add trade to backtest looking results list of dicts - Loop back to look for more trades. - """ - if debug_timing: - t_t = f(st) - print("8a-IfEls", str.format('{0:.17f}', t_t)) - st = s() - # Index will change if incandle stop or look back as close Open and Sell - if t_exit_type == 'stop': - close_index: int = t_exit_ind + 1 - elif t_exit_type == 'sell': - close_index: int = t_exit_ind + 2 - else: - close_index: int = t_exit_ind + 1 - - if debug_timing: - t_t = f(st) - print("8b-Index", str.format('{0:.17f}', t_t)) - st = s() - - # # Profit ABS. - # # sumrecieved((rate * numTokens) * fee) - sumpaid ((rate * numTokens) * fee) - # sumpaid: float = (np_trade_enter_price * stake) - # sumpaid_fee: float = sumpaid * open_fee - # sumrecieved: float = (np_trade_exit_price * stake) - # sumrecieved_fee: float = sumrecieved * close_fee - # profit_abs: float = sumrecieved - sumpaid - sumpaid_fee - sumrecieved_fee - - if debug_timing: - t_t = f(st) - print("8d---ABS", str.format('{0:.17f}', t_t)) - st = s() - + Add trade to backtest looking results list of dicts + Loop back to look for more trades. + """ # Build trade dictionary ## In general if a field can be calculated later from other fields leave blank here - ## Its X(numer of trades faster) to calc all in a single vector than 1 trade at a time + ## Its X(number of trades faster) to calc all in a single vector than 1 trade at a time + # create a new dict + close_index: int = t_exit_ind + bslap_result = {} # Must have at start or we end up with a list of multiple same last result bslap_result["pair"] = pair - bslap_result["profit_percent"] = "1" # To be 1 vector calc across trades when loop complete - bslap_result["profit_abs"] = "1" # To be 1 vector calc across trades when loop complete + bslap_result["profit_percent"] = "" # To be 1 vector calc across trades when loop complete + bslap_result["profit_abs"] = "" # To be 1 vector calc across trades when loop complete bslap_result["open_time"] = np_bslap_dates[t_open_ind + 1] # use numpy array, pandas 20x slower bslap_result["close_time"] = np_bslap_dates[close_index] # use numpy array, pandas 20x slower - bslap_result["open_index"] = t_open_ind + 2 # +1 between np and df, +1 as we buy on next. + bslap_result["open_index"] = t_open_ind + 1 # +1 as we buy on next. bslap_result["close_index"] = close_index - bslap_result["trade_duration"] = "1" # To be 1 vector calc across trades when loop complete + bslap_result["trade_duration"] = "" # To be 1 vector calc across trades when loop complete bslap_result["open_at_end"] = False bslap_result["open_rate"] = round(np_trade_enter_price, 15) bslap_result["close_rate"] = round(np_trade_exit_price, 15) bslap_result["exit_type"] = t_exit_type - - if debug_timing: - t_t = f(st) - print("8e-trade", str.format('{0:.17f}', t_t)) - st = s() - # Add trade dictionary to list + # append the dict to the list and print list bslap_pair_results.append(bslap_result) if debug: - print(bslap_pair_results) - - if debug_timing: - t_t = f(st) - print("8f--list", str.format('{0:.17f}', t_t)) - st = s() + print("The trade dict is: \n", bslap_result) + print("Trades dicts in list after append are: \n ", bslap_pair_results) """ Loop back to start. t_exit_last becomes where loop @@ -802,9 +882,6 @@ class Backtesting(object): # Send back List of trade dicts return bslap_pair_results - - - def start(self) -> None: """ Run a backtesting end-to-end @@ -868,9 +945,9 @@ class Backtesting(object): self._store_backtest_result(self.config.get('exportfilename'), results) logger.info( - '\n================================================= ' - 'BACKTESTING REPORT' - ' ==================================================\n' + '\n====================================================== ' + 'BackSLAP REPORT' + ' =======================================================\n' '%s', self._generate_text_table( data, From eed29a6b8af3a1746c16cb142f6e418072ce6042 Mon Sep 17 00:00:00 2001 From: creslinux Date: Mon, 16 Jul 2018 13:16:18 +0000 Subject: [PATCH 007/699] update --- freqtrade/optimize/backtesting.py | 29 +++++++++++++++++++++++++++-- 1 file changed, 27 insertions(+), 2 deletions(-) diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index 2629ed5fa..d17b7b246 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -24,6 +24,7 @@ from freqtrade.persistence import Trade from profilehooks import profile from collections import OrderedDict import timeit +from time import sleep logger = logging.getLogger(__name__) @@ -191,7 +192,14 @@ class Backtesting(object): return btr return None - @profile + def s(self): + st = timeit.default_timer() + return st + + def f(self, st): + return (timeit.default_timer() - st) + + def backtest(self, args: Dict) -> DataFrame: """ Implements backtesting functionality @@ -207,21 +215,31 @@ class Backtesting(object): realistic: do we try to simulate realistic trades? (default: True) :return: DataFrame """ + debug_timing = False + headers = ['date', 'buy', 'open', 'close', 'sell', 'high', 'low'] processed = args['processed'] max_open_trades = args.get('max_open_trades', 0) realistic = args.get('realistic', False) trades = [] trade_count_lock: Dict = {} + ########################### Call out BSlap instead of using FT bslap_results: list = [] - last_bslap_results: list = [] + for pair, pair_data in processed.items(): + if debug_timing: # Start timer + fl = self.s() + ticker_data = self.populate_sell_trend( self.populate_buy_trend(pair_data))[headers].copy() ticker_data.drop(ticker_data.head(1).index, inplace=True) + if debug_timing: # print time taken + flt = self.f(fl) + print("populate_buy_trend:", pair, round(flt, 10)) + st = self.st() # #dump same DFs to disk for offline testing in scratch # f_pair:str = pair @@ -234,11 +252,18 @@ class Backtesting(object): last_bslap_results = bslap_results bslap_results = last_bslap_results + bslap_pair_results + if debug_timing: # print time taken + tt = self.f(st) + print("Time to Back Slap :", pair, round(tt,10)) + print("-----------------------") + + # Switch List of Trade Dicts (bslap_results) to Dataframe # Fill missing, calculable columns, profit, duration , abs etc. bslap_results_df = DataFrame(bslap_results) bslap_results_df['open_time'] = to_datetime(bslap_results_df['open_time']) bslap_results_df['close_time'] = to_datetime(bslap_results_df['close_time']) + ### don't use this, itll drop exit type field # bslap_results_df = DataFrame(bslap_results, columns=BacktestResult._fields) From fb0edd71ff166a353613a1f14de42a27163ea92b Mon Sep 17 00:00:00 2001 From: creslinux Date: Mon, 16 Jul 2018 14:16:35 +0000 Subject: [PATCH 008/699] in tech test --- freqtrade/optimize/backtesting.py | 276 ++++++++++++++++++------------ 1 file changed, 171 insertions(+), 105 deletions(-) diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index d17b7b246..bbb52d167 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -73,6 +73,35 @@ class Backtesting(object): self.stop_loss_value = self.analyze.strategy.stoploss + #### backslap config + ''' + Numpy arrays are used for 100x speed up + We requires setting Int values for + buy stop triggers and stop calculated on + # buy 0 - open 1 - close 2 - sell 3 - high 4 - low 5 - stop 6 + ''' + self.np_buy: int = 0 + self.np_open: int = 1 + self.np_close: int = 2 + self.np_sell: int = 3 + self.np_high: int = 4 + self.np_low: int = 5 + self.np_stop: int = 6 + self.np_bto: int = self.np_close # buys_triggered_on - should be close + self.np_bco: int = self.np_open # buys calculated on - open of the next candle. + # self.np_sto: int = self.np_low # stops_triggered_on - Should be low, FT uses close + # self.np_sco: int = self.np_stop # stops_calculated_on - Should be stop, FT uses close + self.np_sto: int = self.np_close # stops_triggered_on - Should be low, FT uses close + self.np_sco: int = self.np_close # stops_calculated_on - Should be stop, FT uses close + + self.use_backslap = True # Enable backslap + self.debug = False # Main debug enable, very print heavy, enable 2 loops recommended + self.debug_timing = False # Stages within Backslap + self.debug_2loops = False # Limit each pair to two loops, useful when debugging + self.debug_vector = False # Debug vector calcs + self.debug_timing_main_loop = False # print overall timing per pair + + @staticmethod def get_timeframe(data: Dict[str, DataFrame]) -> Tuple[arrow.Arrow, arrow.Arrow]: """ @@ -215,7 +244,7 @@ class Backtesting(object): realistic: do we try to simulate realistic trades? (default: True) :return: DataFrame """ - debug_timing = False + headers = ['date', 'buy', 'open', 'close', 'sell', 'high', 'low'] processed = args['processed'] @@ -224,99 +253,114 @@ class Backtesting(object): trades = [] trade_count_lock: Dict = {} - ########################### Call out BSlap instead of using FT - bslap_results: list = [] + use_backslap = self.use_backslap + debug_timing = self.debug_timing_main_loop + if use_backslap: # Use Back Slap code + ########################### Call out BSlap Loop instead of Original BT code + bslap_results: list = [] + for pair, pair_data in processed.items(): + if debug_timing: # Start timer + fl = self.s() + + ticker_data = self.populate_sell_trend( + self.populate_buy_trend(pair_data))[headers].copy() + + if debug_timing: # print time taken + flt = self.f(fl) + #print("populate_buy_trend:", pair, round(flt, 10)) + st = self.s() + + # #dump same DFs to disk for offline testing in scratch + # f_pair:str = pair + # csv = f_pair.replace("/", "_") + # csv="/Users/creslin/PycharmProjects/freqtrade_new/frames/" + csv + # ticker_data.to_csv(csv, sep='\t', encoding='utf-8') + + #call bslap - results are a list of dicts + bslap_pair_results = self.backslap_pair(ticker_data, pair) + last_bslap_results = bslap_results + bslap_results = last_bslap_results + bslap_pair_results + + if debug_timing: # print time taken + tt = self.f(st) + print("Time to BackSlap :", pair, round(tt,10)) + print("-----------------------") - for pair, pair_data in processed.items(): - if debug_timing: # Start timer - fl = self.s() + # Switch List of Trade Dicts (bslap_results) to Dataframe + # Fill missing, calculable columns, profit, duration , abs etc. + bslap_results_df = DataFrame(bslap_results) + bslap_results_df['open_time'] = to_datetime(bslap_results_df['open_time']) + bslap_results_df['close_time'] = to_datetime(bslap_results_df['close_time']) - ticker_data = self.populate_sell_trend( + ### don't use this, itll drop exit type field + # bslap_results_df = DataFrame(bslap_results, columns=BacktestResult._fields) + + bslap_results_df = self.vector_fill_results_table(bslap_results_df) + + return bslap_results_df + + else: # use Original Back test code + ########################## Original BT loop + + for pair, pair_data in processed.items(): + if debug_timing: # Start timer + fl = self.s() + + pair_data['buy'], pair_data['sell'] = 0, 0 # cleanup from previous run + + ticker_data = self.populate_sell_trend( self.populate_buy_trend(pair_data))[headers].copy() - ticker_data.drop(ticker_data.head(1).index, inplace=True) - if debug_timing: # print time taken - flt = self.f(fl) - print("populate_buy_trend:", pair, round(flt, 10)) - st = self.st() + # to avoid using data from future, we buy/sell with signal from previous candle + ticker_data.loc[:, 'buy'] = ticker_data['buy'].shift(1) + ticker_data.loc[:, 'sell'] = ticker_data['sell'].shift(1) - # #dump same DFs to disk for offline testing in scratch - # f_pair:str = pair - # csv = f_pair.replace("/", "_") - # csv="/Users/creslin/PycharmProjects/freqtrade_new/frames/" + csv - # ticker_data.to_csv(csv, sep='\t', encoding='utf-8') + ticker_data.drop(ticker_data.head(1).index, inplace=True) - #call bslap - results are a list of dicts - bslap_pair_results = self.backslap_pair(ticker_data, pair) - last_bslap_results = bslap_results - bslap_results = last_bslap_results + bslap_pair_results + if debug_timing: # print time taken + flt = self.f(fl) + #print("populate_buy_trend:", pair, round(flt, 10)) + st = self.s() - if debug_timing: # print time taken - tt = self.f(st) - print("Time to Back Slap :", pair, round(tt,10)) - print("-----------------------") + # Convert from Pandas to list for performance reasons + # (Looping Pandas is slow.) + ticker = [x for x in ticker_data.itertuples()] + + lock_pair_until = None + for index, row in enumerate(ticker): + if row.buy == 0 or row.sell == 1: + continue # skip rows where no buy signal or that would immediately sell off + + if realistic: + if lock_pair_until is not None and row.date <= lock_pair_until: + continue + if max_open_trades > 0: + # Check if max_open_trades has already been reached for the given date + if not trade_count_lock.get(row.date, 0) < max_open_trades: + continue + + trade_count_lock[row.date] = trade_count_lock.get(row.date, 0) + 1 + + trade_entry = self._get_sell_trade_entry(pair, row, ticker[index + 1:], + trade_count_lock, args) - # Switch List of Trade Dicts (bslap_results) to Dataframe - # Fill missing, calculable columns, profit, duration , abs etc. - bslap_results_df = DataFrame(bslap_results) - bslap_results_df['open_time'] = to_datetime(bslap_results_df['open_time']) - bslap_results_df['close_time'] = to_datetime(bslap_results_df['close_time']) + if trade_entry: + lock_pair_until = trade_entry.close_time + trades.append(trade_entry) + else: + # Set lock_pair_until to end of testing period if trade could not be closed + # This happens only if the buy-signal was with the last candle + lock_pair_until = ticker_data.iloc[-1].date - ### don't use this, itll drop exit type field - # bslap_results_df = DataFrame(bslap_results, columns=BacktestResult._fields) + if debug_timing: # print time taken + tt = self.f(st) + print("Time to BackTest :", pair, round(tt, 10)) + print("-----------------------") - bslap_results_df = self.vector_fill_results_table(bslap_results_df) - - return bslap_results_df - - ########################### Original BT loop - # for pair, pair_data in processed.items(): - # pair_data['buy'], pair_data['sell'] = 0, 0 # cleanup from previous run - # - # ticker_data = self.populate_sell_trend( - # self.populate_buy_trend(pair_data))[headers].copy() - # - # # to avoid using data from future, we buy/sell with signal from previous candle - # ticker_data.loc[:, 'buy'] = ticker_data['buy'].shift(1) - # ticker_data.loc[:, 'sell'] = ticker_data['sell'].shift(1) - # - # ticker_data.drop(ticker_data.head(1).index, inplace=True) - # - # # Convert from Pandas to list for performance reasons - # # (Looping Pandas is slow.) - # ticker = [x for x in ticker_data.itertuples()] - # - # lock_pair_until = None - # for index, row in enumerate(ticker): - # if row.buy == 0 or row.sell == 1: - # continue # skip rows where no buy signal or that would immediately sell off - # - # if realistic: - # if lock_pair_until is not None and row.date <= lock_pair_until: - # continue - # if max_open_trades > 0: - # # Check if max_open_trades has already been reached for the given date - # if not trade_count_lock.get(row.date, 0) < max_open_trades: - # continue - # - # trade_count_lock[row.date] = trade_count_lock.get(row.date, 0) + 1 - # - # trade_entry = self._get_sell_trade_entry(pair, row, ticker[index + 1:], - # trade_count_lock, args) - # - # - # if trade_entry: - # lock_pair_until = trade_entry.close_time - # trades.append(trade_entry) - # else: - # # Set lock_pair_until to end of testing period if trade could not be closed - # # This happens only if the buy-signal was with the last candle - # lock_pair_until = ticker_data.iloc[-1].date - # - # return DataFrame.from_records(trades, columns=BacktestResult._fields) - ######################## Original BT loop end + return DataFrame.from_records(trades, columns=BacktestResult._fields) + ####################### Original BT loop end def vector_fill_results_table(self, bslap_results_df: DataFrame): """ @@ -333,13 +377,18 @@ class Backtesting(object): :return: bslap_results Dataframe """ import pandas as pd - debug = False + debug = self.debug_vector + + # stake and fees + # stake = 0.015 + # 0.05% is 0.0005 + #fee = 0.001 + + stake = self.config.get('stake_amount') + fee = self.fee + open_fee = fee / 2 + close_fee = fee / 2 - # stake and fees - stake = 0.015 - # 0.05% is 0.00,05 - open_fee = 0.0000 - close_fee = 0.0000 if debug: print("Stake is,", stake, "the sum of currency to spend per trade") print("The open fee is", open_fee, "The close fee is", close_fee) @@ -420,9 +469,12 @@ class Backtesting(object): from datetime import datetime ### backslap debug wrap - debug_2loops = False # only loop twice, for faster debug - debug_timing = False # print timing for each step - debug = False # print values, to check accuracy + # debug_2loops = False # only loop twice, for faster debug + # debug_timing = False # print timing for each step + # debug = False # print values, to check accuracy + debug_2loops = self.debug_2loops # only loop twice, for faster debug + debug_timing = self.debug_timing # print timing for each step + debug = self.debug # print values, to check accuracy # Read Stop Loss Values and Stake stop = self.stop_loss_value @@ -451,20 +503,34 @@ class Backtesting(object): buy stop triggers and stop calculated on # buy 0 - open 1 - close 2 - sell 3 - high 4 - low 5 - stop 6 ''' - np_buy: int = 0 - np_open: int = 1 - np_close: int = 2 - np_sell: int = 3 - np_high: int = 4 - np_low: int = 5 - np_stop: int = 6 - np_bto: int = np_close # buys_triggered_on - should be close - np_bco: int = np_open # buys calculated on - open of the next candle. - #np_sto: int = np_low # stops_triggered_on - Should be low, FT uses close - #np_sco: int = np_stop # stops_calculated_on - Should be stop, FT uses close - np_sto: int = np_close # stops_triggered_on - Should be low, FT uses close - np_sco: int = np_close # stops_calculated_on - Should be stop, FT uses close - # + # np_buy: int = 0 + # np_open: int = 1 + # np_close: int = 2 + # np_sell: int = 3 + # np_high: int = 4 + # np_low: int = 5 + # np_stop: int = 6 + # np_bto: int = np_close # buys_triggered_on - should be close + # np_bco: int = np_open # buys calculated on - open of the next candle. + # #np_sto: int = np_low # stops_triggered_on - Should be low, FT uses close + # #np_sco: int = np_stop # stops_calculated_on - Should be stop, FT uses close + # np_sto: int = np_close # stops_triggered_on - Should be low, FT uses close + # np_sco: int = np_close # stops_calculated_on - Should be stop, FT uses close + + ####### + # Use vars set at top of backtest + np_buy: int = self.np_buy + np_open: int = self.np_open + np_close: int = self.np_close + np_sell: int = self.np_sell + np_high: int = self.np_high + np_low: int = self.np_low + np_stop: int = self.np_stop + np_bto: int = self.np_bto # buys_triggered_on - should be close + np_bco: int = self.np_bco # buys calculated on - open of the next candle. + np_sto: int = self.np_sto # stops_triggered_on - Should be low, FT uses close + np_sco: int = self.np_sco # stops_calculated_on - Should be stop, FT uses close + ### End Config pair: str = pair From 5aaf454f120e75f4e228ece32ba5a396aa3446b4 Mon Sep 17 00:00:00 2001 From: creslinux Date: Mon, 16 Jul 2018 15:48:06 +0000 Subject: [PATCH 009/699] GAS trades verified from candle data to excel by hand All pass 3 sells 1 stop loss --- freqtrade/optimize/backtesting.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index bbb52d167..f254bc8a4 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -95,10 +95,10 @@ class Backtesting(object): self.np_sco: int = self.np_close # stops_calculated_on - Should be stop, FT uses close self.use_backslap = True # Enable backslap - self.debug = False # Main debug enable, very print heavy, enable 2 loops recommended + self.debug = True # Main debug enable, very print heavy, enable 2 loops recommended self.debug_timing = False # Stages within Backslap self.debug_2loops = False # Limit each pair to two loops, useful when debugging - self.debug_vector = False # Debug vector calcs + self.debug_vector = True # Debug vector calcs self.debug_timing_main_loop = False # print overall timing per pair @@ -255,6 +255,7 @@ class Backtesting(object): use_backslap = self.use_backslap debug_timing = self.debug_timing_main_loop + if use_backslap: # Use Back Slap code ########################### Call out BSlap Loop instead of Original BT code bslap_results: list = [] @@ -395,7 +396,7 @@ class Backtesting(object): if debug: from pandas import set_option set_option('display.max_rows', 5000) - set_option('display.max_columns', 8) + set_option('display.max_columns', 10) pd.set_option('display.width', 1000) pd.set_option('max_colwidth', 40) pd.set_option('precision', 12) @@ -422,8 +423,7 @@ class Backtesting(object): if debug: print("\n") print(bslap_results_df[ - ['buy_sum', 'buy_fee', 'buy_spend', 'sell_sum', 'sell_take', 'profit_percent', 'profit_abs', - 'exit_type']]) + ['buy_sum', 'buy_fee', 'buy_spend', 'sell_sum','sell_fee', 'sell_take', 'profit_percent', 'profit_abs', 'exit_type']]) return bslap_results_df @@ -708,8 +708,8 @@ class Backtesting(object): if debug: print("\n(4) numpy debug\nNext view index with sell = 1 is ", np_t_sell_ind) print("If 0 or less is returned there is no sell found to end of view, then next lines garbage") - print("Row", np_t_sell_ind, np_t_open_v[np_t_stop_ind]) - print("Row", np_t_sell_ind + 1, np_t_open_v[np_t_stop_ind + 1]) + print("Row", np_t_sell_ind, np_t_open_v[np_t_sell_ind]) + print("Row", np_t_sell_ind + 1, np_t_open_v[np_t_sell_ind + 1]) if debug_timing: t_t = f(st) print("4-numpy", str.format('{0:.17f}', t_t)) From 4a39a754f4811aa4531e8ce0a30019d391a72979 Mon Sep 17 00:00:00 2001 From: creslinux Date: Mon, 16 Jul 2018 15:57:15 +0000 Subject: [PATCH 010/699] Fixed: self.use_backslap = Bool on line97 If self.use_backslap = True Backslap executes If self.use_backslap = False Original Backtest Code executes --- freqtrade/optimize/backtesting.py | 57 ++++++++++++++++++++----------- 1 file changed, 38 insertions(+), 19 deletions(-) diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index f254bc8a4..191616c6c 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -94,12 +94,12 @@ class Backtesting(object): self.np_sto: int = self.np_close # stops_triggered_on - Should be low, FT uses close self.np_sco: int = self.np_close # stops_calculated_on - Should be stop, FT uses close - self.use_backslap = True # Enable backslap + self.use_backslap = True # Enable backslap - if false Orginal code is executed. self.debug = True # Main debug enable, very print heavy, enable 2 loops recommended self.debug_timing = False # Stages within Backslap self.debug_2loops = False # Limit each pair to two loops, useful when debugging self.debug_vector = True # Debug vector calcs - self.debug_timing_main_loop = False # print overall timing per pair + self.debug_timing_main_loop = False # print overall timing per pair - works in Backtest and Backslap @staticmethod @@ -245,18 +245,18 @@ class Backtesting(object): :return: DataFrame """ - - headers = ['date', 'buy', 'open', 'close', 'sell', 'high', 'low'] - processed = args['processed'] - max_open_trades = args.get('max_open_trades', 0) - realistic = args.get('realistic', False) - trades = [] - trade_count_lock: Dict = {} - use_backslap = self.use_backslap debug_timing = self.debug_timing_main_loop if use_backslap: # Use Back Slap code + + headers = ['date', 'buy', 'open', 'close', 'sell', 'high', 'low'] + processed = args['processed'] + max_open_trades = args.get('max_open_trades', 0) + realistic = args.get('realistic', False) + trades = [] + trade_count_lock: Dict = {} + ########################### Call out BSlap Loop instead of Original BT code bslap_results: list = [] for pair, pair_data in processed.items(): @@ -304,6 +304,13 @@ class Backtesting(object): else: # use Original Back test code ########################## Original BT loop + headers = ['date', 'buy', 'open', 'close', 'sell'] + processed = args['processed'] + max_open_trades = args.get('max_open_trades', 0) + realistic = args.get('realistic', False) + trades = [] + trade_count_lock: Dict = {} + for pair, pair_data in processed.items(): if debug_timing: # Start timer fl = self.s() @@ -1035,16 +1042,28 @@ class Backtesting(object): if self.config.get('export', False): self._store_backtest_result(self.config.get('exportfilename'), results) - logger.info( - '\n====================================================== ' - 'BackSLAP REPORT' - ' =======================================================\n' - '%s', - self._generate_text_table( - data, - results + if self.use_backslap: + logger.info( + '\n====================================================== ' + 'BackSLAP REPORT' + ' =======================================================\n' + '%s', + self._generate_text_table( + data, + results + ) + ) + else: + logger.info( + '\n================================================= ' + 'BACKTEST REPORT' + ' ==================================================\n' + '%s', + self._generate_text_table( + data, + results + ) ) - ) ## TODO. Catch open trades for this report. # logger.info( From 0f3339f74f30dec2910fc4c36e401bb390fc5053 Mon Sep 17 00:00:00 2001 From: creslinux Date: Mon, 16 Jul 2018 16:09:42 +0000 Subject: [PATCH 011/699] use ujson to load ticker files 30% faster from disk. --- freqtrade/optimize/__init__.py | 24 ++++++++++++++++++++---- requirements.txt | 7 +++++++ 2 files changed, 27 insertions(+), 4 deletions(-) diff --git a/freqtrade/optimize/__init__.py b/freqtrade/optimize/__init__.py index e806ff2b8..90dda79e2 100644 --- a/freqtrade/optimize/__init__.py +++ b/freqtrade/optimize/__init__.py @@ -10,9 +10,16 @@ import arrow from freqtrade import misc, constants, OperationalException from freqtrade.exchange import Exchange from freqtrade.arguments import TimeRange +import importlib +ujson_found = importlib.util.find_spec("ujson") +if ujson_found is not None: + import ujson logger = logging.getLogger(__name__) +if ujson_found is not None: + logger.debug('Loaded UltraJson ujson in optimize.py') + def trim_tickerlist(tickerlist: List[Dict], timerange: TimeRange) -> List[Dict]: if not tickerlist: @@ -63,11 +70,17 @@ def load_tickerdata_file( if os.path.isfile(gzipfile): logger.debug('Loading ticker data from file %s', gzipfile) with gzip.open(gzipfile) as tickerdata: - pairdata = json.load(tickerdata) + if ujson_found is not None: + pairdata = ujson.load(tickerdata, precise_float=True) + else: + pairdata = json.load(tickerdata) elif os.path.isfile(file): logger.debug('Loading ticker data from file %s', file) with open(file) as tickerdata: - pairdata = json.load(tickerdata) + if ujson_found is not None: + pairdata = ujson.load(tickerdata, precise_float=True) + else: + pairdata = json.load(tickerdata) else: return None @@ -163,7 +176,10 @@ def load_cached_data_for_updating(filename: str, # read the cached file if os.path.isfile(filename): with open(filename, "rt") as file: - data = json.load(file) + if ujson_found is not None: + data = ujson.load(file, precise_float=True) + else: + data = json.load(file) # remove the last item, because we are not sure if it is correct # it could be fetched when the candle was incompleted if data: @@ -226,4 +242,4 @@ def download_backtesting_testdata(datadir: str, logger.debug("New Start: %s", misc.format_ms_time(data[0][0])) logger.debug("New End: %s", misc.format_ms_time(data[-1][0])) - misc.file_dump_json(filename, data) + misc.file_dump_json(filename, data) \ No newline at end of file diff --git a/requirements.txt b/requirements.txt index 3fb91888c..99b471b1c 100644 --- a/requirements.txt +++ b/requirements.txt @@ -23,3 +23,10 @@ scikit-optimize==0.5.2 # Required for plotting data #plotly==3.0.0 + +# find first, C search in arrays +py_find_1st==1.1.1 + +#Load ticker files 30% faster +ujson==1.35 + From 059aceb582e04c2a64c5370ffb5381e7c1956385 Mon Sep 17 00:00:00 2001 From: creslinux Date: Mon, 16 Jul 2018 16:12:33 +0000 Subject: [PATCH 012/699] Disabled full debug on in last commit Switched Stops to trigger on Low Switched Stops to pay stop-rate not close. --- freqtrade/optimize/backtesting.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index 191616c6c..3aabfc2ad 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -89,17 +89,17 @@ class Backtesting(object): self.np_stop: int = 6 self.np_bto: int = self.np_close # buys_triggered_on - should be close self.np_bco: int = self.np_open # buys calculated on - open of the next candle. - # self.np_sto: int = self.np_low # stops_triggered_on - Should be low, FT uses close - # self.np_sco: int = self.np_stop # stops_calculated_on - Should be stop, FT uses close - self.np_sto: int = self.np_close # stops_triggered_on - Should be low, FT uses close - self.np_sco: int = self.np_close # stops_calculated_on - Should be stop, FT uses close + self.np_sto: int = self.np_low # stops_triggered_on - Should be low, FT uses close + self.np_sco: int = self.np_stop # stops_calculated_on - Should be stop, FT uses close + #self.np_sto: int = self.np_close # stops_triggered_on - Should be low, FT uses close + #self.np_sco: int = self.np_close # stops_calculated_on - Should be stop, FT uses close self.use_backslap = True # Enable backslap - if false Orginal code is executed. - self.debug = True # Main debug enable, very print heavy, enable 2 loops recommended + self.debug = False # Main debug enable, very print heavy, enable 2 loops recommended self.debug_timing = False # Stages within Backslap self.debug_2loops = False # Limit each pair to two loops, useful when debugging - self.debug_vector = True # Debug vector calcs - self.debug_timing_main_loop = False # print overall timing per pair - works in Backtest and Backslap + self.debug_vector = False # Debug vector calcs + self.debug_timing_main_loop = True # print overall timing per pair - works in Backtest and Backslap @staticmethod From 885a653439bd49e8d5b25d75422f2559dfc225aa Mon Sep 17 00:00:00 2001 From: creslinux Date: Mon, 16 Jul 2018 16:18:54 +0000 Subject: [PATCH 013/699] Disabled full debug on in last commit Switched Stops to trigger on Low Switched Stops to pay stop-rate not close. --- freqtrade/optimize/backtesting.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index 3aabfc2ad..774d4b954 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -98,7 +98,7 @@ class Backtesting(object): self.debug = False # Main debug enable, very print heavy, enable 2 loops recommended self.debug_timing = False # Stages within Backslap self.debug_2loops = False # Limit each pair to two loops, useful when debugging - self.debug_vector = False # Debug vector calcs + self.debug_vector = True # Debug vector calcs self.debug_timing_main_loop = True # print overall timing per pair - works in Backtest and Backslap From 99d16e82c0a4109060abe8e5675b6231bfa2b710 Mon Sep 17 00:00:00 2001 From: creslinux Date: Mon, 16 Jul 2018 16:30:11 +0000 Subject: [PATCH 014/699] disable time calcs output on vector displaying in debug. Excessive. --- freqtrade/optimize/backtesting.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index 774d4b954..0fd21b14e 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -89,10 +89,10 @@ class Backtesting(object): self.np_stop: int = 6 self.np_bto: int = self.np_close # buys_triggered_on - should be close self.np_bco: int = self.np_open # buys calculated on - open of the next candle. - self.np_sto: int = self.np_low # stops_triggered_on - Should be low, FT uses close - self.np_sco: int = self.np_stop # stops_calculated_on - Should be stop, FT uses close - #self.np_sto: int = self.np_close # stops_triggered_on - Should be low, FT uses close - #self.np_sco: int = self.np_close # stops_calculated_on - Should be stop, FT uses close + #self.np_sto: int = self.np_low # stops_triggered_on - Should be low, FT uses close + #self.np_sco: int = self.np_stop # stops_calculated_on - Should be stop, FT uses close + self.np_sto: int = self.np_close # stops_triggered_on - Should be low, FT uses close + self.np_sco: int = self.np_close # stops_calculated_on - Should be stop, FT uses close self.use_backslap = True # Enable backslap - if false Orginal code is executed. self.debug = False # Main debug enable, very print heavy, enable 2 loops recommended @@ -410,8 +410,8 @@ class Backtesting(object): # Populate duration bslap_results_df['trade_duration'] = bslap_results_df['close_time'] - bslap_results_df['open_time'] - if debug: - print(bslap_results_df[['open_time', 'close_time', 'trade_duration']]) + # if debug: + # print(bslap_results_df[['open_time', 'close_time', 'trade_duration']]) ## Spends, Takes, Profit, Absolute Profit # Buy Price From ec1960530b98a2a20d4c2dc6204c2d06765bebd2 Mon Sep 17 00:00:00 2001 From: creslinux Date: Mon, 16 Jul 2018 17:06:06 +0000 Subject: [PATCH 015/699] Added Show trades option If true, prints trades ordered by date after summary. Useful for spotting trends. --- freqtrade/optimize/backtesting.py | 25 +++++++++++++++++++------ 1 file changed, 19 insertions(+), 6 deletions(-) diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index 0fd21b14e..85e42b91a 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -94,12 +94,14 @@ class Backtesting(object): self.np_sto: int = self.np_close # stops_triggered_on - Should be low, FT uses close self.np_sco: int = self.np_close # stops_calculated_on - Should be stop, FT uses close - self.use_backslap = True # Enable backslap - if false Orginal code is executed. - self.debug = False # Main debug enable, very print heavy, enable 2 loops recommended - self.debug_timing = False # Stages within Backslap - self.debug_2loops = False # Limit each pair to two loops, useful when debugging - self.debug_vector = True # Debug vector calcs - self.debug_timing_main_loop = True # print overall timing per pair - works in Backtest and Backslap + self.use_backslap = True # Enable backslap - if false Orginal code is executed. + self.debug = False # Main debug enable, very print heavy, enable 2 loops recommended + self.debug_timing = False # Stages within Backslap + self.debug_2loops = False # Limit each pair to two loops, useful when debugging + self.debug_vector = False # Debug vector calcs + self.debug_timing_main_loop = False # print overall timing per pair - works in Backtest and Backslap + + self.backslap_show_trades = True #prints trades in addition to summary report, also saves to backslap.txt @staticmethod @@ -1053,6 +1055,17 @@ class Backtesting(object): results ) ) + if self.backslap_show_trades: + TradesFrame = results.filter(['open_time', 'pair', 'exit_type', 'profit_percent', 'profit_abs', + 'buy_spend', 'sell_take', 'trade_duration', 'close_time'], axis=1) + + def to_fwf(df, fname): + content = tabulate(df.values.tolist(), list(df.columns), floatfmt=".8f", tablefmt='psql') + print(content) + open(fname, "w").write(content) + + DataFrame.to_fwf = to_fwf(TradesFrame, "backslap.txt") + else: logger.info( '\n================================================= ' From 8d5da4e6ad9ffc93b7ea0b9c6b49cbfa4406ce56 Mon Sep 17 00:00:00 2001 From: creslinux Date: Mon, 16 Jul 2018 17:48:11 +0000 Subject: [PATCH 016/699] changed defaults Seperated save trades and print trades options. --- freqtrade/optimize/backtesting.py | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index 85e42b91a..43e05d70b 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -101,7 +101,8 @@ class Backtesting(object): self.debug_vector = False # Debug vector calcs self.debug_timing_main_loop = False # print overall timing per pair - works in Backtest and Backslap - self.backslap_show_trades = True #prints trades in addition to summary report, also saves to backslap.txt + self.backslap_show_trades = False # prints trades in addition to summary report + self.backslap_save_trades = True # saves trades as a pretty table to backslap.txt @staticmethod @@ -1055,17 +1056,25 @@ class Backtesting(object): results ) ) + #optional print trades if self.backslap_show_trades: TradesFrame = results.filter(['open_time', 'pair', 'exit_type', 'profit_percent', 'profit_abs', 'buy_spend', 'sell_take', 'trade_duration', 'close_time'], axis=1) - def to_fwf(df, fname): content = tabulate(df.values.tolist(), list(df.columns), floatfmt=".8f", tablefmt='psql') print(content) - open(fname, "w").write(content) DataFrame.to_fwf = to_fwf(TradesFrame, "backslap.txt") + #optional save trades + if self.backslap_save_trades: + TradesFrame = results.filter(['open_time', 'pair', 'exit_type', 'profit_percent', 'profit_abs', + 'buy_spend', 'sell_take', 'trade_duration', 'close_time'], axis=1) + def to_fwf(df, fname): + content = tabulate(df.values.tolist(), list(df.columns), floatfmt=".8f", tablefmt='psql') + open(fname, "w").write(content) + DataFrame.to_fwf = to_fwf(TradesFrame, "backslap.txt") + else: logger.info( '\n================================================= ' From 3b0cb7bc333c980523e38abd5d70ba9c6a24113f Mon Sep 17 00:00:00 2001 From: creslinux Date: Mon, 16 Jul 2018 18:06:31 +0000 Subject: [PATCH 017/699] Added ujson and py_find_1st to setup.py --- setup.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/setup.py b/setup.py index cd0574fa2..4ec541674 100644 --- a/setup.py +++ b/setup.py @@ -37,6 +37,8 @@ setup(name='freqtrade', 'cachetools', 'coinmarketcap', 'scikit-optimize', + 'ujson' + 'py_find_1st' ], include_package_data=True, zip_safe=False, From 357c8c0ba0ed902f1757844b0fffb94be45ecc3f Mon Sep 17 00:00:00 2001 From: creslinux Date: Mon, 16 Jul 2018 18:32:41 +0000 Subject: [PATCH 018/699] sensible defaults --- freqtrade/optimize/backtesting.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index 43e05d70b..eb002a7ec 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -89,14 +89,14 @@ class Backtesting(object): self.np_stop: int = 6 self.np_bto: int = self.np_close # buys_triggered_on - should be close self.np_bco: int = self.np_open # buys calculated on - open of the next candle. - #self.np_sto: int = self.np_low # stops_triggered_on - Should be low, FT uses close - #self.np_sco: int = self.np_stop # stops_calculated_on - Should be stop, FT uses close - self.np_sto: int = self.np_close # stops_triggered_on - Should be low, FT uses close - self.np_sco: int = self.np_close # stops_calculated_on - Should be stop, FT uses close + self.np_sto: int = self.np_low # stops_triggered_on - Should be low, FT uses close + self.np_sco: int = self.np_stop # stops_calculated_on - Should be stop, FT uses close + #self.np_sto: int = self.np_close # stops_triggered_on - Should be low, FT uses close + #self.np_sco: int = self.np_close # stops_calculated_on - Should be stop, FT uses close self.use_backslap = True # Enable backslap - if false Orginal code is executed. self.debug = False # Main debug enable, very print heavy, enable 2 loops recommended - self.debug_timing = False # Stages within Backslap + self.debug_timing = True # Stages within Backslap self.debug_2loops = False # Limit each pair to two loops, useful when debugging self.debug_vector = False # Debug vector calcs self.debug_timing_main_loop = False # print overall timing per pair - works in Backtest and Backslap From a31391734798c2f0107b259c9e0b9cf54788c7bf Mon Sep 17 00:00:00 2001 From: creslinux Date: Mon, 16 Jul 2018 18:59:48 +0000 Subject: [PATCH 019/699] Handle a buy on the last candle We will never see this, as buy is on close which is the end of backtest e.g there is no next candle OPEN to buy at, or on --- freqtrade/optimize/backtesting.py | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index eb002a7ec..d6e4d1daf 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -89,10 +89,10 @@ class Backtesting(object): self.np_stop: int = 6 self.np_bto: int = self.np_close # buys_triggered_on - should be close self.np_bco: int = self.np_open # buys calculated on - open of the next candle. - self.np_sto: int = self.np_low # stops_triggered_on - Should be low, FT uses close - self.np_sco: int = self.np_stop # stops_calculated_on - Should be stop, FT uses close - #self.np_sto: int = self.np_close # stops_triggered_on - Should be low, FT uses close - #self.np_sco: int = self.np_close # stops_calculated_on - Should be stop, FT uses close + #self.np_sto: int = self.np_low # stops_triggered_on - Should be low, FT uses close + #self.np_sco: int = self.np_stop # stops_calculated_on - Should be stop, FT uses close + self.np_sto: int = self.np_close # stops_triggered_on - Should be low, FT uses close + self.np_sco: int = self.np_close # stops_calculated_on - Should be stop, FT uses close self.use_backslap = True # Enable backslap - if false Orginal code is executed. self.debug = False # Main debug enable, very print heavy, enable 2 loops recommended @@ -437,7 +437,7 @@ class Backtesting(object): return bslap_results_df - def np_get_t_open_ind(self, np_buy_arr, t_exit_ind: int): + def np_get_t_open_ind(self, np_buy_arr, t_exit_ind: int, np_buy_arr_len: int): import utils_find_1st as utf1st """ The purpose of this def is to return the next "buy" = 1 @@ -469,6 +469,9 @@ class Backtesting(object): if t_open_ind != -1: # send back the -1 if no buys found. otherwise update index t_open_ind = t_open_ind + t_exit_ind # Align numpy index + if t_open_ind == np_buy_arr_len -1 : # If buy found on last candle ignore, there is no OPEN in next to use + t_open_ind = -1 # -1 ends the loop + return t_open_ind def backslap_pair(self, ticker_data, pair): @@ -606,11 +609,12 @@ class Backtesting(object): Requires: np_buy_arr - a 1D array of the 'buy' column. To find next "1" Required: t_exit_ind - Either 0, first loop. Or The index we last exited on + Requires: np_buy_arr_len - length of pair array. Provides: The next "buy" index after t_exit_ind If -1 is returned no buy has been found in remainder of array, skip to exit loop ''' - t_open_ind = self.np_get_t_open_ind(np_buy_arr, t_exit_ind) + t_open_ind = self.np_get_t_open_ind(np_buy_arr, t_exit_ind, np_buy_arr_len) if debug: print("\n(0) numpy debug \nnp_get_t_open, has returned the next valid buy index as", t_open_ind) From baaf0a5b216d61462dbe55e3ac5b0168ece2ae16 Mon Sep 17 00:00:00 2001 From: creslinux Date: Tue, 17 Jul 2018 08:12:21 +0000 Subject: [PATCH 020/699] Handle when 0 trades are found in any pairs being tested. --- freqtrade/optimize/backtesting.py | 55 ++++++++++++++----------------- 1 file changed, 25 insertions(+), 30 deletions(-) diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index d6e4d1daf..3cdc9366b 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -89,14 +89,14 @@ class Backtesting(object): self.np_stop: int = 6 self.np_bto: int = self.np_close # buys_triggered_on - should be close self.np_bco: int = self.np_open # buys calculated on - open of the next candle. - #self.np_sto: int = self.np_low # stops_triggered_on - Should be low, FT uses close - #self.np_sco: int = self.np_stop # stops_calculated_on - Should be stop, FT uses close - self.np_sto: int = self.np_close # stops_triggered_on - Should be low, FT uses close - self.np_sco: int = self.np_close # stops_calculated_on - Should be stop, FT uses close + self.np_sto: int = self.np_low # stops_triggered_on - Should be low, FT uses close + self.np_sco: int = self.np_stop # stops_calculated_on - Should be stop, FT uses close + #self.np_sto: int = self.np_close # stops_triggered_on - Should be low, FT uses close + #self.np_sco: int = self.np_close # stops_calculated_on - Should be stop, FT uses close - self.use_backslap = True # Enable backslap - if false Orginal code is executed. + self.use_backslap = True # Enable backslap - if false Orginal code is executed. self.debug = False # Main debug enable, very print heavy, enable 2 loops recommended - self.debug_timing = True # Stages within Backslap + self.debug_timing = False # Stages within Backslap self.debug_2loops = False # Limit each pair to two loops, useful when debugging self.debug_vector = False # Debug vector calcs self.debug_timing_main_loop = False # print overall timing per pair - works in Backtest and Backslap @@ -294,13 +294,16 @@ class Backtesting(object): # Switch List of Trade Dicts (bslap_results) to Dataframe # Fill missing, calculable columns, profit, duration , abs etc. bslap_results_df = DataFrame(bslap_results) - bslap_results_df['open_time'] = to_datetime(bslap_results_df['open_time']) - bslap_results_df['close_time'] = to_datetime(bslap_results_df['close_time']) - ### don't use this, itll drop exit type field - # bslap_results_df = DataFrame(bslap_results, columns=BacktestResult._fields) + if len(bslap_results_df) > 0: # Only post process a frame if it has a record + bslap_results_df['open_time'] = to_datetime(bslap_results_df['open_time']) + bslap_results_df['close_time'] = to_datetime(bslap_results_df['close_time']) + + bslap_results_df = self.vector_fill_results_table(bslap_results_df, pair) + else: + bslap_results_df = [] + bslap_results_df= DataFrame.from_records(bslap_results_df, columns=BacktestResult._fields) - bslap_results_df = self.vector_fill_results_table(bslap_results_df) return bslap_results_df @@ -370,10 +373,12 @@ class Backtesting(object): print("Time to BackTest :", pair, round(tt, 10)) print("-----------------------") + print("trades") + return DataFrame.from_records(trades, columns=BacktestResult._fields) ####################### Original BT loop end - def vector_fill_results_table(self, bslap_results_df: DataFrame): + def vector_fill_results_table(self, bslap_results_df: DataFrame, pair: str): """ The Results frame contains a number of columns that are calculable from othe columns. These are left blank till all rows are added, @@ -411,6 +416,7 @@ class Backtesting(object): pd.set_option('max_colwidth', 40) pd.set_option('precision', 12) + #Ony do Math on a dataframe that has an open; No result pairs contain the pair string only # Populate duration bslap_results_df['trade_duration'] = bslap_results_df['close_time'] - bslap_results_df['open_time'] # if debug: @@ -516,19 +522,7 @@ class Backtesting(object): buy stop triggers and stop calculated on # buy 0 - open 1 - close 2 - sell 3 - high 4 - low 5 - stop 6 ''' - # np_buy: int = 0 - # np_open: int = 1 - # np_close: int = 2 - # np_sell: int = 3 - # np_high: int = 4 - # np_low: int = 5 - # np_stop: int = 6 - # np_bto: int = np_close # buys_triggered_on - should be close - # np_bco: int = np_open # buys calculated on - open of the next candle. - # #np_sto: int = np_low # stops_triggered_on - Should be low, FT uses close - # #np_sco: int = np_stop # stops_calculated_on - Should be stop, FT uses close - # np_sto: int = np_close # stops_triggered_on - Should be low, FT uses close - # np_sco: int = np_close # stops_calculated_on - Should be stop, FT uses close + ####### # Use vars set at top of backtest @@ -748,10 +742,10 @@ class Backtesting(object): open 6am 98 3 0 0 ----- ------ ------- ----- ----- -1 means not found till end of view i.e no valid Stop found. Exclude from match. - Stop tiggering in 1, candle we bought at OPEN is valid. + Stop tiggering and closing in 96-1, the candle we bought at OPEN in, is valid. Buys and sells are triggered at candle close - Both with action their postions at the open of the next candle Index + 1 + Both will open their postions at the open of the next candle. i/e + 1 index Stop and buy Indexes are on the view. To map to the ticker dataframe the t_open_ind index should be summed. @@ -760,10 +754,10 @@ class Backtesting(object): t_exit_ind : Sell found in view t_open_ind : Where view was started on ticker_data - TODO: fix this frig for logig test,, case/switch/dictionary would be better... + TODO: fix this frig for logic test,, case/switch/dictionary would be better... more so when later testing many options, dynamic stop / roi etc - cludge - Im setting np_t_sell_ind as 9999999999 when -1 (not found) - cludge - Im setting np_t_stop_ind as 9999999999 when -1 (not found) + cludge - Setting np_t_sell_ind as 9999999999 when -1 (not found) + cludge - Setting np_t_stop_ind as 9999999999 when -1 (not found) ''' if debug: @@ -939,6 +933,7 @@ class Backtesting(object): opportunity to close any more trades. """ # TODO :add handing here to record none closed open trades + if debug: print(bslap_pair_results) break From ed4bf32f2af3d6dfd1dc08aa6e7aa5de5e28031a Mon Sep 17 00:00:00 2001 From: creslinux Date: Tue, 17 Jul 2018 10:59:17 +0000 Subject: [PATCH 021/699] Fixed Stop closing in Index 0 when buy opening on Index 1 --- freqtrade/optimize/backtesting.py | 41 ++++++++++++++++++------------- 1 file changed, 24 insertions(+), 17 deletions(-) diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index 3cdc9366b..c0259c8fc 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -95,10 +95,10 @@ class Backtesting(object): #self.np_sco: int = self.np_close # stops_calculated_on - Should be stop, FT uses close self.use_backslap = True # Enable backslap - if false Orginal code is executed. - self.debug = False # Main debug enable, very print heavy, enable 2 loops recommended + self.debug = True # Main debug enable, very print heavy, enable 2 loops recommended self.debug_timing = False # Stages within Backslap self.debug_2loops = False # Limit each pair to two loops, useful when debugging - self.debug_vector = False # Debug vector calcs + self.debug_vector = True # Debug vector calcs self.debug_timing_main_loop = False # print overall timing per pair - works in Backtest and Backslap self.backslap_show_trades = False # prints trades in addition to summary report @@ -296,8 +296,10 @@ class Backtesting(object): bslap_results_df = DataFrame(bslap_results) if len(bslap_results_df) > 0: # Only post process a frame if it has a record - bslap_results_df['open_time'] = to_datetime(bslap_results_df['open_time']) - bslap_results_df['close_time'] = to_datetime(bslap_results_df['close_time']) + # bslap_results_df['open_time'] = to_datetime(bslap_results_df['open_time']) + # bslap_results_df['close_time'] = to_datetime(bslap_results_df['close_time']) + # if debug: + # print("open_time and close_time converted to datetime columns") bslap_results_df = self.vector_fill_results_table(bslap_results_df, pair) else: @@ -373,7 +375,6 @@ class Backtesting(object): print("Time to BackTest :", pair, round(tt, 10)) print("-----------------------") - print("trades") return DataFrame.from_records(trades, columns=BacktestResult._fields) ####################### Original BT loop end @@ -393,6 +394,7 @@ class Backtesting(object): :return: bslap_results Dataframe """ import pandas as pd + import numpy as np debug = self.debug_vector # stake and fees @@ -416,8 +418,6 @@ class Backtesting(object): pd.set_option('max_colwidth', 40) pd.set_option('precision', 12) - #Ony do Math on a dataframe that has an open; No result pairs contain the pair string only - # Populate duration bslap_results_df['trade_duration'] = bslap_results_df['close_time'] - bslap_results_df['open_time'] # if debug: # print(bslap_results_df[['open_time', 'close_time', 'trade_duration']]) @@ -621,9 +621,9 @@ class Backtesting(object): if t_open_ind != -1: """ - 1 - Create view to search within for our open trade + 1 - Create views to search within for our open trade - The view is our search space for the next Stop or Sell + The views are our search space for the next Stop or Sell Numpy view is employed as: 1,000 faster than pandas searches Pandas cannot assure it will always return a view, it may make a slow copy. @@ -633,9 +633,12 @@ class Backtesting(object): Requires: np_bslap is our numpy array of the ticker DataFrame Requires: t_open_ind is the index row with the buy. - Provided: np_t_open_v View of array after trade. + Provides: np_t_open_v View of array after buy. + Provides: np_t_open_v_stop View of array after buy +1 + (Stop will search in here to prevent stopping in the past) """ np_t_open_v = np_bslap[t_open_ind:] + np_t_open_v_stop = np_bslap[t_open_ind +1:] if debug: print("\n(1) numpy debug \nNumpy view row 0 is now Ticker_Data Index", t_open_ind) @@ -675,23 +678,27 @@ class Backtesting(object): where [np_sto] (stop tiggered on variable: "close", "low" etc) < np_t_stop_pri - Requires: np_t_open_v Numpy view of ticker_data after trade open + Requires: np_t_open_v_stop Numpy view of ticker_data after buy row +1 (when trade was opened) Requires: np_sto User Var(STO)StopTriggeredOn. Typically set to "low" or "close" Requires: np_t_stop_pri The stop-loss price STO must fall under to trigger stop Provides: np_t_stop_ind The first candle after trade open where STO is under stop-loss ''' - np_t_stop_ind = utf1st.find_1st(np_t_open_v[:, np_sto], + np_t_stop_ind = utf1st.find_1st(np_t_open_v_stop[:, np_sto], np_t_stop_pri, utf1st.cmp_smaller) + # plus 1 as np_t_open_v_stop is 1 ahead of view np_t_open_v, used from here on out. + np_t_stop_ind = np_t_stop_ind +1 + + if debug: - print("\n(3) numpy debug\nNext view index with STO (stop trigger on) under Stop-Loss is", np_t_stop_ind, + print("\n(3) numpy debug\nNext view index with STO (stop trigger on) under Stop-Loss is", np_t_stop_ind -1, ". STO is using field", np_sto, "\nFrom key: buy 0 - open 1 - close 2 - sell 3 - high 4 - low 5\n") - print("If -1 returned there is no stop found to end of view, then next two array lines are garbage") - print("Row", np_t_stop_ind, np_t_open_v[np_t_stop_ind]) - print("Row", np_t_stop_ind + 1, np_t_open_v[np_t_stop_ind + 1]) + print("If -1 or 0 returned there is no stop found to end of view, then next two array lines are garbage") + print("Row", np_t_stop_ind -1 , np_t_open_v[np_t_stop_ind]) + print("Row", np_t_stop_ind , np_t_open_v[np_t_stop_ind + 1]) if debug_timing: t_t = f(st) print("3-numpy", str.format('{0:.17f}', t_t)) @@ -765,7 +772,7 @@ class Backtesting(object): # cludge for logic test (-1) means it was not found, set crazy high to lose < test np_t_sell_ind = 99999999 if np_t_sell_ind <= 0 else np_t_sell_ind - np_t_stop_ind = 99999999 if np_t_stop_ind == -1 else np_t_stop_ind + np_t_stop_ind = 99999999 if np_t_stop_ind <= 0 else np_t_stop_ind # Stoploss trigger found before a sell =1 if np_t_stop_ind < 99999999 and np_t_stop_ind <= np_t_sell_ind: From 8cea0517eb4e225553afbc8bc450be97a214ef11 Mon Sep 17 00:00:00 2001 From: creslinux Date: Tue, 17 Jul 2018 11:22:38 +0000 Subject: [PATCH 022/699] Added stop_stops stop_stops is an int value when number of stops in a pair reached the int the pair is stopped trading. This allows backtest to align with my pre_trade_mgt that does the same in dry and live operations --- freqtrade/optimize/backtesting.py | 32 ++++++++++++++++++++++++------- 1 file changed, 25 insertions(+), 7 deletions(-) diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index c0259c8fc..8bc6efb4d 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -95,14 +95,16 @@ class Backtesting(object): #self.np_sco: int = self.np_close # stops_calculated_on - Should be stop, FT uses close self.use_backslap = True # Enable backslap - if false Orginal code is executed. - self.debug = True # Main debug enable, very print heavy, enable 2 loops recommended + self.debug = False # Main debug enable, very print heavy, enable 2 loops recommended self.debug_timing = False # Stages within Backslap self.debug_2loops = False # Limit each pair to two loops, useful when debugging - self.debug_vector = True # Debug vector calcs + self.debug_vector = False # Debug vector calcs self.debug_timing_main_loop = False # print overall timing per pair - works in Backtest and Backslap - self.backslap_show_trades = False # prints trades in addition to summary report - self.backslap_save_trades = True # saves trades as a pretty table to backslap.txt + self.backslap_show_trades = True # prints trades in addition to summary report + self.backslap_save_trades = True # saves trades as a pretty table to backslap.txt + + self.stop_stops: int = 9999 # stop back testing any pair with this many stops, set to 999999 to not hit @staticmethod @@ -389,7 +391,6 @@ class Backtesting(object): - Profit - trade duration - profit abs - :param bslap_results Dataframe :return: bslap_results Dataframe """ @@ -443,14 +444,19 @@ class Backtesting(object): return bslap_results_df - def np_get_t_open_ind(self, np_buy_arr, t_exit_ind: int, np_buy_arr_len: int): + def np_get_t_open_ind(self, np_buy_arr, t_exit_ind: int, np_buy_arr_len: int, stop_stops: int, stop_stops_count: int): import utils_find_1st as utf1st """ The purpose of this def is to return the next "buy" = 1 after t_exit_ind. + + This function will also check is the stop limit for the pair has been reached. + if stop_stops is the limit and stop_stops_count it the number of times the stop has been hit. t_exit_ind is the index the last trade exited on or 0 if first time around this loop. + + stop_stops i """ # Timers, to be called if in debug def s(): @@ -478,6 +484,10 @@ class Backtesting(object): if t_open_ind == np_buy_arr_len -1 : # If buy found on last candle ignore, there is no OPEN in next to use t_open_ind = -1 # -1 ends the loop + if stop_stops_count >= stop_stops: # if maximum number of stops allowed in a pair is hit, exit loop + t_open_ind = -1 # -1 ends the loop + print("Max stop limit ", stop_stops, "reached. Moving to next pair") + return t_open_ind def backslap_pair(self, ticker_data, pair): @@ -564,6 +574,9 @@ class Backtesting(object): t_exit_ind = 0 # Start loop from first index t_exit_last = 0 # To test for exit + stop_stops = self.stop_stops # Int of stops within a pair to stop trading a pair at + stop_stops_count = 0 # stop counter per pair + st = s() # Start timer for processing dataframe if debug: print('Processing:', pair) @@ -604,11 +617,13 @@ class Backtesting(object): Requires: np_buy_arr - a 1D array of the 'buy' column. To find next "1" Required: t_exit_ind - Either 0, first loop. Or The index we last exited on Requires: np_buy_arr_len - length of pair array. + Requires: stops_stops - number of stops allowed before stop trading a pair + Requires: stop_stop_counts - count of stops hit in the pair Provides: The next "buy" index after t_exit_ind If -1 is returned no buy has been found in remainder of array, skip to exit loop ''' - t_open_ind = self.np_get_t_open_ind(np_buy_arr, t_exit_ind, np_buy_arr_len) + t_open_ind = self.np_get_t_open_ind(np_buy_arr, t_exit_ind, np_buy_arr_len, stop_stops, stop_stops_count) if debug: print("\n(0) numpy debug \nnp_get_t_open, has returned the next valid buy index as", t_open_ind) @@ -971,6 +986,9 @@ class Backtesting(object): # append the dict to the list and print list bslap_pair_results.append(bslap_result) + if t_exit_type is "stop": + stop_stops_count = stop_stops_count + 1 + if debug: print("The trade dict is: \n", bslap_result) print("Trades dicts in list after append are: \n ", bslap_pair_results) From 3184c85dca50f60c263d8f7d3c4981d41a132e56 Mon Sep 17 00:00:00 2001 From: creslinux Date: Tue, 17 Jul 2018 21:33:11 +0000 Subject: [PATCH 023/699] default settings to trigger low, take stop --- freqtrade/optimize/backtesting.py | 55 +++++++++++++++++-------------- 1 file changed, 31 insertions(+), 24 deletions(-) diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index 8bc6efb4d..6cd271534 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -101,7 +101,7 @@ class Backtesting(object): self.debug_vector = False # Debug vector calcs self.debug_timing_main_loop = False # print overall timing per pair - works in Backtest and Backslap - self.backslap_show_trades = True # prints trades in addition to summary report + self.backslap_show_trades = False # prints trades in addition to summary report self.backslap_save_trades = True # saves trades as a pretty table to backslap.txt self.stop_stops: int = 9999 # stop back testing any pair with this many stops, set to 999999 to not hit @@ -414,33 +414,35 @@ class Backtesting(object): if debug: from pandas import set_option set_option('display.max_rows', 5000) - set_option('display.max_columns', 10) + set_option('display.max_columns', 20) pd.set_option('display.width', 1000) pd.set_option('max_colwidth', 40) pd.set_option('precision', 12) bslap_results_df['trade_duration'] = bslap_results_df['close_time'] - bslap_results_df['open_time'] - # if debug: - # print(bslap_results_df[['open_time', 'close_time', 'trade_duration']]) ## Spends, Takes, Profit, Absolute Profit + print(bslap_results_df) # Buy Price - bslap_results_df['buy_sum'] = stake * bslap_results_df['open_rate'] - bslap_results_df['buy_fee'] = bslap_results_df['buy_sum'] * open_fee - bslap_results_df['buy_spend'] = bslap_results_df['buy_sum'] + bslap_results_df['buy_fee'] + bslap_results_df['buy_vol'] = stake / bslap_results_df['open_rate'] # How many target are we buying + bslap_results_df['buy_fee'] = stake * open_fee + bslap_results_df['buy_spend'] = stake + bslap_results_df['buy_fee'] # How much we're spending + # Sell price - bslap_results_df['sell_sum'] = stake * bslap_results_df['close_rate'] - bslap_results_df['sell_fee'] = bslap_results_df['sell_sum'] * close_fee + bslap_results_df['sell_sum'] = bslap_results_df['buy_vol'] * bslap_results_df['close_rate'] + bslap_results_df['sell_fee'] = bslap_results_df['sell_sum'] * close_fee bslap_results_df['sell_take'] = bslap_results_df['sell_sum'] - bslap_results_df['sell_fee'] # profit_percent - bslap_results_df['profit_percent'] = bslap_results_df['sell_take'] / bslap_results_df['buy_spend'] - 1 + bslap_results_df['profit_percent'] = (bslap_results_df['sell_take'] - bslap_results_df['buy_spend']) \ + / bslap_results_df['buy_spend'] # Absolute profit bslap_results_df['profit_abs'] = bslap_results_df['sell_take'] - bslap_results_df['buy_spend'] + if debug: print("\n") print(bslap_results_df[ - ['buy_sum', 'buy_fee', 'buy_spend', 'sell_sum','sell_fee', 'sell_take', 'profit_percent', 'profit_abs', 'exit_type']]) + ['buy_vol', 'buy_fee', 'buy_spend', 'sell_sum','sell_fee', 'sell_take', 'profit_percent', 'profit_abs', 'exit_type']]) return bslap_results_df @@ -458,6 +460,8 @@ class Backtesting(object): stop_stops i """ + debug = self.debug + # Timers, to be called if in debug def s(): st = timeit.default_timer() @@ -486,7 +490,8 @@ class Backtesting(object): if stop_stops_count >= stop_stops: # if maximum number of stops allowed in a pair is hit, exit loop t_open_ind = -1 # -1 ends the loop - print("Max stop limit ", stop_stops, "reached. Moving to next pair") + if debug: + print("Max stop limit ", stop_stops, "reached. Moving to next pair") return t_open_ind @@ -1026,6 +1031,8 @@ class Backtesting(object): timerange = Arguments.parse_timerange(None if self.config.get( 'timerange') is None else str(self.config.get('timerange'))) + + ld_files = self.s() data = optimize.load_data( self.config['datadir'], pairs=pairs, @@ -1046,6 +1053,8 @@ class Backtesting(object): max_open_trades = 0 preprocessed = self.tickerdata_to_dataframe(data) + t_t = self.f(ld_files) + print("Load from json to file to df in mem took", t_t) # Print timeframe min_date, max_date = self.get_timeframe(preprocessed) @@ -1110,18 +1119,16 @@ class Backtesting(object): results ) ) - - ## TODO. Catch open trades for this report. - # logger.info( - # '\n=============================================== ' - # 'LEFT OPEN TRADES REPORT' - # ' ===============================================\n' - # '%s', - # self._generate_text_table( - # data, - # results.loc[results.open_at_end] - # ) - # ) + logger.info( + '\n=============================================== ' + 'LEFT OPEN TRADES REPORT' + ' ===============================================\n' + '%s', + self._generate_text_table( + data, + results.loc[results.open_at_end] + ) + ) def setup_configuration(args: Namespace) -> Dict[str, Any]: From 79f931f29639e05232ba1044cebceaf2e6a3e332 Mon Sep 17 00:00:00 2001 From: creslin <34645187+creslinux@users.noreply.github.com> Date: Wed, 25 Jul 2018 19:49:25 +0000 Subject: [PATCH 024/699] Update backtesting.py --- freqtrade/optimize/backtesting.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index c0259c8fc..636a95b2d 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -95,10 +95,10 @@ class Backtesting(object): #self.np_sco: int = self.np_close # stops_calculated_on - Should be stop, FT uses close self.use_backslap = True # Enable backslap - if false Orginal code is executed. - self.debug = True # Main debug enable, very print heavy, enable 2 loops recommended + self.debug = False # Main debug enable, very print heavy, enable 2 loops recommended self.debug_timing = False # Stages within Backslap self.debug_2loops = False # Limit each pair to two loops, useful when debugging - self.debug_vector = True # Debug vector calcs + self.debug_vector = False # Debug vector calcs self.debug_timing_main_loop = False # print overall timing per pair - works in Backtest and Backslap self.backslap_show_trades = False # prints trades in addition to summary report From 482b85182ad77a517e79d98d772f93f1f2536061 Mon Sep 17 00:00:00 2001 From: creslin <34645187+creslinux@users.noreply.github.com> Date: Thu, 26 Jul 2018 06:53:43 +0000 Subject: [PATCH 025/699] Update setup.py --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 4ec541674..bfe01e6fe 100644 --- a/setup.py +++ b/setup.py @@ -37,7 +37,7 @@ setup(name='freqtrade', 'cachetools', 'coinmarketcap', 'scikit-optimize', - 'ujson' + 'ujson', 'py_find_1st' ], include_package_data=True, From e39ae45d2f5ff7c954b7a45201bf4a941c6c55cd Mon Sep 17 00:00:00 2001 From: creslinux Date: Thu, 26 Jul 2018 18:40:45 +0000 Subject: [PATCH 026/699] Some reason did not push this... vector calcs redone. --- freqtrade/optimize/backtesting.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index 6cd271534..5b4487762 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -419,6 +419,12 @@ class Backtesting(object): pd.set_option('max_colwidth', 40) pd.set_option('precision', 12) + # # Get before + # csv = "cryptosher_before_debug" + # bslap_results_df.to_csv(csv, sep='\t', encoding='utf-8') + + bslap_results_df.to_csv(csv, sep='\t', encoding='utf-8') + bslap_results_df['trade_duration'] = bslap_results_df['close_time'] - bslap_results_df['open_time'] ## Spends, Takes, Profit, Absolute Profit @@ -438,6 +444,10 @@ class Backtesting(object): # Absolute profit bslap_results_df['profit_abs'] = bslap_results_df['sell_take'] - bslap_results_df['buy_spend'] + # # Get After + # csv="cryptosher_after_debug" + # bslap_results_df.to_csv(csv, sep='\t', encoding='utf-8') + if debug: print("\n") From 2dc3d6a6b857d558beb8b0291f3921d0670783f2 Mon Sep 17 00:00:00 2001 From: creslin <34645187+creslinux@users.noreply.github.com> Date: Thu, 26 Jul 2018 18:42:20 +0000 Subject: [PATCH 027/699] Update backtesting.py --- freqtrade/optimize/backtesting.py | 89 +++++++++++++++++++++---------- 1 file changed, 62 insertions(+), 27 deletions(-) diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index 636a95b2d..5b4487762 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -98,11 +98,13 @@ class Backtesting(object): self.debug = False # Main debug enable, very print heavy, enable 2 loops recommended self.debug_timing = False # Stages within Backslap self.debug_2loops = False # Limit each pair to two loops, useful when debugging - self.debug_vector = False # Debug vector calcs + self.debug_vector = False # Debug vector calcs self.debug_timing_main_loop = False # print overall timing per pair - works in Backtest and Backslap self.backslap_show_trades = False # prints trades in addition to summary report - self.backslap_save_trades = True # saves trades as a pretty table to backslap.txt + self.backslap_save_trades = True # saves trades as a pretty table to backslap.txt + + self.stop_stops: int = 9999 # stop back testing any pair with this many stops, set to 999999 to not hit @staticmethod @@ -389,7 +391,6 @@ class Backtesting(object): - Profit - trade duration - profit abs - :param bslap_results Dataframe :return: bslap_results Dataframe """ @@ -413,45 +414,64 @@ class Backtesting(object): if debug: from pandas import set_option set_option('display.max_rows', 5000) - set_option('display.max_columns', 10) + set_option('display.max_columns', 20) pd.set_option('display.width', 1000) pd.set_option('max_colwidth', 40) pd.set_option('precision', 12) + # # Get before + # csv = "cryptosher_before_debug" + # bslap_results_df.to_csv(csv, sep='\t', encoding='utf-8') + + bslap_results_df.to_csv(csv, sep='\t', encoding='utf-8') + bslap_results_df['trade_duration'] = bslap_results_df['close_time'] - bslap_results_df['open_time'] - # if debug: - # print(bslap_results_df[['open_time', 'close_time', 'trade_duration']]) ## Spends, Takes, Profit, Absolute Profit + print(bslap_results_df) # Buy Price - bslap_results_df['buy_sum'] = stake * bslap_results_df['open_rate'] - bslap_results_df['buy_fee'] = bslap_results_df['buy_sum'] * open_fee - bslap_results_df['buy_spend'] = bslap_results_df['buy_sum'] + bslap_results_df['buy_fee'] + bslap_results_df['buy_vol'] = stake / bslap_results_df['open_rate'] # How many target are we buying + bslap_results_df['buy_fee'] = stake * open_fee + bslap_results_df['buy_spend'] = stake + bslap_results_df['buy_fee'] # How much we're spending + # Sell price - bslap_results_df['sell_sum'] = stake * bslap_results_df['close_rate'] - bslap_results_df['sell_fee'] = bslap_results_df['sell_sum'] * close_fee + bslap_results_df['sell_sum'] = bslap_results_df['buy_vol'] * bslap_results_df['close_rate'] + bslap_results_df['sell_fee'] = bslap_results_df['sell_sum'] * close_fee bslap_results_df['sell_take'] = bslap_results_df['sell_sum'] - bslap_results_df['sell_fee'] # profit_percent - bslap_results_df['profit_percent'] = bslap_results_df['sell_take'] / bslap_results_df['buy_spend'] - 1 + bslap_results_df['profit_percent'] = (bslap_results_df['sell_take'] - bslap_results_df['buy_spend']) \ + / bslap_results_df['buy_spend'] # Absolute profit bslap_results_df['profit_abs'] = bslap_results_df['sell_take'] - bslap_results_df['buy_spend'] + # # Get After + # csv="cryptosher_after_debug" + # bslap_results_df.to_csv(csv, sep='\t', encoding='utf-8') + + if debug: print("\n") print(bslap_results_df[ - ['buy_sum', 'buy_fee', 'buy_spend', 'sell_sum','sell_fee', 'sell_take', 'profit_percent', 'profit_abs', 'exit_type']]) + ['buy_vol', 'buy_fee', 'buy_spend', 'sell_sum','sell_fee', 'sell_take', 'profit_percent', 'profit_abs', 'exit_type']]) return bslap_results_df - def np_get_t_open_ind(self, np_buy_arr, t_exit_ind: int, np_buy_arr_len: int): + def np_get_t_open_ind(self, np_buy_arr, t_exit_ind: int, np_buy_arr_len: int, stop_stops: int, stop_stops_count: int): import utils_find_1st as utf1st """ The purpose of this def is to return the next "buy" = 1 after t_exit_ind. + + This function will also check is the stop limit for the pair has been reached. + if stop_stops is the limit and stop_stops_count it the number of times the stop has been hit. t_exit_ind is the index the last trade exited on or 0 if first time around this loop. + + stop_stops i """ + debug = self.debug + # Timers, to be called if in debug def s(): st = timeit.default_timer() @@ -478,6 +498,11 @@ class Backtesting(object): if t_open_ind == np_buy_arr_len -1 : # If buy found on last candle ignore, there is no OPEN in next to use t_open_ind = -1 # -1 ends the loop + if stop_stops_count >= stop_stops: # if maximum number of stops allowed in a pair is hit, exit loop + t_open_ind = -1 # -1 ends the loop + if debug: + print("Max stop limit ", stop_stops, "reached. Moving to next pair") + return t_open_ind def backslap_pair(self, ticker_data, pair): @@ -564,6 +589,9 @@ class Backtesting(object): t_exit_ind = 0 # Start loop from first index t_exit_last = 0 # To test for exit + stop_stops = self.stop_stops # Int of stops within a pair to stop trading a pair at + stop_stops_count = 0 # stop counter per pair + st = s() # Start timer for processing dataframe if debug: print('Processing:', pair) @@ -604,11 +632,13 @@ class Backtesting(object): Requires: np_buy_arr - a 1D array of the 'buy' column. To find next "1" Required: t_exit_ind - Either 0, first loop. Or The index we last exited on Requires: np_buy_arr_len - length of pair array. + Requires: stops_stops - number of stops allowed before stop trading a pair + Requires: stop_stop_counts - count of stops hit in the pair Provides: The next "buy" index after t_exit_ind If -1 is returned no buy has been found in remainder of array, skip to exit loop ''' - t_open_ind = self.np_get_t_open_ind(np_buy_arr, t_exit_ind, np_buy_arr_len) + t_open_ind = self.np_get_t_open_ind(np_buy_arr, t_exit_ind, np_buy_arr_len, stop_stops, stop_stops_count) if debug: print("\n(0) numpy debug \nnp_get_t_open, has returned the next valid buy index as", t_open_ind) @@ -971,6 +1001,9 @@ class Backtesting(object): # append the dict to the list and print list bslap_pair_results.append(bslap_result) + if t_exit_type is "stop": + stop_stops_count = stop_stops_count + 1 + if debug: print("The trade dict is: \n", bslap_result) print("Trades dicts in list after append are: \n ", bslap_pair_results) @@ -1008,6 +1041,8 @@ class Backtesting(object): timerange = Arguments.parse_timerange(None if self.config.get( 'timerange') is None else str(self.config.get('timerange'))) + + ld_files = self.s() data = optimize.load_data( self.config['datadir'], pairs=pairs, @@ -1028,6 +1063,8 @@ class Backtesting(object): max_open_trades = 0 preprocessed = self.tickerdata_to_dataframe(data) + t_t = self.f(ld_files) + print("Load from json to file to df in mem took", t_t) # Print timeframe min_date, max_date = self.get_timeframe(preprocessed) @@ -1092,18 +1129,16 @@ class Backtesting(object): results ) ) - - ## TODO. Catch open trades for this report. - # logger.info( - # '\n=============================================== ' - # 'LEFT OPEN TRADES REPORT' - # ' ===============================================\n' - # '%s', - # self._generate_text_table( - # data, - # results.loc[results.open_at_end] - # ) - # ) + logger.info( + '\n=============================================== ' + 'LEFT OPEN TRADES REPORT' + ' ===============================================\n' + '%s', + self._generate_text_table( + data, + results.loc[results.open_at_end] + ) + ) def setup_configuration(args: Namespace) -> Dict[str, Any]: From 0372485cf025aa3b4c7e5484d695899681094f05 Mon Sep 17 00:00:00 2001 From: creslinux Date: Thu, 26 Jul 2018 19:17:00 +0000 Subject: [PATCH 028/699] Some reason did not push this... vector calcs redone. --- freqtrade/optimize/backtesting.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index 5b4487762..36ecc6826 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -423,12 +423,12 @@ class Backtesting(object): # csv = "cryptosher_before_debug" # bslap_results_df.to_csv(csv, sep='\t', encoding='utf-8') - bslap_results_df.to_csv(csv, sep='\t', encoding='utf-8') + # bslap_results_df.to_csv(csv, sep='\t', encoding='utf-8') bslap_results_df['trade_duration'] = bslap_results_df['close_time'] - bslap_results_df['open_time'] ## Spends, Takes, Profit, Absolute Profit - print(bslap_results_df) + # print(bslap_results_df) # Buy Price bslap_results_df['buy_vol'] = stake / bslap_results_df['open_rate'] # How many target are we buying bslap_results_df['buy_fee'] = stake * open_fee From 1a673c6ac94e036b9f66343bce9c8d4cb237d91f Mon Sep 17 00:00:00 2001 From: Gert Date: Sat, 28 Jul 2018 14:23:18 -0700 Subject: [PATCH 029/699] working on moving backslap --- freqtrade/optimize/backslapping.py | 785 +++++++++++++++++++++++++++++ freqtrade/optimize/backtesting.py | 745 +-------------------------- 2 files changed, 810 insertions(+), 720 deletions(-) create mode 100644 freqtrade/optimize/backslapping.py diff --git a/freqtrade/optimize/backslapping.py b/freqtrade/optimize/backslapping.py new file mode 100644 index 000000000..dff55648f --- /dev/null +++ b/freqtrade/optimize/backslapping.py @@ -0,0 +1,785 @@ +import timeit +from typing import Dict, Any + +from pandas import DataFrame + +from freqtrade.analyze import Analyze +from freqtrade.exchange import Exchange + + +class Backslapping: + """ + provides a quick way to evaluate strategies over a longer term of time + """ + + def __init__(self, config: Dict[str, Any], exchange = None) -> None: + """ + constructor + """ + + self.config = config + self.analyze = Analyze(self.config) + self.ticker_interval = self.analyze.strategy.ticker_interval + self.tickerdata_to_dataframe = self.analyze.tickerdata_to_dataframe + self.populate_buy_trend = self.analyze.populate_buy_trend + self.populate_sell_trend = self.analyze.populate_sell_trend + + ### + # + ### + if exchange is None: + self.config['exchange']['secret'] = '' + self.config['exchange']['password'] = '' + self.config['exchange']['uid'] = '' + self.config['dry_run'] = True + self.exchange = Exchange(self.config) + else: + self.exchange = exchange + + self.fee = self.exchange.get_fee() + + self.stop_loss_value = self.analyze.strategy.stoploss + + #### backslap config + ''' + Numpy arrays are used for 100x speed up + We requires setting Int values for + buy stop triggers and stop calculated on + # buy 0 - open 1 - close 2 - sell 3 - high 4 - low 5 - stop 6 + ''' + self.np_buy: int = 0 + self.np_open: int = 1 + self.np_close: int = 2 + self.np_sell: int = 3 + self.np_high: int = 4 + self.np_low: int = 5 + self.np_stop: int = 6 + self.np_bto: int = self.np_close # buys_triggered_on - should be close + self.np_bco: int = self.np_open # buys calculated on - open of the next candle. + self.np_sto: int = self.np_low # stops_triggered_on - Should be low, FT uses close + self.np_sco: int = self.np_stop # stops_calculated_on - Should be stop, FT uses close + # self.np_sto: int = self.np_close # stops_triggered_on - Should be low, FT uses close + # self.np_sco: int = self.np_close # stops_calculated_on - Should be stop, FT uses close + + self.debug = False # Main debug enable, very print heavy, enable 2 loops recommended + self.debug_timing = False # Stages within Backslap + self.debug_2loops = False # Limit each pair to two loops, useful when debugging + self.debug_vector = False # Debug vector calcs + self.debug_timing_main_loop = False # print overall timing per pair - works in Backtest and Backslap + + self.backslap_show_trades = False # prints trades in addition to summary report + self.backslap_save_trades = True # saves trades as a pretty table to backslap.txt + + self.stop_stops: int = 9999 # stop back testing any pair with this many stops, set to 999999 to not hit + + def s(self): + st = timeit.default_timer() + return st + + def f(self, st): + return (timeit.default_timer() - st) + + def run(self,args): + + headers = ['date', 'buy', 'open', 'close', 'sell', 'high', 'low'] + processed = args['processed'] + max_open_trades = args.get('max_open_trades', 0) + realistic = args.get('realistic', False) + trades = [] + trade_count_lock: Dict = {} + + ########################### Call out BSlap Loop instead of Original BT code + bslap_results: list = [] + for pair, pair_data in processed.items(): + if self.debug_timing: # Start timer + fl = self.s() + + ticker_data = self.populate_sell_trend( + self.populate_buy_trend(pair_data))[headers].copy() + + if self.debug_timing: # print time taken + flt = self.f(fl) + # print("populate_buy_trend:", pair, round(flt, 10)) + st = self.s() + + # #dump same DFs to disk for offline testing in scratch + # f_pair:str = pair + # csv = f_pair.replace("/", "_") + # csv="/Users/creslin/PycharmProjects/freqtrade_new/frames/" + csv + # ticker_data.to_csv(csv, sep='\t', encoding='utf-8') + + # call bslap - results are a list of dicts + bslap_pair_results = self.backslap_pair(ticker_data, pair) + last_bslap_results = bslap_results + bslap_results = last_bslap_results + bslap_pair_results + + if self.debug_timing: # print time taken + tt = self.f(st) + print("Time to BackSlap :", pair, round(tt, 10)) + print("-----------------------") + + # Switch List of Trade Dicts (bslap_results) to Dataframe + # Fill missing, calculable columns, profit, duration , abs etc. + bslap_results_df = DataFrame(bslap_results) + + if len(bslap_results_df) > 0: # Only post process a frame if it has a record + # bslap_results_df['open_time'] = to_datetime(bslap_results_df['open_time']) + # bslap_results_df['close_time'] = to_datetime(bslap_results_df['close_time']) + # if debug: + # print("open_time and close_time converted to datetime columns") + + bslap_results_df = self.vector_fill_results_table(bslap_results_df, pair) + else: + from freqtrade.optimize.backtesting import BacktestResult + + bslap_results_df = [] + bslap_results_df = DataFrame.from_records(bslap_results_df, columns=BacktestResult._fields) + + return bslap_results_df + + def vector_fill_results_table(self, bslap_results_df: DataFrame, pair: str): + """ + The Results frame contains a number of columns that are calculable + from othe columns. These are left blank till all rows are added, + to be populated in single vector calls. + + Columns to be populated are: + - Profit + - trade duration + - profit abs + :param bslap_results Dataframe + :return: bslap_results Dataframe + """ + import pandas as pd + import numpy as np + debug = self.debug_vector + + # stake and fees + # stake = 0.015 + # 0.05% is 0.0005 + # fee = 0.001 + + stake = self.config.get('stake_amount') + fee = self.fee + open_fee = fee / 2 + close_fee = fee / 2 + + if debug: + print("Stake is,", stake, "the sum of currency to spend per trade") + print("The open fee is", open_fee, "The close fee is", close_fee) + if debug: + from pandas import set_option + set_option('display.max_rows', 5000) + set_option('display.max_columns', 20) + pd.set_option('display.width', 1000) + pd.set_option('max_colwidth', 40) + pd.set_option('precision', 12) + + # # Get before + # csv = "cryptosher_before_debug" + # bslap_results_df.to_csv(csv, sep='\t', encoding='utf-8') + + # bslap_results_df.to_csv(csv, sep='\t', encoding='utf-8') + + bslap_results_df['trade_duration'] = bslap_results_df['close_time'] - bslap_results_df['open_time'] + + ## Spends, Takes, Profit, Absolute Profit + # print(bslap_results_df) + # Buy Price + bslap_results_df['buy_vol'] = stake / bslap_results_df['open_rate'] # How many target are we buying + bslap_results_df['buy_fee'] = stake * open_fee + bslap_results_df['buy_spend'] = stake + bslap_results_df['buy_fee'] # How much we're spending + + # Sell price + bslap_results_df['sell_sum'] = bslap_results_df['buy_vol'] * bslap_results_df['close_rate'] + bslap_results_df['sell_fee'] = bslap_results_df['sell_sum'] * close_fee + bslap_results_df['sell_take'] = bslap_results_df['sell_sum'] - bslap_results_df['sell_fee'] + # profit_percent + bslap_results_df['profit_percent'] = (bslap_results_df['sell_take'] - bslap_results_df['buy_spend']) \ + / bslap_results_df['buy_spend'] + # Absolute profit + bslap_results_df['profit_abs'] = bslap_results_df['sell_take'] - bslap_results_df['buy_spend'] + + # # Get After + # csv="cryptosher_after_debug" + # bslap_results_df.to_csv(csv, sep='\t', encoding='utf-8') + + if debug: + print("\n") + print(bslap_results_df[ + ['buy_vol', 'buy_fee', 'buy_spend', 'sell_sum', 'sell_fee', 'sell_take', 'profit_percent', + 'profit_abs', 'exit_type']]) + + return bslap_results_df + + def np_get_t_open_ind(self, np_buy_arr, t_exit_ind: int, np_buy_arr_len: int, stop_stops: int, + stop_stops_count: int): + import utils_find_1st as utf1st + """ + The purpose of this def is to return the next "buy" = 1 + after t_exit_ind. + + This function will also check is the stop limit for the pair has been reached. + if stop_stops is the limit and stop_stops_count it the number of times the stop has been hit. + + t_exit_ind is the index the last trade exited on + or 0 if first time around this loop. + + stop_stops i + """ + debug = self.debug + + # Timers, to be called if in debug + def s(): + st = timeit.default_timer() + return st + + def f(st): + return (timeit.default_timer() - st) + + st = s() + t_open_ind: int + + """ + Create a view on our buy index starting after last trade exit + Search for next buy + """ + np_buy_arr_v = np_buy_arr[t_exit_ind:] + t_open_ind = utf1st.find_1st(np_buy_arr_v, 1, utf1st.cmp_equal) + + ''' + If -1 is returned no buy has been found, preserve the value + ''' + if t_open_ind != -1: # send back the -1 if no buys found. otherwise update index + t_open_ind = t_open_ind + t_exit_ind # Align numpy index + + if t_open_ind == np_buy_arr_len - 1: # If buy found on last candle ignore, there is no OPEN in next to use + t_open_ind = -1 # -1 ends the loop + + if stop_stops_count >= stop_stops: # if maximum number of stops allowed in a pair is hit, exit loop + t_open_ind = -1 # -1 ends the loop + if debug: + print("Max stop limit ", stop_stops, "reached. Moving to next pair") + + return t_open_ind + + def backslap_pair(self, ticker_data, pair): + import pandas as pd + import numpy as np + import timeit + import utils_find_1st as utf1st + from datetime import datetime + + ### backslap debug wrap + # debug_2loops = False # only loop twice, for faster debug + # debug_timing = False # print timing for each step + # debug = False # print values, to check accuracy + debug_2loops = self.debug_2loops # only loop twice, for faster debug + debug_timing = self.debug_timing # print timing for each step + debug = self.debug # print values, to check accuracy + + # Read Stop Loss Values and Stake + stop = self.stop_loss_value + p_stop = (stop + 1) # What stop really means, e.g 0.01 is 0.99 of price + + if debug: + print("Stop is ", stop, "value from stragey file") + print("p_stop is", p_stop, "value used to multiply to entry price") + + if debug: + from pandas import set_option + set_option('display.max_rows', 5000) + set_option('display.max_columns', 8) + pd.set_option('display.width', 1000) + pd.set_option('max_colwidth', 40) + pd.set_option('precision', 12) + + def s(): + st = timeit.default_timer() + return st + + def f(st): + return (timeit.default_timer() - st) + + #### backslap config + ''' + Numpy arrays are used for 100x speed up + We requires setting Int values for + buy stop triggers and stop calculated on + # buy 0 - open 1 - close 2 - sell 3 - high 4 - low 5 - stop 6 + ''' + + ####### + # Use vars set at top of backtest + np_buy: int = self.np_buy + np_open: int = self.np_open + np_close: int = self.np_close + np_sell: int = self.np_sell + np_high: int = self.np_high + np_low: int = self.np_low + np_stop: int = self.np_stop + np_bto: int = self.np_bto # buys_triggered_on - should be close + np_bco: int = self.np_bco # buys calculated on - open of the next candle. + np_sto: int = self.np_sto # stops_triggered_on - Should be low, FT uses close + np_sco: int = self.np_sco # stops_calculated_on - Should be stop, FT uses close + + ### End Config + + pair: str = pair + + # ticker_data: DataFrame = ticker_dfs[t_file] + bslap: DataFrame = ticker_data + + # Build a single dimension numpy array from "buy" index for faster search + # (500x faster than pandas) + np_buy_arr = bslap['buy'].values + np_buy_arr_len: int = len(np_buy_arr) + + # use numpy array for faster searches in loop, 20x faster than pandas + # buy 0 - open 1 - close 2 - sell 3 - high 4 - low 5 + np_bslap = np.array(bslap[['buy', 'open', 'close', 'sell', 'high', 'low']]) + + # Build a numpy list of date-times. + # We use these when building the trade + # The rationale is to address a value from a pandas cell is thousands of + # times more expensive. Processing time went X25 when trying to use any data from pandas + np_bslap_dates = bslap['date'].values + + loop: int = 0 # how many time around the loop + t_exit_ind = 0 # Start loop from first index + t_exit_last = 0 # To test for exit + + stop_stops = self.stop_stops # Int of stops within a pair to stop trading a pair at + stop_stops_count = 0 # stop counter per pair + + st = s() # Start timer for processing dataframe + if debug: + print('Processing:', pair) + + # Results will be stored in a list of dicts + bslap_pair_results: list = [] + bslap_result: dict = {} + + while t_exit_ind < np_buy_arr_len: + loop = loop + 1 + if debug or debug_timing: + print("-- T_exit_Ind - Numpy Index is", t_exit_ind, " ----------------------- Loop", loop, pair) + if debug_2loops: + if loop == 3: + print( + "++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++Loop debug max met - breaking") + break + ''' + Dev phases + Phase 1 + 1) Manage buy, sell, stop enter/exit + a) Find first buy index + b) Discover first stop and sell hit after buy index + c) Chose first instance as trade exit + + Phase 2 + 2) Manage dynamic Stop and ROI Exit + a) Create trade slice from 1 + b) search within trade slice for dynamice stop hit + c) search within trade slice for ROI hit + ''' + + if debug_timing: + st = s() + ''' + 0 - Find next buy entry + Finds index for first (buy = 1) flag + + Requires: np_buy_arr - a 1D array of the 'buy' column. To find next "1" + Required: t_exit_ind - Either 0, first loop. Or The index we last exited on + Requires: np_buy_arr_len - length of pair array. + Requires: stops_stops - number of stops allowed before stop trading a pair + Requires: stop_stop_counts - count of stops hit in the pair + Provides: The next "buy" index after t_exit_ind + + If -1 is returned no buy has been found in remainder of array, skip to exit loop + ''' + t_open_ind = self.np_get_t_open_ind(np_buy_arr, t_exit_ind, np_buy_arr_len, stop_stops, stop_stops_count) + + if debug: + print("\n(0) numpy debug \nnp_get_t_open, has returned the next valid buy index as", t_open_ind) + print("If -1 there are no valid buys in the remainder of ticker data. Skipping to end of loop") + if debug_timing: + t_t = f(st) + print("0-numpy", str.format('{0:.17f}', t_t)) + st = s() + + if t_open_ind != -1: + + """ + 1 - Create views to search within for our open trade + + The views are our search space for the next Stop or Sell + Numpy view is employed as: + 1,000 faster than pandas searches + Pandas cannot assure it will always return a view, it may make a slow copy. + + The view contains columns: + buy 0 - open 1 - close 2 - sell 3 - high 4 - low 5 + + Requires: np_bslap is our numpy array of the ticker DataFrame + Requires: t_open_ind is the index row with the buy. + Provides: np_t_open_v View of array after buy. + Provides: np_t_open_v_stop View of array after buy +1 + (Stop will search in here to prevent stopping in the past) + """ + np_t_open_v = np_bslap[t_open_ind:] + np_t_open_v_stop = np_bslap[t_open_ind + 1:] + + if debug: + print("\n(1) numpy debug \nNumpy view row 0 is now Ticker_Data Index", t_open_ind) + print("Numpy View: Buy - Open - Close - Sell - High - Low") + print("Row 0", np_t_open_v[0]) + print("Row 1", np_t_open_v[1], ) + if debug_timing: + t_t = f(st) + print("2-numpy", str.format('{0:.17f}', t_t)) + st = s() + + ''' + 2 - Calculate our stop-loss price + + As stop is based on buy price of our trade + - (BTO)Buys are Triggered On np_bto, typically the CLOSE of candle + - (BCO)Buys are Calculated On np_bco, default is OPEN of the next candle. + This is as we only see the CLOSE after it has happened. + The back test assumption is we have bought at first available price, the OPEN + + Requires: np_bslap - is our numpy array of the ticker DataFrame + Requires: t_open_ind - is the index row with the first buy. + Requires: p_stop - is the stop rate, ie. 0.99 is -1% + Provides: np_t_stop_pri - The value stop-loss will be triggered on + ''' + np_t_stop_pri = (np_bslap[t_open_ind + 1, np_bco] * p_stop) + + if debug: + print("\n(2) numpy debug\nStop-Loss has been calculated at:", np_t_stop_pri) + if debug_timing: + t_t = f(st) + print("2-numpy", str.format('{0:.17f}', t_t)) + st = s() + + ''' + 3 - Find candle STO is under Stop-Loss After Trade opened. + + where [np_sto] (stop tiggered on variable: "close", "low" etc) < np_t_stop_pri + + Requires: np_t_open_v_stop Numpy view of ticker_data after buy row +1 (when trade was opened) + Requires: np_sto User Var(STO)StopTriggeredOn. Typically set to "low" or "close" + Requires: np_t_stop_pri The stop-loss price STO must fall under to trigger stop + Provides: np_t_stop_ind The first candle after trade open where STO is under stop-loss + ''' + np_t_stop_ind = utf1st.find_1st(np_t_open_v_stop[:, np_sto], + np_t_stop_pri, + utf1st.cmp_smaller) + + # plus 1 as np_t_open_v_stop is 1 ahead of view np_t_open_v, used from here on out. + np_t_stop_ind = np_t_stop_ind + 1 + + if debug: + print("\n(3) numpy debug\nNext view index with STO (stop trigger on) under Stop-Loss is", + np_t_stop_ind - 1, + ". STO is using field", np_sto, + "\nFrom key: buy 0 - open 1 - close 2 - sell 3 - high 4 - low 5\n") + + print( + "If -1 or 0 returned there is no stop found to end of view, then next two array lines are garbage") + print("Row", np_t_stop_ind - 1, np_t_open_v[np_t_stop_ind]) + print("Row", np_t_stop_ind, np_t_open_v[np_t_stop_ind + 1]) + if debug_timing: + t_t = f(st) + print("3-numpy", str.format('{0:.17f}', t_t)) + st = s() + + ''' + 4 - Find first sell index after trade open + + First index in the view np_t_open_v where ['sell'] = 1 + + Requires: np_t_open_v - view of ticker_data from buy onwards + Requires: no_sell - integer '3', the buy column in the array + Provides: np_t_sell_ind index of view where first sell=1 after buy + ''' + # Use numpy array for faster search for sell + # Sell uses column 3. + # buy 0 - open 1 - close 2 - sell 3 - high 4 - low 5 + # Numpy searches 25-35x quicker than pandas on this data + + np_t_sell_ind = utf1st.find_1st(np_t_open_v[:, np_sell], + 1, utf1st.cmp_equal) + if debug: + print("\n(4) numpy debug\nNext view index with sell = 1 is ", np_t_sell_ind) + print("If 0 or less is returned there is no sell found to end of view, then next lines garbage") + print("Row", np_t_sell_ind, np_t_open_v[np_t_sell_ind]) + print("Row", np_t_sell_ind + 1, np_t_open_v[np_t_sell_ind + 1]) + if debug_timing: + t_t = f(st) + print("4-numpy", str.format('{0:.17f}', t_t)) + st = s() + + ''' + 5 - Determine which was hit first a stop or sell + To then use as exit index price-field (sell on buy, stop on stop) + + STOP takes priority over SELL as would be 'in candle' from tick data + Sell would use Open from Next candle. + So in a draw Stop would be hit first on ticker data in live + + Validity of when types of trades may be executed can be summarised as: + + Tick View + index index Buy Sell open low close high Stop price + open 2am 94 -1 0 0 ----- ------ ------ ----- ----- + open 3am 95 0 1 0 ----- ------ trg buy ----- ----- + open 4am 96 1 0 1 Enter trgstop trg sel ROI out Stop out + open 5am 97 2 0 0 Exit ------ ------- ----- ----- + open 6am 98 3 0 0 ----- ------ ------- ----- ----- + + -1 means not found till end of view i.e no valid Stop found. Exclude from match. + Stop tiggering and closing in 96-1, the candle we bought at OPEN in, is valid. + + Buys and sells are triggered at candle close + Both will open their postions at the open of the next candle. i/e + 1 index + + Stop and buy Indexes are on the view. To map to the ticker dataframe + the t_open_ind index should be summed. + + np_t_stop_ind: Stop Found index in view + t_exit_ind : Sell found in view + t_open_ind : Where view was started on ticker_data + + TODO: fix this frig for logic test,, case/switch/dictionary would be better... + more so when later testing many options, dynamic stop / roi etc + cludge - Setting np_t_sell_ind as 9999999999 when -1 (not found) + cludge - Setting np_t_stop_ind as 9999999999 when -1 (not found) + + ''' + if debug: + print("\n(5) numpy debug\nStop or Sell Logic Processing") + + # cludge for logic test (-1) means it was not found, set crazy high to lose < test + np_t_sell_ind = 99999999 if np_t_sell_ind <= 0 else np_t_sell_ind + np_t_stop_ind = 99999999 if np_t_stop_ind <= 0 else np_t_stop_ind + + # Stoploss trigger found before a sell =1 + if np_t_stop_ind < 99999999 and np_t_stop_ind <= np_t_sell_ind: + t_exit_ind = t_open_ind + np_t_stop_ind # Set Exit row index + t_exit_type = 'stop' # Set Exit type (stop) + np_t_exit_pri = np_sco # The price field our STOP exit will use + if debug: + print("Type STOP is first exit condition. " + "At view index:", np_t_stop_ind, ". Ticker data exit index is", t_exit_ind) + + # Buy = 1 found before a stoploss triggered + elif np_t_sell_ind < 99999999 and np_t_sell_ind < np_t_stop_ind: + # move sell onto next candle, we only look back on sell + # will use the open price later. + t_exit_ind = t_open_ind + np_t_sell_ind + 1 # Set Exit row index + t_exit_type = 'sell' # Set Exit type (sell) + np_t_exit_pri = np_open # The price field our SELL exit will use + if debug: + print("Type SELL is first exit condition. " + "At view index", np_t_sell_ind, ". Ticker data exit index is", t_exit_ind) + + # No stop or buy left in view - set t_exit_last -1 to handle gracefully + else: + t_exit_last: int = -1 # Signal loop to exit, no buys or sells found. + t_exit_type = "No Exit" + np_t_exit_pri = 999 # field price should be calculated on. 999 a non-existent column + if debug: + print("No valid STOP or SELL found. Signalling t_exit_last to gracefully exit") + + # TODO: fix having to cludge/uncludge this .. + # Undo cludge + np_t_sell_ind = -1 if np_t_sell_ind == 99999999 else np_t_sell_ind + np_t_stop_ind = -1 if np_t_stop_ind == 99999999 else np_t_stop_ind + + if debug_timing: + t_t = f(st) + print("5-logic", str.format('{0:.17f}', t_t)) + st = s() + + if debug: + ''' + Print out the buys, stops, sells + Include Line before and after to for easy + Human verification + ''' + # Combine the np_t_stop_pri value to bslap dataframe to make debug + # life easy. This is the current stop price based on buy price_ + # This is slow but don't care about performance in debug + # + # When referencing equiv np_column, as example np_sto, its 5 in numpy and 6 in df, so +1 + # as there is no data column in the numpy array. + bslap['np_stop_pri'] = np_t_stop_pri + + # Buy + print("\n\nDATAFRAME DEBUG =================== BUY ", pair) + print("Numpy Array BUY Index is:", 0) + print("DataFrame BUY Index is:", t_open_ind, "displaying DF \n") + print("HINT, BUY trade should use OPEN price from next candle, i.e ", t_open_ind + 1) + op_is = t_open_ind - 1 # Print open index start, line before + op_if = t_open_ind + 3 # Print open index finish, line after + print(bslap.iloc[op_is:op_if], "\n") + + # Stop - Stops trigger price np_sto (+1 for pandas column), and price received np_sco +1. (Stop Trigger|Calculated On) + if np_t_stop_ind < 0: + print("DATAFRAME DEBUG =================== STOP ", pair) + print("No STOPS were found until the end of ticker data file\n") + else: + print("DATAFRAME DEBUG =================== STOP ", pair) + print("Numpy Array STOP Index is:", np_t_stop_ind, "View starts at index", t_open_ind) + df_stop_index = (t_open_ind + np_t_stop_ind) + + print("DataFrame STOP Index is:", df_stop_index, "displaying DF \n") + print("First Stoploss trigger after Trade entered at OPEN in candle", t_open_ind + 1, "is ", + df_stop_index, ": \n", + str.format('{0:.17f}', bslap.iloc[df_stop_index][np_sto + 1]), + "is less than", str.format('{0:.17f}', np_t_stop_pri)) + + print("A stoploss exit will be calculated at rate:", + str.format('{0:.17f}', bslap.iloc[df_stop_index][np_sco + 1])) + + print("\nHINT, STOPs should exit in-candle, i.e", df_stop_index, + ": As live STOPs are not linked to O-C times") + + st_is = df_stop_index - 1 # Print stop index start, line before + st_if = df_stop_index + 2 # Print stop index finish, line after + print(bslap.iloc[st_is:st_if], "\n") + + # Sell + if np_t_sell_ind < 0: + print("DATAFRAME DEBUG =================== SELL ", pair) + print("No SELLS were found till the end of ticker data file\n") + else: + print("DATAFRAME DEBUG =================== SELL ", pair) + print("Numpy View SELL Index is:", np_t_sell_ind, "View starts at index", t_open_ind) + df_sell_index = (t_open_ind + np_t_sell_ind) + + print("DataFrame SELL Index is:", df_sell_index, "displaying DF \n") + print("First Sell Index after Trade open is in candle", df_sell_index) + print("HINT, if exit is SELL (not stop) trade should use OPEN price from next candle", + df_sell_index + 1) + sl_is = df_sell_index - 1 # Print sell index start, line before + sl_if = df_sell_index + 3 # Print sell index finish, line after + print(bslap.iloc[sl_is:sl_if], "\n") + + # Chosen Exit (stop or sell) + + print("DATAFRAME DEBUG =================== EXIT ", pair) + print("Exit type is :", t_exit_type) + print("trade exit price field is", np_t_exit_pri, "\n") + + if debug_timing: + t_t = f(st) + print("6-depra", str.format('{0:.17f}', t_t)) + st = s() + + ## use numpy view "np_t_open_v" for speed. Columns are + # buy 0 - open 1 - close 2 - sell 3 - high 4 - low 5 + # exception is 6 which is use the stop value. + + # TODO no! this is hard coded bleh fix this open + np_trade_enter_price = np_bslap[t_open_ind + 1, np_open] + if t_exit_type == 'stop': + if np_t_exit_pri == 6: + np_trade_exit_price = np_t_stop_pri + else: + np_trade_exit_price = np_bslap[t_exit_ind, np_t_exit_pri] + if t_exit_type == 'sell': + np_trade_exit_price = np_bslap[t_exit_ind, np_t_exit_pri] + + # Catch no exit found + if t_exit_type == "No Exit": + np_trade_exit_price = 0 + + if debug_timing: + t_t = f(st) + print("7-numpy", str.format('{0:.17f}', t_t)) + st = s() + + if debug: + print("//////////////////////////////////////////////") + print("+++++++++++++++++++++++++++++++++ Trade Enter ") + print("np_trade Enter Price is ", str.format('{0:.17f}', np_trade_enter_price)) + print("--------------------------------- Trade Exit ") + print("Trade Exit Type is ", t_exit_type) + print("np_trade Exit Price is", str.format('{0:.17f}', np_trade_exit_price)) + print("//////////////////////////////////////////////") + + else: # no buys were found, step 0 returned -1 + # Gracefully exit the loop + t_exit_last == -1 + if debug: + print("\n(E) No buys were found in remaining ticker file. Exiting", pair) + + # Loop control - catch no closed trades. + if debug: + print("---------------------------------------- end of loop", loop, + " Dataframe Exit Index is: ", t_exit_ind) + print("Exit Index Last, Exit Index Now Are: ", t_exit_last, t_exit_ind) + + if t_exit_last >= t_exit_ind or t_exit_last == -1: + """ + Break loop and go on to next pair. + + When last trade exit equals index of last exit, there is no + opportunity to close any more trades. + """ + # TODO :add handing here to record none closed open trades + + if debug: + print(bslap_pair_results) + break + else: + """ + Add trade to backtest looking results list of dicts + Loop back to look for more trades. + """ + # Build trade dictionary + ## In general if a field can be calculated later from other fields leave blank here + ## Its X(number of trades faster) to calc all in a single vector than 1 trade at a time + + # create a new dict + close_index: int = t_exit_ind + bslap_result = {} # Must have at start or we end up with a list of multiple same last result + bslap_result["pair"] = pair + bslap_result["profit_percent"] = "" # To be 1 vector calc across trades when loop complete + bslap_result["profit_abs"] = "" # To be 1 vector calc across trades when loop complete + bslap_result["open_time"] = np_bslap_dates[t_open_ind + 1] # use numpy array, pandas 20x slower + bslap_result["close_time"] = np_bslap_dates[close_index] # use numpy array, pandas 20x slower + bslap_result["open_index"] = t_open_ind + 1 # +1 as we buy on next. + bslap_result["close_index"] = close_index + bslap_result["trade_duration"] = "" # To be 1 vector calc across trades when loop complete + bslap_result["open_at_end"] = False + bslap_result["open_rate"] = round(np_trade_enter_price, 15) + bslap_result["close_rate"] = round(np_trade_exit_price, 15) + bslap_result["exit_type"] = t_exit_type + # append the dict to the list and print list + bslap_pair_results.append(bslap_result) + + if t_exit_type is "stop": + stop_stops_count = stop_stops_count + 1 + + if debug: + print("The trade dict is: \n", bslap_result) + print("Trades dicts in list after append are: \n ", bslap_pair_results) + + """ + Loop back to start. t_exit_last becomes where loop + will seek to open new trades from. + Push index on 1 to not open on close + """ + t_exit_last = t_exit_ind + 1 + + if debug_timing: + t_t = f(st) + print("8+trade", str.format('{0:.17f}', t_t)) + + # Send back List of trade dicts + return bslap_pair_results diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index 36ecc6826..b68c544f2 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -20,6 +20,7 @@ from freqtrade.arguments import Arguments from freqtrade.configuration import Configuration from freqtrade.exchange import Exchange from freqtrade.misc import file_dump_json +from freqtrade.optimize.backslapping import Backslapping from freqtrade.persistence import Trade from profilehooks import profile from collections import OrderedDict @@ -54,6 +55,7 @@ class Backtesting(object): backtesting = Backtesting(config) backtesting.start() """ + def __init__(self, config: Dict[str, Any]) -> None: self.config = config self.analyze = Analyze(self.config) @@ -91,21 +93,22 @@ class Backtesting(object): self.np_bco: int = self.np_open # buys calculated on - open of the next candle. self.np_sto: int = self.np_low # stops_triggered_on - Should be low, FT uses close self.np_sco: int = self.np_stop # stops_calculated_on - Should be stop, FT uses close - #self.np_sto: int = self.np_close # stops_triggered_on - Should be low, FT uses close - #self.np_sco: int = self.np_close # stops_calculated_on - Should be stop, FT uses close + # self.np_sto: int = self.np_close # stops_triggered_on - Should be low, FT uses close + # self.np_sco: int = self.np_close # stops_calculated_on - Should be stop, FT uses close - self.use_backslap = True # Enable backslap - if false Orginal code is executed. - self.debug = False # Main debug enable, very print heavy, enable 2 loops recommended - self.debug_timing = False # Stages within Backslap - self.debug_2loops = False # Limit each pair to two loops, useful when debugging - self.debug_vector = False # Debug vector calcs + self.use_backslap = True # Enable backslap - if false Orginal code is executed. + self.debug = False # Main debug enable, very print heavy, enable 2 loops recommended + self.debug_timing = False # Stages within Backslap + self.debug_2loops = False # Limit each pair to two loops, useful when debugging + self.debug_vector = False # Debug vector calcs self.debug_timing_main_loop = False # print overall timing per pair - works in Backtest and Backslap - self.backslap_show_trades = False # prints trades in addition to summary report - self.backslap_save_trades = True # saves trades as a pretty table to backslap.txt + self.backslap_show_trades = False # prints trades in addition to summary report + self.backslap_save_trades = True # saves trades as a pretty table to backslap.txt - self.stop_stops: int = 9999 # stop back testing any pair with this many stops, set to 999999 to not hit + self.stop_stops: int = 9999 # stop back testing any pair with this many stops, set to 999999 to not hit + self.backslap = Backslapping(config) @staticmethod def get_timeframe(data: Dict[str, DataFrame]) -> Tuple[arrow.Arrow, arrow.Arrow]: @@ -119,7 +122,7 @@ class Backtesting(object): for frame in data.values() ] return min(timeframe, key=operator.itemgetter(0))[0], \ - max(timeframe, key=operator.itemgetter(1))[1] + max(timeframe, key=operator.itemgetter(1))[1] def _generate_text_table(self, data: Dict[str, Dict], results: DataFrame) -> str: """ @@ -193,7 +196,6 @@ class Backtesting(object): buy_signal = sell_row.buy if self.analyze.should_sell(trade, sell_row.open, sell_row.date, buy_signal, sell_row.sell): - return BacktestResult(pair=pair, profit_percent=trade.calc_profit_percent(rate=sell_row.open), profit_abs=trade.calc_profit(rate=sell_row.open), @@ -233,7 +235,6 @@ class Backtesting(object): def f(self, st): return (timeit.default_timer() - st) - def backtest(self, args: Dict) -> DataFrame: """ Implements backtesting functionality @@ -253,65 +254,9 @@ class Backtesting(object): use_backslap = self.use_backslap debug_timing = self.debug_timing_main_loop - if use_backslap: # Use Back Slap code - - headers = ['date', 'buy', 'open', 'close', 'sell', 'high', 'low'] - processed = args['processed'] - max_open_trades = args.get('max_open_trades', 0) - realistic = args.get('realistic', False) - trades = [] - trade_count_lock: Dict = {} - - ########################### Call out BSlap Loop instead of Original BT code - bslap_results: list = [] - for pair, pair_data in processed.items(): - if debug_timing: # Start timer - fl = self.s() - - ticker_data = self.populate_sell_trend( - self.populate_buy_trend(pair_data))[headers].copy() - - if debug_timing: # print time taken - flt = self.f(fl) - #print("populate_buy_trend:", pair, round(flt, 10)) - st = self.s() - - # #dump same DFs to disk for offline testing in scratch - # f_pair:str = pair - # csv = f_pair.replace("/", "_") - # csv="/Users/creslin/PycharmProjects/freqtrade_new/frames/" + csv - # ticker_data.to_csv(csv, sep='\t', encoding='utf-8') - - #call bslap - results are a list of dicts - bslap_pair_results = self.backslap_pair(ticker_data, pair) - last_bslap_results = bslap_results - bslap_results = last_bslap_results + bslap_pair_results - - if debug_timing: # print time taken - tt = self.f(st) - print("Time to BackSlap :", pair, round(tt,10)) - print("-----------------------") - - - # Switch List of Trade Dicts (bslap_results) to Dataframe - # Fill missing, calculable columns, profit, duration , abs etc. - bslap_results_df = DataFrame(bslap_results) - - if len(bslap_results_df) > 0: # Only post process a frame if it has a record - # bslap_results_df['open_time'] = to_datetime(bslap_results_df['open_time']) - # bslap_results_df['close_time'] = to_datetime(bslap_results_df['close_time']) - # if debug: - # print("open_time and close_time converted to datetime columns") - - bslap_results_df = self.vector_fill_results_table(bslap_results_df, pair) - else: - bslap_results_df = [] - bslap_results_df= DataFrame.from_records(bslap_results_df, columns=BacktestResult._fields) - - - return bslap_results_df - - else: # use Original Back test code + if use_backslap: # Use Back Slap code + return self.backslap.run(args) + else: # use Original Back test code ########################## Original BT loop headers = ['date', 'buy', 'open', 'close', 'sell'] @@ -322,7 +267,7 @@ class Backtesting(object): trade_count_lock: Dict = {} for pair, pair_data in processed.items(): - if debug_timing: # Start timer + if debug_timing: # Start timer fl = self.s() pair_data['buy'], pair_data['sell'] = 0, 0 # cleanup from previous run @@ -336,9 +281,9 @@ class Backtesting(object): ticker_data.drop(ticker_data.head(1).index, inplace=True) - if debug_timing: # print time taken + if debug_timing: # print time taken flt = self.f(fl) - #print("populate_buy_trend:", pair, round(flt, 10)) + # print("populate_buy_trend:", pair, round(flt, 10)) st = self.s() # Convert from Pandas to list for performance reasons @@ -363,7 +308,6 @@ class Backtesting(object): trade_entry = self._get_sell_trade_entry(pair, row, ticker[index + 1:], trade_count_lock, args) - if trade_entry: lock_pair_until = trade_entry.close_time trades.append(trade_entry) @@ -377,651 +321,9 @@ class Backtesting(object): print("Time to BackTest :", pair, round(tt, 10)) print("-----------------------") - return DataFrame.from_records(trades, columns=BacktestResult._fields) ####################### Original BT loop end - def vector_fill_results_table(self, bslap_results_df: DataFrame, pair: str): - """ - The Results frame contains a number of columns that are calculable - from othe columns. These are left blank till all rows are added, - to be populated in single vector calls. - - Columns to be populated are: - - Profit - - trade duration - - profit abs - :param bslap_results Dataframe - :return: bslap_results Dataframe - """ - import pandas as pd - import numpy as np - debug = self.debug_vector - - # stake and fees - # stake = 0.015 - # 0.05% is 0.0005 - #fee = 0.001 - - stake = self.config.get('stake_amount') - fee = self.fee - open_fee = fee / 2 - close_fee = fee / 2 - - if debug: - print("Stake is,", stake, "the sum of currency to spend per trade") - print("The open fee is", open_fee, "The close fee is", close_fee) - if debug: - from pandas import set_option - set_option('display.max_rows', 5000) - set_option('display.max_columns', 20) - pd.set_option('display.width', 1000) - pd.set_option('max_colwidth', 40) - pd.set_option('precision', 12) - - # # Get before - # csv = "cryptosher_before_debug" - # bslap_results_df.to_csv(csv, sep='\t', encoding='utf-8') - - # bslap_results_df.to_csv(csv, sep='\t', encoding='utf-8') - - bslap_results_df['trade_duration'] = bslap_results_df['close_time'] - bslap_results_df['open_time'] - - ## Spends, Takes, Profit, Absolute Profit - # print(bslap_results_df) - # Buy Price - bslap_results_df['buy_vol'] = stake / bslap_results_df['open_rate'] # How many target are we buying - bslap_results_df['buy_fee'] = stake * open_fee - bslap_results_df['buy_spend'] = stake + bslap_results_df['buy_fee'] # How much we're spending - - # Sell price - bslap_results_df['sell_sum'] = bslap_results_df['buy_vol'] * bslap_results_df['close_rate'] - bslap_results_df['sell_fee'] = bslap_results_df['sell_sum'] * close_fee - bslap_results_df['sell_take'] = bslap_results_df['sell_sum'] - bslap_results_df['sell_fee'] - # profit_percent - bslap_results_df['profit_percent'] = (bslap_results_df['sell_take'] - bslap_results_df['buy_spend']) \ - / bslap_results_df['buy_spend'] - # Absolute profit - bslap_results_df['profit_abs'] = bslap_results_df['sell_take'] - bslap_results_df['buy_spend'] - - # # Get After - # csv="cryptosher_after_debug" - # bslap_results_df.to_csv(csv, sep='\t', encoding='utf-8') - - - if debug: - print("\n") - print(bslap_results_df[ - ['buy_vol', 'buy_fee', 'buy_spend', 'sell_sum','sell_fee', 'sell_take', 'profit_percent', 'profit_abs', 'exit_type']]) - - return bslap_results_df - - def np_get_t_open_ind(self, np_buy_arr, t_exit_ind: int, np_buy_arr_len: int, stop_stops: int, stop_stops_count: int): - import utils_find_1st as utf1st - """ - The purpose of this def is to return the next "buy" = 1 - after t_exit_ind. - - This function will also check is the stop limit for the pair has been reached. - if stop_stops is the limit and stop_stops_count it the number of times the stop has been hit. - - t_exit_ind is the index the last trade exited on - or 0 if first time around this loop. - - stop_stops i - """ - debug = self.debug - - # Timers, to be called if in debug - def s(): - st = timeit.default_timer() - return st - def f(st): - return (timeit.default_timer() - st) - - st = s() - t_open_ind: int - - """ - Create a view on our buy index starting after last trade exit - Search for next buy - """ - np_buy_arr_v = np_buy_arr[t_exit_ind:] - t_open_ind = utf1st.find_1st(np_buy_arr_v, 1, utf1st.cmp_equal) - - ''' - If -1 is returned no buy has been found, preserve the value - ''' - if t_open_ind != -1: # send back the -1 if no buys found. otherwise update index - t_open_ind = t_open_ind + t_exit_ind # Align numpy index - - if t_open_ind == np_buy_arr_len -1 : # If buy found on last candle ignore, there is no OPEN in next to use - t_open_ind = -1 # -1 ends the loop - - if stop_stops_count >= stop_stops: # if maximum number of stops allowed in a pair is hit, exit loop - t_open_ind = -1 # -1 ends the loop - if debug: - print("Max stop limit ", stop_stops, "reached. Moving to next pair") - - return t_open_ind - - def backslap_pair(self, ticker_data, pair): - import pandas as pd - import numpy as np - import timeit - import utils_find_1st as utf1st - from datetime import datetime - - ### backslap debug wrap - # debug_2loops = False # only loop twice, for faster debug - # debug_timing = False # print timing for each step - # debug = False # print values, to check accuracy - debug_2loops = self.debug_2loops # only loop twice, for faster debug - debug_timing = self.debug_timing # print timing for each step - debug = self.debug # print values, to check accuracy - - # Read Stop Loss Values and Stake - stop = self.stop_loss_value - p_stop = (stop + 1) # What stop really means, e.g 0.01 is 0.99 of price - - if debug: - print("Stop is ", stop, "value from stragey file") - print("p_stop is", p_stop, "value used to multiply to entry price") - - if debug: - from pandas import set_option - set_option('display.max_rows', 5000) - set_option('display.max_columns', 8) - pd.set_option('display.width', 1000) - pd.set_option('max_colwidth', 40) - pd.set_option('precision', 12) - def s(): - st = timeit.default_timer() - return st - def f(st): - return (timeit.default_timer() - st) - #### backslap config - ''' - Numpy arrays are used for 100x speed up - We requires setting Int values for - buy stop triggers and stop calculated on - # buy 0 - open 1 - close 2 - sell 3 - high 4 - low 5 - stop 6 - ''' - - - ####### - # Use vars set at top of backtest - np_buy: int = self.np_buy - np_open: int = self.np_open - np_close: int = self.np_close - np_sell: int = self.np_sell - np_high: int = self.np_high - np_low: int = self.np_low - np_stop: int = self.np_stop - np_bto: int = self.np_bto # buys_triggered_on - should be close - np_bco: int = self.np_bco # buys calculated on - open of the next candle. - np_sto: int = self.np_sto # stops_triggered_on - Should be low, FT uses close - np_sco: int = self.np_sco # stops_calculated_on - Should be stop, FT uses close - - ### End Config - - pair: str = pair - - #ticker_data: DataFrame = ticker_dfs[t_file] - bslap: DataFrame = ticker_data - - # Build a single dimension numpy array from "buy" index for faster search - # (500x faster than pandas) - np_buy_arr = bslap['buy'].values - np_buy_arr_len: int = len(np_buy_arr) - - # use numpy array for faster searches in loop, 20x faster than pandas - # buy 0 - open 1 - close 2 - sell 3 - high 4 - low 5 - np_bslap = np.array(bslap[['buy', 'open', 'close', 'sell', 'high', 'low']]) - - # Build a numpy list of date-times. - # We use these when building the trade - # The rationale is to address a value from a pandas cell is thousands of - # times more expensive. Processing time went X25 when trying to use any data from pandas - np_bslap_dates = bslap['date'].values - - loop: int = 0 # how many time around the loop - t_exit_ind = 0 # Start loop from first index - t_exit_last = 0 # To test for exit - - stop_stops = self.stop_stops # Int of stops within a pair to stop trading a pair at - stop_stops_count = 0 # stop counter per pair - - st = s() # Start timer for processing dataframe - if debug: - print('Processing:', pair) - - # Results will be stored in a list of dicts - bslap_pair_results: list = [] - bslap_result: dict = {} - - while t_exit_ind < np_buy_arr_len: - loop = loop + 1 - if debug or debug_timing: - print("-- T_exit_Ind - Numpy Index is", t_exit_ind, " ----------------------- Loop", loop, pair) - if debug_2loops: - if loop == 3: - print("++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++Loop debug max met - breaking") - break - ''' - Dev phases - Phase 1 - 1) Manage buy, sell, stop enter/exit - a) Find first buy index - b) Discover first stop and sell hit after buy index - c) Chose first instance as trade exit - - Phase 2 - 2) Manage dynamic Stop and ROI Exit - a) Create trade slice from 1 - b) search within trade slice for dynamice stop hit - c) search within trade slice for ROI hit - ''' - - if debug_timing: - st = s() - ''' - 0 - Find next buy entry - Finds index for first (buy = 1) flag - - Requires: np_buy_arr - a 1D array of the 'buy' column. To find next "1" - Required: t_exit_ind - Either 0, first loop. Or The index we last exited on - Requires: np_buy_arr_len - length of pair array. - Requires: stops_stops - number of stops allowed before stop trading a pair - Requires: stop_stop_counts - count of stops hit in the pair - Provides: The next "buy" index after t_exit_ind - - If -1 is returned no buy has been found in remainder of array, skip to exit loop - ''' - t_open_ind = self.np_get_t_open_ind(np_buy_arr, t_exit_ind, np_buy_arr_len, stop_stops, stop_stops_count) - - if debug: - print("\n(0) numpy debug \nnp_get_t_open, has returned the next valid buy index as", t_open_ind) - print("If -1 there are no valid buys in the remainder of ticker data. Skipping to end of loop") - if debug_timing: - t_t = f(st) - print("0-numpy", str.format('{0:.17f}', t_t)) - st = s() - - if t_open_ind != -1: - - """ - 1 - Create views to search within for our open trade - - The views are our search space for the next Stop or Sell - Numpy view is employed as: - 1,000 faster than pandas searches - Pandas cannot assure it will always return a view, it may make a slow copy. - - The view contains columns: - buy 0 - open 1 - close 2 - sell 3 - high 4 - low 5 - - Requires: np_bslap is our numpy array of the ticker DataFrame - Requires: t_open_ind is the index row with the buy. - Provides: np_t_open_v View of array after buy. - Provides: np_t_open_v_stop View of array after buy +1 - (Stop will search in here to prevent stopping in the past) - """ - np_t_open_v = np_bslap[t_open_ind:] - np_t_open_v_stop = np_bslap[t_open_ind +1:] - - if debug: - print("\n(1) numpy debug \nNumpy view row 0 is now Ticker_Data Index", t_open_ind) - print("Numpy View: Buy - Open - Close - Sell - High - Low") - print("Row 0", np_t_open_v[0]) - print("Row 1", np_t_open_v[1], ) - if debug_timing: - t_t = f(st) - print("2-numpy", str.format('{0:.17f}', t_t)) - st = s() - - ''' - 2 - Calculate our stop-loss price - - As stop is based on buy price of our trade - - (BTO)Buys are Triggered On np_bto, typically the CLOSE of candle - - (BCO)Buys are Calculated On np_bco, default is OPEN of the next candle. - This is as we only see the CLOSE after it has happened. - The back test assumption is we have bought at first available price, the OPEN - - Requires: np_bslap - is our numpy array of the ticker DataFrame - Requires: t_open_ind - is the index row with the first buy. - Requires: p_stop - is the stop rate, ie. 0.99 is -1% - Provides: np_t_stop_pri - The value stop-loss will be triggered on - ''' - np_t_stop_pri = (np_bslap[t_open_ind + 1, np_bco] * p_stop) - - if debug: - print("\n(2) numpy debug\nStop-Loss has been calculated at:", np_t_stop_pri) - if debug_timing: - t_t = f(st) - print("2-numpy", str.format('{0:.17f}', t_t)) - st = s() - - ''' - 3 - Find candle STO is under Stop-Loss After Trade opened. - - where [np_sto] (stop tiggered on variable: "close", "low" etc) < np_t_stop_pri - - Requires: np_t_open_v_stop Numpy view of ticker_data after buy row +1 (when trade was opened) - Requires: np_sto User Var(STO)StopTriggeredOn. Typically set to "low" or "close" - Requires: np_t_stop_pri The stop-loss price STO must fall under to trigger stop - Provides: np_t_stop_ind The first candle after trade open where STO is under stop-loss - ''' - np_t_stop_ind = utf1st.find_1st(np_t_open_v_stop[:, np_sto], - np_t_stop_pri, - utf1st.cmp_smaller) - - # plus 1 as np_t_open_v_stop is 1 ahead of view np_t_open_v, used from here on out. - np_t_stop_ind = np_t_stop_ind +1 - - - if debug: - print("\n(3) numpy debug\nNext view index with STO (stop trigger on) under Stop-Loss is", np_t_stop_ind -1, - ". STO is using field", np_sto, - "\nFrom key: buy 0 - open 1 - close 2 - sell 3 - high 4 - low 5\n") - - print("If -1 or 0 returned there is no stop found to end of view, then next two array lines are garbage") - print("Row", np_t_stop_ind -1 , np_t_open_v[np_t_stop_ind]) - print("Row", np_t_stop_ind , np_t_open_v[np_t_stop_ind + 1]) - if debug_timing: - t_t = f(st) - print("3-numpy", str.format('{0:.17f}', t_t)) - st = s() - - ''' - 4 - Find first sell index after trade open - - First index in the view np_t_open_v where ['sell'] = 1 - - Requires: np_t_open_v - view of ticker_data from buy onwards - Requires: no_sell - integer '3', the buy column in the array - Provides: np_t_sell_ind index of view where first sell=1 after buy - ''' - # Use numpy array for faster search for sell - # Sell uses column 3. - # buy 0 - open 1 - close 2 - sell 3 - high 4 - low 5 - # Numpy searches 25-35x quicker than pandas on this data - - np_t_sell_ind = utf1st.find_1st(np_t_open_v[:, np_sell], - 1, utf1st.cmp_equal) - if debug: - print("\n(4) numpy debug\nNext view index with sell = 1 is ", np_t_sell_ind) - print("If 0 or less is returned there is no sell found to end of view, then next lines garbage") - print("Row", np_t_sell_ind, np_t_open_v[np_t_sell_ind]) - print("Row", np_t_sell_ind + 1, np_t_open_v[np_t_sell_ind + 1]) - if debug_timing: - t_t = f(st) - print("4-numpy", str.format('{0:.17f}', t_t)) - st = s() - - ''' - 5 - Determine which was hit first a stop or sell - To then use as exit index price-field (sell on buy, stop on stop) - - STOP takes priority over SELL as would be 'in candle' from tick data - Sell would use Open from Next candle. - So in a draw Stop would be hit first on ticker data in live - - Validity of when types of trades may be executed can be summarised as: - - Tick View - index index Buy Sell open low close high Stop price - open 2am 94 -1 0 0 ----- ------ ------ ----- ----- - open 3am 95 0 1 0 ----- ------ trg buy ----- ----- - open 4am 96 1 0 1 Enter trgstop trg sel ROI out Stop out - open 5am 97 2 0 0 Exit ------ ------- ----- ----- - open 6am 98 3 0 0 ----- ------ ------- ----- ----- - - -1 means not found till end of view i.e no valid Stop found. Exclude from match. - Stop tiggering and closing in 96-1, the candle we bought at OPEN in, is valid. - - Buys and sells are triggered at candle close - Both will open their postions at the open of the next candle. i/e + 1 index - - Stop and buy Indexes are on the view. To map to the ticker dataframe - the t_open_ind index should be summed. - - np_t_stop_ind: Stop Found index in view - t_exit_ind : Sell found in view - t_open_ind : Where view was started on ticker_data - - TODO: fix this frig for logic test,, case/switch/dictionary would be better... - more so when later testing many options, dynamic stop / roi etc - cludge - Setting np_t_sell_ind as 9999999999 when -1 (not found) - cludge - Setting np_t_stop_ind as 9999999999 when -1 (not found) - - ''' - if debug: - print("\n(5) numpy debug\nStop or Sell Logic Processing") - - # cludge for logic test (-1) means it was not found, set crazy high to lose < test - np_t_sell_ind = 99999999 if np_t_sell_ind <= 0 else np_t_sell_ind - np_t_stop_ind = 99999999 if np_t_stop_ind <= 0 else np_t_stop_ind - - # Stoploss trigger found before a sell =1 - if np_t_stop_ind < 99999999 and np_t_stop_ind <= np_t_sell_ind: - t_exit_ind = t_open_ind + np_t_stop_ind # Set Exit row index - t_exit_type = 'stop' # Set Exit type (stop) - np_t_exit_pri = np_sco # The price field our STOP exit will use - if debug: - print("Type STOP is first exit condition. " - "At view index:", np_t_stop_ind, ". Ticker data exit index is", t_exit_ind) - - # Buy = 1 found before a stoploss triggered - elif np_t_sell_ind < 99999999 and np_t_sell_ind < np_t_stop_ind: - # move sell onto next candle, we only look back on sell - # will use the open price later. - t_exit_ind = t_open_ind + np_t_sell_ind + 1 # Set Exit row index - t_exit_type = 'sell' # Set Exit type (sell) - np_t_exit_pri = np_open # The price field our SELL exit will use - if debug: - print("Type SELL is first exit condition. " - "At view index", np_t_sell_ind, ". Ticker data exit index is", t_exit_ind) - - # No stop or buy left in view - set t_exit_last -1 to handle gracefully - else: - t_exit_last: int = -1 # Signal loop to exit, no buys or sells found. - t_exit_type = "No Exit" - np_t_exit_pri = 999 # field price should be calculated on. 999 a non-existent column - if debug: - print("No valid STOP or SELL found. Signalling t_exit_last to gracefully exit") - - # TODO: fix having to cludge/uncludge this .. - # Undo cludge - np_t_sell_ind = -1 if np_t_sell_ind == 99999999 else np_t_sell_ind - np_t_stop_ind = -1 if np_t_stop_ind == 99999999 else np_t_stop_ind - - if debug_timing: - t_t = f(st) - print("5-logic", str.format('{0:.17f}', t_t)) - st = s() - - if debug: - ''' - Print out the buys, stops, sells - Include Line before and after to for easy - Human verification - ''' - # Combine the np_t_stop_pri value to bslap dataframe to make debug - # life easy. This is the current stop price based on buy price_ - # This is slow but don't care about performance in debug - # - # When referencing equiv np_column, as example np_sto, its 5 in numpy and 6 in df, so +1 - # as there is no data column in the numpy array. - bslap['np_stop_pri'] = np_t_stop_pri - - # Buy - print("\n\nDATAFRAME DEBUG =================== BUY ", pair) - print("Numpy Array BUY Index is:", 0) - print("DataFrame BUY Index is:", t_open_ind, "displaying DF \n") - print("HINT, BUY trade should use OPEN price from next candle, i.e ", t_open_ind + 1) - op_is = t_open_ind - 1 # Print open index start, line before - op_if = t_open_ind + 3 # Print open index finish, line after - print(bslap.iloc[op_is:op_if], "\n") - - # Stop - Stops trigger price np_sto (+1 for pandas column), and price received np_sco +1. (Stop Trigger|Calculated On) - if np_t_stop_ind < 0: - print("DATAFRAME DEBUG =================== STOP ", pair) - print("No STOPS were found until the end of ticker data file\n") - else: - print("DATAFRAME DEBUG =================== STOP ", pair) - print("Numpy Array STOP Index is:", np_t_stop_ind, "View starts at index", t_open_ind) - df_stop_index = (t_open_ind + np_t_stop_ind) - - print("DataFrame STOP Index is:", df_stop_index, "displaying DF \n") - print("First Stoploss trigger after Trade entered at OPEN in candle", t_open_ind + 1, "is ", - df_stop_index, ": \n", - str.format('{0:.17f}', bslap.iloc[df_stop_index][np_sto + 1]), - "is less than", str.format('{0:.17f}', np_t_stop_pri)) - - print("A stoploss exit will be calculated at rate:", - str.format('{0:.17f}', bslap.iloc[df_stop_index][np_sco + 1])) - - print("\nHINT, STOPs should exit in-candle, i.e", df_stop_index, - ": As live STOPs are not linked to O-C times") - - st_is = df_stop_index - 1 # Print stop index start, line before - st_if = df_stop_index + 2 # Print stop index finish, line after - print(bslap.iloc[st_is:st_if], "\n") - - # Sell - if np_t_sell_ind < 0: - print("DATAFRAME DEBUG =================== SELL ", pair) - print("No SELLS were found till the end of ticker data file\n") - else: - print("DATAFRAME DEBUG =================== SELL ", pair) - print("Numpy View SELL Index is:", np_t_sell_ind, "View starts at index", t_open_ind) - df_sell_index = (t_open_ind + np_t_sell_ind) - - print("DataFrame SELL Index is:", df_sell_index, "displaying DF \n") - print("First Sell Index after Trade open is in candle", df_sell_index) - print("HINT, if exit is SELL (not stop) trade should use OPEN price from next candle", - df_sell_index + 1) - sl_is = df_sell_index - 1 # Print sell index start, line before - sl_if = df_sell_index + 3 # Print sell index finish, line after - print(bslap.iloc[sl_is:sl_if], "\n") - - # Chosen Exit (stop or sell) - - print("DATAFRAME DEBUG =================== EXIT ", pair) - print("Exit type is :", t_exit_type) - print("trade exit price field is", np_t_exit_pri, "\n") - - if debug_timing: - t_t = f(st) - print("6-depra", str.format('{0:.17f}', t_t)) - st = s() - - ## use numpy view "np_t_open_v" for speed. Columns are - # buy 0 - open 1 - close 2 - sell 3 - high 4 - low 5 - # exception is 6 which is use the stop value. - - # TODO no! this is hard coded bleh fix this open - np_trade_enter_price = np_bslap[t_open_ind + 1, np_open] - if t_exit_type == 'stop': - if np_t_exit_pri == 6: - np_trade_exit_price = np_t_stop_pri - else: - np_trade_exit_price = np_bslap[t_exit_ind, np_t_exit_pri] - if t_exit_type == 'sell': - np_trade_exit_price = np_bslap[t_exit_ind, np_t_exit_pri] - - # Catch no exit found - if t_exit_type == "No Exit": - np_trade_exit_price = 0 - - if debug_timing: - t_t = f(st) - print("7-numpy", str.format('{0:.17f}', t_t)) - st = s() - - if debug: - print("//////////////////////////////////////////////") - print("+++++++++++++++++++++++++++++++++ Trade Enter ") - print("np_trade Enter Price is ", str.format('{0:.17f}', np_trade_enter_price)) - print("--------------------------------- Trade Exit ") - print("Trade Exit Type is ", t_exit_type) - print("np_trade Exit Price is", str.format('{0:.17f}', np_trade_exit_price)) - print("//////////////////////////////////////////////") - - else: # no buys were found, step 0 returned -1 - # Gracefully exit the loop - t_exit_last == -1 - if debug: - print("\n(E) No buys were found in remaining ticker file. Exiting", pair) - - # Loop control - catch no closed trades. - if debug: - print("---------------------------------------- end of loop", loop, - " Dataframe Exit Index is: ", t_exit_ind) - print("Exit Index Last, Exit Index Now Are: ", t_exit_last, t_exit_ind) - - if t_exit_last >= t_exit_ind or t_exit_last == -1: - """ - Break loop and go on to next pair. - - When last trade exit equals index of last exit, there is no - opportunity to close any more trades. - """ - # TODO :add handing here to record none closed open trades - - if debug: - print(bslap_pair_results) - break - else: - """ - Add trade to backtest looking results list of dicts - Loop back to look for more trades. - """ - # Build trade dictionary - ## In general if a field can be calculated later from other fields leave blank here - ## Its X(number of trades faster) to calc all in a single vector than 1 trade at a time - - # create a new dict - close_index: int = t_exit_ind - bslap_result = {} # Must have at start or we end up with a list of multiple same last result - bslap_result["pair"] = pair - bslap_result["profit_percent"] = "" # To be 1 vector calc across trades when loop complete - bslap_result["profit_abs"] = "" # To be 1 vector calc across trades when loop complete - bslap_result["open_time"] = np_bslap_dates[t_open_ind + 1] # use numpy array, pandas 20x slower - bslap_result["close_time"] = np_bslap_dates[close_index] # use numpy array, pandas 20x slower - bslap_result["open_index"] = t_open_ind + 1 # +1 as we buy on next. - bslap_result["close_index"] = close_index - bslap_result["trade_duration"] = "" # To be 1 vector calc across trades when loop complete - bslap_result["open_at_end"] = False - bslap_result["open_rate"] = round(np_trade_enter_price, 15) - bslap_result["close_rate"] = round(np_trade_exit_price, 15) - bslap_result["exit_type"] = t_exit_type - # append the dict to the list and print list - bslap_pair_results.append(bslap_result) - - if t_exit_type is "stop": - stop_stops_count = stop_stops_count + 1 - - if debug: - print("The trade dict is: \n", bslap_result) - print("Trades dicts in list after append are: \n ", bslap_pair_results) - - """ - Loop back to start. t_exit_last becomes where loop - will seek to open new trades from. - Push index on 1 to not open on close - """ - t_exit_last = t_exit_ind + 1 - - if debug_timing: - t_t = f(st) - print("8+trade", str.format('{0:.17f}', t_t)) - - # Send back List of trade dicts - return bslap_pair_results - def start(self) -> None: """ Run a backtesting end-to-end @@ -1099,23 +401,26 @@ class Backtesting(object): results ) ) - #optional print trades + # optional print trades if self.backslap_show_trades: TradesFrame = results.filter(['open_time', 'pair', 'exit_type', 'profit_percent', 'profit_abs', 'buy_spend', 'sell_take', 'trade_duration', 'close_time'], axis=1) + def to_fwf(df, fname): content = tabulate(df.values.tolist(), list(df.columns), floatfmt=".8f", tablefmt='psql') print(content) DataFrame.to_fwf = to_fwf(TradesFrame, "backslap.txt") - #optional save trades + # optional save trades if self.backslap_save_trades: TradesFrame = results.filter(['open_time', 'pair', 'exit_type', 'profit_percent', 'profit_abs', 'buy_spend', 'sell_take', 'trade_duration', 'close_time'], axis=1) + def to_fwf(df, fname): content = tabulate(df.values.tolist(), list(df.columns), floatfmt=".8f", tablefmt='psql') open(fname, "w").write(content) + DataFrame.to_fwf = to_fwf(TradesFrame, "backslap.txt") else: From ab66fe1b724cc50c883005282a08bda0f538327d Mon Sep 17 00:00:00 2001 From: Gert Date: Sat, 28 Jul 2018 19:45:33 -0700 Subject: [PATCH 030/699] prepared for tracking signals --- freqtrade/optimize/backslapping.py | 16 +++++++++------- freqtrade/optimize/backtesting.py | 2 +- 2 files changed, 10 insertions(+), 8 deletions(-) diff --git a/freqtrade/optimize/backslapping.py b/freqtrade/optimize/backslapping.py index 3eee5db37..b16515942 100644 --- a/freqtrade/optimize/backslapping.py +++ b/freqtrade/optimize/backslapping.py @@ -5,6 +5,7 @@ from pandas import DataFrame from freqtrade.exchange import Exchange from freqtrade.strategy import IStrategy +from freqtrade.strategy.interface import SellType from freqtrade.strategy.resolver import StrategyResolver @@ -571,7 +572,7 @@ class Backslapping: # Stoploss trigger found before a sell =1 if np_t_stop_ind < 99999999 and np_t_stop_ind <= np_t_sell_ind: t_exit_ind = t_open_ind + np_t_stop_ind # Set Exit row index - t_exit_type = 'stop' # Set Exit type (stop) + t_exit_type = SellType.STOP_LOSS # Set Exit type (stop) np_t_exit_pri = np_sco # The price field our STOP exit will use if debug: print("Type STOP is first exit condition. " @@ -582,7 +583,7 @@ class Backslapping: # move sell onto next candle, we only look back on sell # will use the open price later. t_exit_ind = t_open_ind + np_t_sell_ind + 1 # Set Exit row index - t_exit_type = 'sell' # Set Exit type (sell) + t_exit_type = SellType.SELL_SIGNAL # Set Exit type (sell) np_t_exit_pri = np_open # The price field our SELL exit will use if debug: print("Type SELL is first exit condition. " @@ -591,7 +592,7 @@ class Backslapping: # No stop or buy left in view - set t_exit_last -1 to handle gracefully else: t_exit_last: int = -1 # Signal loop to exit, no buys or sells found. - t_exit_type = "No Exit" + t_exit_type = SellType.NONE np_t_exit_pri = 999 # field price should be calculated on. 999 a non-existent column if debug: print("No valid STOP or SELL found. Signalling t_exit_last to gracefully exit") @@ -688,16 +689,16 @@ class Backslapping: # TODO no! this is hard coded bleh fix this open np_trade_enter_price = np_bslap[t_open_ind + 1, np_open] - if t_exit_type == 'stop': + if t_exit_type == SellType.STOP_LOSS: if np_t_exit_pri == 6: np_trade_exit_price = np_t_stop_pri else: np_trade_exit_price = np_bslap[t_exit_ind, np_t_exit_pri] - if t_exit_type == 'sell': + if t_exit_type == SellType.SELL_SIGNAL: np_trade_exit_price = np_bslap[t_exit_ind, np_t_exit_pri] # Catch no exit found - if t_exit_type == "No Exit": + if t_exit_type == SellType.NONE: np_trade_exit_price = 0 if debug_timing: @@ -762,10 +763,11 @@ class Backslapping: bslap_result["open_rate"] = round(np_trade_enter_price, 15) bslap_result["close_rate"] = round(np_trade_exit_price, 15) bslap_result["exit_type"] = t_exit_type + bslap_result["sell_reason"] = t_exit_type #duplicated, but I don't care # append the dict to the list and print list bslap_pair_results.append(bslap_result) - if t_exit_type is "stop": + if t_exit_type is SellType.STOP_LOSS: stop_stops_count = stop_stops_count + 1 if debug: diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index 4ec3ac82a..8585a2628 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -455,7 +455,7 @@ class Backtesting(object): ) ) - if 'Sell Reason' in results.columns: + if 'sell_reason' in results.columns: logger.info( '\n' + ' SELL READON STATS '.center(119, '=') + From 04d5e857e2e9bc49d66055c1ec25088c45265e59 Mon Sep 17 00:00:00 2001 From: Gert Date: Tue, 31 Jul 2018 18:10:23 -0700 Subject: [PATCH 031/699] added option to easily switch between backtesting and backslapping from the commandline option --- freqtrade/arguments.py | 8 ++++++++ freqtrade/optimize/backtesting.py | 12 +++++++++--- 2 files changed, 17 insertions(+), 3 deletions(-) diff --git a/freqtrade/arguments.py b/freqtrade/arguments.py index 022a2c739..1f6de2052 100644 --- a/freqtrade/arguments.py +++ b/freqtrade/arguments.py @@ -161,6 +161,14 @@ class Arguments(object): dest='exportfilename', metavar='PATH', ) + parser.add_argument( + '--backslap', + help="Utilize the Backslapping approach instead of the default Backtesting. This should provide more " + "accurate results, unless you are utilizing Min/Max function in your strategy.", + required=False, + dest='backslap', + action='store_true' + ) @staticmethod def optimizer_shared_options(parser: argparse.ArgumentParser) -> None: diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index 688b760ca..6ba6d7e3e 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -98,7 +98,13 @@ class Backtesting(object): # self.np_sto: int = self.np_close # stops_triggered_on - Should be low, FT uses close # self.np_sco: int = self.np_close # stops_calculated_on - Should be stop, FT uses close - self.use_backslap = True # Enable backslap - if false Orginal code is executed. + if 'backslap' in config: + self.use_backslap = config['backslap'] # Enable backslap - if false Orginal code is executed. + else: + self.use_backslap = False + + logger.info("using backslap: {}".format(self.use_backslap)) + self.debug = False # Main debug enable, very print heavy, enable 2 loops recommended self.debug_timing = False # Stages within Backslap self.debug_2loops = False # Limit each pair to two loops, useful when debugging @@ -364,7 +370,6 @@ class Backtesting(object): timerange = Arguments.parse_timerange(None if self.config.get( 'timerange') is None else str(self.config.get('timerange'))) - ld_files = self.s() data = optimize.load_data( self.config['datadir'], pairs=pairs, @@ -374,6 +379,7 @@ class Backtesting(object): timerange=timerange ) + ld_files = self.s() if not data: logger.critical("No data found. Terminating.") return @@ -489,7 +495,7 @@ def setup_configuration(args: Namespace) -> Dict[str, Any]: # Ensure we do not use Exchange credentials config['exchange']['key'] = '' config['exchange']['secret'] = '' - + config['backslap'] = args.backslap if config['stake_amount'] == constants.UNLIMITED_STAKE_AMOUNT: raise DependencyException('stake amount could not be "%s" for backtesting' % constants.UNLIMITED_STAKE_AMOUNT) From 721fb3e326e254a5afdc3f81ccaae25d5cbfcbf7 Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 14 Aug 2018 10:12:57 +0200 Subject: [PATCH 032/699] remove unused profile import --- freqtrade/optimize/backtesting.py | 1 - 1 file changed, 1 deletion(-) diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index 6ba6d7e3e..d6de6cb0a 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -23,7 +23,6 @@ from freqtrade.optimize.backslapping import Backslapping from freqtrade.persistence import Trade from freqtrade.strategy.interface import SellType from freqtrade.strategy.resolver import IStrategy, StrategyResolver -from profilehooks import profile from collections import OrderedDict import timeit from time import sleep From bc6b80ff387ac19d1baba01175ada8d199f289c2 Mon Sep 17 00:00:00 2001 From: misagh Date: Fri, 24 Aug 2018 11:59:10 +0200 Subject: [PATCH 033/699] Edge functionality drafted --- freqtrade/edge.py | 6 ++++++ freqtrade/freqtradebot.py | 13 +++++++++++++ 2 files changed, 19 insertions(+) create mode 100644 freqtrade/edge.py diff --git a/freqtrade/edge.py b/freqtrade/edge.py new file mode 100644 index 000000000..4795e8f4d --- /dev/null +++ b/freqtrade/edge.py @@ -0,0 +1,6 @@ +# EDGE +# WinRate and Expected Risk Reward should be calculated for all whitelisted pairs +# ASYNC: For each pair call backslap +# Save WR and ERR in Edge Dict +# Save last time updated for each pair in edge_last_update_time +# Calulate expectancy and position size and stop loss diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index a2090d267..140ab2868 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -342,6 +342,9 @@ class FreqtradeBot(object): :return: True if a trade object has been created and persisted, False otherwise """ interval = self.strategy.ticker_interval + + # EDGE + # STAKE AMOUNT SHOULD COME FROM EDGE stake_amount = self._get_trade_stake_amount() if not stake_amount: @@ -362,6 +365,16 @@ class FreqtradeBot(object): if not whitelist: raise DependencyException('No currency pairs in whitelist') + + # EDGE + # WinRate and Expected Risk Reward should be calculated for all whitelisted pairs + # ASYNC: For each pair call backslap + # Save WR and ERR in Edge Dict + # Save last time updated for each pair in edge_last_update_time + # Calulate expectancy and position size and stop loss + + # whitelist = Edge.filter(whitelist) + # Pick pair based on buy signals for _pair in whitelist: thistory = self.exchange.get_candle_history(_pair, interval) From 601ae0545960a5a6299615621d8ed655211bb59e Mon Sep 17 00:00:00 2001 From: Matthias Date: Wed, 12 Sep 2018 20:24:19 +0200 Subject: [PATCH 034/699] formatting for contributing.md --- CONTRIBUTING.md | 29 +++++++++++++++++++---------- 1 file changed, 19 insertions(+), 10 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index b5b01e020..043686425 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -1,15 +1,17 @@ -# Contribute to freqtrade +# Contributing + +## Contribute to freqtrade Feel like our bot is missing a feature? We welcome your pull requests! Few pointers for contributions: - Create your PR against the `develop` branch, not `master`. - New features need to contain unit tests and must be PEP8 + conformant (max-line-length = 100). If you are unsure, discuss the feature on our [Slack](https://join.slack.com/t/highfrequencybot/shared_invite/enQtMjQ5NTM0OTYzMzY3LWMxYzE3M2MxNDdjMGM3ZTYwNzFjMGIwZGRjNTc3ZGU3MGE3NzdmZGMwNmU3NDM5ZTNmM2Y3NjRiNzk4NmM4OGE) or in a [issue](https://github.com/freqtrade/freqtrade/issues) before a PR. - **Before sending the PR:** ## 1. Run unit tests @@ -17,27 +19,34 @@ or in a [issue](https://github.com/freqtrade/freqtrade/issues) before a PR. All unit tests must pass. If a unit test is broken, change your code to make it pass. It means you have introduced a regression. -**Test the whole project** +### Test the whole project + ```bash pytest freqtrade ``` -**Test only one file** +### Test only one file + ```bash pytest freqtrade/tests/test_.py ``` -**Test only one method from one file** +### Test only one method from one file + ```bash pytest freqtrade/tests/test_.py::test_ ``` ## 2. Test if your code is PEP8 compliant -**Install packages** (If not already installed) + +### Install packages + ```bash pip3.6 install flake8 coveralls -``` -**Run Flake8** +``` + +### Run Flake8 + ```bash flake8 freqtrade ``` @@ -49,13 +58,13 @@ Guide for installing them is [here](http://flake8.pycqa.org/en/latest/user/using ## 3. Test if all type-hints are correct -**Install packages** (If not already installed) +### Install packages ``` bash pip3.6 install mypy ``` -**Run mypy** +### Run mypy ``` bash mypy freqtrade From 3f890335c5ca69f34cbe7bcb86b9e973ba52ed7b Mon Sep 17 00:00:00 2001 From: Matthias Date: Wed, 12 Sep 2018 20:40:52 +0200 Subject: [PATCH 035/699] Introduce Commiter guide --- CONTRIBUTING.md | 53 +++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 53 insertions(+) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 043686425..97130eded 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -69,3 +69,56 @@ pip3.6 install mypy ``` bash mypy freqtrade ``` + +## (Core)-Committer Guide + +### Process: Pull Requests + +How to prioritize pull requests, from most to least important: + +1. Fixes for broken tests. Broken means broken on any supported platform or Python version. +1. Extra tests to cover corner cases. +1. Minor edits to docs. +1. Bug fixes. +1. Major edits to docs. +1. Features. + +Ensure that each pull request meets all requirements in the Contributing document. + +### Process: Issues + +If an issue is a bug that needs an urgent fix, mark it for the next patch release. +Then either fix it or mark as please-help. + +For other issues: encourage friendly discussion, moderate debate, offer your thoughts. + +### Process: Your own code changes + +All code changes, regardless of who does them, need to be reviewed and merged by someone else. +This rule applies to all the core committers. + +Exceptions: + +- Minor corrections and fixes to pull requests submitted by others. +- While making a formal release, the release manager can make necessary, appropriate changes. +- Small documentation changes that reinforce existing subject matter. Most commonly being, but not limited to spelling and grammar corrections. + +### Responsibilities + +- Ensure cross-platform compatibility for every change that's accepted. Windows, Mac & Linux. +- Create issues for any major changes and enhancements that you wish to make. Discuss things transparently and get community feedback. +- Keep feature versions as small as possible, preferably one new feature per version. +- Be welcoming to newcomers and encourage diverse new contributors from all backgrounds. See the Python Community Code of Conduct (https://www.python.org/psf/codeofconduct/). + +### Becoming a Committer + +Contributors may be given commit privileges. Preference will be given to those with: + +1. Past contributions to FreqTrade and other related open-source projects. Contributions to FreqTrade include both code (both accepted and pending) and friendly participation in the issue tracker and Pull request reviews. Quantity and quality are considered. +1. A coding style that the other core committers find simple, minimal, and clean. +1. Access to resources for cross-platform development and testing. +1. Time to devote to the project regularly. + +Beeing a Committer does not grant write permission on `develop` or `master` for security reasons (Users trust FreqTrade with their Exchange API keys). + +After beeing Committer for some time, a Committer may be named Core Committer and given full repository access. From e30d23cf23c43701c16ef7bc0ad1eb2a8a77ee4f Mon Sep 17 00:00:00 2001 From: misagh Date: Fri, 14 Sep 2018 19:04:54 +0200 Subject: [PATCH 036/699] [draft] First version of edge positioning --- freqtrade/optimize/edge.py | 1023 ++++++++++++++++++++++++++++++++++++ 1 file changed, 1023 insertions(+) create mode 100644 freqtrade/optimize/edge.py diff --git a/freqtrade/optimize/edge.py b/freqtrade/optimize/edge.py new file mode 100644 index 000000000..ad37bf49b --- /dev/null +++ b/freqtrade/optimize/edge.py @@ -0,0 +1,1023 @@ +# pragma pylint: disable=missing-docstring, W0212, too-many-arguments + +""" +This module contains the backtesting logic +""" +import logging +import operator +import sys +from argparse import Namespace +from datetime import datetime, timedelta +from typing import Any, Dict, List, NamedTuple, Optional, Tuple + +import arrow +from pandas import DataFrame, to_datetime +from tabulate import tabulate +import numpy as np + +import freqtrade.optimize as optimize +from freqtrade import DependencyException, constants +from freqtrade.arguments import Arguments +from freqtrade.configuration import Configuration +from freqtrade.exchange import Exchange +from freqtrade.misc import file_dump_json +from freqtrade.persistence import Trade +from freqtrade.strategy.interface import SellType +from freqtrade.strategy.resolver import IStrategy, StrategyResolver +from collections import OrderedDict +import timeit +from time import sleep + +import pdb + +logger = logging.getLogger(__name__) + +class Edge: + """ + provides a quick way to evaluate strategies over a longer term of time + """ + + def __init__(self, config: Dict[str, Any], exchange = None) -> None: + """ + constructor + """ + self.config = config + self.strategy: IStrategy = StrategyResolver(self.config).strategy + self.ticker_interval = self.strategy.ticker_interval + self.tickerdata_to_dataframe = self.strategy.tickerdata_to_dataframe + self.populate_buy_trend = self.strategy.populate_buy_trend + self.populate_sell_trend = self.strategy.populate_sell_trend + + ### + # + ### + if exchange is None: + self.config['exchange']['secret'] = '' + self.config['exchange']['password'] = '' + self.config['exchange']['uid'] = '' + self.config['dry_run'] = True + self.exchange = Exchange(self.config) + else: + self.exchange = exchange + + self.fee = self.exchange.get_fee() + + self.stop_loss_value = self.strategy.stoploss + + #### backslap config + ''' + Numpy arrays are used for 100x speed up + We requires setting Int values for + buy stop triggers and stop calculated on + # buy 0 - open 1 - close 2 - sell 3 - high 4 - low 5 - stop 6 + ''' + self.np_buy: int = 0 + self.np_open: int = 1 + self.np_close: int = 2 + self.np_sell: int = 3 + self.np_high: int = 4 + self.np_low: int = 5 + self.np_stop: int = 6 + self.np_bto: int = self.np_close # buys_triggered_on - should be close + self.np_bco: int = self.np_open # buys calculated on - open of the next candle. + self.np_sto: int = self.np_low # stops_triggered_on - Should be low, FT uses close + self.np_sco: int = self.np_stop # stops_calculated_on - Should be stop, FT uses close + # self.np_sto: int = self.np_close # stops_triggered_on - Should be low, FT uses close + # self.np_sco: int = self.np_close # stops_calculated_on - Should be stop, FT uses close + + self.debug = False # Main debug enable, very print heavy, enable 2 loops recommended + self.debug_timing = False # Stages within Backslap + self.debug_2loops = False # Limit each pair to two loops, useful when debugging + self.debug_vector = False # Debug vector calcs + self.debug_timing_main_loop = False # print overall timing per pair - works in Backtest and Backslap + + self.backslap_show_trades = False # prints trades in addition to summary report + self.backslap_save_trades = True # saves trades as a pretty table to backslap.txt + + self.stop_stops: int = 9999 # stop back testing any pair with this many stops, set to 999999 to not hit + + @staticmethod + def get_timeframe(data: Dict[str, DataFrame]) -> Tuple[arrow.Arrow, arrow.Arrow]: + """ + Get the maximum timeframe for the given backtest data + :param data: dictionary with preprocessed backtesting data + :return: tuple containing min_date, max_date + """ + timeframe = [ + (arrow.get(frame['date'].min()), arrow.get(frame['date'].max())) + for frame in data.values() + ] + return min(timeframe, key=operator.itemgetter(0))[0], \ + max(timeframe, key=operator.itemgetter(1))[1] + + def s(self): + st = timeit.default_timer() + return st + + def f(self, st): + return (timeit.default_timer() - st) + + def run(self,args): + + headers = ['date', 'buy', 'open', 'close', 'sell', 'high', 'low'] + processed = args['processed'] + max_open_trades = args.get('max_open_trades', 0) + realistic = args.get('realistic', False) + stoploss_range = args['stoploss_range'] + trades = [] + trade_count_lock: Dict = {} + + ########################### Call out BSlap Loop instead of Original BT code + bslap_results: list = [] + for pair, pair_data in processed.items(): + if self.debug_timing: # Start timer + fl = self.s() + + ticker_data = self.populate_sell_trend( + self.populate_buy_trend(pair_data))[headers].copy() + + if self.debug_timing: # print time taken + flt = self.f(fl) + # print("populate_buy_trend:", pair, round(flt, 10)) + st = self.s() + + # #dump same DFs to disk for offline testing in scratch + # f_pair:str = pair + # csv = f_pair.replace("/", "_") + # csv="/Users/creslin/PycharmProjects/freqtrade_new/frames/" + csv + # ticker_data.to_csv(csv, sep='\t', encoding='utf-8') + + # call bslap - results are a list of dicts + for stoploss in stoploss_range: + bslap_results += self.backslap_pair(ticker_data, pair, round(stoploss, 3)) + + #bslap_results += self.backslap_pair(ticker_data, pair, -0.05) + # bslap_pair_results = self.backslap_pair(ticker_data, pair, -0.05) + # last_bslap_results = bslap_results + # bslap_results = last_bslap_results + bslap_pair_results + + + if self.debug_timing: # print time taken + tt = self.f(st) + print("Time to BackSlap :", pair, round(tt, 10)) + print("-----------------------") + + # Switch List of Trade Dicts (bslap_results) to Dataframe + # Fill missing, calculable columns, profit, duration , abs etc. + bslap_results_df = DataFrame(bslap_results) + + if len(bslap_results_df) > 0: # Only post process a frame if it has a record + # bslap_results_df['open_time'] = to_datetime(bslap_results_df['open_time']) + # bslap_results_df['close_time'] = to_datetime(bslap_results_df['close_time']) + # if debug: + # print("open_time and close_time converted to datetime columns") + + bslap_results_df = self.vector_fill_results_table(bslap_results_df) + else: + from freqtrade.optimize.backtesting import BacktestResult + + bslap_results_df = [] + bslap_results_df = DataFrame.from_records(bslap_results_df, columns=BacktestResult._fields) + + return bslap_results_df + + def vector_fill_results_table(self, bslap_results_df: DataFrame): + """ + The Results frame contains a number of columns that are calculable + from othe columns. These are left blank till all rows are added, + to be populated in single vector calls. + + Columns to be populated are: + - Profit + - trade duration + - profit abs + :param bslap_results Dataframe + :return: bslap_results Dataframe + """ + import pandas as pd + import numpy as np + debug = self.debug_vector + + # stake and fees + # stake = 0.015 + # 0.05% is 0.0005 + # fee = 0.001 + + stake = self.config.get('stake_amount') + fee = self.fee + open_fee = fee / 2 + close_fee = fee / 2 + + if debug: + print("Stake is,", stake, "the sum of currency to spend per trade") + print("The open fee is", open_fee, "The close fee is", close_fee) + if debug: + from pandas import set_option + set_option('display.max_rows', 5000) + set_option('display.max_columns', 20) + pd.set_option('display.width', 1000) + pd.set_option('max_colwidth', 40) + pd.set_option('precision', 12) + + # # Get before + # csv = "cryptosher_before_debug" + # bslap_results_df.to_csv(csv, sep='\t', encoding='utf-8') + + # bslap_results_df.to_csv(csv, sep='\t', encoding='utf-8') + + bslap_results_df['trade_duration'] = bslap_results_df['close_time'] - bslap_results_df['open_time'] + bslap_results_df['trade_duration'] = bslap_results_df['trade_duration'].map(lambda x: int(x.total_seconds() / 60)) + + ## Spends, Takes, Profit, Absolute Profit + # print(bslap_results_df) + # Buy Price + bslap_results_df['buy_vol'] = stake / bslap_results_df['open_rate'] # How many target are we buying + bslap_results_df['buy_fee'] = stake * open_fee + bslap_results_df['buy_spend'] = stake + bslap_results_df['buy_fee'] # How much we're spending + + # Sell price + bslap_results_df['sell_sum'] = bslap_results_df['buy_vol'] * bslap_results_df['close_rate'] + bslap_results_df['sell_fee'] = bslap_results_df['sell_sum'] * close_fee + bslap_results_df['sell_take'] = bslap_results_df['sell_sum'] - bslap_results_df['sell_fee'] + # profit_percent + bslap_results_df['profit_percent'] = (bslap_results_df['sell_take'] - bslap_results_df['buy_spend']) \ + / bslap_results_df['buy_spend'] + # Absolute profit + bslap_results_df['profit_abs'] = bslap_results_df['sell_take'] - bslap_results_df['buy_spend'] + + # # Get After + # csv="cryptosher_after_debug" + # bslap_results_df.to_csv(csv, sep='\t', encoding='utf-8') + + if debug: + print("\n") + print(bslap_results_df[ + ['buy_vol', 'buy_fee', 'buy_spend', 'sell_sum', 'sell_fee', 'sell_take', 'profit_percent', + 'profit_abs', 'exit_type']]) + + #pdb.set_trace() + return bslap_results_df + + def np_get_t_open_ind(self, np_buy_arr, t_exit_ind: int, np_buy_arr_len: int, stop_stops: int, + stop_stops_count: int): + import utils_find_1st as utf1st + """ + The purpose of this def is to return the next "buy" = 1 + after t_exit_ind. + + This function will also check is the stop limit for the pair has been reached. + if stop_stops is the limit and stop_stops_count it the number of times the stop has been hit. + + t_exit_ind is the index the last trade exited on + or 0 if first time around this loop. + + stop_stops i + """ + debug = self.debug + + # Timers, to be called if in debug + def s(): + st = timeit.default_timer() + return st + + def f(st): + return (timeit.default_timer() - st) + + st = s() + t_open_ind: int + + """ + Create a view on our buy index starting after last trade exit + Search for next buy + """ + np_buy_arr_v = np_buy_arr[t_exit_ind:] + t_open_ind = utf1st.find_1st(np_buy_arr_v, 1, utf1st.cmp_equal) + + ''' + If -1 is returned no buy has been found, preserve the value + ''' + if t_open_ind != -1: # send back the -1 if no buys found. otherwise update index + t_open_ind = t_open_ind + t_exit_ind # Align numpy index + + if t_open_ind == np_buy_arr_len - 1: # If buy found on last candle ignore, there is no OPEN in next to use + t_open_ind = -1 # -1 ends the loop + + if stop_stops_count >= stop_stops: # if maximum number of stops allowed in a pair is hit, exit loop + t_open_ind = -1 # -1 ends the loop + if debug: + print("Max stop limit ", stop_stops, "reached. Moving to next pair") + + return t_open_ind + + def backslap_pair(self, ticker_data, pair, stoploss): + import pandas as pd + import numpy as np + import timeit + import utils_find_1st as utf1st + from datetime import datetime + + ### backslap debug wrap + # debug_2loops = False # only loop twice, for faster debug + # debug_timing = False # print timing for each step + # debug = False # print values, to check accuracy + debug_2loops = self.debug_2loops # only loop twice, for faster debug + debug_timing = self.debug_timing # print timing for each step + debug = self.debug # print values, to check accuracy + + # Read Stop Loss Values and Stake + # pdb.set_trace() + #stop = self.stop_loss_value + stop = stoploss + p_stop = (stop + 1) # What stop really means, e.g 0.01 is 0.99 of price + + if debug: + print("Stop is ", stop, "value from stragey file") + print("p_stop is", p_stop, "value used to multiply to entry price") + + if debug: + from pandas import set_option + set_option('display.max_rows', 5000) + set_option('display.max_columns', 8) + pd.set_option('display.width', 1000) + pd.set_option('max_colwidth', 40) + pd.set_option('precision', 12) + + def s(): + st = timeit.default_timer() + return st + + def f(st): + return (timeit.default_timer() - st) + + #### backslap config + ''' + Numpy arrays are used for 100x speed up + We requires setting Int values for + buy stop triggers and stop calculated on + # buy 0 - open 1 - close 2 - sell 3 - high 4 - low 5 - stop 6 + ''' + + ####### + # Use vars set at top of backtest + np_buy: int = self.np_buy + np_open: int = self.np_open + np_close: int = self.np_close + np_sell: int = self.np_sell + np_high: int = self.np_high + np_low: int = self.np_low + np_stop: int = self.np_stop + np_bto: int = self.np_bto # buys_triggered_on - should be close + np_bco: int = self.np_bco # buys calculated on - open of the next candle. + np_sto: int = self.np_sto # stops_triggered_on - Should be low, FT uses close + np_sco: int = self.np_sco # stops_calculated_on - Should be stop, FT uses close + + ### End Config + + pair: str = pair + + # ticker_data: DataFrame = ticker_dfs[t_file] + bslap: DataFrame = ticker_data + + # Build a single dimension numpy array from "buy" index for faster search + # (500x faster than pandas) + np_buy_arr = bslap['buy'].values + np_buy_arr_len: int = len(np_buy_arr) + + # use numpy array for faster searches in loop, 20x faster than pandas + # buy 0 - open 1 - close 2 - sell 3 - high 4 - low 5 + np_bslap = np.array(bslap[['buy', 'open', 'close', 'sell', 'high', 'low']]) + + # Build a numpy list of date-times. + # We use these when building the trade + # The rationale is to address a value from a pandas cell is thousands of + # times more expensive. Processing time went X25 when trying to use any data from pandas + np_bslap_dates = bslap['date'].values + + loop: int = 0 # how many time around the loop + t_exit_ind = 0 # Start loop from first index + t_exit_last = 0 # To test for exit + + stop_stops = self.stop_stops # Int of stops within a pair to stop trading a pair at + stop_stops_count = 0 # stop counter per pair + + st = s() # Start timer for processing dataframe + if debug: + print('Processing:', pair) + + # Results will be stored in a list of dicts + bslap_pair_results: list = [] + bslap_result: dict = {} + + while t_exit_ind < np_buy_arr_len: + loop = loop + 1 + if debug or debug_timing: + print("-- T_exit_Ind - Numpy Index is", t_exit_ind, " ----------------------- Loop", loop, pair) + if debug_2loops: + if loop == 3: + print( + "++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++Loop debug max met - breaking") + break + ''' + Dev phases + Phase 1 + 1) Manage buy, sell, stop enter/exit + a) Find first buy index + b) Discover first stop and sell hit after buy index + c) Chose first instance as trade exit + + Phase 2 + 2) Manage dynamic Stop and ROI Exit + a) Create trade slice from 1 + b) search within trade slice for dynamice stop hit + c) search within trade slice for ROI hit + ''' + + if debug_timing: + st = s() + ''' + 0 - Find next buy entry + Finds index for first (buy = 1) flag + + Requires: np_buy_arr - a 1D array of the 'buy' column. To find next "1" + Required: t_exit_ind - Either 0, first loop. Or The index we last exited on + Requires: np_buy_arr_len - length of pair array. + Requires: stops_stops - number of stops allowed before stop trading a pair + Requires: stop_stop_counts - count of stops hit in the pair + Provides: The next "buy" index after t_exit_ind + + If -1 is returned no buy has been found in remainder of array, skip to exit loop + ''' + t_open_ind = self.np_get_t_open_ind(np_buy_arr, t_exit_ind, np_buy_arr_len, stop_stops, stop_stops_count) + + if debug: + print("\n(0) numpy debug \nnp_get_t_open, has returned the next valid buy index as", t_open_ind) + print("If -1 there are no valid buys in the remainder of ticker data. Skipping to end of loop") + if debug_timing: + t_t = f(st) + print("0-numpy", str.format('{0:.17f}', t_t)) + st = s() + + if t_open_ind != -1: + + """ + 1 - Create views to search within for our open trade + + The views are our search space for the next Stop or Sell + Numpy view is employed as: + 1,000 faster than pandas searches + Pandas cannot assure it will always return a view, it may make a slow copy. + + The view contains columns: + buy 0 - open 1 - close 2 - sell 3 - high 4 - low 5 + + Requires: np_bslap is our numpy array of the ticker DataFrame + Requires: t_open_ind is the index row with the buy. + Provides: np_t_open_v View of array after buy. + Provides: np_t_open_v_stop View of array after buy +1 + (Stop will search in here to prevent stopping in the past) + """ + np_t_open_v = np_bslap[t_open_ind:] + np_t_open_v_stop = np_bslap[t_open_ind + 1:] + + if debug: + print("\n(1) numpy debug \nNumpy view row 0 is now Ticker_Data Index", t_open_ind) + print("Numpy View: Buy - Open - Close - Sell - High - Low") + print("Row 0", np_t_open_v[0]) + print("Row 1", np_t_open_v[1], ) + if debug_timing: + t_t = f(st) + print("2-numpy", str.format('{0:.17f}', t_t)) + st = s() + + ''' + 2 - Calculate our stop-loss price + + As stop is based on buy price of our trade + - (BTO)Buys are Triggered On np_bto, typically the CLOSE of candle + - (BCO)Buys are Calculated On np_bco, default is OPEN of the next candle. + This is as we only see the CLOSE after it has happened. + The back test assumption is we have bought at first available price, the OPEN + + Requires: np_bslap - is our numpy array of the ticker DataFrame + Requires: t_open_ind - is the index row with the first buy. + Requires: p_stop - is the stop rate, ie. 0.99 is -1% + Provides: np_t_stop_pri - The value stop-loss will be triggered on + ''' + np_t_stop_pri = (np_bslap[t_open_ind + 1, np_bco] * p_stop) + + if debug: + print("\n(2) numpy debug\nStop-Loss has been calculated at:", np_t_stop_pri) + if debug_timing: + t_t = f(st) + print("2-numpy", str.format('{0:.17f}', t_t)) + st = s() + + ''' + 3 - Find candle STO is under Stop-Loss After Trade opened. + + where [np_sto] (stop tiggered on variable: "close", "low" etc) < np_t_stop_pri + + Requires: np_t_open_v_stop Numpy view of ticker_data after buy row +1 (when trade was opened) + Requires: np_sto User Var(STO)StopTriggeredOn. Typically set to "low" or "close" + Requires: np_t_stop_pri The stop-loss price STO must fall under to trigger stop + Provides: np_t_stop_ind The first candle after trade open where STO is under stop-loss + ''' + np_t_stop_ind = utf1st.find_1st(np_t_open_v_stop[:, np_sto], + np_t_stop_pri, + utf1st.cmp_smaller) + + # plus 1 as np_t_open_v_stop is 1 ahead of view np_t_open_v, used from here on out. + np_t_stop_ind = np_t_stop_ind + 1 + + if debug: + print("\n(3) numpy debug\nNext view index with STO (stop trigger on) under Stop-Loss is", + np_t_stop_ind - 1, + ". STO is using field", np_sto, + "\nFrom key: buy 0 - open 1 - close 2 - sell 3 - high 4 - low 5\n") + + print( + "If -1 or 0 returned there is no stop found to end of view, then next two array lines are garbage") + print("Row", np_t_stop_ind - 1, np_t_open_v[np_t_stop_ind]) + print("Row", np_t_stop_ind, np_t_open_v[np_t_stop_ind + 1]) + if debug_timing: + t_t = f(st) + print("3-numpy", str.format('{0:.17f}', t_t)) + st = s() + + ''' + 4 - Find first sell index after trade open + + First index in the view np_t_open_v where ['sell'] = 1 + + Requires: np_t_open_v - view of ticker_data from buy onwards + Requires: no_sell - integer '3', the buy column in the array + Provides: np_t_sell_ind index of view where first sell=1 after buy + ''' + # Use numpy array for faster search for sell + # Sell uses column 3. + # buy 0 - open 1 - close 2 - sell 3 - high 4 - low 5 + # Numpy searches 25-35x quicker than pandas on this data + + np_t_sell_ind = utf1st.find_1st(np_t_open_v[:, np_sell], + 1, utf1st.cmp_equal) + if debug: + print("\n(4) numpy debug\nNext view index with sell = 1 is ", np_t_sell_ind) + print("If 0 or less is returned there is no sell found to end of view, then next lines garbage") + print("Row", np_t_sell_ind, np_t_open_v[np_t_sell_ind]) + print("Row", np_t_sell_ind + 1, np_t_open_v[np_t_sell_ind + 1]) + if debug_timing: + t_t = f(st) + print("4-numpy", str.format('{0:.17f}', t_t)) + st = s() + + ''' + 5 - Determine which was hit first a stop or sell + To then use as exit index price-field (sell on buy, stop on stop) + + STOP takes priority over SELL as would be 'in candle' from tick data + Sell would use Open from Next candle. + So in a draw Stop would be hit first on ticker data in live + + Validity of when types of trades may be executed can be summarised as: + + Tick View + index index Buy Sell open low close high Stop price + open 2am 94 -1 0 0 ----- ------ ------ ----- ----- + open 3am 95 0 1 0 ----- ------ trg buy ----- ----- + open 4am 96 1 0 1 Enter trgstop trg sel ROI out Stop out + open 5am 97 2 0 0 Exit ------ ------- ----- ----- + open 6am 98 3 0 0 ----- ------ ------- ----- ----- + + -1 means not found till end of view i.e no valid Stop found. Exclude from match. + Stop tiggering and closing in 96-1, the candle we bought at OPEN in, is valid. + + Buys and sells are triggered at candle close + Both will open their postions at the open of the next candle. i/e + 1 index + + Stop and buy Indexes are on the view. To map to the ticker dataframe + the t_open_ind index should be summed. + + np_t_stop_ind: Stop Found index in view + t_exit_ind : Sell found in view + t_open_ind : Where view was started on ticker_data + + TODO: fix this frig for logic test,, case/switch/dictionary would be better... + more so when later testing many options, dynamic stop / roi etc + cludge - Setting np_t_sell_ind as 9999999999 when -1 (not found) + cludge - Setting np_t_stop_ind as 9999999999 when -1 (not found) + + ''' + if debug: + print("\n(5) numpy debug\nStop or Sell Logic Processing") + + # cludge for logic test (-1) means it was not found, set crazy high to lose < test + np_t_sell_ind = 99999999 if np_t_sell_ind <= 0 else np_t_sell_ind + np_t_stop_ind = 99999999 if np_t_stop_ind <= 0 else np_t_stop_ind + + # Stoploss trigger found before a sell =1 + if np_t_stop_ind < 99999999 and np_t_stop_ind <= np_t_sell_ind: + t_exit_ind = t_open_ind + np_t_stop_ind # Set Exit row index + t_exit_type = SellType.STOP_LOSS # Set Exit type (stop) + np_t_exit_pri = np_sco # The price field our STOP exit will use + if debug: + print("Type STOP is first exit condition. " + "At view index:", np_t_stop_ind, ". Ticker data exit index is", t_exit_ind) + + # Buy = 1 found before a stoploss triggered + elif np_t_sell_ind < 99999999 and np_t_sell_ind < np_t_stop_ind: + # move sell onto next candle, we only look back on sell + # will use the open price later. + t_exit_ind = t_open_ind + np_t_sell_ind + 1 # Set Exit row index + t_exit_type = SellType.SELL_SIGNAL # Set Exit type (sell) + np_t_exit_pri = np_open # The price field our SELL exit will use + if debug: + print("Type SELL is first exit condition. " + "At view index", np_t_sell_ind, ". Ticker data exit index is", t_exit_ind) + + # No stop or buy left in view - set t_exit_last -1 to handle gracefully + else: + t_exit_last: int = -1 # Signal loop to exit, no buys or sells found. + t_exit_type = SellType.NONE + np_t_exit_pri = 999 # field price should be calculated on. 999 a non-existent column + if debug: + print("No valid STOP or SELL found. Signalling t_exit_last to gracefully exit") + + # TODO: fix having to cludge/uncludge this .. + # Undo cludge + np_t_sell_ind = -1 if np_t_sell_ind == 99999999 else np_t_sell_ind + np_t_stop_ind = -1 if np_t_stop_ind == 99999999 else np_t_stop_ind + + if debug_timing: + t_t = f(st) + print("5-logic", str.format('{0:.17f}', t_t)) + st = s() + + if debug: + ''' + Print out the buys, stops, sells + Include Line before and after to for easy + Human verification + ''' + # Combine the np_t_stop_pri value to bslap dataframe to make debug + # life easy. This is the current stop price based on buy price_ + # This is slow but don't care about performance in debug + # + # When referencing equiv np_column, as example np_sto, its 5 in numpy and 6 in df, so +1 + # as there is no data column in the numpy array. + bslap['np_stop_pri'] = np_t_stop_pri + + # Buy + print("\n\nDATAFRAME DEBUG =================== BUY ", pair) + print("Numpy Array BUY Index is:", 0) + print("DataFrame BUY Index is:", t_open_ind, "displaying DF \n") + print("HINT, BUY trade should use OPEN price from next candle, i.e ", t_open_ind + 1) + op_is = t_open_ind - 1 # Print open index start, line before + op_if = t_open_ind + 3 # Print open index finish, line after + print(bslap.iloc[op_is:op_if], "\n") + + # Stop - Stops trigger price np_sto (+1 for pandas column), and price received np_sco +1. (Stop Trigger|Calculated On) + if np_t_stop_ind < 0: + print("DATAFRAME DEBUG =================== STOP ", pair) + print("No STOPS were found until the end of ticker data file\n") + else: + print("DATAFRAME DEBUG =================== STOP ", pair) + print("Numpy Array STOP Index is:", np_t_stop_ind, "View starts at index", t_open_ind) + df_stop_index = (t_open_ind + np_t_stop_ind) + + print("DataFrame STOP Index is:", df_stop_index, "displaying DF \n") + print("First Stoploss trigger after Trade entered at OPEN in candle", t_open_ind + 1, "is ", + df_stop_index, ": \n", + str.format('{0:.17f}', bslap.iloc[df_stop_index][np_sto + 1]), + "is less than", str.format('{0:.17f}', np_t_stop_pri)) + + print("A stoploss exit will be calculated at rate:", + str.format('{0:.17f}', bslap.iloc[df_stop_index][np_sco + 1])) + + print("\nHINT, STOPs should exit in-candle, i.e", df_stop_index, + ": As live STOPs are not linked to O-C times") + + st_is = df_stop_index - 1 # Print stop index start, line before + st_if = df_stop_index + 2 # Print stop index finish, line after + print(bslap.iloc[st_is:st_if], "\n") + + # Sell + if np_t_sell_ind < 0: + print("DATAFRAME DEBUG =================== SELL ", pair) + print("No SELLS were found till the end of ticker data file\n") + else: + print("DATAFRAME DEBUG =================== SELL ", pair) + print("Numpy View SELL Index is:", np_t_sell_ind, "View starts at index", t_open_ind) + df_sell_index = (t_open_ind + np_t_sell_ind) + + print("DataFrame SELL Index is:", df_sell_index, "displaying DF \n") + print("First Sell Index after Trade open is in candle", df_sell_index) + print("HINT, if exit is SELL (not stop) trade should use OPEN price from next candle", + df_sell_index + 1) + sl_is = df_sell_index - 1 # Print sell index start, line before + sl_if = df_sell_index + 3 # Print sell index finish, line after + print(bslap.iloc[sl_is:sl_if], "\n") + + # Chosen Exit (stop or sell) + + print("DATAFRAME DEBUG =================== EXIT ", pair) + print("Exit type is :", t_exit_type) + print("trade exit price field is", np_t_exit_pri, "\n") + + if debug_timing: + t_t = f(st) + print("6-depra", str.format('{0:.17f}', t_t)) + st = s() + + ## use numpy view "np_t_open_v" for speed. Columns are + # buy 0 - open 1 - close 2 - sell 3 - high 4 - low 5 + # exception is 6 which is use the stop value. + + # TODO no! this is hard coded bleh fix this open + np_trade_enter_price = np_bslap[t_open_ind + 1, np_open] + if t_exit_type == SellType.STOP_LOSS: + if np_t_exit_pri == 6: + np_trade_exit_price = np_t_stop_pri + else: + np_trade_exit_price = np_bslap[t_exit_ind, np_t_exit_pri] + if t_exit_type == SellType.SELL_SIGNAL: + np_trade_exit_price = np_bslap[t_exit_ind, np_t_exit_pri] + + # Catch no exit found + if t_exit_type == SellType.NONE: + np_trade_exit_price = 0 + + if debug_timing: + t_t = f(st) + print("7-numpy", str.format('{0:.17f}', t_t)) + st = s() + + if debug: + print("//////////////////////////////////////////////") + print("+++++++++++++++++++++++++++++++++ Trade Enter ") + print("np_trade Enter Price is ", str.format('{0:.17f}', np_trade_enter_price)) + print("--------------------------------- Trade Exit ") + print("Trade Exit Type is ", t_exit_type) + print("np_trade Exit Price is", str.format('{0:.17f}', np_trade_exit_price)) + print("//////////////////////////////////////////////") + + else: # no buys were found, step 0 returned -1 + # Gracefully exit the loop + t_exit_last == -1 + if debug: + print("\n(E) No buys were found in remaining ticker file. Exiting", pair) + + # Loop control - catch no closed trades. + if debug: + print("---------------------------------------- end of loop", loop, + " Dataframe Exit Index is: ", t_exit_ind) + print("Exit Index Last, Exit Index Now Are: ", t_exit_last, t_exit_ind) + + if t_exit_last >= t_exit_ind or t_exit_last == -1: + """ + Break loop and go on to next pair. + + When last trade exit equals index of last exit, there is no + opportunity to close any more trades. + """ + # TODO :add handing here to record none closed open trades + + if debug: + print(bslap_pair_results) + break + else: + """ + Add trade to backtest looking results list of dicts + Loop back to look for more trades. + """ + # Build trade dictionary + ## In general if a field can be calculated later from other fields leave blank here + ## Its X(number of trades faster) to calc all in a single vector than 1 trade at a time + + # create a new dict + close_index: int = t_exit_ind + bslap_result = {} # Must have at start or we end up with a list of multiple same last result + bslap_result["pair"] = pair + bslap_result["stoploss"] = stop + bslap_result["profit_percent"] = "" # To be 1 vector calc across trades when loop complete + bslap_result["profit_abs"] = "" # To be 1 vector calc across trades when loop complete + bslap_result["open_time"] = np_bslap_dates[t_open_ind + 1] # use numpy array, pandas 20x slower + bslap_result["close_time"] = np_bslap_dates[close_index] # use numpy array, pandas 20x slower + bslap_result["open_index"] = t_open_ind + 1 # +1 as we buy on next. + bslap_result["close_index"] = close_index + bslap_result["trade_duration"] = "" # To be 1 vector calc across trades when loop complete + bslap_result["open_at_end"] = False + bslap_result["open_rate"] = round(np_trade_enter_price, 15) + bslap_result["close_rate"] = round(np_trade_exit_price, 15) + bslap_result["exit_type"] = t_exit_type + bslap_result["sell_reason"] = t_exit_type #duplicated, but I don't care + # append the dict to the list and print list + bslap_pair_results.append(bslap_result) + + if t_exit_type is SellType.STOP_LOSS: + stop_stops_count = stop_stops_count + 1 + + if debug: + print("The trade dict is: \n", bslap_result) + print("Trades dicts in list after append are: \n ", bslap_pair_results) + + """ + Loop back to start. t_exit_last becomes where loop + will seek to open new trades from. + Push index on 1 to not open on close + """ + t_exit_last = t_exit_ind + 1 + + if debug_timing: + t_t = f(st) + print("8+trade", str.format('{0:.17f}', t_t)) + + # Send back List of trade dicts + return bslap_pair_results + + def _process_result(self, data: Dict[str, Dict], results: DataFrame, stoploss_range) -> str: + """ + This is a temporary version of edge positioning calculation. + The function will be eventually moved to a plugin called Edge in order to calculate necessary WR, RRR and + other indictaors related to money management periodically (each X minutes) and keep it in a storage. + The calulation will be done per pair and per strategy. + """ + + # Removing open trades from dataset + results = results[results.open_at_end == False] + ################################### + + # Removing pairs having less than min_trades_number + min_trades_number = 50 + results = results.groupby('pair').filter(lambda x: len(x) > min_trades_number) + ################################### + + + # Removing outliers (Pump&Dumps) from the dataset + # The method to detect outliers is to calculate standard deviation + # Then every value more than (standard deviation + 2*average) is out (pump) + # And every value less than (standard deviation - 2*average) is out (dump) + # + # Calculating standard deviation of profits + std = results[["profit_abs"]].std() + # + # Calculating average of profits + avg = results[["profit_abs"]].mean() + # + # Removing Pumps + results = results[results.profit_abs < float(avg + 2*std)] + # + # Removing Dumps + results = results[results.profit_abs > float(avg - 2*std)] + ########################################################################## + + # Removing trades having a duration more than X minutes (set in config) + max_trade_duration = 24*60 + results = results[results.trade_duration < max_trade_duration] + ####################################################################### + + + # Win Rate is the number of profitable trades + # Divided by number of trades + def winrate(x): + x = x[x > 0].count() / x.count() + return x + ############################# + + # Risk Reward Ratio + # 1 / ((loss money / losing trades) / (gained money / winning trades)) + def risk_reward_ratio(x): + x = abs(1/ ((x[x<0].sum() / x[x < 0].count()) / (x[x > 0].sum() / x[x > 0].count()))) + return x + ############################## + + # Required Risk Reward + # (1/(winrate - 1) + def required_risk_reward(x): + x = (1/(x[x > 0].count()/x.count()) -1) + return x + ############################## + + # The difference between risk reward ratio and required risk reward + # We use it as an indicator to find the most interesting pair to trade + def delta(x): + x = (abs(1/ ((x[x < 0].sum() / x[x < 0].count()) / (x[x > 0].sum() / x[x > 0].count())))) - (1/(x[x > 0].count()/x.count()) -1) + return x + ############################## + + + final = results.groupby(['pair', 'stoploss'])['profit_abs'].\ + agg([winrate, risk_reward_ratio, required_risk_reward, delta]).\ + reset_index().sort_values(by=['delta', 'stoploss'], ascending=False)\ + .groupby('pair').first().sort_values(by=['delta'], ascending=False) + + return final + + + def start(self) -> None: + """ + Run a backtesting end-to-end + :return: None + """ + data = {} + pairs = self.config['exchange']['pair_whitelist'] + logger.info('Using stake_currency: %s ...', self.config['stake_currency']) + logger.info('Using stake_amount: %s ...', self.config['stake_amount']) + + if self.config.get('live'): + logger.info('Downloading data for all pairs in whitelist ...') + for pair in pairs: + data[pair] = self.exchange.get_ticker_history(pair, self.ticker_interval) + else: + logger.info('Using local backtesting data (using whitelist in given config) ...') + + timerange = Arguments.parse_timerange(None if self.config.get( + 'timerange') is None else str(self.config.get('timerange'))) + + data = optimize.load_data( + self.config['datadir'], + pairs=pairs, + ticker_interval=self.ticker_interval, + refresh_pairs=self.config.get('refresh_pairs', False), + exchange=self.exchange, + timerange=timerange + ) + + if not data: + logger.critical("No data found. Terminating.") + return + + # Use max_open_trades in backtesting, except --disable-max-market-positions is set + if self.config.get('use_max_market_positions', True): + max_open_trades = self.config['max_open_trades'] + else: + logger.info('Ignoring max_open_trades (--disable-max-market-positions was used) ...') + max_open_trades = 0 + + preprocessed = self.tickerdata_to_dataframe(data) + + # Print timeframe + min_date, max_date = self.get_timeframe(preprocessed) + logger.info( + 'Measuring data from %s up to %s (%s days)..', + min_date.isoformat(), + max_date.isoformat(), + (max_date - min_date).days + ) + + stoploss_range = np.arange(-0.11, -0.00, 0.01) + + # Execute backtest and print results + results = self.run( + { + 'stake_amount': self.config.get('stake_amount'), + 'processed': preprocessed, + 'stoploss_range': stoploss_range, + 'max_open_trades': max_open_trades, + 'position_stacking': self.config.get('position_stacking', False), + } + ) + + logger.info( + '\n====================================================== ' + 'Edge positionning REPORT' + ' =======================================================\n' + '%s', + self._process_result( + data, + results, + stoploss_range + ) + ) + + +def setup_configuration(args: Namespace) -> Dict[str, Any]: + """ + Prepare the configuration for the backtesting + :param args: Cli args from Arguments() + :return: Configuration + """ + configuration = Configuration(args) + config = configuration.get_config() + + # Ensure we do not use Exchange credentials + config['exchange']['key'] = '' + config['exchange']['secret'] = '' + config['backslap'] = args.backslap + if config['stake_amount'] == constants.UNLIMITED_STAKE_AMOUNT: + raise DependencyException('stake amount could not be "%s" for backtesting' % + constants.UNLIMITED_STAKE_AMOUNT) + + return config + +# Initialize configuration +arguments = Arguments( + sys.argv[1:], + 'Simple High Frequency Trading Bot for crypto currencies' + ) +args = arguments.get_parsed_arg() + +config = setup_configuration(args) + +config["strategy"] = "MultiRSI" +edge = Edge(config) +edge.start() From 5d9c7fa82dd261325635f74f666f98bc26af36c4 Mon Sep 17 00:00:00 2001 From: Matthias Date: Fri, 14 Sep 2018 19:56:04 +0200 Subject: [PATCH 037/699] add point about malicious code --- CONTRIBUTING.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 97130eded..1c840fa31 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -106,6 +106,7 @@ Exceptions: ### Responsibilities - Ensure cross-platform compatibility for every change that's accepted. Windows, Mac & Linux. +- Ensure no malicious code is introduced into the core code. - Create issues for any major changes and enhancements that you wish to make. Discuss things transparently and get community feedback. - Keep feature versions as small as possible, preferably one new feature per version. - Be welcoming to newcomers and encourage diverse new contributors from all backgrounds. See the Python Community Code of Conduct (https://www.python.org/psf/codeofconduct/). From 07ba14d1ea150366f953c9ae3e1ca4bc2bd25aec Mon Sep 17 00:00:00 2001 From: misagh Date: Sat, 15 Sep 2018 15:52:10 +0200 Subject: [PATCH 038/699] backslap bug resolved --- freqtrade/optimize/edge.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/optimize/edge.py b/freqtrade/optimize/edge.py index ad37bf49b..88e92f03b 100644 --- a/freqtrade/optimize/edge.py +++ b/freqtrade/optimize/edge.py @@ -627,7 +627,7 @@ class Edge: elif np_t_sell_ind < 99999999 and np_t_sell_ind < np_t_stop_ind: # move sell onto next candle, we only look back on sell # will use the open price later. - t_exit_ind = t_open_ind + np_t_sell_ind + 1 # Set Exit row index + t_exit_ind = t_open_ind + np_t_sell_ind # Set Exit row index t_exit_type = SellType.SELL_SIGNAL # Set Exit type (sell) np_t_exit_pri = np_open # The price field our SELL exit will use if debug: From 88854cba2d2a7da001b37a4292121071c4a04bd7 Mon Sep 17 00:00:00 2001 From: misagh Date: Sat, 15 Sep 2018 15:53:42 +0200 Subject: [PATCH 039/699] removing only pumps from dataset --- freqtrade/optimize/edge.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/freqtrade/optimize/edge.py b/freqtrade/optimize/edge.py index 88e92f03b..cfe1b3012 100644 --- a/freqtrade/optimize/edge.py +++ b/freqtrade/optimize/edge.py @@ -852,10 +852,9 @@ class Edge: ################################### - # Removing outliers (Pump&Dumps) from the dataset + # Removing outliers (Only Pumps) from the dataset # The method to detect outliers is to calculate standard deviation # Then every value more than (standard deviation + 2*average) is out (pump) - # And every value less than (standard deviation - 2*average) is out (dump) # # Calculating standard deviation of profits std = results[["profit_abs"]].std() @@ -865,9 +864,6 @@ class Edge: # # Removing Pumps results = results[results.profit_abs < float(avg + 2*std)] - # - # Removing Dumps - results = results[results.profit_abs > float(avg - 2*std)] ########################################################################## # Removing trades having a duration more than X minutes (set in config) From decaf6c42ea28cfcb932442533dcb94b02bf44f4 Mon Sep 17 00:00:00 2001 From: misagh Date: Thu, 20 Sep 2018 16:15:53 +0200 Subject: [PATCH 040/699] =?UTF-8?q?Backslap=20bug=20on=20=E2=80=9Cstop=20l?= =?UTF-8?q?oss=20triggered=E2=80=9D=20indexes=20resolved?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- freqtrade/optimize/backslapping.py | 24 +++++++++++++++++++++--- freqtrade/optimize/edge.py | 21 +++++++++++++-------- 2 files changed, 34 insertions(+), 11 deletions(-) diff --git a/freqtrade/optimize/backslapping.py b/freqtrade/optimize/backslapping.py index 28b2bd460..7eba39a6e 100644 --- a/freqtrade/optimize/backslapping.py +++ b/freqtrade/optimize/backslapping.py @@ -136,7 +136,7 @@ class Backslapping: bslap_results_df = [] bslap_results_df = DataFrame.from_records(bslap_results_df, columns=BacktestResult._fields) - + return bslap_results_df def vector_fill_results_table(self, bslap_results_df: DataFrame, pair: str): @@ -280,10 +280,19 @@ class Backslapping: # debug = False # print values, to check accuracy debug_2loops = self.debug_2loops # only loop twice, for faster debug debug_timing = self.debug_timing # print timing for each step - debug = self.debug # print values, to check accuracy + #debug = self.debug # print values, to check accuracy + debug = False + + ticker_data = ticker_data.sort_values(by=['date']) + ticker_data = ticker_data.reset_index(drop=True) + + #pandas_df = df.toPandas() + #pandas_df.to_json # Read Stop Loss Values and Stake + # pdb.set_trace() stop = self.stop_loss_value + #stop = stoploss p_stop = (stop + 1) # What stop really means, e.g 0.01 is 0.99 of price if debug: @@ -583,7 +592,7 @@ class Backslapping: elif np_t_sell_ind < 99999999 and np_t_sell_ind < np_t_stop_ind: # move sell onto next candle, we only look back on sell # will use the open price later. - t_exit_ind = t_open_ind + np_t_sell_ind + 1 # Set Exit row index + t_exit_ind = t_open_ind + np_t_sell_ind # Set Exit row index t_exit_type = SellType.SELL_SIGNAL # Set Exit type (sell) np_t_exit_pri = np_open # The price field our SELL exit will use if debug: @@ -693,6 +702,7 @@ class Backslapping: if t_exit_type == SellType.STOP_LOSS: if np_t_exit_pri == 6: np_trade_exit_price = np_t_stop_pri + t_exit_ind = t_exit_ind + 1 else: np_trade_exit_price = np_bslap[t_exit_ind, np_t_exit_pri] if t_exit_type == SellType.SELL_SIGNAL: @@ -736,6 +746,7 @@ class Backslapping: opportunity to close any more trades. """ # TODO :add handing here to record none closed open trades + if debug: print(bslap_pair_results) @@ -745,6 +756,12 @@ class Backslapping: Add trade to backtest looking results list of dicts Loop back to look for more trades. """ + + # We added +1 to t_exit_ind if the exit was a stop-loss, to not exit early in the IF of this ELSE + # removing the +1 here so prices match. + if t_exit_type == SellType.STOP_LOSS: + t_exit_ind = t_exit_ind - 1 + # Build trade dictionary ## In general if a field can be calculated later from other fields leave blank here ## Its X(number of trades faster) to calc all in a single vector than 1 trade at a time @@ -753,6 +770,7 @@ class Backslapping: close_index: int = t_exit_ind bslap_result = {} # Must have at start or we end up with a list of multiple same last result bslap_result["pair"] = pair + bslap_result["stoploss"] = stop bslap_result["profit_percent"] = "" # To be 1 vector calc across trades when loop complete bslap_result["profit_abs"] = "" # To be 1 vector calc across trades when loop complete bslap_result["open_time"] = np_bslap_dates[t_open_ind + 1] # use numpy array, pandas 20x slower diff --git a/freqtrade/optimize/edge.py b/freqtrade/optimize/edge.py index cfe1b3012..86ffb6747 100644 --- a/freqtrade/optimize/edge.py +++ b/freqtrade/optimize/edge.py @@ -133,6 +133,10 @@ class Edge: if self.debug_timing: # Start timer fl = self.s() + # Sorting dataframe by date and reset index + pair_data = pair_data.sort_values(by=['date']) + pair_data = pair_data.reset_index(drop=True) + ticker_data = self.populate_sell_trend( self.populate_buy_trend(pair_data))[headers].copy() @@ -151,11 +155,6 @@ class Edge: for stoploss in stoploss_range: bslap_results += self.backslap_pair(ticker_data, pair, round(stoploss, 3)) - #bslap_results += self.backslap_pair(ticker_data, pair, -0.05) - # bslap_pair_results = self.backslap_pair(ticker_data, pair, -0.05) - # last_bslap_results = bslap_results - # bslap_results = last_bslap_results + bslap_pair_results - if self.debug_timing: # print time taken tt = self.f(st) @@ -627,7 +626,7 @@ class Edge: elif np_t_sell_ind < 99999999 and np_t_sell_ind < np_t_stop_ind: # move sell onto next candle, we only look back on sell # will use the open price later. - t_exit_ind = t_open_ind + np_t_sell_ind # Set Exit row index + t_exit_ind = t_open_ind + np_t_sell_ind # Set Exit row index t_exit_type = SellType.SELL_SIGNAL # Set Exit type (sell) np_t_exit_pri = np_open # The price field our SELL exit will use if debug: @@ -737,6 +736,7 @@ class Edge: if t_exit_type == SellType.STOP_LOSS: if np_t_exit_pri == 6: np_trade_exit_price = np_t_stop_pri + t_exit_ind = t_exit_ind + 1 else: np_trade_exit_price = np_bslap[t_exit_ind, np_t_exit_pri] if t_exit_type == SellType.SELL_SIGNAL: @@ -780,7 +780,7 @@ class Edge: opportunity to close any more trades. """ # TODO :add handing here to record none closed open trades - + if debug: print(bslap_pair_results) break @@ -789,6 +789,12 @@ class Edge: Add trade to backtest looking results list of dicts Loop back to look for more trades. """ + + # We added +1 to t_exit_ind if the exit was a stop-loss, to not exit early in the IF of this ELSE + # removing the +1 here so prices match. + if t_exit_type == SellType.STOP_LOSS: + t_exit_ind = t_exit_ind - 1 + # Build trade dictionary ## In general if a field can be calculated later from other fields leave blank here ## Its X(number of trades faster) to calc all in a single vector than 1 trade at a time @@ -1014,6 +1020,5 @@ args = arguments.get_parsed_arg() config = setup_configuration(args) -config["strategy"] = "MultiRSI" edge = Edge(config) edge.start() From ef52c7b510a80b82d63a079901c93e28cf648b6e Mon Sep 17 00:00:00 2001 From: misagh Date: Fri, 21 Sep 2018 17:41:31 +0200 Subject: [PATCH 041/699] edge positioning put into package --- config.json.example | 7 + freqtrade/edge/__init__.py | 929 ++++++++++++++++++++++++++++++++ freqtrade/freqtradebot.py | 43 +- freqtrade/strategy/interface.py | 11 +- freqtrade/tests/conftest.py | 9 + 5 files changed, 980 insertions(+), 19 deletions(-) create mode 100644 freqtrade/edge/__init__.py diff --git a/config.json.example b/config.json.example index 7a0bb6b9b..801a5137c 100644 --- a/config.json.example +++ b/config.json.example @@ -50,6 +50,13 @@ "sell_profit_only": false, "ignore_roi_if_buy_signal": false }, + "edge": { + "enabled": false, + "process_throttle_secs": 1800, + "maximum_winrate_to_consider": 0.80, + "remove_pumps": true, + "minimum_delta_to_consider": 1 + }, "telegram": { "enabled": true, "token": "your_telegram_token", diff --git a/freqtrade/edge/__init__.py b/freqtrade/edge/__init__.py new file mode 100644 index 000000000..9668e1a44 --- /dev/null +++ b/freqtrade/edge/__init__.py @@ -0,0 +1,929 @@ +import logging +import operator +import sys +from argparse import Namespace +from datetime import datetime, timedelta +from typing import Any, Dict, List, NamedTuple, Optional, Tuple + +import arrow +from pandas import DataFrame, to_datetime +from tabulate import tabulate +import numpy as np + +import freqtrade.optimize as optimize +from freqtrade import DependencyException, constants +from freqtrade.arguments import Arguments +from freqtrade.configuration import Configuration +from freqtrade.exchange import Exchange +from freqtrade.misc import file_dump_json +from freqtrade.persistence import Trade +from freqtrade.strategy.interface import SellType +from freqtrade.strategy.resolver import IStrategy, StrategyResolver +from freqtrade.optimize.backtesting import Backtesting +from collections import OrderedDict +import timeit +from time import sleep + +import pdb + +logger = logging.getLogger(__name__) + +class Edge(): + + config: Dict = {} + + def __init__(self, config: Dict[str, Any], exchange = None) -> None: + """ + constructor + """ + self.config = config + self.strategy: IStrategy = StrategyResolver(self.config).strategy + self.ticker_interval = self.strategy.ticker_interval + self.tickerdata_to_dataframe = self.strategy.tickerdata_to_dataframe + self.get_timeframe = Backtesting.get_timeframe + self.populate_buy_trend = self.strategy.populate_buy_trend + self.populate_sell_trend = self.strategy.populate_sell_trend + + self._last_updated = None + self._cached_pairs = [] + self._total_capital = self.config['edge']['total_capital_in_stake_currency'] + self._allowed_risk = self.config['edge']['allowed_risk'] + + ### + # + ### + if exchange is None: + self.config['exchange']['secret'] = '' + self.config['exchange']['password'] = '' + self.config['exchange']['uid'] = '' + self.config['dry_run'] = True + self.exchange = Exchange(self.config) + else: + self.exchange = exchange + + self.fee = self.exchange.get_fee() + + self.stop_loss_value = self.strategy.stoploss + + #### backslap config + ''' + Numpy arrays are used for 100x speed up + We requires setting Int values for + buy stop triggers and stop calculated on + # buy 0 - open 1 - close 2 - sell 3 - high 4 - low 5 - stop 6 + ''' + self.np_buy: int = 0 + self.np_open: int = 1 + self.np_close: int = 2 + self.np_sell: int = 3 + self.np_high: int = 4 + self.np_low: int = 5 + self.np_stop: int = 6 + self.np_bto: int = self.np_close # buys_triggered_on - should be close + self.np_bco: int = self.np_open # buys calculated on - open of the next candle. + self.np_sto: int = self.np_low # stops_triggered_on - Should be low, FT uses close + self.np_sco: int = self.np_stop # stops_calculated_on - Should be stop, FT uses close + # self.np_sto: int = self.np_close # stops_triggered_on - Should be low, FT uses close + # self.np_sco: int = self.np_close # stops_calculated_on - Should be stop, FT uses close + + self.debug = False # Main debug enable, very print heavy, enable 2 loops recommended + self.debug_timing = False # Stages within Backslap + self.debug_2loops = False # Limit each pair to two loops, useful when debugging + self.debug_vector = False # Debug vector calcs + self.debug_timing_main_loop = False # print overall timing per pair - works in Backtest and Backslap + + self.backslap_show_trades = False # prints trades in addition to summary report + self.backslap_save_trades = True # saves trades as a pretty table to backslap.txt + + self.stop_stops: int = 9999 # stop back testing any pair with this many stops, set to 999999 to not hit + + + def calculate(self) -> bool: + pairs = self.config['exchange']['pair_whitelist'] + heartbeat = self.config['edge']['process_throttle_secs'] + + if ((self._last_updated is not None) and (self._last_updated + heartbeat > arrow.utcnow().timestamp)): + return False + + data = {} + + logger.info('Using stake_currency: %s ...', self.config['stake_currency']) + logger.info('Using stake_amount: %s ...', self.config['stake_amount']) + + logger.info('Using local backtesting data (using whitelist in given config) ...') + + timerange = Arguments.parse_timerange(None if self.config.get( + 'timerange') is None else str(self.config.get('timerange'))) + + data = optimize.load_data( + self.config['datadir'], + pairs=pairs, + ticker_interval=self.ticker_interval, + refresh_pairs=self.config.get('refresh_pairs', False), + exchange=self.exchange, + timerange=timerange + ) + + if not data: + logger.critical("No data found. Edge is stopped ...") + return + + preprocessed = self.tickerdata_to_dataframe(data) + + # Print timeframe + min_date, max_date = self.get_timeframe(preprocessed) + logger.info( + 'Measuring data from %s up to %s (%s days) ...', + min_date.isoformat(), + max_date.isoformat(), + (max_date - min_date).days + ) + + + headers = ['date', 'buy', 'open', 'close', 'sell', 'high', 'low'] + + # Max open trades need not be considered in Edge positioning + max_open_trades = 0 + + realistic = False + stoploss_range = np.arange(-0.11, -0.00, 0.01) + trades = [] + trade_count_lock: Dict = {} + + ########################### Call out BSlap Loop instead of Original BT code + bslap_results: list = [] + for pair, pair_data in preprocessed.items(): + + # Sorting dataframe by date and reset index + pair_data = pair_data.sort_values(by=['date']) + pair_data = pair_data.reset_index(drop=True) + + ticker_data = self.populate_sell_trend( + self.populate_buy_trend(pair_data))[headers].copy() + + # call backslap - results are a list of dicts + for stoploss in stoploss_range: + bslap_results += self.backslap_pair(ticker_data, pair, round(stoploss, 3)) + + # Switch List of Trade Dicts (bslap_results) to Dataframe + # Fill missing, calculable columns, profit, duration , abs etc. + bslap_results_df = DataFrame(bslap_results) + + if len(bslap_results_df) > 0: # Only post process a frame if it has a record + bslap_results_df = self.vector_fill_results_table(bslap_results_df) + else: + from freqtrade.optimize.backtesting import BacktestResult + + bslap_results_df = [] + bslap_results_df = DataFrame.from_records(bslap_results_df, columns=BacktestResult._fields) + + self._cached_pairs = self._process_result(data, bslap_results_df, stoploss_range) + self._last_updated = arrow.utcnow().timestamp + return True + + def sort_pairs(self, pairs) -> bool: + if len(self._cached_pairs) == 0: + self.calculate() + edge_sorted_pairs = [x[0] for x in self._cached_pairs] + return [x for _, x in sorted(zip(edge_sorted_pairs,pairs), key=lambda pair: pair[0])] + + + def vector_fill_results_table(self, bslap_results_df: DataFrame): + """ + The Results frame contains a number of columns that are calculable + from othe columns. These are left blank till all rows are added, + to be populated in single vector calls. + + Columns to be populated are: + - Profit + - trade duration + - profit abs + :param bslap_results Dataframe + :return: bslap_results Dataframe + """ + import pandas as pd + import numpy as np + debug = self.debug_vector + + # stake and fees + # stake = 0.015 + # 0.05% is 0.0005 + # fee = 0.001 + + stake = self.config.get('stake_amount') + fee = self.fee + open_fee = fee / 2 + close_fee = fee / 2 + + + bslap_results_df['trade_duration'] = bslap_results_df['close_time'] - bslap_results_df['open_time'] + bslap_results_df['trade_duration'] = bslap_results_df['trade_duration'].map(lambda x: int(x.total_seconds() / 60)) + + ## Spends, Takes, Profit, Absolute Profit + # print(bslap_results_df) + # Buy Price + bslap_results_df['buy_vol'] = stake / bslap_results_df['open_rate'] # How many target are we buying + bslap_results_df['buy_fee'] = stake * open_fee + bslap_results_df['buy_spend'] = stake + bslap_results_df['buy_fee'] # How much we're spending + + # Sell price + bslap_results_df['sell_sum'] = bslap_results_df['buy_vol'] * bslap_results_df['close_rate'] + bslap_results_df['sell_fee'] = bslap_results_df['sell_sum'] * close_fee + bslap_results_df['sell_take'] = bslap_results_df['sell_sum'] - bslap_results_df['sell_fee'] + # profit_percent + bslap_results_df['profit_percent'] = (bslap_results_df['sell_take'] - bslap_results_df['buy_spend']) \ + / bslap_results_df['buy_spend'] + # Absolute profit + bslap_results_df['profit_abs'] = bslap_results_df['sell_take'] - bslap_results_df['buy_spend'] + + + return bslap_results_df + + def np_get_t_open_ind(self, np_buy_arr, t_exit_ind: int, np_buy_arr_len: int, stop_stops: int, + stop_stops_count: int): + import utils_find_1st as utf1st + """ + The purpose of this def is to return the next "buy" = 1 + after t_exit_ind. + + This function will also check is the stop limit for the pair has been reached. + if stop_stops is the limit and stop_stops_count it the number of times the stop has been hit. + + t_exit_ind is the index the last trade exited on + or 0 if first time around this loop. + + stop_stops i + """ + debug = self.debug + + # Timers, to be called if in debug + def s(): + st = timeit.default_timer() + return st + + def f(st): + return (timeit.default_timer() - st) + + st = s() + t_open_ind: int + + """ + Create a view on our buy index starting after last trade exit + Search for next buy + """ + np_buy_arr_v = np_buy_arr[t_exit_ind:] + t_open_ind = utf1st.find_1st(np_buy_arr_v, 1, utf1st.cmp_equal) + + ''' + If -1 is returned no buy has been found, preserve the value + ''' + if t_open_ind != -1: # send back the -1 if no buys found. otherwise update index + t_open_ind = t_open_ind + t_exit_ind # Align numpy index + + if t_open_ind == np_buy_arr_len - 1: # If buy found on last candle ignore, there is no OPEN in next to use + t_open_ind = -1 # -1 ends the loop + + if stop_stops_count >= stop_stops: # if maximum number of stops allowed in a pair is hit, exit loop + t_open_ind = -1 # -1 ends the loop + if debug: + print("Max stop limit ", stop_stops, "reached. Moving to next pair") + + return t_open_ind + + def _process_result(self, data: Dict[str, Dict], results: DataFrame, stoploss_range) -> str: + """ + This is a temporary version of edge positioning calculation. + The function will be eventually moved to a plugin called Edge in order to calculate necessary WR, RRR and + other indictaors related to money management periodically (each X minutes) and keep it in a storage. + The calulation will be done per pair and per strategy. + """ + + # Removing open trades from dataset + results = results[results.open_at_end == False] + ################################### + + # Removing pairs having less than min_trades_number + min_trades_number = 50 + results = results.groupby('pair').filter(lambda x: len(x) > min_trades_number) + ################################### + + + # Removing outliers (Only Pumps) from the dataset + # The method to detect outliers is to calculate standard deviation + # Then every value more than (standard deviation + 2*average) is out (pump) + # + # Calculating standard deviation of profits + std = results[["profit_abs"]].std() + # + # Calculating average of profits + avg = results[["profit_abs"]].mean() + # + # Removing Pumps + results = results[results.profit_abs < float(avg + 2*std)] + ########################################################################## + + # Removing trades having a duration more than X minutes (set in config) + max_trade_duration = 24*60 + results = results[results.trade_duration < max_trade_duration] + ####################################################################### + + + # Win Rate is the number of profitable trades + # Divided by number of trades + def winrate(x): + x = x[x > 0].count() / x.count() + return x + ############################# + + # Risk Reward Ratio + # 1 / ((loss money / losing trades) / (gained money / winning trades)) + def risk_reward_ratio(x): + x = abs(1/ ((x[x<0].sum() / x[x < 0].count()) / (x[x > 0].sum() / x[x > 0].count()))) + return x + ############################## + + # Required Risk Reward + # (1/(winrate - 1) + def required_risk_reward(x): + x = (1/(x[x > 0].count()/x.count()) -1) + return x + ############################## + + # The difference between risk reward ratio and required risk reward + # We use it as an indicator to find the most interesting pair to trade + def delta(x): + x = (abs(1/ ((x[x < 0].sum() / x[x < 0].count()) / (x[x > 0].sum() / x[x > 0].count())))) - (1/(x[x > 0].count()/x.count()) -1) + return x + ############################## + + + final = results.groupby(['pair', 'stoploss'])['profit_abs'].\ + agg([winrate, risk_reward_ratio, required_risk_reward, delta]).\ + reset_index().sort_values(by=['delta', 'stoploss'], ascending=False)\ + .groupby('pair').first().sort_values(by=['delta'], ascending=False) + + # Returning an array of pairs in order of "delta" + return final.reset_index().values + + def backslap_pair(self, ticker_data, pair, stoploss): + import pandas as pd + import numpy as np + import timeit + import utils_find_1st as utf1st + from datetime import datetime + + ### backslap debug wrap + # debug_2loops = False # only loop twice, for faster debug + # debug_timing = False # print timing for each step + # debug = False # print values, to check accuracy + debug_2loops = self.debug_2loops # only loop twice, for faster debug + debug_timing = self.debug_timing # print timing for each step + debug = self.debug # print values, to check accuracy + + # Read Stop Loss Values and Stake + # pdb.set_trace() + #stop = self.stop_loss_value + stop = stoploss + p_stop = (stop + 1) # What stop really means, e.g 0.01 is 0.99 of price + + if debug: + print("Stop is ", stop, "value from stragey file") + print("p_stop is", p_stop, "value used to multiply to entry price") + + if debug: + from pandas import set_option + set_option('display.max_rows', 5000) + set_option('display.max_columns', 8) + pd.set_option('display.width', 1000) + pd.set_option('max_colwidth', 40) + pd.set_option('precision', 12) + + def s(): + st = timeit.default_timer() + return st + + def f(st): + return (timeit.default_timer() - st) + + #### backslap config + ''' + Numpy arrays are used for 100x speed up + We requires setting Int values for + buy stop triggers and stop calculated on + # buy 0 - open 1 - close 2 - sell 3 - high 4 - low 5 - stop 6 + ''' + + ####### + # Use vars set at top of backtest + np_buy: int = self.np_buy + np_open: int = self.np_open + np_close: int = self.np_close + np_sell: int = self.np_sell + np_high: int = self.np_high + np_low: int = self.np_low + np_stop: int = self.np_stop + np_bto: int = self.np_bto # buys_triggered_on - should be close + np_bco: int = self.np_bco # buys calculated on - open of the next candle. + np_sto: int = self.np_sto # stops_triggered_on - Should be low, FT uses close + np_sco: int = self.np_sco # stops_calculated_on - Should be stop, FT uses close + + ### End Config + + pair: str = pair + + # ticker_data: DataFrame = ticker_dfs[t_file] + bslap: DataFrame = ticker_data + + # Build a single dimension numpy array from "buy" index for faster search + # (500x faster than pandas) + np_buy_arr = bslap['buy'].values + np_buy_arr_len: int = len(np_buy_arr) + + # use numpy array for faster searches in loop, 20x faster than pandas + # buy 0 - open 1 - close 2 - sell 3 - high 4 - low 5 + np_bslap = np.array(bslap[['buy', 'open', 'close', 'sell', 'high', 'low']]) + + # Build a numpy list of date-times. + # We use these when building the trade + # The rationale is to address a value from a pandas cell is thousands of + # times more expensive. Processing time went X25 when trying to use any data from pandas + np_bslap_dates = bslap['date'].values + + loop: int = 0 # how many time around the loop + t_exit_ind = 0 # Start loop from first index + t_exit_last = 0 # To test for exit + + stop_stops = self.stop_stops # Int of stops within a pair to stop trading a pair at + stop_stops_count = 0 # stop counter per pair + + st = s() # Start timer for processing dataframe + if debug: + print('Processing:', pair) + + # Results will be stored in a list of dicts + bslap_pair_results: list = [] + bslap_result: dict = {} + + while t_exit_ind < np_buy_arr_len: + loop = loop + 1 + if debug or debug_timing: + print("-- T_exit_Ind - Numpy Index is", t_exit_ind, " ----------------------- Loop", loop, pair) + if debug_2loops: + if loop == 3: + print( + "++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++Loop debug max met - breaking") + break + ''' + Dev phases + Phase 1 + 1) Manage buy, sell, stop enter/exit + a) Find first buy index + b) Discover first stop and sell hit after buy index + c) Chose first instance as trade exit + + Phase 2 + 2) Manage dynamic Stop and ROI Exit + a) Create trade slice from 1 + b) search within trade slice for dynamice stop hit + c) search within trade slice for ROI hit + ''' + + if debug_timing: + st = s() + ''' + 0 - Find next buy entry + Finds index for first (buy = 1) flag + + Requires: np_buy_arr - a 1D array of the 'buy' column. To find next "1" + Required: t_exit_ind - Either 0, first loop. Or The index we last exited on + Requires: np_buy_arr_len - length of pair array. + Requires: stops_stops - number of stops allowed before stop trading a pair + Requires: stop_stop_counts - count of stops hit in the pair + Provides: The next "buy" index after t_exit_ind + + If -1 is returned no buy has been found in remainder of array, skip to exit loop + ''' + t_open_ind = self.np_get_t_open_ind(np_buy_arr, t_exit_ind, np_buy_arr_len, stop_stops, stop_stops_count) + + if debug: + print("\n(0) numpy debug \nnp_get_t_open, has returned the next valid buy index as", t_open_ind) + print("If -1 there are no valid buys in the remainder of ticker data. Skipping to end of loop") + if debug_timing: + t_t = f(st) + print("0-numpy", str.format('{0:.17f}', t_t)) + st = s() + + if t_open_ind != -1: + + """ + 1 - Create views to search within for our open trade + + The views are our search space for the next Stop or Sell + Numpy view is employed as: + 1,000 faster than pandas searches + Pandas cannot assure it will always return a view, it may make a slow copy. + + The view contains columns: + buy 0 - open 1 - close 2 - sell 3 - high 4 - low 5 + + Requires: np_bslap is our numpy array of the ticker DataFrame + Requires: t_open_ind is the index row with the buy. + Provides: np_t_open_v View of array after buy. + Provides: np_t_open_v_stop View of array after buy +1 + (Stop will search in here to prevent stopping in the past) + """ + np_t_open_v = np_bslap[t_open_ind:] + np_t_open_v_stop = np_bslap[t_open_ind + 1:] + + if debug: + print("\n(1) numpy debug \nNumpy view row 0 is now Ticker_Data Index", t_open_ind) + print("Numpy View: Buy - Open - Close - Sell - High - Low") + print("Row 0", np_t_open_v[0]) + print("Row 1", np_t_open_v[1], ) + if debug_timing: + t_t = f(st) + print("2-numpy", str.format('{0:.17f}', t_t)) + st = s() + + ''' + 2 - Calculate our stop-loss price + + As stop is based on buy price of our trade + - (BTO)Buys are Triggered On np_bto, typically the CLOSE of candle + - (BCO)Buys are Calculated On np_bco, default is OPEN of the next candle. + This is as we only see the CLOSE after it has happened. + The back test assumption is we have bought at first available price, the OPEN + + Requires: np_bslap - is our numpy array of the ticker DataFrame + Requires: t_open_ind - is the index row with the first buy. + Requires: p_stop - is the stop rate, ie. 0.99 is -1% + Provides: np_t_stop_pri - The value stop-loss will be triggered on + ''' + np_t_stop_pri = (np_bslap[t_open_ind + 1, np_bco] * p_stop) + + if debug: + print("\n(2) numpy debug\nStop-Loss has been calculated at:", np_t_stop_pri) + if debug_timing: + t_t = f(st) + print("2-numpy", str.format('{0:.17f}', t_t)) + st = s() + + ''' + 3 - Find candle STO is under Stop-Loss After Trade opened. + + where [np_sto] (stop tiggered on variable: "close", "low" etc) < np_t_stop_pri + + Requires: np_t_open_v_stop Numpy view of ticker_data after buy row +1 (when trade was opened) + Requires: np_sto User Var(STO)StopTriggeredOn. Typically set to "low" or "close" + Requires: np_t_stop_pri The stop-loss price STO must fall under to trigger stop + Provides: np_t_stop_ind The first candle after trade open where STO is under stop-loss + ''' + np_t_stop_ind = utf1st.find_1st(np_t_open_v_stop[:, np_sto], + np_t_stop_pri, + utf1st.cmp_smaller) + + # plus 1 as np_t_open_v_stop is 1 ahead of view np_t_open_v, used from here on out. + np_t_stop_ind = np_t_stop_ind + 1 + + if debug: + print("\n(3) numpy debug\nNext view index with STO (stop trigger on) under Stop-Loss is", + np_t_stop_ind - 1, + ". STO is using field", np_sto, + "\nFrom key: buy 0 - open 1 - close 2 - sell 3 - high 4 - low 5\n") + + print( + "If -1 or 0 returned there is no stop found to end of view, then next two array lines are garbage") + print("Row", np_t_stop_ind - 1, np_t_open_v[np_t_stop_ind]) + print("Row", np_t_stop_ind, np_t_open_v[np_t_stop_ind + 1]) + if debug_timing: + t_t = f(st) + print("3-numpy", str.format('{0:.17f}', t_t)) + st = s() + + ''' + 4 - Find first sell index after trade open + + First index in the view np_t_open_v where ['sell'] = 1 + + Requires: np_t_open_v - view of ticker_data from buy onwards + Requires: no_sell - integer '3', the buy column in the array + Provides: np_t_sell_ind index of view where first sell=1 after buy + ''' + # Use numpy array for faster search for sell + # Sell uses column 3. + # buy 0 - open 1 - close 2 - sell 3 - high 4 - low 5 + # Numpy searches 25-35x quicker than pandas on this data + + np_t_sell_ind = utf1st.find_1st(np_t_open_v[:, np_sell], + 1, utf1st.cmp_equal) + if debug: + print("\n(4) numpy debug\nNext view index with sell = 1 is ", np_t_sell_ind) + print("If 0 or less is returned there is no sell found to end of view, then next lines garbage") + print("Row", np_t_sell_ind, np_t_open_v[np_t_sell_ind]) + print("Row", np_t_sell_ind + 1, np_t_open_v[np_t_sell_ind + 1]) + if debug_timing: + t_t = f(st) + print("4-numpy", str.format('{0:.17f}', t_t)) + st = s() + + ''' + 5 - Determine which was hit first a stop or sell + To then use as exit index price-field (sell on buy, stop on stop) + + STOP takes priority over SELL as would be 'in candle' from tick data + Sell would use Open from Next candle. + So in a draw Stop would be hit first on ticker data in live + + Validity of when types of trades may be executed can be summarised as: + + Tick View + index index Buy Sell open low close high Stop price + open 2am 94 -1 0 0 ----- ------ ------ ----- ----- + open 3am 95 0 1 0 ----- ------ trg buy ----- ----- + open 4am 96 1 0 1 Enter trgstop trg sel ROI out Stop out + open 5am 97 2 0 0 Exit ------ ------- ----- ----- + open 6am 98 3 0 0 ----- ------ ------- ----- ----- + + -1 means not found till end of view i.e no valid Stop found. Exclude from match. + Stop tiggering and closing in 96-1, the candle we bought at OPEN in, is valid. + + Buys and sells are triggered at candle close + Both will open their postions at the open of the next candle. i/e + 1 index + + Stop and buy Indexes are on the view. To map to the ticker dataframe + the t_open_ind index should be summed. + + np_t_stop_ind: Stop Found index in view + t_exit_ind : Sell found in view + t_open_ind : Where view was started on ticker_data + + TODO: fix this frig for logic test,, case/switch/dictionary would be better... + more so when later testing many options, dynamic stop / roi etc + cludge - Setting np_t_sell_ind as 9999999999 when -1 (not found) + cludge - Setting np_t_stop_ind as 9999999999 when -1 (not found) + + ''' + if debug: + print("\n(5) numpy debug\nStop or Sell Logic Processing") + + # cludge for logic test (-1) means it was not found, set crazy high to lose < test + np_t_sell_ind = 99999999 if np_t_sell_ind <= 0 else np_t_sell_ind + np_t_stop_ind = 99999999 if np_t_stop_ind <= 0 else np_t_stop_ind + + # Stoploss trigger found before a sell =1 + if np_t_stop_ind < 99999999 and np_t_stop_ind <= np_t_sell_ind: + t_exit_ind = t_open_ind + np_t_stop_ind # Set Exit row index + t_exit_type = SellType.STOP_LOSS # Set Exit type (stop) + np_t_exit_pri = np_sco # The price field our STOP exit will use + if debug: + print("Type STOP is first exit condition. " + "At view index:", np_t_stop_ind, ". Ticker data exit index is", t_exit_ind) + + # Buy = 1 found before a stoploss triggered + elif np_t_sell_ind < 99999999 and np_t_sell_ind < np_t_stop_ind: + # move sell onto next candle, we only look back on sell + # will use the open price later. + t_exit_ind = t_open_ind + np_t_sell_ind # Set Exit row index + t_exit_type = SellType.SELL_SIGNAL # Set Exit type (sell) + np_t_exit_pri = np_open # The price field our SELL exit will use + if debug: + print("Type SELL is first exit condition. " + "At view index", np_t_sell_ind, ". Ticker data exit index is", t_exit_ind) + + # No stop or buy left in view - set t_exit_last -1 to handle gracefully + else: + t_exit_last: int = -1 # Signal loop to exit, no buys or sells found. + t_exit_type = SellType.NONE + np_t_exit_pri = 999 # field price should be calculated on. 999 a non-existent column + if debug: + print("No valid STOP or SELL found. Signalling t_exit_last to gracefully exit") + + # TODO: fix having to cludge/uncludge this .. + # Undo cludge + np_t_sell_ind = -1 if np_t_sell_ind == 99999999 else np_t_sell_ind + np_t_stop_ind = -1 if np_t_stop_ind == 99999999 else np_t_stop_ind + + if debug_timing: + t_t = f(st) + print("5-logic", str.format('{0:.17f}', t_t)) + st = s() + + if debug: + ''' + Print out the buys, stops, sells + Include Line before and after to for easy + Human verification + ''' + # Combine the np_t_stop_pri value to bslap dataframe to make debug + # life easy. This is the current stop price based on buy price_ + # This is slow but don't care about performance in debug + # + # When referencing equiv np_column, as example np_sto, its 5 in numpy and 6 in df, so +1 + # as there is no data column in the numpy array. + bslap['np_stop_pri'] = np_t_stop_pri + + # Buy + print("\n\nDATAFRAME DEBUG =================== BUY ", pair) + print("Numpy Array BUY Index is:", 0) + print("DataFrame BUY Index is:", t_open_ind, "displaying DF \n") + print("HINT, BUY trade should use OPEN price from next candle, i.e ", t_open_ind + 1) + op_is = t_open_ind - 1 # Print open index start, line before + op_if = t_open_ind + 3 # Print open index finish, line after + print(bslap.iloc[op_is:op_if], "\n") + + # Stop - Stops trigger price np_sto (+1 for pandas column), and price received np_sco +1. (Stop Trigger|Calculated On) + if np_t_stop_ind < 0: + print("DATAFRAME DEBUG =================== STOP ", pair) + print("No STOPS were found until the end of ticker data file\n") + else: + print("DATAFRAME DEBUG =================== STOP ", pair) + print("Numpy Array STOP Index is:", np_t_stop_ind, "View starts at index", t_open_ind) + df_stop_index = (t_open_ind + np_t_stop_ind) + + print("DataFrame STOP Index is:", df_stop_index, "displaying DF \n") + print("First Stoploss trigger after Trade entered at OPEN in candle", t_open_ind + 1, "is ", + df_stop_index, ": \n", + str.format('{0:.17f}', bslap.iloc[df_stop_index][np_sto + 1]), + "is less than", str.format('{0:.17f}', np_t_stop_pri)) + + print("A stoploss exit will be calculated at rate:", + str.format('{0:.17f}', bslap.iloc[df_stop_index][np_sco + 1])) + + print("\nHINT, STOPs should exit in-candle, i.e", df_stop_index, + ": As live STOPs are not linked to O-C times") + + st_is = df_stop_index - 1 # Print stop index start, line before + st_if = df_stop_index + 2 # Print stop index finish, line after + print(bslap.iloc[st_is:st_if], "\n") + + # Sell + if np_t_sell_ind < 0: + print("DATAFRAME DEBUG =================== SELL ", pair) + print("No SELLS were found till the end of ticker data file\n") + else: + print("DATAFRAME DEBUG =================== SELL ", pair) + print("Numpy View SELL Index is:", np_t_sell_ind, "View starts at index", t_open_ind) + df_sell_index = (t_open_ind + np_t_sell_ind) + + print("DataFrame SELL Index is:", df_sell_index, "displaying DF \n") + print("First Sell Index after Trade open is in candle", df_sell_index) + print("HINT, if exit is SELL (not stop) trade should use OPEN price from next candle", + df_sell_index + 1) + sl_is = df_sell_index - 1 # Print sell index start, line before + sl_if = df_sell_index + 3 # Print sell index finish, line after + print(bslap.iloc[sl_is:sl_if], "\n") + + # Chosen Exit (stop or sell) + + print("DATAFRAME DEBUG =================== EXIT ", pair) + print("Exit type is :", t_exit_type) + print("trade exit price field is", np_t_exit_pri, "\n") + + if debug_timing: + t_t = f(st) + print("6-depra", str.format('{0:.17f}', t_t)) + st = s() + + ## use numpy view "np_t_open_v" for speed. Columns are + # buy 0 - open 1 - close 2 - sell 3 - high 4 - low 5 + # exception is 6 which is use the stop value. + + # TODO no! this is hard coded bleh fix this open + np_trade_enter_price = np_bslap[t_open_ind + 1, np_open] + if t_exit_type == SellType.STOP_LOSS: + if np_t_exit_pri == 6: + np_trade_exit_price = np_t_stop_pri + t_exit_ind = t_exit_ind + 1 + else: + np_trade_exit_price = np_bslap[t_exit_ind, np_t_exit_pri] + if t_exit_type == SellType.SELL_SIGNAL: + np_trade_exit_price = np_bslap[t_exit_ind, np_t_exit_pri] + + # Catch no exit found + if t_exit_type == SellType.NONE: + np_trade_exit_price = 0 + + if debug_timing: + t_t = f(st) + print("7-numpy", str.format('{0:.17f}', t_t)) + st = s() + + if debug: + print("//////////////////////////////////////////////") + print("+++++++++++++++++++++++++++++++++ Trade Enter ") + print("np_trade Enter Price is ", str.format('{0:.17f}', np_trade_enter_price)) + print("--------------------------------- Trade Exit ") + print("Trade Exit Type is ", t_exit_type) + print("np_trade Exit Price is", str.format('{0:.17f}', np_trade_exit_price)) + print("//////////////////////////////////////////////") + + else: # no buys were found, step 0 returned -1 + # Gracefully exit the loop + t_exit_last == -1 + if debug: + print("\n(E) No buys were found in remaining ticker file. Exiting", pair) + + # Loop control - catch no closed trades. + if debug: + print("---------------------------------------- end of loop", loop, + " Dataframe Exit Index is: ", t_exit_ind) + print("Exit Index Last, Exit Index Now Are: ", t_exit_last, t_exit_ind) + + if t_exit_last >= t_exit_ind or t_exit_last == -1: + """ + Break loop and go on to next pair. + + When last trade exit equals index of last exit, there is no + opportunity to close any more trades. + """ + # TODO :add handing here to record none closed open trades + + if debug: + print(bslap_pair_results) + break + else: + """ + Add trade to backtest looking results list of dicts + Loop back to look for more trades. + """ + + # We added +1 to t_exit_ind if the exit was a stop-loss, to not exit early in the IF of this ELSE + # removing the +1 here so prices match. + if t_exit_type == SellType.STOP_LOSS: + t_exit_ind = t_exit_ind - 1 + + # Build trade dictionary + ## In general if a field can be calculated later from other fields leave blank here + ## Its X(number of trades faster) to calc all in a single vector than 1 trade at a time + + # create a new dict + close_index: int = t_exit_ind + bslap_result = {} # Must have at start or we end up with a list of multiple same last result + bslap_result["pair"] = pair + bslap_result["stoploss"] = stop + bslap_result["profit_percent"] = "" # To be 1 vector calc across trades when loop complete + bslap_result["profit_abs"] = "" # To be 1 vector calc across trades when loop complete + bslap_result["open_time"] = np_bslap_dates[t_open_ind + 1] # use numpy array, pandas 20x slower + bslap_result["close_time"] = np_bslap_dates[close_index] # use numpy array, pandas 20x slower + bslap_result["open_index"] = t_open_ind + 1 # +1 as we buy on next. + bslap_result["close_index"] = close_index + bslap_result["trade_duration"] = "" # To be 1 vector calc across trades when loop complete + bslap_result["open_at_end"] = False + bslap_result["open_rate"] = round(np_trade_enter_price, 15) + bslap_result["close_rate"] = round(np_trade_exit_price, 15) + bslap_result["exit_type"] = t_exit_type + bslap_result["sell_reason"] = t_exit_type #duplicated, but I don't care + # append the dict to the list and print list + bslap_pair_results.append(bslap_result) + + if t_exit_type is SellType.STOP_LOSS: + stop_stops_count = stop_stops_count + 1 + + if debug: + print("The trade dict is: \n", bslap_result) + print("Trades dicts in list after append are: \n ", bslap_pair_results) + + """ + Loop back to start. t_exit_last becomes where loop + will seek to open new trades from. + Push index on 1 to not open on close + """ + t_exit_last = t_exit_ind + 1 + + if debug_timing: + t_t = f(st) + print("8+trade", str.format('{0:.17f}', t_t)) + + # Send back List of trade dicts + return bslap_pair_results + + def stake_amount(self, pair: str) -> str: + info = [x for x in self._cached_pairs if x[0] == pair][0] + stoploss = info[1] + + allowed_capital_at_risk = round(self._total_capital * self._allowed_risk, 5) + position_size = abs(round((allowed_capital_at_risk / stoploss), 5)) + + return (allowed_dollars_at_risk / symbol_strategy_stop_loss) + + ### stake amount is the same as position size + ### calculate position size + # print("\n~~~~~~~~ Position Size ~~~~~~~~") + # print("bid price is ", bid_price) + # print("stop trigger is ", stop_trigger_price) + + # allowed_dollars_at_risk = total_capital * allowed_risk + # print("allowed capital at risk ", round(allowed_dollars_at_risk, 5)) + + # position_size = (allowed_dollars_at_risk / symbol_strategy_stop_loss) + # print("position_size in dollars", round(position_size, 5 )) + + # buy_amount = position_size / bid_price + # print("amount of tokens to buy ", round(buy_amount,5)) + + # check_risk = (buy_amount * (bid_price - stop_trigger_price)) + # print("check risk capital ", round(check_risk, 5), "** Should not be more than allowed capital at risk") + + def stoploss(self, pair: str) -> float: + info = [x for x in self._cached_pairs if x[0] == pair][0] + return info[1] \ No newline at end of file diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index d34807090..a0abc8665 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -17,6 +17,7 @@ from cachetools import TTLCache, cached from freqtrade import (DependencyException, OperationalException, TemporaryError, __version__, constants, persistence) from freqtrade.exchange import Exchange +from freqtrade.edge import Edge from freqtrade.persistence import Trade from freqtrade.rpc import RPCManager, RPCMessageType from freqtrade.state import State @@ -24,6 +25,8 @@ from freqtrade.strategy.interface import SellType from freqtrade.strategy.resolver import IStrategy, StrategyResolver from freqtrade.exchange.exchange_helpers import order_book_to_dataframe +import pdb + logger = logging.getLogger(__name__) @@ -54,6 +57,7 @@ class FreqtradeBot(object): self.rpc: RPCManager = RPCManager(self) self.persistence = None self.exchange = Exchange(self.config) + self.edge = Edge(self.config, self.exchange) self._init_modules() def _init_modules(self) -> None: @@ -185,6 +189,10 @@ class FreqtradeBot(object): # Refreshing candles self.exchange.refresh_tickers(final_list, self.strategy.ticker_interval) + # Calculating Edge positiong + if self.config['edge']['enabled']: + self.edge.calculate() + # Query trades from persistence layer trades = Trade.query.filter(Trade.is_open.is_(True)).all() @@ -307,13 +315,17 @@ class FreqtradeBot(object): return used_rate - def _get_trade_stake_amount(self) -> Optional[float]: + def _get_trade_stake_amount(self, pair="") -> Optional[float]: """ Check if stake amount can be fulfilled with the available balance for the stake currency :return: float: Stake Amount """ - stake_amount = self.config['stake_amount'] + if self.config['edge']['enabled']: + stake_amount = self.edge.stake_amount(pair) + else: + stake_amount = self.config['stake_amount'] + avaliable_amount = self.exchange.get_balance(self.config['stake_currency']) if stake_amount == constants.UNLIMITED_STAKE_AMOUNT: @@ -372,17 +384,8 @@ class FreqtradeBot(object): """ interval = self.strategy.ticker_interval - # EDGE - # STAKE AMOUNT SHOULD COME FROM EDGE - stake_amount = self._get_trade_stake_amount() + logger.info('Checking buy signals to create a new trade: ...') - if not stake_amount: - return False - - logger.info( - 'Checking buy signals to create a new trade with stake_amount: %f ...', - stake_amount - ) whitelist = copy.deepcopy(self.config['exchange']['pair_whitelist']) # Remove currently opened and latest pairs from whitelist @@ -396,9 +399,13 @@ class FreqtradeBot(object): # running get_signal on historical data fetched # to find buy signals + if self.config['edge']['enabled']: + whitelist = self.edge.sort_pairs(whitelist) + for _pair in whitelist: (buy, sell) = self.strategy.get_signal(_pair, interval, self.exchange.klines.get(_pair)) if buy and not sell: + stake_amount = self._get_trade_stake_amount(_pair) bidstrat_check_depth_of_market = self.config.get('bid_strategy', {}).\ get('check_depth_of_market', {}) if (bidstrat_check_depth_of_market.get('enabled', False)) and\ @@ -436,11 +443,12 @@ class FreqtradeBot(object): """ pair_s = pair.replace('_', '/') pair_url = self.exchange.get_pair_detail_url(pair) - stake_currency = self.config['stake_currency'] - fiat_currency = self.config.get('fiat_display_currency', None) + fiat_currency = self.config.get('fiat_display_currency', None) + # Calculate amount buy_limit = self.get_target_bid(pair, self.exchange.get_ticker(pair)) + stake_currency = self.config['stake_currency'] min_stake_amount = self._get_min_pair_stake_amount(pair_s, buy_limit) if min_stake_amount is not None and min_stake_amount > stake_amount: @@ -622,7 +630,12 @@ class FreqtradeBot(object): return False def check_sell(self, trade: Trade, sell_rate: float, buy: bool, sell: bool) -> bool: - should_sell = self.strategy.should_sell(trade, sell_rate, datetime.utcnow(), buy, sell) + if (self.config['edge']['enabled']): + stoploss = self.edge.stoploss(trade.pair) + should_sell = self.strategy.should_sell(trade, sell_rate, datetime.utcnow(), buy, sell, stoploss) + else: + should_sell = self.strategy.should_sell(trade, sell_rate, datetime.utcnow(), buy, sell) + if should_sell.sell_flag: self.execute_sell(trade, sell_rate, should_sell.sell_type) logger.info('excuted sell') diff --git a/freqtrade/strategy/interface.py b/freqtrade/strategy/interface.py index 6afa4161b..13fe01a70 100644 --- a/freqtrade/strategy/interface.py +++ b/freqtrade/strategy/interface.py @@ -203,7 +203,7 @@ class IStrategy(ABC): return buy, sell def should_sell(self, trade: Trade, rate: float, date: datetime, buy: bool, - sell: bool) -> SellCheckTuple: + sell: bool, force_stoploss=0) -> SellCheckTuple: """ This function evaluate if on the condition required to trigger a sell has been reached if the threshold is reached and updates the trade record. @@ -211,7 +211,7 @@ class IStrategy(ABC): """ current_profit = trade.calc_profit_percent(rate) stoplossflag = self.stop_loss_reached(current_rate=rate, trade=trade, current_time=date, - current_profit=current_profit) + current_profit=current_profit, force_stoploss=force_stoploss) if stoplossflag.sell_flag: return stoplossflag @@ -237,7 +237,7 @@ class IStrategy(ABC): return SellCheckTuple(sell_flag=False, sell_type=SellType.NONE) def stop_loss_reached(self, current_rate: float, trade: Trade, current_time: datetime, - current_profit: float) -> SellCheckTuple: + current_profit: float, force_stoploss: float) -> SellCheckTuple: """ Based on current profit of the trade and configured (trailing) stoploss, decides to sell or not @@ -246,7 +246,10 @@ class IStrategy(ABC): trailing_stop = self.config.get('trailing_stop', False) - trade.adjust_stop_loss(trade.open_rate, self.stoploss, initial=True) + if force_stoploss == 0: + trade.adjust_stop_loss(trade.open_rate, self.stoploss, initial=True) + else: + trade.adjust_stop_loss(trade.open_rate, force_stoploss, initial=True) # evaluate if the stoploss was hit if self.stoploss is not None and trade.stop_loss >= current_rate: diff --git a/freqtrade/tests/conftest.py b/freqtrade/tests/conftest.py index af9062cab..14e8a151d 100644 --- a/freqtrade/tests/conftest.py +++ b/freqtrade/tests/conftest.py @@ -127,6 +127,15 @@ def default_conf(): "NEO/BTC" ] }, + "edge": { + "enabled": False, + "process_throttle_secs": 1800, + "total_capital_in_stake_currency": 0.5, + "allowed_risk": 0.01, + "maximum_winrate_to_consider": 0.80, + "remove_pumps": True, + "minimum_delta_to_consider": 1 + }, "telegram": { "enabled": True, "token": "token", From 4746aea05c79724dfe255bd9574226b6f9454f66 Mon Sep 17 00:00:00 2001 From: misagh Date: Fri, 21 Sep 2018 17:42:04 +0200 Subject: [PATCH 042/699] test file for edge (will be removed) --- freqtrade/optimize/edge.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/freqtrade/optimize/edge.py b/freqtrade/optimize/edge.py index 86ffb6747..efafb8d77 100644 --- a/freqtrade/optimize/edge.py +++ b/freqtrade/optimize/edge.py @@ -912,6 +912,7 @@ class Edge: reset_index().sort_values(by=['delta', 'stoploss'], ascending=False)\ .groupby('pair').first().sort_values(by=['delta'], ascending=False) + pdb.set_trace() return final @@ -1022,3 +1023,15 @@ config = setup_configuration(args) edge = Edge(config) edge.start() + +allowed_dollars_at_risk = total_capital * allowed_risk +print("allowed capital at risk ", round(allowed_dollars_at_risk, 5)) + +position_size = (allowed_dollars_at_risk / symbol_strategy_stop_loss) +print("position_size in dollars", round(position_size, 5 )) + +buy_amount = position_size / bid_price +print("amount of tokens to buy ", round(buy_amount,5)) + +check_risk = (buy_amount * (bid_price - stop_trigger_price)) +print("check risk capital ", round(check_risk, 5), "** Should not be more than allowed capital at risk") \ No newline at end of file From 2d432bfa953dd2d35f2d95edb7580be355266f38 Mon Sep 17 00:00:00 2001 From: misagh Date: Fri, 21 Sep 2018 17:54:37 +0200 Subject: [PATCH 043/699] backtesting rollbacked to develop branch --- freqtrade/optimize/backtesting.py | 442 ++++++++++-------------------- 1 file changed, 141 insertions(+), 301 deletions(-) diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index c539d0154..d0b70afc7 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -6,11 +6,13 @@ This module contains the backtesting logic import logging import operator from argparse import Namespace +from copy import deepcopy from datetime import datetime, timedelta +from pathlib import Path from typing import Any, Dict, List, NamedTuple, Optional, Tuple import arrow -from pandas import DataFrame, to_datetime +from pandas import DataFrame from tabulate import tabulate import freqtrade.optimize as optimize @@ -19,15 +21,9 @@ from freqtrade.arguments import Arguments from freqtrade.configuration import Configuration from freqtrade.exchange import Exchange from freqtrade.misc import file_dump_json -from freqtrade.optimize.backslapping import Backslapping from freqtrade.persistence import Trade from freqtrade.strategy.interface import SellType from freqtrade.strategy.resolver import IStrategy, StrategyResolver -from collections import OrderedDict -import timeit -from time import sleep - -import pdb logger = logging.getLogger(__name__) @@ -61,11 +57,6 @@ class Backtesting(object): def __init__(self, config: Dict[str, Any]) -> None: self.config = config - self.strategy: IStrategy = StrategyResolver(self.config).strategy - self.ticker_interval = self.strategy.ticker_interval - self.tickerdata_to_dataframe = self.strategy.tickerdata_to_dataframe - self.advise_buy = self.strategy.advise_buy - self.advise_sell = self.strategy.advise_sell # Reset keys for backtesting self.config['exchange']['key'] = '' @@ -73,51 +64,35 @@ class Backtesting(object): self.config['exchange']['password'] = '' self.config['exchange']['uid'] = '' self.config['dry_run'] = True + self.strategylist: List[IStrategy] = [] + if self.config.get('strategy_list', None): + # Force one interval + self.ticker_interval = str(self.config.get('ticker_interval')) + for strat in list(self.config['strategy_list']): + stratconf = deepcopy(self.config) + stratconf['strategy'] = strat + self.strategylist.append(StrategyResolver(stratconf).strategy) + + else: + # only one strategy + strat = StrategyResolver(self.config).strategy + + self.strategylist.append(StrategyResolver(self.config).strategy) + # Load one strategy + self._set_strategy(self.strategylist[0]) + self.exchange = Exchange(self.config) self.fee = self.exchange.get_fee() - self.stop_loss_value = self.strategy.stoploss - - #### backslap config - ''' - Numpy arrays are used for 100x speed up - We requires setting Int values for - buy stop triggers and stop calculated on - # buy 0 - open 1 - close 2 - sell 3 - high 4 - low 5 - stop 6 - ''' - self.np_buy: int = 0 - self.np_open: int = 1 - self.np_close: int = 2 - self.np_sell: int = 3 - self.np_high: int = 4 - self.np_low: int = 5 - self.np_stop: int = 6 - self.np_bto: int = self.np_close # buys_triggered_on - should be close - self.np_bco: int = self.np_open # buys calculated on - open of the next candle. - self.np_sto: int = self.np_low # stops_triggered_on - Should be low, FT uses close - self.np_sco: int = self.np_stop # stops_calculated_on - Should be stop, FT uses close - # self.np_sto: int = self.np_close # stops_triggered_on - Should be low, FT uses close - # self.np_sco: int = self.np_close # stops_calculated_on - Should be stop, FT uses close - - if 'backslap' in config: - self.use_backslap = config['backslap'] # Enable backslap - if false Orginal code is executed. - else: - self.use_backslap = False - - logger.info("using backslap: {}".format(self.use_backslap)) - - self.debug = False # Main debug enable, very print heavy, enable 2 loops recommended - self.debug_timing = False # Stages within Backslap - self.debug_2loops = False # Limit each pair to two loops, useful when debugging - self.debug_vector = False # Debug vector calcs - self.debug_timing_main_loop = False # print overall timing per pair - works in Backtest and Backslap - - self.backslap_show_trades = False # prints trades in addition to summary report - self.backslap_save_trades = True # saves trades as a pretty table to backslap.txt - - self.stop_stops: int = 9999 # stop back testing any pair with this many stops, set to 999999 to not hit - - self.backslap = Backslapping(config) + def _set_strategy(self, strategy): + """ + Load strategy into backtesting + """ + self.strategy = strategy + self.ticker_interval = self.config.get('ticker_interval') + self.tickerdata_to_dataframe = strategy.tickerdata_to_dataframe + self.advise_buy = strategy.advise_buy + self.advise_sell = strategy.advise_sell @staticmethod def get_timeframe(data: Dict[str, DataFrame]) -> Tuple[arrow.Arrow, arrow.Arrow]: @@ -131,7 +106,7 @@ class Backtesting(object): for frame in data.values() ] return min(timeframe, key=operator.itemgetter(0))[0], \ - max(timeframe, key=operator.itemgetter(1))[1] + max(timeframe, key=operator.itemgetter(1))[1] def _generate_text_table(self, data: Dict[str, Dict], results: DataFrame) -> str: """ @@ -142,13 +117,10 @@ class Backtesting(object): floatfmt = ('s', 'd', '.2f', '.2f', '.8f', 'd', '.1f', '.1f') tabular_data = [] - # headers = ['pair', 'buy count', 'avg profit %', 'cum profit %', - # 'total profit ' + stake_currency, 'avg duration', 'profit', 'loss', 'total loss ab', 'total profit ab', 'Risk Reward Ratio', 'Win Rate'] headers = ['pair', 'buy count', 'avg profit %', 'cum profit %', - 'total profit ' + stake_currency, 'avg duration', 'profit', 'loss', 'RRR', 'Win Rate %', 'Required RR'] + 'total profit ' + stake_currency, 'avg duration', 'profit', 'loss'] for pair in data: result = results[results.pair == pair] - win_rate = (len(result[result.profit_abs > 0]) / len(result.index)) if (len(result.index) > 0) else None tabular_data.append([ pair, len(result.index), @@ -158,12 +130,7 @@ class Backtesting(object): str(timedelta( minutes=round(result.trade_duration.mean()))) if not result.empty else '0:00', len(result[result.profit_abs > 0]), - len(result[result.profit_abs < 0]), - # result[result.profit_abs < 0]['profit_abs'].sum(), - # result[result.profit_abs > 0]['profit_abs'].sum(), - abs(1 / ((result[result.profit_abs < 0]['profit_abs'].sum() / len(result[result.profit_abs < 0])) / (result[result.profit_abs > 0]['profit_abs'].sum() / len(result[result.profit_abs > 0])))), - win_rate * 100 if win_rate else "nan", - ((1 / win_rate) - 1) if win_rate else "nan" + len(result[result.profit_abs < 0]) ]) # Append Total @@ -180,88 +147,42 @@ class Backtesting(object): ]) return tabulate(tabular_data, headers=headers, floatfmt=floatfmt, tablefmt="pipe") - - def _generate_text_table_edge_positioning(self, data: Dict[str, Dict], results: DataFrame) -> str: - """ - This is a temporary version of edge positioning calculation. - The function will be eventually moved to a plugin called Edge in order to calculate necessary WR, RRR and - other indictaors related to money management periodically (each X minutes) and keep it in a storage. - The calulation will be done per pair and per strategy. - """ - - tabular_data = [] - headers = ['Number of trades', 'RRR', 'Win Rate %', 'Required RR'] - - ### - # The algorithm should be: - # 1) Removing outliers from dataframe. i.e. all profit_percent which are outside (mean -+ (2 * (standard deviation))). - # 2) Removing pairs with less than X trades (X defined in config). - # 3) Calculating RRR and WR. - # 4) Removing pairs for which WR and RRR are not in an acceptable range (e.x. WR > 95%). - # 5) Sorting the result based on the delta between required RR and RRR. - - # Here we assume initial data in order to calculate position size. - # these values will be replaced by exchange info or config - for pair in data: - result = results[results.pair == pair] - - # WinRate is calculated as follows: (Number of profitable trades) / (Total Trades) - win_rate = (len(result[result.profit_abs > 0]) / len(result.index)) if (len(result.index) > 0) else None - - # Risk Reward Ratio is calculated as follows: 1 / ((total loss on losing trades / number of losing trades) / (total gain on profitable trades / number of winning trades)) - risk_reward_ratio = abs(1 / ((result[result.profit_abs < 0]['profit_abs'].sum() / len(result[result.profit_abs < 0])) / (result[result.profit_abs > 0]['profit_abs'].sum() / len(result[result.profit_abs > 0])))) - - # Required Reward Ratio is (1 / WinRate) - 1 - required_risk_reward = ((1 / win_rate) - 1) if win_rate else None - - #pdb.set_trace() - - tabular_data.append([ - pair, - len(result.index), - risk_reward_ratio, - win_rate * 100 if win_rate else "nan", - required_risk_reward - ]) - - # for pair in data: - # result = results[results.pair == pair] - # win_rate = (len(result[result.profit_abs > 0]) / len(result.index)) if (len(result.index) > 0) else None - # tabular_data.append([ - # pair, - # #len(result.index), - # #result.profit_percent.mean() * 100.0, - # #result.profit_percent.sum() * 100.0, - # #result.profit_abs.sum(), - # str(timedelta( - # minutes=round(result.trade_duration.mean()))) if not result.empty else '0:00', - # len(result[result.profit_abs > 0]), - # len(result[result.profit_abs < 0]), - # # result[result.profit_abs < 0]['profit_abs'].sum(), - # # result[result.profit_abs > 0]['profit_abs'].sum(), - # abs(1 / ((result[result.profit_abs < 0]['profit_abs'].sum() / len(result[result.profit_abs < 0])) / (result[result.profit_abs > 0]['profit_abs'].sum() / len(result[result.profit_abs > 0])))), - # win_rate * 100 if win_rate else "nan", - # ((1 / win_rate) - 1) if win_rate else "nan" - # ]) - - #return tabulate(tabular_data, headers=headers, floatfmt=floatfmt, tablefmt="pipe") - return tabulate(tabular_data, headers=headers, tablefmt="pipe") - - - - def _generate_text_table_sell_reason(self, data: Dict[str, Dict], results: DataFrame) -> str: """ Generate small table outlining Backtest results """ - tabular_data = [] headers = ['Sell Reason', 'Count'] for reason, count in results['sell_reason'].value_counts().iteritems(): - tabular_data.append([reason.value, count]) + tabular_data.append([reason.value, count]) return tabulate(tabular_data, headers=headers, tablefmt="pipe") - def _store_backtest_result(self, recordfilename: Optional[str], results: DataFrame) -> None: + def _generate_text_table_strategy(self, all_results: dict) -> str: + """ + Generate summary table per strategy + """ + stake_currency = str(self.config.get('stake_currency')) + + floatfmt = ('s', 'd', '.2f', '.2f', '.8f', 'd', '.1f', '.1f') + tabular_data = [] + headers = ['Strategy', 'buy count', 'avg profit %', 'cum profit %', + 'total profit ' + stake_currency, 'avg duration', 'profit', 'loss'] + for strategy, results in all_results.items(): + tabular_data.append([ + strategy, + len(results.index), + results.profit_percent.mean() * 100.0, + results.profit_percent.sum() * 100.0, + results.profit_abs.sum(), + str(timedelta( + minutes=round(results.trade_duration.mean()))) if not results.empty else '0:00', + len(results[results.profit_abs > 0]), + len(results[results.profit_abs < 0]) + ]) + return tabulate(tabular_data, headers=headers, floatfmt=floatfmt, tablefmt="pipe") + + def _store_backtest_result(self, recordfilename: str, results: DataFrame, + strategyname: Optional[str] = None) -> None: records = [(t.pair, t.profit_percent, t.open_time.timestamp(), t.close_time.timestamp(), t.open_index - 1, t.trade_duration, @@ -269,6 +190,11 @@ class Backtesting(object): for index, t in results.iterrows()] if records: + if strategyname: + # Inject strategyname to filename + recname = Path(recordfilename) + recordfilename = str(Path.joinpath( + recname.parent, f'{recname.stem}-{strategyname}').with_suffix(recname.suffix)) logger.info('Dumping backtest results to %s', recordfilename) file_dump_json(recordfilename, records) @@ -297,13 +223,14 @@ class Backtesting(object): sell = self.strategy.should_sell(trade, sell_row.open, sell_row.date, buy_signal, sell_row.sell) if sell.sell_flag: + return BacktestResult(pair=pair, profit_percent=trade.calc_profit_percent(rate=sell_row.open), profit_abs=trade.calc_profit(rate=sell_row.open), open_time=buy_row.date, close_time=sell_row.date, trade_duration=int(( - sell_row.date - buy_row.date).total_seconds() // 60), + sell_row.date - buy_row.date).total_seconds() // 60), open_index=buy_row.Index, close_index=sell_row.Index, open_at_end=False, @@ -320,7 +247,7 @@ class Backtesting(object): open_time=buy_row.date, close_time=sell_row.date, trade_duration=int(( - sell_row.date - buy_row.date).total_seconds() // 60), + sell_row.date - buy_row.date).total_seconds() // 60), open_index=buy_row.Index, close_index=sell_row.Index, open_at_end=True, @@ -333,13 +260,6 @@ class Backtesting(object): return btr return None - def s(self): - st = timeit.default_timer() - return st - - def f(self, st): - return (timeit.default_timer() - st) - def backtest(self, args: Dict) -> DataFrame: """ Implements backtesting functionality @@ -355,50 +275,32 @@ class Backtesting(object): position_stacking: do we allow position stacking? (default: False) :return: DataFrame """ + headers = ['date', 'buy', 'open', 'close', 'sell'] + processed = args['processed'] + max_open_trades = args.get('max_open_trades', 0) + position_stacking = args.get('position_stacking', False) + trades = [] + trade_count_lock: Dict = {} + for pair, pair_data in processed.items(): + pair_data['buy'], pair_data['sell'] = 0, 0 # cleanup from previous run - use_backslap = self.use_backslap - debug_timing = self.debug_timing_main_loop - - if use_backslap: # Use Back Slap code - return self.backslap.run(args) - else: # use Original Back test code - ########################## Original BT loop - - headers = ['date', 'buy', 'open', 'close', 'sell'] - processed = args['processed'] - max_open_trades = args.get('max_open_trades', 0) - position_stacking = args.get('position_stacking', False) - trades = [] - trade_count_lock: Dict = {} - - for pair, pair_data in processed.items(): - if debug_timing: # Start timer - fl = self.s() - - pair_data['buy'], pair_data['sell'] = 0, 0 # cleanup from previous run - - ticker_data = self.advise_sell( + ticker_data = self.advise_sell( self.advise_buy(pair_data, {'pair': pair}), {'pair': pair})[headers].copy() - # to avoid using data from future, we buy/sell with signal from previous candle - ticker_data.loc[:, 'buy'] = ticker_data['buy'].shift(1) - ticker_data.loc[:, 'sell'] = ticker_data['sell'].shift(1) + # to avoid using data from future, we buy/sell with signal from previous candle + ticker_data.loc[:, 'buy'] = ticker_data['buy'].shift(1) + ticker_data.loc[:, 'sell'] = ticker_data['sell'].shift(1) - ticker_data.drop(ticker_data.head(1).index, inplace=True) + ticker_data.drop(ticker_data.head(1).index, inplace=True) - if debug_timing: # print time taken - flt = self.f(fl) - # print("populate_buy_trend:", pair, round(flt, 10)) - st = self.s() + # Convert from Pandas to list for performance reasons + # (Looping Pandas is slow.) + ticker = [x for x in ticker_data.itertuples()] - # Convert from Pandas to list for performance reasons - # (Looping Pandas is slow.) - ticker = [x for x in ticker_data.itertuples()] - - lock_pair_until = None - for index, row in enumerate(ticker): - if row.buy == 0 or row.sell == 1: - continue # skip rows where no buy signal or that would immediately sell off + lock_pair_until = None + for index, row in enumerate(ticker): + if row.buy == 0 or row.sell == 1: + continue # skip rows where no buy signal or that would immediately sell off if not position_stacking: if lock_pair_until is not None and row.date <= lock_pair_until: @@ -408,26 +310,20 @@ class Backtesting(object): if not trade_count_lock.get(row.date, 0) < max_open_trades: continue - trade_count_lock[row.date] = trade_count_lock.get(row.date, 0) + 1 + trade_count_lock[row.date] = trade_count_lock.get(row.date, 0) + 1 - trade_entry = self._get_sell_trade_entry(pair, row, ticker[index + 1:], - trade_count_lock, args) + trade_entry = self._get_sell_trade_entry(pair, row, ticker[index + 1:], + trade_count_lock, args) - if trade_entry: - lock_pair_until = trade_entry.close_time - trades.append(trade_entry) - else: - # Set lock_pair_until to end of testing period if trade could not be closed - # This happens only if the buy-signal was with the last candle - lock_pair_until = ticker_data.iloc[-1].date + if trade_entry: + lock_pair_until = trade_entry.close_time + trades.append(trade_entry) + else: + # Set lock_pair_until to end of testing period if trade could not be closed + # This happens only if the buy-signal was with the last candle + lock_pair_until = ticker_data.iloc[-1].date - if debug_timing: # print time taken - tt = self.f(st) - print("Time to BackTest :", pair, round(tt, 10)) - print("-----------------------") - - return DataFrame.from_records(trades, columns=BacktestResult._fields) - ####################### Original BT loop end + return DataFrame.from_records(trades, columns=BacktestResult._fields) def start(self) -> None: """ @@ -448,7 +344,6 @@ class Backtesting(object): timerange = Arguments.parse_timerange(None if self.config.get( 'timerange') is None else str(self.config.get('timerange'))) - data = optimize.load_data( self.config['datadir'], pairs=pairs, @@ -458,7 +353,6 @@ class Backtesting(object): timerange=timerange ) - ld_files = self.s() if not data: logger.critical("No data found. Terminating.") return @@ -468,109 +362,55 @@ class Backtesting(object): else: logger.info('Ignoring max_open_trades (--disable-max-market-positions was used) ...') max_open_trades = 0 + all_results = {} - preprocessed = self.tickerdata_to_dataframe(data) - t_t = self.f(ld_files) - print("Load from json to file to df in mem took", t_t) + for strat in self.strategylist: + logger.info("Running backtesting for Strategy %s", strat.get_strategy_name()) + self._set_strategy(strat) - # Print timeframe - min_date, max_date = self.get_timeframe(preprocessed) - logger.info( - 'Measuring data from %s up to %s (%s days)..', - min_date.isoformat(), - max_date.isoformat(), - (max_date - min_date).days - ) - - # Execute backtest and print results - results = self.backtest( - { - 'stake_amount': self.config.get('stake_amount'), - 'processed': preprocessed, - 'max_open_trades': max_open_trades, - 'position_stacking': self.config.get('position_stacking', False), - } - ) - - if self.config.get('export', False): - self._store_backtest_result(self.config.get('exportfilename'), results) - - if self.use_backslap: - # logger.info( - # '\n====================================================== ' - # 'BackSLAP REPORT' - # ' =======================================================\n' - # '%s', - # self._generate_text_table( - # data, - # results - # ) - # ) + # need to reprocess data every time to populate signals + preprocessed = self.tickerdata_to_dataframe(data) + # Print timeframe + min_date, max_date = self.get_timeframe(preprocessed) logger.info( - '\n====================================================== ' - 'Edge positionning REPORT' - ' =======================================================\n' - '%s', - self._generate_text_table_edge_positioning( - data, - results - ) - ) - # optional print trades - if self.backslap_show_trades: - TradesFrame = results.filter(['open_time', 'pair', 'exit_type', 'profit_percent', 'profit_abs', - 'buy_spend', 'sell_take', 'trade_duration', 'close_time'], axis=1) - - def to_fwf(df, fname): - content = tabulate(df.values.tolist(), list(df.columns), floatfmt=".8f", tablefmt='psql') - print(content) - - DataFrame.to_fwf = to_fwf(TradesFrame, "backslap.txt") - - # optional save trades - if self.backslap_save_trades: - TradesFrame = results.filter(['open_time', 'pair', 'exit_type', 'profit_percent', 'profit_abs', - 'buy_spend', 'sell_take', 'trade_duration', 'close_time'], axis=1) - - def to_fwf(df, fname): - content = tabulate(df.values.tolist(), list(df.columns), floatfmt=".8f", tablefmt='psql') - open(fname, "w").write(content) - - DataFrame.to_fwf = to_fwf(TradesFrame, "backslap.txt") - - else: - logger.info( - '\n================================================= ' - 'BACKTEST REPORT' - ' ==================================================\n' - '%s', - self._generate_text_table( - data, - results - ) + 'Measuring data from %s up to %s (%s days)..', + min_date.isoformat(), + max_date.isoformat(), + (max_date - min_date).days ) - if 'sell_reason' in results.columns: - logger.info( - '\n' + - ' SELL READON STATS '.center(119, '=') + - '\n%s \n', - self._generate_text_table_sell_reason(data, results) - + # Execute backtest and print results + all_results[self.strategy.get_strategy_name()] = self.backtest( + { + 'stake_amount': self.config.get('stake_amount'), + 'processed': preprocessed, + 'max_open_trades': max_open_trades, + 'position_stacking': self.config.get('position_stacking', False), + } ) - else: - logger.info("no sell reasons available!") - logger.info( - '\n' + - ' LEFT OPEN TRADES REPORT '.center(119, '=') + - '\n%s', - self._generate_text_table( - data, - results.loc[results.open_at_end] - ) - ) + for strategy, results in all_results.items(): + + if self.config.get('export', False): + self._store_backtest_result(self.config['exportfilename'], results, + strategy if len(self.strategylist) > 1 else None) + + print(f"Result for strategy {strategy}") + print(' BACKTESTING REPORT '.center(119, '=')) + print(self._generate_text_table(data, results)) + + print(' SELL REASON STATS '.center(119, '=')) + print(self._generate_text_table_sell_reason(data, results)) + + print(' LEFT OPEN TRADES REPORT '.center(119, '=')) + print(self._generate_text_table(data, results.loc[results.open_at_end])) + print() + if len(all_results) > 1: + # Print Strategy summary table + print(' Strategy Summary '.center(119, '=')) + print(self._generate_text_table_strategy(all_results)) + print('\nFor more details, please look at the detail tables above') def setup_configuration(args: Namespace) -> Dict[str, Any]: @@ -585,7 +425,7 @@ def setup_configuration(args: Namespace) -> Dict[str, Any]: # Ensure we do not use Exchange credentials config['exchange']['key'] = '' config['exchange']['secret'] = '' - config['backslap'] = args.backslap + if config['stake_amount'] == constants.UNLIMITED_STAKE_AMOUNT: raise DependencyException('stake amount could not be "%s" for backtesting' % constants.UNLIMITED_STAKE_AMOUNT) @@ -605,4 +445,4 @@ def start(args: Namespace) -> None: # Initialize backtesting object backtesting = Backtesting(config) - backtesting.start() \ No newline at end of file + backtesting.start() From 74979943baaa3d870c64e35eb59194533bdcd32b Mon Sep 17 00:00:00 2001 From: misagh Date: Fri, 21 Sep 2018 17:57:29 +0200 Subject: [PATCH 044/699] backslap removed from arguments --- freqtrade/arguments.py | 9 --------- 1 file changed, 9 deletions(-) diff --git a/freqtrade/arguments.py b/freqtrade/arguments.py index 4f47849d8..501c1784f 100644 --- a/freqtrade/arguments.py +++ b/freqtrade/arguments.py @@ -171,14 +171,6 @@ class Arguments(object): dest='exportfilename', metavar='PATH', ) - parser.add_argument( - '--backslap', - help="Utilize the Backslapping approach instead of the default Backtesting. This should provide more " - "accurate results, unless you are utilizing Min/Max function in your strategy.", - required=False, - dest='backslap', - action='store_true' - ) @staticmethod def optimizer_shared_options(parser: argparse.ArgumentParser) -> None: @@ -262,7 +254,6 @@ class Arguments(object): self.optimizer_shared_options(hyperopt_cmd) self.hyperopt_options(hyperopt_cmd) - @staticmethod def parse_timerange(text: Optional[str]) -> TimeRange: """ From 4bd956d5b1da5f4ab7ed1d3a3e5051c10875a073 Mon Sep 17 00:00:00 2001 From: misagh Date: Fri, 21 Sep 2018 17:58:20 +0200 Subject: [PATCH 045/699] test file removed --- freqtrade/edge.py | 6 ------ 1 file changed, 6 deletions(-) delete mode 100644 freqtrade/edge.py diff --git a/freqtrade/edge.py b/freqtrade/edge.py deleted file mode 100644 index 4795e8f4d..000000000 --- a/freqtrade/edge.py +++ /dev/null @@ -1,6 +0,0 @@ -# EDGE -# WinRate and Expected Risk Reward should be calculated for all whitelisted pairs -# ASYNC: For each pair call backslap -# Save WR and ERR in Edge Dict -# Save last time updated for each pair in edge_last_update_time -# Calulate expectancy and position size and stop loss From 4fd037f83fb92ecb05f501eb2622b0394fbd05a1 Mon Sep 17 00:00:00 2001 From: misagh Date: Fri, 21 Sep 2018 18:00:37 +0200 Subject: [PATCH 046/699] removing pdb --- freqtrade/freqtradebot.py | 1 - 1 file changed, 1 deletion(-) diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index a0abc8665..febdcea40 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -25,7 +25,6 @@ from freqtrade.strategy.interface import SellType from freqtrade.strategy.resolver import IStrategy, StrategyResolver from freqtrade.exchange.exchange_helpers import order_book_to_dataframe -import pdb logger = logging.getLogger(__name__) From 61095db071e81d4e7a8845f95b499acc1e76d8fa Mon Sep 17 00:00:00 2001 From: misagh Date: Fri, 21 Sep 2018 21:36:26 +0200 Subject: [PATCH 047/699] edge config enriched --- config.json.example | 7 +++++-- freqtrade/tests/conftest.py | 7 +++---- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/config.json.example b/config.json.example index 801a5137c..175a537f1 100644 --- a/config.json.example +++ b/config.json.example @@ -53,9 +53,12 @@ "edge": { "enabled": false, "process_throttle_secs": 1800, - "maximum_winrate_to_consider": 0.80, + "total_capital_in_stake_currency": 0.5, + "allowed_risk": 0.01, + "maximum_winrate": 0.80, + "min_trade_number": 15, "remove_pumps": true, - "minimum_delta_to_consider": 1 + "minimum_delta": 1 }, "telegram": { "enabled": true, diff --git a/freqtrade/tests/conftest.py b/freqtrade/tests/conftest.py index 14e8a151d..09c746df2 100644 --- a/freqtrade/tests/conftest.py +++ b/freqtrade/tests/conftest.py @@ -130,11 +130,10 @@ def default_conf(): "edge": { "enabled": False, "process_throttle_secs": 1800, - "total_capital_in_stake_currency": 0.5, - "allowed_risk": 0.01, - "maximum_winrate_to_consider": 0.80, + "maximum_winrate": 0.80, + "min_trade_number": 15, "remove_pumps": True, - "minimum_delta_to_consider": 1 + "minimum_delta": 1 }, "telegram": { "enabled": True, From 3e3ed947cc67d0093de093029db79275e2250e24 Mon Sep 17 00:00:00 2001 From: misagh Date: Fri, 21 Sep 2018 21:46:18 +0200 Subject: [PATCH 048/699] =?UTF-8?q?added=20=E2=80=9Cmax=5Ftrade=5Fduration?= =?UTF-8?q?=E2=80=9D=20config=20+=20using=20=E2=80=9Cremove=5Fdumps?= =?UTF-8?q?=E2=80=9D=20config?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- config.json.example | 3 ++- freqtrade/edge/__init__.py | 30 ++++++++---------------------- freqtrade/tests/conftest.py | 3 +++ 3 files changed, 13 insertions(+), 23 deletions(-) diff --git a/config.json.example b/config.json.example index 175a537f1..f90b4f470 100644 --- a/config.json.example +++ b/config.json.example @@ -55,8 +55,9 @@ "process_throttle_secs": 1800, "total_capital_in_stake_currency": 0.5, "allowed_risk": 0.01, - "maximum_winrate": 0.80, + "maximum_winrate": 0.80, "min_trade_number": 15, + "max_trade_duration_minute": 1440, "remove_pumps": true, "minimum_delta": 1 }, diff --git a/freqtrade/edge/__init__.py b/freqtrade/edge/__init__.py index 9668e1a44..46cdaaf32 100644 --- a/freqtrade/edge/__init__.py +++ b/freqtrade/edge/__init__.py @@ -43,11 +43,13 @@ class Edge(): self.get_timeframe = Backtesting.get_timeframe self.populate_buy_trend = self.strategy.populate_buy_trend self.populate_sell_trend = self.strategy.populate_sell_trend + + self.edge_config = self.config.get('edge', {}) self._last_updated = None self._cached_pairs = [] - self._total_capital = self.config['edge']['total_capital_in_stake_currency'] - self._allowed_risk = self.config['edge']['allowed_risk'] + self._total_capital = self.edge_config['total_capital_in_stake_currency'] + self._allowed_risk = self.edge_config['allowed_risk'] ### # @@ -303,7 +305,7 @@ class Edge(): ################################### # Removing pairs having less than min_trades_number - min_trades_number = 50 + min_trades_number = self.edge_config.get('min_trade_number', 15) results = results.groupby('pair').filter(lambda x: len(x) > min_trades_number) ################################### @@ -319,11 +321,12 @@ class Edge(): avg = results[["profit_abs"]].mean() # # Removing Pumps - results = results[results.profit_abs < float(avg + 2*std)] + if self.edge_config.get('remove_pumps', True): + results = results[results.profit_abs < float(avg + 2*std)] ########################################################################## # Removing trades having a duration more than X minutes (set in config) - max_trade_duration = 24*60 + max_trade_duration = self.edge_config.get('max_trade_duration_minute', 1440) results = results[results.trade_duration < max_trade_duration] ####################################################################### @@ -906,23 +909,6 @@ class Edge(): return (allowed_dollars_at_risk / symbol_strategy_stop_loss) - ### stake amount is the same as position size - ### calculate position size - # print("\n~~~~~~~~ Position Size ~~~~~~~~") - # print("bid price is ", bid_price) - # print("stop trigger is ", stop_trigger_price) - - # allowed_dollars_at_risk = total_capital * allowed_risk - # print("allowed capital at risk ", round(allowed_dollars_at_risk, 5)) - - # position_size = (allowed_dollars_at_risk / symbol_strategy_stop_loss) - # print("position_size in dollars", round(position_size, 5 )) - - # buy_amount = position_size / bid_price - # print("amount of tokens to buy ", round(buy_amount,5)) - - # check_risk = (buy_amount * (bid_price - stop_trigger_price)) - # print("check risk capital ", round(check_risk, 5), "** Should not be more than allowed capital at risk") def stoploss(self, pair: str) -> float: info = [x for x in self._cached_pairs if x[0] == pair][0] diff --git a/freqtrade/tests/conftest.py b/freqtrade/tests/conftest.py index 09c746df2..8a3f71f51 100644 --- a/freqtrade/tests/conftest.py +++ b/freqtrade/tests/conftest.py @@ -130,8 +130,11 @@ def default_conf(): "edge": { "enabled": False, "process_throttle_secs": 1800, + "total_capital_in_stake_currency": 0.5, + "allowed_risk": 0.01, "maximum_winrate": 0.80, "min_trade_number": 15, + "max_trade_duration_minute": 1440, "remove_pumps": True, "minimum_delta": 1 }, From 3b925e46be86e38c0208b73f0f8f1555cd6c720e Mon Sep 17 00:00:00 2001 From: misagh Date: Fri, 21 Sep 2018 21:48:27 +0200 Subject: [PATCH 049/699] removing default pair value of _get_trade_stake_amount --- freqtrade/freqtradebot.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index febdcea40..7ac4f5522 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -314,7 +314,7 @@ class FreqtradeBot(object): return used_rate - def _get_trade_stake_amount(self, pair="") -> Optional[float]: + def _get_trade_stake_amount(self, pair) -> Optional[float]: """ Check if stake amount can be fulfilled with the available balance for the stake currency From fbc77c1f28d677d089812e22b03f8add571240de Mon Sep 17 00:00:00 2001 From: misagh Date: Fri, 21 Sep 2018 21:55:36 +0200 Subject: [PATCH 050/699] moving stake_currency line back to its initial place --- freqtrade/freqtradebot.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index 7ac4f5522..182a42819 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -442,13 +442,12 @@ class FreqtradeBot(object): """ pair_s = pair.replace('_', '/') pair_url = self.exchange.get_pair_detail_url(pair) - + stake_currency = self.config['stake_currency'] fiat_currency = self.config.get('fiat_display_currency', None) # Calculate amount buy_limit = self.get_target_bid(pair, self.exchange.get_ticker(pair)) - stake_currency = self.config['stake_currency'] - + min_stake_amount = self._get_min_pair_stake_amount(pair_s, buy_limit) if min_stake_amount is not None and min_stake_amount > stake_amount: logger.warning( From e1ca80734d1f3609e0e2cd9117ca45b9afa2dec7 Mon Sep 17 00:00:00 2001 From: misagh Date: Fri, 21 Sep 2018 21:58:37 +0200 Subject: [PATCH 051/699] removing unnecessary ujson import --- freqtrade/optimize/__init__.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/freqtrade/optimize/__init__.py b/freqtrade/optimize/__init__.py index 646f893e6..61710944b 100644 --- a/freqtrade/optimize/__init__.py +++ b/freqtrade/optimize/__init__.py @@ -17,9 +17,6 @@ from freqtrade import misc, constants, OperationalException from freqtrade.exchange import Exchange from freqtrade.arguments import TimeRange import importlib -ujson_found = importlib.util.find_spec("ujson") -if ujson_found is not None: - import ujson logger = logging.getLogger(__name__) From 66b1eac1db03c2c07ef3cb7206a6e5253ee60447 Mon Sep 17 00:00:00 2001 From: misagh Date: Fri, 21 Sep 2018 21:59:35 +0200 Subject: [PATCH 052/699] removing unnecessary ujson import --- freqtrade/optimize/__init__.py | 7 ------- 1 file changed, 7 deletions(-) diff --git a/freqtrade/optimize/__init__.py b/freqtrade/optimize/__init__.py index 646f893e6..aeb56811e 100644 --- a/freqtrade/optimize/__init__.py +++ b/freqtrade/optimize/__init__.py @@ -17,16 +17,9 @@ from freqtrade import misc, constants, OperationalException from freqtrade.exchange import Exchange from freqtrade.arguments import TimeRange import importlib -ujson_found = importlib.util.find_spec("ujson") -if ujson_found is not None: - import ujson logger = logging.getLogger(__name__) -if ujson_found is not None: - logger.debug('Loaded UltraJson ujson in optimize.py') - - def json_load(data): """Try to load data with ujson""" if _UJSON: From d6d3dfdcc2c22fb256596b5f9e2ef429586331cf Mon Sep 17 00:00:00 2001 From: misagh Date: Fri, 21 Sep 2018 22:06:09 +0200 Subject: [PATCH 053/699] =?UTF-8?q?removing=20=E2=80=9Cif=20ujson=5Ffound?= =?UTF-8?q?=20is=20not=20None:=E2=80=9D=20as=20=E2=80=9Cjson=E2=80=9D=20re?= =?UTF-8?q?fers=20to=20=E2=80=9Cujson=E2=80=9D=20if=20it=20exists?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- freqtrade/optimize/__init__.py | 15 +++------------ 1 file changed, 3 insertions(+), 12 deletions(-) diff --git a/freqtrade/optimize/__init__.py b/freqtrade/optimize/__init__.py index aeb56811e..7e44b3bac 100644 --- a/freqtrade/optimize/__init__.py +++ b/freqtrade/optimize/__init__.py @@ -77,17 +77,11 @@ def load_tickerdata_file( if os.path.isfile(gzipfile): logger.debug('Loading ticker data from file %s', gzipfile) with gzip.open(gzipfile) as tickerdata: - if ujson_found is not None: - pairdata = ujson.load(tickerdata, precise_float=True) - else: - pairdata = json.load(tickerdata) + pairdata = json.load(tickerdata) elif os.path.isfile(file): logger.debug('Loading ticker data from file %s', file) with open(file) as tickerdata: - if ujson_found is not None: - pairdata = ujson.load(tickerdata, precise_float=True) - else: - pairdata = json.load(tickerdata) + pairdata = json.load(tickerdata) else: return None @@ -183,10 +177,7 @@ def load_cached_data_for_updating(filename: str, # read the cached file if os.path.isfile(filename): with open(filename, "rt") as file: - if ujson_found is not None: - data = ujson.load(file, precise_float=True) - else: - data = json.load(file) + data = json.load(file) # remove the last item, because we are not sure if it is correct # it could be fetched when the candle was incompleted if data: From cf37093e5a267dc0a47349e84047a20d2811b75a Mon Sep 17 00:00:00 2001 From: misagh Date: Fri, 21 Sep 2018 22:07:12 +0200 Subject: [PATCH 054/699] empty dict default removed --- freqtrade/strategy/default_strategy.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/freqtrade/strategy/default_strategy.py b/freqtrade/strategy/default_strategy.py index 323a7d95f..f1646779b 100644 --- a/freqtrade/strategy/default_strategy.py +++ b/freqtrade/strategy/default_strategy.py @@ -28,7 +28,7 @@ class DefaultStrategy(IStrategy): # Optimal ticker interval for the strategy ticker_interval = '5m' - def populate_indicators(self, dataframe: DataFrame, metadata: dict = {}) -> DataFrame: + def populate_indicators(self, dataframe: DataFrame, metadata: dict) -> DataFrame: """ Adds several different TA indicators to the given DataFrame @@ -199,7 +199,7 @@ class DefaultStrategy(IStrategy): return dataframe - def populate_buy_trend(self, dataframe: DataFrame, metadata: dict = {}) -> DataFrame: + def populate_buy_trend(self, dataframe: DataFrame, metadata: dict) -> DataFrame: """ Based on TA indicators, populates the buy signal for the given dataframe :param dataframe: DataFrame @@ -221,7 +221,7 @@ class DefaultStrategy(IStrategy): return dataframe - def populate_sell_trend(self, dataframe: DataFrame, metadata: dict = {}) -> DataFrame: + def populate_sell_trend(self, dataframe: DataFrame, metadata: dict) -> DataFrame: """ Based on TA indicators, populates the sell signal for the given dataframe :param dataframe: DataFrame From f1b4e4b36caf33f72128d05f29ca927b6f4bd746 Mon Sep 17 00:00:00 2001 From: misagh Date: Sat, 22 Sep 2018 15:43:41 +0200 Subject: [PATCH 055/699] =?UTF-8?q?stop=20loss=20range=20=E2=80=9Cstart,?= =?UTF-8?q?=20end,=20step=E2=80=9D=20configurable=20for=20Edge?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- config.json.example | 3 +++ freqtrade/edge/__init__.py | 8 ++++++-- freqtrade/tests/conftest.py | 3 +++ 3 files changed, 12 insertions(+), 2 deletions(-) diff --git a/config.json.example b/config.json.example index f90b4f470..83e9e0d38 100644 --- a/config.json.example +++ b/config.json.example @@ -55,6 +55,9 @@ "process_throttle_secs": 1800, "total_capital_in_stake_currency": 0.5, "allowed_risk": 0.01, + "stoploss_range_min": -0.01, + "stoploss_range_max": -0.1, + "stoploss_range_step": -0.001, "maximum_winrate": 0.80, "min_trade_number": 15, "max_trade_duration_minute": 1440, diff --git a/freqtrade/edge/__init__.py b/freqtrade/edge/__init__.py index 46cdaaf32..f18e84f3f 100644 --- a/freqtrade/edge/__init__.py +++ b/freqtrade/edge/__init__.py @@ -148,7 +148,11 @@ class Edge(): max_open_trades = 0 realistic = False - stoploss_range = np.arange(-0.11, -0.00, 0.01) + stoploss_range_min = float(self.edge_config.get('stoploss_range_min', -0.01)) + stoploss_range_max = float(self.edge_config.get('stoploss_range_max', -0.05)) + stoploss_range_step = float(self.edge_config.get('stoploss_range_step', -0.001)) + + stoploss_range = np.arange(stoploss_range_min, stoploss_range_max, stoploss_range_step) trades = [] trade_count_lock: Dict = {} @@ -165,7 +169,7 @@ class Edge(): # call backslap - results are a list of dicts for stoploss in stoploss_range: - bslap_results += self.backslap_pair(ticker_data, pair, round(stoploss, 3)) + bslap_results += self.backslap_pair(ticker_data, pair, round(stoploss, 6)) # Switch List of Trade Dicts (bslap_results) to Dataframe # Fill missing, calculable columns, profit, duration , abs etc. diff --git a/freqtrade/tests/conftest.py b/freqtrade/tests/conftest.py index 8a3f71f51..bece82436 100644 --- a/freqtrade/tests/conftest.py +++ b/freqtrade/tests/conftest.py @@ -132,6 +132,9 @@ def default_conf(): "process_throttle_secs": 1800, "total_capital_in_stake_currency": 0.5, "allowed_risk": 0.01, + "stoploss_range_min": -0.01, + "stoploss_range_max": -0.1, + "stoploss_range_step": -0.001, "maximum_winrate": 0.80, "min_trade_number": 15, "max_trade_duration_minute": 1440, From 29459d7d30b8e2019d033740d0b1200163b2c397 Mon Sep 17 00:00:00 2001 From: misagh Date: Sun, 23 Sep 2018 04:51:53 +0200 Subject: [PATCH 056/699] import libraries organized. --- freqtrade/edge/__init__.py | 29 ++++++++++------------------- 1 file changed, 10 insertions(+), 19 deletions(-) diff --git a/freqtrade/edge/__init__.py b/freqtrade/edge/__init__.py index f18e84f3f..a9d049ee4 100644 --- a/freqtrade/edge/__init__.py +++ b/freqtrade/edge/__init__.py @@ -6,11 +6,13 @@ from datetime import datetime, timedelta from typing import Any, Dict, List, NamedTuple, Optional, Tuple import arrow -from pandas import DataFrame, to_datetime -from tabulate import tabulate -import numpy as np +from pandas import DataFrame, to_datetime +import pandas as pd + +from tabulate import tabulate import freqtrade.optimize as optimize +from freqtrade.optimize.backtesting import BacktestResult from freqtrade import DependencyException, constants from freqtrade.arguments import Arguments from freqtrade.configuration import Configuration @@ -21,10 +23,12 @@ from freqtrade.strategy.interface import SellType from freqtrade.strategy.resolver import IStrategy, StrategyResolver from freqtrade.optimize.backtesting import Backtesting from collections import OrderedDict -import timeit -from time import sleep -import pdb +import numpy as np +import timeit +import utils_find_1st as utf1st +from time import sleep +from pandas import set_option logger = logging.getLogger(__name__) @@ -178,8 +182,6 @@ class Edge(): if len(bslap_results_df) > 0: # Only post process a frame if it has a record bslap_results_df = self.vector_fill_results_table(bslap_results_df) else: - from freqtrade.optimize.backtesting import BacktestResult - bslap_results_df = [] bslap_results_df = DataFrame.from_records(bslap_results_df, columns=BacktestResult._fields) @@ -207,8 +209,6 @@ class Edge(): :param bslap_results Dataframe :return: bslap_results Dataframe """ - import pandas as pd - import numpy as np debug = self.debug_vector # stake and fees @@ -247,7 +247,6 @@ class Edge(): def np_get_t_open_ind(self, np_buy_arr, t_exit_ind: int, np_buy_arr_len: int, stop_stops: int, stop_stops_count: int): - import utils_find_1st as utf1st """ The purpose of this def is to return the next "buy" = 1 after t_exit_ind. @@ -373,12 +372,6 @@ class Edge(): return final.reset_index().values def backslap_pair(self, ticker_data, pair, stoploss): - import pandas as pd - import numpy as np - import timeit - import utils_find_1st as utf1st - from datetime import datetime - ### backslap debug wrap # debug_2loops = False # only loop twice, for faster debug # debug_timing = False # print timing for each step @@ -388,7 +381,6 @@ class Edge(): debug = self.debug # print values, to check accuracy # Read Stop Loss Values and Stake - # pdb.set_trace() #stop = self.stop_loss_value stop = stoploss p_stop = (stop + 1) # What stop really means, e.g 0.01 is 0.99 of price @@ -398,7 +390,6 @@ class Edge(): print("p_stop is", p_stop, "value used to multiply to entry price") if debug: - from pandas import set_option set_option('display.max_rows', 5000) set_option('display.max_columns', 8) pd.set_option('display.width', 1000) From a26131cea359822d9290a46ec3d6f0017648ffab Mon Sep 17 00:00:00 2001 From: misagh Date: Mon, 24 Sep 2018 14:21:37 +0200 Subject: [PATCH 057/699] .travis: install future before requirements.txt --- .travis.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.travis.yml b/.travis.yml index 981eedcf8..c81c96460 100644 --- a/.travis.yml +++ b/.travis.yml @@ -14,6 +14,7 @@ install: - ./install_ta-lib.sh - export LD_LIBRARY_PATH=/usr/local/lib:$LD_LIBRARY_PATH - pip install --upgrade flake8 coveralls pytest-random-order pytest-asyncio mypy +- pip install future - pip install -r requirements.txt - pip install -e . jobs: From 136678351765ef8921405277dabb6d075dedcdf2 Mon Sep 17 00:00:00 2001 From: misagh Date: Mon, 24 Sep 2018 14:28:16 +0200 Subject: [PATCH 058/699] Dockerfile: installing future before requirements.txt --- Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/Dockerfile b/Dockerfile index 2506665ab..0e90f0720 100644 --- a/Dockerfile +++ b/Dockerfile @@ -17,6 +17,7 @@ WORKDIR /freqtrade # Install dependencies COPY requirements.txt /freqtrade/ RUN pip install numpy --no-cache-dir \ + && pip install future \ && pip install -r requirements.txt --no-cache-dir # Install and execute From 303eefda76e04f9c9b714d1875a406dd1ec52cb3 Mon Sep 17 00:00:00 2001 From: misagh Date: Mon, 24 Sep 2018 14:55:49 +0200 Subject: [PATCH 059/699] =?UTF-8?q?test=5Fget=5Ftrade=5Fstake=5Famount=5Fu?= =?UTF-8?q?nlimited=5Famount=20fixed:=20=E2=80=9Cpair=E2=80=9D=20argument?= =?UTF-8?q?=20added=20to=20=5Fget=5Ftrade=5Fstake=5Famount?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- freqtrade/tests/test_freqtradebot.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/freqtrade/tests/test_freqtradebot.py b/freqtrade/tests/test_freqtradebot.py index 5e982f11a..c246cc858 100644 --- a/freqtrade/tests/test_freqtradebot.py +++ b/freqtrade/tests/test_freqtradebot.py @@ -229,25 +229,25 @@ def test_get_trade_stake_amount_unlimited_amount(default_conf, patch_get_signal(freqtrade) # no open trades, order amount should be 'balance / max_open_trades' - result = freqtrade._get_trade_stake_amount() + result = freqtrade._get_trade_stake_amount('ETH/BTC') assert result == default_conf['stake_amount'] / conf['max_open_trades'] # create one trade, order amount should be 'balance / (max_open_trades - num_open_trades)' freqtrade.create_trade() - result = freqtrade._get_trade_stake_amount() + result = freqtrade._get_trade_stake_amount('LTC/BTC') assert result == default_conf['stake_amount'] / (conf['max_open_trades'] - 1) # create 2 trades, order amount should be None freqtrade.create_trade() - result = freqtrade._get_trade_stake_amount() + result = freqtrade._get_trade_stake_amount('XRP/BTC') assert result is None # set max_open_trades = None, so do not trade conf['max_open_trades'] = 0 freqtrade = FreqtradeBot(conf) - result = freqtrade._get_trade_stake_amount() + result = freqtrade._get_trade_stake_amount('NEO/BTC') assert result is None From 76dd7549636fca2f8e17f82270e9efc2a7d3bb58 Mon Sep 17 00:00:00 2001 From: misagh Date: Mon, 24 Sep 2018 15:02:50 +0200 Subject: [PATCH 060/699] =?UTF-8?q?test=5Fget=5Ftrade=5Fstake=5Famount=20a?= =?UTF-8?q?nd=20test=5Fget=5Ftrade=5Fstake=5Famount=5Fno=5Fstake=5Famount?= =?UTF-8?q?=20fixed:=20=E2=80=9Cpair=E2=80=9D=20arg=20added=20to=20=5Fget?= =?UTF-8?q?=5Ftrade=5Fstake=5Famount?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- freqtrade/tests/test_freqtradebot.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/freqtrade/tests/test_freqtradebot.py b/freqtrade/tests/test_freqtradebot.py index c246cc858..4d5039449 100644 --- a/freqtrade/tests/test_freqtradebot.py +++ b/freqtrade/tests/test_freqtradebot.py @@ -183,7 +183,7 @@ def test_get_trade_stake_amount(default_conf, ticker, limit_buy_order, fee, mock freqtrade = FreqtradeBot(default_conf) - result = freqtrade._get_trade_stake_amount() + result = freqtrade._get_trade_stake_amount('ETH/BTC') assert result == default_conf['stake_amount'] @@ -201,7 +201,7 @@ def test_get_trade_stake_amount_no_stake_amount(default_conf, freqtrade = FreqtradeBot(default_conf) with pytest.raises(DependencyException, match=r'.*stake amount.*'): - freqtrade._get_trade_stake_amount() + freqtrade._get_trade_stake_amount('ETH/BTC') def test_get_trade_stake_amount_unlimited_amount(default_conf, @@ -499,7 +499,7 @@ def test_create_trade_limit_reached(default_conf, ticker, limit_buy_order, patch_get_signal(freqtrade) assert freqtrade.create_trade() is False - assert freqtrade._get_trade_stake_amount() is None + assert freqtrade._get_trade_stake_amount('ETH/BTC') is None def test_create_trade_no_pairs(default_conf, ticker, limit_buy_order, fee, markets, mocker) -> None: From 308428644b2f7fcb8a241f68bb09b3730e8309f9 Mon Sep 17 00:00:00 2001 From: misagh Date: Mon, 24 Sep 2018 15:27:26 +0200 Subject: [PATCH 061/699] test_process_trade_creation log message changed: in reality the buy signal is actually triggered --- freqtrade/freqtradebot.py | 4 +--- freqtrade/tests/test_freqtradebot.py | 2 +- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index 182a42819..e3cc669d9 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -382,9 +382,6 @@ class FreqtradeBot(object): :return: True if a trade object has been created and persisted, False otherwise """ interval = self.strategy.ticker_interval - - logger.info('Checking buy signals to create a new trade: ...') - whitelist = copy.deepcopy(self.config['exchange']['pair_whitelist']) # Remove currently opened and latest pairs from whitelist @@ -405,6 +402,7 @@ class FreqtradeBot(object): (buy, sell) = self.strategy.get_signal(_pair, interval, self.exchange.klines.get(_pair)) if buy and not sell: stake_amount = self._get_trade_stake_amount(_pair) + logger.info('Buy signal found: about create a new trade with stake_amount: %f ...', stake_amount) bidstrat_check_depth_of_market = self.config.get('bid_strategy', {}).\ get('check_depth_of_market', {}) if (bidstrat_check_depth_of_market.get('enabled', False)) and\ diff --git a/freqtrade/tests/test_freqtradebot.py b/freqtrade/tests/test_freqtradebot.py index 4d5039449..79a09fde3 100644 --- a/freqtrade/tests/test_freqtradebot.py +++ b/freqtrade/tests/test_freqtradebot.py @@ -598,7 +598,7 @@ def test_process_trade_creation(default_conf, ticker, limit_buy_order, assert trade.amount == 90.99181073703367 assert log_has( - 'Checking buy signals to create a new trade with stake_amount: 0.001000 ...', + 'Buy signal found: about create a new trade with stake_amount: 0.001000 ...', caplog.record_tuples ) From 027ec4d98eda93679e929655ad3688e9dcde0cd2 Mon Sep 17 00:00:00 2001 From: misagh Date: Mon, 24 Sep 2018 15:47:07 +0200 Subject: [PATCH 062/699] test_sell_profit_only_enable_loss and test_create_trade_limit_reached fixed --- freqtrade/freqtradebot.py | 2 ++ freqtrade/tests/test_freqtradebot.py | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index e3cc669d9..60d286af1 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -402,6 +402,8 @@ class FreqtradeBot(object): (buy, sell) = self.strategy.get_signal(_pair, interval, self.exchange.klines.get(_pair)) if buy and not sell: stake_amount = self._get_trade_stake_amount(_pair) + if not stake_amount: + return False logger.info('Buy signal found: about create a new trade with stake_amount: %f ...', stake_amount) bidstrat_check_depth_of_market = self.config.get('bid_strategy', {}).\ get('check_depth_of_market', {}) diff --git a/freqtrade/tests/test_freqtradebot.py b/freqtrade/tests/test_freqtradebot.py index 79a09fde3..b12ec1018 100644 --- a/freqtrade/tests/test_freqtradebot.py +++ b/freqtrade/tests/test_freqtradebot.py @@ -1423,7 +1423,7 @@ def test_sell_profit_only_enable_loss(default_conf, limit_buy_order, fee, market freqtrade = FreqtradeBot(default_conf) patch_get_signal(freqtrade) freqtrade.strategy.stop_loss_reached = \ - lambda current_rate, trade, current_time, current_profit: SellCheckTuple( + lambda current_rate, trade, current_time, force_stoploss, current_profit: SellCheckTuple( sell_flag=False, sell_type=SellType.NONE) freqtrade.create_trade() From a806dd45f27a8f023d148221f81937e4970f6b4e Mon Sep 17 00:00:00 2001 From: misagh Date: Mon, 24 Sep 2018 16:02:29 +0200 Subject: [PATCH 063/699] lost in branches ! typo for some magical unknown reasons --- freqtrade/optimize/__init__.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/freqtrade/optimize/__init__.py b/freqtrade/optimize/__init__.py index 7e44b3bac..967227805 100644 --- a/freqtrade/optimize/__init__.py +++ b/freqtrade/optimize/__init__.py @@ -20,6 +20,7 @@ import importlib logger = logging.getLogger(__name__) + def json_load(data): """Try to load data with ujson""" if _UJSON: @@ -177,7 +178,7 @@ def load_cached_data_for_updating(filename: str, # read the cached file if os.path.isfile(filename): with open(filename, "rt") as file: - data = json.load(file) + data = json_load(file) # remove the last item, because we are not sure if it is correct # it could be fetched when the candle was incompleted if data: @@ -242,4 +243,4 @@ def download_backtesting_testdata(datadir: str, logger.debug("New Start: %s", misc.format_ms_time(data[0][0])) logger.debug("New End: %s", misc.format_ms_time(data[-1][0])) - misc.file_dump_json(filename, data) \ No newline at end of file + misc.file_dump_json(filename, data) From e8716f16ad6b868cef60d149128ec91f195770c1 Mon Sep 17 00:00:00 2001 From: misagh Date: Mon, 24 Sep 2018 17:47:50 +0200 Subject: [PATCH 064/699] calculating expectancy and sort pairs accordingly instead of delta --- freqtrade/edge/__init__.py | 21 ++++++++++++--------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/freqtrade/edge/__init__.py b/freqtrade/edge/__init__.py index a9d049ee4..77350feed 100644 --- a/freqtrade/edge/__init__.py +++ b/freqtrade/edge/__init__.py @@ -355,20 +355,23 @@ class Edge(): return x ############################## - # The difference between risk reward ratio and required risk reward - # We use it as an indicator to find the most interesting pair to trade - def delta(x): - x = (abs(1/ ((x[x < 0].sum() / x[x < 0].count()) / (x[x > 0].sum() / x[x > 0].count())))) - (1/(x[x > 0].count()/x.count()) -1) + # Expectancy + # Tells you the interest percentage you should hope + # E.x. if expectancy is 0.35, on $1 trade you should expect a target of $1.35 + def expectancy(x): + average_win = float(x[x > 0].sum() / x[x > 0].count()) + average_loss = float(abs(x[x < 0].sum() / x[x < 0].count())) + winrate = float(x[x > 0].count()/x.count()) + x = ((1 + average_win/average_loss) * winrate) - 1 return x ############################## - final = results.groupby(['pair', 'stoploss'])['profit_abs'].\ - agg([winrate, risk_reward_ratio, required_risk_reward, delta]).\ - reset_index().sort_values(by=['delta', 'stoploss'], ascending=False)\ - .groupby('pair').first().sort_values(by=['delta'], ascending=False) + agg([winrate, risk_reward_ratio, required_risk_reward, expectancy]).\ + reset_index().sort_values(by=['expectancy', 'stoploss'], ascending=False)\ + .groupby('pair').first().sort_values(by=['expectancy'], ascending=False) - # Returning an array of pairs in order of "delta" + # Returning an array of pairs in order of "expectancy" return final.reset_index().values def backslap_pair(self, ticker_data, pair, stoploss): From 40d73de357812868b982cd55aba09d677e050609 Mon Sep 17 00:00:00 2001 From: misagh Date: Mon, 24 Sep 2018 19:22:30 +0200 Subject: [PATCH 065/699] refactoring backslap (round one) --- freqtrade/edge/__init__.py | 370 ++++++------------------------------- 1 file changed, 52 insertions(+), 318 deletions(-) diff --git a/freqtrade/edge/__init__.py b/freqtrade/edge/__init__.py index 77350feed..7a45b7b78 100644 --- a/freqtrade/edge/__init__.py +++ b/freqtrade/edge/__init__.py @@ -1,42 +1,33 @@ +# pragma pylint: disable=W0603 +""" Edge positioning package """ import logging -import operator -import sys -from argparse import Namespace -from datetime import datetime, timedelta -from typing import Any, Dict, List, NamedTuple, Optional, Tuple - +from typing import Any, Dict import arrow -from pandas import DataFrame, to_datetime +from pandas import DataFrame import pandas as pd -from tabulate import tabulate import freqtrade.optimize as optimize from freqtrade.optimize.backtesting import BacktestResult -from freqtrade import DependencyException, constants from freqtrade.arguments import Arguments -from freqtrade.configuration import Configuration from freqtrade.exchange import Exchange -from freqtrade.misc import file_dump_json -from freqtrade.persistence import Trade from freqtrade.strategy.interface import SellType from freqtrade.strategy.resolver import IStrategy, StrategyResolver from freqtrade.optimize.backtesting import Backtesting -from collections import OrderedDict import numpy as np import timeit import utils_find_1st as utf1st -from time import sleep -from pandas import set_option +import pdb logger = logging.getLogger(__name__) + class Edge(): config: Dict = {} - def __init__(self, config: Dict[str, Any], exchange = None) -> None: + def __init__(self, config: Dict[str, Any], exchange=None) -> None: """ constructor """ @@ -47,7 +38,7 @@ class Edge(): self.get_timeframe = Backtesting.get_timeframe self.populate_buy_trend = self.strategy.populate_buy_trend self.populate_sell_trend = self.strategy.populate_sell_trend - + self.edge_config = self.config.get('edge', {}) self._last_updated = None @@ -103,7 +94,6 @@ class Edge(): self.stop_stops: int = 9999 # stop back testing any pair with this many stops, set to 999999 to not hit - def calculate(self) -> bool: pairs = self.config['exchange']['pair_whitelist'] heartbeat = self.config['edge']['process_throttle_secs'] @@ -115,9 +105,8 @@ class Edge(): logger.info('Using stake_currency: %s ...', self.config['stake_currency']) logger.info('Using stake_amount: %s ...', self.config['stake_amount']) - logger.info('Using local backtesting data (using whitelist in given config) ...') - + #TODO: add "timerange" to Edge config timerange = Arguments.parse_timerange(None if self.config.get( 'timerange') is None else str(self.config.get('timerange'))) @@ -133,7 +122,7 @@ class Edge(): if not data: logger.critical("No data found. Edge is stopped ...") return - + preprocessed = self.tickerdata_to_dataframe(data) # Print timeframe @@ -144,27 +133,17 @@ class Edge(): max_date.isoformat(), (max_date - min_date).days ) - - headers = ['date', 'buy', 'open', 'close', 'sell', 'high', 'low'] - # Max open trades need not be considered in Edge positioning - max_open_trades = 0 - - realistic = False stoploss_range_min = float(self.edge_config.get('stoploss_range_min', -0.01)) stoploss_range_max = float(self.edge_config.get('stoploss_range_max', -0.05)) stoploss_range_step = float(self.edge_config.get('stoploss_range_step', -0.001)) - stoploss_range = np.arange(stoploss_range_min, stoploss_range_max, stoploss_range_step) - trades = [] - trade_count_lock: Dict = {} ########################### Call out BSlap Loop instead of Original BT code bslap_results: list = [] for pair, pair_data in preprocessed.items(): - - # Sorting dataframe by date and reset index + # Sorting dataframe by date and reset index pair_data = pair_data.sort_values(by=['date']) pair_data = pair_data.reset_index(drop=True) @@ -184,18 +163,17 @@ class Edge(): else: bslap_results_df = [] bslap_results_df = DataFrame.from_records(bslap_results_df, columns=BacktestResult._fields) - + self._cached_pairs = self._process_result(data, bslap_results_df, stoploss_range) self._last_updated = arrow.utcnow().timestamp return True - + def sort_pairs(self, pairs) -> bool: if len(self._cached_pairs) == 0: self.calculate() edge_sorted_pairs = [x[0] for x in self._cached_pairs] return [x for _, x in sorted(zip(edge_sorted_pairs,pairs), key=lambda pair: pair[0])] - - + def vector_fill_results_table(self, bslap_results_df: DataFrame): """ The Results frame contains a number of columns that are calculable @@ -209,7 +187,6 @@ class Edge(): :param bslap_results Dataframe :return: bslap_results Dataframe """ - debug = self.debug_vector # stake and fees # stake = 0.015 @@ -221,7 +198,6 @@ class Edge(): open_fee = fee / 2 close_fee = fee / 2 - bslap_results_df['trade_duration'] = bslap_results_df['close_time'] - bslap_results_df['open_time'] bslap_results_df['trade_duration'] = bslap_results_df['trade_duration'].map(lambda x: int(x.total_seconds() / 60)) @@ -242,7 +218,6 @@ class Edge(): # Absolute profit bslap_results_df['profit_abs'] = bslap_results_df['sell_take'] - bslap_results_df['buy_spend'] - return bslap_results_df def np_get_t_open_ind(self, np_buy_arr, t_exit_ind: int, np_buy_arr_len: int, stop_stops: int, @@ -250,26 +225,12 @@ class Edge(): """ The purpose of this def is to return the next "buy" = 1 after t_exit_ind. - - This function will also check is the stop limit for the pair has been reached. + This function will also check is the stop limit for the pair has been reached. if stop_stops is the limit and stop_stops_count it the number of times the stop has been hit. - t_exit_ind is the index the last trade exited on or 0 if first time around this loop. - stop_stops i """ - debug = self.debug - - # Timers, to be called if in debug - def s(): - st = timeit.default_timer() - return st - - def f(st): - return (timeit.default_timer() - st) - - st = s() t_open_ind: int """ @@ -290,17 +251,15 @@ class Edge(): if stop_stops_count >= stop_stops: # if maximum number of stops allowed in a pair is hit, exit loop t_open_ind = -1 # -1 ends the loop - if debug: - print("Max stop limit ", stop_stops, "reached. Moving to next pair") return t_open_ind def _process_result(self, data: Dict[str, Dict], results: DataFrame, stoploss_range) -> str: """ - This is a temporary version of edge positioning calculation. - The function will be eventually moved to a plugin called Edge in order to calculate necessary WR, RRR and - other indictaors related to money management periodically (each X minutes) and keep it in a storage. - The calulation will be done per pair and per strategy. + This is a temporary version of edge positioning calculation. + The function will be eventually moved to a plugin called Edge in order to calculate necessary WR, RRR and + other indictaors related to money management periodically (each X minutes) and keep it in a storage. + The calulation will be done per pair and per strategy. """ # Removing open trades from dataset @@ -312,8 +271,7 @@ class Edge(): results = results.groupby('pair').filter(lambda x: len(x) > min_trades_number) ################################### - - # Removing outliers (Only Pumps) from the dataset + # Removing outliers (Only Pumps) from the dataset # The method to detect outliers is to calculate standard deviation # Then every value more than (standard deviation + 2*average) is out (pump) # @@ -333,7 +291,6 @@ class Edge(): results = results[results.trade_duration < max_trade_duration] ####################################################################### - # Win Rate is the number of profitable trades # Divided by number of trades def winrate(x): @@ -354,7 +311,11 @@ class Edge(): x = (1/(x[x > 0].count()/x.count()) -1) return x ############################## - + + def delta(x): + x = (abs(1/ ((x[x < 0].sum() / x[x < 0].count()) / (x[x > 0].sum() / x[x > 0].count())))) - (1/(x[x > 0].count()/x.count()) -1) + return x + # Expectancy # Tells you the interest percentage you should hope # E.x. if expectancy is 0.35, on $1 trade you should expect a target of $1.35 @@ -365,47 +326,20 @@ class Edge(): x = ((1 + average_win/average_loss) * winrate) - 1 return x ############################## - + final = results.groupby(['pair', 'stoploss'])['profit_abs'].\ - agg([winrate, risk_reward_ratio, required_risk_reward, expectancy]).\ + agg([winrate, risk_reward_ratio, required_risk_reward, expectancy, delta]).\ reset_index().sort_values(by=['expectancy', 'stoploss'], ascending=False)\ .groupby('pair').first().sort_values(by=['expectancy'], ascending=False) - + # Returning an array of pairs in order of "expectancy" return final.reset_index().values def backslap_pair(self, ticker_data, pair, stoploss): - ### backslap debug wrap - # debug_2loops = False # only loop twice, for faster debug - # debug_timing = False # print timing for each step - # debug = False # print values, to check accuracy - debug_2loops = self.debug_2loops # only loop twice, for faster debug - debug_timing = self.debug_timing # print timing for each step - debug = self.debug # print values, to check accuracy - # Read Stop Loss Values and Stake - #stop = self.stop_loss_value stop = stoploss p_stop = (stop + 1) # What stop really means, e.g 0.01 is 0.99 of price - if debug: - print("Stop is ", stop, "value from stragey file") - print("p_stop is", p_stop, "value used to multiply to entry price") - - if debug: - set_option('display.max_rows', 5000) - set_option('display.max_columns', 8) - pd.set_option('display.width', 1000) - pd.set_option('max_colwidth', 40) - pd.set_option('precision', 12) - - def s(): - st = timeit.default_timer() - return st - - def f(st): - return (timeit.default_timer() - st) - #### backslap config ''' Numpy arrays are used for 100x speed up @@ -416,22 +350,12 @@ class Edge(): ####### # Use vars set at top of backtest - np_buy: int = self.np_buy np_open: int = self.np_open - np_close: int = self.np_close np_sell: int = self.np_sell - np_high: int = self.np_high - np_low: int = self.np_low - np_stop: int = self.np_stop - np_bto: int = self.np_bto # buys_triggered_on - should be close np_bco: int = self.np_bco # buys calculated on - open of the next candle. np_sto: int = self.np_sto # stops_triggered_on - Should be low, FT uses close np_sco: int = self.np_sco # stops_calculated_on - Should be stop, FT uses close - ### End Config - - pair: str = pair - # ticker_data: DataFrame = ticker_dfs[t_file] bslap: DataFrame = ticker_data @@ -457,23 +381,12 @@ class Edge(): stop_stops = self.stop_stops # Int of stops within a pair to stop trading a pair at stop_stops_count = 0 # stop counter per pair - st = s() # Start timer for processing dataframe - if debug: - print('Processing:', pair) - # Results will be stored in a list of dicts bslap_pair_results: list = [] bslap_result: dict = {} while t_exit_ind < np_buy_arr_len: loop = loop + 1 - if debug or debug_timing: - print("-- T_exit_Ind - Numpy Index is", t_exit_ind, " ----------------------- Loop", loop, pair) - if debug_2loops: - if loop == 3: - print( - "++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++Loop debug max met - breaking") - break ''' Dev phases Phase 1 @@ -481,7 +394,7 @@ class Edge(): a) Find first buy index b) Discover first stop and sell hit after buy index c) Chose first instance as trade exit - + Phase 2 2) Manage dynamic Stop and ROI Exit a) Create trade slice from 1 @@ -489,72 +402,50 @@ class Edge(): c) search within trade slice for ROI hit ''' - if debug_timing: - st = s() ''' 0 - Find next buy entry Finds index for first (buy = 1) flag - + Requires: np_buy_arr - a 1D array of the 'buy' column. To find next "1" Required: t_exit_ind - Either 0, first loop. Or The index we last exited on - Requires: np_buy_arr_len - length of pair array. - Requires: stops_stops - number of stops allowed before stop trading a pair + Requires: np_buy_arr_len - length of pair array. + Requires: stops_stops - number of stops allowed before stop trading a pair Requires: stop_stop_counts - count of stops hit in the pair Provides: The next "buy" index after t_exit_ind - + If -1 is returned no buy has been found in remainder of array, skip to exit loop ''' t_open_ind = self.np_get_t_open_ind(np_buy_arr, t_exit_ind, np_buy_arr_len, stop_stops, stop_stops_count) - if debug: - print("\n(0) numpy debug \nnp_get_t_open, has returned the next valid buy index as", t_open_ind) - print("If -1 there are no valid buys in the remainder of ticker data. Skipping to end of loop") - if debug_timing: - t_t = f(st) - print("0-numpy", str.format('{0:.17f}', t_t)) - st = s() - if t_open_ind != -1: """ 1 - Create views to search within for our open trade - The views are our search space for the next Stop or Sell Numpy view is employed as: 1,000 faster than pandas searches Pandas cannot assure it will always return a view, it may make a slow copy. - The view contains columns: buy 0 - open 1 - close 2 - sell 3 - high 4 - low 5 - + Requires: np_bslap is our numpy array of the ticker DataFrame Requires: t_open_ind is the index row with the buy. Provides: np_t_open_v View of array after buy. - Provides: np_t_open_v_stop View of array after buy +1 + Provides: np_t_open_v_stop View of array after buy +1 (Stop will search in here to prevent stopping in the past) """ np_t_open_v = np_bslap[t_open_ind:] np_t_open_v_stop = np_bslap[t_open_ind + 1:] - if debug: - print("\n(1) numpy debug \nNumpy view row 0 is now Ticker_Data Index", t_open_ind) - print("Numpy View: Buy - Open - Close - Sell - High - Low") - print("Row 0", np_t_open_v[0]) - print("Row 1", np_t_open_v[1], ) - if debug_timing: - t_t = f(st) - print("2-numpy", str.format('{0:.17f}', t_t)) - st = s() - ''' 2 - Calculate our stop-loss price - + As stop is based on buy price of our trade - (BTO)Buys are Triggered On np_bto, typically the CLOSE of candle - (BCO)Buys are Calculated On np_bco, default is OPEN of the next candle. This is as we only see the CLOSE after it has happened. The back test assumption is we have bought at first available price, the OPEN - + Requires: np_bslap - is our numpy array of the ticker DataFrame Requires: t_open_ind - is the index row with the first buy. Requires: p_stop - is the stop rate, ie. 0.99 is -1% @@ -562,18 +453,11 @@ class Edge(): ''' np_t_stop_pri = (np_bslap[t_open_ind + 1, np_bco] * p_stop) - if debug: - print("\n(2) numpy debug\nStop-Loss has been calculated at:", np_t_stop_pri) - if debug_timing: - t_t = f(st) - print("2-numpy", str.format('{0:.17f}', t_t)) - st = s() - ''' 3 - Find candle STO is under Stop-Loss After Trade opened. - + where [np_sto] (stop tiggered on variable: "close", "low" etc) < np_t_stop_pri - + Requires: np_t_open_v_stop Numpy view of ticker_data after buy row +1 (when trade was opened) Requires: np_sto User Var(STO)StopTriggeredOn. Typically set to "low" or "close" Requires: np_t_stop_pri The stop-loss price STO must fall under to trigger stop @@ -586,26 +470,11 @@ class Edge(): # plus 1 as np_t_open_v_stop is 1 ahead of view np_t_open_v, used from here on out. np_t_stop_ind = np_t_stop_ind + 1 - if debug: - print("\n(3) numpy debug\nNext view index with STO (stop trigger on) under Stop-Loss is", - np_t_stop_ind - 1, - ". STO is using field", np_sto, - "\nFrom key: buy 0 - open 1 - close 2 - sell 3 - high 4 - low 5\n") - - print( - "If -1 or 0 returned there is no stop found to end of view, then next two array lines are garbage") - print("Row", np_t_stop_ind - 1, np_t_open_v[np_t_stop_ind]) - print("Row", np_t_stop_ind, np_t_open_v[np_t_stop_ind + 1]) - if debug_timing: - t_t = f(st) - print("3-numpy", str.format('{0:.17f}', t_t)) - st = s() - ''' 4 - Find first sell index after trade open - + First index in the view np_t_open_v where ['sell'] = 1 - + Requires: np_t_open_v - view of ticker_data from buy onwards Requires: no_sell - integer '3', the buy column in the array Provides: np_t_sell_ind index of view where first sell=1 after buy @@ -617,26 +486,17 @@ class Edge(): np_t_sell_ind = utf1st.find_1st(np_t_open_v[:, np_sell], 1, utf1st.cmp_equal) - if debug: - print("\n(4) numpy debug\nNext view index with sell = 1 is ", np_t_sell_ind) - print("If 0 or less is returned there is no sell found to end of view, then next lines garbage") - print("Row", np_t_sell_ind, np_t_open_v[np_t_sell_ind]) - print("Row", np_t_sell_ind + 1, np_t_open_v[np_t_sell_ind + 1]) - if debug_timing: - t_t = f(st) - print("4-numpy", str.format('{0:.17f}', t_t)) - st = s() ''' 5 - Determine which was hit first a stop or sell To then use as exit index price-field (sell on buy, stop on stop) - + STOP takes priority over SELL as would be 'in candle' from tick data Sell would use Open from Next candle. So in a draw Stop would be hit first on ticker data in live - + Validity of when types of trades may be executed can be summarised as: - + Tick View index index Buy Sell open low close high Stop price open 2am 94 -1 0 0 ----- ------ ------ ----- ----- @@ -644,28 +504,26 @@ class Edge(): open 4am 96 1 0 1 Enter trgstop trg sel ROI out Stop out open 5am 97 2 0 0 Exit ------ ------- ----- ----- open 6am 98 3 0 0 ----- ------ ------- ----- ----- - + -1 means not found till end of view i.e no valid Stop found. Exclude from match. Stop tiggering and closing in 96-1, the candle we bought at OPEN in, is valid. - + Buys and sells are triggered at candle close Both will open their postions at the open of the next candle. i/e + 1 index - + Stop and buy Indexes are on the view. To map to the ticker dataframe the t_open_ind index should be summed. - + np_t_stop_ind: Stop Found index in view t_exit_ind : Sell found in view t_open_ind : Where view was started on ticker_data - + TODO: fix this frig for logic test,, case/switch/dictionary would be better... more so when later testing many options, dynamic stop / roi etc cludge - Setting np_t_sell_ind as 9999999999 when -1 (not found) cludge - Setting np_t_stop_ind as 9999999999 when -1 (not found) - + ''' - if debug: - print("\n(5) numpy debug\nStop or Sell Logic Processing") # cludge for logic test (-1) means it was not found, set crazy high to lose < test np_t_sell_ind = 99999999 if np_t_sell_ind <= 0 else np_t_sell_ind @@ -676,9 +534,6 @@ class Edge(): t_exit_ind = t_open_ind + np_t_stop_ind # Set Exit row index t_exit_type = SellType.STOP_LOSS # Set Exit type (stop) np_t_exit_pri = np_sco # The price field our STOP exit will use - if debug: - print("Type STOP is first exit condition. " - "At view index:", np_t_stop_ind, ". Ticker data exit index is", t_exit_ind) # Buy = 1 found before a stoploss triggered elif np_t_sell_ind < 99999999 and np_t_sell_ind < np_t_stop_ind: @@ -687,104 +542,18 @@ class Edge(): t_exit_ind = t_open_ind + np_t_sell_ind # Set Exit row index t_exit_type = SellType.SELL_SIGNAL # Set Exit type (sell) np_t_exit_pri = np_open # The price field our SELL exit will use - if debug: - print("Type SELL is first exit condition. " - "At view index", np_t_sell_ind, ". Ticker data exit index is", t_exit_ind) # No stop or buy left in view - set t_exit_last -1 to handle gracefully else: t_exit_last: int = -1 # Signal loop to exit, no buys or sells found. t_exit_type = SellType.NONE np_t_exit_pri = 999 # field price should be calculated on. 999 a non-existent column - if debug: - print("No valid STOP or SELL found. Signalling t_exit_last to gracefully exit") # TODO: fix having to cludge/uncludge this .. # Undo cludge np_t_sell_ind = -1 if np_t_sell_ind == 99999999 else np_t_sell_ind np_t_stop_ind = -1 if np_t_stop_ind == 99999999 else np_t_stop_ind - if debug_timing: - t_t = f(st) - print("5-logic", str.format('{0:.17f}', t_t)) - st = s() - - if debug: - ''' - Print out the buys, stops, sells - Include Line before and after to for easy - Human verification - ''' - # Combine the np_t_stop_pri value to bslap dataframe to make debug - # life easy. This is the current stop price based on buy price_ - # This is slow but don't care about performance in debug - # - # When referencing equiv np_column, as example np_sto, its 5 in numpy and 6 in df, so +1 - # as there is no data column in the numpy array. - bslap['np_stop_pri'] = np_t_stop_pri - - # Buy - print("\n\nDATAFRAME DEBUG =================== BUY ", pair) - print("Numpy Array BUY Index is:", 0) - print("DataFrame BUY Index is:", t_open_ind, "displaying DF \n") - print("HINT, BUY trade should use OPEN price from next candle, i.e ", t_open_ind + 1) - op_is = t_open_ind - 1 # Print open index start, line before - op_if = t_open_ind + 3 # Print open index finish, line after - print(bslap.iloc[op_is:op_if], "\n") - - # Stop - Stops trigger price np_sto (+1 for pandas column), and price received np_sco +1. (Stop Trigger|Calculated On) - if np_t_stop_ind < 0: - print("DATAFRAME DEBUG =================== STOP ", pair) - print("No STOPS were found until the end of ticker data file\n") - else: - print("DATAFRAME DEBUG =================== STOP ", pair) - print("Numpy Array STOP Index is:", np_t_stop_ind, "View starts at index", t_open_ind) - df_stop_index = (t_open_ind + np_t_stop_ind) - - print("DataFrame STOP Index is:", df_stop_index, "displaying DF \n") - print("First Stoploss trigger after Trade entered at OPEN in candle", t_open_ind + 1, "is ", - df_stop_index, ": \n", - str.format('{0:.17f}', bslap.iloc[df_stop_index][np_sto + 1]), - "is less than", str.format('{0:.17f}', np_t_stop_pri)) - - print("A stoploss exit will be calculated at rate:", - str.format('{0:.17f}', bslap.iloc[df_stop_index][np_sco + 1])) - - print("\nHINT, STOPs should exit in-candle, i.e", df_stop_index, - ": As live STOPs are not linked to O-C times") - - st_is = df_stop_index - 1 # Print stop index start, line before - st_if = df_stop_index + 2 # Print stop index finish, line after - print(bslap.iloc[st_is:st_if], "\n") - - # Sell - if np_t_sell_ind < 0: - print("DATAFRAME DEBUG =================== SELL ", pair) - print("No SELLS were found till the end of ticker data file\n") - else: - print("DATAFRAME DEBUG =================== SELL ", pair) - print("Numpy View SELL Index is:", np_t_sell_ind, "View starts at index", t_open_ind) - df_sell_index = (t_open_ind + np_t_sell_ind) - - print("DataFrame SELL Index is:", df_sell_index, "displaying DF \n") - print("First Sell Index after Trade open is in candle", df_sell_index) - print("HINT, if exit is SELL (not stop) trade should use OPEN price from next candle", - df_sell_index + 1) - sl_is = df_sell_index - 1 # Print sell index start, line before - sl_if = df_sell_index + 3 # Print sell index finish, line after - print(bslap.iloc[sl_is:sl_if], "\n") - - # Chosen Exit (stop or sell) - - print("DATAFRAME DEBUG =================== EXIT ", pair) - print("Exit type is :", t_exit_type) - print("trade exit price field is", np_t_exit_pri, "\n") - - if debug_timing: - t_t = f(st) - print("6-depra", str.format('{0:.17f}', t_t)) - st = s() - ## use numpy view "np_t_open_v" for speed. Columns are # buy 0 - open 1 - close 2 - sell 3 - high 4 - low 5 # exception is 6 which is use the stop value. @@ -804,43 +573,19 @@ class Edge(): if t_exit_type == SellType.NONE: np_trade_exit_price = 0 - if debug_timing: - t_t = f(st) - print("7-numpy", str.format('{0:.17f}', t_t)) - st = s() - - if debug: - print("//////////////////////////////////////////////") - print("+++++++++++++++++++++++++++++++++ Trade Enter ") - print("np_trade Enter Price is ", str.format('{0:.17f}', np_trade_enter_price)) - print("--------------------------------- Trade Exit ") - print("Trade Exit Type is ", t_exit_type) - print("np_trade Exit Price is", str.format('{0:.17f}', np_trade_exit_price)) - print("//////////////////////////////////////////////") - else: # no buys were found, step 0 returned -1 # Gracefully exit the loop t_exit_last == -1 - if debug: - print("\n(E) No buys were found in remaining ticker file. Exiting", pair) # Loop control - catch no closed trades. - if debug: - print("---------------------------------------- end of loop", loop, - " Dataframe Exit Index is: ", t_exit_ind) - print("Exit Index Last, Exit Index Now Are: ", t_exit_last, t_exit_ind) - if t_exit_last >= t_exit_ind or t_exit_last == -1: """ Break loop and go on to next pair. - + When last trade exit equals index of last exit, there is no opportunity to close any more trades. """ # TODO :add handing here to record none closed open trades - - if debug: - print(bslap_pair_results) break else: """ @@ -880,10 +625,6 @@ class Edge(): if t_exit_type is SellType.STOP_LOSS: stop_stops_count = stop_stops_count + 1 - if debug: - print("The trade dict is: \n", bslap_result) - print("Trades dicts in list after append are: \n ", bslap_pair_results) - """ Loop back to start. t_exit_last becomes where loop will seek to open new trades from. @@ -891,23 +632,16 @@ class Edge(): """ t_exit_last = t_exit_ind + 1 - if debug_timing: - t_t = f(st) - print("8+trade", str.format('{0:.17f}', t_t)) - # Send back List of trade dicts return bslap_pair_results def stake_amount(self, pair: str) -> str: info = [x for x in self._cached_pairs if x[0] == pair][0] stoploss = info[1] - allowed_capital_at_risk = round(self._total_capital * self._allowed_risk, 5) position_size = abs(round((allowed_capital_at_risk / stoploss), 5)) + return position_size - return (allowed_dollars_at_risk / symbol_strategy_stop_loss) - - def stoploss(self, pair: str) -> float: info = [x for x in self._cached_pairs if x[0] == pair][0] return info[1] \ No newline at end of file From 87df4e455663e0ac64030ada8bc6fd70d6b7dcbd Mon Sep 17 00:00:00 2001 From: misagh Date: Wed, 26 Sep 2018 15:20:53 +0200 Subject: [PATCH 066/699] refactoring backslap (round 2) --- freqtrade/edge/__init__.py | 509 +++----------- freqtrade/optimize/backslapping.py | 808 ---------------------- freqtrade/optimize/edge.py | 1037 ---------------------------- 3 files changed, 109 insertions(+), 2245 deletions(-) delete mode 100644 freqtrade/optimize/backslapping.py delete mode 100644 freqtrade/optimize/edge.py diff --git a/freqtrade/edge/__init__.py b/freqtrade/edge/__init__.py index 7a45b7b78..7a0b715ab 100644 --- a/freqtrade/edge/__init__.py +++ b/freqtrade/edge/__init__.py @@ -16,9 +16,7 @@ from freqtrade.strategy.resolver import IStrategy, StrategyResolver from freqtrade.optimize.backtesting import Backtesting import numpy as np -import timeit import utils_find_1st as utf1st -import pdb logger = logging.getLogger(__name__) @@ -60,40 +58,6 @@ class Edge(): self.fee = self.exchange.get_fee() - self.stop_loss_value = self.strategy.stoploss - - #### backslap config - ''' - Numpy arrays are used for 100x speed up - We requires setting Int values for - buy stop triggers and stop calculated on - # buy 0 - open 1 - close 2 - sell 3 - high 4 - low 5 - stop 6 - ''' - self.np_buy: int = 0 - self.np_open: int = 1 - self.np_close: int = 2 - self.np_sell: int = 3 - self.np_high: int = 4 - self.np_low: int = 5 - self.np_stop: int = 6 - self.np_bto: int = self.np_close # buys_triggered_on - should be close - self.np_bco: int = self.np_open # buys calculated on - open of the next candle. - self.np_sto: int = self.np_low # stops_triggered_on - Should be low, FT uses close - self.np_sco: int = self.np_stop # stops_calculated_on - Should be stop, FT uses close - # self.np_sto: int = self.np_close # stops_triggered_on - Should be low, FT uses close - # self.np_sco: int = self.np_close # stops_calculated_on - Should be stop, FT uses close - - self.debug = False # Main debug enable, very print heavy, enable 2 loops recommended - self.debug_timing = False # Stages within Backslap - self.debug_2loops = False # Limit each pair to two loops, useful when debugging - self.debug_vector = False # Debug vector calcs - self.debug_timing_main_loop = False # print overall timing per pair - works in Backtest and Backslap - - self.backslap_show_trades = False # prints trades in addition to summary report - self.backslap_save_trades = True # saves trades as a pretty table to backslap.txt - - self.stop_stops: int = 9999 # stop back testing any pair with this many stops, set to 999999 to not hit - def calculate(self) -> bool: pairs = self.config['exchange']['pair_whitelist'] heartbeat = self.config['edge']['process_throttle_secs'] @@ -102,7 +66,6 @@ class Edge(): return False data = {} - logger.info('Using stake_currency: %s ...', self.config['stake_currency']) logger.info('Using stake_amount: %s ...', self.config['stake_amount']) logger.info('Using local backtesting data (using whitelist in given config) ...') @@ -141,7 +104,7 @@ class Edge(): stoploss_range = np.arange(stoploss_range_min, stoploss_range_max, stoploss_range_step) ########################### Call out BSlap Loop instead of Original BT code - bslap_results: list = [] + trades: list = [] for pair, pair_data in preprocessed.items(): # Sorting dataframe by date and reset index pair_data = pair_data.sort_values(by=['date']) @@ -150,33 +113,44 @@ class Edge(): ticker_data = self.populate_sell_trend( self.populate_buy_trend(pair_data))[headers].copy() - # call backslap - results are a list of dicts - for stoploss in stoploss_range: - bslap_results += self.backslap_pair(ticker_data, pair, round(stoploss, 6)) + trades += self._find_trades_for_stoploss_range(ticker_data, pair, stoploss_range) + - # Switch List of Trade Dicts (bslap_results) to Dataframe + # Switch List of Trade Dicts (trades) to Dataframe # Fill missing, calculable columns, profit, duration , abs etc. - bslap_results_df = DataFrame(bslap_results) + trades_df = DataFrame(trades) - if len(bslap_results_df) > 0: # Only post process a frame if it has a record - bslap_results_df = self.vector_fill_results_table(bslap_results_df) + if len(trades_df) > 0: # Only post process a frame if it has a record + trades_df = self._fill_calculable_fields(trades_df) else: - bslap_results_df = [] - bslap_results_df = DataFrame.from_records(bslap_results_df, columns=BacktestResult._fields) + trades_df = [] + trades_df = DataFrame.from_records(trades_df, columns=BacktestResult._fields) - self._cached_pairs = self._process_result(data, bslap_results_df, stoploss_range) + + self._cached_pairs = self._process_expectancy(trades_df) self._last_updated = arrow.utcnow().timestamp return True + def stake_amount(self, pair: str) -> str: + info = [x for x in self._cached_pairs if x[0] == pair][0] + stoploss = info[1] + allowed_capital_at_risk = round(self._total_capital * self._allowed_risk, 5) + position_size = abs(round((allowed_capital_at_risk / stoploss), 5)) + return position_size + + def stoploss(self, pair: str) -> float: + info = [x for x in self._cached_pairs if x[0] == pair][0] + return info[1] + def sort_pairs(self, pairs) -> bool: if len(self._cached_pairs) == 0: self.calculate() edge_sorted_pairs = [x[0] for x in self._cached_pairs] return [x for _, x in sorted(zip(edge_sorted_pairs,pairs), key=lambda pair: pair[0])] - def vector_fill_results_table(self, bslap_results_df: DataFrame): + def _fill_calculable_fields(self, result: DataFrame): """ - The Results frame contains a number of columns that are calculable + The result frame contains a number of columns that are calculable from othe columns. These are left blank till all rows are added, to be populated in single vector calls. @@ -184,8 +158,8 @@ class Edge(): - Profit - trade duration - profit abs - :param bslap_results Dataframe - :return: bslap_results Dataframe + :param result Dataframe + :return: result Dataframe """ # stake and fees @@ -198,63 +172,30 @@ class Edge(): open_fee = fee / 2 close_fee = fee / 2 - bslap_results_df['trade_duration'] = bslap_results_df['close_time'] - bslap_results_df['open_time'] - bslap_results_df['trade_duration'] = bslap_results_df['trade_duration'].map(lambda x: int(x.total_seconds() / 60)) + result['trade_duration'] = result['close_time'] - result['open_time'] + result['trade_duration'] = result['trade_duration'].map(lambda x: int(x.total_seconds() / 60)) ## Spends, Takes, Profit, Absolute Profit - # print(bslap_results_df) + # Buy Price - bslap_results_df['buy_vol'] = stake / bslap_results_df['open_rate'] # How many target are we buying - bslap_results_df['buy_fee'] = stake * open_fee - bslap_results_df['buy_spend'] = stake + bslap_results_df['buy_fee'] # How much we're spending + result['buy_vol'] = stake / result['open_rate'] # How many target are we buying + result['buy_fee'] = stake * open_fee + result['buy_spend'] = stake + result['buy_fee'] # How much we're spending # Sell price - bslap_results_df['sell_sum'] = bslap_results_df['buy_vol'] * bslap_results_df['close_rate'] - bslap_results_df['sell_fee'] = bslap_results_df['sell_sum'] * close_fee - bslap_results_df['sell_take'] = bslap_results_df['sell_sum'] - bslap_results_df['sell_fee'] + result['sell_sum'] = result['buy_vol'] * result['close_rate'] + result['sell_fee'] = result['sell_sum'] * close_fee + result['sell_take'] = result['sell_sum'] - result['sell_fee'] + # profit_percent - bslap_results_df['profit_percent'] = (bslap_results_df['sell_take'] - bslap_results_df['buy_spend']) \ - / bslap_results_df['buy_spend'] + result['profit_percent'] = (result['sell_take'] - result['buy_spend']) \ + / result['buy_spend'] # Absolute profit - bslap_results_df['profit_abs'] = bslap_results_df['sell_take'] - bslap_results_df['buy_spend'] + result['profit_abs'] = result['sell_take'] - result['buy_spend'] - return bslap_results_df + return result - def np_get_t_open_ind(self, np_buy_arr, t_exit_ind: int, np_buy_arr_len: int, stop_stops: int, - stop_stops_count: int): - """ - The purpose of this def is to return the next "buy" = 1 - after t_exit_ind. - This function will also check is the stop limit for the pair has been reached. - if stop_stops is the limit and stop_stops_count it the number of times the stop has been hit. - t_exit_ind is the index the last trade exited on - or 0 if first time around this loop. - stop_stops i - """ - t_open_ind: int - - """ - Create a view on our buy index starting after last trade exit - Search for next buy - """ - np_buy_arr_v = np_buy_arr[t_exit_ind:] - t_open_ind = utf1st.find_1st(np_buy_arr_v, 1, utf1st.cmp_equal) - - ''' - If -1 is returned no buy has been found, preserve the value - ''' - if t_open_ind != -1: # send back the -1 if no buys found. otherwise update index - t_open_ind = t_open_ind + t_exit_ind # Align numpy index - - if t_open_ind == np_buy_arr_len - 1: # If buy found on last candle ignore, there is no OPEN in next to use - t_open_ind = -1 # -1 ends the loop - - if stop_stops_count >= stop_stops: # if maximum number of stops allowed in a pair is hit, exit loop - t_open_ind = -1 # -1 ends the loop - - return t_open_ind - - def _process_result(self, data: Dict[str, Dict], results: DataFrame, stoploss_range) -> str: + def _process_expectancy(self, results: DataFrame) -> str: """ This is a temporary version of edge positioning calculation. The function will be eventually moved to a plugin called Edge in order to calculate necessary WR, RRR and @@ -262,10 +203,6 @@ class Edge(): The calulation will be done per pair and per strategy. """ - # Removing open trades from dataset - results = results[results.open_at_end == False] - ################################### - # Removing pairs having less than min_trades_number min_trades_number = self.edge_config.get('min_trade_number', 15) results = results.groupby('pair').filter(lambda x: len(x) > min_trades_number) @@ -335,313 +272,85 @@ class Edge(): # Returning an array of pairs in order of "expectancy" return final.reset_index().values - def backslap_pair(self, ticker_data, pair, stoploss): - # Read Stop Loss Values and Stake - stop = stoploss - p_stop = (stop + 1) # What stop really means, e.g 0.01 is 0.99 of price + def _find_trades_for_stoploss_range(self, ticker_data, pair, stoploss_range): + buy_column = ticker_data['buy'].values + sell_column = ticker_data['sell'].values + date_column = ticker_data['date'].values + ohlc_columns = ticker_data[['open', 'high', 'low', 'close']].values + + result: list = [] + for stoploss in stoploss_range: + result += self._detect_stop_and_sell_points(buy_column, sell_column, date_column, ohlc_columns, round(stoploss, 6), pair) - #### backslap config - ''' - Numpy arrays are used for 100x speed up - We requires setting Int values for - buy stop triggers and stop calculated on - # buy 0 - open 1 - close 2 - sell 3 - high 4 - low 5 - stop 6 - ''' + return result - ####### - # Use vars set at top of backtest - np_open: int = self.np_open - np_sell: int = self.np_sell - np_bco: int = self.np_bco # buys calculated on - open of the next candle. - np_sto: int = self.np_sto # stops_triggered_on - Should be low, FT uses close - np_sco: int = self.np_sco # stops_calculated_on - Should be stop, FT uses close + def _detect_stop_and_sell_points(self, buy_column, sell_column, date_column, ohlc_columns, stoploss, pair, start_point=0): + result: list = [] + open_trade_index = utf1st.find_1st(buy_column, 1, utf1st.cmp_equal) + #open_trade_index = np.argmax(buy_column == 1) - # ticker_data: DataFrame = ticker_dfs[t_file] - bslap: DataFrame = ticker_data + # return empty if we don't find trade entry (i.e. buy==1) + if open_trade_index == -1: + return [] - # Build a single dimension numpy array from "buy" index for faster search - # (500x faster than pandas) - np_buy_arr = bslap['buy'].values - np_buy_arr_len: int = len(np_buy_arr) + stop_price_percentage = stoploss + 1 + open_price = ohlc_columns[open_trade_index + 1, 0] + stop_price = (open_price * stop_price_percentage) - # use numpy array for faster searches in loop, 20x faster than pandas - # buy 0 - open 1 - close 2 - sell 3 - high 4 - low 5 - np_bslap = np.array(bslap[['buy', 'open', 'close', 'sell', 'high', 'low']]) + # Searching for the index where stoploss is hit + stop_index = utf1st.find_1st(ohlc_columns[open_trade_index + 1:, 2], stop_price, utf1st.cmp_smaller) - # Build a numpy list of date-times. - # We use these when building the trade - # The rationale is to address a value from a pandas cell is thousands of - # times more expensive. Processing time went X25 when trying to use any data from pandas - np_bslap_dates = bslap['date'].values + # If we don't find it then we assume stop_index will be far in future (infinite number) + if stop_index == -1: + stop_index = float('inf') - loop: int = 0 # how many time around the loop - t_exit_ind = 0 # Start loop from first index - t_exit_last = 0 # To test for exit + #stop_index = np.argmax((ohlc_columns[open_trade_index + 1:, 2] < stop_price) == True) - stop_stops = self.stop_stops # Int of stops within a pair to stop trading a pair at - stop_stops_count = 0 # stop counter per pair + # Searching for the index where sell is hit + sell_index = utf1st.find_1st(sell_column[open_trade_index + 1:], 1, utf1st.cmp_equal) - # Results will be stored in a list of dicts - bslap_pair_results: list = [] - bslap_result: dict = {} + # If we don't find it then we assume sell_index will be far in future (infinite number) + if sell_index == -1: + sell_index = float('inf') - while t_exit_ind < np_buy_arr_len: - loop = loop + 1 - ''' - Dev phases - Phase 1 - 1) Manage buy, sell, stop enter/exit - a) Find first buy index - b) Discover first stop and sell hit after buy index - c) Chose first instance as trade exit + #sell_index = np.argmax(sell_column[open_trade_index + 1:] == 1) - Phase 2 - 2) Manage dynamic Stop and ROI Exit - a) Create trade slice from 1 - b) search within trade slice for dynamice stop hit - c) search within trade slice for ROI hit - ''' + # Check if we don't find any stop or sell point (in that case trade remains open) + # It is not interesting for Edge to consider it so we simply ignore the trade + # And stop iterating as the party is over + if stop_index == sell_index == float('inf'): + return [] - ''' - 0 - Find next buy entry - Finds index for first (buy = 1) flag + if stop_index <= sell_index: + exit_index = open_trade_index + stop_index + 1 + exit_type = SellType.STOP_LOSS + exit_price = stop_price + elif stop_index > sell_index: + exit_index = open_trade_index + sell_index + 1 + exit_type = SellType.SELL_SIGNAL + exit_price = ohlc_columns[open_trade_index + sell_index + 1, 0] - Requires: np_buy_arr - a 1D array of the 'buy' column. To find next "1" - Required: t_exit_ind - Either 0, first loop. Or The index we last exited on - Requires: np_buy_arr_len - length of pair array. - Requires: stops_stops - number of stops allowed before stop trading a pair - Requires: stop_stop_counts - count of stops hit in the pair - Provides: The next "buy" index after t_exit_ind + trade = {} + trade["pair"] = pair + trade["stoploss"] = stoploss + trade["profit_percent"] = "" # To be 1 vector calculation across trades when loop complete + trade["profit_abs"] = "" # To be 1 vector calculation across trades when loop complete + trade["open_time"] = date_column[open_trade_index] + trade["close_time"] = date_column[exit_index] + trade["open_index"] = start_point + open_trade_index + 1 # +1 as we buy on next. + trade["close_index"] = start_point + exit_index + trade["trade_duration"] = "" # To be 1 vector calculation across trades when loop complete + trade["open_rate"] = round(open_price, 15) + trade["close_rate"] = round(exit_price, 15) + trade["exit_type"] = exit_type + result.append(trade) - If -1 is returned no buy has been found in remainder of array, skip to exit loop - ''' - t_open_ind = self.np_get_t_open_ind(np_buy_arr, t_exit_ind, np_buy_arr_len, stop_stops, stop_stops_count) - - if t_open_ind != -1: - - """ - 1 - Create views to search within for our open trade - The views are our search space for the next Stop or Sell - Numpy view is employed as: - 1,000 faster than pandas searches - Pandas cannot assure it will always return a view, it may make a slow copy. - The view contains columns: - buy 0 - open 1 - close 2 - sell 3 - high 4 - low 5 - - Requires: np_bslap is our numpy array of the ticker DataFrame - Requires: t_open_ind is the index row with the buy. - Provides: np_t_open_v View of array after buy. - Provides: np_t_open_v_stop View of array after buy +1 - (Stop will search in here to prevent stopping in the past) - """ - np_t_open_v = np_bslap[t_open_ind:] - np_t_open_v_stop = np_bslap[t_open_ind + 1:] - - ''' - 2 - Calculate our stop-loss price - - As stop is based on buy price of our trade - - (BTO)Buys are Triggered On np_bto, typically the CLOSE of candle - - (BCO)Buys are Calculated On np_bco, default is OPEN of the next candle. - This is as we only see the CLOSE after it has happened. - The back test assumption is we have bought at first available price, the OPEN - - Requires: np_bslap - is our numpy array of the ticker DataFrame - Requires: t_open_ind - is the index row with the first buy. - Requires: p_stop - is the stop rate, ie. 0.99 is -1% - Provides: np_t_stop_pri - The value stop-loss will be triggered on - ''' - np_t_stop_pri = (np_bslap[t_open_ind + 1, np_bco] * p_stop) - - ''' - 3 - Find candle STO is under Stop-Loss After Trade opened. - - where [np_sto] (stop tiggered on variable: "close", "low" etc) < np_t_stop_pri - - Requires: np_t_open_v_stop Numpy view of ticker_data after buy row +1 (when trade was opened) - Requires: np_sto User Var(STO)StopTriggeredOn. Typically set to "low" or "close" - Requires: np_t_stop_pri The stop-loss price STO must fall under to trigger stop - Provides: np_t_stop_ind The first candle after trade open where STO is under stop-loss - ''' - np_t_stop_ind = utf1st.find_1st(np_t_open_v_stop[:, np_sto], - np_t_stop_pri, - utf1st.cmp_smaller) - - # plus 1 as np_t_open_v_stop is 1 ahead of view np_t_open_v, used from here on out. - np_t_stop_ind = np_t_stop_ind + 1 - - ''' - 4 - Find first sell index after trade open - - First index in the view np_t_open_v where ['sell'] = 1 - - Requires: np_t_open_v - view of ticker_data from buy onwards - Requires: no_sell - integer '3', the buy column in the array - Provides: np_t_sell_ind index of view where first sell=1 after buy - ''' - # Use numpy array for faster search for sell - # Sell uses column 3. - # buy 0 - open 1 - close 2 - sell 3 - high 4 - low 5 - # Numpy searches 25-35x quicker than pandas on this data - - np_t_sell_ind = utf1st.find_1st(np_t_open_v[:, np_sell], - 1, utf1st.cmp_equal) - - ''' - 5 - Determine which was hit first a stop or sell - To then use as exit index price-field (sell on buy, stop on stop) - - STOP takes priority over SELL as would be 'in candle' from tick data - Sell would use Open from Next candle. - So in a draw Stop would be hit first on ticker data in live - - Validity of when types of trades may be executed can be summarised as: - - Tick View - index index Buy Sell open low close high Stop price - open 2am 94 -1 0 0 ----- ------ ------ ----- ----- - open 3am 95 0 1 0 ----- ------ trg buy ----- ----- - open 4am 96 1 0 1 Enter trgstop trg sel ROI out Stop out - open 5am 97 2 0 0 Exit ------ ------- ----- ----- - open 6am 98 3 0 0 ----- ------ ------- ----- ----- - - -1 means not found till end of view i.e no valid Stop found. Exclude from match. - Stop tiggering and closing in 96-1, the candle we bought at OPEN in, is valid. - - Buys and sells are triggered at candle close - Both will open their postions at the open of the next candle. i/e + 1 index - - Stop and buy Indexes are on the view. To map to the ticker dataframe - the t_open_ind index should be summed. - - np_t_stop_ind: Stop Found index in view - t_exit_ind : Sell found in view - t_open_ind : Where view was started on ticker_data - - TODO: fix this frig for logic test,, case/switch/dictionary would be better... - more so when later testing many options, dynamic stop / roi etc - cludge - Setting np_t_sell_ind as 9999999999 when -1 (not found) - cludge - Setting np_t_stop_ind as 9999999999 when -1 (not found) - - ''' - - # cludge for logic test (-1) means it was not found, set crazy high to lose < test - np_t_sell_ind = 99999999 if np_t_sell_ind <= 0 else np_t_sell_ind - np_t_stop_ind = 99999999 if np_t_stop_ind <= 0 else np_t_stop_ind - - # Stoploss trigger found before a sell =1 - if np_t_stop_ind < 99999999 and np_t_stop_ind <= np_t_sell_ind: - t_exit_ind = t_open_ind + np_t_stop_ind # Set Exit row index - t_exit_type = SellType.STOP_LOSS # Set Exit type (stop) - np_t_exit_pri = np_sco # The price field our STOP exit will use - - # Buy = 1 found before a stoploss triggered - elif np_t_sell_ind < 99999999 and np_t_sell_ind < np_t_stop_ind: - # move sell onto next candle, we only look back on sell - # will use the open price later. - t_exit_ind = t_open_ind + np_t_sell_ind # Set Exit row index - t_exit_type = SellType.SELL_SIGNAL # Set Exit type (sell) - np_t_exit_pri = np_open # The price field our SELL exit will use - - # No stop or buy left in view - set t_exit_last -1 to handle gracefully - else: - t_exit_last: int = -1 # Signal loop to exit, no buys or sells found. - t_exit_type = SellType.NONE - np_t_exit_pri = 999 # field price should be calculated on. 999 a non-existent column - - # TODO: fix having to cludge/uncludge this .. - # Undo cludge - np_t_sell_ind = -1 if np_t_sell_ind == 99999999 else np_t_sell_ind - np_t_stop_ind = -1 if np_t_stop_ind == 99999999 else np_t_stop_ind - - ## use numpy view "np_t_open_v" for speed. Columns are - # buy 0 - open 1 - close 2 - sell 3 - high 4 - low 5 - # exception is 6 which is use the stop value. - - # TODO no! this is hard coded bleh fix this open - np_trade_enter_price = np_bslap[t_open_ind + 1, np_open] - if t_exit_type == SellType.STOP_LOSS: - if np_t_exit_pri == 6: - np_trade_exit_price = np_t_stop_pri - t_exit_ind = t_exit_ind + 1 - else: - np_trade_exit_price = np_bslap[t_exit_ind, np_t_exit_pri] - if t_exit_type == SellType.SELL_SIGNAL: - np_trade_exit_price = np_bslap[t_exit_ind, np_t_exit_pri] - - # Catch no exit found - if t_exit_type == SellType.NONE: - np_trade_exit_price = 0 - - else: # no buys were found, step 0 returned -1 - # Gracefully exit the loop - t_exit_last == -1 - - # Loop control - catch no closed trades. - if t_exit_last >= t_exit_ind or t_exit_last == -1: - """ - Break loop and go on to next pair. - - When last trade exit equals index of last exit, there is no - opportunity to close any more trades. - """ - # TODO :add handing here to record none closed open trades - break - else: - """ - Add trade to backtest looking results list of dicts - Loop back to look for more trades. - """ - - # We added +1 to t_exit_ind if the exit was a stop-loss, to not exit early in the IF of this ELSE - # removing the +1 here so prices match. - if t_exit_type == SellType.STOP_LOSS: - t_exit_ind = t_exit_ind - 1 - - # Build trade dictionary - ## In general if a field can be calculated later from other fields leave blank here - ## Its X(number of trades faster) to calc all in a single vector than 1 trade at a time - - # create a new dict - close_index: int = t_exit_ind - bslap_result = {} # Must have at start or we end up with a list of multiple same last result - bslap_result["pair"] = pair - bslap_result["stoploss"] = stop - bslap_result["profit_percent"] = "" # To be 1 vector calc across trades when loop complete - bslap_result["profit_abs"] = "" # To be 1 vector calc across trades when loop complete - bslap_result["open_time"] = np_bslap_dates[t_open_ind + 1] # use numpy array, pandas 20x slower - bslap_result["close_time"] = np_bslap_dates[close_index] # use numpy array, pandas 20x slower - bslap_result["open_index"] = t_open_ind + 1 # +1 as we buy on next. - bslap_result["close_index"] = close_index - bslap_result["trade_duration"] = "" # To be 1 vector calc across trades when loop complete - bslap_result["open_at_end"] = False - bslap_result["open_rate"] = round(np_trade_enter_price, 15) - bslap_result["close_rate"] = round(np_trade_exit_price, 15) - bslap_result["exit_type"] = t_exit_type - bslap_result["sell_reason"] = t_exit_type #duplicated, but I don't care - # append the dict to the list and print list - bslap_pair_results.append(bslap_result) - - if t_exit_type is SellType.STOP_LOSS: - stop_stops_count = stop_stops_count + 1 - - """ - Loop back to start. t_exit_last becomes where loop - will seek to open new trades from. - Push index on 1 to not open on close - """ - t_exit_last = t_exit_ind + 1 - - # Send back List of trade dicts - return bslap_pair_results - - def stake_amount(self, pair: str) -> str: - info = [x for x in self._cached_pairs if x[0] == pair][0] - stoploss = info[1] - allowed_capital_at_risk = round(self._total_capital * self._allowed_risk, 5) - position_size = abs(round((allowed_capital_at_risk / stoploss), 5)) - return position_size - - def stoploss(self, pair: str) -> float: - info = [x for x in self._cached_pairs if x[0] == pair][0] - return info[1] \ No newline at end of file + return result + self._detect_stop_and_sell_points( + buy_column[exit_index:], + sell_column[exit_index:], + date_column[exit_index:], + ohlc_columns[exit_index:], + stoploss, + pair, + (start_point + exit_index) + ) diff --git a/freqtrade/optimize/backslapping.py b/freqtrade/optimize/backslapping.py deleted file mode 100644 index 7eba39a6e..000000000 --- a/freqtrade/optimize/backslapping.py +++ /dev/null @@ -1,808 +0,0 @@ -import timeit -from typing import Dict, Any - -from pandas import DataFrame - -from freqtrade.exchange import Exchange -from freqtrade.strategy import IStrategy -from freqtrade.strategy.interface import SellType -from freqtrade.strategy.resolver import StrategyResolver -import pdb - -class Backslapping: - """ - provides a quick way to evaluate strategies over a longer term of time - """ - - def __init__(self, config: Dict[str, Any], exchange = None) -> None: - """ - constructor - """ - - self.config = config - self.strategy: IStrategy = StrategyResolver(self.config).strategy - self.ticker_interval = self.strategy.ticker_interval - self.tickerdata_to_dataframe = self.strategy.tickerdata_to_dataframe - self.populate_buy_trend = self.strategy.populate_buy_trend - self.populate_sell_trend = self.strategy.populate_sell_trend - - ### - # - ### - if exchange is None: - self.config['exchange']['secret'] = '' - self.config['exchange']['password'] = '' - self.config['exchange']['uid'] = '' - self.config['dry_run'] = True - self.exchange = Exchange(self.config) - else: - self.exchange = exchange - - self.fee = self.exchange.get_fee() - - self.stop_loss_value = self.strategy.stoploss - - #### backslap config - ''' - Numpy arrays are used for 100x speed up - We requires setting Int values for - buy stop triggers and stop calculated on - # buy 0 - open 1 - close 2 - sell 3 - high 4 - low 5 - stop 6 - ''' - self.np_buy: int = 0 - self.np_open: int = 1 - self.np_close: int = 2 - self.np_sell: int = 3 - self.np_high: int = 4 - self.np_low: int = 5 - self.np_stop: int = 6 - self.np_bto: int = self.np_close # buys_triggered_on - should be close - self.np_bco: int = self.np_open # buys calculated on - open of the next candle. - self.np_sto: int = self.np_low # stops_triggered_on - Should be low, FT uses close - self.np_sco: int = self.np_stop # stops_calculated_on - Should be stop, FT uses close - # self.np_sto: int = self.np_close # stops_triggered_on - Should be low, FT uses close - # self.np_sco: int = self.np_close # stops_calculated_on - Should be stop, FT uses close - - self.debug = False # Main debug enable, very print heavy, enable 2 loops recommended - self.debug_timing = False # Stages within Backslap - self.debug_2loops = False # Limit each pair to two loops, useful when debugging - self.debug_vector = False # Debug vector calcs - self.debug_timing_main_loop = False # print overall timing per pair - works in Backtest and Backslap - - self.backslap_show_trades = False # prints trades in addition to summary report - self.backslap_save_trades = True # saves trades as a pretty table to backslap.txt - - self.stop_stops: int = 9999 # stop back testing any pair with this many stops, set to 999999 to not hit - - def s(self): - st = timeit.default_timer() - return st - - def f(self, st): - return (timeit.default_timer() - st) - - def run(self,args): - - headers = ['date', 'buy', 'open', 'close', 'sell', 'high', 'low'] - processed = args['processed'] - max_open_trades = args.get('max_open_trades', 0) - realistic = args.get('realistic', False) - trades = [] - trade_count_lock: Dict = {} - - ########################### Call out BSlap Loop instead of Original BT code - bslap_results: list = [] - for pair, pair_data in processed.items(): - if self.debug_timing: # Start timer - fl = self.s() - - ticker_data = self.populate_sell_trend( - self.populate_buy_trend(pair_data))[headers].copy() - - if self.debug_timing: # print time taken - flt = self.f(fl) - # print("populate_buy_trend:", pair, round(flt, 10)) - st = self.s() - - # #dump same DFs to disk for offline testing in scratch - # f_pair:str = pair - # csv = f_pair.replace("/", "_") - # csv="/Users/creslin/PycharmProjects/freqtrade_new/frames/" + csv - # ticker_data.to_csv(csv, sep='\t', encoding='utf-8') - - # call bslap - results are a list of dicts - bslap_pair_results = self.backslap_pair(ticker_data, pair) - last_bslap_results = bslap_results - bslap_results = last_bslap_results + bslap_pair_results - - if self.debug_timing: # print time taken - tt = self.f(st) - print("Time to BackSlap :", pair, round(tt, 10)) - print("-----------------------") - - # Switch List of Trade Dicts (bslap_results) to Dataframe - # Fill missing, calculable columns, profit, duration , abs etc. - bslap_results_df = DataFrame(bslap_results) - - if len(bslap_results_df) > 0: # Only post process a frame if it has a record - # bslap_results_df['open_time'] = to_datetime(bslap_results_df['open_time']) - # bslap_results_df['close_time'] = to_datetime(bslap_results_df['close_time']) - # if debug: - # print("open_time and close_time converted to datetime columns") - - bslap_results_df = self.vector_fill_results_table(bslap_results_df, pair) - else: - from freqtrade.optimize.backtesting import BacktestResult - - bslap_results_df = [] - bslap_results_df = DataFrame.from_records(bslap_results_df, columns=BacktestResult._fields) - - return bslap_results_df - - def vector_fill_results_table(self, bslap_results_df: DataFrame, pair: str): - """ - The Results frame contains a number of columns that are calculable - from othe columns. These are left blank till all rows are added, - to be populated in single vector calls. - - Columns to be populated are: - - Profit - - trade duration - - profit abs - :param bslap_results Dataframe - :return: bslap_results Dataframe - """ - import pandas as pd - import numpy as np - debug = self.debug_vector - - # stake and fees - # stake = 0.015 - # 0.05% is 0.0005 - # fee = 0.001 - - stake = self.config.get('stake_amount') - fee = self.fee - open_fee = fee / 2 - close_fee = fee / 2 - - if debug: - print("Stake is,", stake, "the sum of currency to spend per trade") - print("The open fee is", open_fee, "The close fee is", close_fee) - if debug: - from pandas import set_option - set_option('display.max_rows', 5000) - set_option('display.max_columns', 20) - pd.set_option('display.width', 1000) - pd.set_option('max_colwidth', 40) - pd.set_option('precision', 12) - - # # Get before - # csv = "cryptosher_before_debug" - # bslap_results_df.to_csv(csv, sep='\t', encoding='utf-8') - - # bslap_results_df.to_csv(csv, sep='\t', encoding='utf-8') - - bslap_results_df['trade_duration'] = bslap_results_df['close_time'] - bslap_results_df['open_time'] - bslap_results_df['trade_duration'] = bslap_results_df['trade_duration'].map(lambda x: int(x.total_seconds() / 60)) - - ## Spends, Takes, Profit, Absolute Profit - # print(bslap_results_df) - # Buy Price - bslap_results_df['buy_vol'] = stake / bslap_results_df['open_rate'] # How many target are we buying - bslap_results_df['buy_fee'] = stake * open_fee - bslap_results_df['buy_spend'] = stake + bslap_results_df['buy_fee'] # How much we're spending - - # Sell price - bslap_results_df['sell_sum'] = bslap_results_df['buy_vol'] * bslap_results_df['close_rate'] - bslap_results_df['sell_fee'] = bslap_results_df['sell_sum'] * close_fee - bslap_results_df['sell_take'] = bslap_results_df['sell_sum'] - bslap_results_df['sell_fee'] - # profit_percent - bslap_results_df['profit_percent'] = (bslap_results_df['sell_take'] - bslap_results_df['buy_spend']) \ - / bslap_results_df['buy_spend'] - # Absolute profit - bslap_results_df['profit_abs'] = bslap_results_df['sell_take'] - bslap_results_df['buy_spend'] - - # # Get After - # csv="cryptosher_after_debug" - # bslap_results_df.to_csv(csv, sep='\t', encoding='utf-8') - - if debug: - print("\n") - print(bslap_results_df[ - ['buy_vol', 'buy_fee', 'buy_spend', 'sell_sum', 'sell_fee', 'sell_take', 'profit_percent', - 'profit_abs', 'exit_type']]) - - #pdb.set_trace() - return bslap_results_df - - def np_get_t_open_ind(self, np_buy_arr, t_exit_ind: int, np_buy_arr_len: int, stop_stops: int, - stop_stops_count: int): - import utils_find_1st as utf1st - """ - The purpose of this def is to return the next "buy" = 1 - after t_exit_ind. - - This function will also check is the stop limit for the pair has been reached. - if stop_stops is the limit and stop_stops_count it the number of times the stop has been hit. - - t_exit_ind is the index the last trade exited on - or 0 if first time around this loop. - - stop_stops i - """ - debug = self.debug - - # Timers, to be called if in debug - def s(): - st = timeit.default_timer() - return st - - def f(st): - return (timeit.default_timer() - st) - - st = s() - t_open_ind: int - - """ - Create a view on our buy index starting after last trade exit - Search for next buy - """ - np_buy_arr_v = np_buy_arr[t_exit_ind:] - t_open_ind = utf1st.find_1st(np_buy_arr_v, 1, utf1st.cmp_equal) - - ''' - If -1 is returned no buy has been found, preserve the value - ''' - if t_open_ind != -1: # send back the -1 if no buys found. otherwise update index - t_open_ind = t_open_ind + t_exit_ind # Align numpy index - - if t_open_ind == np_buy_arr_len - 1: # If buy found on last candle ignore, there is no OPEN in next to use - t_open_ind = -1 # -1 ends the loop - - if stop_stops_count >= stop_stops: # if maximum number of stops allowed in a pair is hit, exit loop - t_open_ind = -1 # -1 ends the loop - if debug: - print("Max stop limit ", stop_stops, "reached. Moving to next pair") - - return t_open_ind - - def backslap_pair(self, ticker_data, pair): - import pandas as pd - import numpy as np - import timeit - import utils_find_1st as utf1st - from datetime import datetime - - ### backslap debug wrap - # debug_2loops = False # only loop twice, for faster debug - # debug_timing = False # print timing for each step - # debug = False # print values, to check accuracy - debug_2loops = self.debug_2loops # only loop twice, for faster debug - debug_timing = self.debug_timing # print timing for each step - #debug = self.debug # print values, to check accuracy - debug = False - - ticker_data = ticker_data.sort_values(by=['date']) - ticker_data = ticker_data.reset_index(drop=True) - - #pandas_df = df.toPandas() - #pandas_df.to_json - - # Read Stop Loss Values and Stake - # pdb.set_trace() - stop = self.stop_loss_value - #stop = stoploss - p_stop = (stop + 1) # What stop really means, e.g 0.01 is 0.99 of price - - if debug: - print("Stop is ", stop, "value from stragey file") - print("p_stop is", p_stop, "value used to multiply to entry price") - - if debug: - from pandas import set_option - set_option('display.max_rows', 5000) - set_option('display.max_columns', 8) - pd.set_option('display.width', 1000) - pd.set_option('max_colwidth', 40) - pd.set_option('precision', 12) - - def s(): - st = timeit.default_timer() - return st - - def f(st): - return (timeit.default_timer() - st) - - #### backslap config - ''' - Numpy arrays are used for 100x speed up - We requires setting Int values for - buy stop triggers and stop calculated on - # buy 0 - open 1 - close 2 - sell 3 - high 4 - low 5 - stop 6 - ''' - - ####### - # Use vars set at top of backtest - np_buy: int = self.np_buy - np_open: int = self.np_open - np_close: int = self.np_close - np_sell: int = self.np_sell - np_high: int = self.np_high - np_low: int = self.np_low - np_stop: int = self.np_stop - np_bto: int = self.np_bto # buys_triggered_on - should be close - np_bco: int = self.np_bco # buys calculated on - open of the next candle. - np_sto: int = self.np_sto # stops_triggered_on - Should be low, FT uses close - np_sco: int = self.np_sco # stops_calculated_on - Should be stop, FT uses close - - ### End Config - - pair: str = pair - - # ticker_data: DataFrame = ticker_dfs[t_file] - bslap: DataFrame = ticker_data - - # Build a single dimension numpy array from "buy" index for faster search - # (500x faster than pandas) - np_buy_arr = bslap['buy'].values - np_buy_arr_len: int = len(np_buy_arr) - - # use numpy array for faster searches in loop, 20x faster than pandas - # buy 0 - open 1 - close 2 - sell 3 - high 4 - low 5 - np_bslap = np.array(bslap[['buy', 'open', 'close', 'sell', 'high', 'low']]) - - # Build a numpy list of date-times. - # We use these when building the trade - # The rationale is to address a value from a pandas cell is thousands of - # times more expensive. Processing time went X25 when trying to use any data from pandas - np_bslap_dates = bslap['date'].values - - loop: int = 0 # how many time around the loop - t_exit_ind = 0 # Start loop from first index - t_exit_last = 0 # To test for exit - - stop_stops = self.stop_stops # Int of stops within a pair to stop trading a pair at - stop_stops_count = 0 # stop counter per pair - - st = s() # Start timer for processing dataframe - if debug: - print('Processing:', pair) - - # Results will be stored in a list of dicts - bslap_pair_results: list = [] - bslap_result: dict = {} - - while t_exit_ind < np_buy_arr_len: - loop = loop + 1 - if debug or debug_timing: - print("-- T_exit_Ind - Numpy Index is", t_exit_ind, " ----------------------- Loop", loop, pair) - if debug_2loops: - if loop == 3: - print( - "++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++Loop debug max met - breaking") - break - ''' - Dev phases - Phase 1 - 1) Manage buy, sell, stop enter/exit - a) Find first buy index - b) Discover first stop and sell hit after buy index - c) Chose first instance as trade exit - - Phase 2 - 2) Manage dynamic Stop and ROI Exit - a) Create trade slice from 1 - b) search within trade slice for dynamice stop hit - c) search within trade slice for ROI hit - ''' - - if debug_timing: - st = s() - ''' - 0 - Find next buy entry - Finds index for first (buy = 1) flag - - Requires: np_buy_arr - a 1D array of the 'buy' column. To find next "1" - Required: t_exit_ind - Either 0, first loop. Or The index we last exited on - Requires: np_buy_arr_len - length of pair array. - Requires: stops_stops - number of stops allowed before stop trading a pair - Requires: stop_stop_counts - count of stops hit in the pair - Provides: The next "buy" index after t_exit_ind - - If -1 is returned no buy has been found in remainder of array, skip to exit loop - ''' - t_open_ind = self.np_get_t_open_ind(np_buy_arr, t_exit_ind, np_buy_arr_len, stop_stops, stop_stops_count) - - if debug: - print("\n(0) numpy debug \nnp_get_t_open, has returned the next valid buy index as", t_open_ind) - print("If -1 there are no valid buys in the remainder of ticker data. Skipping to end of loop") - if debug_timing: - t_t = f(st) - print("0-numpy", str.format('{0:.17f}', t_t)) - st = s() - - if t_open_ind != -1: - - """ - 1 - Create views to search within for our open trade - - The views are our search space for the next Stop or Sell - Numpy view is employed as: - 1,000 faster than pandas searches - Pandas cannot assure it will always return a view, it may make a slow copy. - - The view contains columns: - buy 0 - open 1 - close 2 - sell 3 - high 4 - low 5 - - Requires: np_bslap is our numpy array of the ticker DataFrame - Requires: t_open_ind is the index row with the buy. - Provides: np_t_open_v View of array after buy. - Provides: np_t_open_v_stop View of array after buy +1 - (Stop will search in here to prevent stopping in the past) - """ - np_t_open_v = np_bslap[t_open_ind:] - np_t_open_v_stop = np_bslap[t_open_ind + 1:] - - if debug: - print("\n(1) numpy debug \nNumpy view row 0 is now Ticker_Data Index", t_open_ind) - print("Numpy View: Buy - Open - Close - Sell - High - Low") - print("Row 0", np_t_open_v[0]) - print("Row 1", np_t_open_v[1], ) - if debug_timing: - t_t = f(st) - print("2-numpy", str.format('{0:.17f}', t_t)) - st = s() - - ''' - 2 - Calculate our stop-loss price - - As stop is based on buy price of our trade - - (BTO)Buys are Triggered On np_bto, typically the CLOSE of candle - - (BCO)Buys are Calculated On np_bco, default is OPEN of the next candle. - This is as we only see the CLOSE after it has happened. - The back test assumption is we have bought at first available price, the OPEN - - Requires: np_bslap - is our numpy array of the ticker DataFrame - Requires: t_open_ind - is the index row with the first buy. - Requires: p_stop - is the stop rate, ie. 0.99 is -1% - Provides: np_t_stop_pri - The value stop-loss will be triggered on - ''' - np_t_stop_pri = (np_bslap[t_open_ind + 1, np_bco] * p_stop) - - if debug: - print("\n(2) numpy debug\nStop-Loss has been calculated at:", np_t_stop_pri) - if debug_timing: - t_t = f(st) - print("2-numpy", str.format('{0:.17f}', t_t)) - st = s() - - ''' - 3 - Find candle STO is under Stop-Loss After Trade opened. - - where [np_sto] (stop tiggered on variable: "close", "low" etc) < np_t_stop_pri - - Requires: np_t_open_v_stop Numpy view of ticker_data after buy row +1 (when trade was opened) - Requires: np_sto User Var(STO)StopTriggeredOn. Typically set to "low" or "close" - Requires: np_t_stop_pri The stop-loss price STO must fall under to trigger stop - Provides: np_t_stop_ind The first candle after trade open where STO is under stop-loss - ''' - np_t_stop_ind = utf1st.find_1st(np_t_open_v_stop[:, np_sto], - np_t_stop_pri, - utf1st.cmp_smaller) - - # plus 1 as np_t_open_v_stop is 1 ahead of view np_t_open_v, used from here on out. - np_t_stop_ind = np_t_stop_ind + 1 - - if debug: - print("\n(3) numpy debug\nNext view index with STO (stop trigger on) under Stop-Loss is", - np_t_stop_ind - 1, - ". STO is using field", np_sto, - "\nFrom key: buy 0 - open 1 - close 2 - sell 3 - high 4 - low 5\n") - - print( - "If -1 or 0 returned there is no stop found to end of view, then next two array lines are garbage") - print("Row", np_t_stop_ind - 1, np_t_open_v[np_t_stop_ind]) - print("Row", np_t_stop_ind, np_t_open_v[np_t_stop_ind + 1]) - if debug_timing: - t_t = f(st) - print("3-numpy", str.format('{0:.17f}', t_t)) - st = s() - - ''' - 4 - Find first sell index after trade open - - First index in the view np_t_open_v where ['sell'] = 1 - - Requires: np_t_open_v - view of ticker_data from buy onwards - Requires: no_sell - integer '3', the buy column in the array - Provides: np_t_sell_ind index of view where first sell=1 after buy - ''' - # Use numpy array for faster search for sell - # Sell uses column 3. - # buy 0 - open 1 - close 2 - sell 3 - high 4 - low 5 - # Numpy searches 25-35x quicker than pandas on this data - - np_t_sell_ind = utf1st.find_1st(np_t_open_v[:, np_sell], - 1, utf1st.cmp_equal) - if debug: - print("\n(4) numpy debug\nNext view index with sell = 1 is ", np_t_sell_ind) - print("If 0 or less is returned there is no sell found to end of view, then next lines garbage") - print("Row", np_t_sell_ind, np_t_open_v[np_t_sell_ind]) - print("Row", np_t_sell_ind + 1, np_t_open_v[np_t_sell_ind + 1]) - if debug_timing: - t_t = f(st) - print("4-numpy", str.format('{0:.17f}', t_t)) - st = s() - - ''' - 5 - Determine which was hit first a stop or sell - To then use as exit index price-field (sell on buy, stop on stop) - - STOP takes priority over SELL as would be 'in candle' from tick data - Sell would use Open from Next candle. - So in a draw Stop would be hit first on ticker data in live - - Validity of when types of trades may be executed can be summarised as: - - Tick View - index index Buy Sell open low close high Stop price - open 2am 94 -1 0 0 ----- ------ ------ ----- ----- - open 3am 95 0 1 0 ----- ------ trg buy ----- ----- - open 4am 96 1 0 1 Enter trgstop trg sel ROI out Stop out - open 5am 97 2 0 0 Exit ------ ------- ----- ----- - open 6am 98 3 0 0 ----- ------ ------- ----- ----- - - -1 means not found till end of view i.e no valid Stop found. Exclude from match. - Stop tiggering and closing in 96-1, the candle we bought at OPEN in, is valid. - - Buys and sells are triggered at candle close - Both will open their postions at the open of the next candle. i/e + 1 index - - Stop and buy Indexes are on the view. To map to the ticker dataframe - the t_open_ind index should be summed. - - np_t_stop_ind: Stop Found index in view - t_exit_ind : Sell found in view - t_open_ind : Where view was started on ticker_data - - TODO: fix this frig for logic test,, case/switch/dictionary would be better... - more so when later testing many options, dynamic stop / roi etc - cludge - Setting np_t_sell_ind as 9999999999 when -1 (not found) - cludge - Setting np_t_stop_ind as 9999999999 when -1 (not found) - - ''' - if debug: - print("\n(5) numpy debug\nStop or Sell Logic Processing") - - # cludge for logic test (-1) means it was not found, set crazy high to lose < test - np_t_sell_ind = 99999999 if np_t_sell_ind <= 0 else np_t_sell_ind - np_t_stop_ind = 99999999 if np_t_stop_ind <= 0 else np_t_stop_ind - - # Stoploss trigger found before a sell =1 - if np_t_stop_ind < 99999999 and np_t_stop_ind <= np_t_sell_ind: - t_exit_ind = t_open_ind + np_t_stop_ind # Set Exit row index - t_exit_type = SellType.STOP_LOSS # Set Exit type (stop) - np_t_exit_pri = np_sco # The price field our STOP exit will use - if debug: - print("Type STOP is first exit condition. " - "At view index:", np_t_stop_ind, ". Ticker data exit index is", t_exit_ind) - - # Buy = 1 found before a stoploss triggered - elif np_t_sell_ind < 99999999 and np_t_sell_ind < np_t_stop_ind: - # move sell onto next candle, we only look back on sell - # will use the open price later. - t_exit_ind = t_open_ind + np_t_sell_ind # Set Exit row index - t_exit_type = SellType.SELL_SIGNAL # Set Exit type (sell) - np_t_exit_pri = np_open # The price field our SELL exit will use - if debug: - print("Type SELL is first exit condition. " - "At view index", np_t_sell_ind, ". Ticker data exit index is", t_exit_ind) - - # No stop or buy left in view - set t_exit_last -1 to handle gracefully - else: - t_exit_last: int = -1 # Signal loop to exit, no buys or sells found. - t_exit_type = SellType.NONE - np_t_exit_pri = 999 # field price should be calculated on. 999 a non-existent column - if debug: - print("No valid STOP or SELL found. Signalling t_exit_last to gracefully exit") - - # TODO: fix having to cludge/uncludge this .. - # Undo cludge - np_t_sell_ind = -1 if np_t_sell_ind == 99999999 else np_t_sell_ind - np_t_stop_ind = -1 if np_t_stop_ind == 99999999 else np_t_stop_ind - - if debug_timing: - t_t = f(st) - print("5-logic", str.format('{0:.17f}', t_t)) - st = s() - - if debug: - ''' - Print out the buys, stops, sells - Include Line before and after to for easy - Human verification - ''' - # Combine the np_t_stop_pri value to bslap dataframe to make debug - # life easy. This is the current stop price based on buy price_ - # This is slow but don't care about performance in debug - # - # When referencing equiv np_column, as example np_sto, its 5 in numpy and 6 in df, so +1 - # as there is no data column in the numpy array. - bslap['np_stop_pri'] = np_t_stop_pri - - # Buy - print("\n\nDATAFRAME DEBUG =================== BUY ", pair) - print("Numpy Array BUY Index is:", 0) - print("DataFrame BUY Index is:", t_open_ind, "displaying DF \n") - print("HINT, BUY trade should use OPEN price from next candle, i.e ", t_open_ind + 1) - op_is = t_open_ind - 1 # Print open index start, line before - op_if = t_open_ind + 3 # Print open index finish, line after - print(bslap.iloc[op_is:op_if], "\n") - - # Stop - Stops trigger price np_sto (+1 for pandas column), and price received np_sco +1. (Stop Trigger|Calculated On) - if np_t_stop_ind < 0: - print("DATAFRAME DEBUG =================== STOP ", pair) - print("No STOPS were found until the end of ticker data file\n") - else: - print("DATAFRAME DEBUG =================== STOP ", pair) - print("Numpy Array STOP Index is:", np_t_stop_ind, "View starts at index", t_open_ind) - df_stop_index = (t_open_ind + np_t_stop_ind) - - print("DataFrame STOP Index is:", df_stop_index, "displaying DF \n") - print("First Stoploss trigger after Trade entered at OPEN in candle", t_open_ind + 1, "is ", - df_stop_index, ": \n", - str.format('{0:.17f}', bslap.iloc[df_stop_index][np_sto + 1]), - "is less than", str.format('{0:.17f}', np_t_stop_pri)) - - print("A stoploss exit will be calculated at rate:", - str.format('{0:.17f}', bslap.iloc[df_stop_index][np_sco + 1])) - - print("\nHINT, STOPs should exit in-candle, i.e", df_stop_index, - ": As live STOPs are not linked to O-C times") - - st_is = df_stop_index - 1 # Print stop index start, line before - st_if = df_stop_index + 2 # Print stop index finish, line after - print(bslap.iloc[st_is:st_if], "\n") - - # Sell - if np_t_sell_ind < 0: - print("DATAFRAME DEBUG =================== SELL ", pair) - print("No SELLS were found till the end of ticker data file\n") - else: - print("DATAFRAME DEBUG =================== SELL ", pair) - print("Numpy View SELL Index is:", np_t_sell_ind, "View starts at index", t_open_ind) - df_sell_index = (t_open_ind + np_t_sell_ind) - - print("DataFrame SELL Index is:", df_sell_index, "displaying DF \n") - print("First Sell Index after Trade open is in candle", df_sell_index) - print("HINT, if exit is SELL (not stop) trade should use OPEN price from next candle", - df_sell_index + 1) - sl_is = df_sell_index - 1 # Print sell index start, line before - sl_if = df_sell_index + 3 # Print sell index finish, line after - print(bslap.iloc[sl_is:sl_if], "\n") - - # Chosen Exit (stop or sell) - - print("DATAFRAME DEBUG =================== EXIT ", pair) - print("Exit type is :", t_exit_type) - print("trade exit price field is", np_t_exit_pri, "\n") - - if debug_timing: - t_t = f(st) - print("6-depra", str.format('{0:.17f}', t_t)) - st = s() - - ## use numpy view "np_t_open_v" for speed. Columns are - # buy 0 - open 1 - close 2 - sell 3 - high 4 - low 5 - # exception is 6 which is use the stop value. - - # TODO no! this is hard coded bleh fix this open - np_trade_enter_price = np_bslap[t_open_ind + 1, np_open] - if t_exit_type == SellType.STOP_LOSS: - if np_t_exit_pri == 6: - np_trade_exit_price = np_t_stop_pri - t_exit_ind = t_exit_ind + 1 - else: - np_trade_exit_price = np_bslap[t_exit_ind, np_t_exit_pri] - if t_exit_type == SellType.SELL_SIGNAL: - np_trade_exit_price = np_bslap[t_exit_ind, np_t_exit_pri] - - # Catch no exit found - if t_exit_type == SellType.NONE: - np_trade_exit_price = 0 - - if debug_timing: - t_t = f(st) - print("7-numpy", str.format('{0:.17f}', t_t)) - st = s() - - if debug: - print("//////////////////////////////////////////////") - print("+++++++++++++++++++++++++++++++++ Trade Enter ") - print("np_trade Enter Price is ", str.format('{0:.17f}', np_trade_enter_price)) - print("--------------------------------- Trade Exit ") - print("Trade Exit Type is ", t_exit_type) - print("np_trade Exit Price is", str.format('{0:.17f}', np_trade_exit_price)) - print("//////////////////////////////////////////////") - - else: # no buys were found, step 0 returned -1 - # Gracefully exit the loop - t_exit_last == -1 - if debug: - print("\n(E) No buys were found in remaining ticker file. Exiting", pair) - - # Loop control - catch no closed trades. - if debug: - print("---------------------------------------- end of loop", loop, - " Dataframe Exit Index is: ", t_exit_ind) - print("Exit Index Last, Exit Index Now Are: ", t_exit_last, t_exit_ind) - - if t_exit_last >= t_exit_ind or t_exit_last == -1: - """ - Break loop and go on to next pair. - - When last trade exit equals index of last exit, there is no - opportunity to close any more trades. - """ - # TODO :add handing here to record none closed open trades - - - if debug: - print(bslap_pair_results) - break - else: - """ - Add trade to backtest looking results list of dicts - Loop back to look for more trades. - """ - - # We added +1 to t_exit_ind if the exit was a stop-loss, to not exit early in the IF of this ELSE - # removing the +1 here so prices match. - if t_exit_type == SellType.STOP_LOSS: - t_exit_ind = t_exit_ind - 1 - - # Build trade dictionary - ## In general if a field can be calculated later from other fields leave blank here - ## Its X(number of trades faster) to calc all in a single vector than 1 trade at a time - - # create a new dict - close_index: int = t_exit_ind - bslap_result = {} # Must have at start or we end up with a list of multiple same last result - bslap_result["pair"] = pair - bslap_result["stoploss"] = stop - bslap_result["profit_percent"] = "" # To be 1 vector calc across trades when loop complete - bslap_result["profit_abs"] = "" # To be 1 vector calc across trades when loop complete - bslap_result["open_time"] = np_bslap_dates[t_open_ind + 1] # use numpy array, pandas 20x slower - bslap_result["close_time"] = np_bslap_dates[close_index] # use numpy array, pandas 20x slower - bslap_result["open_index"] = t_open_ind + 1 # +1 as we buy on next. - bslap_result["close_index"] = close_index - bslap_result["trade_duration"] = "" # To be 1 vector calc across trades when loop complete - bslap_result["open_at_end"] = False - bslap_result["open_rate"] = round(np_trade_enter_price, 15) - bslap_result["close_rate"] = round(np_trade_exit_price, 15) - bslap_result["exit_type"] = t_exit_type - bslap_result["sell_reason"] = t_exit_type #duplicated, but I don't care - # append the dict to the list and print list - bslap_pair_results.append(bslap_result) - - if t_exit_type is SellType.STOP_LOSS: - stop_stops_count = stop_stops_count + 1 - - if debug: - print("The trade dict is: \n", bslap_result) - print("Trades dicts in list after append are: \n ", bslap_pair_results) - - """ - Loop back to start. t_exit_last becomes where loop - will seek to open new trades from. - Push index on 1 to not open on close - """ - t_exit_last = t_exit_ind + 1 - - if debug_timing: - t_t = f(st) - print("8+trade", str.format('{0:.17f}', t_t)) - - # Send back List of trade dicts - return bslap_pair_results \ No newline at end of file diff --git a/freqtrade/optimize/edge.py b/freqtrade/optimize/edge.py deleted file mode 100644 index efafb8d77..000000000 --- a/freqtrade/optimize/edge.py +++ /dev/null @@ -1,1037 +0,0 @@ -# pragma pylint: disable=missing-docstring, W0212, too-many-arguments - -""" -This module contains the backtesting logic -""" -import logging -import operator -import sys -from argparse import Namespace -from datetime import datetime, timedelta -from typing import Any, Dict, List, NamedTuple, Optional, Tuple - -import arrow -from pandas import DataFrame, to_datetime -from tabulate import tabulate -import numpy as np - -import freqtrade.optimize as optimize -from freqtrade import DependencyException, constants -from freqtrade.arguments import Arguments -from freqtrade.configuration import Configuration -from freqtrade.exchange import Exchange -from freqtrade.misc import file_dump_json -from freqtrade.persistence import Trade -from freqtrade.strategy.interface import SellType -from freqtrade.strategy.resolver import IStrategy, StrategyResolver -from collections import OrderedDict -import timeit -from time import sleep - -import pdb - -logger = logging.getLogger(__name__) - -class Edge: - """ - provides a quick way to evaluate strategies over a longer term of time - """ - - def __init__(self, config: Dict[str, Any], exchange = None) -> None: - """ - constructor - """ - self.config = config - self.strategy: IStrategy = StrategyResolver(self.config).strategy - self.ticker_interval = self.strategy.ticker_interval - self.tickerdata_to_dataframe = self.strategy.tickerdata_to_dataframe - self.populate_buy_trend = self.strategy.populate_buy_trend - self.populate_sell_trend = self.strategy.populate_sell_trend - - ### - # - ### - if exchange is None: - self.config['exchange']['secret'] = '' - self.config['exchange']['password'] = '' - self.config['exchange']['uid'] = '' - self.config['dry_run'] = True - self.exchange = Exchange(self.config) - else: - self.exchange = exchange - - self.fee = self.exchange.get_fee() - - self.stop_loss_value = self.strategy.stoploss - - #### backslap config - ''' - Numpy arrays are used for 100x speed up - We requires setting Int values for - buy stop triggers and stop calculated on - # buy 0 - open 1 - close 2 - sell 3 - high 4 - low 5 - stop 6 - ''' - self.np_buy: int = 0 - self.np_open: int = 1 - self.np_close: int = 2 - self.np_sell: int = 3 - self.np_high: int = 4 - self.np_low: int = 5 - self.np_stop: int = 6 - self.np_bto: int = self.np_close # buys_triggered_on - should be close - self.np_bco: int = self.np_open # buys calculated on - open of the next candle. - self.np_sto: int = self.np_low # stops_triggered_on - Should be low, FT uses close - self.np_sco: int = self.np_stop # stops_calculated_on - Should be stop, FT uses close - # self.np_sto: int = self.np_close # stops_triggered_on - Should be low, FT uses close - # self.np_sco: int = self.np_close # stops_calculated_on - Should be stop, FT uses close - - self.debug = False # Main debug enable, very print heavy, enable 2 loops recommended - self.debug_timing = False # Stages within Backslap - self.debug_2loops = False # Limit each pair to two loops, useful when debugging - self.debug_vector = False # Debug vector calcs - self.debug_timing_main_loop = False # print overall timing per pair - works in Backtest and Backslap - - self.backslap_show_trades = False # prints trades in addition to summary report - self.backslap_save_trades = True # saves trades as a pretty table to backslap.txt - - self.stop_stops: int = 9999 # stop back testing any pair with this many stops, set to 999999 to not hit - - @staticmethod - def get_timeframe(data: Dict[str, DataFrame]) -> Tuple[arrow.Arrow, arrow.Arrow]: - """ - Get the maximum timeframe for the given backtest data - :param data: dictionary with preprocessed backtesting data - :return: tuple containing min_date, max_date - """ - timeframe = [ - (arrow.get(frame['date'].min()), arrow.get(frame['date'].max())) - for frame in data.values() - ] - return min(timeframe, key=operator.itemgetter(0))[0], \ - max(timeframe, key=operator.itemgetter(1))[1] - - def s(self): - st = timeit.default_timer() - return st - - def f(self, st): - return (timeit.default_timer() - st) - - def run(self,args): - - headers = ['date', 'buy', 'open', 'close', 'sell', 'high', 'low'] - processed = args['processed'] - max_open_trades = args.get('max_open_trades', 0) - realistic = args.get('realistic', False) - stoploss_range = args['stoploss_range'] - trades = [] - trade_count_lock: Dict = {} - - ########################### Call out BSlap Loop instead of Original BT code - bslap_results: list = [] - for pair, pair_data in processed.items(): - if self.debug_timing: # Start timer - fl = self.s() - - # Sorting dataframe by date and reset index - pair_data = pair_data.sort_values(by=['date']) - pair_data = pair_data.reset_index(drop=True) - - ticker_data = self.populate_sell_trend( - self.populate_buy_trend(pair_data))[headers].copy() - - if self.debug_timing: # print time taken - flt = self.f(fl) - # print("populate_buy_trend:", pair, round(flt, 10)) - st = self.s() - - # #dump same DFs to disk for offline testing in scratch - # f_pair:str = pair - # csv = f_pair.replace("/", "_") - # csv="/Users/creslin/PycharmProjects/freqtrade_new/frames/" + csv - # ticker_data.to_csv(csv, sep='\t', encoding='utf-8') - - # call bslap - results are a list of dicts - for stoploss in stoploss_range: - bslap_results += self.backslap_pair(ticker_data, pair, round(stoploss, 3)) - - - if self.debug_timing: # print time taken - tt = self.f(st) - print("Time to BackSlap :", pair, round(tt, 10)) - print("-----------------------") - - # Switch List of Trade Dicts (bslap_results) to Dataframe - # Fill missing, calculable columns, profit, duration , abs etc. - bslap_results_df = DataFrame(bslap_results) - - if len(bslap_results_df) > 0: # Only post process a frame if it has a record - # bslap_results_df['open_time'] = to_datetime(bslap_results_df['open_time']) - # bslap_results_df['close_time'] = to_datetime(bslap_results_df['close_time']) - # if debug: - # print("open_time and close_time converted to datetime columns") - - bslap_results_df = self.vector_fill_results_table(bslap_results_df) - else: - from freqtrade.optimize.backtesting import BacktestResult - - bslap_results_df = [] - bslap_results_df = DataFrame.from_records(bslap_results_df, columns=BacktestResult._fields) - - return bslap_results_df - - def vector_fill_results_table(self, bslap_results_df: DataFrame): - """ - The Results frame contains a number of columns that are calculable - from othe columns. These are left blank till all rows are added, - to be populated in single vector calls. - - Columns to be populated are: - - Profit - - trade duration - - profit abs - :param bslap_results Dataframe - :return: bslap_results Dataframe - """ - import pandas as pd - import numpy as np - debug = self.debug_vector - - # stake and fees - # stake = 0.015 - # 0.05% is 0.0005 - # fee = 0.001 - - stake = self.config.get('stake_amount') - fee = self.fee - open_fee = fee / 2 - close_fee = fee / 2 - - if debug: - print("Stake is,", stake, "the sum of currency to spend per trade") - print("The open fee is", open_fee, "The close fee is", close_fee) - if debug: - from pandas import set_option - set_option('display.max_rows', 5000) - set_option('display.max_columns', 20) - pd.set_option('display.width', 1000) - pd.set_option('max_colwidth', 40) - pd.set_option('precision', 12) - - # # Get before - # csv = "cryptosher_before_debug" - # bslap_results_df.to_csv(csv, sep='\t', encoding='utf-8') - - # bslap_results_df.to_csv(csv, sep='\t', encoding='utf-8') - - bslap_results_df['trade_duration'] = bslap_results_df['close_time'] - bslap_results_df['open_time'] - bslap_results_df['trade_duration'] = bslap_results_df['trade_duration'].map(lambda x: int(x.total_seconds() / 60)) - - ## Spends, Takes, Profit, Absolute Profit - # print(bslap_results_df) - # Buy Price - bslap_results_df['buy_vol'] = stake / bslap_results_df['open_rate'] # How many target are we buying - bslap_results_df['buy_fee'] = stake * open_fee - bslap_results_df['buy_spend'] = stake + bslap_results_df['buy_fee'] # How much we're spending - - # Sell price - bslap_results_df['sell_sum'] = bslap_results_df['buy_vol'] * bslap_results_df['close_rate'] - bslap_results_df['sell_fee'] = bslap_results_df['sell_sum'] * close_fee - bslap_results_df['sell_take'] = bslap_results_df['sell_sum'] - bslap_results_df['sell_fee'] - # profit_percent - bslap_results_df['profit_percent'] = (bslap_results_df['sell_take'] - bslap_results_df['buy_spend']) \ - / bslap_results_df['buy_spend'] - # Absolute profit - bslap_results_df['profit_abs'] = bslap_results_df['sell_take'] - bslap_results_df['buy_spend'] - - # # Get After - # csv="cryptosher_after_debug" - # bslap_results_df.to_csv(csv, sep='\t', encoding='utf-8') - - if debug: - print("\n") - print(bslap_results_df[ - ['buy_vol', 'buy_fee', 'buy_spend', 'sell_sum', 'sell_fee', 'sell_take', 'profit_percent', - 'profit_abs', 'exit_type']]) - - #pdb.set_trace() - return bslap_results_df - - def np_get_t_open_ind(self, np_buy_arr, t_exit_ind: int, np_buy_arr_len: int, stop_stops: int, - stop_stops_count: int): - import utils_find_1st as utf1st - """ - The purpose of this def is to return the next "buy" = 1 - after t_exit_ind. - - This function will also check is the stop limit for the pair has been reached. - if stop_stops is the limit and stop_stops_count it the number of times the stop has been hit. - - t_exit_ind is the index the last trade exited on - or 0 if first time around this loop. - - stop_stops i - """ - debug = self.debug - - # Timers, to be called if in debug - def s(): - st = timeit.default_timer() - return st - - def f(st): - return (timeit.default_timer() - st) - - st = s() - t_open_ind: int - - """ - Create a view on our buy index starting after last trade exit - Search for next buy - """ - np_buy_arr_v = np_buy_arr[t_exit_ind:] - t_open_ind = utf1st.find_1st(np_buy_arr_v, 1, utf1st.cmp_equal) - - ''' - If -1 is returned no buy has been found, preserve the value - ''' - if t_open_ind != -1: # send back the -1 if no buys found. otherwise update index - t_open_ind = t_open_ind + t_exit_ind # Align numpy index - - if t_open_ind == np_buy_arr_len - 1: # If buy found on last candle ignore, there is no OPEN in next to use - t_open_ind = -1 # -1 ends the loop - - if stop_stops_count >= stop_stops: # if maximum number of stops allowed in a pair is hit, exit loop - t_open_ind = -1 # -1 ends the loop - if debug: - print("Max stop limit ", stop_stops, "reached. Moving to next pair") - - return t_open_ind - - def backslap_pair(self, ticker_data, pair, stoploss): - import pandas as pd - import numpy as np - import timeit - import utils_find_1st as utf1st - from datetime import datetime - - ### backslap debug wrap - # debug_2loops = False # only loop twice, for faster debug - # debug_timing = False # print timing for each step - # debug = False # print values, to check accuracy - debug_2loops = self.debug_2loops # only loop twice, for faster debug - debug_timing = self.debug_timing # print timing for each step - debug = self.debug # print values, to check accuracy - - # Read Stop Loss Values and Stake - # pdb.set_trace() - #stop = self.stop_loss_value - stop = stoploss - p_stop = (stop + 1) # What stop really means, e.g 0.01 is 0.99 of price - - if debug: - print("Stop is ", stop, "value from stragey file") - print("p_stop is", p_stop, "value used to multiply to entry price") - - if debug: - from pandas import set_option - set_option('display.max_rows', 5000) - set_option('display.max_columns', 8) - pd.set_option('display.width', 1000) - pd.set_option('max_colwidth', 40) - pd.set_option('precision', 12) - - def s(): - st = timeit.default_timer() - return st - - def f(st): - return (timeit.default_timer() - st) - - #### backslap config - ''' - Numpy arrays are used for 100x speed up - We requires setting Int values for - buy stop triggers and stop calculated on - # buy 0 - open 1 - close 2 - sell 3 - high 4 - low 5 - stop 6 - ''' - - ####### - # Use vars set at top of backtest - np_buy: int = self.np_buy - np_open: int = self.np_open - np_close: int = self.np_close - np_sell: int = self.np_sell - np_high: int = self.np_high - np_low: int = self.np_low - np_stop: int = self.np_stop - np_bto: int = self.np_bto # buys_triggered_on - should be close - np_bco: int = self.np_bco # buys calculated on - open of the next candle. - np_sto: int = self.np_sto # stops_triggered_on - Should be low, FT uses close - np_sco: int = self.np_sco # stops_calculated_on - Should be stop, FT uses close - - ### End Config - - pair: str = pair - - # ticker_data: DataFrame = ticker_dfs[t_file] - bslap: DataFrame = ticker_data - - # Build a single dimension numpy array from "buy" index for faster search - # (500x faster than pandas) - np_buy_arr = bslap['buy'].values - np_buy_arr_len: int = len(np_buy_arr) - - # use numpy array for faster searches in loop, 20x faster than pandas - # buy 0 - open 1 - close 2 - sell 3 - high 4 - low 5 - np_bslap = np.array(bslap[['buy', 'open', 'close', 'sell', 'high', 'low']]) - - # Build a numpy list of date-times. - # We use these when building the trade - # The rationale is to address a value from a pandas cell is thousands of - # times more expensive. Processing time went X25 when trying to use any data from pandas - np_bslap_dates = bslap['date'].values - - loop: int = 0 # how many time around the loop - t_exit_ind = 0 # Start loop from first index - t_exit_last = 0 # To test for exit - - stop_stops = self.stop_stops # Int of stops within a pair to stop trading a pair at - stop_stops_count = 0 # stop counter per pair - - st = s() # Start timer for processing dataframe - if debug: - print('Processing:', pair) - - # Results will be stored in a list of dicts - bslap_pair_results: list = [] - bslap_result: dict = {} - - while t_exit_ind < np_buy_arr_len: - loop = loop + 1 - if debug or debug_timing: - print("-- T_exit_Ind - Numpy Index is", t_exit_ind, " ----------------------- Loop", loop, pair) - if debug_2loops: - if loop == 3: - print( - "++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++Loop debug max met - breaking") - break - ''' - Dev phases - Phase 1 - 1) Manage buy, sell, stop enter/exit - a) Find first buy index - b) Discover first stop and sell hit after buy index - c) Chose first instance as trade exit - - Phase 2 - 2) Manage dynamic Stop and ROI Exit - a) Create trade slice from 1 - b) search within trade slice for dynamice stop hit - c) search within trade slice for ROI hit - ''' - - if debug_timing: - st = s() - ''' - 0 - Find next buy entry - Finds index for first (buy = 1) flag - - Requires: np_buy_arr - a 1D array of the 'buy' column. To find next "1" - Required: t_exit_ind - Either 0, first loop. Or The index we last exited on - Requires: np_buy_arr_len - length of pair array. - Requires: stops_stops - number of stops allowed before stop trading a pair - Requires: stop_stop_counts - count of stops hit in the pair - Provides: The next "buy" index after t_exit_ind - - If -1 is returned no buy has been found in remainder of array, skip to exit loop - ''' - t_open_ind = self.np_get_t_open_ind(np_buy_arr, t_exit_ind, np_buy_arr_len, stop_stops, stop_stops_count) - - if debug: - print("\n(0) numpy debug \nnp_get_t_open, has returned the next valid buy index as", t_open_ind) - print("If -1 there are no valid buys in the remainder of ticker data. Skipping to end of loop") - if debug_timing: - t_t = f(st) - print("0-numpy", str.format('{0:.17f}', t_t)) - st = s() - - if t_open_ind != -1: - - """ - 1 - Create views to search within for our open trade - - The views are our search space for the next Stop or Sell - Numpy view is employed as: - 1,000 faster than pandas searches - Pandas cannot assure it will always return a view, it may make a slow copy. - - The view contains columns: - buy 0 - open 1 - close 2 - sell 3 - high 4 - low 5 - - Requires: np_bslap is our numpy array of the ticker DataFrame - Requires: t_open_ind is the index row with the buy. - Provides: np_t_open_v View of array after buy. - Provides: np_t_open_v_stop View of array after buy +1 - (Stop will search in here to prevent stopping in the past) - """ - np_t_open_v = np_bslap[t_open_ind:] - np_t_open_v_stop = np_bslap[t_open_ind + 1:] - - if debug: - print("\n(1) numpy debug \nNumpy view row 0 is now Ticker_Data Index", t_open_ind) - print("Numpy View: Buy - Open - Close - Sell - High - Low") - print("Row 0", np_t_open_v[0]) - print("Row 1", np_t_open_v[1], ) - if debug_timing: - t_t = f(st) - print("2-numpy", str.format('{0:.17f}', t_t)) - st = s() - - ''' - 2 - Calculate our stop-loss price - - As stop is based on buy price of our trade - - (BTO)Buys are Triggered On np_bto, typically the CLOSE of candle - - (BCO)Buys are Calculated On np_bco, default is OPEN of the next candle. - This is as we only see the CLOSE after it has happened. - The back test assumption is we have bought at first available price, the OPEN - - Requires: np_bslap - is our numpy array of the ticker DataFrame - Requires: t_open_ind - is the index row with the first buy. - Requires: p_stop - is the stop rate, ie. 0.99 is -1% - Provides: np_t_stop_pri - The value stop-loss will be triggered on - ''' - np_t_stop_pri = (np_bslap[t_open_ind + 1, np_bco] * p_stop) - - if debug: - print("\n(2) numpy debug\nStop-Loss has been calculated at:", np_t_stop_pri) - if debug_timing: - t_t = f(st) - print("2-numpy", str.format('{0:.17f}', t_t)) - st = s() - - ''' - 3 - Find candle STO is under Stop-Loss After Trade opened. - - where [np_sto] (stop tiggered on variable: "close", "low" etc) < np_t_stop_pri - - Requires: np_t_open_v_stop Numpy view of ticker_data after buy row +1 (when trade was opened) - Requires: np_sto User Var(STO)StopTriggeredOn. Typically set to "low" or "close" - Requires: np_t_stop_pri The stop-loss price STO must fall under to trigger stop - Provides: np_t_stop_ind The first candle after trade open where STO is under stop-loss - ''' - np_t_stop_ind = utf1st.find_1st(np_t_open_v_stop[:, np_sto], - np_t_stop_pri, - utf1st.cmp_smaller) - - # plus 1 as np_t_open_v_stop is 1 ahead of view np_t_open_v, used from here on out. - np_t_stop_ind = np_t_stop_ind + 1 - - if debug: - print("\n(3) numpy debug\nNext view index with STO (stop trigger on) under Stop-Loss is", - np_t_stop_ind - 1, - ". STO is using field", np_sto, - "\nFrom key: buy 0 - open 1 - close 2 - sell 3 - high 4 - low 5\n") - - print( - "If -1 or 0 returned there is no stop found to end of view, then next two array lines are garbage") - print("Row", np_t_stop_ind - 1, np_t_open_v[np_t_stop_ind]) - print("Row", np_t_stop_ind, np_t_open_v[np_t_stop_ind + 1]) - if debug_timing: - t_t = f(st) - print("3-numpy", str.format('{0:.17f}', t_t)) - st = s() - - ''' - 4 - Find first sell index after trade open - - First index in the view np_t_open_v where ['sell'] = 1 - - Requires: np_t_open_v - view of ticker_data from buy onwards - Requires: no_sell - integer '3', the buy column in the array - Provides: np_t_sell_ind index of view where first sell=1 after buy - ''' - # Use numpy array for faster search for sell - # Sell uses column 3. - # buy 0 - open 1 - close 2 - sell 3 - high 4 - low 5 - # Numpy searches 25-35x quicker than pandas on this data - - np_t_sell_ind = utf1st.find_1st(np_t_open_v[:, np_sell], - 1, utf1st.cmp_equal) - if debug: - print("\n(4) numpy debug\nNext view index with sell = 1 is ", np_t_sell_ind) - print("If 0 or less is returned there is no sell found to end of view, then next lines garbage") - print("Row", np_t_sell_ind, np_t_open_v[np_t_sell_ind]) - print("Row", np_t_sell_ind + 1, np_t_open_v[np_t_sell_ind + 1]) - if debug_timing: - t_t = f(st) - print("4-numpy", str.format('{0:.17f}', t_t)) - st = s() - - ''' - 5 - Determine which was hit first a stop or sell - To then use as exit index price-field (sell on buy, stop on stop) - - STOP takes priority over SELL as would be 'in candle' from tick data - Sell would use Open from Next candle. - So in a draw Stop would be hit first on ticker data in live - - Validity of when types of trades may be executed can be summarised as: - - Tick View - index index Buy Sell open low close high Stop price - open 2am 94 -1 0 0 ----- ------ ------ ----- ----- - open 3am 95 0 1 0 ----- ------ trg buy ----- ----- - open 4am 96 1 0 1 Enter trgstop trg sel ROI out Stop out - open 5am 97 2 0 0 Exit ------ ------- ----- ----- - open 6am 98 3 0 0 ----- ------ ------- ----- ----- - - -1 means not found till end of view i.e no valid Stop found. Exclude from match. - Stop tiggering and closing in 96-1, the candle we bought at OPEN in, is valid. - - Buys and sells are triggered at candle close - Both will open their postions at the open of the next candle. i/e + 1 index - - Stop and buy Indexes are on the view. To map to the ticker dataframe - the t_open_ind index should be summed. - - np_t_stop_ind: Stop Found index in view - t_exit_ind : Sell found in view - t_open_ind : Where view was started on ticker_data - - TODO: fix this frig for logic test,, case/switch/dictionary would be better... - more so when later testing many options, dynamic stop / roi etc - cludge - Setting np_t_sell_ind as 9999999999 when -1 (not found) - cludge - Setting np_t_stop_ind as 9999999999 when -1 (not found) - - ''' - if debug: - print("\n(5) numpy debug\nStop or Sell Logic Processing") - - # cludge for logic test (-1) means it was not found, set crazy high to lose < test - np_t_sell_ind = 99999999 if np_t_sell_ind <= 0 else np_t_sell_ind - np_t_stop_ind = 99999999 if np_t_stop_ind <= 0 else np_t_stop_ind - - # Stoploss trigger found before a sell =1 - if np_t_stop_ind < 99999999 and np_t_stop_ind <= np_t_sell_ind: - t_exit_ind = t_open_ind + np_t_stop_ind # Set Exit row index - t_exit_type = SellType.STOP_LOSS # Set Exit type (stop) - np_t_exit_pri = np_sco # The price field our STOP exit will use - if debug: - print("Type STOP is first exit condition. " - "At view index:", np_t_stop_ind, ". Ticker data exit index is", t_exit_ind) - - # Buy = 1 found before a stoploss triggered - elif np_t_sell_ind < 99999999 and np_t_sell_ind < np_t_stop_ind: - # move sell onto next candle, we only look back on sell - # will use the open price later. - t_exit_ind = t_open_ind + np_t_sell_ind # Set Exit row index - t_exit_type = SellType.SELL_SIGNAL # Set Exit type (sell) - np_t_exit_pri = np_open # The price field our SELL exit will use - if debug: - print("Type SELL is first exit condition. " - "At view index", np_t_sell_ind, ". Ticker data exit index is", t_exit_ind) - - # No stop or buy left in view - set t_exit_last -1 to handle gracefully - else: - t_exit_last: int = -1 # Signal loop to exit, no buys or sells found. - t_exit_type = SellType.NONE - np_t_exit_pri = 999 # field price should be calculated on. 999 a non-existent column - if debug: - print("No valid STOP or SELL found. Signalling t_exit_last to gracefully exit") - - # TODO: fix having to cludge/uncludge this .. - # Undo cludge - np_t_sell_ind = -1 if np_t_sell_ind == 99999999 else np_t_sell_ind - np_t_stop_ind = -1 if np_t_stop_ind == 99999999 else np_t_stop_ind - - if debug_timing: - t_t = f(st) - print("5-logic", str.format('{0:.17f}', t_t)) - st = s() - - if debug: - ''' - Print out the buys, stops, sells - Include Line before and after to for easy - Human verification - ''' - # Combine the np_t_stop_pri value to bslap dataframe to make debug - # life easy. This is the current stop price based on buy price_ - # This is slow but don't care about performance in debug - # - # When referencing equiv np_column, as example np_sto, its 5 in numpy and 6 in df, so +1 - # as there is no data column in the numpy array. - bslap['np_stop_pri'] = np_t_stop_pri - - # Buy - print("\n\nDATAFRAME DEBUG =================== BUY ", pair) - print("Numpy Array BUY Index is:", 0) - print("DataFrame BUY Index is:", t_open_ind, "displaying DF \n") - print("HINT, BUY trade should use OPEN price from next candle, i.e ", t_open_ind + 1) - op_is = t_open_ind - 1 # Print open index start, line before - op_if = t_open_ind + 3 # Print open index finish, line after - print(bslap.iloc[op_is:op_if], "\n") - - # Stop - Stops trigger price np_sto (+1 for pandas column), and price received np_sco +1. (Stop Trigger|Calculated On) - if np_t_stop_ind < 0: - print("DATAFRAME DEBUG =================== STOP ", pair) - print("No STOPS were found until the end of ticker data file\n") - else: - print("DATAFRAME DEBUG =================== STOP ", pair) - print("Numpy Array STOP Index is:", np_t_stop_ind, "View starts at index", t_open_ind) - df_stop_index = (t_open_ind + np_t_stop_ind) - - print("DataFrame STOP Index is:", df_stop_index, "displaying DF \n") - print("First Stoploss trigger after Trade entered at OPEN in candle", t_open_ind + 1, "is ", - df_stop_index, ": \n", - str.format('{0:.17f}', bslap.iloc[df_stop_index][np_sto + 1]), - "is less than", str.format('{0:.17f}', np_t_stop_pri)) - - print("A stoploss exit will be calculated at rate:", - str.format('{0:.17f}', bslap.iloc[df_stop_index][np_sco + 1])) - - print("\nHINT, STOPs should exit in-candle, i.e", df_stop_index, - ": As live STOPs are not linked to O-C times") - - st_is = df_stop_index - 1 # Print stop index start, line before - st_if = df_stop_index + 2 # Print stop index finish, line after - print(bslap.iloc[st_is:st_if], "\n") - - # Sell - if np_t_sell_ind < 0: - print("DATAFRAME DEBUG =================== SELL ", pair) - print("No SELLS were found till the end of ticker data file\n") - else: - print("DATAFRAME DEBUG =================== SELL ", pair) - print("Numpy View SELL Index is:", np_t_sell_ind, "View starts at index", t_open_ind) - df_sell_index = (t_open_ind + np_t_sell_ind) - - print("DataFrame SELL Index is:", df_sell_index, "displaying DF \n") - print("First Sell Index after Trade open is in candle", df_sell_index) - print("HINT, if exit is SELL (not stop) trade should use OPEN price from next candle", - df_sell_index + 1) - sl_is = df_sell_index - 1 # Print sell index start, line before - sl_if = df_sell_index + 3 # Print sell index finish, line after - print(bslap.iloc[sl_is:sl_if], "\n") - - # Chosen Exit (stop or sell) - - print("DATAFRAME DEBUG =================== EXIT ", pair) - print("Exit type is :", t_exit_type) - print("trade exit price field is", np_t_exit_pri, "\n") - - if debug_timing: - t_t = f(st) - print("6-depra", str.format('{0:.17f}', t_t)) - st = s() - - ## use numpy view "np_t_open_v" for speed. Columns are - # buy 0 - open 1 - close 2 - sell 3 - high 4 - low 5 - # exception is 6 which is use the stop value. - - # TODO no! this is hard coded bleh fix this open - np_trade_enter_price = np_bslap[t_open_ind + 1, np_open] - if t_exit_type == SellType.STOP_LOSS: - if np_t_exit_pri == 6: - np_trade_exit_price = np_t_stop_pri - t_exit_ind = t_exit_ind + 1 - else: - np_trade_exit_price = np_bslap[t_exit_ind, np_t_exit_pri] - if t_exit_type == SellType.SELL_SIGNAL: - np_trade_exit_price = np_bslap[t_exit_ind, np_t_exit_pri] - - # Catch no exit found - if t_exit_type == SellType.NONE: - np_trade_exit_price = 0 - - if debug_timing: - t_t = f(st) - print("7-numpy", str.format('{0:.17f}', t_t)) - st = s() - - if debug: - print("//////////////////////////////////////////////") - print("+++++++++++++++++++++++++++++++++ Trade Enter ") - print("np_trade Enter Price is ", str.format('{0:.17f}', np_trade_enter_price)) - print("--------------------------------- Trade Exit ") - print("Trade Exit Type is ", t_exit_type) - print("np_trade Exit Price is", str.format('{0:.17f}', np_trade_exit_price)) - print("//////////////////////////////////////////////") - - else: # no buys were found, step 0 returned -1 - # Gracefully exit the loop - t_exit_last == -1 - if debug: - print("\n(E) No buys were found in remaining ticker file. Exiting", pair) - - # Loop control - catch no closed trades. - if debug: - print("---------------------------------------- end of loop", loop, - " Dataframe Exit Index is: ", t_exit_ind) - print("Exit Index Last, Exit Index Now Are: ", t_exit_last, t_exit_ind) - - if t_exit_last >= t_exit_ind or t_exit_last == -1: - """ - Break loop and go on to next pair. - - When last trade exit equals index of last exit, there is no - opportunity to close any more trades. - """ - # TODO :add handing here to record none closed open trades - - if debug: - print(bslap_pair_results) - break - else: - """ - Add trade to backtest looking results list of dicts - Loop back to look for more trades. - """ - - # We added +1 to t_exit_ind if the exit was a stop-loss, to not exit early in the IF of this ELSE - # removing the +1 here so prices match. - if t_exit_type == SellType.STOP_LOSS: - t_exit_ind = t_exit_ind - 1 - - # Build trade dictionary - ## In general if a field can be calculated later from other fields leave blank here - ## Its X(number of trades faster) to calc all in a single vector than 1 trade at a time - - # create a new dict - close_index: int = t_exit_ind - bslap_result = {} # Must have at start or we end up with a list of multiple same last result - bslap_result["pair"] = pair - bslap_result["stoploss"] = stop - bslap_result["profit_percent"] = "" # To be 1 vector calc across trades when loop complete - bslap_result["profit_abs"] = "" # To be 1 vector calc across trades when loop complete - bslap_result["open_time"] = np_bslap_dates[t_open_ind + 1] # use numpy array, pandas 20x slower - bslap_result["close_time"] = np_bslap_dates[close_index] # use numpy array, pandas 20x slower - bslap_result["open_index"] = t_open_ind + 1 # +1 as we buy on next. - bslap_result["close_index"] = close_index - bslap_result["trade_duration"] = "" # To be 1 vector calc across trades when loop complete - bslap_result["open_at_end"] = False - bslap_result["open_rate"] = round(np_trade_enter_price, 15) - bslap_result["close_rate"] = round(np_trade_exit_price, 15) - bslap_result["exit_type"] = t_exit_type - bslap_result["sell_reason"] = t_exit_type #duplicated, but I don't care - # append the dict to the list and print list - bslap_pair_results.append(bslap_result) - - if t_exit_type is SellType.STOP_LOSS: - stop_stops_count = stop_stops_count + 1 - - if debug: - print("The trade dict is: \n", bslap_result) - print("Trades dicts in list after append are: \n ", bslap_pair_results) - - """ - Loop back to start. t_exit_last becomes where loop - will seek to open new trades from. - Push index on 1 to not open on close - """ - t_exit_last = t_exit_ind + 1 - - if debug_timing: - t_t = f(st) - print("8+trade", str.format('{0:.17f}', t_t)) - - # Send back List of trade dicts - return bslap_pair_results - - def _process_result(self, data: Dict[str, Dict], results: DataFrame, stoploss_range) -> str: - """ - This is a temporary version of edge positioning calculation. - The function will be eventually moved to a plugin called Edge in order to calculate necessary WR, RRR and - other indictaors related to money management periodically (each X minutes) and keep it in a storage. - The calulation will be done per pair and per strategy. - """ - - # Removing open trades from dataset - results = results[results.open_at_end == False] - ################################### - - # Removing pairs having less than min_trades_number - min_trades_number = 50 - results = results.groupby('pair').filter(lambda x: len(x) > min_trades_number) - ################################### - - - # Removing outliers (Only Pumps) from the dataset - # The method to detect outliers is to calculate standard deviation - # Then every value more than (standard deviation + 2*average) is out (pump) - # - # Calculating standard deviation of profits - std = results[["profit_abs"]].std() - # - # Calculating average of profits - avg = results[["profit_abs"]].mean() - # - # Removing Pumps - results = results[results.profit_abs < float(avg + 2*std)] - ########################################################################## - - # Removing trades having a duration more than X minutes (set in config) - max_trade_duration = 24*60 - results = results[results.trade_duration < max_trade_duration] - ####################################################################### - - - # Win Rate is the number of profitable trades - # Divided by number of trades - def winrate(x): - x = x[x > 0].count() / x.count() - return x - ############################# - - # Risk Reward Ratio - # 1 / ((loss money / losing trades) / (gained money / winning trades)) - def risk_reward_ratio(x): - x = abs(1/ ((x[x<0].sum() / x[x < 0].count()) / (x[x > 0].sum() / x[x > 0].count()))) - return x - ############################## - - # Required Risk Reward - # (1/(winrate - 1) - def required_risk_reward(x): - x = (1/(x[x > 0].count()/x.count()) -1) - return x - ############################## - - # The difference between risk reward ratio and required risk reward - # We use it as an indicator to find the most interesting pair to trade - def delta(x): - x = (abs(1/ ((x[x < 0].sum() / x[x < 0].count()) / (x[x > 0].sum() / x[x > 0].count())))) - (1/(x[x > 0].count()/x.count()) -1) - return x - ############################## - - - final = results.groupby(['pair', 'stoploss'])['profit_abs'].\ - agg([winrate, risk_reward_ratio, required_risk_reward, delta]).\ - reset_index().sort_values(by=['delta', 'stoploss'], ascending=False)\ - .groupby('pair').first().sort_values(by=['delta'], ascending=False) - - pdb.set_trace() - return final - - - def start(self) -> None: - """ - Run a backtesting end-to-end - :return: None - """ - data = {} - pairs = self.config['exchange']['pair_whitelist'] - logger.info('Using stake_currency: %s ...', self.config['stake_currency']) - logger.info('Using stake_amount: %s ...', self.config['stake_amount']) - - if self.config.get('live'): - logger.info('Downloading data for all pairs in whitelist ...') - for pair in pairs: - data[pair] = self.exchange.get_ticker_history(pair, self.ticker_interval) - else: - logger.info('Using local backtesting data (using whitelist in given config) ...') - - timerange = Arguments.parse_timerange(None if self.config.get( - 'timerange') is None else str(self.config.get('timerange'))) - - data = optimize.load_data( - self.config['datadir'], - pairs=pairs, - ticker_interval=self.ticker_interval, - refresh_pairs=self.config.get('refresh_pairs', False), - exchange=self.exchange, - timerange=timerange - ) - - if not data: - logger.critical("No data found. Terminating.") - return - - # Use max_open_trades in backtesting, except --disable-max-market-positions is set - if self.config.get('use_max_market_positions', True): - max_open_trades = self.config['max_open_trades'] - else: - logger.info('Ignoring max_open_trades (--disable-max-market-positions was used) ...') - max_open_trades = 0 - - preprocessed = self.tickerdata_to_dataframe(data) - - # Print timeframe - min_date, max_date = self.get_timeframe(preprocessed) - logger.info( - 'Measuring data from %s up to %s (%s days)..', - min_date.isoformat(), - max_date.isoformat(), - (max_date - min_date).days - ) - - stoploss_range = np.arange(-0.11, -0.00, 0.01) - - # Execute backtest and print results - results = self.run( - { - 'stake_amount': self.config.get('stake_amount'), - 'processed': preprocessed, - 'stoploss_range': stoploss_range, - 'max_open_trades': max_open_trades, - 'position_stacking': self.config.get('position_stacking', False), - } - ) - - logger.info( - '\n====================================================== ' - 'Edge positionning REPORT' - ' =======================================================\n' - '%s', - self._process_result( - data, - results, - stoploss_range - ) - ) - - -def setup_configuration(args: Namespace) -> Dict[str, Any]: - """ - Prepare the configuration for the backtesting - :param args: Cli args from Arguments() - :return: Configuration - """ - configuration = Configuration(args) - config = configuration.get_config() - - # Ensure we do not use Exchange credentials - config['exchange']['key'] = '' - config['exchange']['secret'] = '' - config['backslap'] = args.backslap - if config['stake_amount'] == constants.UNLIMITED_STAKE_AMOUNT: - raise DependencyException('stake amount could not be "%s" for backtesting' % - constants.UNLIMITED_STAKE_AMOUNT) - - return config - -# Initialize configuration -arguments = Arguments( - sys.argv[1:], - 'Simple High Frequency Trading Bot for crypto currencies' - ) -args = arguments.get_parsed_arg() - -config = setup_configuration(args) - -edge = Edge(config) -edge.start() - -allowed_dollars_at_risk = total_capital * allowed_risk -print("allowed capital at risk ", round(allowed_dollars_at_risk, 5)) - -position_size = (allowed_dollars_at_risk / symbol_strategy_stop_loss) -print("position_size in dollars", round(position_size, 5 )) - -buy_amount = position_size / bid_price -print("amount of tokens to buy ", round(buy_amount,5)) - -check_risk = (buy_amount * (bid_price - stop_trigger_price)) -print("check risk capital ", round(check_risk, 5), "** Should not be more than allowed capital at risk") \ No newline at end of file From fcf837bfda77192924417e351ab64c987fd5b4a1 Mon Sep 17 00:00:00 2001 From: misagh Date: Wed, 26 Sep 2018 16:03:51 +0200 Subject: [PATCH 067/699] refactoring variable declaration --- freqtrade/edge/__init__.py | 37 +++++++++++++++++++------------------ 1 file changed, 19 insertions(+), 18 deletions(-) diff --git a/freqtrade/edge/__init__.py b/freqtrade/edge/__init__.py index 7a0b715ab..7cf2e62de 100644 --- a/freqtrade/edge/__init__.py +++ b/freqtrade/edge/__init__.py @@ -40,7 +40,7 @@ class Edge(): self.edge_config = self.config.get('edge', {}) self._last_updated = None - self._cached_pairs = [] + self._cached_pairs : list = [] self._total_capital = self.edge_config['total_capital_in_stake_currency'] self._allowed_risk = self.edge_config['allowed_risk'] @@ -65,7 +65,7 @@ class Edge(): if ((self._last_updated is not None) and (self._last_updated + heartbeat > arrow.utcnow().timestamp)): return False - data = {} + data: Dict[str, Any] = {} logger.info('Using stake_currency: %s ...', self.config['stake_currency']) logger.info('Using stake_amount: %s ...', self.config['stake_amount']) logger.info('Using local backtesting data (using whitelist in given config) ...') @@ -84,7 +84,7 @@ class Edge(): if not data: logger.critical("No data found. Edge is stopped ...") - return + return False preprocessed = self.tickerdata_to_dataframe(data) @@ -142,7 +142,7 @@ class Edge(): info = [x for x in self._cached_pairs if x[0] == pair][0] return info[1] - def sort_pairs(self, pairs) -> bool: + def sort_pairs(self, pairs) -> list: if len(self._cached_pairs) == 0: self.calculate() edge_sorted_pairs = [x[0] for x in self._cached_pairs] @@ -195,7 +195,7 @@ class Edge(): return result - def _process_expectancy(self, results: DataFrame) -> str: + def _process_expectancy(self, results: DataFrame) -> list: """ This is a temporary version of edge positioning calculation. The function will be eventually moved to a plugin called Edge in order to calculate necessary WR, RRR and @@ -330,19 +330,20 @@ class Edge(): exit_type = SellType.SELL_SIGNAL exit_price = ohlc_columns[open_trade_index + sell_index + 1, 0] - trade = {} - trade["pair"] = pair - trade["stoploss"] = stoploss - trade["profit_percent"] = "" # To be 1 vector calculation across trades when loop complete - trade["profit_abs"] = "" # To be 1 vector calculation across trades when loop complete - trade["open_time"] = date_column[open_trade_index] - trade["close_time"] = date_column[exit_index] - trade["open_index"] = start_point + open_trade_index + 1 # +1 as we buy on next. - trade["close_index"] = start_point + exit_index - trade["trade_duration"] = "" # To be 1 vector calculation across trades when loop complete - trade["open_rate"] = round(open_price, 15) - trade["close_rate"] = round(exit_price, 15) - trade["exit_type"] = exit_type + trade = {'pair': pair, + 'stoploss': stoploss, + 'profit_percent': '', + 'profit_abs': '', + 'open_time': date_column[open_trade_index], + 'close_time': date_column[exit_index], + 'open_index': start_point + open_trade_index + 1, + 'close_index': start_point + exit_index, + 'trade_duration': '', + 'open_rate': round(open_price, 15), + 'close_rate': round(exit_price, 15), + 'exit_type': exit_type + } + result.append(trade) return result + self._detect_stop_and_sell_points( From 75ba6578a31e65bc25ba780fd826e2d55cd9785d Mon Sep 17 00:00:00 2001 From: misagh Date: Wed, 26 Sep 2018 16:36:41 +0200 Subject: [PATCH 068/699] unused library + trailing whitespaces removed. --- freqtrade/freqtradebot.py | 17 ++++++++++++----- freqtrade/optimize/__init__.py | 1 - freqtrade/strategy/interface.py | 6 ++++-- freqtrade/tests/conftest.py | 2 +- 4 files changed, 17 insertions(+), 9 deletions(-) diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index 60d286af1..be02db9a8 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -404,7 +404,12 @@ class FreqtradeBot(object): stake_amount = self._get_trade_stake_amount(_pair) if not stake_amount: return False - logger.info('Buy signal found: about create a new trade with stake_amount: %f ...', stake_amount) + + logger.info( + 'Buy signal found: about create a new trade with stake_amount: %f ...', + stake_amount + ) + bidstrat_check_depth_of_market = self.config.get('bid_strategy', {}).\ get('check_depth_of_market', {}) if (bidstrat_check_depth_of_market.get('enabled', False)) and\ @@ -444,10 +449,10 @@ class FreqtradeBot(object): pair_url = self.exchange.get_pair_detail_url(pair) stake_currency = self.config['stake_currency'] fiat_currency = self.config.get('fiat_display_currency', None) - + # Calculate amount buy_limit = self.get_target_bid(pair, self.exchange.get_ticker(pair)) - + min_stake_amount = self._get_min_pair_stake_amount(pair_s, buy_limit) if min_stake_amount is not None and min_stake_amount > stake_amount: logger.warning( @@ -630,9 +635,11 @@ class FreqtradeBot(object): def check_sell(self, trade: Trade, sell_rate: float, buy: bool, sell: bool) -> bool: if (self.config['edge']['enabled']): stoploss = self.edge.stoploss(trade.pair) - should_sell = self.strategy.should_sell(trade, sell_rate, datetime.utcnow(), buy, sell, stoploss) + should_sell = \ + self.strategy.should_sell(trade, sell_rate, datetime.utcnow(), buy, sell, stoploss) else: - should_sell = self.strategy.should_sell(trade, sell_rate, datetime.utcnow(), buy, sell) + should_sell = \ + self.strategy.should_sell(trade, sell_rate, datetime.utcnow(), buy, sell) if should_sell.sell_flag: self.execute_sell(trade, sell_rate, should_sell.sell_type) diff --git a/freqtrade/optimize/__init__.py b/freqtrade/optimize/__init__.py index 967227805..74c842427 100644 --- a/freqtrade/optimize/__init__.py +++ b/freqtrade/optimize/__init__.py @@ -16,7 +16,6 @@ import arrow from freqtrade import misc, constants, OperationalException from freqtrade.exchange import Exchange from freqtrade.arguments import TimeRange -import importlib logger = logging.getLogger(__name__) diff --git a/freqtrade/strategy/interface.py b/freqtrade/strategy/interface.py index 13fe01a70..73bf2313d 100644 --- a/freqtrade/strategy/interface.py +++ b/freqtrade/strategy/interface.py @@ -210,8 +210,10 @@ class IStrategy(ABC): :return: True if trade should be sold, False otherwise """ current_profit = trade.calc_profit_percent(rate) - stoplossflag = self.stop_loss_reached(current_rate=rate, trade=trade, current_time=date, - current_profit=current_profit, force_stoploss=force_stoploss) + stoplossflag = \ + self.stop_loss_reached(current_rate=rate, trade=trade, current_time=date, + current_profit=current_profit, force_stoploss=force_stoploss) + if stoplossflag.sell_flag: return stoplossflag diff --git a/freqtrade/tests/conftest.py b/freqtrade/tests/conftest.py index f8f7729eb..99c90d00a 100644 --- a/freqtrade/tests/conftest.py +++ b/freqtrade/tests/conftest.py @@ -137,7 +137,7 @@ def default_conf(): "stoploss_range_min": -0.01, "stoploss_range_max": -0.1, "stoploss_range_step": -0.001, - "maximum_winrate": 0.80, + "maximum_winrate": 0.80, "min_trade_number": 15, "max_trade_duration_minute": 1440, "remove_pumps": True, From 0594deafc6df2c82fb1792ce21074be2d53e5a94 Mon Sep 17 00:00:00 2001 From: misagh Date: Wed, 26 Sep 2018 16:50:17 +0200 Subject: [PATCH 069/699] removing whitespaces and long lines --- freqtrade/edge/__init__.py | 67 ++++++++++++++++++++++---------------- 1 file changed, 39 insertions(+), 28 deletions(-) diff --git a/freqtrade/edge/__init__.py b/freqtrade/edge/__init__.py index 7cf2e62de..8990c6b35 100644 --- a/freqtrade/edge/__init__.py +++ b/freqtrade/edge/__init__.py @@ -5,7 +5,6 @@ from typing import Any, Dict import arrow from pandas import DataFrame -import pandas as pd import freqtrade.optimize as optimize from freqtrade.optimize.backtesting import BacktestResult @@ -40,7 +39,7 @@ class Edge(): self.edge_config = self.config.get('edge', {}) self._last_updated = None - self._cached_pairs : list = [] + self._cached_pairs: list = [] self._total_capital = self.edge_config['total_capital_in_stake_currency'] self._allowed_risk = self.edge_config['allowed_risk'] @@ -62,14 +61,15 @@ class Edge(): pairs = self.config['exchange']['pair_whitelist'] heartbeat = self.config['edge']['process_throttle_secs'] - if ((self._last_updated is not None) and (self._last_updated + heartbeat > arrow.utcnow().timestamp)): + if (self._last_updated is not None) and \ + (self._last_updated + heartbeat > arrow.utcnow().timestamp): return False data: Dict[str, Any] = {} logger.info('Using stake_currency: %s ...', self.config['stake_currency']) logger.info('Using stake_amount: %s ...', self.config['stake_amount']) logger.info('Using local backtesting data (using whitelist in given config) ...') - #TODO: add "timerange" to Edge config + # TODO: add "timerange" to Edge config timerange = Arguments.parse_timerange(None if self.config.get( 'timerange') is None else str(self.config.get('timerange'))) @@ -103,7 +103,6 @@ class Edge(): stoploss_range_step = float(self.edge_config.get('stoploss_range_step', -0.001)) stoploss_range = np.arange(stoploss_range_min, stoploss_range_max, stoploss_range_step) - ########################### Call out BSlap Loop instead of Original BT code trades: list = [] for pair, pair_data in preprocessed.items(): # Sorting dataframe by date and reset index @@ -114,7 +113,6 @@ class Edge(): self.populate_buy_trend(pair_data))[headers].copy() trades += self._find_trades_for_stoploss_range(ticker_data, pair, stoploss_range) - # Switch List of Trade Dicts (trades) to Dataframe # Fill missing, calculable columns, profit, duration , abs etc. @@ -126,7 +124,6 @@ class Edge(): trades_df = [] trades_df = DataFrame.from_records(trades_df, columns=BacktestResult._fields) - self._cached_pairs = self._process_expectancy(trades_df) self._last_updated = arrow.utcnow().timestamp return True @@ -146,7 +143,7 @@ class Edge(): if len(self._cached_pairs) == 0: self.calculate() edge_sorted_pairs = [x[0] for x in self._cached_pairs] - return [x for _, x in sorted(zip(edge_sorted_pairs,pairs), key=lambda pair: pair[0])] + return [x for _, x in sorted(zip(edge_sorted_pairs, pairs), key=lambda pair: pair[0])] def _fill_calculable_fields(self, result: DataFrame): """ @@ -173,9 +170,11 @@ class Edge(): close_fee = fee / 2 result['trade_duration'] = result['close_time'] - result['open_time'] - result['trade_duration'] = result['trade_duration'].map(lambda x: int(x.total_seconds() / 60)) - ## Spends, Takes, Profit, Absolute Profit + result['trade_duration'] = \ + result['trade_duration'].map(lambda x: int(x.total_seconds() / 60)) + + # Spends, Takes, Profit, Absolute Profit # Buy Price result['buy_vol'] = stake / result['open_rate'] # How many target are we buying @@ -188,8 +187,9 @@ class Edge(): result['sell_take'] = result['sell_sum'] - result['sell_fee'] # profit_percent - result['profit_percent'] = (result['sell_take'] - result['buy_spend']) \ - / result['buy_spend'] + result['profit_percent'] = \ + (result['sell_take'] - result['buy_spend']) / result['buy_spend'] + # Absolute profit result['profit_abs'] = result['sell_take'] - result['buy_spend'] @@ -198,8 +198,10 @@ class Edge(): def _process_expectancy(self, results: DataFrame) -> list: """ This is a temporary version of edge positioning calculation. - The function will be eventually moved to a plugin called Edge in order to calculate necessary WR, RRR and - other indictaors related to money management periodically (each X minutes) and keep it in a storage. + The function will be eventually moved to a plugin called Edge in order + to calculate necessary WR, RRR and + other indictaors related to money management periodically (each X minutes) + and keep it in a storage. The calulation will be done per pair and per strategy. """ @@ -238,21 +240,17 @@ class Edge(): # Risk Reward Ratio # 1 / ((loss money / losing trades) / (gained money / winning trades)) def risk_reward_ratio(x): - x = abs(1/ ((x[x<0].sum() / x[x < 0].count()) / (x[x > 0].sum() / x[x > 0].count()))) + x = abs(1 / ((x[x < 0].sum() / x[x < 0].count()) / (x[x > 0].sum() / x[x > 0].count()))) return x ############################## # Required Risk Reward # (1/(winrate - 1) def required_risk_reward(x): - x = (1/(x[x > 0].count()/x.count()) -1) + x = (1 / (x[x > 0].count() / x.count()) - 1) return x ############################## - def delta(x): - x = (abs(1/ ((x[x < 0].sum() / x[x < 0].count()) / (x[x > 0].sum() / x[x > 0].count())))) - (1/(x[x > 0].count()/x.count()) -1) - return x - # Expectancy # Tells you the interest percentage you should hope # E.x. if expectancy is 0.35, on $1 trade you should expect a target of $1.35 @@ -265,7 +263,7 @@ class Edge(): ############################## final = results.groupby(['pair', 'stoploss'])['profit_abs'].\ - agg([winrate, risk_reward_ratio, required_risk_reward, expectancy, delta]).\ + agg([winrate, risk_reward_ratio, required_risk_reward, expectancy]).\ reset_index().sort_values(by=['expectancy', 'stoploss'], ascending=False)\ .groupby('pair').first().sort_values(by=['expectancy'], ascending=False) @@ -277,17 +275,29 @@ class Edge(): sell_column = ticker_data['sell'].values date_column = ticker_data['date'].values ohlc_columns = ticker_data[['open', 'high', 'low', 'close']].values - + result: list = [] for stoploss in stoploss_range: - result += self._detect_stop_and_sell_points(buy_column, sell_column, date_column, ohlc_columns, round(stoploss, 6), pair) + result += self._detect_stop_and_sell_points( + buy_column, sell_column, date_column, ohlc_columns, round(stoploss, 6), pair + ) return result - def _detect_stop_and_sell_points(self, buy_column, sell_column, date_column, ohlc_columns, stoploss, pair, start_point=0): + def _detect_stop_and_sell_points( + self, + buy_column, + sell_column, + date_column, + ohlc_columns, + stoploss, + pair, + start_point=0 + ): + result: list = [] open_trade_index = utf1st.find_1st(buy_column, 1, utf1st.cmp_equal) - #open_trade_index = np.argmax(buy_column == 1) + # open_trade_index = np.argmax(buy_column == 1) # return empty if we don't find trade entry (i.e. buy==1) if open_trade_index == -1: @@ -298,13 +308,14 @@ class Edge(): stop_price = (open_price * stop_price_percentage) # Searching for the index where stoploss is hit - stop_index = utf1st.find_1st(ohlc_columns[open_trade_index + 1:, 2], stop_price, utf1st.cmp_smaller) + stop_index = \ + utf1st.find_1st(ohlc_columns[open_trade_index + 1:, 2], stop_price, utf1st.cmp_smaller) # If we don't find it then we assume stop_index will be far in future (infinite number) if stop_index == -1: stop_index = float('inf') - #stop_index = np.argmax((ohlc_columns[open_trade_index + 1:, 2] < stop_price) == True) + # stop_index = np.argmax((ohlc_columns[open_trade_index + 1:, 2] < stop_price) == True) # Searching for the index where sell is hit sell_index = utf1st.find_1st(sell_column[open_trade_index + 1:], 1, utf1st.cmp_equal) @@ -313,7 +324,7 @@ class Edge(): if sell_index == -1: sell_index = float('inf') - #sell_index = np.argmax(sell_column[open_trade_index + 1:] == 1) + # sell_index = np.argmax(sell_column[open_trade_index + 1:] == 1) # Check if we don't find any stop or sell point (in that case trade remains open) # It is not interesting for Edge to consider it so we simply ignore the trade From 24364a56ea7dfcbef99eb50be69a334fff7fa43a Mon Sep 17 00:00:00 2001 From: misagh Date: Wed, 26 Sep 2018 17:03:10 +0200 Subject: [PATCH 070/699] keeping mypy happy --- freqtrade/edge/__init__.py | 10 +++++----- freqtrade/freqtradebot.py | 6 +++--- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/freqtrade/edge/__init__.py b/freqtrade/edge/__init__.py index 8990c6b35..1f2633dde 100644 --- a/freqtrade/edge/__init__.py +++ b/freqtrade/edge/__init__.py @@ -33,8 +33,8 @@ class Edge(): self.ticker_interval = self.strategy.ticker_interval self.tickerdata_to_dataframe = self.strategy.tickerdata_to_dataframe self.get_timeframe = Backtesting.get_timeframe - self.populate_buy_trend = self.strategy.populate_buy_trend - self.populate_sell_trend = self.strategy.populate_sell_trend + self.advise_sell = self.strategy.advise_sell + self.advise_buy = self.strategy.advise_buy self.edge_config = self.config.get('edge', {}) @@ -108,9 +108,9 @@ class Edge(): # Sorting dataframe by date and reset index pair_data = pair_data.sort_values(by=['date']) pair_data = pair_data.reset_index(drop=True) - - ticker_data = self.populate_sell_trend( - self.populate_buy_trend(pair_data))[headers].copy() + + ticker_data = self.advise_sell( + self.advise_buy(pair_data, {'pair': pair}), {'pair': pair})[headers].copy() trades += self._find_trades_for_stoploss_range(ticker_data, pair, stoploss_range) diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index be02db9a8..77e2b4915 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -338,11 +338,11 @@ class FreqtradeBot(object): if avaliable_amount < stake_amount: raise DependencyException( 'Available balance(%f %s) is lower than stake amount(%f %s)' % ( - avaliable_amount, self.config['stake_currency'], - stake_amount, self.config['stake_currency']) + float(avaliable_amount), self.config['stake_currency'], + float(stake_amount), self.config['stake_currency']) ) - return stake_amount + return float(stake_amount) def _get_min_pair_stake_amount(self, pair: str, price: float) -> Optional[float]: markets = self.exchange.get_markets() From 25d6ed319a871fbf749a6ec2c5b30fa6280ac4cf Mon Sep 17 00:00:00 2001 From: misagh Date: Wed, 26 Sep 2018 17:09:20 +0200 Subject: [PATCH 071/699] whitespace removed --- freqtrade/edge/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/edge/__init__.py b/freqtrade/edge/__init__.py index 1f2633dde..369b90ce9 100644 --- a/freqtrade/edge/__init__.py +++ b/freqtrade/edge/__init__.py @@ -108,7 +108,7 @@ class Edge(): # Sorting dataframe by date and reset index pair_data = pair_data.sort_values(by=['date']) pair_data = pair_data.reset_index(drop=True) - + ticker_data = self.advise_sell( self.advise_buy(pair_data, {'pair': pair}), {'pair': pair})[headers].copy() From 21f5a94ecaf5dca251bc44efe53c58c5f0a5028c Mon Sep 17 00:00:00 2001 From: misagh Date: Thu, 27 Sep 2018 12:23:46 +0200 Subject: [PATCH 072/699] using autopep8 for formatting file --- freqtrade/edge/__init__.py | 31 +++++++++++++++---------------- 1 file changed, 15 insertions(+), 16 deletions(-) diff --git a/freqtrade/edge/__init__.py b/freqtrade/edge/__init__.py index 369b90ce9..e525b224e 100644 --- a/freqtrade/edge/__init__.py +++ b/freqtrade/edge/__init__.py @@ -4,6 +4,8 @@ import logging from typing import Any, Dict import arrow +import numpy as np +import utils_find_1st as utf1st from pandas import DataFrame import freqtrade.optimize as optimize @@ -14,8 +16,6 @@ from freqtrade.strategy.interface import SellType from freqtrade.strategy.resolver import IStrategy, StrategyResolver from freqtrade.optimize.backtesting import Backtesting -import numpy as np -import utils_find_1st as utf1st logger = logging.getLogger(__name__) @@ -61,8 +61,8 @@ class Edge(): pairs = self.config['exchange']['pair_whitelist'] heartbeat = self.config['edge']['process_throttle_secs'] - if (self._last_updated is not None) and \ - (self._last_updated + heartbeat > arrow.utcnow().timestamp): + if (self._last_updated is not None) and ( + self._last_updated + heartbeat > arrow.utcnow().timestamp): return False data: Dict[str, Any] = {} @@ -78,6 +78,7 @@ class Edge(): pairs=pairs, ticker_interval=self.ticker_interval, refresh_pairs=self.config.get('refresh_pairs', False), + # refresh_pairs=True, exchange=self.exchange, timerange=timerange ) @@ -171,8 +172,8 @@ class Edge(): result['trade_duration'] = result['close_time'] - result['open_time'] - result['trade_duration'] = \ - result['trade_duration'].map(lambda x: int(x.total_seconds() / 60)) + result['trade_duration'] = result['trade_duration'].map( + lambda x: int(x.total_seconds() / 60)) # Spends, Takes, Profit, Absolute Profit @@ -187,8 +188,7 @@ class Edge(): result['sell_take'] = result['sell_sum'] - result['sell_fee'] # profit_percent - result['profit_percent'] = \ - (result['sell_take'] - result['buy_spend']) / result['buy_spend'] + result['profit_percent'] = (result['sell_take'] - result['buy_spend']) / result['buy_spend'] # Absolute profit result['profit_abs'] = result['sell_take'] - result['buy_spend'] @@ -222,7 +222,7 @@ class Edge(): # # Removing Pumps if self.edge_config.get('remove_pumps', True): - results = results[results.profit_abs < float(avg + 2*std)] + results = results[results.profit_abs < float(avg + 2 * std)] ########################################################################## # Removing trades having a duration more than X minutes (set in config) @@ -257,8 +257,8 @@ class Edge(): def expectancy(x): average_win = float(x[x > 0].sum() / x[x > 0].count()) average_loss = float(abs(x[x < 0].sum() / x[x < 0].count())) - winrate = float(x[x > 0].count()/x.count()) - x = ((1 + average_win/average_loss) * winrate) - 1 + winrate = float(x[x > 0].count() / x.count()) + x = ((1 + average_win / average_loss) * winrate) - 1 return x ############################## @@ -280,7 +280,7 @@ class Edge(): for stoploss in stoploss_range: result += self._detect_stop_and_sell_points( buy_column, sell_column, date_column, ohlc_columns, round(stoploss, 6), pair - ) + ) return result @@ -292,8 +292,7 @@ class Edge(): ohlc_columns, stoploss, pair, - start_point=0 - ): + start_point=0): result: list = [] open_trade_index = utf1st.find_1st(buy_column, 1, utf1st.cmp_equal) @@ -308,8 +307,8 @@ class Edge(): stop_price = (open_price * stop_price_percentage) # Searching for the index where stoploss is hit - stop_index = \ - utf1st.find_1st(ohlc_columns[open_trade_index + 1:, 2], stop_price, utf1st.cmp_smaller) + stop_index = utf1st.find_1st( + ohlc_columns[open_trade_index + 1:, 2], stop_price, utf1st.cmp_smaller) # If we don't find it then we assume stop_index will be far in future (infinite number) if stop_index == -1: From 96a0fc88cbbcd446aa4db0f31d2fbb8ee48bd9b5 Mon Sep 17 00:00:00 2001 From: misagh Date: Fri, 28 Sep 2018 14:19:22 +0200 Subject: [PATCH 073/699] Moving Edge before refresh_pairs see comments on edge (line 129) --- freqtrade/edge/__init__.py | 16 ++++++++++++++-- freqtrade/freqtradebot.py | 9 ++++++--- 2 files changed, 20 insertions(+), 5 deletions(-) diff --git a/freqtrade/edge/__init__.py b/freqtrade/edge/__init__.py index e525b224e..7d915a984 100644 --- a/freqtrade/edge/__init__.py +++ b/freqtrade/edge/__init__.py @@ -77,8 +77,7 @@ class Edge(): self.config['datadir'], pairs=pairs, ticker_interval=self.ticker_interval, - refresh_pairs=self.config.get('refresh_pairs', False), - # refresh_pairs=True, + refresh_pairs=True, exchange=self.exchange, timerange=timerange ) @@ -127,6 +126,19 @@ class Edge(): self._cached_pairs = self._process_expectancy(trades_df) self._last_updated = arrow.utcnow().timestamp + + # Not a nice hack but probably simplest solution: + # When backtest load data it loads the delta between disk and exchange + # The problem is that exchange consider that recent. it is but it is incomplete (c.f. _async_get_candle_history) + # So it causes get_signal to exit cause incomplete ticker_hist + # A patch to that would be update _pairs_last_refresh_time of exchange so it will download again all pairs + # Another solution is to add new data to klines instead of reassigning it: + # self.klines[pair].update(data) instead of self.klines[pair] = data in exchange package. + # But that means indexing timestamp and having a verification so that + # there is no empty range between two timestaps (recently added and last + # one) + self.exchange._pairs_last_refresh_time = {} + return True def stake_amount(self, pair: str) -> str: diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index cc9343659..9ac0fd02f 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -185,13 +185,16 @@ class FreqtradeBot(object): final_list = sanitized_list[:nb_assets] if nb_assets else sanitized_list self.config['exchange']['pair_whitelist'] = final_list - # Refreshing candles - self.exchange.refresh_tickers(final_list, self.strategy.ticker_interval) - # Calculating Edge positiong + # Should be called before refresh_tickers + # Otherwise it will override cached klines in exchange + # with delta value (klines only from last refresh_pairs) if self.config['edge']['enabled']: self.edge.calculate() + # Refreshing candles + self.exchange.refresh_tickers(final_list, self.strategy.ticker_interval) + # Query trades from persistence layer trades = Trade.query.filter(Trade.is_open.is_(True)).all() From e822d5d721874eec94192a36fd598c56b9a26f50 Mon Sep 17 00:00:00 2001 From: misagh Date: Fri, 28 Sep 2018 14:23:39 +0200 Subject: [PATCH 074/699] upgrading py_first_1st to 1.1.2: ez_setup.py removed --- requirements.txt | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/requirements.txt b/requirements.txt index 3b36fa5d3..d09167155 100644 --- a/requirements.txt +++ b/requirements.txt @@ -26,8 +26,7 @@ scikit-optimize==0.5.2 #plotly==3.1.1 # find first, C search in arrays -py_find_1st==1.1.1 +py_find_1st==1.1.2 #Load ticker files 30% faster ujson==1.35 - From f15825e3a772a02194f9a509a0ca8bbac0fe31f2 Mon Sep 17 00:00:00 2001 From: misagh Date: Fri, 28 Sep 2018 14:28:05 +0200 Subject: [PATCH 075/699] long line broken to two --- freqtrade/edge/__init__.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/freqtrade/edge/__init__.py b/freqtrade/edge/__init__.py index 7d915a984..f9549cd90 100644 --- a/freqtrade/edge/__init__.py +++ b/freqtrade/edge/__init__.py @@ -129,9 +129,11 @@ class Edge(): # Not a nice hack but probably simplest solution: # When backtest load data it loads the delta between disk and exchange - # The problem is that exchange consider that recent. it is but it is incomplete (c.f. _async_get_candle_history) + # The problem is that exchange consider that recent. + # it is but it is incomplete (c.f. _async_get_candle_history) # So it causes get_signal to exit cause incomplete ticker_hist - # A patch to that would be update _pairs_last_refresh_time of exchange so it will download again all pairs + # A patch to that would be update _pairs_last_refresh_time of exchange + # so it will download again all pairs # Another solution is to add new data to klines instead of reassigning it: # self.klines[pair].update(data) instead of self.klines[pair] = data in exchange package. # But that means indexing timestamp and having a verification so that From c8d06e2b0e4db96e87f09a31a0ea07fe6dcfa900 Mon Sep 17 00:00:00 2001 From: misagh Date: Fri, 28 Sep 2018 16:40:34 +0200 Subject: [PATCH 076/699] filter pairs according to expectancy + bug at the end of array resolved --- config.json.example | 2 ++ freqtrade/edge/__init__.py | 18 +++++++++++------- freqtrade/freqtradebot.py | 2 +- freqtrade/tests/conftest.py | 2 ++ 4 files changed, 16 insertions(+), 8 deletions(-) diff --git a/config.json.example b/config.json.example index 83e9e0d38..2c1ef4fd0 100644 --- a/config.json.example +++ b/config.json.example @@ -53,12 +53,14 @@ "edge": { "enabled": false, "process_throttle_secs": 1800, + "calculate_since_number_of_days": 14, "total_capital_in_stake_currency": 0.5, "allowed_risk": 0.01, "stoploss_range_min": -0.01, "stoploss_range_max": -0.1, "stoploss_range_step": -0.001, "maximum_winrate": 0.80, + "minimum_expectancy": 0.20, "min_trade_number": 15, "max_trade_duration_minute": 1440, "remove_pumps": true, diff --git a/freqtrade/edge/__init__.py b/freqtrade/edge/__init__.py index f9549cd90..9789f2991 100644 --- a/freqtrade/edge/__init__.py +++ b/freqtrade/edge/__init__.py @@ -154,11 +154,14 @@ class Edge(): info = [x for x in self._cached_pairs if x[0] == pair][0] return info[1] - def sort_pairs(self, pairs) -> list: - if len(self._cached_pairs) == 0: - self.calculate() - edge_sorted_pairs = [x[0] for x in self._cached_pairs] - return [x for _, x in sorted(zip(edge_sorted_pairs, pairs), key=lambda pair: pair[0])] + def filter(self, pairs) -> list: + # Filtering pairs acccording to the expectancy + filtered_expectancy: list = [] + filtered_expectancy = [x[0] for x in self._cached_pairs if x[5] > float(self.edge_config.get('minimum_expectancy', 0.2))] + + # Only return pairs which are included in "pairs" argument list + final = [x for x in filtered_expectancy if x in pairs] + return final def _fill_calculable_fields(self, result: DataFrame): """ @@ -312,8 +315,9 @@ class Edge(): open_trade_index = utf1st.find_1st(buy_column, 1, utf1st.cmp_equal) # open_trade_index = np.argmax(buy_column == 1) - # return empty if we don't find trade entry (i.e. buy==1) - if open_trade_index == -1: + # return empty if we don't find trade entry (i.e. buy==1) or + # we find a buy but at the of array + if open_trade_index == -1 or open_trade_index == len(buy_column) - 1: return [] stop_price_percentage = stoploss + 1 diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index 9ac0fd02f..299954e49 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -399,7 +399,7 @@ class FreqtradeBot(object): # running get_signal on historical data fetched # to find buy signals if self.config['edge']['enabled']: - whitelist = self.edge.sort_pairs(whitelist) + whitelist = self.edge.filter(whitelist) for _pair in whitelist: (buy, sell) = self.strategy.get_signal(_pair, interval, self.exchange.klines.get(_pair)) diff --git a/freqtrade/tests/conftest.py b/freqtrade/tests/conftest.py index 99c90d00a..c72cc5a40 100644 --- a/freqtrade/tests/conftest.py +++ b/freqtrade/tests/conftest.py @@ -132,12 +132,14 @@ def default_conf(): "edge": { "enabled": False, "process_throttle_secs": 1800, + "calculate_since_number_of_days": 14, "total_capital_in_stake_currency": 0.5, "allowed_risk": 0.01, "stoploss_range_min": -0.01, "stoploss_range_max": -0.1, "stoploss_range_step": -0.001, "maximum_winrate": 0.80, + "minimum_expectancy": 0.20, "min_trade_number": 15, "max_trade_duration_minute": 1440, "remove_pumps": True, From cff83d3e6f2ff31ab7a4f4c17adfc6c2e05f7f82 Mon Sep 17 00:00:00 2001 From: misagh Date: Fri, 28 Sep 2018 16:46:42 +0200 Subject: [PATCH 077/699] bloody autopep8 again --- freqtrade/edge/__init__.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/freqtrade/edge/__init__.py b/freqtrade/edge/__init__.py index 9789f2991..d675d1659 100644 --- a/freqtrade/edge/__init__.py +++ b/freqtrade/edge/__init__.py @@ -157,7 +157,10 @@ class Edge(): def filter(self, pairs) -> list: # Filtering pairs acccording to the expectancy filtered_expectancy: list = [] - filtered_expectancy = [x[0] for x in self._cached_pairs if x[5] > float(self.edge_config.get('minimum_expectancy', 0.2))] + filtered_expectancy = [ + x[0] for x in self._cached_pairs if x[5] > float( + self.edge_config.get( + 'minimum_expectancy', 0.2))] # Only return pairs which are included in "pairs" argument list final = [x for x in filtered_expectancy if x in pairs] From 2a9ca9a3dc43adf14cca2861ca04ea295d25142f Mon Sep 17 00:00:00 2001 From: misagh Date: Mon, 1 Oct 2018 17:09:08 +0200 Subject: [PATCH 078/699] Removing future from travis and dockerfile --- .travis.yml | 1 - Dockerfile | 1 - 2 files changed, 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index c81c96460..981eedcf8 100644 --- a/.travis.yml +++ b/.travis.yml @@ -14,7 +14,6 @@ install: - ./install_ta-lib.sh - export LD_LIBRARY_PATH=/usr/local/lib:$LD_LIBRARY_PATH - pip install --upgrade flake8 coveralls pytest-random-order pytest-asyncio mypy -- pip install future - pip install -r requirements.txt - pip install -e . jobs: diff --git a/Dockerfile b/Dockerfile index 0e90f0720..2506665ab 100644 --- a/Dockerfile +++ b/Dockerfile @@ -17,7 +17,6 @@ WORKDIR /freqtrade # Install dependencies COPY requirements.txt /freqtrade/ RUN pip install numpy --no-cache-dir \ - && pip install future \ && pip install -r requirements.txt --no-cache-dir # Install and execute From aa1948750f16ec1d748c09117913b23e8077ab23 Mon Sep 17 00:00:00 2001 From: misagh Date: Mon, 1 Oct 2018 17:11:48 +0200 Subject: [PATCH 079/699] removing unnecessary constructor docstring --- freqtrade/edge/__init__.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/freqtrade/edge/__init__.py b/freqtrade/edge/__init__.py index d675d1659..c52700d44 100644 --- a/freqtrade/edge/__init__.py +++ b/freqtrade/edge/__init__.py @@ -23,11 +23,9 @@ logger = logging.getLogger(__name__) class Edge(): config: Dict = {} + _cached_pairs: list = [] def __init__(self, config: Dict[str, Any], exchange=None) -> None: - """ - constructor - """ self.config = config self.strategy: IStrategy = StrategyResolver(self.config).strategy self.ticker_interval = self.strategy.ticker_interval From 114fd7feef0d21c579e109a1d8be2a3b992e7ffa Mon Sep 17 00:00:00 2001 From: misagh Date: Mon, 1 Oct 2018 17:21:40 +0200 Subject: [PATCH 080/699] declaring local variables. using get for configuration --- freqtrade/edge/__init__.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/freqtrade/edge/__init__.py b/freqtrade/edge/__init__.py index c52700d44..f07c34427 100644 --- a/freqtrade/edge/__init__.py +++ b/freqtrade/edge/__init__.py @@ -23,7 +23,12 @@ logger = logging.getLogger(__name__) class Edge(): config: Dict = {} - _cached_pairs: list = [] + _last_updated: int # Timestamp of pairs last updated time + _cached_pairs: list = [] # Keeps an array of + # [pair, winrate, risk reward ratio, required risk reward, expectancy] + + _total_capital: float + _allowed_risk: float def __init__(self, config: Dict[str, Any], exchange=None) -> None: self.config = config @@ -35,11 +40,10 @@ class Edge(): self.advise_buy = self.strategy.advise_buy self.edge_config = self.config.get('edge', {}) - self._last_updated = None self._cached_pairs: list = [] - self._total_capital = self.edge_config['total_capital_in_stake_currency'] - self._allowed_risk = self.edge_config['allowed_risk'] + self._total_capital = self.edge_config.get('total_capital_in_stake_currency') + self._allowed_risk = self.edge_config.get('allowed_risk') ### # From f72fb0ad04faa56b6a53f1e8f382bfeba7f05017 Mon Sep 17 00:00:00 2001 From: misagh Date: Mon, 1 Oct 2018 17:29:33 +0200 Subject: [PATCH 081/699] =?UTF-8?q?exchange=20=E2=80=9CNone=E2=80=9D=20con?= =?UTF-8?q?dition=20removed=20as=20Edge=20is=20after=20Exchange=20anyway?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- freqtrade/edge/__init__.py | 22 ++++------------------ 1 file changed, 4 insertions(+), 18 deletions(-) diff --git a/freqtrade/edge/__init__.py b/freqtrade/edge/__init__.py index f07c34427..a452e3751 100644 --- a/freqtrade/edge/__init__.py +++ b/freqtrade/edge/__init__.py @@ -32,6 +32,7 @@ class Edge(): def __init__(self, config: Dict[str, Any], exchange=None) -> None: self.config = config + self.exchange = exchange self.strategy: IStrategy = StrategyResolver(self.config).strategy self.ticker_interval = self.strategy.ticker_interval self.tickerdata_to_dataframe = self.strategy.tickerdata_to_dataframe @@ -45,23 +46,11 @@ class Edge(): self._total_capital = self.edge_config.get('total_capital_in_stake_currency') self._allowed_risk = self.edge_config.get('allowed_risk') - ### - # - ### - if exchange is None: - self.config['exchange']['secret'] = '' - self.config['exchange']['password'] = '' - self.config['exchange']['uid'] = '' - self.config['dry_run'] = True - self.exchange = Exchange(self.config) - else: - self.exchange = exchange - self.fee = self.exchange.get_fee() def calculate(self) -> bool: pairs = self.config['exchange']['pair_whitelist'] - heartbeat = self.config['edge']['process_throttle_secs'] + heartbeat = self.edge_config.get('process_throttle_secs') if (self._last_updated is not None) and ( self._last_updated + heartbeat > arrow.utcnow().timestamp): @@ -318,7 +307,6 @@ class Edge(): result: list = [] open_trade_index = utf1st.find_1st(buy_column, 1, utf1st.cmp_equal) - # open_trade_index = np.argmax(buy_column == 1) # return empty if we don't find trade entry (i.e. buy==1) or # we find a buy but at the of array @@ -337,8 +325,6 @@ class Edge(): if stop_index == -1: stop_index = float('inf') - # stop_index = np.argmax((ohlc_columns[open_trade_index + 1:, 2] < stop_price) == True) - # Searching for the index where sell is hit sell_index = utf1st.find_1st(sell_column[open_trade_index + 1:], 1, utf1st.cmp_equal) @@ -346,8 +332,6 @@ class Edge(): if sell_index == -1: sell_index = float('inf') - # sell_index = np.argmax(sell_column[open_trade_index + 1:] == 1) - # Check if we don't find any stop or sell point (in that case trade remains open) # It is not interesting for Edge to consider it so we simply ignore the trade # And stop iterating as the party is over @@ -379,6 +363,8 @@ class Edge(): result.append(trade) + # Calling again the same function recursively but giving + # it a view of exit_index till the end of array return result + self._detect_stop_and_sell_points( buy_column[exit_index:], sell_column[exit_index:], From ad666ac65c6a5bd1f5cd5a1357dbf9903e176b34 Mon Sep 17 00:00:00 2001 From: misagh Date: Mon, 1 Oct 2018 17:33:18 +0200 Subject: [PATCH 082/699] autopep8 corrected --- freqtrade/freqtradebot.py | 7 +++---- freqtrade/strategy/interface.py | 9 ++++++--- 2 files changed, 9 insertions(+), 7 deletions(-) diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index 299954e49..6981dc6df 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -638,11 +638,10 @@ class FreqtradeBot(object): def check_sell(self, trade: Trade, sell_rate: float, buy: bool, sell: bool) -> bool: if (self.config['edge']['enabled']): stoploss = self.edge.stoploss(trade.pair) - should_sell = \ - self.strategy.should_sell(trade, sell_rate, datetime.utcnow(), buy, sell, stoploss) + should_sell = self.strategy.should_sell( + trade, sell_rate, datetime.utcnow(), buy, sell, stoploss) else: - should_sell = \ - self.strategy.should_sell(trade, sell_rate, datetime.utcnow(), buy, sell) + should_sell = self.strategy.should_sell(trade, sell_rate, datetime.utcnow(), buy, sell) if should_sell.sell_flag: self.execute_sell(trade, sell_rate, should_sell.sell_type) diff --git a/freqtrade/strategy/interface.py b/freqtrade/strategy/interface.py index 73bf2313d..bd42f6c7e 100644 --- a/freqtrade/strategy/interface.py +++ b/freqtrade/strategy/interface.py @@ -210,9 +210,12 @@ class IStrategy(ABC): :return: True if trade should be sold, False otherwise """ current_profit = trade.calc_profit_percent(rate) - stoplossflag = \ - self.stop_loss_reached(current_rate=rate, trade=trade, current_time=date, - current_profit=current_profit, force_stoploss=force_stoploss) + stoplossflag = self.stop_loss_reached( + current_rate=rate, + trade=trade, + current_time=date, + current_profit=current_profit, + force_stoploss=force_stoploss) if stoplossflag.sell_flag: return stoplossflag From 2056b6f5f1b95d443cca9878a0abb58a8171b646 Mon Sep 17 00:00:00 2001 From: misagh Date: Mon, 1 Oct 2018 17:35:27 +0200 Subject: [PATCH 083/699] no need to initialize a variable with None --- freqtrade/edge/__init__.py | 1 - 1 file changed, 1 deletion(-) diff --git a/freqtrade/edge/__init__.py b/freqtrade/edge/__init__.py index a452e3751..3ab0f302a 100644 --- a/freqtrade/edge/__init__.py +++ b/freqtrade/edge/__init__.py @@ -41,7 +41,6 @@ class Edge(): self.advise_buy = self.strategy.advise_buy self.edge_config = self.config.get('edge', {}) - self._last_updated = None self._cached_pairs: list = [] self._total_capital = self.edge_config.get('total_capital_in_stake_currency') self._allowed_risk = self.edge_config.get('allowed_risk') From 8b3631d1ac3b490456dd77d8a2e1ee828408db97 Mon Sep 17 00:00:00 2001 From: misagh Date: Mon, 1 Oct 2018 17:49:27 +0200 Subject: [PATCH 084/699] =?UTF-8?q?make=20=E2=80=9Cif=20condition=E2=80=9D?= =?UTF-8?q?=20more=20readable?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- freqtrade/strategy/interface.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/freqtrade/strategy/interface.py b/freqtrade/strategy/interface.py index bd42f6c7e..4d613265a 100644 --- a/freqtrade/strategy/interface.py +++ b/freqtrade/strategy/interface.py @@ -251,10 +251,8 @@ class IStrategy(ABC): trailing_stop = self.config.get('trailing_stop', False) - if force_stoploss == 0: - trade.adjust_stop_loss(trade.open_rate, self.stoploss, initial=True) - else: - trade.adjust_stop_loss(trade.open_rate, force_stoploss, initial=True) + trade.adjust_stop_loss(trade.open_rate, force_stoploss if force_stoploss + else self.stoploss, initial=True) # evaluate if the stoploss was hit if self.stoploss is not None and trade.stop_loss >= current_rate: From f306abb3ee19fb17b42b462d5b05ae284f0a9289 Mon Sep 17 00:00:00 2001 From: misagh Date: Mon, 1 Oct 2018 17:52:07 +0200 Subject: [PATCH 085/699] No need for Exchange class in Edge --- freqtrade/edge/__init__.py | 1 - 1 file changed, 1 deletion(-) diff --git a/freqtrade/edge/__init__.py b/freqtrade/edge/__init__.py index 3ab0f302a..029f2eb43 100644 --- a/freqtrade/edge/__init__.py +++ b/freqtrade/edge/__init__.py @@ -11,7 +11,6 @@ from pandas import DataFrame import freqtrade.optimize as optimize from freqtrade.optimize.backtesting import BacktestResult from freqtrade.arguments import Arguments -from freqtrade.exchange import Exchange from freqtrade.strategy.interface import SellType from freqtrade.strategy.resolver import IStrategy, StrategyResolver from freqtrade.optimize.backtesting import Backtesting From 4a9ed02b9b1b5c92f9438d078a72b1091ebc4f89 Mon Sep 17 00:00:00 2001 From: Samuel Husso Date: Tue, 2 Oct 2018 09:18:54 +0300 Subject: [PATCH 086/699] develop to version 0.17.3 --- freqtrade/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/__init__.py b/freqtrade/__init__.py index 46825f548..89145d9e2 100644 --- a/freqtrade/__init__.py +++ b/freqtrade/__init__.py @@ -1,5 +1,5 @@ """ FreqTrade bot """ -__version__ = '0.17.2' +__version__ = '0.17.3' class DependencyException(BaseException): From a6c2e40bd43ac0b4be94031edacace028ab95c26 Mon Sep 17 00:00:00 2001 From: misagh Date: Tue, 2 Oct 2018 11:49:49 +0200 Subject: [PATCH 087/699] moving time range to initializer as we have to calculate it once --- freqtrade/edge/__init__.py | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/freqtrade/edge/__init__.py b/freqtrade/edge/__init__.py index 029f2eb43..d2f0ff4a2 100644 --- a/freqtrade/edge/__init__.py +++ b/freqtrade/edge/__init__.py @@ -11,6 +11,7 @@ from pandas import DataFrame import freqtrade.optimize as optimize from freqtrade.optimize.backtesting import BacktestResult from freqtrade.arguments import Arguments +from freqtrade.arguments import TimeRange from freqtrade.strategy.interface import SellType from freqtrade.strategy.resolver import IStrategy, StrategyResolver from freqtrade.optimize.backtesting import Backtesting @@ -28,6 +29,8 @@ class Edge(): _total_capital: float _allowed_risk: float + _since_number_of_days: int + _timerange: TimeRange def __init__(self, config: Dict[str, Any], exchange=None) -> None: self.config = config @@ -43,6 +46,11 @@ class Edge(): self._cached_pairs: list = [] self._total_capital = self.edge_config.get('total_capital_in_stake_currency') self._allowed_risk = self.edge_config.get('allowed_risk') + self._since_number_of_days = self.edge_config.get('since_number_of_days', 14) + self._last_updated = 0 + + self._timerange = Arguments.parse_timerange("%s-" % arrow.now().shift( + days=-1 * self._since_number_of_days).format('YYYYMMDD')) self.fee = self.exchange.get_fee() @@ -50,17 +58,13 @@ class Edge(): pairs = self.config['exchange']['pair_whitelist'] heartbeat = self.edge_config.get('process_throttle_secs') - if (self._last_updated is not None) and ( + if (self._last_updated > 0) and ( self._last_updated + heartbeat > arrow.utcnow().timestamp): return False data: Dict[str, Any] = {} logger.info('Using stake_currency: %s ...', self.config['stake_currency']) - logger.info('Using stake_amount: %s ...', self.config['stake_amount']) logger.info('Using local backtesting data (using whitelist in given config) ...') - # TODO: add "timerange" to Edge config - timerange = Arguments.parse_timerange(None if self.config.get( - 'timerange') is None else str(self.config.get('timerange'))) data = optimize.load_data( self.config['datadir'], @@ -68,7 +72,7 @@ class Edge(): ticker_interval=self.ticker_interval, refresh_pairs=True, exchange=self.exchange, - timerange=timerange + timerange=self._timerange ) if not data: @@ -332,7 +336,7 @@ class Edge(): # Check if we don't find any stop or sell point (in that case trade remains open) # It is not interesting for Edge to consider it so we simply ignore the trade - # And stop iterating as the party is over + # And stop iterating there is no more entry if stop_index == sell_index == float('inf'): return [] @@ -343,7 +347,7 @@ class Edge(): elif stop_index > sell_index: exit_index = open_trade_index + sell_index + 1 exit_type = SellType.SELL_SIGNAL - exit_price = ohlc_columns[open_trade_index + sell_index + 1, 0] + exit_price = ohlc_columns[exit_index, 0] trade = {'pair': pair, 'stoploss': stoploss, From 11c3b3fdb90a4d95d2e07d38e8ecd25b2d1fec3d Mon Sep 17 00:00:00 2001 From: misagh Date: Tue, 2 Oct 2018 11:53:16 +0200 Subject: [PATCH 088/699] trade_df unnecessary type removed --- freqtrade/edge/__init__.py | 1 - 1 file changed, 1 deletion(-) diff --git a/freqtrade/edge/__init__.py b/freqtrade/edge/__init__.py index d2f0ff4a2..3d85e2286 100644 --- a/freqtrade/edge/__init__.py +++ b/freqtrade/edge/__init__.py @@ -114,7 +114,6 @@ class Edge(): if len(trades_df) > 0: # Only post process a frame if it has a record trades_df = self._fill_calculable_fields(trades_df) else: - trades_df = [] trades_df = DataFrame.from_records(trades_df, columns=BacktestResult._fields) self._cached_pairs = self._process_expectancy(trades_df) From e4fc298bd651ae1df588af3c9e684fefa4262e1b Mon Sep 17 00:00:00 2001 From: misagh Date: Tue, 2 Oct 2018 11:53:59 +0200 Subject: [PATCH 089/699] typo corrected --- freqtrade/edge/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/edge/__init__.py b/freqtrade/edge/__init__.py index 3d85e2286..90dd0d1ba 100644 --- a/freqtrade/edge/__init__.py +++ b/freqtrade/edge/__init__.py @@ -161,7 +161,7 @@ class Edge(): def _fill_calculable_fields(self, result: DataFrame): """ The result frame contains a number of columns that are calculable - from othe columns. These are left blank till all rows are added, + from other columns. These are left blank till all rows are added, to be populated in single vector calls. Columns to be populated are: From d634a034550077297fe7a1c2493041723997287f Mon Sep 17 00:00:00 2001 From: misagh Date: Tue, 2 Oct 2018 11:55:14 +0200 Subject: [PATCH 090/699] adding DataFrame type --- freqtrade/edge/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/edge/__init__.py b/freqtrade/edge/__init__.py index 90dd0d1ba..8e9d5861a 100644 --- a/freqtrade/edge/__init__.py +++ b/freqtrade/edge/__init__.py @@ -158,7 +158,7 @@ class Edge(): final = [x for x in filtered_expectancy if x in pairs] return final - def _fill_calculable_fields(self, result: DataFrame): + def _fill_calculable_fields(self, result: DataFrame) -> DataFrame: """ The result frame contains a number of columns that are calculable from other columns. These are left blank till all rows are added, From 9c4fdc1bc59d72938fa64097347b67a38806be26 Mon Sep 17 00:00:00 2001 From: misagh Date: Tue, 2 Oct 2018 12:15:54 +0200 Subject: [PATCH 091/699] initializing Edge in Freqtradebot only if it is enabled --- freqtrade/freqtradebot.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index 6981dc6df..ad3d7f5db 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -56,7 +56,11 @@ class FreqtradeBot(object): self.rpc: RPCManager = RPCManager(self) self.persistence = None self.exchange = Exchange(self.config) - self.edge = Edge(self.config, self.exchange) + + # Initializing Edge only if enabled + if self.config.get('edge', {}).get('enabled', False): + self.edge = Edge(self.config, self.exchange) + self._init_modules() def _init_modules(self) -> None: @@ -189,7 +193,7 @@ class FreqtradeBot(object): # Should be called before refresh_tickers # Otherwise it will override cached klines in exchange # with delta value (klines only from last refresh_pairs) - if self.config['edge']['enabled']: + if self.config.get('edge', {}).get('enabled', False): self.edge.calculate() # Refreshing candles From 3b57aef168d16a2a15c5db67ee24bee420ac84e7 Mon Sep 17 00:00:00 2001 From: misagh Date: Tue, 2 Oct 2018 12:16:09 +0200 Subject: [PATCH 092/699] config name refactored --- freqtrade/edge/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/edge/__init__.py b/freqtrade/edge/__init__.py index 8e9d5861a..ecc7544a9 100644 --- a/freqtrade/edge/__init__.py +++ b/freqtrade/edge/__init__.py @@ -46,7 +46,7 @@ class Edge(): self._cached_pairs: list = [] self._total_capital = self.edge_config.get('total_capital_in_stake_currency') self._allowed_risk = self.edge_config.get('allowed_risk') - self._since_number_of_days = self.edge_config.get('since_number_of_days', 14) + self._since_number_of_days = self.edge_config.get('calculate_since_number_of_days', 14) self._last_updated = 0 self._timerange = Arguments.parse_timerange("%s-" % arrow.now().shift( From 26b3c3f7a8f8068443d3adaa29b31d340bc55fe7 Mon Sep 17 00:00:00 2001 From: misagh Date: Tue, 2 Oct 2018 12:20:30 +0200 Subject: [PATCH 093/699] removing unnecessary typing --- freqtrade/freqtradebot.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index ad3d7f5db..be4b35a20 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -345,8 +345,8 @@ class FreqtradeBot(object): if avaliable_amount < stake_amount: raise DependencyException( 'Available balance(%f %s) is lower than stake amount(%f %s)' % ( - float(avaliable_amount), self.config['stake_currency'], - float(stake_amount), self.config['stake_currency']) + avaliable_amount, self.config['stake_currency'], + stake_amount, self.config['stake_currency']) ) return float(stake_amount) From 8741a63783c11ec3035a81ea4ba3b40ad15a38c3 Mon Sep 17 00:00:00 2001 From: misagh Date: Tue, 2 Oct 2018 12:20:48 +0200 Subject: [PATCH 094/699] return type of stake_amount set to float --- freqtrade/edge/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/edge/__init__.py b/freqtrade/edge/__init__.py index ecc7544a9..edcd66337 100644 --- a/freqtrade/edge/__init__.py +++ b/freqtrade/edge/__init__.py @@ -135,7 +135,7 @@ class Edge(): return True - def stake_amount(self, pair: str) -> str: + def stake_amount(self, pair: str) -> float: info = [x for x in self._cached_pairs if x[0] == pair][0] stoploss = info[1] allowed_capital_at_risk = round(self._total_capital * self._allowed_risk, 5) From 23f8980973343dc95748ff3f7fc71b7640f7b48e Mon Sep 17 00:00:00 2001 From: misagh Date: Tue, 2 Oct 2018 12:42:59 +0200 Subject: [PATCH 095/699] edge config added to CONF_SCHEMA and config_full.json.example --- config_full.json.example | 15 +++++++++++++++ freqtrade/constants.py | 21 ++++++++++++++++++++- freqtrade/tests/conftest.py | 5 ++--- 3 files changed, 37 insertions(+), 4 deletions(-) diff --git a/config_full.json.example b/config_full.json.example index 7083bada6..8541d984b 100644 --- a/config_full.json.example +++ b/config_full.json.example @@ -55,6 +55,21 @@ ], "outdated_offset": 5 }, + "edge": { + "enabled": false, + "process_throttle_secs": 1800, + "calculate_since_number_of_days": 14, + "total_capital_in_stake_currency": 0.5, + "allowed_risk": 0.01, + "stoploss_range_min": -0.01, + "stoploss_range_max": -0.1, + "stoploss_range_step": -0.01, + "maximum_winrate": 0.80, + "minimum_expectancy": 0.20, + "min_trade_number": 15, + "max_trade_duration_minute": 1440, + "remove_pumps": true + }, "experimental": { "use_sell_signal": false, "sell_profit_only": false, diff --git a/freqtrade/constants.py b/freqtrade/constants.py index eadfa6eba..5a03f81cd 100644 --- a/freqtrade/constants.py +++ b/freqtrade/constants.py @@ -37,7 +37,7 @@ SUPPORTED_FIAT = [ "KRW", "MXN", "MYR", "NOK", "NZD", "PHP", "PKR", "PLN", "RUB", "SEK", "SGD", "THB", "TRY", "TWD", "ZAR", "USD", "BTC", "XBT", "ETH", "XRP", "LTC", "BCH", "USDT" - ] +] # Required json-schema for user specified config CONF_SCHEMA = { @@ -102,6 +102,7 @@ CONF_SCHEMA = { } }, 'exchange': {'$ref': '#/definitions/exchange'}, + 'edge': {'$ref': '#/definitions/edge'}, 'experimental': { 'type': 'object', 'properties': { @@ -167,6 +168,24 @@ CONF_SCHEMA = { 'outdated_offset': {'type': 'integer', 'minimum': 1} }, 'required': ['name', 'key', 'secret', 'pair_whitelist'] + }, + 'edge': { + 'type': 'object', + 'properties': { + "enabled": {'type': 'boolean'}, + "process_throttle_secs": {'type': 'integer', 'minimum': 600}, + "calculate_since_number_of_days": {'type': 'integer'}, + "total_capital_in_stake_currency": {'type': 'number'}, + "allowed_risk": {'type': 'number'}, + "stoploss_range_min": {'type': 'number'}, + "stoploss_range_max": {'type': 'number'}, + "stoploss_range_step": {'type': 'number'}, + "maximum_winrate": {'type': 'number'}, + "minimum_expectancy": {'type': 'number'}, + "min_trade_number": {'type': 'number'}, + "max_trade_duration_minute": {'type': 'integer'}, + "remove_pumps": {'type': 'boolean'} + } } }, 'anyOf': [ diff --git a/freqtrade/tests/conftest.py b/freqtrade/tests/conftest.py index c72cc5a40..f6f067a41 100644 --- a/freqtrade/tests/conftest.py +++ b/freqtrade/tests/conftest.py @@ -137,13 +137,12 @@ def default_conf(): "allowed_risk": 0.01, "stoploss_range_min": -0.01, "stoploss_range_max": -0.1, - "stoploss_range_step": -0.001, + "stoploss_range_step": -0.01, "maximum_winrate": 0.80, "minimum_expectancy": 0.20, "min_trade_number": 15, "max_trade_duration_minute": 1440, - "remove_pumps": True, - "minimum_delta": 1 + "remove_pumps": True }, "telegram": { "enabled": True, From 697493bd017445339eaacfc26258a2ed2520cc69 Mon Sep 17 00:00:00 2001 From: misagh Date: Tue, 2 Oct 2018 16:07:33 +0200 Subject: [PATCH 096/699] test cases for Edge package drafted --- freqtrade/tests/edge/test_edge.py | 52 +++++++++++++++++++++++++++++++ 1 file changed, 52 insertions(+) create mode 100644 freqtrade/tests/edge/test_edge.py diff --git a/freqtrade/tests/edge/test_edge.py b/freqtrade/tests/edge/test_edge.py new file mode 100644 index 000000000..57f026304 --- /dev/null +++ b/freqtrade/tests/edge/test_edge.py @@ -0,0 +1,52 @@ +# pragma pylint: disable=missing-docstring, W0212, line-too-long, C0103, unused-argument + +import json +import math +import random +from typing import List +from unittest.mock import MagicMock + +import numpy as np +import pandas as pd +import pytest +from arrow import Arrow + +from freqtrade import DependencyException, constants, optimize +from freqtrade.arguments import Arguments, TimeRange +from freqtrade.optimize.backtesting import (Backtesting, setup_configuration, + start) +from freqtrade.tests.conftest import log_has, patch_exchange, get_patched_exchange +from freqtrade.strategy.interface import SellType +from freqtrade.strategy.default_strategy import DefaultStrategy + +from freqtrade.exchange import Exchange +from freqtrade.freqtradebot import FreqtradeBot + +from freqtrade.edge import Edge + + +# Cases to be tested: +# 1) Three complete trades within dataframe (with sell or buy hit for all) +# 2) Two open trades but one without sell/buy hit +# 3) Two complete trades and one which should not be considered as it happend while +# there was an already open trade on pair +# 4) Three complete trades with buy=1 on the last frame +# 5) Candle drops 8%, stoploss at 1%: Trade closed, 1% loss +# 6) Candle drops 4% but recovers to 1% loss, stoploss at 3%: Trade closed, 3% loss +# 7) Candle drops 4% recovers to 1% entry criteria are met candle drops +# 20%, stoploss at 2%: Trade 1 closed, Loss 2%, Trade 2 opened, Trade 2 closed, Loss 2% + + +def test_filter(mocker, default_conf): + exchange = get_patched_exchange(mocker, default_conf) + edge = Edge(default_conf, exchange) + mocker.patch('freqtrade.edge.Edge._cached_pairs', mocker.PropertyMock( + return_value=[ + ['E/F', -0.01, 0.66, 3.71, 0.50, 1.71], + ['C/D', -0.01, 0.66, 3.71, 0.50, 1.71], + ['N/O', -0.01, 0.66, 3.71, 0.50, 1.71] + ] + )) + + pairs = ['A/B', 'C/D', 'E/F', 'G/H'] + assert(edge.filter(pairs) == ['E/F', 'C/D']) From a364a1e40d654269312f38141500f9183cfa4f6f Mon Sep 17 00:00:00 2001 From: misagh Date: Tue, 2 Oct 2018 16:32:57 +0200 Subject: [PATCH 097/699] Edge package test cases drafted --- freqtrade/tests/edge/test_edge.py | 25 +------------------------ 1 file changed, 1 insertion(+), 24 deletions(-) diff --git a/freqtrade/tests/edge/test_edge.py b/freqtrade/tests/edge/test_edge.py index 57f026304..ba9b71d46 100644 --- a/freqtrade/tests/edge/test_edge.py +++ b/freqtrade/tests/edge/test_edge.py @@ -1,27 +1,4 @@ -# pragma pylint: disable=missing-docstring, W0212, line-too-long, C0103, unused-argument - -import json -import math -import random -from typing import List -from unittest.mock import MagicMock - -import numpy as np -import pandas as pd -import pytest -from arrow import Arrow - -from freqtrade import DependencyException, constants, optimize -from freqtrade.arguments import Arguments, TimeRange -from freqtrade.optimize.backtesting import (Backtesting, setup_configuration, - start) -from freqtrade.tests.conftest import log_has, patch_exchange, get_patched_exchange -from freqtrade.strategy.interface import SellType -from freqtrade.strategy.default_strategy import DefaultStrategy - -from freqtrade.exchange import Exchange -from freqtrade.freqtradebot import FreqtradeBot - +from freqtrade.tests.conftest import log_has, get_patched_exchange from freqtrade.edge import Edge From de20e142a0461ad3d0c32cc99684fdf4d0de795b Mon Sep 17 00:00:00 2001 From: misagh Date: Tue, 2 Oct 2018 18:05:24 +0200 Subject: [PATCH 098/699] added 9 use cased for testing Edge --- freqtrade/tests/edge/test_edge.py | 30 +++++++++++++++++++++++------- 1 file changed, 23 insertions(+), 7 deletions(-) diff --git a/freqtrade/tests/edge/test_edge.py b/freqtrade/tests/edge/test_edge.py index ba9b71d46..c95cd07e5 100644 --- a/freqtrade/tests/edge/test_edge.py +++ b/freqtrade/tests/edge/test_edge.py @@ -3,16 +3,18 @@ from freqtrade.edge import Edge # Cases to be tested: -# 1) Three complete trades within dataframe (with sell or buy hit for all) -# 2) Two open trades but one without sell/buy hit -# 3) Two complete trades and one which should not be considered as it happend while -# there was an already open trade on pair -# 4) Three complete trades with buy=1 on the last frame +############################### SELL POINTS ##################################### +# 1) Three complete trades within dataframe (with sell hit for all) +# 2) Two complete trades but one without sell hit (remains open) +# 3) Two complete trades and one buy signal while one trade is open +# 4) Two complete trades with buy=1 on the last frame +################################# STOPLOSS ###################################### # 5) Candle drops 8%, stoploss at 1%: Trade closed, 1% loss # 6) Candle drops 4% but recovers to 1% loss, stoploss at 3%: Trade closed, 3% loss -# 7) Candle drops 4% recovers to 1% entry criteria are met candle drops +# 7) Candle drops 4% recovers to 1% entry criteria are met, candle drops # 20%, stoploss at 2%: Trade 1 closed, Loss 2%, Trade 2 opened, Trade 2 closed, Loss 2% - +############################ PRIORITY TO STOPLOSS ################################ +# 8) Stoploss and sell are hit. should sell on stoploss def test_filter(mocker, default_conf): exchange = get_patched_exchange(mocker, default_conf) @@ -27,3 +29,17 @@ def test_filter(mocker, default_conf): pairs = ['A/B', 'C/D', 'E/F', 'G/H'] assert(edge.filter(pairs) == ['E/F', 'C/D']) + + +def test_three_complete_trades(): + stoploss = -0.01 + three_sell_points_hit = [ + # Buy, O, H, L, C, Sell + [1, 15, 20, 12, 17, 0], # -> should enter the trade + [1, 17, 18, 13, 14, 1], # -> should sell (trade 1 completed) + [0, 14, 15, 11, 12, 0], # -> no action + [1, 12, 25, 11, 20, 0], # -> should enter the trade + [0, 20, 30, 21, 25, 1], # -> should sell (trade 2 completed) + [1, 25, 27, 22, 26, 1], # -> buy and sell, should enter the trade + [0, 26, 36, 25, 35, 1], # -> should sell (trade 3 completed) + ] From a46b3ec9e7191fbec37caa8e5d7402e1cdedff41 Mon Sep 17 00:00:00 2001 From: misagh Date: Wed, 3 Oct 2018 10:37:36 +0200 Subject: [PATCH 099/699] first test completed --- freqtrade/tests/edge/test_edge.py | 79 +++++++++++++++++++++++++------ 1 file changed, 65 insertions(+), 14 deletions(-) diff --git a/freqtrade/tests/edge/test_edge.py b/freqtrade/tests/edge/test_edge.py index c95cd07e5..5714b1530 100644 --- a/freqtrade/tests/edge/test_edge.py +++ b/freqtrade/tests/edge/test_edge.py @@ -1,20 +1,24 @@ -from freqtrade.tests.conftest import log_has, get_patched_exchange +from freqtrade.tests.conftest import get_patched_exchange from freqtrade.edge import Edge +from pandas import DataFrame # Cases to be tested: -############################### SELL POINTS ##################################### +# SELL POINTS: # 1) Three complete trades within dataframe (with sell hit for all) # 2) Two complete trades but one without sell hit (remains open) # 3) Two complete trades and one buy signal while one trade is open # 4) Two complete trades with buy=1 on the last frame -################################# STOPLOSS ###################################### +################################################################### +# STOPLOSS: # 5) Candle drops 8%, stoploss at 1%: Trade closed, 1% loss # 6) Candle drops 4% but recovers to 1% loss, stoploss at 3%: Trade closed, 3% loss # 7) Candle drops 4% recovers to 1% entry criteria are met, candle drops # 20%, stoploss at 2%: Trade 1 closed, Loss 2%, Trade 2 opened, Trade 2 closed, Loss 2% -############################ PRIORITY TO STOPLOSS ################################ +#################################################################### +# PRIORITY TO STOPLOSS: # 8) Stoploss and sell are hit. should sell on stoploss +#################################################################### def test_filter(mocker, default_conf): exchange = get_patched_exchange(mocker, default_conf) @@ -31,15 +35,62 @@ def test_filter(mocker, default_conf): assert(edge.filter(pairs) == ['E/F', 'C/D']) -def test_three_complete_trades(): - stoploss = -0.01 +def _validate_ohlc(buy_ohlc_sell_matrice): + for index, ohlc in enumerate(buy_ohlc_sell_matrice): + # if not high < open < low or not high < close < low + if not ohlc[3] > ohlc[2] > ohlc[4] or not ohlc[3] > ohlc[5] > ohlc[4]: + raise Exception('Line ' + str(index + 1) + ' of ohlc has invalid values!') + return True + + +def _build_dataframe(buy_ohlc_sell_matrice): + _validate_ohlc(buy_ohlc_sell_matrice) + + tickers = [] + for ohlc in buy_ohlc_sell_matrice: + ticker = { + 'date': ohlc[0], + 'buy': ohlc[1], + 'open': ohlc[2], + 'high': ohlc[3], + 'low': ohlc[4], + 'close': ohlc[5], + 'sell': ohlc[6] + } + tickers.append(ticker) + return DataFrame(tickers) + + +def test_three_complete_trades(mocker, default_conf): + exchange = get_patched_exchange(mocker, default_conf) + edge = Edge(default_conf, exchange) + + stoploss = -0.90 # we don't want stoploss to be hit in this test three_sell_points_hit = [ - # Buy, O, H, L, C, Sell - [1, 15, 20, 12, 17, 0], # -> should enter the trade - [1, 17, 18, 13, 14, 1], # -> should sell (trade 1 completed) - [0, 14, 15, 11, 12, 0], # -> no action - [1, 12, 25, 11, 20, 0], # -> should enter the trade - [0, 20, 30, 21, 25, 1], # -> should sell (trade 2 completed) - [1, 25, 27, 22, 26, 1], # -> buy and sell, should enter the trade - [0, 26, 36, 25, 35, 1], # -> should sell (trade 3 completed) + # Date, Buy, O, H, L, C, Sell + [1, 1, 15, 20, 12, 17, 0], # -> should enter the trade + [2, 0, 17, 18, 13, 14, 1], # -> should sell (trade 1 completed) + [3, 0, 14, 15, 11, 12, 0], # -> no action + [4, 1, 12, 25, 11, 20, 0], # -> should enter the trade + [5, 0, 20, 30, 19, 25, 1], # -> should sell (trade 2 completed) + [6, 1, 25, 27, 22, 26, 1], # -> buy and sell, should enter the trade + [7, 0, 26, 36, 25, 35, 1], # -> should sell (trade 3 completed) ] + + ticker_df = _build_dataframe(three_sell_points_hit) + trades = edge._find_trades_for_stoploss_range(ticker_df, 'TEST/BTC', [stoploss]) + + # Three trades must have happened + assert len(trades) == 3 + + # First trade check + assert trades[0]['open_time'] == 1 + assert trades[0]['close_time'] == 2 + + # Second trade check + assert trades[1]['open_time'] == 4 + assert trades[1]['close_time'] == 5 + + # Third trade check + assert trades[2]['open_time'] == 6 + assert trades[2]['close_time'] == 7 From 6f79b55845d26169e504bbf71655526201515c0a Mon Sep 17 00:00:00 2001 From: misagh Date: Wed, 3 Oct 2018 14:22:27 +0200 Subject: [PATCH 100/699] - function renamed to be more readable - expectancy bug resolved --- freqtrade/edge/__init__.py | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/freqtrade/edge/__init__.py b/freqtrade/edge/__init__.py index edcd66337..bf855ef43 100644 --- a/freqtrade/edge/__init__.py +++ b/freqtrade/edge/__init__.py @@ -25,7 +25,7 @@ class Edge(): config: Dict = {} _last_updated: int # Timestamp of pairs last updated time _cached_pairs: list = [] # Keeps an array of - # [pair, winrate, risk reward ratio, required risk reward, expectancy] + # [pair, stoploss, winrate, risk reward ratio, required risk reward, expectancy] _total_capital: float _allowed_risk: float @@ -156,6 +156,14 @@ class Edge(): # Only return pairs which are included in "pairs" argument list final = [x for x in filtered_expectancy if x in pairs] + if final: + logger.info( + 'Edge validated only %s', + final + ) + else: + logger.info('Edge removed all pairs as no pair with minimum expectancy was found !') + return final def _fill_calculable_fields(self, result: DataFrame) -> DataFrame: @@ -234,7 +242,7 @@ class Edge(): # # Removing Pumps if self.edge_config.get('remove_pumps', True): - results = results[results.profit_abs < float(avg + 2 * std)] + results = results[results.profit_abs <= float(avg + 2 * std)] ########################################################################## # Removing trades having a duration more than X minutes (set in config) @@ -290,13 +298,13 @@ class Edge(): result: list = [] for stoploss in stoploss_range: - result += self._detect_stop_and_sell_points( + result += self._detect_next_stop_or_sell_point( buy_column, sell_column, date_column, ohlc_columns, round(stoploss, 6), pair ) return result - def _detect_stop_and_sell_points( + def _detect_next_stop_or_sell_point( self, buy_column, sell_column, @@ -366,7 +374,7 @@ class Edge(): # Calling again the same function recursively but giving # it a view of exit_index till the end of array - return result + self._detect_stop_and_sell_points( + return result + self._detect_next_stop_or_sell_point( buy_column[exit_index:], sell_column[exit_index:], date_column[exit_index:], From b57d9edda8b28722e382652a0ed329593a8d22ae Mon Sep 17 00:00:00 2001 From: misagh Date: Wed, 3 Oct 2018 14:23:10 +0200 Subject: [PATCH 101/699] Edge test expectancy function (round 1) --- freqtrade/tests/edge/test_edge.py | 107 +++++++++++++++++++++++++++--- 1 file changed, 96 insertions(+), 11 deletions(-) diff --git a/freqtrade/tests/edge/test_edge.py b/freqtrade/tests/edge/test_edge.py index 5714b1530..15fc38114 100644 --- a/freqtrade/tests/edge/test_edge.py +++ b/freqtrade/tests/edge/test_edge.py @@ -1,6 +1,8 @@ from freqtrade.tests.conftest import get_patched_exchange from freqtrade.edge import Edge -from pandas import DataFrame +from pandas import DataFrame, to_datetime +import arrow +import numpy as np # Cases to be tested: @@ -20,6 +22,10 @@ from pandas import DataFrame # 8) Stoploss and sell are hit. should sell on stoploss #################################################################### +ticker_start_time = arrow.get(2018, 10, 3) +ticker_interval_in_minute = 5 + + def test_filter(mocker, default_conf): exchange = get_patched_exchange(mocker, default_conf) edge = Edge(default_conf, exchange) @@ -45,11 +51,11 @@ def _validate_ohlc(buy_ohlc_sell_matrice): def _build_dataframe(buy_ohlc_sell_matrice): _validate_ohlc(buy_ohlc_sell_matrice) - tickers = [] for ohlc in buy_ohlc_sell_matrice: ticker = { - 'date': ohlc[0], + # ticker every 5 min + 'date': ticker_start_time.shift(minutes=(ohlc[0] * 5)).timestamp * 1000, 'buy': ohlc[1], 'open': ohlc[2], 'high': ohlc[3], @@ -58,7 +64,71 @@ def _build_dataframe(buy_ohlc_sell_matrice): 'sell': ohlc[6] } tickers.append(ticker) - return DataFrame(tickers) + + frame = DataFrame(tickers) + frame['date'] = to_datetime(frame['date'], + unit='ms', + utc=True, + infer_datetime_format=True) + + return frame + + +def test_process_expectancy(mocker, default_conf): + default_conf['edge']['min_trade_number'] = 2 + exchange = get_patched_exchange(mocker, default_conf) + + def get_fee(): + return 0.001 + + exchange.get_fee = get_fee + edge = Edge(default_conf, exchange) + + trades = [ + {'pair': 'TEST/BTC', + 'stoploss': -0.9, + 'profit_percent': '', + 'profit_abs': '', + 'open_time': np.datetime64('2018-10-03T00:05:00.000000000'), + 'close_time': np.datetime64('2018-10-03T00:10:00.000000000'), + 'open_index': 1, + 'close_index': 1, + 'trade_duration': '', + 'open_rate': 17, + 'close_rate': 17, + 'exit_type': 'sell_signal'}, # sdfsdf + + {'pair': 'TEST/BTC', + 'stoploss': -0.9, + 'profit_percent': '', + 'profit_abs': '', + 'open_time': np.datetime64('2018-10-03T00:20:00.000000000'), + 'close_time': np.datetime64('2018-10-03T00:25:00.000000000'), + 'open_index': 4, + 'close_index': 4, + 'trade_duration': '', + 'open_rate': 20, + 'close_rate': 20, + 'exit_type': 'sell_signal'}, + + {'pair': 'TEST/BTC', + 'stoploss': -0.9, + 'profit_percent': '', + 'profit_abs': '', + 'open_time': np.datetime64('2018-10-03T00:30:00.000000000'), + 'close_time': np.datetime64('2018-10-03T00:40:00.000000000'), + 'open_index': 6, + 'close_index': 7, + 'trade_duration': '', + 'open_rate': 26, + 'close_rate': 34, + 'exit_type': 'sell_signal'} + ] + + trades_df = DataFrame(trades) + trades_df = edge._fill_calculable_fields(trades_df) + final = edge._process_expectancy(trades_df) + assert len(final) == 1 def test_three_complete_trades(mocker, default_conf): @@ -80,17 +150,32 @@ def test_three_complete_trades(mocker, default_conf): ticker_df = _build_dataframe(three_sell_points_hit) trades = edge._find_trades_for_stoploss_range(ticker_df, 'TEST/BTC', [stoploss]) - # Three trades must have happened + # Three trades must have occured assert len(trades) == 3 # First trade check - assert trades[0]['open_time'] == 1 - assert trades[0]['close_time'] == 2 + # open time should be on line 1 + assert trades[0]['open_time'] == np.datetime64(ticker_start_time.shift( + minutes=(1 * ticker_interval_in_minute)).timestamp * 1000, 'ms') + + # close time should be on line 2 + assert trades[0]['close_time'] == np.datetime64(ticker_start_time.shift( + minutes=(2 * ticker_interval_in_minute)).timestamp * 1000, 'ms') # Second trade check - assert trades[1]['open_time'] == 4 - assert trades[1]['close_time'] == 5 + # open time should be on line 4 + assert trades[1]['open_time'] == np.datetime64(ticker_start_time.shift( + minutes=(4 * ticker_interval_in_minute)).timestamp * 1000, 'ms') + + # close time should be on line 5 + assert trades[1]['close_time'] == np.datetime64(ticker_start_time.shift( + minutes=(5 * ticker_interval_in_minute)).timestamp * 1000, 'ms') # Third trade check - assert trades[2]['open_time'] == 6 - assert trades[2]['close_time'] == 7 + # open time should be on line 6 + assert trades[2]['open_time'] == np.datetime64(ticker_start_time.shift( + minutes=(6 * ticker_interval_in_minute)).timestamp * 1000, 'ms') + + # close time should be on line 7 + assert trades[2]['close_time'] == np.datetime64(ticker_start_time.shift( + minutes=(7 * ticker_interval_in_minute)).timestamp * 1000, 'ms') From 1e669c72289d680e245c6e381c30284326d600c6 Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Wed, 3 Oct 2018 14:29:06 +0200 Subject: [PATCH 102/699] Update ccxt from 1.17.363 to 1.17.365 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index fc5d936b7..dec89be51 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -ccxt==1.17.363 +ccxt==1.17.365 SQLAlchemy==1.2.12 python-telegram-bot==11.1.0 arrow==0.12.1 From fa38772942bd03072e0d4fd22dfb3f4ef56dbd35 Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Wed, 3 Oct 2018 14:29:07 +0200 Subject: [PATCH 103/699] Update pytest from 3.8.1 to 3.8.2 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index dec89be51..272892e99 100644 --- a/requirements.txt +++ b/requirements.txt @@ -12,7 +12,7 @@ scipy==1.1.0 jsonschema==2.6.0 numpy==1.15.2 TA-Lib==0.4.17 -pytest==3.8.1 +pytest==3.8.2 pytest-mock==1.10.0 pytest-asyncio==0.9.0 pytest-cov==2.6.0 From 3ed486f3a0c1995a551c18a00d2674340eb5ce40 Mon Sep 17 00:00:00 2001 From: Matthias Date: Wed, 3 Oct 2018 19:32:14 +0200 Subject: [PATCH 104/699] update documentation for raspberry to match as shown in #1236 --- docs/installation.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/installation.md b/docs/installation.md index 1ceda6b1c..1000600e6 100644 --- a/docs/installation.md +++ b/docs/installation.md @@ -245,8 +245,8 @@ conda install python=3.6 conda create -n freqtrade python=3.6 conda install scipy pandas -pip install -r requirements.txt -pip install -e . +python3 -m pip install -r requirements.txt +python3 -m pip install -e . ``` ### MacOS From 9723300a0758e089de88e1532b388876cbe5e9e8 Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Thu, 4 Oct 2018 14:29:06 +0200 Subject: [PATCH 105/699] Update ccxt from 1.17.365 to 1.17.368 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 272892e99..1ec5bdf26 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -ccxt==1.17.365 +ccxt==1.17.368 SQLAlchemy==1.2.12 python-telegram-bot==11.1.0 arrow==0.12.1 From 06d75a8bad9581837678c06a77f24090ce851238 Mon Sep 17 00:00:00 2001 From: misagh Date: Thu, 4 Oct 2018 18:05:46 +0200 Subject: [PATCH 106/699] test cases added: force_stoploss by Edge --- freqtrade/freqtradebot.py | 2 +- freqtrade/tests/conftest.py | 20 ++++++ freqtrade/tests/edge/test_edge.py | 2 + freqtrade/tests/test_freqtradebot.py | 96 +++++++++++++++++++++++++++- 4 files changed, 118 insertions(+), 2 deletions(-) diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index be4b35a20..8b7f16334 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -649,7 +649,7 @@ class FreqtradeBot(object): if should_sell.sell_flag: self.execute_sell(trade, sell_rate, should_sell.sell_type) - logger.info('excuted sell') + logger.info('executed sell, reason: %s', should_sell.sell_type) return True return False diff --git a/freqtrade/tests/conftest.py b/freqtrade/tests/conftest.py index f6f067a41..8e691e3c8 100644 --- a/freqtrade/tests/conftest.py +++ b/freqtrade/tests/conftest.py @@ -12,6 +12,7 @@ from telegram import Chat, Message, Update from freqtrade.exchange.exchange_helpers import parse_ticker_dataframe from freqtrade.exchange import Exchange +from freqtrade.edge import Edge from freqtrade.freqtradebot import FreqtradeBot logging.getLogger('').setLevel(logging.INFO) @@ -42,6 +43,25 @@ def get_patched_exchange(mocker, config, api_mock=None) -> Exchange: return exchange +def patch_edge(mocker) -> None: + # "ETH/BTC", + # "LTC/BTC", + # "XRP/BTC", + # "NEO/BTC" + mocker.patch('freqtrade.edge.Edge._cached_pairs', mocker.PropertyMock( + return_value=[ + ['NEO/BTC', -0.20, 0.66, 3.71, 0.50, 1.71], + ['LTC/BTC', -0.21, 0.66, 3.71, 0.50, 1.71], + ] + )) + mocker.patch('freqtrade.edge.Edge.stoploss', MagicMock(return_value=-0.20)) + mocker.patch('freqtrade.edge.Edge.calculate', MagicMock(return_value=True)) + +def get_patched_edge(mocker, config) -> Edge: + patch_edge(mocker) + edge = Edge(config) + return edge + # Functions for recurrent object patching def get_patched_freqtradebot(mocker, config) -> FreqtradeBot: """ diff --git a/freqtrade/tests/edge/test_edge.py b/freqtrade/tests/edge/test_edge.py index 15fc38114..b6dc60000 100644 --- a/freqtrade/tests/edge/test_edge.py +++ b/freqtrade/tests/edge/test_edge.py @@ -130,6 +130,8 @@ def test_process_expectancy(mocker, default_conf): final = edge._process_expectancy(trades_df) assert len(final) == 1 + # TODO: check expectancy + win rate etc + def test_three_complete_trades(mocker, default_conf): exchange = get_patched_exchange(mocker, default_conf) diff --git a/freqtrade/tests/test_freqtradebot.py b/freqtrade/tests/test_freqtradebot.py index c302eeb43..9d5de004d 100644 --- a/freqtrade/tests/test_freqtradebot.py +++ b/freqtrade/tests/test_freqtradebot.py @@ -18,7 +18,7 @@ from freqtrade.persistence import Trade from freqtrade.rpc import RPCMessageType from freqtrade.state import State from freqtrade.strategy.interface import SellType, SellCheckTuple -from freqtrade.tests.conftest import log_has, patch_exchange +from freqtrade.tests.conftest import log_has, patch_exchange, patch_edge # Functions for recurrent object patching @@ -251,6 +251,100 @@ def test_get_trade_stake_amount_unlimited_amount(default_conf, assert result is None +def test_edge_overrides_stake_amount(mocker, default_conf) -> None: + default_conf['edge']['enabled'] = True + patch_RPCManager(mocker) + patch_exchange(mocker) + patch_edge(mocker) + freqtrade = FreqtradeBot(default_conf) + + # strategy stoploss should be ignored + freqtrade.strategy.stoploss = -0.05 + + with pytest.raises(IndexError): + freqtrade._get_trade_stake_amount('ETH/BTC') + + assert freqtrade._get_trade_stake_amount('NEO/BTC') == 0.025 + assert freqtrade._get_trade_stake_amount('LTC/BTC') == 0.02381 + + +def test_edge_overrides_stoploss(limit_buy_order, fee, markets, caplog, mocker, default_conf) -> None: + default_conf['edge']['enabled'] = True + patch_RPCManager(mocker) + patch_exchange(mocker) + patch_edge(mocker) + + # Strategy stoploss is -0.1 but Edge imposes a stoploss at -0.2 + # Thus, if price falls 21%, stoploss should be triggered + # + # mocking the ticker: price is falling ... + buy_price = limit_buy_order['price'] + mocker.patch.multiple( + 'freqtrade.exchange.Exchange', + get_ticker=MagicMock(return_value={ + 'bid': buy_price * 0.79, + 'ask': buy_price * 0.79, + 'last': buy_price * 0.79 + }), + buy=MagicMock(return_value={'id': limit_buy_order['id']}), + get_fee=fee, + get_markets=markets, + ) + ############################################# + + # Create a trade with "limit_buy_order" price + freqtrade = FreqtradeBot(default_conf) + patch_get_signal(freqtrade) + freqtrade.strategy.min_roi_reached = lambda trade, current_profit, current_time: False + freqtrade.create_trade() + trade = Trade.query.first() + trade.update(limit_buy_order) + ############################################# + + # stoploss shoud be hit + assert freqtrade.handle_trade(trade) is True + + assert log_has('executed sell, reason: SellType.STOP_LOSS', caplog.record_tuples) + assert trade.sell_reason == SellType.STOP_LOSS.value + + +def test_edge_should_ignore_strategy_stoploss(limit_buy_order, fee, markets, + mocker, default_conf) -> None: + default_conf['edge']['enabled'] = True + patch_RPCManager(mocker) + patch_exchange(mocker) + patch_edge(mocker) + + # Strategy stoploss is -0.1 but Edge imposes a stoploss at -0.2 + # Thus, if price falls 15%, stoploss should not be triggered + # + # mocking the ticker: price is falling ... + buy_price = limit_buy_order['price'] + mocker.patch.multiple( + 'freqtrade.exchange.Exchange', + get_ticker=MagicMock(return_value={ + 'bid': buy_price * 0.85, + 'ask': buy_price * 0.85, + 'last': buy_price * 0.85 + }), + buy=MagicMock(return_value={'id': limit_buy_order['id']}), + get_fee=fee, + get_markets=markets, + ) + ############################################# + + # Create a trade with "limit_buy_order" price + freqtrade = FreqtradeBot(default_conf) + patch_get_signal(freqtrade) + freqtrade.strategy.min_roi_reached = lambda trade, current_profit, current_time: False + freqtrade.create_trade() + trade = Trade.query.first() + trade.update(limit_buy_order) + ############################################# + + # stoploss shoud be hit + assert freqtrade.handle_trade(trade) is False + def test_get_min_pair_stake_amount(mocker, default_conf) -> None: patch_RPCManager(mocker) patch_exchange(mocker) From 77cac9e562a9bc6b430fbc4cd68b77281630a538 Mon Sep 17 00:00:00 2001 From: misagh Date: Thu, 4 Oct 2018 18:07:47 +0200 Subject: [PATCH 107/699] autopep8 applied --- freqtrade/tests/conftest.py | 5 ++++- freqtrade/tests/test_freqtradebot.py | 19 +++++++++++++------ 2 files changed, 17 insertions(+), 7 deletions(-) diff --git a/freqtrade/tests/conftest.py b/freqtrade/tests/conftest.py index 8e691e3c8..86e7e73c8 100644 --- a/freqtrade/tests/conftest.py +++ b/freqtrade/tests/conftest.py @@ -53,16 +53,19 @@ def patch_edge(mocker) -> None: ['NEO/BTC', -0.20, 0.66, 3.71, 0.50, 1.71], ['LTC/BTC', -0.21, 0.66, 3.71, 0.50, 1.71], ] - )) + )) mocker.patch('freqtrade.edge.Edge.stoploss', MagicMock(return_value=-0.20)) mocker.patch('freqtrade.edge.Edge.calculate', MagicMock(return_value=True)) + def get_patched_edge(mocker, config) -> Edge: patch_edge(mocker) edge = Edge(config) return edge # Functions for recurrent object patching + + def get_patched_freqtradebot(mocker, config) -> FreqtradeBot: """ This function patch _init_modules() to not call dependencies diff --git a/freqtrade/tests/test_freqtradebot.py b/freqtrade/tests/test_freqtradebot.py index 9d5de004d..f38ecd78a 100644 --- a/freqtrade/tests/test_freqtradebot.py +++ b/freqtrade/tests/test_freqtradebot.py @@ -268,7 +268,13 @@ def test_edge_overrides_stake_amount(mocker, default_conf) -> None: assert freqtrade._get_trade_stake_amount('LTC/BTC') == 0.02381 -def test_edge_overrides_stoploss(limit_buy_order, fee, markets, caplog, mocker, default_conf) -> None: +def test_edge_overrides_stoploss( + limit_buy_order, + fee, + markets, + caplog, + mocker, + default_conf) -> None: default_conf['edge']['enabled'] = True patch_RPCManager(mocker) patch_exchange(mocker) @@ -309,7 +315,7 @@ def test_edge_overrides_stoploss(limit_buy_order, fee, markets, caplog, mocker, def test_edge_should_ignore_strategy_stoploss(limit_buy_order, fee, markets, - mocker, default_conf) -> None: + mocker, default_conf) -> None: default_conf['edge']['enabled'] = True patch_RPCManager(mocker) patch_exchange(mocker) @@ -345,6 +351,7 @@ def test_edge_should_ignore_strategy_stoploss(limit_buy_order, fee, markets, # stoploss shoud be hit assert freqtrade.handle_trade(trade) is False + def test_get_min_pair_stake_amount(mocker, default_conf) -> None: patch_RPCManager(mocker) patch_exchange(mocker) @@ -1825,7 +1832,7 @@ def test_get_real_amount_quote(default_conf, trades_for_order, buy_order_fee, ca exchange='binance', open_rate=0.245441, open_order_id="123456" - ) + ) freqtrade = FreqtradeBot(default_conf) patch_get_signal(freqtrade) @@ -2101,9 +2108,9 @@ def test_order_book_bid_strategy2(mocker, default_conf, order_book_l2, markets) """ patch_exchange(mocker) mocker.patch.multiple( - 'freqtrade.exchange.Exchange', - get_markets=markets, - get_order_book=order_book_l2 + 'freqtrade.exchange.Exchange', + get_markets=markets, + get_order_book=order_book_l2 ) default_conf['exchange']['name'] = 'binance' default_conf['bid_strategy']['use_order_book'] = True From d3078d756487e99ed22c2b69981a32fe68b48dc0 Mon Sep 17 00:00:00 2001 From: misagh Date: Thu, 4 Oct 2018 18:51:59 +0200 Subject: [PATCH 108/699] test case added: edge stop loss for pair --- freqtrade/tests/edge/test_edge.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/freqtrade/tests/edge/test_edge.py b/freqtrade/tests/edge/test_edge.py index b6dc60000..3cc2fd76c 100644 --- a/freqtrade/tests/edge/test_edge.py +++ b/freqtrade/tests/edge/test_edge.py @@ -41,6 +41,21 @@ def test_filter(mocker, default_conf): assert(edge.filter(pairs) == ['E/F', 'C/D']) +def test_stoploss(mocker, default_conf): + exchange = get_patched_exchange(mocker, default_conf) + edge = Edge(default_conf, exchange) + mocker.patch('freqtrade.edge.Edge._cached_pairs', mocker.PropertyMock( + return_value=[ + ['E/F', -0.01, 0.66, 3.71, 0.50, 1.71], + ['C/D', -0.01, 0.66, 3.71, 0.50, 1.71], + ['N/O', -0.01, 0.66, 3.71, 0.50, 1.71] + ] + )) + + pairs = ['A/B', 'C/D', 'E/F', 'G/H'] + assert edge.stoploss('E/F') == -0.01 + + def _validate_ohlc(buy_ohlc_sell_matrice): for index, ohlc in enumerate(buy_ohlc_sell_matrice): # if not high < open < low or not high < close < low From e7d5cf9d9dde6bce0bbf643538c4873e53fb568b Mon Sep 17 00:00:00 2001 From: Matthias Date: Thu, 4 Oct 2018 20:11:02 +0200 Subject: [PATCH 109/699] Allow loading of any additional configuration to ccxt seperated by async and non-async --- freqtrade/exchange/__init__.py | 20 ++++++++++++----- freqtrade/tests/exchange/test_exchange.py | 27 +++++++++++++++++++++++ 2 files changed, 41 insertions(+), 6 deletions(-) diff --git a/freqtrade/exchange/__init__.py b/freqtrade/exchange/__init__.py index d3c60c256..7fd0e5f43 100644 --- a/freqtrade/exchange/__init__.py +++ b/freqtrade/exchange/__init__.py @@ -93,8 +93,9 @@ class Exchange(object): logger.info('Instance is running with dry_run enabled') exchange_config = config['exchange'] - self._api = self._init_ccxt(exchange_config) - self._api_async = self._init_ccxt(exchange_config, ccxt_async) + self._api = self._init_ccxt(exchange_config, ccxt_kwargs=exchange_config.get('ccxt_config')) + self._api_async = self._init_ccxt(exchange_config, ccxt_async, + ccxt_kwargs=exchange_config.get('ccxt_async_config')) logger.info('Using Exchange "%s"', self.name) @@ -114,7 +115,8 @@ class Exchange(object): if self._api_async and inspect.iscoroutinefunction(self._api_async.close): asyncio.get_event_loop().run_until_complete(self._api_async.close()) - def _init_ccxt(self, exchange_config: dict, ccxt_module=ccxt) -> ccxt.Exchange: + def _init_ccxt(self, exchange_config: dict, ccxt_module=ccxt, + ccxt_kwargs: dict = None) -> ccxt.Exchange: """ Initialize ccxt with given config and return valid ccxt instance. @@ -124,14 +126,20 @@ class Exchange(object): if name not in ccxt_module.exchanges: raise OperationalException(f'Exchange {name} is not supported') - try: - api = getattr(ccxt_module, name.lower())({ + + ex_config = { 'apiKey': exchange_config.get('key'), 'secret': exchange_config.get('secret'), 'password': exchange_config.get('password'), 'uid': exchange_config.get('uid', ''), 'enableRateLimit': exchange_config.get('ccxt_rate_limit', True) - }) + } + if ccxt_kwargs: + logger.info('Applying additional ccxt config: %s', ccxt_kwargs) + ex_config.update(ccxt_kwargs) + try: + + api = getattr(ccxt_module, name.lower())(ex_config) except (KeyError, AttributeError): raise OperationalException(f'Exchange {name} is not supported') diff --git a/freqtrade/tests/exchange/test_exchange.py b/freqtrade/tests/exchange/test_exchange.py index d9d68c3b8..93cd1e546 100644 --- a/freqtrade/tests/exchange/test_exchange.py +++ b/freqtrade/tests/exchange/test_exchange.py @@ -1,5 +1,6 @@ # pragma pylint: disable=missing-docstring, C0103, bad-continuation, global-statement # pragma pylint: disable=protected-access +import copy import logging from datetime import datetime from random import randint @@ -56,6 +57,32 @@ def test_init(default_conf, mocker, caplog): assert log_has('Instance is running with dry_run enabled', caplog.record_tuples) +def test_init_ccxt_kwargs(default_conf, mocker, caplog): + mocker.patch('freqtrade.exchange.Exchange._load_markets', MagicMock(return_value={})) + caplog.set_level(logging.INFO) + conf = copy.deepcopy(default_conf) + conf['exchange']['ccxt_async_config'] = {'aiohttp_trust_env': True} + ex = Exchange(conf) + assert log_has("Applying additional ccxt config: {'aiohttp_trust_env': True}", + caplog.record_tuples) + assert ex._api_async.aiohttp_trust_env + assert not ex._api.aiohttp_trust_env + + # Reset logging and config + caplog.clear() + conf = copy.deepcopy(default_conf) + conf['exchange']['ccxt_config'] = {'TestKWARG': 11} + ex = Exchange(conf) + assert not log_has("Applying additional ccxt config: {'aiohttp_trust_env': True}", + caplog.record_tuples) + assert not ex._api_async.aiohttp_trust_env + assert hasattr(ex._api, 'TestKWARG') + assert ex._api.TestKWARG == 11 + assert not hasattr(ex._api_async, 'TestKWARG') + assert log_has("Applying additional ccxt config: {'TestKWARG': 11}", + caplog.record_tuples) + + def test_destroy(default_conf, mocker, caplog): caplog.set_level(logging.DEBUG) get_patched_exchange(mocker, default_conf) From d1edcf9dcd990c4b9ab0be0f65777b7eae419bfd Mon Sep 17 00:00:00 2001 From: Matthias Date: Thu, 4 Oct 2018 20:17:19 +0200 Subject: [PATCH 110/699] Add documentation for ccxt_config --- config_full.json.example | 6 +++++- docs/configuration.md | 25 ++++++++++++++++++++++++- docs/telegram-usage.md | 6 ------ 3 files changed, 29 insertions(+), 8 deletions(-) diff --git a/config_full.json.example b/config_full.json.example index 7083bada6..71b119fb5 100644 --- a/config_full.json.example +++ b/config_full.json.example @@ -53,7 +53,11 @@ "pair_blacklist": [ "DOGE/BTC" ], - "outdated_offset": 5 + "outdated_offset": 5, + "ccxt_config": {}, + "ccxt_async_config": { + "aiohttp_trust_env": false + } }, "experimental": { "use_sell_signal": false, diff --git a/docs/configuration.md b/docs/configuration.md index 010e693d4..7c380e992 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -45,6 +45,8 @@ The table below will list all configuration parameters. | `exchange.pair_whitelist` | [] | No | List of currency to use by the bot. Can be overrided with `--dynamic-whitelist` param. | `exchange.pair_blacklist` | [] | No | List of currency the bot must avoid. Useful when using `--dynamic-whitelist` param. | `exchange.ccxt_rate_limit` | True | No | Have CCXT handle Exchange rate limits. Depending on the exchange, having this to false can lead to temporary bans from the exchange. +| `exchange.ccxt_config` | None | No | Additional CCXT parameters passed to the regular ccxt instance. Parameters may differ from exchange to exchange and are documented in the [ccxt documentation](https://ccxt.readthedocs.io/en/latest/manual.html#instantiation) +| `exchange.ccxt_async_config` | None | No | Additional CCXT parameters passed to the async ccxt instance. Parameters may differ from exchange to exchange and are documented in the [ccxt documentation](https://ccxt.readthedocs.io/en/latest/manual.html#instantiation) | `experimental.use_sell_signal` | false | No | Use your sell strategy in addition of the `minimal_roi`. | `experimental.sell_profit_only` | false | No | waits until you have made a positive profit before taking a sell decision. | `experimental.ignore_roi_if_buy_signal` | false | No | Does not sell if the buy-signal is still active. Takes preference over `minimal_roi` and `use_sell_signal` @@ -204,8 +206,29 @@ you run it in production mode. } ``` + If you have not your Bittrex API key yet, [see our tutorial](https://github.com/freqtrade/freqtrade/blob/develop/docs/pre-requisite.md). +### Using proxy with FreqTrade + +To use a proxy with freqtrade, add the kwarg `"aiohttp_trust_env"=true` to the `"ccxt_async_kwargs"` dict in the exchange section of the configuration. + +An example for this can be found in `config_full.json.example` + +``` json +"ccxt_async_config": { + "aiohttp_trust_env": true +} +``` + +Then, export your proxy settings using the variables `"HTTP_PROXY"` and `"HTTPS_PROXY"` set to the appropriate values + +``` bash +export HTTP_PROXY="http://addr:port" +export HTTPS_PROXY="http://addr:port" +freqtrade +``` + ### Embedding Strategies @@ -213,7 +236,7 @@ FreqTrade provides you with with an easy way to embed the strategy into your con This is done by utilizing BASE64 encoding and providing this string at the strategy configuration field, in your chosen config file. -##### Encoding a string as BASE64 +#### Encoding a string as BASE64 This is a quick example, how to generate the BASE64 string in python diff --git a/docs/telegram-usage.md b/docs/telegram-usage.md index ca764c9b4..945e31f9c 100644 --- a/docs/telegram-usage.md +++ b/docs/telegram-usage.md @@ -129,9 +129,3 @@ Day Profit BTC Profit USD ## /version > **Version:** `0.14.3` -### using proxy with telegram -``` -$ export HTTP_PROXY="http://addr:port" -$ export HTTPS_PROXY="http://addr:port" -$ freqtrade -``` From ddc15132866681063873d2c6dc0fe307cc0ca66d Mon Sep 17 00:00:00 2001 From: Matthias Date: Thu, 4 Oct 2018 20:34:33 +0200 Subject: [PATCH 111/699] Add ccxt_config to both config_samples --- config.json.example | 5 ++++- config_full.json.example | 12 ++++++------ 2 files changed, 10 insertions(+), 7 deletions(-) diff --git a/config.json.example b/config.json.example index 7a0bb6b9b..5198b9d81 100644 --- a/config.json.example +++ b/config.json.example @@ -28,7 +28,10 @@ "name": "bittrex", "key": "your_exchange_key", "secret": "your_exchange_secret", - "ccxt_rate_limit": true, + "ccxt_config": {"enableRateLimit": true}, + "ccxt_async_config": { + "enableRateLimit": false + }, "pair_whitelist": [ "ETH/BTC", "LTC/BTC", diff --git a/config_full.json.example b/config_full.json.example index 71b119fb5..bdc6361d2 100644 --- a/config_full.json.example +++ b/config_full.json.example @@ -37,7 +37,11 @@ "name": "bittrex", "key": "your_exchange_key", "secret": "your_exchange_secret", - "ccxt_rate_limit": true, + "ccxt_config": {"enableRateLimit": true}, + "ccxt_async_config": { + "enableRateLimit": false, + "aiohttp_trust_env": false + }, "pair_whitelist": [ "ETH/BTC", "LTC/BTC", @@ -53,11 +57,7 @@ "pair_blacklist": [ "DOGE/BTC" ], - "outdated_offset": 5, - "ccxt_config": {}, - "ccxt_async_config": { - "aiohttp_trust_env": false - } + "outdated_offset": 5 }, "experimental": { "use_sell_signal": false, From 37088cfb393d225c856bc9102aa44d9e7530f5bb Mon Sep 17 00:00:00 2001 From: Matthias Date: Thu, 4 Oct 2018 20:34:48 +0200 Subject: [PATCH 112/699] add to constants --- freqtrade/constants.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/freqtrade/constants.py b/freqtrade/constants.py index eadfa6eba..12f10d3b9 100644 --- a/freqtrade/constants.py +++ b/freqtrade/constants.py @@ -164,7 +164,9 @@ CONF_SCHEMA = { }, 'uniqueItems': True }, - 'outdated_offset': {'type': 'integer', 'minimum': 1} + 'outdated_offset': {'type': 'integer', 'minimum': 1}, + 'ccxt_config': {'type': 'object'}, + 'ccxt_async_config': {'type': 'object'} }, 'required': ['name', 'key', 'secret', 'pair_whitelist'] } From 3973d3697c7f964435af545bda25f2106988de2d Mon Sep 17 00:00:00 2001 From: Matthias Date: Thu, 4 Oct 2018 20:35:28 +0200 Subject: [PATCH 113/699] deprecate ccxt_rate_limt --- docs/configuration.md | 2 +- freqtrade/configuration.py | 5 +++++ freqtrade/tests/test_configuration.py | 11 ++++++++++- 3 files changed, 16 insertions(+), 2 deletions(-) diff --git a/docs/configuration.md b/docs/configuration.md index 7c380e992..3512f891b 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -44,7 +44,7 @@ The table below will list all configuration parameters. | `exchange.secret` | secret | No | API secret to use for the exchange. Only required when you are in production mode. | `exchange.pair_whitelist` | [] | No | List of currency to use by the bot. Can be overrided with `--dynamic-whitelist` param. | `exchange.pair_blacklist` | [] | No | List of currency the bot must avoid. Useful when using `--dynamic-whitelist` param. -| `exchange.ccxt_rate_limit` | True | No | Have CCXT handle Exchange rate limits. Depending on the exchange, having this to false can lead to temporary bans from the exchange. +| `exchange.ccxt_rate_limit` | True | No | DEPRECATED!! Have CCXT handle Exchange rate limits. Depending on the exchange, having this to false can lead to temporary bans from the exchange. | `exchange.ccxt_config` | None | No | Additional CCXT parameters passed to the regular ccxt instance. Parameters may differ from exchange to exchange and are documented in the [ccxt documentation](https://ccxt.readthedocs.io/en/latest/manual.html#instantiation) | `exchange.ccxt_async_config` | None | No | Additional CCXT parameters passed to the async ccxt instance. Parameters may differ from exchange to exchange and are documented in the [ccxt documentation](https://ccxt.readthedocs.io/en/latest/manual.html#instantiation) | `experimental.use_sell_signal` | false | No | Use your sell strategy in addition of the `minimal_roi`. diff --git a/freqtrade/configuration.py b/freqtrade/configuration.py index 4e1137f33..93ae3deab 100644 --- a/freqtrade/configuration.py +++ b/freqtrade/configuration.py @@ -271,6 +271,11 @@ class Configuration(object): raise OperationalException( exception_msg ) + # Depreciation warning + if 'ccxt_rate_limit' in config.get('exchange', {}): + logger.warning("`ccxt_rate_limit` has been deprecated in favor of " + "`ccxt_config` and `ccxt_async_config` and will be removed " + "in future a future version.") logger.debug('Exchange "%s" supported', exchange) return True diff --git a/freqtrade/tests/test_configuration.py b/freqtrade/tests/test_configuration.py index bf41aab83..2c487c165 100644 --- a/freqtrade/tests/test_configuration.py +++ b/freqtrade/tests/test_configuration.py @@ -371,7 +371,7 @@ def test_hyperopt_with_arguments(mocker, default_conf, caplog) -> None: assert log_has('Parameter -s/--spaces detected: [\'all\']', caplog.record_tuples) -def test_check_exchange(default_conf) -> None: +def test_check_exchange(default_conf, caplog) -> None: configuration = Configuration(Namespace()) # Test a valid exchange @@ -392,6 +392,15 @@ def test_check_exchange(default_conf) -> None: ): configuration.check_exchange(default_conf) + # Test ccxt_rate_limit depreciation + default_conf.get('exchange').update({'name': 'binance'}) + default_conf['exchange']['ccxt_rate_limit'] = True + configuration.check_exchange(default_conf) + assert log_has("`ccxt_rate_limit` has been deprecated in favor of " + "`ccxt_config` and `ccxt_async_config` and will be removed " + "in future a future version.", + caplog.record_tuples) + def test_cli_verbose_with_params(default_conf, mocker, caplog) -> None: mocker.patch('freqtrade.configuration.open', mocker.mock_open( From ce4f0696e1b46a9e1337095d5242da0d590e4344 Mon Sep 17 00:00:00 2001 From: Matthias Date: Thu, 4 Oct 2018 20:38:30 +0200 Subject: [PATCH 114/699] Add logging to download script and enable ccxt_async_config --- scripts/download_backtest_data.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/scripts/download_backtest_data.py b/scripts/download_backtest_data.py index 27c4c1e1c..b83eb2c4b 100755 --- a/scripts/download_backtest_data.py +++ b/scripts/download_backtest_data.py @@ -10,7 +10,14 @@ from freqtrade import arguments from freqtrade.arguments import TimeRange from freqtrade.exchange import Exchange from freqtrade.optimize import download_backtesting_testdata +from freqtrade.configuration import set_loggers +import logging +logging.basicConfig( + level=logging.INFO, + format='%(asctime)s - %(name)s - %(levelname)s - %(message)s', +) +set_loggers(0) DEFAULT_DL_PATH = 'user_data/data' @@ -54,7 +61,9 @@ exchange = Exchange({'key': '', 'exchange': { 'name': args.exchange, 'pair_whitelist': [], - 'ccxt_rate_limit': False + 'ccxt_async_config': { + "enableRateLimit": False + } } }) pairs_not_available = [] From 18c04ab4e21c33a2a32004ce43a2ee49a405a180 Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Fri, 5 Oct 2018 14:29:06 +0200 Subject: [PATCH 115/699] Update ccxt from 1.17.368 to 1.17.369 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 1ec5bdf26..60dcf6c6d 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -ccxt==1.17.368 +ccxt==1.17.369 SQLAlchemy==1.2.12 python-telegram-bot==11.1.0 arrow==0.12.1 From 36d928d411ffaf0e75bf5b074e5de83493ed84e3 Mon Sep 17 00:00:00 2001 From: misagh Date: Fri, 5 Oct 2018 17:06:17 +0200 Subject: [PATCH 116/699] unnecessary if removed --- freqtrade/edge/__init__.py | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/freqtrade/edge/__init__.py b/freqtrade/edge/__init__.py index bf855ef43..91594f4fb 100644 --- a/freqtrade/edge/__init__.py +++ b/freqtrade/edge/__init__.py @@ -107,15 +107,12 @@ class Edge(): trades += self._find_trades_for_stoploss_range(ticker_data, pair, stoploss_range) - # Switch List of Trade Dicts (trades) to Dataframe + # If no trade found then exit + if len(trades) == 0: + return False + # Fill missing, calculable columns, profit, duration , abs etc. - trades_df = DataFrame(trades) - - if len(trades_df) > 0: # Only post process a frame if it has a record - trades_df = self._fill_calculable_fields(trades_df) - else: - trades_df = DataFrame.from_records(trades_df, columns=BacktestResult._fields) - + trades_df = self._fill_calculable_fields(DataFrame(trades)) self._cached_pairs = self._process_expectancy(trades_df) self._last_updated = arrow.utcnow().timestamp @@ -187,6 +184,7 @@ class Edge(): stake = self.config.get('stake_amount') fee = self.fee + open_fee = fee / 2 close_fee = fee / 2 From bd25212bd6ee83bd4b6bfe16db5cf79e7c0e0707 Mon Sep 17 00:00:00 2001 From: misagh Date: Fri, 5 Oct 2018 17:07:20 +0200 Subject: [PATCH 117/699] test case added: edge calculate function --- freqtrade/tests/edge/test_edge.py | 59 +++++++++++++++++++++++++++++++ 1 file changed, 59 insertions(+) diff --git a/freqtrade/tests/edge/test_edge.py b/freqtrade/tests/edge/test_edge.py index 3cc2fd76c..8ddadeb58 100644 --- a/freqtrade/tests/edge/test_edge.py +++ b/freqtrade/tests/edge/test_edge.py @@ -1,8 +1,12 @@ from freqtrade.tests.conftest import get_patched_exchange from freqtrade.edge import Edge +from freqtrade import optimize from pandas import DataFrame, to_datetime import arrow import numpy as np +import math + +from unittest.mock import MagicMock # Cases to be tested: @@ -89,6 +93,61 @@ def _build_dataframe(buy_ohlc_sell_matrice): return frame +def test_edge_heartbeat_calculate(mocker, default_conf): + exchange = get_patched_exchange(mocker, default_conf) + edge = Edge(default_conf, exchange) + heartbeat = default_conf['edge']['process_throttle_secs'] + + # should not recalculate if heartbeat not reached + edge._last_updated = arrow.utcnow().timestamp - heartbeat + 1 + + assert edge.calculate() == False + + +def mocked_load_data(datadir, pairs=[], ticker_interval='0m', refresh_pairs=False, + timerange=None, exchange=None): + hz = 0.1 + base = 0.001 + + ETHBTC = [ + [ + ticker_start_time.shift(minutes=(x * ticker_interval_in_minute)).timestamp * 1000, + math.sin(x * hz) / 1000 + base, # But replace O,H,L,C + math.sin(x * hz) / 1000 + base + 0.0001, + math.sin(x * hz) / 1000 + base - 0.0001, + math.sin(x * hz) / 1000 + base, + 123.45 + ] for x in range(0, 500)] + + hz = 0.2 + base = 0.002 + LTCBTC = [ + [ + ticker_start_time.shift(minutes=(x * ticker_interval_in_minute)).timestamp * 1000, + math.sin(x * hz) / 1000 + base, # But replace O,H,L,C + math.sin(x * hz) / 1000 + base + 0.0001, + math.sin(x * hz) / 1000 + base - 0.0001, + math.sin(x * hz) / 1000 + base, + 123.45 + ] for x in range(0, 500)] + + pairdata = {'NEO/BTC': ETHBTC, 'LTC/BTC': LTCBTC} + return pairdata + + +def test_edge_process_downloaded_data(mocker, default_conf): + default_conf['datadir'] = None + exchange = get_patched_exchange(mocker, default_conf) + mocker.patch('freqtrade.exchange.Exchange.get_fee', MagicMock(return_value=0.001)) + mocker.patch('freqtrade.optimize.load_data', mocked_load_data) + mocker.patch('freqtrade.exchange.Exchange.refresh_tickers', MagicMock()) + edge = Edge(default_conf, exchange) + + assert edge.calculate() + assert len(edge._cached_pairs) == 2 + assert edge._last_updated <= arrow.utcnow().timestamp + 2 + + def test_process_expectancy(mocker, default_conf): default_conf['edge']['min_trade_number'] = 2 exchange = get_patched_exchange(mocker, default_conf) From 9e44b260e29d2370ecf03d3a3dff556cda8cc0e4 Mon Sep 17 00:00:00 2001 From: misagh Date: Fri, 5 Oct 2018 17:19:20 +0200 Subject: [PATCH 118/699] BacktestResult removed as it is not used --- freqtrade/edge/__init__.py | 1 - 1 file changed, 1 deletion(-) diff --git a/freqtrade/edge/__init__.py b/freqtrade/edge/__init__.py index 91594f4fb..490bc065e 100644 --- a/freqtrade/edge/__init__.py +++ b/freqtrade/edge/__init__.py @@ -9,7 +9,6 @@ import utils_find_1st as utf1st from pandas import DataFrame import freqtrade.optimize as optimize -from freqtrade.optimize.backtesting import BacktestResult from freqtrade.arguments import Arguments from freqtrade.arguments import TimeRange from freqtrade.strategy.interface import SellType From 6d4f68fcdbeb8cbbde0cdb824762eb76b718c283 Mon Sep 17 00:00:00 2001 From: misagh Date: Fri, 5 Oct 2018 17:25:56 +0200 Subject: [PATCH 119/699] unnecessary variables removed --- freqtrade/tests/edge/test_edge.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/freqtrade/tests/edge/test_edge.py b/freqtrade/tests/edge/test_edge.py index 8ddadeb58..7d750cc31 100644 --- a/freqtrade/tests/edge/test_edge.py +++ b/freqtrade/tests/edge/test_edge.py @@ -1,6 +1,5 @@ from freqtrade.tests.conftest import get_patched_exchange from freqtrade.edge import Edge -from freqtrade import optimize from pandas import DataFrame, to_datetime import arrow import numpy as np @@ -56,7 +55,6 @@ def test_stoploss(mocker, default_conf): ] )) - pairs = ['A/B', 'C/D', 'E/F', 'G/H'] assert edge.stoploss('E/F') == -0.01 @@ -101,7 +99,7 @@ def test_edge_heartbeat_calculate(mocker, default_conf): # should not recalculate if heartbeat not reached edge._last_updated = arrow.utcnow().timestamp - heartbeat + 1 - assert edge.calculate() == False + assert edge.calculate() is False def mocked_load_data(datadir, pairs=[], ticker_interval='0m', refresh_pairs=False, From 1d38c35e6a6f7db5d2e69f94f02f76fed6c897d3 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 6 Oct 2018 09:23:40 +0200 Subject: [PATCH 120/699] Fix typo / word repetition --- freqtrade/configuration.py | 2 +- freqtrade/tests/test_configuration.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/freqtrade/configuration.py b/freqtrade/configuration.py index 93ae3deab..d61ff4dd7 100644 --- a/freqtrade/configuration.py +++ b/freqtrade/configuration.py @@ -275,7 +275,7 @@ class Configuration(object): if 'ccxt_rate_limit' in config.get('exchange', {}): logger.warning("`ccxt_rate_limit` has been deprecated in favor of " "`ccxt_config` and `ccxt_async_config` and will be removed " - "in future a future version.") + "in a future version.") logger.debug('Exchange "%s" supported', exchange) return True diff --git a/freqtrade/tests/test_configuration.py b/freqtrade/tests/test_configuration.py index 2c487c165..547c38de9 100644 --- a/freqtrade/tests/test_configuration.py +++ b/freqtrade/tests/test_configuration.py @@ -398,7 +398,7 @@ def test_check_exchange(default_conf, caplog) -> None: configuration.check_exchange(default_conf) assert log_has("`ccxt_rate_limit` has been deprecated in favor of " "`ccxt_config` and `ccxt_async_config` and will be removed " - "in future a future version.", + "in a future version.", caplog.record_tuples) From d5409287e099b6707210dcdc6d8fb805410fb948 Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Sat, 6 Oct 2018 14:29:05 +0200 Subject: [PATCH 121/699] Update ccxt from 1.17.369 to 1.17.371 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 60dcf6c6d..f914f819f 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -ccxt==1.17.369 +ccxt==1.17.371 SQLAlchemy==1.2.12 python-telegram-bot==11.1.0 arrow==0.12.1 From 3af655d1702c16c94045fc28e66a66850d9a6c4f Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Mon, 8 Oct 2018 14:29:07 +0200 Subject: [PATCH 122/699] Update ccxt from 1.17.371 to 1.17.373 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index f914f819f..28e47b9e9 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -ccxt==1.17.371 +ccxt==1.17.373 SQLAlchemy==1.2.12 python-telegram-bot==11.1.0 arrow==0.12.1 From 2cd7b40b3805a8df58217c059dbec24cbe2a79fb Mon Sep 17 00:00:00 2001 From: Matthias Date: Mon, 8 Oct 2018 15:39:21 +0200 Subject: [PATCH 123/699] Fix spelling in stoploss.md --- docs/stoploss.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/stoploss.md b/docs/stoploss.md index 9740672cd..014de32f2 100644 --- a/docs/stoploss.md +++ b/docs/stoploss.md @@ -48,4 +48,4 @@ Both values can be configured in the main configuration file and requires `"trai The 0.01 would translate to a 1% stop loss, once you hit 1.1% profit. -You should also make sure to have this value higher than your minimal ROI, otherwise minimal ROI will apply first and sell your trade. +You should also make sure to have this value lower than your minimal ROI, otherwise minimal ROI will apply first and sell your trade. From 21480d4219b6d8a81f37a486a77a358e46b658b5 Mon Sep 17 00:00:00 2001 From: Matthias Date: Mon, 8 Oct 2018 15:41:07 +0200 Subject: [PATCH 124/699] be more expressive on what "this value" is --- docs/stoploss.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/stoploss.md b/docs/stoploss.md index 014de32f2..f34050a85 100644 --- a/docs/stoploss.md +++ b/docs/stoploss.md @@ -48,4 +48,4 @@ Both values can be configured in the main configuration file and requires `"trai The 0.01 would translate to a 1% stop loss, once you hit 1.1% profit. -You should also make sure to have this value lower than your minimal ROI, otherwise minimal ROI will apply first and sell your trade. +You should also make sure to have this value (`trailing_stop_positive_offset`) lower than your minimal ROI, otherwise minimal ROI will apply first and sell your trade. From a20ceb9e31912fb64fbd05652d7a09eda6def191 Mon Sep 17 00:00:00 2001 From: Matthias Date: Mon, 8 Oct 2018 19:43:37 +0200 Subject: [PATCH 125/699] Add reload_conf to telegram help --- freqtrade/rpc/telegram.py | 1 + 1 file changed, 1 insertion(+) diff --git a/freqtrade/rpc/telegram.py b/freqtrade/rpc/telegram.py index 64708ef74..b6ad047b5 100644 --- a/freqtrade/rpc/telegram.py +++ b/freqtrade/rpc/telegram.py @@ -437,6 +437,7 @@ class Telegram(RPC): "*/count:* `Show number of trades running compared to allowed number of trades`" \ "\n" \ "*/balance:* `Show account balance per currency`\n" \ + "*/reload_conf:* `Reload configuration file` \n" \ "*/help:* `This help message`\n" \ "*/version:* `Show version`" From 5029003957cc0d3541f93783e69886e42b3e6e4e Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 9 Oct 2018 07:06:11 +0200 Subject: [PATCH 126/699] Allow passing price to buy function --- freqtrade/freqtradebot.py | 9 ++++-- freqtrade/tests/test_freqtradebot.py | 48 ++++++++++++++++++++++++++++ 2 files changed, 54 insertions(+), 3 deletions(-) diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index fa803bda7..1d09643af 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -425,7 +425,7 @@ class FreqtradeBot(object): return True return False - def execute_buy(self, pair: str, stake_amount: float) -> bool: + def execute_buy(self, pair: str, stake_amount: float, price: Optional[float] = None) -> bool: """ Executes a limit buy for the given pair :param pair: pair for which we want to create a LIMIT_BUY @@ -436,8 +436,11 @@ class FreqtradeBot(object): stake_currency = self.config['stake_currency'] fiat_currency = self.config.get('fiat_display_currency', None) - # Calculate amount - buy_limit = self.get_target_bid(pair, self.exchange.get_ticker(pair)) + if price: + buy_limit = price + else: + # Calculate amount + buy_limit = self.get_target_bid(pair, self.exchange.get_ticker(pair)) min_stake_amount = self._get_min_pair_stake_amount(pair_s, buy_limit) if min_stake_amount is not None and min_stake_amount > stake_amount: diff --git a/freqtrade/tests/test_freqtradebot.py b/freqtrade/tests/test_freqtradebot.py index cad2f654d..b147d34e7 100644 --- a/freqtrade/tests/test_freqtradebot.py +++ b/freqtrade/tests/test_freqtradebot.py @@ -689,6 +689,54 @@ def test_balance_bigger_last_ask(mocker, default_conf) -> None: assert freqtrade.get_target_bid('ETH/BTC', {'ask': 5, 'last': 10}) == 5 +def test_execute_buy(mocker, default_conf, fee, markets, limit_buy_order) -> None: + patch_RPCManager(mocker) + patch_exchange(mocker) + freqtrade = FreqtradeBot(default_conf) + stake_amount = 2 + bid = 0.11 + get_bid = MagicMock(return_value=bid) + mocker.patch.multiple( + 'freqtrade.freqtradebot.FreqtradeBot', + get_target_bid=get_bid, + _get_min_pair_stake_amount=MagicMock(return_value=1) + ) + buy_mm = MagicMock(return_value={'id': limit_buy_order['id']}) + mocker.patch.multiple( + 'freqtrade.exchange.Exchange', + get_ticker=MagicMock(return_value={ + 'bid': 0.00001172, + 'ask': 0.00001173, + 'last': 0.00001172 + }), + buy=buy_mm, + get_fee=fee, + get_markets=markets + ) + pair = 'ETH/BTC' + print(buy_mm.call_args_list) + + assert freqtrade.execute_buy(pair, stake_amount) + assert get_bid.call_count == 1 + assert buy_mm.call_count == 1 + call_args = buy_mm.call_args_list[0][0] + assert call_args[0] == pair + assert call_args[1] == bid + assert call_args[2] == stake_amount / bid + + # Test calling with price + fix_price = 0.06 + assert freqtrade.execute_buy(pair, stake_amount, fix_price) + # Make sure get_target_bid wasn't called again + assert get_bid.call_count == 1 + + assert buy_mm.call_count == 2 + call_args = buy_mm.call_args_list[1][0] + assert call_args[0] == pair + assert call_args[1] == fix_price + assert call_args[2] == stake_amount / fix_price + + def test_process_maybe_execute_buy(mocker, default_conf) -> None: freqtrade = get_patched_freqtradebot(mocker, default_conf) From b1f016b9c035f784ee3d51df1886fc8989d809ab Mon Sep 17 00:00:00 2001 From: fapaydin Date: Tue, 9 Oct 2018 11:49:25 +0300 Subject: [PATCH 127/699] Update hyperopt.md Invalid argument in description. Replace --timeperiod with --timerange correct format. --- docs/hyperopt.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/hyperopt.md b/docs/hyperopt.md index 3f568d82e..7444d32b7 100644 --- a/docs/hyperopt.md +++ b/docs/hyperopt.md @@ -131,12 +131,12 @@ you have on-disk, use the `--datadir PATH` option. Default hyperopt will use data from directory `user_data/data`. ### Running Hyperopt with Smaller Testset -Use the `--timeperiod` argument to change how much of the testset +Use the `--timerange` argument to change how much of the testset you want to use. The last N ticks/timeframes will be used. Example: ```bash -python3 ./freqtrade/main.py hyperopt --timeperiod -200 +python3 ./freqtrade/main.py hyperopt --timerange -200 ``` ### Running Hyperopt with Smaller Search Space From 98c1706cdd64c48b59970c3604a222ed35eca725 Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Tue, 9 Oct 2018 14:29:07 +0200 Subject: [PATCH 128/699] Update ccxt from 1.17.373 to 1.17.375 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 28e47b9e9..807d042db 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -ccxt==1.17.373 +ccxt==1.17.375 SQLAlchemy==1.2.12 python-telegram-bot==11.1.0 arrow==0.12.1 From fbe69cee3f040af8527b7793684bce39af9d6d20 Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 9 Oct 2018 19:25:43 +0200 Subject: [PATCH 129/699] Add /forcebuy to telegram --- freqtrade/rpc/rpc.py | 34 ++++++++++++++++++++++++++++++++++ freqtrade/rpc/telegram.py | 19 +++++++++++++++++++ 2 files changed, 53 insertions(+) diff --git a/freqtrade/rpc/rpc.py b/freqtrade/rpc/rpc.py index dc3d9bd65..d33e4bdb1 100644 --- a/freqtrade/rpc/rpc.py +++ b/freqtrade/rpc/rpc.py @@ -379,6 +379,40 @@ class RPC(object): _exec_forcesell(trade) Trade.session.flush() + def _rpc_forcebuy(self, pair: str, price: Optional[float]) -> Optional[Trade]: + """ + Handler for forcebuy + Buys a pair trade at the given or current price + """ + + if not self._freqtrade.config.get('forcebuy_enable', False): + raise RPCException('Forcebuy not enabled.') + + if self._freqtrade.state != State.RUNNING: + raise RPCException('trader is not running') + + # Check pair is in stake currency + stake_currency = self._freqtrade.config.get('stake_currency') + if not pair.endswith(stake_currency): + raise RPCException( + f'Wrong pair selected. Please pairs with stake {stake_currency} pairs only') + # check if valid pair + + # check if pair already has an open pair + trade = Trade.query.filter(Trade.is_open.is_(True)).filter(Trade.pair.is_(pair)).first() + if trade: + raise RPCException(f'position for {pair} already open - id: {trade.id}') + + # gen stake amount + stakeamount = self._freqtrade._get_trade_stake_amount() + + # execute buy + if self._freqtrade.execute_buy(pair, stakeamount, price): + trade = Trade.query.filter(Trade.is_open.is_(True)).filter(Trade.pair.is_(pair)).first() + return trade + else: + return None + def _rpc_performance(self) -> List[Dict]: """ Handler for performance. diff --git a/freqtrade/rpc/telegram.py b/freqtrade/rpc/telegram.py index 64708ef74..3354d6e6f 100644 --- a/freqtrade/rpc/telegram.py +++ b/freqtrade/rpc/telegram.py @@ -86,6 +86,7 @@ class Telegram(RPC): CommandHandler('start', self._start), CommandHandler('stop', self._stop), CommandHandler('forcesell', self._forcesell), + CommandHandler('forcebuy', self._forcebuy), CommandHandler('performance', self._performance), CommandHandler('daily', self._daily), CommandHandler('count', self._count), @@ -372,6 +373,24 @@ class Telegram(RPC): except RPCException as e: self._send_msg(str(e), bot=bot) + @authorized_only + def _forcebuy(self, bot: Bot, update: Update) -> None: + """ + Handler for /forcebuy . + Buys a pair trade at the given or current price + :param bot: telegram bot + :param update: message update + :return: None + """ + + message = update.message.text.replace('/forcebuy', '').strip().split() + pair = message[0] + price = float(message[1]) if len(message) > 1 else None + try: + self._rpc_forcebuy(pair, price) + except RPCException as e: + self._send_msg(str(e), bot=bot) + @authorized_only def _performance(self, bot: Bot, update: Update) -> None: """ From 8c6d7c48ad8b49fc2fb2796ac5f33448f881d385 Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 9 Oct 2018 20:04:53 +0200 Subject: [PATCH 130/699] Add tests for /forcebuy --- freqtrade/tests/rpc/test_rpc.py | 76 ++++++++++++++++++++++++ freqtrade/tests/rpc/test_rpc_telegram.py | 61 ++++++++++++++++++- 2 files changed, 135 insertions(+), 2 deletions(-) diff --git a/freqtrade/tests/rpc/test_rpc.py b/freqtrade/tests/rpc/test_rpc.py index efc136777..e1db34347 100644 --- a/freqtrade/tests/rpc/test_rpc.py +++ b/freqtrade/tests/rpc/test_rpc.py @@ -532,3 +532,79 @@ def test_rpc_count(mocker, default_conf, ticker, fee, markets) -> None: trades = rpc._rpc_count() nb_trades = len(trades) assert nb_trades == 1 + + +def test_rpcforcebuy(mocker, default_conf, ticker, fee, markets, limit_buy_order) -> None: + default_conf['forcebuy_enable'] = True + patch_coinmarketcap(mocker) + patch_exchange(mocker) + mocker.patch('freqtrade.rpc.telegram.Telegram', MagicMock()) + buy_mm = MagicMock(return_value={'id': limit_buy_order['id']}) + mocker.patch.multiple( + 'freqtrade.exchange.Exchange', + get_balances=MagicMock(return_value=ticker), + get_ticker=ticker, + get_fee=fee, + get_markets=markets, + buy=buy_mm + ) + + freqtradebot = FreqtradeBot(default_conf) + patch_get_signal(freqtradebot, (True, False)) + rpc = RPC(freqtradebot) + pair = 'ETH/BTC' + trade = rpc._rpc_forcebuy(pair, None) + assert isinstance(trade, Trade) + assert trade.pair == pair + assert trade.open_rate == ticker()['ask'] + + # Test buy duplicate + with pytest.raises(RPCException, match=r'position for ETH/BTC already open - id: 1'): + rpc._rpc_forcebuy(pair, 0.0001) + pair = 'XRP/BTC' + trade = rpc._rpc_forcebuy(pair, 0.0001) + assert isinstance(trade, Trade) + assert trade.pair == pair + assert trade.open_rate == 0.0001 + + # Test buy pair not with stakes + with pytest.raises(RPCException, match=r'Wrong pair selected. Please pairs with stake.*'): + rpc._rpc_forcebuy('XRP/ETH', 0.0001) + pair = 'XRP/BTC' + + # Test not buying + default_conf['stake_amount'] = 0.0000001 + freqtradebot = FreqtradeBot(default_conf) + patch_get_signal(freqtradebot, (True, False)) + rpc = RPC(freqtradebot) + pair = 'TKN/BTC' + trade = rpc._rpc_forcebuy(pair, None) + assert trade is None + + +def test_rpcforcebuy_stopped(mocker, default_conf) -> None: + default_conf['forcebuy_enable'] = True + default_conf['initial_state'] = 'stopped' + patch_coinmarketcap(mocker) + patch_exchange(mocker) + mocker.patch('freqtrade.rpc.telegram.Telegram', MagicMock()) + + freqtradebot = FreqtradeBot(default_conf) + patch_get_signal(freqtradebot, (True, False)) + rpc = RPC(freqtradebot) + pair = 'ETH/BTC' + with pytest.raises(RPCException, match=r'trader is not running'): + rpc._rpc_forcebuy(pair, None) + + +def test_rpcforcebuy_disabled(mocker, default_conf) -> None: + patch_coinmarketcap(mocker) + patch_exchange(mocker) + mocker.patch('freqtrade.rpc.telegram.Telegram', MagicMock()) + + freqtradebot = FreqtradeBot(default_conf) + patch_get_signal(freqtradebot, (True, False)) + rpc = RPC(freqtradebot) + pair = 'ETH/BTC' + with pytest.raises(RPCException, match=r'Forcebuy not enabled.'): + rpc._rpc_forcebuy(pair, None) diff --git a/freqtrade/tests/rpc/test_rpc_telegram.py b/freqtrade/tests/rpc/test_rpc_telegram.py index 182c1d2e7..25f2be350 100644 --- a/freqtrade/tests/rpc/test_rpc_telegram.py +++ b/freqtrade/tests/rpc/test_rpc_telegram.py @@ -71,8 +71,8 @@ def test_init(default_conf, mocker, caplog) -> None: assert start_polling.start_polling.call_count == 1 message_str = "rpc.telegram is listening for following commands: [['status'], ['profit'], " \ - "['balance'], ['start'], ['stop'], ['forcesell'], ['performance'], ['daily'], " \ - "['count'], ['reload_conf'], ['help'], ['version']]" + "['balance'], ['start'], ['stop'], ['forcesell'], ['forcebuy'], " \ + "['performance'], ['daily'], ['count'], ['reload_conf'], ['help'], ['version']]" assert log_has(message_str, caplog.record_tuples) @@ -855,6 +855,63 @@ def test_forcesell_handle_invalid(default_conf, update, mocker) -> None: assert 'invalid argument' in msg_mock.call_args_list[0][0][0] +def test_forcebuy_handle(default_conf, update, markets, mocker) -> None: + patch_coinmarketcap(mocker, value={'price_usd': 15000.0}) + mocker.patch('freqtrade.rpc.rpc.CryptoToFiatConverter._find_price', return_value=15000.0) + mocker.patch('freqtrade.rpc.telegram.Telegram._send_msg', MagicMock()) + mocker.patch('freqtrade.rpc.telegram.Telegram._init', MagicMock()) + mocker.patch.multiple( + 'freqtrade.exchange.Exchange', + _load_markets=MagicMock(return_value={}), + get_markets=markets + ) + fbuy_mock = MagicMock(return_value=None) + mocker.patch('freqtrade.rpc.RPC._rpc_forcebuy', fbuy_mock) + + freqtradebot = FreqtradeBot(default_conf) + patch_get_signal(freqtradebot, (True, False)) + telegram = Telegram(freqtradebot) + + update.message.text = '/forcebuy ETH/BTC' + telegram._forcebuy(bot=MagicMock(), update=update) + + assert fbuy_mock.call_count == 1 + assert fbuy_mock.call_args_list[0][0][0] == 'ETH/BTC' + assert fbuy_mock.call_args_list[0][0][1] is None + + # Reset and retry with specified price + fbuy_mock = MagicMock(return_value=None) + mocker.patch('freqtrade.rpc.RPC._rpc_forcebuy', fbuy_mock) + update.message.text = '/forcebuy ETH/BTC 0.055' + telegram._forcebuy(bot=MagicMock(), update=update) + + assert fbuy_mock.call_count == 1 + assert fbuy_mock.call_args_list[0][0][0] == 'ETH/BTC' + assert isinstance(fbuy_mock.call_args_list[0][0][1], float) + assert fbuy_mock.call_args_list[0][0][1] == 0.055 + + +def test_forcebuy_handle_exception(default_conf, update, markets, mocker) -> None: + patch_coinmarketcap(mocker, value={'price_usd': 15000.0}) + mocker.patch('freqtrade.rpc.rpc.CryptoToFiatConverter._find_price', return_value=15000.0) + rpc_mock = mocker.patch('freqtrade.rpc.telegram.Telegram._send_msg', MagicMock()) + mocker.patch('freqtrade.rpc.telegram.Telegram._init', MagicMock()) + mocker.patch.multiple( + 'freqtrade.exchange.Exchange', + _load_markets=MagicMock(return_value={}), + get_markets=markets + ) + freqtradebot = FreqtradeBot(default_conf) + patch_get_signal(freqtradebot, (True, False)) + telegram = Telegram(freqtradebot) + + update.message.text = '/forcebuy ETH/Nonepair' + telegram._forcebuy(bot=MagicMock(), update=update) + + assert rpc_mock.call_count == 1 + assert rpc_mock.call_args_list[0][0][0] == 'Forcebuy not enabled.' + + def test_performance_handle(default_conf, update, ticker, fee, limit_buy_order, limit_sell_order, markets, mocker) -> None: patch_coinmarketcap(mocker) From 44c275c8015dbe66ded193221db8275a100b5531 Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 9 Oct 2018 21:08:56 +0200 Subject: [PATCH 131/699] flush session for /forcesell all --- freqtrade/rpc/rpc.py | 1 + 1 file changed, 1 insertion(+) diff --git a/freqtrade/rpc/rpc.py b/freqtrade/rpc/rpc.py index dc3d9bd65..7031086d8 100644 --- a/freqtrade/rpc/rpc.py +++ b/freqtrade/rpc/rpc.py @@ -363,6 +363,7 @@ class RPC(object): # Execute sell for all open orders for trade in Trade.query.filter(Trade.is_open.is_(True)).all(): _exec_forcesell(trade) + Trade.session.flush() return # Query for trade From a541d0a9319813c9962c02d3d3c4048f06ea9053 Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 9 Oct 2018 21:13:30 +0200 Subject: [PATCH 132/699] convert tests to packages source: https://docs.pytest.org/en/latest/goodpractices.html If you need to have test modules with the same name, you might add __init__.py files to your tests folder and subfolders, changing them to packages: --- freqtrade/tests/exchange/__init__.py | 0 freqtrade/tests/optimize/__init__.py | 0 freqtrade/tests/rpc/__init__.py | 0 freqtrade/tests/strategy/__init__.py | 0 4 files changed, 0 insertions(+), 0 deletions(-) create mode 100644 freqtrade/tests/exchange/__init__.py create mode 100644 freqtrade/tests/optimize/__init__.py create mode 100644 freqtrade/tests/rpc/__init__.py create mode 100644 freqtrade/tests/strategy/__init__.py diff --git a/freqtrade/tests/exchange/__init__.py b/freqtrade/tests/exchange/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/freqtrade/tests/optimize/__init__.py b/freqtrade/tests/optimize/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/freqtrade/tests/rpc/__init__.py b/freqtrade/tests/rpc/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/freqtrade/tests/strategy/__init__.py b/freqtrade/tests/strategy/__init__.py new file mode 100644 index 000000000..e69de29bb From 03fb188555863f8054c7cdd066975c1b9c4cba0d Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Wed, 10 Oct 2018 14:29:06 +0200 Subject: [PATCH 133/699] Update urllib3 from 1.22 to 1.23 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 807d042db..a5f22fec2 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,7 +4,7 @@ python-telegram-bot==11.1.0 arrow==0.12.1 cachetools==2.1.0 requests==2.19.1 -urllib3==1.22 +urllib3==1.23 wrapt==1.10.11 pandas==0.23.4 scikit-learn==0.20.0 From 6ff4c9b88839dd5c94a2e0013157587274333686 Mon Sep 17 00:00:00 2001 From: Matthias Date: Wed, 10 Oct 2018 20:08:29 +0200 Subject: [PATCH 134/699] Update docs for /forcesell --- config.json.example | 1 + config_full.json.example | 1 + docs/configuration.md | 12 +++++++++++- docs/telegram-usage.md | 18 ++++++++++++++++++ freqtrade/constants.py | 1 + 5 files changed, 32 insertions(+), 1 deletion(-) diff --git a/config.json.example b/config.json.example index 5198b9d81..d6806c0e9 100644 --- a/config.json.example +++ b/config.json.example @@ -59,6 +59,7 @@ "chat_id": "your_telegram_chat_id" }, "initial_state": "running", + "forcebuy_enable": false, "internals": { "process_throttle_secs": 5 } diff --git a/config_full.json.example b/config_full.json.example index bdc6361d2..af6c7c045 100644 --- a/config_full.json.example +++ b/config_full.json.example @@ -71,6 +71,7 @@ }, "db_url": "sqlite:///tradesv3.sqlite", "initial_state": "running", + "forcebuy_enable": false, "internals": { "process_throttle_secs": 5 }, diff --git a/docs/configuration.md b/docs/configuration.md index 3512f891b..90a39ccc2 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -53,13 +53,14 @@ The table below will list all configuration parameters. | `telegram.enabled` | true | Yes | Enable or not the usage of Telegram. | `telegram.token` | token | No | Your Telegram bot token. Only required if `telegram.enabled` is `true`. | `telegram.chat_id` | chat_id | No | Your personal Telegram account id. Only required if `telegram.enabled` is `true`. -| `webhook.enabled` | false | No | Enable useage of Webhook notifications +| `webhook.enabled` | false | No | Enable usage of Webhook notifications | `webhook.url` | false | No | URL for the webhook. Only required if `webhook.enabled` is `true`. See the [webhook documentation](webhook-config.md) for more details. | `webhook.webhookbuy` | false | No | Payload to send on buy. Only required if `webhook.enabled` is `true`. See the [webhook documentationV](webhook-config.md) for more details. | `webhook.webhooksell` | false | No | Payload to send on sell. Only required if `webhook.enabled` is `true`. See the [webhook documentationV](webhook-config.md) for more details. | `webhook.webhookstatus` | false | No | Payload to send on status calls. Only required if `webhook.enabled` is `true`. See the [webhook documentationV](webhook-config.md) for more details. | `db_url` | `sqlite:///tradesv3.sqlite` | No | Declares database URL to use. NOTE: This defaults to `sqlite://` if `dry_run` is `True`. | `initial_state` | running | No | Defines the initial application state. More information below. +| `forcebuy_enable` | false | No | Enables the RPC Commands to force a buy. More information below. | `strategy` | DefaultStrategy | No | Defines Strategy class to use. | `strategy_path` | null | No | Adds an additional strategy lookup path (must be a folder). | `internals.process_throttle_secs` | 5 | Yes | Set the process throttle. Value in second. @@ -113,6 +114,15 @@ Go to the [trailing stoploss Documentation](stoploss.md) for details on trailing Possible values are `running` or `stopped`. (default=`running`) If the value is `stopped` the bot has to be started with `/start` first. +### Understand forcebuy_enable + +`forcebuy_enable` enables the usage of forcebuy commands via Telegram. +This is disabled for security reasons by default. +You send `/forcebuy ETH/BTC` to the bot, who buys the pair and holds it until a regular sell-signal appears (ROI, stoploss, /forcesell). + +Can be dangerous with some strategies, so use with care +See [the telegram documentation](telegram-usage.md) for details on usage. + ### Understand process_throttle_secs `process_throttle_secs` is an optional field that defines in seconds how long the bot should wait diff --git a/docs/telegram-usage.md b/docs/telegram-usage.md index 945e31f9c..28213fb5d 100644 --- a/docs/telegram-usage.md +++ b/docs/telegram-usage.md @@ -23,6 +23,7 @@ official commands. You can ask at any moment for help with `/help`. | `/profit` | | Display a summary of your profit/loss from close trades and some stats about your performance | `/forcesell ` | | Instantly sells the given trade (Ignoring `minimum_roi`). | `/forcesell all` | | Instantly sells all open trades (Ignoring `minimum_roi`). +| `/forcebuy [rate]` | | Instantly buys the given pair. Rate is optional. (`forcebuy_enable` must be set to True) | `/performance` | | Show performance of each finished trade grouped by pair | `/balance` | | Show account balance per currency | `/daily ` | 7 | Shows profit or loss per day, over the last n days @@ -30,16 +31,20 @@ official commands. You can ask at any moment for help with `/help`. | `/version` | | Show version ## Telegram commands in action + Below, example of Telegram message you will receive for each command. ### /start + > **Status:** `running` ### /stop + > `Stopping trader ...` > **Status:** `stopped` ## /status + For each open trade, the bot will send you the following message. > **Trade ID:** `123` @@ -54,6 +59,7 @@ For each open trade, the bot will send you the following message. > **Open Order:** `None` ## /status table + Return the status of all open trades in a table format. ``` ID Pair Since Profit @@ -63,6 +69,7 @@ Return the status of all open trades in a table format. ``` ## /count + Return the number of trades used and available. ``` current max @@ -71,6 +78,7 @@ current max ``` ## /profit + Return a summary of your profit/loss and performance. > **ROI:** Close trades @@ -90,7 +98,14 @@ Return a summary of your profit/loss and performance. > **BITTREX:** Selling BTC/LTC with limit `0.01650000 (profit: ~-4.07%, -0.00008168)` +## /forcebuy + +> **BITTREX**: Buying ETH/BTC with limit `0.03400000` (`1.000000 ETH`, `225.290 USD`) + +Note that for this to work, `forcebuy_enable` needs to be set to true. + ## /performance + Return the performance of each crypto-currency the bot has sold. > Performance: > 1. `RCN/BTC 57.77%` @@ -101,6 +116,7 @@ Return the performance of each crypto-currency the bot has sold. > ... ## /balance + Return the balance of all crypto-currency your have on the exchange. > **Currency:** BTC @@ -114,6 +130,7 @@ Return the balance of all crypto-currency your have on the exchange. > **Pending:** 0.0 ## /daily + Per default `/daily` will return the 7 last days. The example below if for `/daily 3`: @@ -127,5 +144,6 @@ Day Profit BTC Profit USD ``` ## /version + > **Version:** `0.14.3` diff --git a/freqtrade/constants.py b/freqtrade/constants.py index 12f10d3b9..2b09aa6c9 100644 --- a/freqtrade/constants.py +++ b/freqtrade/constants.py @@ -130,6 +130,7 @@ CONF_SCHEMA = { }, 'db_url': {'type': 'string'}, 'initial_state': {'type': 'string', 'enum': ['running', 'stopped']}, + 'forcebuy_enable': {'type': 'boolean'}, 'internals': { 'type': 'object', 'properties': { From 3de3c246b404fffdf5589c8d99830c20eccf55d8 Mon Sep 17 00:00:00 2001 From: Matthias Date: Wed, 10 Oct 2018 20:13:56 +0200 Subject: [PATCH 135/699] add warning-message when forcebuy_enable is true --- docs/configuration.md | 2 +- freqtrade/configuration.py | 3 +++ freqtrade/tests/test_configuration.py | 14 ++++++++++++++ 3 files changed, 18 insertions(+), 1 deletion(-) diff --git a/docs/configuration.md b/docs/configuration.md index 90a39ccc2..15ba4b48d 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -117,7 +117,7 @@ If the value is `stopped` the bot has to be started with `/start` first. ### Understand forcebuy_enable `forcebuy_enable` enables the usage of forcebuy commands via Telegram. -This is disabled for security reasons by default. +This is disabled for security reasons by default, and will show a warning message on startup if enabled. You send `/forcebuy ETH/BTC` to the bot, who buys the pair and holds it until a regular sell-signal appears (ROI, stoploss, /forcesell). Can be dangerous with some strategies, so use with care diff --git a/freqtrade/configuration.py b/freqtrade/configuration.py index d61ff4dd7..e043525a7 100644 --- a/freqtrade/configuration.py +++ b/freqtrade/configuration.py @@ -127,6 +127,9 @@ class Configuration(object): config['db_url'] = constants.DEFAULT_DB_PROD_URL logger.info('Dry run is disabled') + if config.get('forcebuy_enable', False): + logger.warning('`forcebuy` RPC message enabled.') + logger.info(f'Using DB: "{config["db_url"]}"') # Check if the exchange set by the user is supported diff --git a/freqtrade/tests/test_configuration.py b/freqtrade/tests/test_configuration.py index 547c38de9..daaaec090 100644 --- a/freqtrade/tests/test_configuration.py +++ b/freqtrade/tests/test_configuration.py @@ -455,5 +455,19 @@ def test_set_loggers() -> None: assert logging.getLogger('telegram').level is logging.INFO +def test_load_config_warn_forcebuy(default_conf, mocker, caplog) -> None: + default_conf['forcebuy_enable'] = True + mocker.patch('freqtrade.configuration.open', mocker.mock_open( + read_data=json.dumps(default_conf) + )) + + args = Arguments([], '').get_parsed_arg() + configuration = Configuration(args) + validated_conf = configuration.load_config() + + assert validated_conf.get('forcebuy_enable') + assert log_has('`forcebuy` RPC message enabled.', caplog.record_tuples) + + def test_validate_default_conf(default_conf) -> None: validate(default_conf, constants.CONF_SCHEMA) From 3e8e8a55faae7f42765c38593de933cc01496c89 Mon Sep 17 00:00:00 2001 From: Matthias Date: Wed, 10 Oct 2018 20:58:21 +0200 Subject: [PATCH 136/699] Enable analytical telegram commands when stopped --- freqtrade/rpc/rpc.py | 10 ++------- freqtrade/tests/rpc/test_rpc.py | 8 -------- freqtrade/tests/rpc/test_rpc_telegram.py | 26 ++++-------------------- 3 files changed, 6 insertions(+), 38 deletions(-) diff --git a/freqtrade/rpc/rpc.py b/freqtrade/rpc/rpc.py index 7031086d8..d653ea176 100644 --- a/freqtrade/rpc/rpc.py +++ b/freqtrade/rpc/rpc.py @@ -84,9 +84,7 @@ class RPC(object): """ # Fetch open trade trades = Trade.query.filter(Trade.is_open.is_(True)).all() - if self._freqtrade.state != State.RUNNING: - raise RPCException('trader is not running') - elif not trades: + if not trades: raise RPCException('no active trade') else: results = [] @@ -118,9 +116,7 @@ class RPC(object): def _rpc_status_table(self) -> DataFrame: trades = Trade.query.filter(Trade.is_open.is_(True)).all() - if self._freqtrade.state != State.RUNNING: - raise RPCException('trader is not running') - elif not trades: + if not trades: raise RPCException('no active order') else: trades_list = [] @@ -385,8 +381,6 @@ class RPC(object): Handler for performance. Shows a performance statistic from finished trades """ - if self._freqtrade.state != State.RUNNING: - raise RPCException('trader is not running') pair_rates = Trade.session.query(Trade.pair, sql.func.sum(Trade.close_profit).label('profit_sum'), diff --git a/freqtrade/tests/rpc/test_rpc.py b/freqtrade/tests/rpc/test_rpc.py index efc136777..88bf5e9ad 100644 --- a/freqtrade/tests/rpc/test_rpc.py +++ b/freqtrade/tests/rpc/test_rpc.py @@ -40,10 +40,6 @@ def test_rpc_trade_status(default_conf, ticker, fee, markets, mocker) -> None: patch_get_signal(freqtradebot, (True, False)) rpc = RPC(freqtradebot) - freqtradebot.state = State.STOPPED - with pytest.raises(RPCException, match=r'.*trader is not running*'): - rpc._rpc_trade_status() - freqtradebot.state = State.RUNNING with pytest.raises(RPCException, match=r'.*no active trade*'): rpc._rpc_trade_status() @@ -81,10 +77,6 @@ def test_rpc_status_table(default_conf, ticker, fee, markets, mocker) -> None: patch_get_signal(freqtradebot, (True, False)) rpc = RPC(freqtradebot) - freqtradebot.state = State.STOPPED - with pytest.raises(RPCException, match=r'.*trader is not running*'): - rpc._rpc_status_table() - freqtradebot.state = State.RUNNING with pytest.raises(RPCException, match=r'.*no active order*'): rpc._rpc_status_table() diff --git a/freqtrade/tests/rpc/test_rpc_telegram.py b/freqtrade/tests/rpc/test_rpc_telegram.py index 182c1d2e7..3072d1cfe 100644 --- a/freqtrade/tests/rpc/test_rpc_telegram.py +++ b/freqtrade/tests/rpc/test_rpc_telegram.py @@ -250,9 +250,10 @@ def test_status_handle(default_conf, update, ticker, fee, markets, mocker) -> No telegram = Telegram(freqtradebot) freqtradebot.state = State.STOPPED + # Status is also enabled when stopped telegram._status(bot=MagicMock(), update=update) assert msg_mock.call_count == 1 - assert 'trader is not running' in msg_mock.call_args_list[0][0][0] + assert 'no active trade' in msg_mock.call_args_list[0][0][0] msg_mock.reset_mock() freqtradebot.state = State.RUNNING @@ -295,9 +296,10 @@ def test_status_table_handle(default_conf, update, ticker, fee, markets, mocker) telegram = Telegram(freqtradebot) freqtradebot.state = State.STOPPED + # Status table is also enabled when stopped telegram._status_table(bot=MagicMock(), update=update) assert msg_mock.call_count == 1 - assert 'trader is not running' in msg_mock.call_args_list[0][0][0] + assert 'no active order' in msg_mock.call_args_list[0][0][0] msg_mock.reset_mock() freqtradebot.state = State.RUNNING @@ -895,26 +897,6 @@ def test_performance_handle(default_conf, update, ticker, fee, assert 'ETH/BTC\t6.20% (1)' in msg_mock.call_args_list[0][0][0] -def test_performance_handle_invalid(default_conf, update, mocker) -> None: - patch_coinmarketcap(mocker) - patch_exchange(mocker) - msg_mock = MagicMock() - mocker.patch.multiple( - 'freqtrade.rpc.telegram.Telegram', - _init=MagicMock(), - _send_msg=msg_mock - ) - freqtradebot = FreqtradeBot(default_conf) - patch_get_signal(freqtradebot, (True, False)) - telegram = Telegram(freqtradebot) - - # Trader is not running - freqtradebot.state = State.STOPPED - telegram._performance(bot=MagicMock(), update=update) - assert msg_mock.call_count == 1 - assert 'not running' in msg_mock.call_args_list[0][0][0] - - def test_count_handle(default_conf, update, ticker, fee, markets, mocker) -> None: patch_coinmarketcap(mocker) patch_exchange(mocker) From a4d2bb6f294d2562ae9b90105fae9e486d0630d7 Mon Sep 17 00:00:00 2001 From: Matthias Date: Wed, 10 Oct 2018 21:28:48 +0200 Subject: [PATCH 137/699] Fix "No market symbol" exception in telegram calls --- freqtrade/exchange/__init__.py | 2 ++ freqtrade/rpc/rpc.py | 21 +++++++++++++++------ freqtrade/tests/exchange/test_exchange.py | 4 ++++ 3 files changed, 21 insertions(+), 6 deletions(-) diff --git a/freqtrade/exchange/__init__.py b/freqtrade/exchange/__init__.py index 7fd0e5f43..4af9db6db 100644 --- a/freqtrade/exchange/__init__.py +++ b/freqtrade/exchange/__init__.py @@ -375,6 +375,8 @@ class Exchange(object): def get_ticker(self, pair: str, refresh: Optional[bool] = True) -> dict: if refresh or pair not in self._cached_ticker.keys(): try: + if pair not in self._api.markets: + raise DependencyException(f"Pair {pair} not available") data = self._api.fetch_ticker(pair) try: self._cached_ticker[pair] = { diff --git a/freqtrade/rpc/rpc.py b/freqtrade/rpc/rpc.py index d653ea176..900ad1998 100644 --- a/freqtrade/rpc/rpc.py +++ b/freqtrade/rpc/rpc.py @@ -10,10 +10,10 @@ from typing import Dict, Any, List, Optional import arrow import sqlalchemy as sql -from numpy import mean, nan_to_num +from numpy import mean, nan_to_num, NAN from pandas import DataFrame -from freqtrade import TemporaryError +from freqtrade import TemporaryError, DependencyException from freqtrade.fiat_convert import CryptoToFiatConverter from freqtrade.misc import shorten_date from freqtrade.persistence import Trade @@ -93,7 +93,10 @@ class RPC(object): if trade.open_order_id: order = self._freqtrade.exchange.get_order(trade.open_order_id, trade.pair) # calculate profit and send message to user - current_rate = self._freqtrade.exchange.get_ticker(trade.pair, False)['bid'] + try: + current_rate = self._freqtrade.exchange.get_ticker(trade.pair, False)['bid'] + except DependencyException: + current_rate = NAN current_profit = trade.calc_profit_percent(current_rate) fmt_close_profit = (f'{round(trade.close_profit * 100, 2):.2f}%' if trade.close_profit else None) @@ -122,7 +125,10 @@ class RPC(object): trades_list = [] for trade in trades: # calculate profit and send message to user - current_rate = self._freqtrade.exchange.get_ticker(trade.pair, False)['bid'] + try: + current_rate = self._freqtrade.exchange.get_ticker(trade.pair, False)['bid'] + except DependencyException: + current_rate = NAN trade_perc = (100 * trade.calc_profit_percent(current_rate)) trades_list.append([ trade.id, @@ -207,7 +213,10 @@ class RPC(object): profit_closed_percent.append(profit_percent) else: # Get current rate - current_rate = self._freqtrade.exchange.get_ticker(trade.pair, False)['bid'] + try: + current_rate = self._freqtrade.exchange.get_ticker(trade.pair, False)['bid'] + except DependencyException: + current_rate = NAN profit_percent = trade.calc_profit_percent(rate=current_rate) profit_all_coin.append( @@ -275,7 +284,7 @@ class RPC(object): rate = 1.0 / self._freqtrade.exchange.get_ticker('BTC/USDT', False)['bid'] else: rate = self._freqtrade.exchange.get_ticker(coin + '/BTC', False)['bid'] - except TemporaryError: + except (TemporaryError, DependencyException): continue est_btc: float = rate * balance['total'] total = total + est_btc diff --git a/freqtrade/tests/exchange/test_exchange.py b/freqtrade/tests/exchange/test_exchange.py index 93cd1e546..788ef4518 100644 --- a/freqtrade/tests/exchange/test_exchange.py +++ b/freqtrade/tests/exchange/test_exchange.py @@ -572,6 +572,7 @@ def test_get_ticker(default_conf, mocker): 'last': 0.0001, } api_mock.fetch_ticker = MagicMock(return_value=tick) + api_mock.markets = {'ETH/BTC': {}} exchange = get_patched_exchange(mocker, default_conf, api_mock) # retrieve original ticker ticker = exchange.get_ticker(pair='ETH/BTC') @@ -614,6 +615,9 @@ def test_get_ticker(default_conf, mocker): exchange = get_patched_exchange(mocker, default_conf, api_mock) exchange.get_ticker(pair='ETH/BTC', refresh=True) + with pytest.raises(DependencyException, match=r'Pair XRP/ETH not available'): + exchange.get_ticker(pair='XRP/ETH', refresh=True) + def test_get_history(default_conf, mocker, caplog): exchange = get_patched_exchange(mocker, default_conf) From 792d2dbe3222e388f15088cfc6301fb0b0d408de Mon Sep 17 00:00:00 2001 From: Matthias Date: Wed, 10 Oct 2018 21:29:40 +0200 Subject: [PATCH 138/699] Hide "dust" from /balance --- freqtrade/rpc/telegram.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/freqtrade/rpc/telegram.py b/freqtrade/rpc/telegram.py index b6ad047b5..996137728 100644 --- a/freqtrade/rpc/telegram.py +++ b/freqtrade/rpc/telegram.py @@ -307,11 +307,14 @@ class Telegram(RPC): result = self._rpc_balance(self._config.get('fiat_display_currency', '')) output = '' for currency in result['currencies']: - output += "*{currency}:*\n" \ - "\t`Available: {available: .8f}`\n" \ - "\t`Balance: {balance: .8f}`\n" \ - "\t`Pending: {pending: .8f}`\n" \ - "\t`Est. BTC: {est_btc: .8f}`\n".format(**currency) + if currency['est_btc'] > 0.0001: + output += "*{currency}:*\n" \ + "\t`Available: {available: .8f}`\n" \ + "\t`Balance: {balance: .8f}`\n" \ + "\t`Pending: {pending: .8f}`\n" \ + "\t`Est. BTC: {est_btc: .8f}`\n".format(**currency) + else: + output += "*{currency}*: not showing <1$ amount \n".format(**currency) output += "\n*Estimated Value*:\n" \ "\t`BTC: {total: .8f}`\n" \ From 362865981060c44cdef6586ae28ce117f93a0031 Mon Sep 17 00:00:00 2001 From: Matthias Date: Wed, 10 Oct 2018 21:50:59 +0200 Subject: [PATCH 139/699] Add tests to check if no failure occurs when pair is not available --- freqtrade/tests/rpc/test_rpc.py | 46 ++++++++++++++++++++++++++++++++- 1 file changed, 45 insertions(+), 1 deletion(-) diff --git a/freqtrade/tests/rpc/test_rpc.py b/freqtrade/tests/rpc/test_rpc.py index 88bf5e9ad..14320cd54 100644 --- a/freqtrade/tests/rpc/test_rpc.py +++ b/freqtrade/tests/rpc/test_rpc.py @@ -5,8 +5,9 @@ from datetime import datetime from unittest.mock import MagicMock, ANY import pytest +from numpy import NAN, isnan -from freqtrade import TemporaryError +from freqtrade import TemporaryError, DependencyException from freqtrade.fiat_convert import CryptoToFiatConverter from freqtrade.freqtradebot import FreqtradeBot from freqtrade.persistence import Trade @@ -61,6 +62,27 @@ def test_rpc_trade_status(default_conf, ticker, fee, markets, mocker) -> None: 'open_order': '(limit buy rem=0.00000000)' } == results[0] + mocker.patch('freqtrade.exchange.Exchange.get_ticker', + MagicMock(side_effect=DependencyException(f"Pair 'ETH/BTC' not available"))) + # invalidate ticker cache + rpc._freqtrade.exchange._cached_ticker = {} + results = rpc._rpc_trade_status() + assert isnan(results[0]['current_profit']) + assert isnan(results[0]['current_rate']) + assert { + 'trade_id': 1, + 'pair': 'ETH/BTC', + 'market_url': 'https://bittrex.com/Market/Index?MarketName=BTC-ETH', + 'date': ANY, + 'open_rate': 1.099e-05, + 'close_rate': None, + 'current_rate': ANY, + 'amount': 90.99181074, + 'close_profit': None, + 'current_profit': ANY, + 'open_order': '(limit buy rem=0.00000000)' + } == results[0] + def test_rpc_status_table(default_conf, ticker, fee, markets, mocker) -> None: patch_coinmarketcap(mocker) @@ -87,6 +109,15 @@ def test_rpc_status_table(default_conf, ticker, fee, markets, mocker) -> None: assert 'ETH/BTC' in result['Pair'].all() assert '-0.59%' in result['Profit'].all() + mocker.patch('freqtrade.exchange.Exchange.get_ticker', + MagicMock(side_effect=DependencyException(f"Pair 'ETH/BTC' not available"))) + # invalidate ticker cache + rpc._freqtrade.exchange._cached_ticker = {} + result = rpc._rpc_status_table() + assert 'just now' in result['Since'].all() + assert 'ETH/BTC' in result['Pair'].all() + assert 'nan%' in result['Profit'].all() + def test_rpc_daily_profit(default_conf, update, ticker, fee, limit_buy_order, limit_sell_order, markets, mocker) -> None: @@ -208,6 +239,19 @@ def test_rpc_trade_statistics(default_conf, ticker, ticker_sell_up, fee, assert stats['best_pair'] == 'ETH/BTC' assert prec_satoshi(stats['best_rate'], 6.2) + # Test non-available pair + mocker.patch('freqtrade.exchange.Exchange.get_ticker', + MagicMock(side_effect=DependencyException(f"Pair 'ETH/BTC' not available"))) + # invalidate ticker cache + rpc._freqtrade.exchange._cached_ticker = {} + stats = rpc._rpc_trade_statistics(stake_currency, fiat_display_currency) + assert stats['trade_count'] == 2 + assert stats['first_trade_date'] == 'just now' + assert stats['latest_trade_date'] == 'just now' + assert stats['avg_duration'] == '0:00:00' + assert stats['best_pair'] == 'ETH/BTC' + assert prec_satoshi(stats['best_rate'], 6.2) + assert isnan(stats['profit_all_coin']) # Test that rpc_trade_statistics can handle trades that lacks # trade.open_rate (it is set to None) From 701978a4b1e8ee2238ded16d7a5276957cb05403 Mon Sep 17 00:00:00 2001 From: Matthias Date: Wed, 10 Oct 2018 22:01:22 +0200 Subject: [PATCH 140/699] Add test for dust hiding --- freqtrade/rpc/telegram.py | 2 +- freqtrade/tests/rpc/test_rpc_telegram.py | 17 ++++++++++++++--- 2 files changed, 15 insertions(+), 4 deletions(-) diff --git a/freqtrade/rpc/telegram.py b/freqtrade/rpc/telegram.py index 996137728..040f053f1 100644 --- a/freqtrade/rpc/telegram.py +++ b/freqtrade/rpc/telegram.py @@ -314,7 +314,7 @@ class Telegram(RPC): "\t`Pending: {pending: .8f}`\n" \ "\t`Est. BTC: {est_btc: .8f}`\n".format(**currency) else: - output += "*{currency}*: not showing <1$ amount \n".format(**currency) + output += "*{currency}:* not showing <1$ amount \n".format(**currency) output += "\n*Estimated Value*:\n" \ "\t`BTC: {total: .8f}`\n" \ diff --git a/freqtrade/tests/rpc/test_rpc_telegram.py b/freqtrade/tests/rpc/test_rpc_telegram.py index 182c1d2e7..892c851bd 100644 --- a/freqtrade/tests/rpc/test_rpc_telegram.py +++ b/freqtrade/tests/rpc/test_rpc_telegram.py @@ -507,7 +507,12 @@ def test_telegram_balance_handle(default_conf, update, mocker) -> None: 'total': 10.0, 'free': 10.0, 'used': 0.0 - } + }, + 'XRP': { + 'total': 1.0, + 'free': 1.0, + 'used': 0.0 + } } def mock_ticker(symbol, refresh): @@ -517,7 +522,12 @@ def test_telegram_balance_handle(default_conf, update, mocker) -> None: 'ask': 10000.00, 'last': 10000.00, } - + elif symbol == 'XRP/BTC': + return { + 'bid': 0.00001, + 'ask': 0.00001, + 'last': 0.00001, + } return { 'bid': 0.1, 'ask': 0.1, @@ -548,7 +558,8 @@ def test_telegram_balance_handle(default_conf, update, mocker) -> None: assert '*USDT:*' in result assert 'Balance:' in result assert 'Est. BTC:' in result - assert 'BTC: 14.00000000' in result + assert 'BTC: 12.00000000' in result + assert '*XRP:* not showing <1$ amount' in result def test_balance_handle_empty_response(default_conf, update, mocker) -> None: From 138c8152c20831f836f1a82c759ec86ce49cb27f Mon Sep 17 00:00:00 2001 From: Matthias Date: Wed, 10 Oct 2018 22:03:54 +0200 Subject: [PATCH 141/699] remove unused import --- freqtrade/tests/rpc/test_rpc.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/freqtrade/tests/rpc/test_rpc.py b/freqtrade/tests/rpc/test_rpc.py index 14320cd54..b181231c8 100644 --- a/freqtrade/tests/rpc/test_rpc.py +++ b/freqtrade/tests/rpc/test_rpc.py @@ -5,7 +5,7 @@ from datetime import datetime from unittest.mock import MagicMock, ANY import pytest -from numpy import NAN, isnan +from numpy import isnan from freqtrade import TemporaryError, DependencyException from freqtrade.fiat_convert import CryptoToFiatConverter @@ -253,6 +253,7 @@ def test_rpc_trade_statistics(default_conf, ticker, ticker_sell_up, fee, assert prec_satoshi(stats['best_rate'], 6.2) assert isnan(stats['profit_all_coin']) + # Test that rpc_trade_statistics can handle trades that lacks # trade.open_rate (it is set to None) def test_rpc_trade_statistics_closed(mocker, default_conf, ticker, fee, markets, From 912e9bd15ceb584ada3e70ac55883da760928cc6 Mon Sep 17 00:00:00 2001 From: misagh Date: Thu, 11 Oct 2018 19:12:12 +0200 Subject: [PATCH 142/699] mac installation path --- docs/installation.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/installation.md b/docs/installation.md index 1000600e6..482209933 100644 --- a/docs/installation.md +++ b/docs/installation.md @@ -268,7 +268,7 @@ wget http://prdownloads.sourceforge.net/ta-lib/ta-lib-0.4.0-src.tar.gz tar xvzf ta-lib-0.4.0-src.tar.gz cd ta-lib sed -i.bak "s|0.00000001|0.000000000000000001 |g" src/ta_func/ta_utility.h -./configure --prefix=/usr +./configure --prefix=/usr # On MacOS this should be --prefix=/usr/local make make install cd .. From fb3fd7cb152f399e8aa1ae96315658ab953407b6 Mon Sep 17 00:00:00 2001 From: misagh Date: Thu, 11 Oct 2018 19:26:19 +0200 Subject: [PATCH 143/699] setup script and documentation fixed for TA-Lib and MacOS --- docs/installation.md | 6 +++--- setup.sh | 10 +++++++--- 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/docs/installation.md b/docs/installation.md index 482209933..d2002035e 100644 --- a/docs/installation.md +++ b/docs/installation.md @@ -251,7 +251,7 @@ python3 -m pip install -e . ### MacOS -#### Install Python 3.6, git, wget and ta-lib +#### Install Python 3.6, git and wget ```bash brew install python3 git wget @@ -268,9 +268,9 @@ wget http://prdownloads.sourceforge.net/ta-lib/ta-lib-0.4.0-src.tar.gz tar xvzf ta-lib-0.4.0-src.tar.gz cd ta-lib sed -i.bak "s|0.00000001|0.000000000000000001 |g" src/ta_func/ta_utility.h -./configure --prefix=/usr # On MacOS this should be --prefix=/usr/local +./configure --prefix=/usr/local make -make install +sudo make install cd .. rm -rf ./ta-lib* ``` diff --git a/setup.sh b/setup.sh index bd58edbee..472260148 100755 --- a/setup.sh +++ b/setup.sh @@ -37,7 +37,11 @@ function updateenv () { function install_talib () { curl -O -L http://prdownloads.sourceforge.net/ta-lib/ta-lib-0.4.0-src.tar.gz tar zxvf ta-lib-0.4.0-src.tar.gz - cd ta-lib && ./configure --prefix=/usr && make && sudo make install + cd ta-lib + sed -i.bak "s|0.00000001|0.000000000000000001 |g" src/ta_func/ta_utility.h + ./configure --prefix=/usr/local + make + sudo make install cd .. && rm -rf ./ta-lib* } @@ -50,8 +54,8 @@ function install_macos () { echo "-------------------------" /usr/bin/ruby -e "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/master/install)" fi - brew install python3 wget ta-lib - + brew install python3 wget + install_talib test_and_fix_python_on_mac } From 6aa9cd106088e89e451690a97fc2e63ab60dcc5e Mon Sep 17 00:00:00 2001 From: misagh Date: Fri, 12 Oct 2018 19:37:23 +0200 Subject: [PATCH 144/699] removing outliers per pair and not across all pairs --- freqtrade/edge/__init__.py | 13 +++---------- 1 file changed, 3 insertions(+), 10 deletions(-) diff --git a/freqtrade/edge/__init__.py b/freqtrade/edge/__init__.py index 490bc065e..e2af560c9 100644 --- a/freqtrade/edge/__init__.py +++ b/freqtrade/edge/__init__.py @@ -15,7 +15,6 @@ from freqtrade.strategy.interface import SellType from freqtrade.strategy.resolver import IStrategy, StrategyResolver from freqtrade.optimize.backtesting import Backtesting - logger = logging.getLogger(__name__) @@ -221,25 +220,19 @@ class Edge(): and keep it in a storage. The calulation will be done per pair and per strategy. """ - # Removing pairs having less than min_trades_number min_trades_number = self.edge_config.get('min_trade_number', 15) - results = results.groupby('pair').filter(lambda x: len(x) > min_trades_number) + results = results.groupby(['pair', 'stoploss']).filter(lambda x: len(x) > min_trades_number) ################################### # Removing outliers (Only Pumps) from the dataset # The method to detect outliers is to calculate standard deviation # Then every value more than (standard deviation + 2*average) is out (pump) # - # Calculating standard deviation of profits - std = results[["profit_abs"]].std() - # - # Calculating average of profits - avg = results[["profit_abs"]].mean() - # # Removing Pumps if self.edge_config.get('remove_pumps', True): - results = results[results.profit_abs <= float(avg + 2 * std)] + results = results.groupby(['pair', 'stoploss']).apply( + lambda x: x[x['profit_abs'] < 2 * x['profit_abs'].std() + x['profit_abs'].mean()]) ########################################################################## # Removing trades having a duration more than X minutes (set in config) From 93503d60517a8bf74ea1e9483051cc2dc22c2167 Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Sat, 13 Oct 2018 14:33:06 +0200 Subject: [PATCH 145/699] Update ccxt from 1.17.375 to 1.17.376 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index a5f22fec2..0ce7f65cb 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -ccxt==1.17.375 +ccxt==1.17.376 SQLAlchemy==1.2.12 python-telegram-bot==11.1.0 arrow==0.12.1 From 4b9d04a2ca33c15c7f8dd492c48e8df775234d06 Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Sun, 14 Oct 2018 14:33:06 +0200 Subject: [PATCH 146/699] Update ccxt from 1.17.376 to 1.17.381 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 0ce7f65cb..81c508014 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -ccxt==1.17.376 +ccxt==1.17.381 SQLAlchemy==1.2.12 python-telegram-bot==11.1.0 arrow==0.12.1 From 631ba464f3512fd628aa364b3159f6ae2fffadee Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 14 Oct 2018 14:40:03 +0200 Subject: [PATCH 147/699] Show warning if part of backtest data is missing --- freqtrade/optimize/__init__.py | 8 ++++++ freqtrade/tests/optimize/test_optimize.py | 32 +++++++++++++++++++++++ 2 files changed, 40 insertions(+) diff --git a/freqtrade/optimize/__init__.py b/freqtrade/optimize/__init__.py index 74c842427..9178139c9 100644 --- a/freqtrade/optimize/__init__.py +++ b/freqtrade/optimize/__init__.py @@ -113,6 +113,14 @@ def load_data(datadir: str, for pair in pairs: pairdata = load_tickerdata_file(datadir, pair, ticker_interval, timerange=timerange) if pairdata: + if timerange.starttype == 'date' and pairdata[0][0] > timerange.startts * 1000: + logger.warning('Missing data at start for pair %s, data starts at %s', + pair, + arrow.get(pairdata[0][0] // 1000).strftime('%Y-%m-%d %H:%M:%S')) + if timerange.stoptype == 'date' and pairdata[-1][0] < timerange.stopts * 1000: + logger.warning('Missing data at end for pair %s, data ends at %s', + pair, + arrow.get(pairdata[-1][0] // 1000).strftime('%Y-%m-%d %H:%M:%S')) result[pair] = pairdata else: logger.warning( diff --git a/freqtrade/tests/optimize/test_optimize.py b/freqtrade/tests/optimize/test_optimize.py index 77fa3e3b1..1e5e8a94e 100644 --- a/freqtrade/tests/optimize/test_optimize.py +++ b/freqtrade/tests/optimize/test_optimize.py @@ -322,6 +322,38 @@ def test_load_tickerdata_file() -> None: assert _BTC_UNITTEST_LENGTH == len(tickerdata) +def test_load_partial_missing(caplog) -> None: + # Make sure we start fresh - test missing data at start + start = arrow.get('2018-01-01T00:00:00') + end = arrow.get('2018-01-11T00:00:00') + tickerdata = optimize.load_data(None, '5m', ['UNITTEST/BTC'], + refresh_pairs=False, + timerange=TimeRange('date', 'date', + start.timestamp, end.timestamp)) + # timedifference in 5 minutes + td = ((end - start).total_seconds() // 60 // 5) + 1 + assert td != len(tickerdata['UNITTEST/BTC']) + start_real = arrow.get(tickerdata['UNITTEST/BTC'][0][0] / 1000) + assert log_has(f'Missing data at start for pair ' + f'UNITTEST/BTC, data starts at {start_real.strftime("%Y-%m-%d %H:%M:%S")}', + caplog.record_tuples) + # Make sure we start fresh - test missing data at end + caplog.clear() + start = arrow.get('2018-01-10T00:00:00') + end = arrow.get('2018-02-20T00:00:00') + tickerdata = optimize.load_data(None, '5m', ['UNITTEST/BTC'], + refresh_pairs=False, + timerange=TimeRange('date', 'date', + start.timestamp, end.timestamp)) + # timedifference in 5 minutes + td = ((end - start).total_seconds() // 60 // 5) + 1 + assert td != len(tickerdata['UNITTEST/BTC']) + end_real = arrow.get(tickerdata['UNITTEST/BTC'][-1][0] / 1000) + assert log_has(f'Missing data at end for pair ' + f'UNITTEST/BTC, data ends at {end_real.strftime("%Y-%m-%d %H:%M:%S")}', + caplog.record_tuples) + + def test_init(default_conf, mocker) -> None: exchange = get_patched_exchange(mocker, default_conf) assert {} == optimize.load_data( From b278dcd6db22cd749341603945c7686f04a4f0e7 Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Mon, 15 Oct 2018 14:33:06 +0200 Subject: [PATCH 148/699] Update ccxt from 1.17.381 to 1.17.383 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 81c508014..4b2d474ca 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -ccxt==1.17.381 +ccxt==1.17.383 SQLAlchemy==1.2.12 python-telegram-bot==11.1.0 arrow==0.12.1 From b546f0302e549db70d129db0a3caa4877a83f678 Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Tue, 16 Oct 2018 14:33:06 +0200 Subject: [PATCH 149/699] Update ccxt from 1.17.383 to 1.17.388 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 4b2d474ca..918a56891 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -ccxt==1.17.383 +ccxt==1.17.388 SQLAlchemy==1.2.12 python-telegram-bot==11.1.0 arrow==0.12.1 From 5134736c61a08bc0d5b5048ee154e543acb14c0a Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Wed, 17 Oct 2018 14:34:07 +0200 Subject: [PATCH 150/699] Update ccxt from 1.17.388 to 1.17.392 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 918a56891..202d41abe 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -ccxt==1.17.388 +ccxt==1.17.392 SQLAlchemy==1.2.12 python-telegram-bot==11.1.0 arrow==0.12.1 From 14e58169750a510d668a5e44d20d3272258a1e3c Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Wed, 17 Oct 2018 14:34:09 +0200 Subject: [PATCH 151/699] Update urllib3 from 1.23 to 1.24 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 202d41abe..b47c02865 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,7 +4,7 @@ python-telegram-bot==11.1.0 arrow==0.12.1 cachetools==2.1.0 requests==2.19.1 -urllib3==1.23 +urllib3==1.24 wrapt==1.10.11 pandas==0.23.4 scikit-learn==0.20.0 From d953190ca5e73fd249f18ecb65e027d305bcff71 Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Wed, 17 Oct 2018 14:34:10 +0200 Subject: [PATCH 152/699] Update pytest from 3.8.2 to 3.9.1 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index b47c02865..86a654ffb 100644 --- a/requirements.txt +++ b/requirements.txt @@ -12,7 +12,7 @@ scipy==1.1.0 jsonschema==2.6.0 numpy==1.15.2 TA-Lib==0.4.17 -pytest==3.8.2 +pytest==3.9.1 pytest-mock==1.10.0 pytest-asyncio==0.9.0 pytest-cov==2.6.0 From 8a3272e7c54ec31572d039124a8fe052ec83a3d9 Mon Sep 17 00:00:00 2001 From: Matthias Date: Wed, 17 Oct 2018 19:47:19 +0200 Subject: [PATCH 153/699] don't copy tickerdata_to_dataframe into backtesting it's used only once, so this does not make sense and hides the origin of the function --- freqtrade/optimize/backtesting.py | 3 +-- freqtrade/optimize/hyperopt.py | 2 +- freqtrade/tests/optimize/test_backtesting.py | 16 ++++++++-------- freqtrade/tests/optimize/test_hyperopt.py | 6 +++--- 4 files changed, 13 insertions(+), 14 deletions(-) diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index cd822023f..df2f6834a 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -88,7 +88,6 @@ class Backtesting(object): """ self.strategy = strategy self.ticker_interval = self.config.get('ticker_interval') - self.tickerdata_to_dataframe = strategy.tickerdata_to_dataframe self.advise_buy = strategy.advise_buy self.advise_sell = strategy.advise_sell @@ -371,7 +370,7 @@ class Backtesting(object): self._set_strategy(strat) # need to reprocess data every time to populate signals - preprocessed = self.tickerdata_to_dataframe(data) + preprocessed = self.strategy.tickerdata_to_dataframe(data) # Print timeframe min_date, max_date = self.get_timeframe(preprocessed) diff --git a/freqtrade/optimize/hyperopt.py b/freqtrade/optimize/hyperopt.py index 4a239ab28..b2d05d603 100644 --- a/freqtrade/optimize/hyperopt.py +++ b/freqtrade/optimize/hyperopt.py @@ -352,7 +352,7 @@ class Hyperopt(Backtesting): if self.has_space('buy'): self.strategy.advise_indicators = Hyperopt.populate_indicators # type: ignore - dump(self.tickerdata_to_dataframe(data), TICKERDATA_PICKLE) + dump(self.strategy.tickerdata_to_dataframe(data), TICKERDATA_PICKLE) self.exchange = None # type: ignore self.load_previous_results() diff --git a/freqtrade/tests/optimize/test_backtesting.py b/freqtrade/tests/optimize/test_backtesting.py index a17867b3a..36b2fcdd3 100644 --- a/freqtrade/tests/optimize/test_backtesting.py +++ b/freqtrade/tests/optimize/test_backtesting.py @@ -89,7 +89,7 @@ def simple_backtest(config, contour, num_results, mocker) -> None: backtesting = Backtesting(config) data = load_data_test(contour) - processed = backtesting.tickerdata_to_dataframe(data) + processed = backtesting.strategy.tickerdata_to_dataframe(data) assert isinstance(processed, dict) results = backtesting.backtest( { @@ -125,7 +125,7 @@ def _make_backtest_conf(mocker, conf=None, pair='UNITTEST/BTC', record=None): backtesting = Backtesting(conf) return { 'stake_amount': conf['stake_amount'], - 'processed': backtesting.tickerdata_to_dataframe(data), + 'processed': backtesting.strategy.tickerdata_to_dataframe(data), 'max_open_trades': 10, 'position_stacking': False, 'record': record @@ -313,7 +313,7 @@ def test_backtesting_init(mocker, default_conf) -> None: backtesting = Backtesting(default_conf) assert backtesting.config == default_conf assert backtesting.ticker_interval == '5m' - assert callable(backtesting.tickerdata_to_dataframe) + assert callable(backtesting.strategy.tickerdata_to_dataframe) assert callable(backtesting.advise_buy) assert callable(backtesting.advise_sell) get_fee.assert_called() @@ -327,7 +327,7 @@ def test_tickerdata_to_dataframe(default_conf, mocker) -> None: tickerlist = {'UNITTEST/BTC': tick} backtesting = Backtesting(default_conf) - data = backtesting.tickerdata_to_dataframe(tickerlist) + data = backtesting.strategy.tickerdata_to_dataframe(tickerlist) assert len(data['UNITTEST/BTC']) == 99 # Load strategy to compare the result between Backtesting function and strategy are the same @@ -340,7 +340,7 @@ def test_get_timeframe(default_conf, mocker) -> None: patch_exchange(mocker) backtesting = Backtesting(default_conf) - data = backtesting.tickerdata_to_dataframe( + data = backtesting.strategy.tickerdata_to_dataframe( optimize.load_data( None, ticker_interval='1m', @@ -520,7 +520,7 @@ def test_backtest(default_conf, fee, mocker) -> None: pair = 'UNITTEST/BTC' data = optimize.load_data(None, ticker_interval='5m', pairs=['UNITTEST/BTC']) data = trim_dictlist(data, -200) - data_processed = backtesting.tickerdata_to_dataframe(data) + data_processed = backtesting.strategy.tickerdata_to_dataframe(data) results = backtesting.backtest( { 'stake_amount': default_conf['stake_amount'], @@ -571,7 +571,7 @@ def test_backtest_1min_ticker_interval(default_conf, fee, mocker) -> None: results = backtesting.backtest( { 'stake_amount': default_conf['stake_amount'], - 'processed': backtesting.tickerdata_to_dataframe(data), + 'processed': backtesting.strategy.tickerdata_to_dataframe(data), 'max_open_trades': 1, 'position_stacking': False } @@ -585,7 +585,7 @@ def test_processed(default_conf, mocker) -> None: backtesting = Backtesting(default_conf) dict_of_tickerrows = load_data_test('raise') - dataframes = backtesting.tickerdata_to_dataframe(dict_of_tickerrows) + dataframes = backtesting.strategy.tickerdata_to_dataframe(dict_of_tickerrows) dataframe = dataframes['UNITTEST/BTC'] cols = dataframe.columns # assert the dataframe got some of the indicator columns diff --git a/freqtrade/tests/optimize/test_hyperopt.py b/freqtrade/tests/optimize/test_hyperopt.py index 2035e23df..c93f2d316 100644 --- a/freqtrade/tests/optimize/test_hyperopt.py +++ b/freqtrade/tests/optimize/test_hyperopt.py @@ -194,7 +194,7 @@ def test_start_calls_optimizer(mocker, default_conf, caplog) -> None: default_conf.update({'spaces': 'all'}) hyperopt = Hyperopt(default_conf) - hyperopt.tickerdata_to_dataframe = MagicMock() + hyperopt.strategy.tickerdata_to_dataframe = MagicMock() hyperopt.start() parallel.assert_called_once() @@ -242,7 +242,7 @@ def test_has_space(hyperopt): def test_populate_indicators(hyperopt) -> None: tick = load_tickerdata_file(None, 'UNITTEST/BTC', '1m') tickerlist = {'UNITTEST/BTC': tick} - dataframes = hyperopt.tickerdata_to_dataframe(tickerlist) + dataframes = hyperopt.strategy.tickerdata_to_dataframe(tickerlist) dataframe = hyperopt.populate_indicators(dataframes['UNITTEST/BTC'], {'pair': 'UNITTEST/BTC'}) # Check if some indicators are generated. We will not test all of them @@ -254,7 +254,7 @@ def test_populate_indicators(hyperopt) -> None: def test_buy_strategy_generator(hyperopt) -> None: tick = load_tickerdata_file(None, 'UNITTEST/BTC', '1m') tickerlist = {'UNITTEST/BTC': tick} - dataframes = hyperopt.tickerdata_to_dataframe(tickerlist) + dataframes = hyperopt.strategy.tickerdata_to_dataframe(tickerlist) dataframe = hyperopt.populate_indicators(dataframes['UNITTEST/BTC'], {'pair': 'UNITTEST/BTC'}) populate_buy_trend = hyperopt.buy_strategy_generator( From d7459bbbf3c60695c22201b2273c3a77b4385dea Mon Sep 17 00:00:00 2001 From: Matthias Date: Wed, 17 Oct 2018 19:59:33 +0200 Subject: [PATCH 154/699] refactor get_timeframe out of backtesting class --- freqtrade/optimize/__init__.py | 17 ++++++++++++++ freqtrade/optimize/backtesting.py | 20 ++-------------- freqtrade/tests/optimize/test_backtesting.py | 24 ++++---------------- freqtrade/tests/optimize/test_optimize.py | 19 +++++++++++++++- 4 files changed, 41 insertions(+), 39 deletions(-) diff --git a/freqtrade/optimize/__init__.py b/freqtrade/optimize/__init__.py index 74c842427..5367f7663 100644 --- a/freqtrade/optimize/__init__.py +++ b/freqtrade/optimize/__init__.py @@ -11,7 +11,10 @@ except ImportError: import logging import os from typing import Optional, List, Dict, Tuple, Any +import operator + import arrow +from pandas import DataFrame from freqtrade import misc, constants, OperationalException from freqtrade.exchange import Exchange @@ -59,6 +62,20 @@ def trim_tickerlist(tickerlist: List[Dict], timerange: TimeRange) -> List[Dict]: return tickerlist[start_index:stop_index] +def get_timeframe(data: Dict[str, DataFrame]) -> Tuple[arrow.Arrow, arrow.Arrow]: + """ + Get the maximum timeframe for the given backtest data + :param data: dictionary with preprocessed backtesting data + :return: tuple containing min_date, max_date + """ + timeframe = [ + (arrow.get(frame['date'].min()), arrow.get(frame['date'].max())) + for frame in data.values() + ] + return min(timeframe, key=operator.itemgetter(0))[0], \ + max(timeframe, key=operator.itemgetter(1))[1] + + def load_tickerdata_file( datadir: str, pair: str, ticker_interval: str, diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index df2f6834a..695a52052 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -4,14 +4,12 @@ This module contains the backtesting logic """ import logging -import operator from argparse import Namespace from copy import deepcopy from datetime import datetime, timedelta from pathlib import Path -from typing import Any, Dict, List, NamedTuple, Optional, Tuple +from typing import Any, Dict, List, NamedTuple, Optional -import arrow from pandas import DataFrame from tabulate import tabulate @@ -91,20 +89,6 @@ class Backtesting(object): self.advise_buy = strategy.advise_buy self.advise_sell = strategy.advise_sell - @staticmethod - def get_timeframe(data: Dict[str, DataFrame]) -> Tuple[arrow.Arrow, arrow.Arrow]: - """ - Get the maximum timeframe for the given backtest data - :param data: dictionary with preprocessed backtesting data - :return: tuple containing min_date, max_date - """ - timeframe = [ - (arrow.get(frame['date'].min()), arrow.get(frame['date'].max())) - for frame in data.values() - ] - return min(timeframe, key=operator.itemgetter(0))[0], \ - max(timeframe, key=operator.itemgetter(1))[1] - def _generate_text_table(self, data: Dict[str, Dict], results: DataFrame, skip_nan: bool = False) -> str: """ @@ -373,7 +357,7 @@ class Backtesting(object): preprocessed = self.strategy.tickerdata_to_dataframe(data) # Print timeframe - min_date, max_date = self.get_timeframe(preprocessed) + min_date, max_date = optimize.get_timeframe(preprocessed) logger.info( 'Measuring data from %s up to %s (%s days)..', min_date.isoformat(), diff --git a/freqtrade/tests/optimize/test_backtesting.py b/freqtrade/tests/optimize/test_backtesting.py index 36b2fcdd3..ff6a0666d 100644 --- a/freqtrade/tests/optimize/test_backtesting.py +++ b/freqtrade/tests/optimize/test_backtesting.py @@ -336,22 +336,6 @@ def test_tickerdata_to_dataframe(default_conf, mocker) -> None: assert data['UNITTEST/BTC'].equals(data2['UNITTEST/BTC']) -def test_get_timeframe(default_conf, mocker) -> None: - patch_exchange(mocker) - backtesting = Backtesting(default_conf) - - data = backtesting.strategy.tickerdata_to_dataframe( - optimize.load_data( - None, - ticker_interval='1m', - pairs=['UNITTEST/BTC'] - ) - ) - min_date, max_date = backtesting.get_timeframe(data) - assert min_date.isoformat() == '2017-11-04T23:02:00+00:00' - assert max_date.isoformat() == '2017-11-14T22:58:00+00:00' - - def test_generate_text_table(default_conf, mocker): patch_exchange(mocker) backtesting = Backtesting(default_conf) @@ -451,17 +435,17 @@ def test_generate_text_table_strategyn(default_conf, mocker): def test_backtesting_start(default_conf, mocker, caplog) -> None: - def get_timeframe(input1, input2): + def get_timeframe(input1): return Arrow(2017, 11, 14, 21, 17), Arrow(2017, 11, 14, 22, 59) mocker.patch('freqtrade.optimize.load_data', mocked_load_data) + mocker.patch('freqtrade.optimize.get_timeframe', get_timeframe) mocker.patch('freqtrade.exchange.Exchange.refresh_tickers', MagicMock()) patch_exchange(mocker) mocker.patch.multiple( 'freqtrade.optimize.backtesting.Backtesting', backtest=MagicMock(), _generate_text_table=MagicMock(return_value='1'), - get_timeframe=get_timeframe, ) default_conf['exchange']['pair_whitelist'] = ['UNITTEST/BTC'] @@ -486,17 +470,17 @@ def test_backtesting_start(default_conf, mocker, caplog) -> None: def test_backtesting_start_no_data(default_conf, mocker, caplog) -> None: - def get_timeframe(input1, input2): + def get_timeframe(input1): return Arrow(2017, 11, 14, 21, 17), Arrow(2017, 11, 14, 22, 59) mocker.patch('freqtrade.optimize.load_data', MagicMock(return_value={})) + mocker.patch('freqtrade.optimize.get_timeframe', get_timeframe) mocker.patch('freqtrade.exchange.Exchange.refresh_tickers', MagicMock()) patch_exchange(mocker) mocker.patch.multiple( 'freqtrade.optimize.backtesting.Backtesting', backtest=MagicMock(), _generate_text_table=MagicMock(return_value='1'), - get_timeframe=get_timeframe, ) default_conf['exchange']['pair_whitelist'] = ['UNITTEST/BTC'] diff --git a/freqtrade/tests/optimize/test_optimize.py b/freqtrade/tests/optimize/test_optimize.py index 77fa3e3b1..061caf70b 100644 --- a/freqtrade/tests/optimize/test_optimize.py +++ b/freqtrade/tests/optimize/test_optimize.py @@ -15,7 +15,8 @@ from freqtrade.optimize.__init__ import (download_backtesting_testdata, load_cached_data_for_updating, load_tickerdata_file, make_testdata_path, trim_tickerlist) -from freqtrade.tests.conftest import get_patched_exchange, log_has +from freqtrade.strategy.default_strategy import DefaultStrategy +from freqtrade.tests.conftest import get_patched_exchange, log_has, patch_exchange # Change this if modifying UNITTEST/BTC testdatafile _BTC_UNITTEST_LENGTH = 13681 @@ -433,3 +434,19 @@ def test_file_dump_json() -> None: # Remove the file _clean_test_file(file) + + +def test_get_timeframe(default_conf, mocker) -> None: + patch_exchange(mocker) + strategy = DefaultStrategy(default_conf) + + data = strategy.tickerdata_to_dataframe( + optimize.load_data( + None, + ticker_interval='1m', + pairs=['UNITTEST/BTC'] + ) + ) + min_date, max_date = optimize.get_timeframe(data) + assert min_date.isoformat() == '2017-11-04T23:02:00+00:00' + assert max_date.isoformat() == '2017-11-14T22:58:00+00:00' From 57bc4a866ac835dee52e6cccdba314d40b510b27 Mon Sep 17 00:00:00 2001 From: misagh Date: Thu, 18 Oct 2018 11:09:10 +0200 Subject: [PATCH 155/699] average trade duration added --- freqtrade/edge/__init__.py | 20 ++++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/freqtrade/edge/__init__.py b/freqtrade/edge/__init__.py index e2af560c9..4e3c2c7c8 100644 --- a/freqtrade/edge/__init__.py +++ b/freqtrade/edge/__init__.py @@ -14,6 +14,7 @@ from freqtrade.arguments import TimeRange from freqtrade.strategy.interface import SellType from freqtrade.strategy.resolver import IStrategy, StrategyResolver from freqtrade.optimize.backtesting import Backtesting +import sys logger = logging.getLogger(__name__) @@ -31,6 +32,7 @@ class Edge(): _timerange: TimeRange def __init__(self, config: Dict[str, Any], exchange=None) -> None: + sys.setrecursionlimit(10000) self.config = config self.exchange = exchange self.strategy: IStrategy = StrategyResolver(self.config).strategy @@ -68,7 +70,7 @@ class Edge(): self.config['datadir'], pairs=pairs, ticker_interval=self.ticker_interval, - refresh_pairs=True, + refresh_pairs=False, exchange=self.exchange, timerange=self._timerange ) @@ -272,13 +274,19 @@ class Edge(): return x ############################## - final = results.groupby(['pair', 'stoploss'])['profit_abs'].\ - agg([winrate, risk_reward_ratio, required_risk_reward, expectancy]).\ - reset_index().sort_values(by=['expectancy', 'stoploss'], ascending=False)\ - .groupby('pair').first().sort_values(by=['expectancy'], ascending=False) + if results.empty: + return [] + + groupby_aggregator = {'profit_abs': [winrate, risk_reward_ratio, required_risk_reward, expectancy, 'count'], 'trade_duration': ['mean']} + final = results.groupby(['pair', 'stoploss'])['profit_abs','trade_duration'].agg(groupby_aggregator).reset_index(col_level=1) + final.columns = final.columns.droplevel(0) + final = final.sort_values(by=['expectancy', 'stoploss'], ascending=False).groupby( + 'pair').first().sort_values(by=['expectancy'], ascending=False).reset_index() + + final.rename(columns={'mean': 'avg_duration(min)'}, inplace=True) # Returning an array of pairs in order of "expectancy" - return final.reset_index().values + return final.values def _find_trades_for_stoploss_range(self, ticker_data, pair, stoploss_range): buy_column = ticker_data['buy'].values From fb52d322966382e09c39367c2a10ffbd8efc0aaa Mon Sep 17 00:00:00 2001 From: Matthias Date: Thu, 18 Oct 2018 19:42:54 +0200 Subject: [PATCH 156/699] Add validate_backtest_data function --- freqtrade/optimize/__init__.py | 19 ++++++++++ freqtrade/optimize/backtesting.py | 4 ++- freqtrade/tests/optimize/test_optimize.py | 44 ++++++++++++++++++++++- 3 files changed, 65 insertions(+), 2 deletions(-) diff --git a/freqtrade/optimize/__init__.py b/freqtrade/optimize/__init__.py index 5367f7663..d4cb6c067 100644 --- a/freqtrade/optimize/__init__.py +++ b/freqtrade/optimize/__init__.py @@ -10,6 +10,7 @@ except ImportError: _UJSON = False import logging import os +from datetime import datetime from typing import Optional, List, Dict, Tuple, Any import operator @@ -76,6 +77,24 @@ def get_timeframe(data: Dict[str, DataFrame]) -> Tuple[arrow.Arrow, arrow.Arrow] max(timeframe, key=operator.itemgetter(1))[1] +def validate_backtest_data(data: Dict[str, DataFrame], min_date: datetime, + max_date: datetime, ticker_interval_mins: int) -> None: + """ + Validates preprocessed backtesting data for missing values and shows warnings about it that. + + :param data: dictionary with preprocessed backtesting data + :param min_date: start-date of the data + :param max_date: end-date of the data + :param ticker_interval_mins: ticker interval in minutes + """ + # total difference in minutes / interval-minutes + expected_frames = int((max_date - min_date).total_seconds() // 60 // ticker_interval_mins) + for pair, df in data.items(): + if len(df) < expected_frames: + logger.warning('%s has missing frames: expected %s, got %s', + pair, expected_frames, len(df)) + + def load_tickerdata_file( datadir: str, pair: str, ticker_interval: str, diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index 695a52052..961cfb092 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -356,8 +356,10 @@ class Backtesting(object): # need to reprocess data every time to populate signals preprocessed = self.strategy.tickerdata_to_dataframe(data) - # Print timeframe min_date, max_date = optimize.get_timeframe(preprocessed) + # Validate dataframe for missing values + optimize.validate_backtest_data(preprocessed, min_date, max_date, + constants.TICKER_INTERVAL_MINUTES[self.ticker_interval]) logger.info( 'Measuring data from %s up to %s (%s days)..', min_date.isoformat(), diff --git a/freqtrade/tests/optimize/test_optimize.py b/freqtrade/tests/optimize/test_optimize.py index 061caf70b..7b13b498a 100644 --- a/freqtrade/tests/optimize/test_optimize.py +++ b/freqtrade/tests/optimize/test_optimize.py @@ -7,7 +7,7 @@ from shutil import copyfile import arrow -from freqtrade import optimize +from freqtrade import optimize, constants from freqtrade.arguments import TimeRange from freqtrade.misc import file_dump_json from freqtrade.optimize.__init__ import (download_backtesting_testdata, @@ -450,3 +450,45 @@ def test_get_timeframe(default_conf, mocker) -> None: min_date, max_date = optimize.get_timeframe(data) assert min_date.isoformat() == '2017-11-04T23:02:00+00:00' assert max_date.isoformat() == '2017-11-14T22:58:00+00:00' + + +def test_validate_backtest_data_warn(default_conf, mocker, caplog) -> None: + patch_exchange(mocker) + strategy = DefaultStrategy(default_conf) + + data = strategy.tickerdata_to_dataframe( + optimize.load_data( + None, + ticker_interval='1m', + pairs=['UNITTEST/BTC'] + ) + ) + min_date, max_date = optimize.get_timeframe(data) + caplog.clear() + optimize.validate_backtest_data(data, min_date, max_date, + constants.TICKER_INTERVAL_MINUTES["1m"]) + assert len(caplog.record_tuples) == 1 + assert log_has('UNITTEST/BTC has missing frames: expected 14396, got 13680', + caplog.record_tuples) + + +def test_validate_backtest_data(default_conf, mocker, caplog) -> None: + patch_exchange(mocker) + strategy = DefaultStrategy(default_conf) + + timerange = TimeRange('index', 'index', 200, 250) + data = strategy.tickerdata_to_dataframe( + optimize.load_data( + None, + ticker_interval='5m', + pairs=['UNITTEST/BTC'], + timerange=timerange + ) + ) + + min_date, max_date = optimize.get_timeframe(data) + caplog.clear() + optimize.validate_backtest_data(data, min_date, max_date, + constants.TICKER_INTERVAL_MINUTES["5m"]) + assert len(caplog.record_tuples) == 0 + From 518dcf5209034e4c7fdf14f99257d406363fdcdb Mon Sep 17 00:00:00 2001 From: Matthias Date: Thu, 18 Oct 2018 19:43:04 +0200 Subject: [PATCH 157/699] Cleanup some tests 8m is not a valid ticker value not in constants.TICKER_INTERVAL_MINUTES map --- freqtrade/tests/optimize/test_backtesting.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/freqtrade/tests/optimize/test_backtesting.py b/freqtrade/tests/optimize/test_backtesting.py index ff6a0666d..83fe52ed3 100644 --- a/freqtrade/tests/optimize/test_backtesting.py +++ b/freqtrade/tests/optimize/test_backtesting.py @@ -119,7 +119,7 @@ def _load_pair_as_ticks(pair, tickfreq): # FIX: fixturize this? def _make_backtest_conf(mocker, conf=None, pair='UNITTEST/BTC', record=None): - data = optimize.load_data(None, ticker_interval='8m', pairs=[pair]) + data = optimize.load_data(None, ticker_interval='1m', pairs=[pair]) data = trim_dictlist(data, -201) patch_exchange(mocker) backtesting = Backtesting(conf) @@ -449,7 +449,7 @@ def test_backtesting_start(default_conf, mocker, caplog) -> None: ) default_conf['exchange']['pair_whitelist'] = ['UNITTEST/BTC'] - default_conf['ticker_interval'] = 1 + default_conf['ticker_interval'] = "1m" default_conf['live'] = False default_conf['datadir'] = None default_conf['export'] = None From bc356c4d6511034f691793f54f26c59fdf2a2dc9 Mon Sep 17 00:00:00 2001 From: Matthias Date: Thu, 18 Oct 2018 19:48:54 +0200 Subject: [PATCH 158/699] Return true/false for validation function --- freqtrade/optimize/__init__.py | 5 ++++- freqtrade/tests/optimize/test_optimize.py | 8 ++++---- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/freqtrade/optimize/__init__.py b/freqtrade/optimize/__init__.py index d4cb6c067..3376d0075 100644 --- a/freqtrade/optimize/__init__.py +++ b/freqtrade/optimize/__init__.py @@ -78,7 +78,7 @@ def get_timeframe(data: Dict[str, DataFrame]) -> Tuple[arrow.Arrow, arrow.Arrow] def validate_backtest_data(data: Dict[str, DataFrame], min_date: datetime, - max_date: datetime, ticker_interval_mins: int) -> None: + max_date: datetime, ticker_interval_mins: int) -> bool: """ Validates preprocessed backtesting data for missing values and shows warnings about it that. @@ -89,10 +89,13 @@ def validate_backtest_data(data: Dict[str, DataFrame], min_date: datetime, """ # total difference in minutes / interval-minutes expected_frames = int((max_date - min_date).total_seconds() // 60 // ticker_interval_mins) + found_missing = False for pair, df in data.items(): if len(df) < expected_frames: + found_missing = True logger.warning('%s has missing frames: expected %s, got %s', pair, expected_frames, len(df)) + return found_missing def load_tickerdata_file( diff --git a/freqtrade/tests/optimize/test_optimize.py b/freqtrade/tests/optimize/test_optimize.py index 7b13b498a..32db5abd0 100644 --- a/freqtrade/tests/optimize/test_optimize.py +++ b/freqtrade/tests/optimize/test_optimize.py @@ -465,8 +465,8 @@ def test_validate_backtest_data_warn(default_conf, mocker, caplog) -> None: ) min_date, max_date = optimize.get_timeframe(data) caplog.clear() - optimize.validate_backtest_data(data, min_date, max_date, - constants.TICKER_INTERVAL_MINUTES["1m"]) + assert optimize.validate_backtest_data(data, min_date, max_date, + constants.TICKER_INTERVAL_MINUTES["1m"]) assert len(caplog.record_tuples) == 1 assert log_has('UNITTEST/BTC has missing frames: expected 14396, got 13680', caplog.record_tuples) @@ -488,7 +488,7 @@ def test_validate_backtest_data(default_conf, mocker, caplog) -> None: min_date, max_date = optimize.get_timeframe(data) caplog.clear() - optimize.validate_backtest_data(data, min_date, max_date, - constants.TICKER_INTERVAL_MINUTES["5m"]) + assert not optimize.validate_backtest_data(data, min_date, max_date, + constants.TICKER_INTERVAL_MINUTES["5m"]) assert len(caplog.record_tuples) == 0 From 3c6d10f03e95b4ede9d63328e4d2f0a65cd2a294 Mon Sep 17 00:00:00 2001 From: Matthias Date: Thu, 18 Oct 2018 20:05:57 +0200 Subject: [PATCH 159/699] Print missing value count too --- freqtrade/optimize/__init__.py | 7 ++++--- freqtrade/tests/optimize/test_optimize.py | 6 +++--- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/freqtrade/optimize/__init__.py b/freqtrade/optimize/__init__.py index 3376d0075..e837a09bd 100644 --- a/freqtrade/optimize/__init__.py +++ b/freqtrade/optimize/__init__.py @@ -91,10 +91,11 @@ def validate_backtest_data(data: Dict[str, DataFrame], min_date: datetime, expected_frames = int((max_date - min_date).total_seconds() // 60 // ticker_interval_mins) found_missing = False for pair, df in data.items(): - if len(df) < expected_frames: + dflen = len(df) + if dflen < expected_frames: found_missing = True - logger.warning('%s has missing frames: expected %s, got %s', - pair, expected_frames, len(df)) + logger.warning("%s has missing frames: expected %s, got %s, that's %s missing values", + pair, expected_frames, dflen, expected_frames - dflen) return found_missing diff --git a/freqtrade/tests/optimize/test_optimize.py b/freqtrade/tests/optimize/test_optimize.py index 32db5abd0..2975a3e4b 100644 --- a/freqtrade/tests/optimize/test_optimize.py +++ b/freqtrade/tests/optimize/test_optimize.py @@ -468,8 +468,9 @@ def test_validate_backtest_data_warn(default_conf, mocker, caplog) -> None: assert optimize.validate_backtest_data(data, min_date, max_date, constants.TICKER_INTERVAL_MINUTES["1m"]) assert len(caplog.record_tuples) == 1 - assert log_has('UNITTEST/BTC has missing frames: expected 14396, got 13680', - caplog.record_tuples) + assert log_has( + "UNITTEST/BTC has missing frames: expected 14396, got 13680, that's 716 missing values", + caplog.record_tuples) def test_validate_backtest_data(default_conf, mocker, caplog) -> None: @@ -491,4 +492,3 @@ def test_validate_backtest_data(default_conf, mocker, caplog) -> None: assert not optimize.validate_backtest_data(data, min_date, max_date, constants.TICKER_INTERVAL_MINUTES["5m"]) assert len(caplog.record_tuples) == 0 - From 7f9f53248c1aa87a2d7483a9e465d0fc93d68b1e Mon Sep 17 00:00:00 2001 From: Matthias Date: Thu, 18 Oct 2018 20:25:21 +0200 Subject: [PATCH 160/699] Add validate_backtest_data script --- scripts/plot_dataframe.py | 381 -------------------------------------- 1 file changed, 381 deletions(-) delete mode 100755 scripts/plot_dataframe.py diff --git a/scripts/plot_dataframe.py b/scripts/plot_dataframe.py deleted file mode 100755 index 68713f296..000000000 --- a/scripts/plot_dataframe.py +++ /dev/null @@ -1,381 +0,0 @@ -#!/usr/bin/env python3 -""" -Script to display when the bot will buy a specific pair - -Mandatory Cli parameters: --p / --pair: pair to examine - -Option but recommended --s / --strategy: strategy to use - - -Optional Cli parameters --d / --datadir: path to pair backtest data ---timerange: specify what timerange of data to use. --l / --live: Live, to download the latest ticker for the pair --db / --db-url: Show trades stored in database - - -Indicators recommended -Row 1: sma, ema3, ema5, ema10, ema50 -Row 3: macd, rsi, fisher_rsi, mfi, slowd, slowk, fastd, fastk - -Example of usage: -> python3 scripts/plot_dataframe.py --pair BTC/EUR -d user_data/data/ --indicators1 sma,ema3 ---indicators2 fastk,fastd -""" -import json -import logging -import sys -from argparse import Namespace -from pathlib import Path -from typing import Dict, List, Any - -import pandas as pd -import plotly.graph_objs as go -import pytz - -from plotly import tools -from plotly.offline import plot - -import freqtrade.optimize as optimize -from freqtrade import persistence -from freqtrade.arguments import Arguments, TimeRange -from freqtrade.exchange import Exchange -from freqtrade.optimize.backtesting import setup_configuration -from freqtrade.persistence import Trade -from freqtrade.strategy.resolver import StrategyResolver - -logger = logging.getLogger(__name__) -_CONF: Dict[str, Any] = {} - -timeZone = pytz.UTC - - -def load_trades(args: Namespace, pair: str, timerange: TimeRange) -> pd.DataFrame: - trades: pd.DataFrame = pd.DataFrame() - if args.db_url: - persistence.init(_CONF) - columns = ["pair", "profit", "opents", "closets", "open_rate", "close_rate", "duration"] - - for x in Trade.query.all(): - print("date: {}".format(x.open_date)) - - trades = pd.DataFrame([(t.pair, t.calc_profit(), - t.open_date.replace(tzinfo=timeZone), - t.close_date.replace(tzinfo=timeZone) if t.close_date else None, - t.open_rate, t.close_rate, - t.close_date.timestamp() - t.open_date.timestamp() if t.close_date else None) - for t in Trade.query.filter(Trade.pair.is_(pair)).all()], - columns=columns) - - elif args.exportfilename: - file = Path(args.exportfilename) - # must align with columns in backtest.py - columns = ["pair", "profit", "opents", "closets", "index", "duration", - "open_rate", "close_rate", "open_at_end", "sell_reason"] - with file.open() as f: - data = json.load(f) - trades = pd.DataFrame(data, columns=columns) - trades = trades.loc[trades["pair"] == pair] - if timerange: - if timerange.starttype == 'date': - trades = trades.loc[trades["opents"] >= timerange.startts] - if timerange.stoptype == 'date': - trades = trades.loc[trades["opents"] <= timerange.stopts] - - trades['opents'] = pd.to_datetime(trades['opents'], - unit='s', - utc=True, - infer_datetime_format=True) - trades['closets'] = pd.to_datetime(trades['closets'], - unit='s', - utc=True, - infer_datetime_format=True) - return trades - - -def plot_analyzed_dataframe(args: Namespace) -> None: - """ - Calls analyze() and plots the returned dataframe - :return: None - """ - global _CONF - - # Load the configuration - _CONF.update(setup_configuration(args)) - - print(_CONF) - # Set the pair to audit - pair = args.pair - - if pair is None: - logger.critical('Parameter --pair mandatory;. E.g --pair ETH/BTC') - exit() - - if '/' not in pair: - logger.critical('--pair format must be XXX/YYY') - exit() - - # Set timerange to use - timerange = Arguments.parse_timerange(args.timerange) - - # Load the strategy - try: - strategy = StrategyResolver(_CONF).strategy - exchange = Exchange(_CONF) - except AttributeError: - logger.critical( - 'Impossible to load the strategy. Please check the file "user_data/strategies/%s.py"', - args.strategy - ) - exit() - - # Set the ticker to use - tick_interval = strategy.ticker_interval - - # Load pair tickers - tickers = {} - if args.live: - logger.info('Downloading pair.') - exchange.refresh_tickers([pair], tick_interval) - tickers[pair] = exchange.klines[pair] - else: - tickers = optimize.load_data( - datadir=_CONF.get("datadir"), - pairs=[pair], - ticker_interval=tick_interval, - refresh_pairs=_CONF.get('refresh_pairs', False), - timerange=timerange, - exchange=Exchange(_CONF) - ) - - # No ticker found, or impossible to download - if tickers == {}: - exit() - - # Get trades already made from the DB - trades = load_trades(args, pair, timerange) - - dataframes = strategy.tickerdata_to_dataframe(tickers) - - dataframe = dataframes[pair] - dataframe = strategy.advise_buy(dataframe, {'pair': pair}) - dataframe = strategy.advise_sell(dataframe, {'pair': pair}) - - if len(dataframe.index) > args.plot_limit: - logger.warning('Ticker contained more than %s candles as defined ' - 'with --plot-limit, clipping.', args.plot_limit) - dataframe = dataframe.tail(args.plot_limit) - - trades = trades.loc[trades['opents'] >= dataframe.iloc[0]['date']] - fig = generate_graph( - pair=pair, - trades=trades, - data=dataframe, - args=args - ) - - plot(fig, filename=str(Path('user_data').joinpath('freqtrade-plot.html'))) - - -def generate_graph(pair, trades: pd.DataFrame, data: pd.DataFrame, args) -> tools.make_subplots: - """ - Generate the graph from the data generated by Backtesting or from DB - :param pair: Pair to Display on the graph - :param trades: All trades created - :param data: Dataframe - :param args: sys.argv that contrains the two params indicators1, and indicators2 - :return: None - """ - - # Define the graph - fig = tools.make_subplots( - rows=3, - cols=1, - shared_xaxes=True, - row_width=[1, 1, 4], - vertical_spacing=0.0001, - ) - fig['layout'].update(title=pair) - fig['layout']['yaxis1'].update(title='Price') - fig['layout']['yaxis2'].update(title='Volume') - fig['layout']['yaxis3'].update(title='Other') - - # Common information - candles = go.Candlestick( - x=data.date, - open=data.open, - high=data.high, - low=data.low, - close=data.close, - name='Price' - ) - - df_buy = data[data['buy'] == 1] - buys = go.Scattergl( - x=df_buy.date, - y=df_buy.close, - mode='markers', - name='buy', - marker=dict( - symbol='triangle-up-dot', - size=9, - line=dict(width=1), - color='green', - ) - ) - df_sell = data[data['sell'] == 1] - sells = go.Scattergl( - x=df_sell.date, - y=df_sell.close, - mode='markers', - name='sell', - marker=dict( - symbol='triangle-down-dot', - size=9, - line=dict(width=1), - color='red', - ) - ) - - trade_buys = go.Scattergl( - x=trades["opents"], - y=trades["open_rate"], - mode='markers', - name='trade_buy', - marker=dict( - symbol='square-open', - size=11, - line=dict(width=2), - color='green' - ) - ) - trade_sells = go.Scattergl( - x=trades["closets"], - y=trades["close_rate"], - mode='markers', - name='trade_sell', - marker=dict( - symbol='square-open', - size=11, - line=dict(width=2), - color='red' - ) - ) - - # Row 1 - fig.append_trace(candles, 1, 1) - - if 'bb_lowerband' in data and 'bb_upperband' in data: - bb_lower = go.Scatter( - x=data.date, - y=data.bb_lowerband, - name='BB lower', - line={'color': 'rgba(255,255,255,0)'}, - ) - bb_upper = go.Scatter( - x=data.date, - y=data.bb_upperband, - name='BB upper', - fill="tonexty", - fillcolor="rgba(0,176,246,0.2)", - line={'color': 'rgba(255,255,255,0)'}, - ) - fig.append_trace(bb_lower, 1, 1) - fig.append_trace(bb_upper, 1, 1) - - fig = generate_row(fig=fig, row=1, raw_indicators=args.indicators1, data=data) - fig.append_trace(buys, 1, 1) - fig.append_trace(sells, 1, 1) - fig.append_trace(trade_buys, 1, 1) - fig.append_trace(trade_sells, 1, 1) - - # Row 2 - volume = go.Bar( - x=data['date'], - y=data['volume'], - name='Volume' - ) - fig.append_trace(volume, 2, 1) - - # Row 3 - fig = generate_row(fig=fig, row=3, raw_indicators=args.indicators2, data=data) - - return fig - - -def generate_row(fig, row, raw_indicators, data) -> tools.make_subplots: - """ - Generator all the indicator selected by the user for a specific row - """ - for indicator in raw_indicators.split(','): - if indicator in data: - scattergl = go.Scattergl( - x=data['date'], - y=data[indicator], - name=indicator - ) - fig.append_trace(scattergl, row, 1) - else: - logger.info( - 'Indicator "%s" ignored. Reason: This indicator is not found ' - 'in your strategy.', - indicator - ) - - return fig - - -def plot_parse_args(args: List[str]) -> Namespace: - """ - Parse args passed to the script - :param args: Cli arguments - :return: args: Array with all arguments - """ - arguments = Arguments(args, 'Graph dataframe') - arguments.scripts_options() - arguments.parser.add_argument( - '--indicators1', - help='Set indicators from your strategy you want in the first row of the graph. Separate ' - 'them with a coma. E.g: ema3,ema5 (default: %(default)s)', - type=str, - default='sma,ema3,ema5', - dest='indicators1', - ) - - arguments.parser.add_argument( - '--indicators2', - help='Set indicators from your strategy you want in the third row of the graph. Separate ' - 'them with a coma. E.g: fastd,fastk (default: %(default)s)', - type=str, - default='macd', - dest='indicators2', - ) - arguments.parser.add_argument( - '--plot-limit', - help='Specify tick limit for plotting - too high values cause huge files - ' - 'Default: %(default)s', - dest='plot_limit', - default=750, - type=int, - ) - arguments.common_args_parser() - arguments.optimizer_shared_options(arguments.parser) - arguments.backtesting_options(arguments.parser) - return arguments.parse_args() - - -def main(sysargv: List[str]) -> None: - """ - This function will initiate the bot and start the trading loop. - :return: None - """ - logger.info('Starting Plot Dataframe') - plot_analyzed_dataframe( - plot_parse_args(sysargv) - ) - - -if __name__ == '__main__': - main(sys.argv[1:]) From c69b87914d6666797353c0d48f1fd2c7ec386c82 Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Fri, 19 Oct 2018 14:35:07 +0200 Subject: [PATCH 161/699] Update ccxt from 1.17.392 to 1.17.393 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 86a654ffb..9359dc1a7 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -ccxt==1.17.392 +ccxt==1.17.393 SQLAlchemy==1.2.12 python-telegram-bot==11.1.0 arrow==0.12.1 From 71814ae2d6806c2afefe09f8745dc656168cfd95 Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Fri, 19 Oct 2018 14:35:09 +0200 Subject: [PATCH 162/699] Update requests from 2.19.1 to 2.20.0 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 9359dc1a7..e750cf893 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,7 +3,7 @@ SQLAlchemy==1.2.12 python-telegram-bot==11.1.0 arrow==0.12.1 cachetools==2.1.0 -requests==2.19.1 +requests==2.20.0 urllib3==1.24 wrapt==1.10.11 pandas==0.23.4 From 202b1d1f0bef02ad6f373babb9d995d774a1ab98 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 21 Oct 2018 09:21:32 +0200 Subject: [PATCH 163/699] fix #1289 - we should not modify decimal context --- freqtrade/persistence.py | 6 +---- freqtrade/tests/optimize/test_backtesting.py | 2 +- freqtrade/tests/rpc/test_rpc_telegram.py | 6 ++--- freqtrade/tests/test_freqtradebot.py | 10 ++++---- freqtrade/tests/test_persistence.py | 26 ++++++++++---------- 5 files changed, 23 insertions(+), 27 deletions(-) diff --git a/freqtrade/persistence.py b/freqtrade/persistence.py index c26d74015..02267ac21 100644 --- a/freqtrade/persistence.py +++ b/freqtrade/persistence.py @@ -4,7 +4,7 @@ This module contains the class to persist trades into SQLite import logging from datetime import datetime -from decimal import Decimal, getcontext +from decimal import Decimal from typing import Any, Dict, Optional import arrow @@ -241,7 +241,6 @@ class Trade(_DECL_BASE): logger.info('Updating trade (id=%d) ...', self.id) - getcontext().prec = 8 # Bittrex do not go above 8 decimal if order_type == 'limit' and order['side'] == 'buy': # Update open rate and actual amount self.open_rate = Decimal(order['price']) @@ -278,7 +277,6 @@ class Trade(_DECL_BASE): If rate is not set self.fee will be used :return: Price in BTC of the open trade """ - getcontext().prec = 8 buy_trade = (Decimal(self.amount) * Decimal(self.open_rate)) fees = buy_trade * Decimal(fee or self.fee_open) @@ -296,7 +294,6 @@ class Trade(_DECL_BASE): If rate is not set self.close_rate will be used :return: Price in BTC of the open trade """ - getcontext().prec = 8 if rate is None and not self.close_rate: return 0.0 @@ -336,7 +333,6 @@ class Trade(_DECL_BASE): :param fee: fee to use on the close rate (optional). :return: profit in percentage as float """ - getcontext().prec = 8 open_trade_price = self.calc_open_trade_price() close_trade_price = self.calc_close_trade_price( diff --git a/freqtrade/tests/optimize/test_backtesting.py b/freqtrade/tests/optimize/test_backtesting.py index a17867b3a..3ea0f240c 100644 --- a/freqtrade/tests/optimize/test_backtesting.py +++ b/freqtrade/tests/optimize/test_backtesting.py @@ -534,7 +534,7 @@ def test_backtest(default_conf, fee, mocker) -> None: expected = pd.DataFrame( {'pair': [pair, pair], - 'profit_percent': [0.00029975, 0.00056708], + 'profit_percent': [0.00029977, 0.00056716], 'profit_abs': [1.49e-06, 7.6e-07], 'open_time': [Arrow(2018, 1, 29, 18, 40, 0).datetime, Arrow(2018, 1, 30, 3, 30, 0).datetime], diff --git a/freqtrade/tests/rpc/test_rpc_telegram.py b/freqtrade/tests/rpc/test_rpc_telegram.py index 708ed4478..45e01aa57 100644 --- a/freqtrade/tests/rpc/test_rpc_telegram.py +++ b/freqtrade/tests/rpc/test_rpc_telegram.py @@ -725,7 +725,7 @@ def test_forcesell_handle(default_conf, update, ticker, fee, 'open_rate': 1.099e-05, 'current_rate': 1.172e-05, 'profit_amount': 6.126e-05, - 'profit_percent': 0.06110514, + 'profit_percent': 0.0611052, 'stake_currency': 'BTC', 'fiat_currency': 'USD', } == last_msg @@ -778,7 +778,7 @@ def test_forcesell_down_handle(default_conf, update, ticker, fee, 'open_rate': 1.099e-05, 'current_rate': 1.044e-05, 'profit_amount': -5.492e-05, - 'profit_percent': -0.05478343, + 'profit_percent': -0.05478342, 'stake_currency': 'BTC', 'fiat_currency': 'USD', } == last_msg @@ -823,7 +823,7 @@ def test_forcesell_all_handle(default_conf, update, ticker, fee, markets, mocker 'open_rate': 1.099e-05, 'current_rate': 1.098e-05, 'profit_amount': -5.91e-06, - 'profit_percent': -0.00589292, + 'profit_percent': -0.00589291, 'stake_currency': 'BTC', 'fiat_currency': 'USD', } == msg diff --git a/freqtrade/tests/test_freqtradebot.py b/freqtrade/tests/test_freqtradebot.py index cad2f654d..383b07864 100644 --- a/freqtrade/tests/test_freqtradebot.py +++ b/freqtrade/tests/test_freqtradebot.py @@ -804,7 +804,7 @@ def test_handle_trade(default_conf, limit_buy_order, limit_sell_order, trade.update(limit_sell_order) assert trade.close_rate == 0.00001173 - assert trade.close_profit == 0.06201057 + assert trade.close_profit == 0.06201058 assert trade.calc_profit() == 0.00006217 assert trade.close_date is not None @@ -1231,7 +1231,7 @@ def test_execute_sell_up(default_conf, ticker, fee, ticker_sell_up, markets, moc 'open_rate': 1.099e-05, 'current_rate': 1.172e-05, 'profit_amount': 6.126e-05, - 'profit_percent': 0.06110514, + 'profit_percent': 0.0611052, 'stake_currency': 'BTC', 'fiat_currency': 'USD', } == last_msg @@ -1277,7 +1277,7 @@ def test_execute_sell_down(default_conf, ticker, fee, ticker_sell_down, markets, 'open_rate': 1.099e-05, 'current_rate': 1.044e-05, 'profit_amount': -5.492e-05, - 'profit_percent': -0.05478343, + 'profit_percent': -0.05478342, 'stake_currency': 'BTC', 'fiat_currency': 'USD', } == last_msg @@ -1324,7 +1324,7 @@ def test_execute_sell_without_conf_sell_up(default_conf, ticker, fee, 'open_rate': 1.099e-05, 'current_rate': 1.172e-05, 'profit_amount': 6.126e-05, - 'profit_percent': 0.06110514, + 'profit_percent': 0.0611052, } == last_msg @@ -1370,7 +1370,7 @@ def test_execute_sell_without_conf_sell_down(default_conf, ticker, fee, 'open_rate': 1.099e-05, 'current_rate': 1.044e-05, 'profit_amount': -5.492e-05, - 'profit_percent': -0.05478343, + 'profit_percent': -0.05478342, } == last_msg diff --git a/freqtrade/tests/test_persistence.py b/freqtrade/tests/test_persistence.py index 7584537e2..5e0647dff 100644 --- a/freqtrade/tests/test_persistence.py +++ b/freqtrade/tests/test_persistence.py @@ -113,7 +113,7 @@ def test_update_with_bittrex(limit_buy_order, limit_sell_order, fee): trade.update(limit_sell_order) assert trade.open_order_id is None assert trade.close_rate == 0.00001173 - assert trade.close_profit == 0.06201057 + assert trade.close_profit == 0.06201058 assert trade.close_date is not None @@ -129,16 +129,16 @@ def test_calc_open_close_trade_price(limit_buy_order, limit_sell_order, fee): trade.open_order_id = 'something' trade.update(limit_buy_order) - assert trade.calc_open_trade_price() == 0.001002500 + assert trade.calc_open_trade_price() == 0.0010024999999225068 trade.update(limit_sell_order) - assert trade.calc_close_trade_price() == 0.0010646656 + assert trade.calc_close_trade_price() == 0.0010646656050132426 # Profit in BTC assert trade.calc_profit() == 0.00006217 # Profit in percent - assert trade.calc_profit_percent() == 0.06201057 + assert trade.calc_profit_percent() == 0.06201058 @pytest.mark.usefixtures("init_persistence") @@ -207,10 +207,10 @@ def test_calc_open_trade_price(limit_buy_order, fee): trade.update(limit_buy_order) # Buy @ 0.00001099 # Get the open rate price with the standard fee rate - assert trade.calc_open_trade_price() == 0.001002500 + assert trade.calc_open_trade_price() == 0.0010024999999225068 # Get the open rate price with a custom fee rate - assert trade.calc_open_trade_price(fee=0.003) == 0.001003000 + assert trade.calc_open_trade_price(fee=0.003) == 0.001002999999922468 @pytest.mark.usefixtures("init_persistence") @@ -226,14 +226,14 @@ def test_calc_close_trade_price(limit_buy_order, limit_sell_order, fee): trade.update(limit_buy_order) # Buy @ 0.00001099 # Get the close rate price with a custom close rate and a regular fee rate - assert trade.calc_close_trade_price(rate=0.00001234) == 0.0011200318 + assert trade.calc_close_trade_price(rate=0.00001234) == 0.0011200318470471794 # Get the close rate price with a custom close rate and a custom fee rate - assert trade.calc_close_trade_price(rate=0.00001234, fee=0.003) == 0.0011194704 + assert trade.calc_close_trade_price(rate=0.00001234, fee=0.003) == 0.0011194704275749754 # Test when we apply a Sell order, and ask price with a custom fee rate trade.update(limit_sell_order) - assert trade.calc_close_trade_price(fee=0.005) == 0.0010619972 + assert trade.calc_close_trade_price(fee=0.005) == 0.0010619972701635854 @pytest.mark.usefixtures("init_persistence") @@ -281,17 +281,17 @@ def test_calc_profit_percent(limit_buy_order, limit_sell_order, fee): trade.update(limit_buy_order) # Buy @ 0.00001099 # Get percent of profit with a custom rate (Higher than open rate) - assert trade.calc_profit_percent(rate=0.00001234) == 0.1172387 + assert trade.calc_profit_percent(rate=0.00001234) == 0.11723875 # Get percent of profit with a custom rate (Lower than open rate) - assert trade.calc_profit_percent(rate=0.00000123) == -0.88863827 + assert trade.calc_profit_percent(rate=0.00000123) == -0.88863828 # Test when we apply a Sell order. Sell higher than open rate @ 0.00001173 trade.update(limit_sell_order) - assert trade.calc_profit_percent() == 0.06201057 + assert trade.calc_profit_percent() == 0.06201058 # Test with a custom fee rate on the close trade - assert trade.calc_profit_percent(fee=0.003) == 0.0614782 + assert trade.calc_profit_percent(fee=0.003) == 0.06147824 def test_clean_dry_run_db(default_conf, fee): From 677a9e56afd83141f7e2a3a6cc5acc2737fdffaa Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 21 Oct 2018 09:23:07 +0200 Subject: [PATCH 164/699] remove skipped test (refresh_whitelist is tested in test_acl_pair) --- freqtrade/tests/test_freqtradebot.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/freqtrade/tests/test_freqtradebot.py b/freqtrade/tests/test_freqtradebot.py index 383b07864..6b13da35f 100644 --- a/freqtrade/tests/test_freqtradebot.py +++ b/freqtrade/tests/test_freqtradebot.py @@ -167,11 +167,6 @@ def test_gen_pair_whitelist_not_supported(mocker, default_conf, tickers) -> None freqtrade._gen_pair_whitelist(base_currency='BTC') -@pytest.mark.skip(reason="Test not implemented") -def test_refresh_whitelist() -> None: - pass - - def test_get_trade_stake_amount(default_conf, ticker, limit_buy_order, fee, mocker) -> None: patch_RPCManager(mocker) patch_exchange(mocker) From 184b5ca3fc5ad3d102be4846db52a5b1a469b1b2 Mon Sep 17 00:00:00 2001 From: Matthias Date: Mon, 13 Aug 2018 19:59:56 +0200 Subject: [PATCH 165/699] cleanup root dir and create build_helpers --- .../install_ta-lib.sh | 0 build_helpers/publish_docker.sh | 53 ++++++++++++++++++ .../ta-lib-0.4.0-src.tar.gz | Bin 3 files changed, 53 insertions(+) rename install_ta-lib.sh => build_helpers/install_ta-lib.sh (100%) create mode 100755 build_helpers/publish_docker.sh rename ta-lib-0.4.0-src.tar.gz => build_helpers/ta-lib-0.4.0-src.tar.gz (100%) diff --git a/install_ta-lib.sh b/build_helpers/install_ta-lib.sh similarity index 100% rename from install_ta-lib.sh rename to build_helpers/install_ta-lib.sh diff --git a/build_helpers/publish_docker.sh b/build_helpers/publish_docker.sh new file mode 100755 index 000000000..a398a8719 --- /dev/null +++ b/build_helpers/publish_docker.sh @@ -0,0 +1,53 @@ +#!/bin/sh +# Tag with travis build +TAG=$TRAVIS_BUILD_NUMBER +# - export TAG=`if [ "$TRAVIS_BRANCH" == "develop" ]; then echo "latest"; else echo $TRAVIS_BRANCH ; fi` +# Replace / with _ to create a valid tag +TAG=$(echo "${TRAVIS_BRANCH}" | sed -e "s/\//_/") + + +# Pull last build to avoid rebuilding the whole image +docker pull ${REPO}:${TAG} + +docker build --cache-from ${IMAGE_NAME}:${TAG} -t freqtrade:${TAG} . +if [ $? -ne 0 ]; then + echo "failed building image" + return 1 +fi + +# Run backtest +docker run --rm -it -v $(pwd)/config.json.example:/freqtrade/config.json:ro freqtrade:${TAG} --datadir freqtrade/tests/testdata backtesting + +if [ $? -ne 0 ]; then + echo "failed running backtest" + return 1 +fi + +# Tag image for upload +docker tag freqtrade:$TAG ${IMAGE_NAME}:$TAG +if [ $? -ne 0 ]; then + echo "failed tagging image" + return 1 +fi + +# Tag as latest for develop builds +if [ "${TRAVIS_BRANCH}" == "develop" ]; then + docker tag freqtrade:$TAG ${IMAGE_NAME}:latest +fi + +# Login +echo "$DOCKER_PASS" | docker login -u $DOCKER_USER --password-stdin + +if [ $? -ne 0 ]; then + echo "failed login" + return 1 +fi + +# Show all available images +docker images + +docker push ${IMAGE_NAME} +if [ $? -ne 0 ]; then + echo "failed pushing repo" + return 1 +fi diff --git a/ta-lib-0.4.0-src.tar.gz b/build_helpers/ta-lib-0.4.0-src.tar.gz similarity index 100% rename from ta-lib-0.4.0-src.tar.gz rename to build_helpers/ta-lib-0.4.0-src.tar.gz From 98738c482a7e52ecc9d7d55929e21b54ff2558c9 Mon Sep 17 00:00:00 2001 From: Matthias Date: Mon, 13 Aug 2018 20:01:57 +0200 Subject: [PATCH 166/699] modify install-ta-lib script to support running in docker --- build_helpers/install_ta-lib.sh | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/build_helpers/install_ta-lib.sh b/build_helpers/install_ta-lib.sh index d8ae2eeaa..4d4f37c17 100755 --- a/build_helpers/install_ta-lib.sh +++ b/build_helpers/install_ta-lib.sh @@ -1,7 +1,13 @@ if [ ! -f "ta-lib/CHANGELOG.TXT" ]; then tar zxvf ta-lib-0.4.0-src.tar.gz - cd ta-lib && sed -i.bak "s|0.00000001|0.000000000000000001 |g" src/ta_func/ta_utility.h && ./configure && make && sudo make install && cd .. + cd ta-lib \ + && sed -i.bak "s|0.00000001|0.000000000000000001 |g" src/ta_func/ta_utility.h \ + && ./configure \ + && make \ + && which sudo && sudo make install || make install \ + && cd .. else echo "TA-lib already installed, skipping download and build." cd ta-lib && sudo make install && cd .. + fi From 907761f9941e73f931e3b8e97e8d1366539f90ca Mon Sep 17 00:00:00 2001 From: Matthias Date: Mon, 13 Aug 2018 20:03:20 +0200 Subject: [PATCH 167/699] Install ta-lib in Docker with script works for travis, works for Docker --- Dockerfile | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/Dockerfile b/Dockerfile index 2506665ab..24cce0049 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,19 +1,20 @@ FROM python:3.7.0-slim-stretch -# Install TA-lib -RUN apt-get update && apt-get -y install curl build-essential && apt-get clean -RUN curl -L http://prdownloads.sourceforge.net/ta-lib/ta-lib-0.4.0-src.tar.gz | \ - tar xzvf - && \ - cd ta-lib && \ - sed -i "s|0.00000001|0.000000000000000001 |g" src/ta_func/ta_utility.h && \ - ./configure && make && make install && \ - cd .. && rm -rf ta-lib -ENV LD_LIBRARY_PATH /usr/local/lib +RUN apt-get update \ + && apt-get -y install curl build-essential \ + && apt-get clean \ + && pip install --upgrade pip # Prepare environment RUN mkdir /freqtrade WORKDIR /freqtrade +# Install TA-lib +COPY build_helpers/* /tmp/ +RUN cd /tmp && /tmp/install_ta-lib.sh && rm -r /tmp/*ta-lib* + +ENV LD_LIBRARY_PATH /usr/local/lib + # Install dependencies COPY requirements.txt /freqtrade/ RUN pip install numpy --no-cache-dir \ From af7283017b331f6c5f396b7757f7c84674796a3e Mon Sep 17 00:00:00 2001 From: Matthias Date: Mon, 13 Aug 2018 20:04:25 +0200 Subject: [PATCH 168/699] modify travis to build and push docker * name steps * only build for master / develop and this branch (for now) --- .travis.yml | 24 ++++++++++++++++++++++-- 1 file changed, 22 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 981eedcf8..f1192e80c 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,9 +1,15 @@ sudo: true os: - linux +dist: trusty language: python python: - 3.6 +services: + - docker +env: + global: + - IMAGE_NAME=freqtradeorg/freqtrade addons: apt: packages: @@ -11,24 +17,38 @@ addons: - libdw-dev - binutils-dev install: -- ./install_ta-lib.sh +- ./build_helpers/install_ta-lib.sh - export LD_LIBRARY_PATH=/usr/local/lib:$LD_LIBRARY_PATH - pip install --upgrade flake8 coveralls pytest-random-order pytest-asyncio mypy - pip install -r requirements.txt - pip install -e . jobs: include: - - script: + - stage: tests + script: - pytest --cov=freqtrade --cov-config=.coveragerc freqtrade/tests/ - coveralls + name: pytest - script: - cp config.json.example config.json - python freqtrade/main.py --datadir freqtrade/tests/testdata backtesting + name: backtest - script: - cp config.json.example config.json - python freqtrade/main.py --datadir freqtrade/tests/testdata hyperopt -e 5 + name: hyperopt - script: flake8 freqtrade + name: flake8 - script: mypy freqtrade + name: mypy + + - stage: docker + if: branch in (master, develop, feat/improve_travis) AND (type in (push, cron)) + script: + - build_helpers/publish_docker.sh + name: "Build and test and push docker image" + + notifications: slack: secure: bKLXmOrx8e2aPZl7W8DA5BdPAXWGpI5UzST33oc1G/thegXcDVmHBTJrBs4sZak6bgAclQQrdZIsRd2eFYzHLalJEaw6pk7hoAw8SvLnZO0ZurWboz7qg2+aZZXfK4eKl/VUe4sM9M4e/qxjkK+yWG7Marg69c4v1ypF7ezUi1fPYILYw8u0paaiX0N5UX8XNlXy+PBlga2MxDjUY70MuajSZhPsY2pDUvYnMY1D/7XN3cFW0g+3O8zXjF0IF4q1Z/1ASQe+eYjKwPQacE+O8KDD+ZJYoTOFBAPllrtpO1jnOPFjNGf3JIbVMZw4bFjIL0mSQaiSUaUErbU3sFZ5Or79rF93XZ81V7uEZ55vD8KMfR2CB1cQJcZcj0v50BxLo0InkFqa0Y8Nra3sbpV4fV5Oe8pDmomPJrNFJnX6ULQhQ1gTCe0M5beKgVms5SITEpt4/Y0CmLUr6iHDT0CUiyMIRWAXdIgbGh1jfaWOMksybeRevlgDsIsNBjXmYI1Sw2ZZR2Eo2u4R6zyfyjOMLwYJ3vgq9IrACv2w5nmf0+oguMWHf6iWi2hiOqhlAN1W74+3HsYQcqnuM3LGOmuCnPprV1oGBqkPXjIFGpy21gNx4vHfO1noLUyJnMnlu2L7SSuN1CdLsnjJ1hVjpJjPfqB4nn8g12x87TqM1bOm+3Q= From 0535660db722342d11e3a7a2da4d3aca6880025c Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 14 Aug 2018 09:44:21 +0200 Subject: [PATCH 169/699] build technical image --- Dockerfile.technical | 6 ++++++ build_helpers/publish_docker.sh | 10 ++++++---- 2 files changed, 12 insertions(+), 4 deletions(-) create mode 100644 Dockerfile.technical diff --git a/Dockerfile.technical b/Dockerfile.technical new file mode 100644 index 000000000..5339eb232 --- /dev/null +++ b/Dockerfile.technical @@ -0,0 +1,6 @@ +FROM freqtradeorg/freqtrade:develop + +RUN apt-get update \ + && apt-get -y install git \ + && apt-get clean \ + && pip install git+https://github.com/berlinguyinca/technical diff --git a/build_helpers/publish_docker.sh b/build_helpers/publish_docker.sh index a398a8719..95aae0f4f 100755 --- a/build_helpers/publish_docker.sh +++ b/build_helpers/publish_docker.sh @@ -1,10 +1,8 @@ #!/bin/sh -# Tag with travis build -TAG=$TRAVIS_BUILD_NUMBER # - export TAG=`if [ "$TRAVIS_BRANCH" == "develop" ]; then echo "latest"; else echo $TRAVIS_BRANCH ; fi` # Replace / with _ to create a valid tag TAG=$(echo "${TRAVIS_BRANCH}" | sed -e "s/\//_/") - +TAG_TECH="${TAG}_technical" # Pull last build to avoid rebuilding the whole image docker pull ${REPO}:${TAG} @@ -23,6 +21,10 @@ if [ $? -ne 0 ]; then return 1 fi +# build technical image +sed -i Dockerfile.technical -e "s/FROM freqtradeorg\/freqtrade:develop/FROM freqtradeorg\/freqtrade:${TAG}/" +docker build --cache-from freqtrade:${TAG} -t ${IMAGE_NAME}:${TAG_TECH} -f Dockerfile.technical . + # Tag image for upload docker tag freqtrade:$TAG ${IMAGE_NAME}:$TAG if [ $? -ne 0 ]; then @@ -31,7 +33,7 @@ if [ $? -ne 0 ]; then fi # Tag as latest for develop builds -if [ "${TRAVIS_BRANCH}" == "develop" ]; then +if [ "${TRAVIS_BRANCH}" = "develop" ]; then docker tag freqtrade:$TAG ${IMAGE_NAME}:latest fi From 7301d76cff46c23a0e9b7c7aee04d83ca124bb27 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 21 Oct 2018 13:20:51 +0200 Subject: [PATCH 170/699] Remove autobuild for technical as it's not versioned as it's not versioned and installed from github, we cannot guarantee which version is in the image. --- build_helpers/publish_docker.sh | 4 ---- 1 file changed, 4 deletions(-) diff --git a/build_helpers/publish_docker.sh b/build_helpers/publish_docker.sh index 95aae0f4f..5e0809fcf 100755 --- a/build_helpers/publish_docker.sh +++ b/build_helpers/publish_docker.sh @@ -21,10 +21,6 @@ if [ $? -ne 0 ]; then return 1 fi -# build technical image -sed -i Dockerfile.technical -e "s/FROM freqtradeorg\/freqtrade:develop/FROM freqtradeorg\/freqtrade:${TAG}/" -docker build --cache-from freqtrade:${TAG} -t ${IMAGE_NAME}:${TAG_TECH} -f Dockerfile.technical . - # Tag image for upload docker tag freqtrade:$TAG ${IMAGE_NAME}:$TAG if [ $? -ne 0 ]; then From 39efda19f4df040576ce55f33e60699a5e1dbe73 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 21 Oct 2018 13:33:23 +0200 Subject: [PATCH 171/699] Add freqtradeorg/freqtrade docker images to the documentation --- docs/installation.md | 20 +++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/docs/installation.md b/docs/installation.md index 1000600e6..15cfb1467 100644 --- a/docs/installation.md +++ b/docs/installation.md @@ -109,7 +109,25 @@ Dry-Run touch tradesv3.dryrun.sqlite ``` -### 2. Build the Docker image +### 2. Download or build the docker image + +Either use the prebuilt image from docker hub - or build the image yourself if you would like more control on which version is used. + +Branches / tags available can be checked out on [Dockerhub](https://hub.docker.com/r/freqtradeorg/freqtrade/tags/). + +#### 2.1. Download the docker image + +Pull the image from docker hub and (optionally) change the name of the image + +```bash +docker pull freqtradeorg/freqtrade:develop +# Optionally tag the repository so the run-commands remain shorter +docker tag freqtradeorg/freqtrade:develop freqtrade +``` + +To update the image, simply run the above commands again and restart your running container. + +#### 2.2. Build the Docker image ```bash cd freqtrade From 530d521d78bc9305face56e2fb8308c6b58f328c Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 21 Oct 2018 14:00:01 +0200 Subject: [PATCH 172/699] Rebuild complete image on "cron" events --- build_helpers/publish_docker.sh | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/build_helpers/publish_docker.sh b/build_helpers/publish_docker.sh index 5e0809fcf..c2b40ba81 100755 --- a/build_helpers/publish_docker.sh +++ b/build_helpers/publish_docker.sh @@ -2,12 +2,18 @@ # - export TAG=`if [ "$TRAVIS_BRANCH" == "develop" ]; then echo "latest"; else echo $TRAVIS_BRANCH ; fi` # Replace / with _ to create a valid tag TAG=$(echo "${TRAVIS_BRANCH}" | sed -e "s/\//_/") -TAG_TECH="${TAG}_technical" -# Pull last build to avoid rebuilding the whole image -docker pull ${REPO}:${TAG} -docker build --cache-from ${IMAGE_NAME}:${TAG} -t freqtrade:${TAG} . +if [ "${TRAVIS_EVENT_TYPE}" = "cron" ]; then + echo "event ${TRAVIS_EVENT_TYPE}: full rebuild - skipping cache" + docker build -t freqtrade:${TAG} . +else + echo "event ${TRAVIS_EVENT_TYPE}: building with cache" + # Pull last build to avoid rebuilding the whole image + docker pull ${REPO}:${TAG} + docker build --cache-from ${IMAGE_NAME}:${TAG} -t freqtrade:${TAG} . +fi + if [ $? -ne 0 ]; then echo "failed building image" return 1 @@ -29,7 +35,7 @@ if [ $? -ne 0 ]; then fi # Tag as latest for develop builds -if [ "${TRAVIS_BRANCH}" = "develop" ]; then +if [ "${TRAVIS_BRANCH}" = "develop" ]; then docker tag freqtrade:$TAG ${IMAGE_NAME}:latest fi From ee697d609cce6a81f55b5b02cdf10eebdf39d43a Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Sun, 21 Oct 2018 14:34:06 +0200 Subject: [PATCH 173/699] Update ccxt from 1.17.393 to 1.17.395 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index e750cf893..70759fc6f 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -ccxt==1.17.393 +ccxt==1.17.395 SQLAlchemy==1.2.12 python-telegram-bot==11.1.0 arrow==0.12.1 From 91dc8644bfb99f2eedd83c5563149f11ab610aa0 Mon Sep 17 00:00:00 2001 From: wingsgb Date: Mon, 22 Oct 2018 14:30:01 +1300 Subject: [PATCH 174/699] Update hyperopt.md --- docs/hyperopt.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/hyperopt.md b/docs/hyperopt.md index 7444d32b7..d1f363733 100644 --- a/docs/hyperopt.md +++ b/docs/hyperopt.md @@ -20,8 +20,8 @@ We recommend you start by taking a look at `hyperopt.py` file located in [freqtr ### Configure your Guards and Triggers There are two places you need to change to add a new buy strategy for testing: -- Inside [populate_buy_trend()](https://github.com/freqtrade/freqtrade/blob/develop/freqtrade/optimize/hyperopt.py#L278-L294). -- Inside [hyperopt_space()](https://github.com/freqtrade/freqtrade/blob/develop/freqtrade/optimize/hyperopt.py#L218-L229) +- Inside [populate_buy_trend()](https://github.com/freqtrade/freqtrade/blob/develop/freqtrade/optimize/hyperopt.py#L231-L264). +- Inside [hyperopt_space()](https://github.com/freqtrade/freqtrade/blob/develop/freqtrade/optimize/hyperopt.py#L213-L224) and the associated methods `indicator_space`, `roi_space`, `stoploss_space`. There you have two different type of indicators: 1. `guards` and 2. `triggers`. From 764aed2c37dee6b1676c9b96c9b8bd27fe8e7487 Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Mon, 22 Oct 2018 14:34:08 +0200 Subject: [PATCH 175/699] Update ccxt from 1.17.395 to 1.17.399 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 70759fc6f..a33a388d2 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -ccxt==1.17.395 +ccxt==1.17.399 SQLAlchemy==1.2.12 python-telegram-bot==11.1.0 arrow==0.12.1 From 49d1687229627d820c5ee999bd8644ca43df6c23 Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Tue, 23 Oct 2018 14:33:09 +0200 Subject: [PATCH 176/699] Update ccxt from 1.17.399 to 1.17.402 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index a33a388d2..69868ab27 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -ccxt==1.17.399 +ccxt==1.17.402 SQLAlchemy==1.2.12 python-telegram-bot==11.1.0 arrow==0.12.1 From 59545013c1150a2e27a0bfd14c745d270cc4b7cd Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Tue, 23 Oct 2018 14:33:10 +0200 Subject: [PATCH 177/699] Update numpy from 1.15.2 to 1.15.3 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 69868ab27..15929c089 100644 --- a/requirements.txt +++ b/requirements.txt @@ -10,7 +10,7 @@ pandas==0.23.4 scikit-learn==0.20.0 scipy==1.1.0 jsonschema==2.6.0 -numpy==1.15.2 +numpy==1.15.3 TA-Lib==0.4.17 pytest==3.9.1 pytest-mock==1.10.0 From b90392f9be285c823b90ed9ab473ba0348986f70 Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Tue, 23 Oct 2018 14:33:12 +0200 Subject: [PATCH 178/699] Update pytest from 3.9.1 to 3.9.2 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 15929c089..461125418 100644 --- a/requirements.txt +++ b/requirements.txt @@ -12,7 +12,7 @@ scipy==1.1.0 jsonschema==2.6.0 numpy==1.15.3 TA-Lib==0.4.17 -pytest==3.9.1 +pytest==3.9.2 pytest-mock==1.10.0 pytest-asyncio==0.9.0 pytest-cov==2.6.0 From 67ace0a76cfad000faee7c7ef4bf4ef7e1131ffc Mon Sep 17 00:00:00 2001 From: misagh Date: Tue, 23 Oct 2018 19:32:20 +0200 Subject: [PATCH 179/699] trade open time bug resolved (was behind of the market) --- freqtrade/edge/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/edge/__init__.py b/freqtrade/edge/__init__.py index 4e3c2c7c8..a5be19a73 100644 --- a/freqtrade/edge/__init__.py +++ b/freqtrade/edge/__init__.py @@ -358,7 +358,7 @@ class Edge(): 'stoploss': stoploss, 'profit_percent': '', 'profit_abs': '', - 'open_time': date_column[open_trade_index], + 'open_time': date_column[open_trade_index + 1], 'close_time': date_column[exit_index], 'open_index': start_point + open_trade_index + 1, 'close_index': start_point + exit_index, From b09a1d1abed8ce828e40166f00e8c7bf3d28f6a8 Mon Sep 17 00:00:00 2001 From: misagh Date: Tue, 23 Oct 2018 19:36:57 +0200 Subject: [PATCH 180/699] 1) do not download ta-lib as we have it offline. 2) removing ta-lib directory but not the file --- setup.sh | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/setup.sh b/setup.sh index 472260148..1bb2ba397 100755 --- a/setup.sh +++ b/setup.sh @@ -35,14 +35,13 @@ function updateenv () { # Install tab lib function install_talib () { - curl -O -L http://prdownloads.sourceforge.net/ta-lib/ta-lib-0.4.0-src.tar.gz tar zxvf ta-lib-0.4.0-src.tar.gz cd ta-lib sed -i.bak "s|0.00000001|0.000000000000000001 |g" src/ta_func/ta_utility.h ./configure --prefix=/usr/local make sudo make install - cd .. && rm -rf ./ta-lib* + cd .. && rm -rf ./ta-lib/ } # Install bot MacOS From 5c77dc6b3b212c3c21139ad2696500ea333f0a6b Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Wed, 24 Oct 2018 14:33:06 +0200 Subject: [PATCH 181/699] Update ccxt from 1.17.402 to 1.17.411 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 461125418..a08f55beb 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -ccxt==1.17.402 +ccxt==1.17.411 SQLAlchemy==1.2.12 python-telegram-bot==11.1.0 arrow==0.12.1 From c5474794d194f4583d95f300444f6b37d8ce60ad Mon Sep 17 00:00:00 2001 From: misagh Date: Thu, 25 Oct 2018 16:57:49 +0200 Subject: [PATCH 182/699] 1) open_trade_index refactored 2) sell index is shifted by 1 --- freqtrade/edge/__init__.py | 37 ++++++++++++++++++++++++++++--------- 1 file changed, 28 insertions(+), 9 deletions(-) diff --git a/freqtrade/edge/__init__.py b/freqtrade/edge/__init__.py index a5be19a73..5c074ceb1 100644 --- a/freqtrade/edge/__init__.py +++ b/freqtrade/edge/__init__.py @@ -50,7 +50,7 @@ class Edge(): self._last_updated = 0 self._timerange = Arguments.parse_timerange("%s-" % arrow.now().shift( - days=-1 * self._since_number_of_days).format('YYYYMMDD')) + days=-1 * self._since_number_of_days).format('YYYYMMDD')) self.fee = self.exchange.get_fee() @@ -277,8 +277,18 @@ class Edge(): if results.empty: return [] - groupby_aggregator = {'profit_abs': [winrate, risk_reward_ratio, required_risk_reward, expectancy, 'count'], 'trade_duration': ['mean']} - final = results.groupby(['pair', 'stoploss'])['profit_abs','trade_duration'].agg(groupby_aggregator).reset_index(col_level=1) + groupby_aggregator = { + 'profit_abs': [ + winrate, + risk_reward_ratio, + required_risk_reward, + expectancy, + 'count'], + 'trade_duration': ['mean']} + + final = results.groupby(['pair', 'stoploss'])['profit_abs', 'trade_duration'].agg( + groupby_aggregator).reset_index(col_level=1) + final.columns = final.columns.droplevel(0) final = final.sort_values(by=['expectancy', 'stoploss'], ascending=False).groupby( 'pair').first().sort_values(by=['expectancy'], ascending=False).reset_index() @@ -319,21 +329,24 @@ class Edge(): # we find a buy but at the of array if open_trade_index == -1 or open_trade_index == len(buy_column) - 1: return [] + else: + open_trade_index += 1 # when a buy signal is seen, + # trade opens in reality on the next candle stop_price_percentage = stoploss + 1 - open_price = ohlc_columns[open_trade_index + 1, 0] + open_price = ohlc_columns[open_trade_index, 0] stop_price = (open_price * stop_price_percentage) # Searching for the index where stoploss is hit stop_index = utf1st.find_1st( - ohlc_columns[open_trade_index + 1:, 2], stop_price, utf1st.cmp_smaller) + ohlc_columns[open_trade_index:, 2], stop_price, utf1st.cmp_smaller) # If we don't find it then we assume stop_index will be far in future (infinite number) if stop_index == -1: stop_index = float('inf') # Searching for the index where sell is hit - sell_index = utf1st.find_1st(sell_column[open_trade_index + 1:], 1, utf1st.cmp_equal) + sell_index = utf1st.find_1st(sell_column[open_trade_index:], 1, utf1st.cmp_equal) # If we don't find it then we assume sell_index will be far in future (infinite number) if sell_index == -1: @@ -346,11 +359,17 @@ class Edge(): return [] if stop_index <= sell_index: - exit_index = open_trade_index + stop_index + 1 + exit_index = open_trade_index + stop_index exit_type = SellType.STOP_LOSS exit_price = stop_price elif stop_index > sell_index: + # if exit is SELL then we exit at the next candle exit_index = open_trade_index + sell_index + 1 + + # check if we have the next candle + if len(ohlc_columns) - 1 < exit_index: + return [] + exit_type = SellType.SELL_SIGNAL exit_price = ohlc_columns[exit_index, 0] @@ -358,9 +377,9 @@ class Edge(): 'stoploss': stoploss, 'profit_percent': '', 'profit_abs': '', - 'open_time': date_column[open_trade_index + 1], + 'open_time': date_column[open_trade_index], 'close_time': date_column[exit_index], - 'open_index': start_point + open_trade_index + 1, + 'open_index': start_point + open_trade_index, 'close_index': start_point + exit_index, 'trade_duration': '', 'open_rate': round(open_price, 15), From dfeabcf7e56ca1c2eef69f9b07fc5ed3f378985a Mon Sep 17 00:00:00 2001 From: misagh Date: Thu, 25 Oct 2018 16:59:05 +0200 Subject: [PATCH 183/699] Edge tests template refactored to be more readable --- freqtrade/tests/edge/test_edge.py | 91 +++++++++++++++++-------------- 1 file changed, 51 insertions(+), 40 deletions(-) diff --git a/freqtrade/tests/edge/test_edge.py b/freqtrade/tests/edge/test_edge.py index 7d750cc31..d2f2b4b13 100644 --- a/freqtrade/tests/edge/test_edge.py +++ b/freqtrade/tests/edge/test_edge.py @@ -10,7 +10,7 @@ from unittest.mock import MagicMock # Cases to be tested: # SELL POINTS: -# 1) Three complete trades within dataframe (with sell hit for all) +# 1) Two complete trades within dataframe (with sell hit for all) # 2) Two complete trades but one without sell hit (remains open) # 3) Two complete trades and one buy signal while one trade is open # 4) Two complete trades with buy=1 on the last frame @@ -26,7 +26,8 @@ from unittest.mock import MagicMock #################################################################### ticker_start_time = arrow.get(2018, 10, 3) -ticker_interval_in_minute = 5 +ticker_interval_in_minute = 60 +_ohlc = {'date': 0, 'buy': 1, 'open': 2, 'high': 3, 'low': 4, 'close': 5, 'sell': 6, 'volume': 7} def test_filter(mocker, default_conf): @@ -72,7 +73,7 @@ def _build_dataframe(buy_ohlc_sell_matrice): for ohlc in buy_ohlc_sell_matrice: ticker = { # ticker every 5 min - 'date': ticker_start_time.shift(minutes=(ohlc[0] * 5)).timestamp * 1000, + 'date': ticker_start_time.shift(minutes=(ohlc[0] * ticker_interval_in_minute)).timestamp * 1000, 'buy': ohlc[1], 'open': ohlc[2], 'high': ohlc[3], @@ -90,6 +91,9 @@ def _build_dataframe(buy_ohlc_sell_matrice): return frame +def _time_on_candle(number): + return np.datetime64(ticker_start_time.shift( + minutes=(number * ticker_interval_in_minute)).timestamp * 1000, 'ms') def test_edge_heartbeat_calculate(mocker, default_conf): exchange = get_patched_exchange(mocker, default_conf) @@ -110,7 +114,7 @@ def mocked_load_data(datadir, pairs=[], ticker_interval='0m', refresh_pairs=Fals ETHBTC = [ [ ticker_start_time.shift(minutes=(x * ticker_interval_in_minute)).timestamp * 1000, - math.sin(x * hz) / 1000 + base, # But replace O,H,L,C + math.sin(x * hz) / 1000 + base, math.sin(x * hz) / 1000 + base + 0.0001, math.sin(x * hz) / 1000 + base - 0.0001, math.sin(x * hz) / 1000 + base, @@ -122,7 +126,7 @@ def mocked_load_data(datadir, pairs=[], ticker_interval='0m', refresh_pairs=Fals LTCBTC = [ [ ticker_start_time.shift(minutes=(x * ticker_interval_in_minute)).timestamp * 1000, - math.sin(x * hz) / 1000 + base, # But replace O,H,L,C + math.sin(x * hz) / 1000 + base, math.sin(x * hz) / 1000 + base + 0.0001, math.sin(x * hz) / 1000 + base - 0.0001, math.sin(x * hz) / 1000 + base, @@ -205,51 +209,58 @@ def test_process_expectancy(mocker, default_conf): # TODO: check expectancy + win rate etc -def test_three_complete_trades(mocker, default_conf): +def test_remove_open_trade_at_the_end(mocker, default_conf): exchange = get_patched_exchange(mocker, default_conf) edge = Edge(default_conf, exchange) - stoploss = -0.90 # we don't want stoploss to be hit in this test - three_sell_points_hit = [ - # Date, Buy, O, H, L, C, Sell - [1, 1, 15, 20, 12, 17, 0], # -> should enter the trade - [2, 0, 17, 18, 13, 14, 1], # -> should sell (trade 1 completed) - [3, 0, 14, 15, 11, 12, 0], # -> no action - [4, 1, 12, 25, 11, 20, 0], # -> should enter the trade - [5, 0, 20, 30, 19, 25, 1], # -> should sell (trade 2 completed) - [6, 1, 25, 27, 22, 26, 1], # -> buy and sell, should enter the trade - [7, 0, 26, 36, 25, 35, 1], # -> should sell (trade 3 completed) + stoploss = -0.99 # we don't want stoploss to be hit in this test + ticker = [ + #D=Date, B=Buy, O=Open, H=High, L=Low, C=Close, S=Sell + #D, B, O, H, L, C, S + [3, 1, 12, 25, 11, 20, 0], # -> + [4, 0, 20, 30, 19, 25, 1], # -> should enter the trade ] - ticker_df = _build_dataframe(three_sell_points_hit) + ticker_df = _build_dataframe(ticker) trades = edge._find_trades_for_stoploss_range(ticker_df, 'TEST/BTC', [stoploss]) - # Three trades must have occured - assert len(trades) == 3 + # No trade should be found + assert len(trades) == 0 - # First trade check - # open time should be on line 1 - assert trades[0]['open_time'] == np.datetime64(ticker_start_time.shift( - minutes=(1 * ticker_interval_in_minute)).timestamp * 1000, 'ms') +def test_two_complete_trades(mocker, default_conf): + exchange = get_patched_exchange(mocker, default_conf) + edge = Edge(default_conf, exchange) - # close time should be on line 2 - assert trades[0]['close_time'] == np.datetime64(ticker_start_time.shift( - minutes=(2 * ticker_interval_in_minute)).timestamp * 1000, 'ms') + stoploss = -0.99 # we don't want stoploss to be hit in this test + ticker = [ + #D=Date, B=Buy, O=Open, H=High, L=Low, C=Close, S=Sell + #D, B, O, H, L, C, S + [0, 1, 15, 20, 12, 17, 0], # -> no action + [1, 0, 17, 18, 13, 14, 1], # -> should enter the trade as B signal recieved on last candle + [2, 0, 14, 15, 11, 12, 0], # -> exit the trade as the sell signal recieved on last candle + [3, 1, 12, 25, 11, 20, 0], # -> no action + [4, 0, 20, 30, 19, 25, 0], # -> should enter the trade + [5, 0, 25, 27, 22, 26, 1], # -> no action + [6, 0, 26, 36, 25, 35, 0], # -> should sell + ] - # Second trade check - # open time should be on line 4 - assert trades[1]['open_time'] == np.datetime64(ticker_start_time.shift( - minutes=(4 * ticker_interval_in_minute)).timestamp * 1000, 'ms') + ticker_df = _build_dataframe(ticker) + ticker_df.to_json('/Users/misaghshakeri/Projects/freq/misagh/bslap_test_df.json') + trades = edge._find_trades_for_stoploss_range(ticker_df, 'TEST/BTC', [stoploss]) - # close time should be on line 5 - assert trades[1]['close_time'] == np.datetime64(ticker_start_time.shift( - minutes=(5 * ticker_interval_in_minute)).timestamp * 1000, 'ms') + # Two trades must have occured + assert len(trades) == 2 - # Third trade check - # open time should be on line 6 - assert trades[2]['open_time'] == np.datetime64(ticker_start_time.shift( - minutes=(6 * ticker_interval_in_minute)).timestamp * 1000, 'ms') + ################### First trade check ######################## + assert trades[0]['open_time'] == _time_on_candle(1) + assert trades[0]['close_time'] == _time_on_candle(2) + assert trades[0]['open_rate'] == ticker[1][_ohlc['open']] + assert trades[0]['close_rate'] == ticker[2][_ohlc['open']] + ############################################################## - # close time should be on line 7 - assert trades[2]['close_time'] == np.datetime64(ticker_start_time.shift( - minutes=(7 * ticker_interval_in_minute)).timestamp * 1000, 'ms') + ################### Second trade check ######################## + assert trades[1]['open_time'] == _time_on_candle(4) + assert trades[1]['close_time'] == _time_on_candle(6) + assert trades[1]['open_rate'] == ticker[4][_ohlc['open']] + assert trades[1]['close_rate'] == ticker[6][_ohlc['open']] + ############################################################## From 426db721260e209ffabcc49599da035da74931cd Mon Sep 17 00:00:00 2001 From: misagh Date: Thu, 25 Oct 2018 17:24:33 +0200 Subject: [PATCH 184/699] removing test line --- freqtrade/tests/edge/test_edge.py | 49 ++++++++++++++++++------------- 1 file changed, 29 insertions(+), 20 deletions(-) diff --git a/freqtrade/tests/edge/test_edge.py b/freqtrade/tests/edge/test_edge.py index d2f2b4b13..ec8af1200 100644 --- a/freqtrade/tests/edge/test_edge.py +++ b/freqtrade/tests/edge/test_edge.py @@ -1,6 +1,7 @@ from freqtrade.tests.conftest import get_patched_exchange from freqtrade.edge import Edge from pandas import DataFrame, to_datetime +from freqtrade.strategy.interface import SellType import arrow import numpy as np import math @@ -9,20 +10,22 @@ from unittest.mock import MagicMock # Cases to be tested: -# SELL POINTS: -# 1) Two complete trades within dataframe (with sell hit for all) -# 2) Two complete trades but one without sell hit (remains open) -# 3) Two complete trades and one buy signal while one trade is open -# 4) Two complete trades with buy=1 on the last frame +# 1) Open trade should be removed from the end +# 2) Two complete trades within dataframe (with sell hit for all) +# 3) Entered, sl 1%, candle drops 8% => Trade closed, 1% loss +# 4) Entered, sl 3%, candle drops 4%, recovers to 1% => Trade closed, 3% loss +# 5) Entered, sl 2%, candle drops 4%, recovers to 1%, entry met, candle drops 20% => +# Trade 1 closed: loss 2%, Trade 2 opened, Trade 2 closed: loss 2% +# 6) ################################################################### # STOPLOSS: -# 5) Candle drops 8%, stoploss at 1%: Trade closed, 1% loss -# 6) Candle drops 4% but recovers to 1% loss, stoploss at 3%: Trade closed, 3% loss -# 7) Candle drops 4% recovers to 1% entry criteria are met, candle drops +# 6) Candle drops 8%, stoploss at 1%: Trade closed, 1% loss +# 7) Candle drops 4% but recovers to 1% loss, stoploss at 3%: Trade closed, 3% loss +# 8) Candle drops 4% recovers to 1% entry criteria are met, candle drops # 20%, stoploss at 2%: Trade 1 closed, Loss 2%, Trade 2 opened, Trade 2 closed, Loss 2% #################################################################### # PRIORITY TO STOPLOSS: -# 8) Stoploss and sell are hit. should sell on stoploss +# 9) Stoploss and sell are hit. should sell on stoploss #################################################################### ticker_start_time = arrow.get(2018, 10, 3) @@ -72,15 +75,17 @@ def _build_dataframe(buy_ohlc_sell_matrice): tickers = [] for ohlc in buy_ohlc_sell_matrice: ticker = { - # ticker every 5 min - 'date': ticker_start_time.shift(minutes=(ohlc[0] * ticker_interval_in_minute)).timestamp * 1000, + 'date': ticker_start_time.shift( + minutes=( + ohlc[0] * + ticker_interval_in_minute)).timestamp * + 1000, 'buy': ohlc[1], 'open': ohlc[2], 'high': ohlc[3], 'low': ohlc[4], 'close': ohlc[5], - 'sell': ohlc[6] - } + 'sell': ohlc[6]} tickers.append(ticker) frame = DataFrame(tickers) @@ -91,10 +96,12 @@ def _build_dataframe(buy_ohlc_sell_matrice): return frame + def _time_on_candle(number): return np.datetime64(ticker_start_time.shift( minutes=(number * ticker_interval_in_minute)).timestamp * 1000, 'ms') + def test_edge_heartbeat_calculate(mocker, default_conf): exchange = get_patched_exchange(mocker, default_conf) edge = Edge(default_conf, exchange) @@ -215,8 +222,8 @@ def test_remove_open_trade_at_the_end(mocker, default_conf): stoploss = -0.99 # we don't want stoploss to be hit in this test ticker = [ - #D=Date, B=Buy, O=Open, H=High, L=Low, C=Close, S=Sell - #D, B, O, H, L, C, S + # D=Date, B=Buy, O=Open, H=High, L=Low, C=Close, S=Sell + # D, B, O, H, L, C, S [3, 1, 12, 25, 11, 20, 0], # -> [4, 0, 20, 30, 19, 25, 1], # -> should enter the trade ] @@ -227,14 +234,15 @@ def test_remove_open_trade_at_the_end(mocker, default_conf): # No trade should be found assert len(trades) == 0 + def test_two_complete_trades(mocker, default_conf): exchange = get_patched_exchange(mocker, default_conf) edge = Edge(default_conf, exchange) stoploss = -0.99 # we don't want stoploss to be hit in this test ticker = [ - #D=Date, B=Buy, O=Open, H=High, L=Low, C=Close, S=Sell - #D, B, O, H, L, C, S + # D=Date, B=Buy, O=Open, H=High, L=Low, C=Close, S=Sell + # D, B, O, H, L, C, S [0, 1, 15, 20, 12, 17, 0], # -> no action [1, 0, 17, 18, 13, 14, 1], # -> should enter the trade as B signal recieved on last candle [2, 0, 14, 15, 11, 12, 0], # -> exit the trade as the sell signal recieved on last candle @@ -245,22 +253,23 @@ def test_two_complete_trades(mocker, default_conf): ] ticker_df = _build_dataframe(ticker) - ticker_df.to_json('/Users/misaghshakeri/Projects/freq/misagh/bslap_test_df.json') trades = edge._find_trades_for_stoploss_range(ticker_df, 'TEST/BTC', [stoploss]) # Two trades must have occured assert len(trades) == 2 - ################### First trade check ######################## + # First trade check assert trades[0]['open_time'] == _time_on_candle(1) assert trades[0]['close_time'] == _time_on_candle(2) assert trades[0]['open_rate'] == ticker[1][_ohlc['open']] assert trades[0]['close_rate'] == ticker[2][_ohlc['open']] + assert trades[0]['exit_type'] == SellType.SELL_SIGNAL ############################################################## - ################### Second trade check ######################## + # Second trade check assert trades[1]['open_time'] == _time_on_candle(4) assert trades[1]['close_time'] == _time_on_candle(6) assert trades[1]['open_rate'] == ticker[4][_ohlc['open']] assert trades[1]['close_rate'] == ticker[6][_ohlc['open']] + assert trades[1]['exit_type'] == SellType.SELL_SIGNAL ############################################################## From f860aab094019e3a5fabe7c0861bda945e76b194 Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Fri, 26 Oct 2018 14:33:07 +0200 Subject: [PATCH 185/699] Update ccxt from 1.17.411 to 1.17.421 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index a08f55beb..40ab21f61 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -ccxt==1.17.411 +ccxt==1.17.421 SQLAlchemy==1.2.12 python-telegram-bot==11.1.0 arrow==0.12.1 From 57d3a6f7a7c39faeee577481025e5a6bca73d7e5 Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Sat, 27 Oct 2018 14:33:06 +0200 Subject: [PATCH 186/699] Update ccxt from 1.17.421 to 1.17.427 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 40ab21f61..79662c483 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -ccxt==1.17.421 +ccxt==1.17.427 SQLAlchemy==1.2.12 python-telegram-bot==11.1.0 arrow==0.12.1 From a4fc5afb66bc6adf3e638469b2c1196ca1194ac2 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 27 Oct 2018 17:35:08 +0200 Subject: [PATCH 187/699] Add hyperopt ROI documentation, add note on methology for hyperopt --- docs/hyperopt.md | 62 ++++++++++++++++++++++++++++++++++++++++-------- 1 file changed, 52 insertions(+), 10 deletions(-) diff --git a/docs/hyperopt.md b/docs/hyperopt.md index d1f363733..51fcbfaad 100644 --- a/docs/hyperopt.md +++ b/docs/hyperopt.md @@ -1,4 +1,5 @@ # Hyperopt + This page explains how to tune your strategy by finding the optimal parameters, a process called hyperparameter optimization. The bot uses several algorithms included in the `scikit-optimize` package to accomplish this. The @@ -8,17 +9,20 @@ and still take a long time. *Note:* Hyperopt will crash when used with only 1 CPU Core as found out in [Issue #1133](https://github.com/freqtrade/freqtrade/issues/1133) ## Table of Contents + - [Prepare your Hyperopt](#prepare-hyperopt) - [Configure your Guards and Triggers](#configure-your-guards-and-triggers) - [Solving a Mystery](#solving-a-mystery) - [Adding New Indicators](#adding-new-indicators) - [Execute Hyperopt](#execute-hyperopt) -- [Understand the hyperopts result](#understand-the-backtesting-result) +- [Understand the hyperopt result](#understand-the-hyperopt-result) ## Prepare Hyperopting + We recommend you start by taking a look at `hyperopt.py` file located in [freqtrade/optimize](https://github.com/freqtrade/freqtrade/blob/develop/freqtrade/optimize/hyperopt.py) ### Configure your Guards and Triggers + There are two places you need to change to add a new buy strategy for testing: - Inside [populate_buy_trend()](https://github.com/freqtrade/freqtrade/blob/develop/freqtrade/optimize/hyperopt.py#L231-L264). - Inside [hyperopt_space()](https://github.com/freqtrade/freqtrade/blob/develop/freqtrade/optimize/hyperopt.py#L213-L224) @@ -113,11 +117,12 @@ When you want to test an indicator that isn't used by the bot currently, remembe add it to the `populate_indicators()` method in `hyperopt.py`. ## Execute Hyperopt -Once you have updated your hyperopt configuration you can run it. -Because hyperopt tries a lot of combination to find the best parameters -it will take time you will have the result (more than 30 mins). -We strongly recommend to use `screen` to prevent any connection loss. +Once you have updated your hyperopt configuration you can run it. +Because hyperopt tries a lot of combinations to find the best parameters it will take time you will have the result (more than 30 mins). + +We strongly recommend to use `screen` or `tmux` to prevent any connection loss. + ```bash python3 ./freqtrade/main.py -c config.json hyperopt -e 5000 ``` @@ -126,11 +131,13 @@ The `-e` flag will set how many evaluations hyperopt will do. We recommend running at least several thousand evaluations. ### Execute Hyperopt with Different Ticker-Data Source + If you would like to hyperopt parameters using an alternate ticker data that you have on-disk, use the `--datadir PATH` option. Default hyperopt will use data from directory `user_data/data`. ### Running Hyperopt with Smaller Testset + Use the `--timerange` argument to change how much of the testset you want to use. The last N ticks/timeframes will be used. Example: @@ -140,6 +147,7 @@ python3 ./freqtrade/main.py hyperopt --timerange -200 ``` ### Running Hyperopt with Smaller Search Space + Use the `--spaces` argument to limit the search space used by hyperopt. Letting Hyperopt optimize everything is a huuuuge search space. Often it might make more sense to start by just searching for initial buy algorithm. @@ -154,7 +162,8 @@ Legal values are: - `stoploss`: search for the best stoploss value - space-separated list of any of the above values for example `--spaces roi stoploss` -## Understand the Hyperopts Result +## Understand the Hyperopt Result + Once Hyperopt is completed you can use the result to create a new strategy. Given the following result from hyperopt: @@ -166,22 +175,24 @@ with values: ``` You should understand this result like: + - The buy trigger that worked best was `bb_lower`. - You should not use ADX because `adx-enabled: False`) - You should **consider** using the RSI indicator (`rsi-enabled: True` and the best value is `29.0` (`rsi-value: 29.0`) You have to look inside your strategy file into `buy_strategy_generator()` -method, what those values match to. +method, what those values match to. -So for example you had `rsi-value: 29.0` so we would look -at `rsi`-block, that translates to the following code block: +So for example you had `rsi-value: 29.0` so we would look at `rsi`-block, that translates to the following code block: + ``` (dataframe['rsi'] < 29.0) ``` Translating your whole hyperopt result as the new buy-signal would then look like: -``` + +```python def populate_buy_trend(self, dataframe: DataFrame) -> DataFrame: dataframe.loc[ ( @@ -192,6 +203,37 @@ def populate_buy_trend(self, dataframe: DataFrame) -> DataFrame: return dataframe ``` +### Understand Hyperopt ROI results + +If you are optimizing ROI, you're result will look as follows and include a ROI table. + +``` +Best result: + 135 trades. Avg profit 0.57%. Total profit 0.03871918 BTC (0.7722Σ%). Avg duration 180.4 mins. +with values: +{'adx-value': 44, 'rsi-value': 29, 'adx-enabled': False, 'rsi-enabled': True, 'trigger': 'bb_lower', 'roi_t1': 40, 'roi_t2': 57, 'roi_t3': 21, 'roi_p1': 0.03634636907306948, 'roi_p2': 0.055237357937802885, 'roi_p3': 0.015163796015548354, 'stoploss': -0.37996664668703606} +ROI table: +{0: 0.10674752302642071, 21: 0.09158372701087236, 78: 0.03634636907306948, 118: 0} +``` + +This would translate to the following ROI table: + +``` python + minimal_roi = { + "118": 0, + "78": 0.0363463, + "21": 0.0915, + "0": 0.106 + } +``` + +### Validate backtest result + +Once the optimized strategy has been implemented into your strategy, you should backtest this strategy to make sure everything is working as expected. +To archive the same results (number of trades, ...) than during hyperopt, please use the command line flag `--disable-max-market-positions`. +This setting is the default for hyperopt for speed reasons. you can overwrite this in the configuration by setting `"position_stacking"=false` or by changing the relevant line in your hyperopt file [here](https://github.com/freqtrade/freqtrade/blob/develop/freqtrade/optimize/hyperopt.py#L283) + ## Next Step + Now you have a perfect bot and want to control it from Telegram. Your next step is to learn the [Telegram usage](https://github.com/freqtrade/freqtrade/blob/develop/docs/telegram-usage.md). From 7e4a0baef252d24d50564061e25433a80a8b0594 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 27 Oct 2018 17:38:15 +0200 Subject: [PATCH 188/699] improve hyperopt.md --- docs/hyperopt.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/docs/hyperopt.md b/docs/hyperopt.md index 51fcbfaad..e2dcf3e95 100644 --- a/docs/hyperopt.md +++ b/docs/hyperopt.md @@ -231,7 +231,9 @@ This would translate to the following ROI table: Once the optimized strategy has been implemented into your strategy, you should backtest this strategy to make sure everything is working as expected. To archive the same results (number of trades, ...) than during hyperopt, please use the command line flag `--disable-max-market-positions`. -This setting is the default for hyperopt for speed reasons. you can overwrite this in the configuration by setting `"position_stacking"=false` or by changing the relevant line in your hyperopt file [here](https://github.com/freqtrade/freqtrade/blob/develop/freqtrade/optimize/hyperopt.py#L283) +This setting is the default for hyperopt for speed reasons. You can overwrite this in the configuration by setting `"position_stacking"=false` or by changing the relevant line in your hyperopt file [here](https://github.com/freqtrade/freqtrade/blob/develop/freqtrade/optimize/hyperopt.py#L283). + +Dry/live runs will **NOT** use position stacking - therefore it does make sense to also validate the strategy without this as it's closer to reality. ## Next Step From 551dc79cf7692191b797351fb03ebf67a5771c2b Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 28 Oct 2018 13:15:49 +0100 Subject: [PATCH 189/699] Don't overwrite pair_whitelist in config dict Doing that will result in an empty whitelist after a short Exchange downtime --- freqtrade/freqtradebot.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index fa803bda7..3881f78d1 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -54,6 +54,7 @@ class FreqtradeBot(object): self.rpc: RPCManager = RPCManager(self) self.persistence = None self.exchange = Exchange(self.config) + self.active_pair_whitelist: List[str] = self.config['exchange']['pair_whitelist'] self._init_modules() def _init_modules(self) -> None: @@ -179,11 +180,10 @@ class FreqtradeBot(object): ) # Keep only the subsets of pairs wanted (up to nb_assets) - final_list = sanitized_list[:nb_assets] if nb_assets else sanitized_list - self.config['exchange']['pair_whitelist'] = final_list + self.active_pair_whitelist = sanitized_list[:nb_assets] if nb_assets else sanitized_list # Refreshing candles - self.exchange.refresh_tickers(final_list, self.strategy.ticker_interval) + self.exchange.refresh_tickers(self.active_pair_whitelist, self.strategy.ticker_interval) # Query trades from persistence layer trades = Trade.query.filter(Trade.is_open.is_(True)).all() @@ -380,7 +380,7 @@ class FreqtradeBot(object): 'Checking buy signals to create a new trade with stake_amount: %f ...', stake_amount ) - whitelist = copy.deepcopy(self.config['exchange']['pair_whitelist']) + whitelist = copy.deepcopy(self.active_pair_whitelist) # Remove currently opened and latest pairs from whitelist for trade in Trade.query.filter(Trade.is_open.is_(True)).all(): From d3387dec456b3dfafacfcc29c11a1b34ed5d4ffd Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Sun, 28 Oct 2018 13:33:09 +0100 Subject: [PATCH 190/699] Update ccxt from 1.17.427 to 1.17.429 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 79662c483..e69ca09c2 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -ccxt==1.17.427 +ccxt==1.17.429 SQLAlchemy==1.2.12 python-telegram-bot==11.1.0 arrow==0.12.1 From 86ad0c047c79bc7f38b86d08fb34b003faa5ac90 Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Sun, 28 Oct 2018 13:33:10 +0100 Subject: [PATCH 191/699] Update pytest from 3.9.2 to 3.9.3 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index e69ca09c2..d9eb58c25 100644 --- a/requirements.txt +++ b/requirements.txt @@ -12,7 +12,7 @@ scipy==1.1.0 jsonschema==2.6.0 numpy==1.15.3 TA-Lib==0.4.17 -pytest==3.9.2 +pytest==3.9.3 pytest-mock==1.10.0 pytest-asyncio==0.9.0 pytest-cov==2.6.0 From 1121ec0724f0d69ee1d7c8b5791882cd8e6db01e Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 28 Oct 2018 14:43:35 +0100 Subject: [PATCH 192/699] don't have nb_assets as parameter - it's a config setting as any other --- freqtrade/freqtradebot.py | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index 3881f78d1..ca503c89c 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -108,11 +108,8 @@ class FreqtradeBot(object): constants.PROCESS_THROTTLE_SECS ) - nb_assets = self.config.get('dynamic_whitelist', None) - self._throttle(func=self._process, - min_secs=min_secs, - nb_assets=nb_assets) + min_secs=min_secs) return state def _startup_messages(self) -> None: @@ -163,15 +160,15 @@ class FreqtradeBot(object): time.sleep(duration) return result - def _process(self, nb_assets: Optional[int] = 0) -> bool: + def _process(self) -> bool: """ Queries the persistence layer for open trades and handles them, otherwise a new trade is created. - :param: nb_assets: the maximum number of pairs to be traded at the same time :return: True if one or more trades has been created or closed, False otherwise """ state_changed = False try: + nb_assets = self.config.get('dynamic_whitelist', None) # Refresh whitelist based on wallet maintenance sanitized_list = self._refresh_whitelist( self._gen_pair_whitelist( From 35759b372d336ba6d6add798b439c72eefe01590 Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Mon, 29 Oct 2018 13:33:10 +0100 Subject: [PATCH 193/699] Update ccxt from 1.17.429 to 1.17.432 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index d9eb58c25..da382e82b 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -ccxt==1.17.429 +ccxt==1.17.432 SQLAlchemy==1.2.12 python-telegram-bot==11.1.0 arrow==0.12.1 From 2f55cbde35ebc839df94e414e7ddd02554a9c5be Mon Sep 17 00:00:00 2001 From: Matthias Date: Mon, 29 Oct 2018 19:23:56 +0100 Subject: [PATCH 194/699] fix #1298 --- freqtrade/freqtradebot.py | 11 +++++-- freqtrade/tests/test_freqtradebot.py | 46 ++++++++++++++++++++++++++++ 2 files changed, 54 insertions(+), 3 deletions(-) diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index ca503c89c..6cc46a07e 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -179,12 +179,17 @@ class FreqtradeBot(object): # Keep only the subsets of pairs wanted (up to nb_assets) self.active_pair_whitelist = sanitized_list[:nb_assets] if nb_assets else sanitized_list - # Refreshing candles - self.exchange.refresh_tickers(self.active_pair_whitelist, self.strategy.ticker_interval) - # Query trades from persistence layer trades = Trade.query.filter(Trade.is_open.is_(True)).all() + # Extend active-pair whitelist with pairs from open trades + # ensures that tickers are downloaded for open trades + self.active_pair_whitelist.extend([trade.pair for trade in trades + if trade.pair not in self.active_pair_whitelist]) + + # Refreshing candles + self.exchange.refresh_tickers(self.active_pair_whitelist, self.strategy.ticker_interval) + # First process current opened trades for trade in trades: state_changed |= self.process_maybe_execute_sell(trade) diff --git a/freqtrade/tests/test_freqtradebot.py b/freqtrade/tests/test_freqtradebot.py index 6b13da35f..871e59240 100644 --- a/freqtrade/tests/test_freqtradebot.py +++ b/freqtrade/tests/test_freqtradebot.py @@ -663,6 +663,52 @@ def test_process_trade_handling( assert result is False +def test_process_trade_no_whitelist_pair( + default_conf, ticker, limit_buy_order, markets, fee, mocker) -> None: + """ Test _process with trade not in pair list """ + patch_RPCManager(mocker) + patch_exchange(mocker) + mocker.patch.multiple( + 'freqtrade.exchange.Exchange', + get_ticker=ticker, + get_markets=markets, + buy=MagicMock(return_value={'id': limit_buy_order['id']}), + get_order=MagicMock(return_value=limit_buy_order), + get_fee=fee, + ) + freqtrade = FreqtradeBot(default_conf) + patch_get_signal(freqtrade) + pair = 'NOCLUE/BTC' + # create open trade not in whitelist + Trade.session.add(Trade( + pair=pair, + stake_amount=0.001, + fee_open=fee.return_value, + fee_close=fee.return_value, + is_open=True, + amount=20, + open_rate=0.01, + exchange='bittrex', + )) + Trade.session.add(Trade( + pair='ETH/BTC', + stake_amount=0.001, + fee_open=fee.return_value, + fee_close=fee.return_value, + is_open=True, + amount=12, + open_rate=0.001, + exchange='bittrex', + )) + + assert pair not in freqtrade.active_pair_whitelist + result = freqtrade._process() + assert pair in freqtrade.active_pair_whitelist + # Make sure each pair is only in the list once + assert len(freqtrade.active_pair_whitelist) == len(set(freqtrade.active_pair_whitelist)) + assert result is True + + def test_balance_fully_ask_side(mocker, default_conf) -> None: default_conf['bid_strategy']['ask_last_balance'] = 0.0 freqtrade = get_patched_freqtradebot(mocker, default_conf) From e0fda7a5ddac9624276256343975a5ba17a9d0d4 Mon Sep 17 00:00:00 2001 From: Matthias Date: Mon, 9 Jul 2018 21:38:49 +0200 Subject: [PATCH 195/699] Add tests validating backtest details --- .../tests/optimize/test_backtest_detail.py | 77 +++++++++++++++++++ 1 file changed, 77 insertions(+) create mode 100644 freqtrade/tests/optimize/test_backtest_detail.py diff --git a/freqtrade/tests/optimize/test_backtest_detail.py b/freqtrade/tests/optimize/test_backtest_detail.py new file mode 100644 index 000000000..fb88fe2a7 --- /dev/null +++ b/freqtrade/tests/optimize/test_backtest_detail.py @@ -0,0 +1,77 @@ +# pragma pylint: disable=missing-docstring, W0212, line-too-long, C0103, unused-argument +import logging +from unittest.mock import MagicMock + +import pandas as pd +import pytest +from arrow import get as getdate + +from freqtrade.optimize.backtesting import Backtesting +from freqtrade.tests.conftest import patch_exchange, log_has + + +columns = ['date', 'open', 'high', 'low', 'close', 'volume', 'buy', 'sell'] +data_profit = pd.DataFrame([[getdate('2018-07-08 18:00:00').datetime, + 0.0009910, 0.001011, 0.00098618, 0.001000, 47027.0, 1, 0], + [getdate('2018-07-08 19:00:00').datetime, + 0.001000, 0.001010, 0.0009900, 0.0009900, 87116.0, 0, 0], + [getdate('2018-07-08 20:00:00').datetime, + 0.0009900, 0.001011, 0.00091618, 0.0009900, 58539.0, 0, 0], + [getdate('2018-07-08 21:00:00').datetime, + 0.001000, 0.001011, 0.00098618, 0.001100, 37498.0, 0, 1], + [getdate('2018-07-08 22:00:00').datetime, + 0.001000, 0.001011, 0.00098618, 0.0009900, 59792.0, 0, 0]], + columns=columns) + +data_loss = pd.DataFrame([[getdate('2018-07-08 18:00:00').datetime, + 0.0009910, 0.001011, 0.00098618, 0.001000, 47027.0, 1, 0], + [getdate('2018-07-08 19:00:00').datetime, + 0.001000, 0.001010, 0.0009900, 0.001000, 87116.0, 0, 0], + [getdate('2018-07-08 20:00:00').datetime, + 0.001000, 0.001011, 0.0010618, 0.00091618, 58539.0, 0, 0], + [getdate('2018-07-08 21:00:00').datetime, + 0.001000, 0.001011, 0.00098618, 0.00091618, 37498.0, 0, 0], + [getdate('2018-07-08 22:00:00').datetime, + 0.001000, 0.001011, 0.00098618, 0.00091618, 59792.0, 0, 0]], + columns=columns) + + +@pytest.mark.parametrize("data, stoploss, tradecount, profit_perc, sl", [ + (data_profit, -0.01, 1, 0.10557, False), # should be stoploss - drops 8% + # (data_profit, -0.10, 1, 0.10557, True), # win + (data_loss, -0.05, 1, -0.08839, True), # Stoploss ... + ]) +def test_backtest_results(default_conf, fee, mocker, caplog, + data, stoploss, tradecount, profit_perc, sl) -> None: + """ + run functional tests + """ + default_conf["stoploss"] = stoploss + mocker.patch('freqtrade.exchange.Exchange.get_fee', fee) + mocker.patch('freqtrade.analyze.Analyze.populate_sell_trend', MagicMock(return_value=data)) + mocker.patch('freqtrade.analyze.Analyze.populate_buy_trend', MagicMock(return_value=data)) + patch_exchange(mocker) + + backtesting = Backtesting(default_conf) + caplog.set_level(logging.DEBUG) + + pair = 'UNITTEST/BTC' + # Dummy data as we mock the analyze functions + data_processed = {pair: pd.DataFrame()} + results = backtesting.backtest( + { + 'stake_amount': default_conf['stake_amount'], + 'processed': data_processed, + 'max_open_trades': 10, + 'realistic': True + } + ) + print(results.T) + + assert len(results) == tradecount + assert round(results["profit_percent"].sum(), 5) == profit_perc + if sl: + assert log_has("Stop loss hit.", caplog.record_tuples) + else: + + assert not log_has("Stop loss hit.", caplog.record_tuples) From b8f78cb1878ef4fff90353d63305a0dc56be19dd Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 10 Jul 2018 21:08:44 +0200 Subject: [PATCH 196/699] Refactor tests, implement @creslinux's data --- .../tests/optimize/test_backtest_detail.py | 196 ++++++++++++++---- 1 file changed, 159 insertions(+), 37 deletions(-) diff --git a/freqtrade/tests/optimize/test_backtest_detail.py b/freqtrade/tests/optimize/test_backtest_detail.py index fb88fe2a7..5023f4b24 100644 --- a/freqtrade/tests/optimize/test_backtest_detail.py +++ b/freqtrade/tests/optimize/test_backtest_detail.py @@ -1,55 +1,177 @@ # pragma pylint: disable=missing-docstring, W0212, line-too-long, C0103, unused-argument import logging from unittest.mock import MagicMock +from typing import NamedTuple -import pandas as pd +from pandas import DataFrame import pytest from arrow import get as getdate + from freqtrade.optimize.backtesting import Backtesting from freqtrade.tests.conftest import patch_exchange, log_has +class BTContainer(NamedTuple): + """ + NamedTuple Defining BacktestResults inputs. + """ + data: DataFrame + stop_loss: float + roi: float + trades: int + profit_perc: float + sl: float + + columns = ['date', 'open', 'high', 'low', 'close', 'volume', 'buy', 'sell'] -data_profit = pd.DataFrame([[getdate('2018-07-08 18:00:00').datetime, - 0.0009910, 0.001011, 0.00098618, 0.001000, 47027.0, 1, 0], - [getdate('2018-07-08 19:00:00').datetime, - 0.001000, 0.001010, 0.0009900, 0.0009900, 87116.0, 0, 0], - [getdate('2018-07-08 20:00:00').datetime, - 0.0009900, 0.001011, 0.00091618, 0.0009900, 58539.0, 0, 0], - [getdate('2018-07-08 21:00:00').datetime, - 0.001000, 0.001011, 0.00098618, 0.001100, 37498.0, 0, 1], - [getdate('2018-07-08 22:00:00').datetime, - 0.001000, 0.001011, 0.00098618, 0.0009900, 59792.0, 0, 0]], - columns=columns) +data_profit = DataFrame([ + [getdate('2018-07-08 18:00:00').datetime, 0.0009910, + 0.001011, 0.00098618, 0.001000, 12345, 1, 0], + [getdate('2018-07-08 19:00:00').datetime, 0.001000, + 0.001010, 0.0009900, 0.0009900, 12345, 0, 0], + [getdate('2018-07-08 20:00:00').datetime, 0.0009900, + 0.001011, 0.00091618, 0.0009900, 12345, 0, 0], + [getdate('2018-07-08 21:00:00').datetime, 0.001000, + 0.001011, 0.00098618, 0.001100, 12345, 0, 1], + [getdate('2018-07-08 22:00:00').datetime, 0.001000, + 0.001011, 0.00098618, 0.0009900, 12345, 0, 0] +], columns=columns) -data_loss = pd.DataFrame([[getdate('2018-07-08 18:00:00').datetime, - 0.0009910, 0.001011, 0.00098618, 0.001000, 47027.0, 1, 0], - [getdate('2018-07-08 19:00:00').datetime, - 0.001000, 0.001010, 0.0009900, 0.001000, 87116.0, 0, 0], - [getdate('2018-07-08 20:00:00').datetime, - 0.001000, 0.001011, 0.0010618, 0.00091618, 58539.0, 0, 0], - [getdate('2018-07-08 21:00:00').datetime, - 0.001000, 0.001011, 0.00098618, 0.00091618, 37498.0, 0, 0], - [getdate('2018-07-08 22:00:00').datetime, - 0.001000, 0.001011, 0.00098618, 0.00091618, 59792.0, 0, 0]], - columns=columns) +tc_profit1 = BTContainer(data=data_profit, stop_loss=-0.01, roi=1, trades=1, + profit_perc=0.10557, sl=False) # should be stoploss - drops 8% +tc_profit2 = BTContainer(data=data_profit, stop_loss=-0.10, roi=1, + trades=1, profit_perc=0.10557, sl=True) -@pytest.mark.parametrize("data, stoploss, tradecount, profit_perc, sl", [ - (data_profit, -0.01, 1, 0.10557, False), # should be stoploss - drops 8% - # (data_profit, -0.10, 1, 0.10557, True), # win - (data_loss, -0.05, 1, -0.08839, True), # Stoploss ... - ]) -def test_backtest_results(default_conf, fee, mocker, caplog, - data, stoploss, tradecount, profit_perc, sl) -> None: +tc_loss0 = BTContainer(data=DataFrame([ + [getdate('2018-07-08 18:00:00').datetime, 0.0009910, + 0.001011, 0.00098618, 0.001000, 12345, 1, 0], + [getdate('2018-07-08 19:00:00').datetime, 0.001000, + 0.001010, 0.0009900, 0.001000, 12345, 0, 0], + [getdate('2018-07-08 20:00:00').datetime, 0.001000, + 0.001011, 0.0010618, 0.00091618, 12345, 0, 0], + [getdate('2018-07-08 21:00:00').datetime, 0.001000, + 0.001011, 0.00098618, 0.00091618, 12345, 0, 0], + [getdate('2018-07-08 22:00:00').datetime, 0.001000, + 0.001011, 0.00098618, 0.00091618, 12345, 0, 0] +], columns=columns), + stop_loss=-0.05, roi=1, trades=1, profit_perc=-0.08839, sl=True) + + +# Test 1 Minus 8% Close +# Candle Data for test 1 – close at -8% (9200) +# Test with Stop-loss at 1% +tc1 = BTContainer(data=DataFrame([ + [getdate('2018-06-10 08:00:00').datetime, 10000, 10050, 9950, 9975, 12345, 1, 0], + [getdate('2018-06-10 09:00:00').datetime, 9975, 10025, 9925, 9950, 12345, 0, 0], + [getdate('2018-06-10 10:00:00').datetime, 9950, 10000, 9960, 9955, 12345, 0, 0], + [getdate('2018-06-10 11:00:00').datetime, 9955, 9975, 9955, 9990, 12345, 0, 0], + [getdate('2018-06-10 12:00:00').datetime, 9990, 9990, 9200, 9200, 12345, 0, 0] +], columns=columns), + stop_loss=-0.01, roi=1, trades=1, profit_perc=-0.07999, sl=True) + +# Test 2 Minus 4% Low, minus 1% close +# Candle Data for test 2 +# Test with Stop-Loss at 3% +tc2 = BTContainer(data=DataFrame([ + [getdate('2018-06-10 08:00:00').datetime, 10000, 10050, 9950, 9975, 12345, 1, 0], + [getdate('2018-06-10 09:00:00').datetime, 9975, 10025, 9925, 9950, 12345, 0, 0], + [getdate('2018-06-10 10:00:00').datetime, 9950, 10000, 9600, 9925, 12345, 0, 0], + [getdate('2018-06-10 11:00:00').datetime, 9925, 9975, 9875, 9900, 12345, 0, 0], + [getdate('2018-06-10 12:00:00').datetime, 9900, 9950, 9850, 9900, 12345, 0, 0] +], columns=columns), stop_loss=-0.03, roi=1, trades=1, profit_perc=-0.00999, sl=False) # + + +# Test 3 Candle drops 4%, Recovers 1%. +# Entry Criteria Met +# Candle drops 20% +# Candle Data for test 3 +# Test with Stop-Loss at 2% +tc3 = BTContainer(data=DataFrame([ + [getdate('2018-06-10 08:00:00').datetime, 10000, 10050, 9950, 9975, 12345, 1, 0], + [getdate('2018-06-10 09:00:00').datetime, 9975, 10025, 9600, 9950, 12345, 0, 0], + [getdate('2018-06-10 10:00:00').datetime, 9950, 10000, 9900, 9925, 12345, 1, 0], + [getdate('2018-06-10 11:00:00').datetime, 9925, 9975, 8000, 8000, 12345, 0, 0], + [getdate('2018-06-10 12:00:00').datetime, 9900, 9950, 9950, 9900, 12345, 0, 0] +], columns=columns), stop_loss=-0.02, roi=1, trades=1, profit_perc=-0.19999, sl=True) # + + +# Test 4 Minus 3% / recovery +15% +# Candle Data for test 4 – Candle drops 3% Closed 15% up +# Test with Stop-loss at 2% ROI 6% + +tc4 = BTContainer(data=DataFrame([ + [getdate('2018-06-10 08:00:00').datetime, 10000, 10050, 9950, 9975, 12345, 1, 0], + [getdate('2018-06-10 09:00:00').datetime, 9975, 11500, 9700, 11500, 12345, 0, 0], + [getdate('2018-06-10 10:00:00').datetime, 9950, 10000, 9900, 9925, 12345, 0, 0], + [getdate('2018-06-10 11:00:00').datetime, 9925, 9975, 9875, 9900, 12345, 0, 0], + [getdate('2018-06-10 12:00:00').datetime, 9900, 9950, 9850, 9900, 12345, 0, 0] +], columns=columns), + stop_loss=-0.02, roi=0.06, trades=1, profit_perc=-0.141, sl=True) + +# Test 5 / Drops 0.5% Closes +20% +# Candle Data for test 5 +# Set stop-loss at 1% ROI 3% +tc5 = BTContainer(data=DataFrame([ + [getdate('2018-06-10 08:00:00').datetime, 10000, 10050, 9950, 9975, 12345, 1, 0], + [getdate('2018-06-10 09:00:00').datetime, 9975, 12000, 9950, 12000, 12345, 0, 0], + [getdate('2018-06-10 10:00:00').datetime, 9950, 10000, 9900, 9925, 12345, 0, 0], + [getdate('2018-06-10 11:00:00').datetime, 9925, 9975, 9945, 9900, 12345, 0, 0], + [getdate('2018-06-10 12:00:00').datetime, 9900, 9950, 9850, 9900, 12345, 0, 0] +], columns=columns), + stop_loss=-0.01, roi=0.03, trades=1, profit_perc=-0.177, sl=True) + +# Test 6 / Drops 3% / Recovers 6% Positive / Closes 1% positve +# Candle Data for test 6 +# Set stop-loss at 2% ROI at 5% +tc6 = BTContainer(data=DataFrame([ + [getdate('2018-06-10 08:00:00').datetime, 10000, 10050, 9950, 9975, 12345, 1, 0], + [getdate('2018-06-10 09:00:00').datetime, 9975, 10600, 9700, 10100, 12345, 0, 0], + [getdate('2018-06-10 10:00:00').datetime, 9950, 10000, 9900, 9925, 12345, 0, 0], + [getdate('2018-06-10 11:00:00').datetime, 9925, 9975, 9945, 9900, 12345, 0, 0], + [getdate('2018-06-10 12:00:00').datetime, 9900, 9950, 9850, 9900, 12345, 0, 0] +], columns=columns), stop_loss=-0.02, roi=0.05, + trades=1, profit_perc=-0.025, sl=False) + +# Test 7 - 6% Positive / 1% Negative / Close 1% Positve +# Candle Data for test 7 +# Set stop-loss at 2% ROI at 3% + +tc7 = BTContainer(data=DataFrame([ + [getdate('2018-06-10 08:00:00').datetime, 10000, 10050, 9950, 9975, 12345, 1, 0], + [getdate('2018-06-10 09:00:00').datetime, 9975, 10600, 9900, 10100, 12345, 0, 0], + [getdate('2018-06-10 10:00:00').datetime, 9950, 10000, 9900, 9925, 12345, 0, 0], + [getdate('2018-06-10 11:00:00').datetime, 9925, 9975, 9945, 9900, 12345, 0, 0], + [getdate('2018-06-10 12:00:00').datetime, 9900, 9950, 9850, 9900, 12345, 0, 0] +], columns=columns), stop_loss=-0.02, roi=0.03, + trades=1, profit_perc=-0.025, sl=False) + +TESTS = [ + # tc_profit1, + # tc_profit2, + tc_loss0, + tc1, + tc2, + tc3, + tc4, + tc5, + tc6, + tc7, +] + + +@pytest.mark.parametrize("data", TESTS) +def test_backtest_results(default_conf, fee, mocker, caplog, data) -> None: """ run functional tests """ - default_conf["stoploss"] = stoploss + default_conf["stoploss"] = data.stop_loss + default_conf["minimal_roi"] = {"0": data.roi} mocker.patch('freqtrade.exchange.Exchange.get_fee', fee) - mocker.patch('freqtrade.analyze.Analyze.populate_sell_trend', MagicMock(return_value=data)) - mocker.patch('freqtrade.analyze.Analyze.populate_buy_trend', MagicMock(return_value=data)) + mocker.patch.multiple('freqtrade.analyze.Analyze', + populate_sell_trend=MagicMock(return_value=data.data), + populate_buy_trend=MagicMock(return_value=data.data)) patch_exchange(mocker) backtesting = Backtesting(default_conf) @@ -57,7 +179,7 @@ def test_backtest_results(default_conf, fee, mocker, caplog, pair = 'UNITTEST/BTC' # Dummy data as we mock the analyze functions - data_processed = {pair: pd.DataFrame()} + data_processed = {pair: DataFrame()} results = backtesting.backtest( { 'stake_amount': default_conf['stake_amount'], @@ -68,9 +190,9 @@ def test_backtest_results(default_conf, fee, mocker, caplog, ) print(results.T) - assert len(results) == tradecount - assert round(results["profit_percent"].sum(), 5) == profit_perc - if sl: + assert len(results) == data.trades + assert round(results["profit_percent"].sum(), 3) == round(data.profit_perc, 3) + if data.sl: assert log_has("Stop loss hit.", caplog.record_tuples) else: From 30a6e684a66956c5e715076244d9d15daab223a0 Mon Sep 17 00:00:00 2001 From: Matthias Date: Wed, 11 Jul 2018 07:18:52 +0200 Subject: [PATCH 197/699] update with new comments and new data for tc5 --- .../tests/optimize/test_backtest_detail.py | 68 +++++++++++++------ 1 file changed, 46 insertions(+), 22 deletions(-) diff --git a/freqtrade/tests/optimize/test_backtest_detail.py b/freqtrade/tests/optimize/test_backtest_detail.py index 5023f4b24..f8fe2cd16 100644 --- a/freqtrade/tests/optimize/test_backtest_detail.py +++ b/freqtrade/tests/optimize/test_backtest_detail.py @@ -21,7 +21,8 @@ class BTContainer(NamedTuple): roi: float trades: int profit_perc: float - sl: float + sl: bool + remains: bool columns = ['date', 'open', 'high', 'low', 'close', 'volume', 'buy', 'sell'] @@ -39,9 +40,9 @@ data_profit = DataFrame([ ], columns=columns) tc_profit1 = BTContainer(data=data_profit, stop_loss=-0.01, roi=1, trades=1, - profit_perc=0.10557, sl=False) # should be stoploss - drops 8% + profit_perc=0.10557, sl=False, remains=False) # should be stoploss - drops 8% tc_profit2 = BTContainer(data=data_profit, stop_loss=-0.10, roi=1, - trades=1, profit_perc=0.10557, sl=True) + trades=1, profit_perc=0.10557, sl=True, remains=False) tc_loss0 = BTContainer(data=DataFrame([ @@ -56,31 +57,37 @@ tc_loss0 = BTContainer(data=DataFrame([ [getdate('2018-07-08 22:00:00').datetime, 0.001000, 0.001011, 0.00098618, 0.00091618, 12345, 0, 0] ], columns=columns), - stop_loss=-0.05, roi=1, trades=1, profit_perc=-0.08839, sl=True) + stop_loss=-0.05, roi=1, trades=1, profit_perc=-0.08839, sl=True, remains=False) # Test 1 Minus 8% Close # Candle Data for test 1 – close at -8% (9200) # Test with Stop-loss at 1% +# TC1: Stop-Loss Triggered 1% loss tc1 = BTContainer(data=DataFrame([ [getdate('2018-06-10 08:00:00').datetime, 10000, 10050, 9950, 9975, 12345, 1, 0], - [getdate('2018-06-10 09:00:00').datetime, 9975, 10025, 9925, 9950, 12345, 0, 0], + [getdate('2018-06-10 09:00:00').datetime, 9975, 10025, 9200, 9200, 12345, 0, 0], [getdate('2018-06-10 10:00:00').datetime, 9950, 10000, 9960, 9955, 12345, 0, 0], [getdate('2018-06-10 11:00:00').datetime, 9955, 9975, 9955, 9990, 12345, 0, 0], - [getdate('2018-06-10 12:00:00').datetime, 9990, 9990, 9200, 9200, 12345, 0, 0] + [getdate('2018-06-10 12:00:00').datetime, 9990, 9990, 9990, 9900, 12345, 0, 0] ], columns=columns), - stop_loss=-0.01, roi=1, trades=1, profit_perc=-0.07999, sl=True) + # stop_loss=-0.01, roi=1, trades=1, profit_perc=-0.01, sl=True, remains=False) # should be + stop_loss=-0.01, roi=1, trades=1, profit_perc=0.071, sl=False, remains=True) # + # Test 2 Minus 4% Low, minus 1% close # Candle Data for test 2 # Test with Stop-Loss at 3% +# TC2: Stop-Loss Triggered 3% Loss tc2 = BTContainer(data=DataFrame([ [getdate('2018-06-10 08:00:00').datetime, 10000, 10050, 9950, 9975, 12345, 1, 0], [getdate('2018-06-10 09:00:00').datetime, 9975, 10025, 9925, 9950, 12345, 0, 0], [getdate('2018-06-10 10:00:00').datetime, 9950, 10000, 9600, 9925, 12345, 0, 0], [getdate('2018-06-10 11:00:00').datetime, 9925, 9975, 9875, 9900, 12345, 0, 0], [getdate('2018-06-10 12:00:00').datetime, 9900, 9950, 9850, 9900, 12345, 0, 0] -], columns=columns), stop_loss=-0.03, roi=1, trades=1, profit_perc=-0.00999, sl=False) # +], columns=columns), + # stop_loss=-0.03, roi=1, trades=1, profit_perc=-0.03, sl=True, remains=False) #should be + stop_loss=-0.03, roi=1, trades=1, profit_perc=-0.00999, sl=False, remains=True) # # Test 3 Candle drops 4%, Recovers 1%. @@ -88,19 +95,23 @@ tc2 = BTContainer(data=DataFrame([ # Candle drops 20% # Candle Data for test 3 # Test with Stop-Loss at 2% +# TC3: Trade-A: Stop-Loss Triggered 2% Loss +# Trade-B: Stop-Loss Triggered 2% Loss tc3 = BTContainer(data=DataFrame([ [getdate('2018-06-10 08:00:00').datetime, 10000, 10050, 9950, 9975, 12345, 1, 0], [getdate('2018-06-10 09:00:00').datetime, 9975, 10025, 9600, 9950, 12345, 0, 0], [getdate('2018-06-10 10:00:00').datetime, 9950, 10000, 9900, 9925, 12345, 1, 0], [getdate('2018-06-10 11:00:00').datetime, 9925, 9975, 8000, 8000, 12345, 0, 0], [getdate('2018-06-10 12:00:00').datetime, 9900, 9950, 9950, 9900, 12345, 0, 0] -], columns=columns), stop_loss=-0.02, roi=1, trades=1, profit_perc=-0.19999, sl=True) # +], columns=columns), + # stop_loss=-0.02, roi=1, trades=2, profit_perc=-0.4, sl=True, remains=False) #should be + stop_loss=-0.02, roi=1, trades=1, profit_perc=-0.19999, sl=True, remains=False) # # Test 4 Minus 3% / recovery +15% # Candle Data for test 4 – Candle drops 3% Closed 15% up # Test with Stop-loss at 2% ROI 6% - +# TC4: Stop-Loss Triggered 2% Loss tc4 = BTContainer(data=DataFrame([ [getdate('2018-06-10 08:00:00').datetime, 10000, 10050, 9950, 9975, 12345, 1, 0], [getdate('2018-06-10 09:00:00').datetime, 9975, 11500, 9700, 11500, 12345, 0, 0], @@ -108,49 +119,55 @@ tc4 = BTContainer(data=DataFrame([ [getdate('2018-06-10 11:00:00').datetime, 9925, 9975, 9875, 9900, 12345, 0, 0], [getdate('2018-06-10 12:00:00').datetime, 9900, 9950, 9850, 9900, 12345, 0, 0] ], columns=columns), - stop_loss=-0.02, roi=0.06, trades=1, profit_perc=-0.141, sl=True) + # stop_loss=-0.02, roi=0.06, trades=1, profit_perc=-0.02, sl=False, remains=False) #should be + stop_loss=-0.02, roi=0.06, trades=1, profit_perc=-0.141, sl=True, remains=False) # Test 5 / Drops 0.5% Closes +20% # Candle Data for test 5 # Set stop-loss at 1% ROI 3% +# TC5: ROI triggers 3% Gain tc5 = BTContainer(data=DataFrame([ - [getdate('2018-06-10 08:00:00').datetime, 10000, 10050, 9950, 9975, 12345, 1, 0], - [getdate('2018-06-10 09:00:00').datetime, 9975, 12000, 9950, 12000, 12345, 0, 0], - [getdate('2018-06-10 10:00:00').datetime, 9950, 10000, 9900, 9925, 12345, 0, 0], + [getdate('2018-06-10 08:00:00').datetime, 10000, 10050, 9960, 9975, 12345, 1, 0], + [getdate('2018-06-10 09:00:00').datetime, 9975, 10050, 9950, 9975, 12345, 0, 0], + [getdate('2018-06-10 10:00:00').datetime, 9950, 12000, 9950, 12000, 12345, 0, 0], [getdate('2018-06-10 11:00:00').datetime, 9925, 9975, 9945, 9900, 12345, 0, 0], [getdate('2018-06-10 12:00:00').datetime, 9900, 9950, 9850, 9900, 12345, 0, 0] ], columns=columns), - stop_loss=-0.01, roi=0.03, trades=1, profit_perc=-0.177, sl=True) + # stop_loss=-0.01, roi=0.03, trades=1, profit_perc=0.03, sl=False, remains=False) #should be + stop_loss=-0.01, roi=0.03, trades=1, profit_perc=0.197, sl=False, remains=False) # Test 6 / Drops 3% / Recovers 6% Positive / Closes 1% positve # Candle Data for test 6 # Set stop-loss at 2% ROI at 5% +# TC6: Stop-Loss triggers 2% Loss tc6 = BTContainer(data=DataFrame([ [getdate('2018-06-10 08:00:00').datetime, 10000, 10050, 9950, 9975, 12345, 1, 0], [getdate('2018-06-10 09:00:00').datetime, 9975, 10600, 9700, 10100, 12345, 0, 0], [getdate('2018-06-10 10:00:00').datetime, 9950, 10000, 9900, 9925, 12345, 0, 0], [getdate('2018-06-10 11:00:00').datetime, 9925, 9975, 9945, 9900, 12345, 0, 0], [getdate('2018-06-10 12:00:00').datetime, 9900, 9950, 9850, 9900, 12345, 0, 0] -], columns=columns), stop_loss=-0.02, roi=0.05, - trades=1, profit_perc=-0.025, sl=False) +], columns=columns), + # stop_loss=-0.02, roi=0.05, trades=1, profit_perc=-0.02, sl=False, remains=False) #should be + stop_loss=-0.02, roi=0.05, trades=1, profit_perc=-0.025, sl=False, remains=True) # # Test 7 - 6% Positive / 1% Negative / Close 1% Positve # Candle Data for test 7 # Set stop-loss at 2% ROI at 3% - +# TC7: ROI Triggers 3% Gain tc7 = BTContainer(data=DataFrame([ [getdate('2018-06-10 08:00:00').datetime, 10000, 10050, 9950, 9975, 12345, 1, 0], [getdate('2018-06-10 09:00:00').datetime, 9975, 10600, 9900, 10100, 12345, 0, 0], [getdate('2018-06-10 10:00:00').datetime, 9950, 10000, 9900, 9925, 12345, 0, 0], [getdate('2018-06-10 11:00:00').datetime, 9925, 9975, 9945, 9900, 12345, 0, 0], [getdate('2018-06-10 12:00:00').datetime, 9900, 9950, 9850, 9900, 12345, 0, 0] -], columns=columns), stop_loss=-0.02, roi=0.03, - trades=1, profit_perc=-0.025, sl=False) +], columns=columns), + # stop_loss=-0.02, roi=0.03, trades=1, profit_perc=-0.03, sl=False, remains=False) #should be + stop_loss=-0.02, roi=0.03, trades=1, profit_perc=-0.025, sl=False, remains=True) # TESTS = [ # tc_profit1, # tc_profit2, - tc_loss0, + # tc_loss0, tc1, tc2, tc3, @@ -195,5 +212,12 @@ def test_backtest_results(default_conf, fee, mocker, caplog, data) -> None: if data.sl: assert log_has("Stop loss hit.", caplog.record_tuples) else: - assert not log_has("Stop loss hit.", caplog.record_tuples) + log_test = (f'Force_selling still open trade UNITTEST/BTC with ' + f'{results.iloc[-1].profit_percent} perc - {results.iloc[-1].profit_abs}') + if data.remains: + assert log_has(log_test, + caplog.record_tuples) + else: + assert not log_has(log_test, + caplog.record_tuples) From 409465ac8e7580a3bacd04b61cd0539a80d8c9d5 Mon Sep 17 00:00:00 2001 From: Matthias Date: Mon, 30 Jul 2018 21:32:54 +0200 Subject: [PATCH 198/699] adapt functional tests for new version after rebase --- .../tests/optimize/test_backtest_detail.py | 48 +++++++++---------- 1 file changed, 23 insertions(+), 25 deletions(-) diff --git a/freqtrade/tests/optimize/test_backtest_detail.py b/freqtrade/tests/optimize/test_backtest_detail.py index f8fe2cd16..779687f10 100644 --- a/freqtrade/tests/optimize/test_backtest_detail.py +++ b/freqtrade/tests/optimize/test_backtest_detail.py @@ -9,6 +9,7 @@ from arrow import get as getdate from freqtrade.optimize.backtesting import Backtesting +from freqtrade.strategy.interface import SellType from freqtrade.tests.conftest import patch_exchange, log_has @@ -21,8 +22,7 @@ class BTContainer(NamedTuple): roi: float trades: int profit_perc: float - sl: bool - remains: bool + sell_r: SellType columns = ['date', 'open', 'high', 'low', 'close', 'volume', 'buy', 'sell'] @@ -40,9 +40,9 @@ data_profit = DataFrame([ ], columns=columns) tc_profit1 = BTContainer(data=data_profit, stop_loss=-0.01, roi=1, trades=1, - profit_perc=0.10557, sl=False, remains=False) # should be stoploss - drops 8% + profit_perc=0.10557, sell_r=SellType.STOP_LOSS) # should be stoploss - drops 8% tc_profit2 = BTContainer(data=data_profit, stop_loss=-0.10, roi=1, - trades=1, profit_perc=0.10557, sl=True, remains=False) + trades=1, profit_perc=0.10557, sell_r=SellType.STOP_LOSS) tc_loss0 = BTContainer(data=DataFrame([ @@ -57,7 +57,7 @@ tc_loss0 = BTContainer(data=DataFrame([ [getdate('2018-07-08 22:00:00').datetime, 0.001000, 0.001011, 0.00098618, 0.00091618, 12345, 0, 0] ], columns=columns), - stop_loss=-0.05, roi=1, trades=1, profit_perc=-0.08839, sl=True, remains=False) + stop_loss=-0.05, roi=1, trades=1, profit_perc=-0.08839, sell_r=SellType.STOP_LOSS) # Test 1 Minus 8% Close @@ -71,8 +71,8 @@ tc1 = BTContainer(data=DataFrame([ [getdate('2018-06-10 11:00:00').datetime, 9955, 9975, 9955, 9990, 12345, 0, 0], [getdate('2018-06-10 12:00:00').datetime, 9990, 9990, 9990, 9900, 12345, 0, 0] ], columns=columns), - # stop_loss=-0.01, roi=1, trades=1, profit_perc=-0.01, sl=True, remains=False) # should be - stop_loss=-0.01, roi=1, trades=1, profit_perc=0.071, sl=False, remains=True) # + # stop_loss=-0.01, roi=1, trades=1, profit_perc=-0.01, sell_r=SellType.STOP_LOSS) # should be + stop_loss=-0.01, roi=1, trades=1, profit_perc=-0.003, sell_r=SellType.FORCE_SELL) # # Test 2 Minus 4% Low, minus 1% close @@ -86,8 +86,8 @@ tc2 = BTContainer(data=DataFrame([ [getdate('2018-06-10 11:00:00').datetime, 9925, 9975, 9875, 9900, 12345, 0, 0], [getdate('2018-06-10 12:00:00').datetime, 9900, 9950, 9850, 9900, 12345, 0, 0] ], columns=columns), - # stop_loss=-0.03, roi=1, trades=1, profit_perc=-0.03, sl=True, remains=False) #should be - stop_loss=-0.03, roi=1, trades=1, profit_perc=-0.00999, sl=False, remains=True) # + # stop_loss=-0.03, roi=1, trades=1, profit_perc=-0.03, sell_r=SellType.STOP_LOSS) #should be + stop_loss=-0.03, roi=1, trades=1, profit_perc=-0.012, sell_r=SellType.FORCE_SELL) # # Test 3 Candle drops 4%, Recovers 1%. @@ -104,8 +104,8 @@ tc3 = BTContainer(data=DataFrame([ [getdate('2018-06-10 11:00:00').datetime, 9925, 9975, 8000, 8000, 12345, 0, 0], [getdate('2018-06-10 12:00:00').datetime, 9900, 9950, 9950, 9900, 12345, 0, 0] ], columns=columns), - # stop_loss=-0.02, roi=1, trades=2, profit_perc=-0.4, sl=True, remains=False) #should be - stop_loss=-0.02, roi=1, trades=1, profit_perc=-0.19999, sl=True, remains=False) # + # stop_loss=-0.02, roi=1, trades=2, profit_perc=-0.4, sell_r=SellType.STOP_LOSS) #should be + stop_loss=-0.02, roi=1, trades=1, profit_perc=-0.012, sell_r=SellType.FORCE_SELL) # # Test 4 Minus 3% / recovery +15% @@ -119,8 +119,8 @@ tc4 = BTContainer(data=DataFrame([ [getdate('2018-06-10 11:00:00').datetime, 9925, 9975, 9875, 9900, 12345, 0, 0], [getdate('2018-06-10 12:00:00').datetime, 9900, 9950, 9850, 9900, 12345, 0, 0] ], columns=columns), - # stop_loss=-0.02, roi=0.06, trades=1, profit_perc=-0.02, sl=False, remains=False) #should be - stop_loss=-0.02, roi=0.06, trades=1, profit_perc=-0.141, sl=True, remains=False) + # stop_loss=-0.02, roi=0.06, trades=1, profit_perc=-0.02, sell_r=SellType.STOP_LOSS) #should be + stop_loss=-0.02, roi=0.06, trades=1, profit_perc=-0.012, sell_r=SellType.FORCE_SELL) # Test 5 / Drops 0.5% Closes +20% # Candle Data for test 5 @@ -133,8 +133,8 @@ tc5 = BTContainer(data=DataFrame([ [getdate('2018-06-10 11:00:00').datetime, 9925, 9975, 9945, 9900, 12345, 0, 0], [getdate('2018-06-10 12:00:00').datetime, 9900, 9950, 9850, 9900, 12345, 0, 0] ], columns=columns), - # stop_loss=-0.01, roi=0.03, trades=1, profit_perc=0.03, sl=False, remains=False) #should be - stop_loss=-0.01, roi=0.03, trades=1, profit_perc=0.197, sl=False, remains=False) + # stop_loss=-0.01, roi=0.03, trades=1, profit_perc=0.03, sell_r=SellType.ROI) #should be + stop_loss=-0.01, roi=0.03, trades=1, profit_perc=-0.012, sell_r=SellType.FORCE_SELL) # Test 6 / Drops 3% / Recovers 6% Positive / Closes 1% positve # Candle Data for test 6 @@ -147,8 +147,8 @@ tc6 = BTContainer(data=DataFrame([ [getdate('2018-06-10 11:00:00').datetime, 9925, 9975, 9945, 9900, 12345, 0, 0], [getdate('2018-06-10 12:00:00').datetime, 9900, 9950, 9850, 9900, 12345, 0, 0] ], columns=columns), - # stop_loss=-0.02, roi=0.05, trades=1, profit_perc=-0.02, sl=False, remains=False) #should be - stop_loss=-0.02, roi=0.05, trades=1, profit_perc=-0.025, sl=False, remains=True) # + # stop_loss=-0.02, roi=0.05, trades=1, profit_perc=-0.02, sell_r=SellType.STOP_LOSS) #should be + stop_loss=-0.02, roi=0.05, trades=1, profit_perc=-0.012, sell_r=SellType.FORCE_SELL) # # Test 7 - 6% Positive / 1% Negative / Close 1% Positve # Candle Data for test 7 @@ -161,8 +161,8 @@ tc7 = BTContainer(data=DataFrame([ [getdate('2018-06-10 11:00:00').datetime, 9925, 9975, 9945, 9900, 12345, 0, 0], [getdate('2018-06-10 12:00:00').datetime, 9900, 9950, 9850, 9900, 12345, 0, 0] ], columns=columns), - # stop_loss=-0.02, roi=0.03, trades=1, profit_perc=-0.03, sl=False, remains=False) #should be - stop_loss=-0.02, roi=0.03, trades=1, profit_perc=-0.025, sl=False, remains=True) # + # stop_loss=-0.02, roi=0.03, trades=1, profit_perc=0.03, sell_r=SellType.ROI) #should be + stop_loss=-0.02, roi=0.03, trades=1, profit_perc=-0.012, sell_r=SellType.FORCE_SELL) # TESTS = [ # tc_profit1, @@ -186,12 +186,11 @@ def test_backtest_results(default_conf, fee, mocker, caplog, data) -> None: default_conf["stoploss"] = data.stop_loss default_conf["minimal_roi"] = {"0": data.roi} mocker.patch('freqtrade.exchange.Exchange.get_fee', fee) - mocker.patch.multiple('freqtrade.analyze.Analyze', - populate_sell_trend=MagicMock(return_value=data.data), - populate_buy_trend=MagicMock(return_value=data.data)) patch_exchange(mocker) backtesting = Backtesting(default_conf) + backtesting.advise_buy = lambda a, m: data.data + backtesting.advise_sell = lambda a, m: data.data caplog.set_level(logging.DEBUG) pair = 'UNITTEST/BTC' @@ -202,20 +201,19 @@ def test_backtest_results(default_conf, fee, mocker, caplog, data) -> None: 'stake_amount': default_conf['stake_amount'], 'processed': data_processed, 'max_open_trades': 10, - 'realistic': True } ) print(results.T) assert len(results) == data.trades assert round(results["profit_percent"].sum(), 3) == round(data.profit_perc, 3) - if data.sl: + if data.sell_r == SellType.STOP_LOSS: assert log_has("Stop loss hit.", caplog.record_tuples) else: assert not log_has("Stop loss hit.", caplog.record_tuples) log_test = (f'Force_selling still open trade UNITTEST/BTC with ' f'{results.iloc[-1].profit_percent} perc - {results.iloc[-1].profit_abs}') - if data.remains: + if data.sell_r == SellType.FORCE_SELL: assert log_has(log_test, caplog.record_tuples) else: From a0e8bfbd77daa21d2af1dd83f865e7099f5869b6 Mon Sep 17 00:00:00 2001 From: Matthias Date: Thu, 16 Aug 2018 11:28:25 +0200 Subject: [PATCH 199/699] shift buy-signal to one earlier (backtest shifts it forward to avoid lookahead) --- .../tests/optimize/test_backtest_detail.py | 26 ++++++++++++------- 1 file changed, 17 insertions(+), 9 deletions(-) diff --git a/freqtrade/tests/optimize/test_backtest_detail.py b/freqtrade/tests/optimize/test_backtest_detail.py index 779687f10..bb24f1602 100644 --- a/freqtrade/tests/optimize/test_backtest_detail.py +++ b/freqtrade/tests/optimize/test_backtest_detail.py @@ -65,7 +65,8 @@ tc_loss0 = BTContainer(data=DataFrame([ # Test with Stop-loss at 1% # TC1: Stop-Loss Triggered 1% loss tc1 = BTContainer(data=DataFrame([ - [getdate('2018-06-10 08:00:00').datetime, 10000, 10050, 9950, 9975, 12345, 1, 0], + [getdate('2018-06-10 07:00:00').datetime, 10000, 10050, 9950, 9975, 12345, 1, 0], + [getdate('2018-06-10 08:00:00').datetime, 10000, 10050, 9950, 9975, 12345, 0, 0], [getdate('2018-06-10 09:00:00').datetime, 9975, 10025, 9200, 9200, 12345, 0, 0], [getdate('2018-06-10 10:00:00').datetime, 9950, 10000, 9960, 9955, 12345, 0, 0], [getdate('2018-06-10 11:00:00').datetime, 9955, 9975, 9955, 9990, 12345, 0, 0], @@ -80,7 +81,8 @@ tc1 = BTContainer(data=DataFrame([ # Test with Stop-Loss at 3% # TC2: Stop-Loss Triggered 3% Loss tc2 = BTContainer(data=DataFrame([ - [getdate('2018-06-10 08:00:00').datetime, 10000, 10050, 9950, 9975, 12345, 1, 0], + [getdate('2018-06-10 07:00:00').datetime, 10000, 10050, 9950, 9975, 12345, 1, 0], + [getdate('2018-06-10 08:00:00').datetime, 10000, 10050, 9950, 9975, 12345, 0, 0], [getdate('2018-06-10 09:00:00').datetime, 9975, 10025, 9925, 9950, 12345, 0, 0], [getdate('2018-06-10 10:00:00').datetime, 9950, 10000, 9600, 9925, 12345, 0, 0], [getdate('2018-06-10 11:00:00').datetime, 9925, 9975, 9875, 9900, 12345, 0, 0], @@ -98,11 +100,13 @@ tc2 = BTContainer(data=DataFrame([ # TC3: Trade-A: Stop-Loss Triggered 2% Loss # Trade-B: Stop-Loss Triggered 2% Loss tc3 = BTContainer(data=DataFrame([ - [getdate('2018-06-10 08:00:00').datetime, 10000, 10050, 9950, 9975, 12345, 1, 0], + [getdate('2018-06-10 07:00:00').datetime, 10000, 10050, 9950, 9975, 12345, 1, 0], + [getdate('2018-06-10 08:00:00').datetime, 10000, 10050, 9950, 9975, 12345, 0, 0], [getdate('2018-06-10 09:00:00').datetime, 9975, 10025, 9600, 9950, 12345, 0, 0], [getdate('2018-06-10 10:00:00').datetime, 9950, 10000, 9900, 9925, 12345, 1, 0], - [getdate('2018-06-10 11:00:00').datetime, 9925, 9975, 8000, 8000, 12345, 0, 0], - [getdate('2018-06-10 12:00:00').datetime, 9900, 9950, 9950, 9900, 12345, 0, 0] + [getdate('2018-06-10 11:00:00').datetime, 9950, 10000, 9900, 9925, 12345, 0, 0], + [getdate('2018-06-10 12:00:00').datetime, 9925, 9975, 8000, 8000, 12345, 0, 0], + [getdate('2018-06-10 13:00:00').datetime, 9900, 9950, 9950, 9900, 12345, 0, 0] ], columns=columns), # stop_loss=-0.02, roi=1, trades=2, profit_perc=-0.4, sell_r=SellType.STOP_LOSS) #should be stop_loss=-0.02, roi=1, trades=1, profit_perc=-0.012, sell_r=SellType.FORCE_SELL) # @@ -113,7 +117,8 @@ tc3 = BTContainer(data=DataFrame([ # Test with Stop-loss at 2% ROI 6% # TC4: Stop-Loss Triggered 2% Loss tc4 = BTContainer(data=DataFrame([ - [getdate('2018-06-10 08:00:00').datetime, 10000, 10050, 9950, 9975, 12345, 1, 0], + [getdate('2018-06-10 07:00:00').datetime, 10000, 10050, 9950, 9975, 12345, 1, 0], + [getdate('2018-06-10 08:00:00').datetime, 10000, 10050, 9950, 9975, 12345, 0, 0], [getdate('2018-06-10 09:00:00').datetime, 9975, 11500, 9700, 11500, 12345, 0, 0], [getdate('2018-06-10 10:00:00').datetime, 9950, 10000, 9900, 9925, 12345, 0, 0], [getdate('2018-06-10 11:00:00').datetime, 9925, 9975, 9875, 9900, 12345, 0, 0], @@ -127,7 +132,8 @@ tc4 = BTContainer(data=DataFrame([ # Set stop-loss at 1% ROI 3% # TC5: ROI triggers 3% Gain tc5 = BTContainer(data=DataFrame([ - [getdate('2018-06-10 08:00:00').datetime, 10000, 10050, 9960, 9975, 12345, 1, 0], + [getdate('2018-06-10 07:00:00').datetime, 10000, 10050, 9960, 9975, 12345, 1, 0], + [getdate('2018-06-10 08:00:00').datetime, 10000, 10050, 9960, 9975, 12345, 0, 0], [getdate('2018-06-10 09:00:00').datetime, 9975, 10050, 9950, 9975, 12345, 0, 0], [getdate('2018-06-10 10:00:00').datetime, 9950, 12000, 9950, 12000, 12345, 0, 0], [getdate('2018-06-10 11:00:00').datetime, 9925, 9975, 9945, 9900, 12345, 0, 0], @@ -141,7 +147,8 @@ tc5 = BTContainer(data=DataFrame([ # Set stop-loss at 2% ROI at 5% # TC6: Stop-Loss triggers 2% Loss tc6 = BTContainer(data=DataFrame([ - [getdate('2018-06-10 08:00:00').datetime, 10000, 10050, 9950, 9975, 12345, 1, 0], + [getdate('2018-06-10 07:00:00').datetime, 10000, 10050, 9950, 9975, 12345, 1, 0], + [getdate('2018-06-10 08:00:00').datetime, 10000, 10050, 9950, 9975, 12345, 0, 0], [getdate('2018-06-10 09:00:00').datetime, 9975, 10600, 9700, 10100, 12345, 0, 0], [getdate('2018-06-10 10:00:00').datetime, 9950, 10000, 9900, 9925, 12345, 0, 0], [getdate('2018-06-10 11:00:00').datetime, 9925, 9975, 9945, 9900, 12345, 0, 0], @@ -155,7 +162,8 @@ tc6 = BTContainer(data=DataFrame([ # Set stop-loss at 2% ROI at 3% # TC7: ROI Triggers 3% Gain tc7 = BTContainer(data=DataFrame([ - [getdate('2018-06-10 08:00:00').datetime, 10000, 10050, 9950, 9975, 12345, 1, 0], + [getdate('2018-06-10 07:00:00').datetime, 10000, 10050, 9950, 9975, 12345, 1, 0], + [getdate('2018-06-10 08:00:00').datetime, 10000, 10050, 9950, 9975, 12345, 0, 0], [getdate('2018-06-10 09:00:00').datetime, 9975, 10600, 9900, 10100, 12345, 0, 0], [getdate('2018-06-10 10:00:00').datetime, 9950, 10000, 9900, 9925, 12345, 0, 0], [getdate('2018-06-10 11:00:00').datetime, 9925, 9975, 9945, 9900, 12345, 0, 0], From 233c442af97631bd525ef55559a4df9356258e29 Mon Sep 17 00:00:00 2001 From: Matthias Date: Thu, 16 Aug 2018 11:31:41 +0200 Subject: [PATCH 200/699] Adjust backtest so sell uses stop-loss or roi value as closerate --- freqtrade/optimize/backtesting.py | 18 +++++++--- freqtrade/strategy/interface.py | 12 ++++--- .../tests/optimize/test_backtest_detail.py | 33 ++++++++++--------- 3 files changed, 39 insertions(+), 24 deletions(-) diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index 961cfb092..a203fef2e 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -206,12 +206,20 @@ class Backtesting(object): buy_signal = sell_row.buy sell = self.strategy.should_sell(trade, sell_row.open, sell_row.date, buy_signal, - sell_row.sell) + sell_row.sell, low=sell_row.low, high=sell_row.high) if sell.sell_flag: + if sell.sell_type in (SellType.STOP_LOSS, SellType.TRAILING_STOP_LOSS): + # Set close_rate to stoploss + closerate = trade.stop_loss + elif sell.sell_type == (SellType.ROI): + # set close-rate to min-roi + closerate = trade.open_rate + trade.open_rate * self.strategy.minimal_roi[0] + else: + closerate = sell_row.open return BacktestResult(pair=pair, - profit_percent=trade.calc_profit_percent(rate=sell_row.open), - profit_abs=trade.calc_profit(rate=sell_row.open), + profit_percent=trade.calc_profit_percent(rate=closerate), + profit_abs=trade.calc_profit(rate=closerate), open_time=buy_row.date, close_time=sell_row.date, trade_duration=int(( @@ -220,7 +228,7 @@ class Backtesting(object): close_index=sell_row.Index, open_at_end=False, open_rate=buy_row.open, - close_rate=sell_row.open, + close_rate=closerate, sell_reason=sell.sell_type ) if partial_ticker: @@ -260,7 +268,7 @@ class Backtesting(object): position_stacking: do we allow position stacking? (default: False) :return: DataFrame """ - headers = ['date', 'buy', 'open', 'close', 'sell'] + headers = ['date', 'buy', 'open', 'close', 'sell', 'low', 'high'] processed = args['processed'] max_open_trades = args.get('max_open_trades', 0) position_stacking = args.get('position_stacking', False) diff --git a/freqtrade/strategy/interface.py b/freqtrade/strategy/interface.py index 6afa4161b..21efa09d6 100644 --- a/freqtrade/strategy/interface.py +++ b/freqtrade/strategy/interface.py @@ -203,18 +203,22 @@ class IStrategy(ABC): return buy, sell def should_sell(self, trade: Trade, rate: float, date: datetime, buy: bool, - sell: bool) -> SellCheckTuple: + sell: bool, low: float=None, high: float=None) -> SellCheckTuple: """ This function evaluate if on the condition required to trigger a sell has been reached if the threshold is reached and updates the trade record. :return: True if trade should be sold, False otherwise """ - current_profit = trade.calc_profit_percent(rate) - stoplossflag = self.stop_loss_reached(current_rate=rate, trade=trade, current_time=date, + # Set current rate to low for backtesting sell + current_rate = rate if not low else low + current_profit = trade.calc_profit_percent(current_rate) + stoplossflag = self.stop_loss_reached(current_rate=current_rate, trade=trade, current_time=date, current_profit=current_profit) if stoplossflag.sell_flag: return stoplossflag - + # Set current rate to low for backtesting sell + current_rate = rate if not high else high + current_profit = trade.calc_profit_percent(current_rate) experimental = self.config.get('experimental', {}) if buy and experimental.get('ignore_roi_if_buy_signal', False): diff --git a/freqtrade/tests/optimize/test_backtest_detail.py b/freqtrade/tests/optimize/test_backtest_detail.py index bb24f1602..6430f6c1e 100644 --- a/freqtrade/tests/optimize/test_backtest_detail.py +++ b/freqtrade/tests/optimize/test_backtest_detail.py @@ -72,8 +72,8 @@ tc1 = BTContainer(data=DataFrame([ [getdate('2018-06-10 11:00:00').datetime, 9955, 9975, 9955, 9990, 12345, 0, 0], [getdate('2018-06-10 12:00:00').datetime, 9990, 9990, 9990, 9900, 12345, 0, 0] ], columns=columns), - # stop_loss=-0.01, roi=1, trades=1, profit_perc=-0.01, sell_r=SellType.STOP_LOSS) # should be - stop_loss=-0.01, roi=1, trades=1, profit_perc=-0.003, sell_r=SellType.FORCE_SELL) # + stop_loss=-0.01, roi=1, trades=1, profit_perc=-0.01, sell_r=SellType.STOP_LOSS) # should be + # stop_loss=-0.01, roi=1, trades=1, profit_perc=-0.003, sell_r=SellType.FORCE_SELL) # # Test 2 Minus 4% Low, minus 1% close @@ -88,8 +88,8 @@ tc2 = BTContainer(data=DataFrame([ [getdate('2018-06-10 11:00:00').datetime, 9925, 9975, 9875, 9900, 12345, 0, 0], [getdate('2018-06-10 12:00:00').datetime, 9900, 9950, 9850, 9900, 12345, 0, 0] ], columns=columns), - # stop_loss=-0.03, roi=1, trades=1, profit_perc=-0.03, sell_r=SellType.STOP_LOSS) #should be - stop_loss=-0.03, roi=1, trades=1, profit_perc=-0.012, sell_r=SellType.FORCE_SELL) # + stop_loss=-0.03, roi=1, trades=1, profit_perc=-0.03, sell_r=SellType.STOP_LOSS) #should be + # stop_loss=-0.03, roi=1, trades=1, profit_perc=-0.007, sell_r=SellType.FORCE_SELL) # # Test 3 Candle drops 4%, Recovers 1%. @@ -108,8 +108,9 @@ tc3 = BTContainer(data=DataFrame([ [getdate('2018-06-10 12:00:00').datetime, 9925, 9975, 8000, 8000, 12345, 0, 0], [getdate('2018-06-10 13:00:00').datetime, 9900, 9950, 9950, 9900, 12345, 0, 0] ], columns=columns), - # stop_loss=-0.02, roi=1, trades=2, profit_perc=-0.4, sell_r=SellType.STOP_LOSS) #should be - stop_loss=-0.02, roi=1, trades=1, profit_perc=-0.012, sell_r=SellType.FORCE_SELL) # + stop_loss=-0.02, roi=1, trades=2, profit_perc=-0.04, sell_r=SellType.STOP_LOSS) #should be + # stop_loss=-0.02, roi=1, trades=1, profit_perc=-0.02, sell_r=SellType.STOP_LOSS) #should be + # stop_loss=-0.02, roi=1, trades=1, profit_perc=-0.012, sell_r=SellType.FORCE_SELL) # # Test 4 Minus 3% / recovery +15% @@ -124,8 +125,8 @@ tc4 = BTContainer(data=DataFrame([ [getdate('2018-06-10 11:00:00').datetime, 9925, 9975, 9875, 9900, 12345, 0, 0], [getdate('2018-06-10 12:00:00').datetime, 9900, 9950, 9850, 9900, 12345, 0, 0] ], columns=columns), - # stop_loss=-0.02, roi=0.06, trades=1, profit_perc=-0.02, sell_r=SellType.STOP_LOSS) #should be - stop_loss=-0.02, roi=0.06, trades=1, profit_perc=-0.012, sell_r=SellType.FORCE_SELL) + stop_loss=-0.02, roi=0.06, trades=1, profit_perc=-0.02, sell_r=SellType.STOP_LOSS) #should be + # stop_loss=-0.02, roi=0.06, trades=1, profit_perc=-0.012, sell_r=SellType.FORCE_SELL) # Test 5 / Drops 0.5% Closes +20% # Candle Data for test 5 @@ -139,8 +140,8 @@ tc5 = BTContainer(data=DataFrame([ [getdate('2018-06-10 11:00:00').datetime, 9925, 9975, 9945, 9900, 12345, 0, 0], [getdate('2018-06-10 12:00:00').datetime, 9900, 9950, 9850, 9900, 12345, 0, 0] ], columns=columns), - # stop_loss=-0.01, roi=0.03, trades=1, profit_perc=0.03, sell_r=SellType.ROI) #should be - stop_loss=-0.01, roi=0.03, trades=1, profit_perc=-0.012, sell_r=SellType.FORCE_SELL) + stop_loss=-0.01, roi=0.03, trades=1, profit_perc=0.03, sell_r=SellType.ROI) #should be + # stop_loss=-0.01, roi=0.03, trades=1, profit_perc=-0.012, sell_r=SellType.FORCE_SELL) # Test 6 / Drops 3% / Recovers 6% Positive / Closes 1% positve # Candle Data for test 6 @@ -154,8 +155,8 @@ tc6 = BTContainer(data=DataFrame([ [getdate('2018-06-10 11:00:00').datetime, 9925, 9975, 9945, 9900, 12345, 0, 0], [getdate('2018-06-10 12:00:00').datetime, 9900, 9950, 9850, 9900, 12345, 0, 0] ], columns=columns), - # stop_loss=-0.02, roi=0.05, trades=1, profit_perc=-0.02, sell_r=SellType.STOP_LOSS) #should be - stop_loss=-0.02, roi=0.05, trades=1, profit_perc=-0.012, sell_r=SellType.FORCE_SELL) # + stop_loss=-0.02, roi=0.05, trades=1, profit_perc=-0.02, sell_r=SellType.STOP_LOSS) #should be + # stop_loss=-0.02, roi=0.05, trades=1, profit_perc=-0.012, sell_r=SellType.FORCE_SELL) # # Test 7 - 6% Positive / 1% Negative / Close 1% Positve # Candle Data for test 7 @@ -169,8 +170,8 @@ tc7 = BTContainer(data=DataFrame([ [getdate('2018-06-10 11:00:00').datetime, 9925, 9975, 9945, 9900, 12345, 0, 0], [getdate('2018-06-10 12:00:00').datetime, 9900, 9950, 9850, 9900, 12345, 0, 0] ], columns=columns), - # stop_loss=-0.02, roi=0.03, trades=1, profit_perc=0.03, sell_r=SellType.ROI) #should be - stop_loss=-0.02, roi=0.03, trades=1, profit_perc=-0.012, sell_r=SellType.FORCE_SELL) # + stop_loss=-0.02, roi=0.03, trades=1, profit_perc=0.03, sell_r=SellType.ROI) #should be + # stop_loss=-0.02, roi=0.03, trades=1, profit_perc=-0.012, sell_r=SellType.FORCE_SELL) # TESTS = [ # tc_profit1, @@ -193,7 +194,9 @@ def test_backtest_results(default_conf, fee, mocker, caplog, data) -> None: """ default_conf["stoploss"] = data.stop_loss default_conf["minimal_roi"] = {"0": data.roi} - mocker.patch('freqtrade.exchange.Exchange.get_fee', fee) + # mocker.patch('freqtrade.exchange.Exchange.get_fee', fee) + # TODO: don't Mock fee to for now + mocker.patch('freqtrade.exchange.Exchange.get_fee', MagicMock(return_value=0.0)) patch_exchange(mocker) backtesting = Backtesting(default_conf) From 98050ff594d36914488c6285faed8d3719785029 Mon Sep 17 00:00:00 2001 From: Matthias Date: Fri, 17 Aug 2018 07:07:50 +0200 Subject: [PATCH 201/699] use all min_roi entries --- freqtrade/optimize/backtesting.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index a203fef2e..9678b6d09 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -208,12 +208,17 @@ class Backtesting(object): sell = self.strategy.should_sell(trade, sell_row.open, sell_row.date, buy_signal, sell_row.sell, low=sell_row.low, high=sell_row.high) if sell.sell_flag: + trade_dur = int((sell_row.date - buy_row.date).total_seconds() // 60) if sell.sell_type in (SellType.STOP_LOSS, SellType.TRAILING_STOP_LOSS): # Set close_rate to stoploss closerate = trade.stop_loss elif sell.sell_type == (SellType.ROI): + # get entry in min_roi >= to trade duration + roi_entry = max(list(filter(lambda x: trade_dur >= x, + list(self.strategy.minimal_roi.keys())))) # set close-rate to min-roi - closerate = trade.open_rate + trade.open_rate * self.strategy.minimal_roi[0] + closerate = trade.open_rate + trade.open_rate * \ + self.strategy.minimal_roi[roi_entry] else: closerate = sell_row.open @@ -222,8 +227,7 @@ class Backtesting(object): profit_abs=trade.calc_profit(rate=closerate), open_time=buy_row.date, close_time=sell_row.date, - trade_duration=int(( - sell_row.date - buy_row.date).total_seconds() // 60), + trade_duration=trade_dur, open_index=buy_row.Index, close_index=sell_row.Index, open_at_end=False, From 6096f3ca47c74e5aa609720273f004c5f13a3da4 Mon Sep 17 00:00:00 2001 From: Matthias Date: Mon, 29 Oct 2018 20:17:15 +0100 Subject: [PATCH 202/699] Simplify functional tests --- .../tests/optimize/test_backtest_detail.py | 175 +++++++++--------- 1 file changed, 85 insertions(+), 90 deletions(-) diff --git a/freqtrade/tests/optimize/test_backtest_detail.py b/freqtrade/tests/optimize/test_backtest_detail.py index 6430f6c1e..f2b97c744 100644 --- a/freqtrade/tests/optimize/test_backtest_detail.py +++ b/freqtrade/tests/optimize/test_backtest_detail.py @@ -1,11 +1,11 @@ # pragma pylint: disable=missing-docstring, W0212, line-too-long, C0103, unused-argument import logging from unittest.mock import MagicMock -from typing import NamedTuple +from typing import NamedTuple, List from pandas import DataFrame import pytest -from arrow import get as getdate +import arrow from freqtrade.optimize.backtesting import Backtesting @@ -13,11 +13,15 @@ from freqtrade.strategy.interface import SellType from freqtrade.tests.conftest import patch_exchange, log_has +ticker_start_time = arrow.get(2018, 10, 3) +ticker_interval_in_minute = 60 + + class BTContainer(NamedTuple): """ NamedTuple Defining BacktestResults inputs. """ - data: DataFrame + data: List[float] stop_loss: float roi: float trades: int @@ -25,19 +29,24 @@ class BTContainer(NamedTuple): sell_r: SellType -columns = ['date', 'open', 'high', 'low', 'close', 'volume', 'buy', 'sell'] -data_profit = DataFrame([ - [getdate('2018-07-08 18:00:00').datetime, 0.0009910, - 0.001011, 0.00098618, 0.001000, 12345, 1, 0], - [getdate('2018-07-08 19:00:00').datetime, 0.001000, - 0.001010, 0.0009900, 0.0009900, 12345, 0, 0], - [getdate('2018-07-08 20:00:00').datetime, 0.0009900, - 0.001011, 0.00091618, 0.0009900, 12345, 0, 0], - [getdate('2018-07-08 21:00:00').datetime, 0.001000, - 0.001011, 0.00098618, 0.001100, 12345, 0, 1], - [getdate('2018-07-08 22:00:00').datetime, 0.001000, - 0.001011, 0.00098618, 0.0009900, 12345, 0, 0] -], columns=columns) +def _build_dataframe(ticker_with_signals): + columns = ['date', 'open', 'high', 'low', 'close', 'volume', 'buy', 'sell'] + + frame = DataFrame.from_records(ticker_with_signals, columns=columns) + frame['date'] = frame['date'].apply(lambda x: ticker_start_time.shift( + minutes=(x * ticker_interval_in_minute)).datetime) + # Ensure floats are in place + for column in ['open', 'high', 'low', 'close', 'volume']: + frame[column] = frame[column].astype('float64') + return frame + + +data_profit = [ + [0, 0.0009910, 0.001011, 0.00098618, 0.001000, 12345, 1, 0], + [1, 0.001000, 0.001010, 0.0009900, 0.0009900, 12345, 0, 0], + [2, 0.0009900, 0.001011, 0.00091618, 0.0009900, 12345, 0, 0], + [3, 0.001000, 0.001011, 0.00098618, 0.001100, 12345, 0, 1], + [4, 0.001000, 0.001011, 0.00098618, 0.0009900, 12345, 0, 0]] tc_profit1 = BTContainer(data=data_profit, stop_loss=-0.01, roi=1, trades=1, profit_perc=0.10557, sell_r=SellType.STOP_LOSS) # should be stoploss - drops 8% @@ -45,18 +54,12 @@ tc_profit2 = BTContainer(data=data_profit, stop_loss=-0.10, roi=1, trades=1, profit_perc=0.10557, sell_r=SellType.STOP_LOSS) -tc_loss0 = BTContainer(data=DataFrame([ - [getdate('2018-07-08 18:00:00').datetime, 0.0009910, - 0.001011, 0.00098618, 0.001000, 12345, 1, 0], - [getdate('2018-07-08 19:00:00').datetime, 0.001000, - 0.001010, 0.0009900, 0.001000, 12345, 0, 0], - [getdate('2018-07-08 20:00:00').datetime, 0.001000, - 0.001011, 0.0010618, 0.00091618, 12345, 0, 0], - [getdate('2018-07-08 21:00:00').datetime, 0.001000, - 0.001011, 0.00098618, 0.00091618, 12345, 0, 0], - [getdate('2018-07-08 22:00:00').datetime, 0.001000, - 0.001011, 0.00098618, 0.00091618, 12345, 0, 0] -], columns=columns), +tc_loss0 = BTContainer(data=[ + [0, 0.0009910, 0.001011, 0.00098618, 0.001000, 12345, 1, 0], + [1, 0.001000, 0.001010, 0.0009900, 0.001000, 12345, 0, 0], + [2, 0.001000, 0.001011, 0.0010618, 0.00091618, 12345, 0, 0], + [3, 0.001000, 0.001011, 0.00098618, 0.00091618, 12345, 0, 0], + [4, 0.001000, 0.001011, 0.00098618, 0.00091618, 12345, 0, 0]], stop_loss=-0.05, roi=1, trades=1, profit_perc=-0.08839, sell_r=SellType.STOP_LOSS) @@ -64,30 +67,27 @@ tc_loss0 = BTContainer(data=DataFrame([ # Candle Data for test 1 – close at -8% (9200) # Test with Stop-loss at 1% # TC1: Stop-Loss Triggered 1% loss -tc1 = BTContainer(data=DataFrame([ - [getdate('2018-06-10 07:00:00').datetime, 10000, 10050, 9950, 9975, 12345, 1, 0], - [getdate('2018-06-10 08:00:00').datetime, 10000, 10050, 9950, 9975, 12345, 0, 0], - [getdate('2018-06-10 09:00:00').datetime, 9975, 10025, 9200, 9200, 12345, 0, 0], - [getdate('2018-06-10 10:00:00').datetime, 9950, 10000, 9960, 9955, 12345, 0, 0], - [getdate('2018-06-10 11:00:00').datetime, 9955, 9975, 9955, 9990, 12345, 0, 0], - [getdate('2018-06-10 12:00:00').datetime, 9990, 9990, 9990, 9900, 12345, 0, 0] -], columns=columns), - stop_loss=-0.01, roi=1, trades=1, profit_perc=-0.01, sell_r=SellType.STOP_LOSS) # should be - # stop_loss=-0.01, roi=1, trades=1, profit_perc=-0.003, sell_r=SellType.FORCE_SELL) # +tc1 = BTContainer(data=[ + [0, 10000.0, 10050, 9950, 9975, 12345, 1, 0], + [1, 10000, 10050, 9950, 9975, 12345, 0, 0], + [2, 9975, 10025, 9200, 9200, 12345, 0, 0], + [3, 9950, 10000, 9960, 9955, 12345, 0, 0], + [4, 9955, 9975, 9955, 9990, 12345, 0, 0], + [5, 9990, 9990, 9990, 9900, 12345, 0, 0]], + stop_loss=-0.01, roi=1, trades=1, profit_perc=-0.01, sell_r=SellType.STOP_LOSS) # Test 2 Minus 4% Low, minus 1% close # Candle Data for test 2 # Test with Stop-Loss at 3% # TC2: Stop-Loss Triggered 3% Loss -tc2 = BTContainer(data=DataFrame([ - [getdate('2018-06-10 07:00:00').datetime, 10000, 10050, 9950, 9975, 12345, 1, 0], - [getdate('2018-06-10 08:00:00').datetime, 10000, 10050, 9950, 9975, 12345, 0, 0], - [getdate('2018-06-10 09:00:00').datetime, 9975, 10025, 9925, 9950, 12345, 0, 0], - [getdate('2018-06-10 10:00:00').datetime, 9950, 10000, 9600, 9925, 12345, 0, 0], - [getdate('2018-06-10 11:00:00').datetime, 9925, 9975, 9875, 9900, 12345, 0, 0], - [getdate('2018-06-10 12:00:00').datetime, 9900, 9950, 9850, 9900, 12345, 0, 0] -], columns=columns), +tc2 = BTContainer(data=[ + [0, 10000, 10050, 9950, 9975, 12345, 1, 0], + [1, 10000, 10050, 9950, 9975, 12345, 0, 0], + [2, 9975, 10025, 9925, 9950, 12345, 0, 0], + [3, 9950, 10000, 9600, 9925, 12345, 0, 0], + [4, 9925, 9975, 9875, 9900, 12345, 0, 0], + [5, 9900, 9950, 9850, 9900, 12345, 0, 0]], stop_loss=-0.03, roi=1, trades=1, profit_perc=-0.03, sell_r=SellType.STOP_LOSS) #should be # stop_loss=-0.03, roi=1, trades=1, profit_perc=-0.007, sell_r=SellType.FORCE_SELL) # @@ -99,15 +99,14 @@ tc2 = BTContainer(data=DataFrame([ # Test with Stop-Loss at 2% # TC3: Trade-A: Stop-Loss Triggered 2% Loss # Trade-B: Stop-Loss Triggered 2% Loss -tc3 = BTContainer(data=DataFrame([ - [getdate('2018-06-10 07:00:00').datetime, 10000, 10050, 9950, 9975, 12345, 1, 0], - [getdate('2018-06-10 08:00:00').datetime, 10000, 10050, 9950, 9975, 12345, 0, 0], - [getdate('2018-06-10 09:00:00').datetime, 9975, 10025, 9600, 9950, 12345, 0, 0], - [getdate('2018-06-10 10:00:00').datetime, 9950, 10000, 9900, 9925, 12345, 1, 0], - [getdate('2018-06-10 11:00:00').datetime, 9950, 10000, 9900, 9925, 12345, 0, 0], - [getdate('2018-06-10 12:00:00').datetime, 9925, 9975, 8000, 8000, 12345, 0, 0], - [getdate('2018-06-10 13:00:00').datetime, 9900, 9950, 9950, 9900, 12345, 0, 0] -], columns=columns), +tc3 = BTContainer(data=[ + [0, 10000, 10050, 9950, 9975, 12345, 1, 0], + [1, 10000, 10050, 9950, 9975, 12345, 0, 0], + [2, 9975, 10025, 9600, 9950, 12345, 0, 0], + [3, 9950, 10000, 9900, 9925, 12345, 1, 0], + [4, 9950, 10000, 9900, 9925, 12345, 0, 0], + [5, 9925, 9975, 8000, 8000, 12345, 0, 0], + [6, 9900, 9950, 9950, 9900, 12345, 0, 0]], stop_loss=-0.02, roi=1, trades=2, profit_perc=-0.04, sell_r=SellType.STOP_LOSS) #should be # stop_loss=-0.02, roi=1, trades=1, profit_perc=-0.02, sell_r=SellType.STOP_LOSS) #should be # stop_loss=-0.02, roi=1, trades=1, profit_perc=-0.012, sell_r=SellType.FORCE_SELL) # @@ -117,14 +116,13 @@ tc3 = BTContainer(data=DataFrame([ # Candle Data for test 4 – Candle drops 3% Closed 15% up # Test with Stop-loss at 2% ROI 6% # TC4: Stop-Loss Triggered 2% Loss -tc4 = BTContainer(data=DataFrame([ - [getdate('2018-06-10 07:00:00').datetime, 10000, 10050, 9950, 9975, 12345, 1, 0], - [getdate('2018-06-10 08:00:00').datetime, 10000, 10050, 9950, 9975, 12345, 0, 0], - [getdate('2018-06-10 09:00:00').datetime, 9975, 11500, 9700, 11500, 12345, 0, 0], - [getdate('2018-06-10 10:00:00').datetime, 9950, 10000, 9900, 9925, 12345, 0, 0], - [getdate('2018-06-10 11:00:00').datetime, 9925, 9975, 9875, 9900, 12345, 0, 0], - [getdate('2018-06-10 12:00:00').datetime, 9900, 9950, 9850, 9900, 12345, 0, 0] -], columns=columns), +tc4 = BTContainer(data=[ + [0, 10000, 10050, 9950, 9975, 12345, 1, 0], + [1, 10000, 10050, 9950, 9975, 12345, 0, 0], + [2, 9975, 11500, 9700, 11500, 12345, 0, 0], + [3, 9950, 10000, 9900, 9925, 12345, 0, 0], + [4, 9925, 9975, 9875, 9900, 12345, 0, 0], + [5, 9900, 9950, 9850, 9900, 12345, 0, 0]], stop_loss=-0.02, roi=0.06, trades=1, profit_perc=-0.02, sell_r=SellType.STOP_LOSS) #should be # stop_loss=-0.02, roi=0.06, trades=1, profit_perc=-0.012, sell_r=SellType.FORCE_SELL) @@ -132,14 +130,13 @@ tc4 = BTContainer(data=DataFrame([ # Candle Data for test 5 # Set stop-loss at 1% ROI 3% # TC5: ROI triggers 3% Gain -tc5 = BTContainer(data=DataFrame([ - [getdate('2018-06-10 07:00:00').datetime, 10000, 10050, 9960, 9975, 12345, 1, 0], - [getdate('2018-06-10 08:00:00').datetime, 10000, 10050, 9960, 9975, 12345, 0, 0], - [getdate('2018-06-10 09:00:00').datetime, 9975, 10050, 9950, 9975, 12345, 0, 0], - [getdate('2018-06-10 10:00:00').datetime, 9950, 12000, 9950, 12000, 12345, 0, 0], - [getdate('2018-06-10 11:00:00').datetime, 9925, 9975, 9945, 9900, 12345, 0, 0], - [getdate('2018-06-10 12:00:00').datetime, 9900, 9950, 9850, 9900, 12345, 0, 0] -], columns=columns), +tc5 = BTContainer(data=[ + [0, 10000, 10050, 9960, 9975, 12345, 1, 0], + [1, 10000, 10050, 9960, 9975, 12345, 0, 0], + [2, 9975, 10050, 9950, 9975, 12345, 0, 0], + [3, 9950, 12000, 9950, 12000, 12345, 0, 0], + [4, 9925, 9975, 9945, 9900, 12345, 0, 0], + [5, 9900, 9950, 9850, 9900, 12345, 0, 0]], stop_loss=-0.01, roi=0.03, trades=1, profit_perc=0.03, sell_r=SellType.ROI) #should be # stop_loss=-0.01, roi=0.03, trades=1, profit_perc=-0.012, sell_r=SellType.FORCE_SELL) @@ -147,14 +144,13 @@ tc5 = BTContainer(data=DataFrame([ # Candle Data for test 6 # Set stop-loss at 2% ROI at 5% # TC6: Stop-Loss triggers 2% Loss -tc6 = BTContainer(data=DataFrame([ - [getdate('2018-06-10 07:00:00').datetime, 10000, 10050, 9950, 9975, 12345, 1, 0], - [getdate('2018-06-10 08:00:00').datetime, 10000, 10050, 9950, 9975, 12345, 0, 0], - [getdate('2018-06-10 09:00:00').datetime, 9975, 10600, 9700, 10100, 12345, 0, 0], - [getdate('2018-06-10 10:00:00').datetime, 9950, 10000, 9900, 9925, 12345, 0, 0], - [getdate('2018-06-10 11:00:00').datetime, 9925, 9975, 9945, 9900, 12345, 0, 0], - [getdate('2018-06-10 12:00:00').datetime, 9900, 9950, 9850, 9900, 12345, 0, 0] -], columns=columns), +tc6 = BTContainer(data=[ + [0, 10000, 10050, 9950, 9975, 12345, 1, 0], + [1, 10000, 10050, 9950, 9975, 12345, 0, 0], + [2, 9975, 10600, 9700, 10100, 12345, 0, 0], + [3, 9950, 10000, 9900, 9925, 12345, 0, 0], + [4, 9925, 9975, 9945, 9900, 12345, 0, 0], + [5, 9900, 9950, 9850, 9900, 12345, 0, 0]], stop_loss=-0.02, roi=0.05, trades=1, profit_perc=-0.02, sell_r=SellType.STOP_LOSS) #should be # stop_loss=-0.02, roi=0.05, trades=1, profit_perc=-0.012, sell_r=SellType.FORCE_SELL) # @@ -162,14 +158,13 @@ tc6 = BTContainer(data=DataFrame([ # Candle Data for test 7 # Set stop-loss at 2% ROI at 3% # TC7: ROI Triggers 3% Gain -tc7 = BTContainer(data=DataFrame([ - [getdate('2018-06-10 07:00:00').datetime, 10000, 10050, 9950, 9975, 12345, 1, 0], - [getdate('2018-06-10 08:00:00').datetime, 10000, 10050, 9950, 9975, 12345, 0, 0], - [getdate('2018-06-10 09:00:00').datetime, 9975, 10600, 9900, 10100, 12345, 0, 0], - [getdate('2018-06-10 10:00:00').datetime, 9950, 10000, 9900, 9925, 12345, 0, 0], - [getdate('2018-06-10 11:00:00').datetime, 9925, 9975, 9945, 9900, 12345, 0, 0], - [getdate('2018-06-10 12:00:00').datetime, 9900, 9950, 9850, 9900, 12345, 0, 0] -], columns=columns), +tc7 = BTContainer(data=[ + [0, 10000, 10050, 9950, 9975, 12345, 1, 0], + [1, 10000, 10050, 9950, 9975, 12345, 0, 0], + [2, 9975, 10600, 9900, 10100, 12345, 0, 0], + [3, 9950, 10000, 9900, 9925, 12345, 0, 0], + [4, 9925, 9975, 9945, 9900, 12345, 0, 0], + [5, 9900, 9950, 9850, 9900, 12345, 0, 0]], stop_loss=-0.02, roi=0.03, trades=1, profit_perc=0.03, sell_r=SellType.ROI) #should be # stop_loss=-0.02, roi=0.03, trades=1, profit_perc=-0.012, sell_r=SellType.FORCE_SELL) # @@ -198,10 +193,10 @@ def test_backtest_results(default_conf, fee, mocker, caplog, data) -> None: # TODO: don't Mock fee to for now mocker.patch('freqtrade.exchange.Exchange.get_fee', MagicMock(return_value=0.0)) patch_exchange(mocker) - + frame = _build_dataframe(data.data) backtesting = Backtesting(default_conf) - backtesting.advise_buy = lambda a, m: data.data - backtesting.advise_sell = lambda a, m: data.data + backtesting.advise_buy = lambda a, m: frame + backtesting.advise_sell = lambda a, m: frame caplog.set_level(logging.DEBUG) pair = 'UNITTEST/BTC' From 9065e79f531d0a6b7338b5f6afbc6927e455ec12 Mon Sep 17 00:00:00 2001 From: Matthias Date: Mon, 29 Oct 2018 20:33:27 +0100 Subject: [PATCH 203/699] Cleanup and add some comments on what's happening in the sample snippets --- .../tests/optimize/test_backtest_detail.py | 50 ++++++++----------- 1 file changed, 21 insertions(+), 29 deletions(-) diff --git a/freqtrade/tests/optimize/test_backtest_detail.py b/freqtrade/tests/optimize/test_backtest_detail.py index f2b97c744..dda3be93f 100644 --- a/freqtrade/tests/optimize/test_backtest_detail.py +++ b/freqtrade/tests/optimize/test_backtest_detail.py @@ -69,8 +69,8 @@ tc_loss0 = BTContainer(data=[ # TC1: Stop-Loss Triggered 1% loss tc1 = BTContainer(data=[ [0, 10000.0, 10050, 9950, 9975, 12345, 1, 0], - [1, 10000, 10050, 9950, 9975, 12345, 0, 0], - [2, 9975, 10025, 9200, 9200, 12345, 0, 0], + [1, 10000, 10050, 9950, 9975, 12345, 0, 0], # enter trade (signal on last candle) + [2, 9975, 10025, 9200, 9200, 12345, 0, 0], # Exit with stoploss hit [3, 9950, 10000, 9960, 9955, 12345, 0, 0], [4, 9955, 9975, 9955, 9990, 12345, 0, 0], [5, 9990, 9990, 9990, 9900, 12345, 0, 0]], @@ -83,13 +83,12 @@ tc1 = BTContainer(data=[ # TC2: Stop-Loss Triggered 3% Loss tc2 = BTContainer(data=[ [0, 10000, 10050, 9950, 9975, 12345, 1, 0], - [1, 10000, 10050, 9950, 9975, 12345, 0, 0], + [1, 10000, 10050, 9950, 9975, 12345, 0, 0], # enter trade (signal on last candle) [2, 9975, 10025, 9925, 9950, 12345, 0, 0], - [3, 9950, 10000, 9600, 9925, 12345, 0, 0], + [3, 9950, 10000, 9600, 9925, 12345, 0, 0], # Exit with stoploss hit [4, 9925, 9975, 9875, 9900, 12345, 0, 0], [5, 9900, 9950, 9850, 9900, 12345, 0, 0]], - stop_loss=-0.03, roi=1, trades=1, profit_perc=-0.03, sell_r=SellType.STOP_LOSS) #should be - # stop_loss=-0.03, roi=1, trades=1, profit_perc=-0.007, sell_r=SellType.FORCE_SELL) # + stop_loss=-0.03, roi=1, trades=1, profit_perc=-0.03, sell_r=SellType.STOP_LOSS) # Test 3 Candle drops 4%, Recovers 1%. @@ -101,16 +100,13 @@ tc2 = BTContainer(data=[ # Trade-B: Stop-Loss Triggered 2% Loss tc3 = BTContainer(data=[ [0, 10000, 10050, 9950, 9975, 12345, 1, 0], - [1, 10000, 10050, 9950, 9975, 12345, 0, 0], + [1, 10000, 10050, 9950, 9975, 12345, 0, 0], # enter trade (signal on last candle) [2, 9975, 10025, 9600, 9950, 12345, 0, 0], - [3, 9950, 10000, 9900, 9925, 12345, 1, 0], + [3, 9950, 10000, 9900, 9925, 12345, 1, 0], # enter trade 2 (signal on last candle) [4, 9950, 10000, 9900, 9925, 12345, 0, 0], [5, 9925, 9975, 8000, 8000, 12345, 0, 0], [6, 9900, 9950, 9950, 9900, 12345, 0, 0]], - stop_loss=-0.02, roi=1, trades=2, profit_perc=-0.04, sell_r=SellType.STOP_LOSS) #should be - # stop_loss=-0.02, roi=1, trades=1, profit_perc=-0.02, sell_r=SellType.STOP_LOSS) #should be - # stop_loss=-0.02, roi=1, trades=1, profit_perc=-0.012, sell_r=SellType.FORCE_SELL) # - + stop_loss=-0.02, roi=1, trades=2, profit_perc=-0.04, sell_r=SellType.STOP_LOSS) # Test 4 Minus 3% / recovery +15% # Candle Data for test 4 – Candle drops 3% Closed 15% up @@ -118,13 +114,12 @@ tc3 = BTContainer(data=[ # TC4: Stop-Loss Triggered 2% Loss tc4 = BTContainer(data=[ [0, 10000, 10050, 9950, 9975, 12345, 1, 0], - [1, 10000, 10050, 9950, 9975, 12345, 0, 0], + [1, 10000, 10050, 9950, 9975, 12345, 0, 0], # enter trade (signal on last candle) [2, 9975, 11500, 9700, 11500, 12345, 0, 0], [3, 9950, 10000, 9900, 9925, 12345, 0, 0], - [4, 9925, 9975, 9875, 9900, 12345, 0, 0], + [4, 9925, 9975, 9875, 9900, 12345, 0, 0], # Exit with stoploss hit [5, 9900, 9950, 9850, 9900, 12345, 0, 0]], - stop_loss=-0.02, roi=0.06, trades=1, profit_perc=-0.02, sell_r=SellType.STOP_LOSS) #should be - # stop_loss=-0.02, roi=0.06, trades=1, profit_perc=-0.012, sell_r=SellType.FORCE_SELL) + stop_loss=-0.02, roi=0.06, trades=1, profit_perc=-0.02, sell_r=SellType.STOP_LOSS) # Test 5 / Drops 0.5% Closes +20% # Candle Data for test 5 @@ -132,13 +127,12 @@ tc4 = BTContainer(data=[ # TC5: ROI triggers 3% Gain tc5 = BTContainer(data=[ [0, 10000, 10050, 9960, 9975, 12345, 1, 0], - [1, 10000, 10050, 9960, 9975, 12345, 0, 0], + [1, 10000, 10050, 9960, 9975, 12345, 0, 0], # enter trade (signal on last candle) [2, 9975, 10050, 9950, 9975, 12345, 0, 0], - [3, 9950, 12000, 9950, 12000, 12345, 0, 0], + [3, 9950, 12000, 9950, 12000, 12345, 0, 0], # ROI [4, 9925, 9975, 9945, 9900, 12345, 0, 0], [5, 9900, 9950, 9850, 9900, 12345, 0, 0]], - stop_loss=-0.01, roi=0.03, trades=1, profit_perc=0.03, sell_r=SellType.ROI) #should be - # stop_loss=-0.01, roi=0.03, trades=1, profit_perc=-0.012, sell_r=SellType.FORCE_SELL) + stop_loss=-0.01, roi=0.03, trades=1, profit_perc=0.03, sell_r=SellType.ROI) # Test 6 / Drops 3% / Recovers 6% Positive / Closes 1% positve # Candle Data for test 6 @@ -146,13 +140,12 @@ tc5 = BTContainer(data=[ # TC6: Stop-Loss triggers 2% Loss tc6 = BTContainer(data=[ [0, 10000, 10050, 9950, 9975, 12345, 1, 0], - [1, 10000, 10050, 9950, 9975, 12345, 0, 0], - [2, 9975, 10600, 9700, 10100, 12345, 0, 0], + [1, 10000, 10050, 9950, 9975, 12345, 0, 0], # enter trade (signal on last candle) + [2, 9975, 10600, 9700, 10100, 12345, 0, 0], # Exit with stoploss [3, 9950, 10000, 9900, 9925, 12345, 0, 0], [4, 9925, 9975, 9945, 9900, 12345, 0, 0], [5, 9900, 9950, 9850, 9900, 12345, 0, 0]], - stop_loss=-0.02, roi=0.05, trades=1, profit_perc=-0.02, sell_r=SellType.STOP_LOSS) #should be - # stop_loss=-0.02, roi=0.05, trades=1, profit_perc=-0.012, sell_r=SellType.FORCE_SELL) # + stop_loss=-0.02, roi=0.05, trades=1, profit_perc=-0.02, sell_r=SellType.STOP_LOSS) # Test 7 - 6% Positive / 1% Negative / Close 1% Positve # Candle Data for test 7 @@ -160,13 +153,12 @@ tc6 = BTContainer(data=[ # TC7: ROI Triggers 3% Gain tc7 = BTContainer(data=[ [0, 10000, 10050, 9950, 9975, 12345, 1, 0], - [1, 10000, 10050, 9950, 9975, 12345, 0, 0], - [2, 9975, 10600, 9900, 10100, 12345, 0, 0], + [1, 10000, 10050, 9950, 9975, 12345, 0, 0], # enter trade (signal on last candle) + [2, 9975, 10600, 9900, 10100, 12345, 0, 0], # ROI [3, 9950, 10000, 9900, 9925, 12345, 0, 0], [4, 9925, 9975, 9945, 9900, 12345, 0, 0], [5, 9900, 9950, 9850, 9900, 12345, 0, 0]], - stop_loss=-0.02, roi=0.03, trades=1, profit_perc=0.03, sell_r=SellType.ROI) #should be - # stop_loss=-0.02, roi=0.03, trades=1, profit_perc=-0.012, sell_r=SellType.FORCE_SELL) # + stop_loss=-0.02, roi=0.03, trades=1, profit_perc=0.03, sell_r=SellType.ROI) TESTS = [ # tc_profit1, @@ -189,7 +181,7 @@ def test_backtest_results(default_conf, fee, mocker, caplog, data) -> None: """ default_conf["stoploss"] = data.stop_loss default_conf["minimal_roi"] = {"0": data.roi} - # mocker.patch('freqtrade.exchange.Exchange.get_fee', fee) + mocker.patch('freqtrade.exchange.Exchange.get_fee', fee) # TODO: don't Mock fee to for now mocker.patch('freqtrade.exchange.Exchange.get_fee', MagicMock(return_value=0.0)) patch_exchange(mocker) From 936441a853bc67a2628e3a8cb36f036de4662968 Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Tue, 30 Oct 2018 13:33:07 +0100 Subject: [PATCH 204/699] Update ccxt from 1.17.432 to 1.17.436 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index da382e82b..5244b54a8 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -ccxt==1.17.432 +ccxt==1.17.436 SQLAlchemy==1.2.12 python-telegram-bot==11.1.0 arrow==0.12.1 From b383113d6cfbce19db1188a15e8af921bc201b63 Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 30 Oct 2018 19:33:32 +0100 Subject: [PATCH 205/699] Test open / close time - small refactorings --- .../tests/optimize/test_backtest_detail.py | 119 +++++++++--------- 1 file changed, 63 insertions(+), 56 deletions(-) diff --git a/freqtrade/tests/optimize/test_backtest_detail.py b/freqtrade/tests/optimize/test_backtest_detail.py index dda3be93f..705a803cc 100644 --- a/freqtrade/tests/optimize/test_backtest_detail.py +++ b/freqtrade/tests/optimize/test_backtest_detail.py @@ -17,6 +17,12 @@ ticker_start_time = arrow.get(2018, 10, 3) ticker_interval_in_minute = 60 +class BTrade(NamedTuple): + sell_r: SellType + open_tick: int + close_tick: int + + class BTContainer(NamedTuple): """ NamedTuple Defining BacktestResults inputs. @@ -24,71 +30,56 @@ class BTContainer(NamedTuple): data: List[float] stop_loss: float roi: float - trades: int + trades: List[BTrade] profit_perc: float - sell_r: SellType + + +def _get_frame_time(offset): + return ticker_start_time.shift( + minutes=(offset * ticker_interval_in_minute)).datetime def _build_dataframe(ticker_with_signals): columns = ['date', 'open', 'high', 'low', 'close', 'volume', 'buy', 'sell'] frame = DataFrame.from_records(ticker_with_signals, columns=columns) - frame['date'] = frame['date'].apply(lambda x: ticker_start_time.shift( - minutes=(x * ticker_interval_in_minute)).datetime) + frame['date'] = frame['date'].apply(_get_frame_time) # Ensure floats are in place for column in ['open', 'high', 'low', 'close', 'volume']: frame[column] = frame[column].astype('float64') return frame -data_profit = [ - [0, 0.0009910, 0.001011, 0.00098618, 0.001000, 12345, 1, 0], - [1, 0.001000, 0.001010, 0.0009900, 0.0009900, 12345, 0, 0], - [2, 0.0009900, 0.001011, 0.00091618, 0.0009900, 12345, 0, 0], - [3, 0.001000, 0.001011, 0.00098618, 0.001100, 12345, 0, 1], - [4, 0.001000, 0.001011, 0.00098618, 0.0009900, 12345, 0, 0]] - -tc_profit1 = BTContainer(data=data_profit, stop_loss=-0.01, roi=1, trades=1, - profit_perc=0.10557, sell_r=SellType.STOP_LOSS) # should be stoploss - drops 8% -tc_profit2 = BTContainer(data=data_profit, stop_loss=-0.10, roi=1, - trades=1, profit_perc=0.10557, sell_r=SellType.STOP_LOSS) - - -tc_loss0 = BTContainer(data=[ - [0, 0.0009910, 0.001011, 0.00098618, 0.001000, 12345, 1, 0], - [1, 0.001000, 0.001010, 0.0009900, 0.001000, 12345, 0, 0], - [2, 0.001000, 0.001011, 0.0010618, 0.00091618, 12345, 0, 0], - [3, 0.001000, 0.001011, 0.00098618, 0.00091618, 12345, 0, 0], - [4, 0.001000, 0.001011, 0.00098618, 0.00091618, 12345, 0, 0]], - stop_loss=-0.05, roi=1, trades=1, profit_perc=-0.08839, sell_r=SellType.STOP_LOSS) - - # Test 1 Minus 8% Close # Candle Data for test 1 – close at -8% (9200) # Test with Stop-loss at 1% # TC1: Stop-Loss Triggered 1% loss -tc1 = BTContainer(data=[ +tc0 = BTContainer(data=[ [0, 10000.0, 10050, 9950, 9975, 12345, 1, 0], [1, 10000, 10050, 9950, 9975, 12345, 0, 0], # enter trade (signal on last candle) [2, 9975, 10025, 9200, 9200, 12345, 0, 0], # Exit with stoploss hit [3, 9950, 10000, 9960, 9955, 12345, 0, 0], [4, 9955, 9975, 9955, 9990, 12345, 0, 0], [5, 9990, 9990, 9990, 9900, 12345, 0, 0]], - stop_loss=-0.01, roi=1, trades=1, profit_perc=-0.01, sell_r=SellType.STOP_LOSS) + stop_loss=-0.01, roi=1, profit_perc=-0.01, + trades=[BTrade(sell_r=SellType.STOP_LOSS, open_tick=1, close_tick=2)] +) # Test 2 Minus 4% Low, minus 1% close # Candle Data for test 2 # Test with Stop-Loss at 3% # TC2: Stop-Loss Triggered 3% Loss -tc2 = BTContainer(data=[ +tc1 = BTContainer(data=[ [0, 10000, 10050, 9950, 9975, 12345, 1, 0], [1, 10000, 10050, 9950, 9975, 12345, 0, 0], # enter trade (signal on last candle) [2, 9975, 10025, 9925, 9950, 12345, 0, 0], [3, 9950, 10000, 9600, 9925, 12345, 0, 0], # Exit with stoploss hit [4, 9925, 9975, 9875, 9900, 12345, 0, 0], [5, 9900, 9950, 9850, 9900, 12345, 0, 0]], - stop_loss=-0.03, roi=1, trades=1, profit_perc=-0.03, sell_r=SellType.STOP_LOSS) + stop_loss=-0.03, roi=1, profit_perc=-0.03, + trades=[BTrade(sell_r=SellType.STOP_LOSS, open_tick=1, close_tick=3)] + ) # Test 3 Candle drops 4%, Recovers 1%. @@ -98,79 +89,90 @@ tc2 = BTContainer(data=[ # Test with Stop-Loss at 2% # TC3: Trade-A: Stop-Loss Triggered 2% Loss # Trade-B: Stop-Loss Triggered 2% Loss -tc3 = BTContainer(data=[ +tc2 = BTContainer(data=[ [0, 10000, 10050, 9950, 9975, 12345, 1, 0], [1, 10000, 10050, 9950, 9975, 12345, 0, 0], # enter trade (signal on last candle) [2, 9975, 10025, 9600, 9950, 12345, 0, 0], - [3, 9950, 10000, 9900, 9925, 12345, 1, 0], # enter trade 2 (signal on last candle) - [4, 9950, 10000, 9900, 9925, 12345, 0, 0], + [3, 9950, 10000, 9900, 9925, 12345, 1, 0], + [4, 9950, 10000, 9900, 9925, 12345, 0, 0], # enter trade 2 (signal on last candle) [5, 9925, 9975, 8000, 8000, 12345, 0, 0], [6, 9900, 9950, 9950, 9900, 12345, 0, 0]], - stop_loss=-0.02, roi=1, trades=2, profit_perc=-0.04, sell_r=SellType.STOP_LOSS) + stop_loss=-0.02, roi=1, profit_perc=-0.04, + trades=[BTrade(sell_r=SellType.STOP_LOSS, open_tick=1, close_tick=2), + BTrade(sell_r=SellType.STOP_LOSS, open_tick=4, close_tick=5)] +) # Test 4 Minus 3% / recovery +15% # Candle Data for test 4 – Candle drops 3% Closed 15% up # Test with Stop-loss at 2% ROI 6% # TC4: Stop-Loss Triggered 2% Loss -tc4 = BTContainer(data=[ +tc3 = BTContainer(data=[ [0, 10000, 10050, 9950, 9975, 12345, 1, 0], [1, 10000, 10050, 9950, 9975, 12345, 0, 0], # enter trade (signal on last candle) [2, 9975, 11500, 9700, 11500, 12345, 0, 0], [3, 9950, 10000, 9900, 9925, 12345, 0, 0], [4, 9925, 9975, 9875, 9900, 12345, 0, 0], # Exit with stoploss hit [5, 9900, 9950, 9850, 9900, 12345, 0, 0]], - stop_loss=-0.02, roi=0.06, trades=1, profit_perc=-0.02, sell_r=SellType.STOP_LOSS) + stop_loss=-0.02, roi=0.06, profit_perc=-0.02, + trades=[BTrade(sell_r=SellType.STOP_LOSS, open_tick=1, close_tick=2)] +) # Test 5 / Drops 0.5% Closes +20% # Candle Data for test 5 # Set stop-loss at 1% ROI 3% # TC5: ROI triggers 3% Gain -tc5 = BTContainer(data=[ +tc4 = BTContainer(data=[ [0, 10000, 10050, 9960, 9975, 12345, 1, 0], [1, 10000, 10050, 9960, 9975, 12345, 0, 0], # enter trade (signal on last candle) [2, 9975, 10050, 9950, 9975, 12345, 0, 0], [3, 9950, 12000, 9950, 12000, 12345, 0, 0], # ROI [4, 9925, 9975, 9945, 9900, 12345, 0, 0], [5, 9900, 9950, 9850, 9900, 12345, 0, 0]], - stop_loss=-0.01, roi=0.03, trades=1, profit_perc=0.03, sell_r=SellType.ROI) + stop_loss=-0.01, roi=0.03, profit_perc=0.03, + trades=[BTrade(sell_r=SellType.ROI, open_tick=1, close_tick=3)] +) # Test 6 / Drops 3% / Recovers 6% Positive / Closes 1% positve # Candle Data for test 6 # Set stop-loss at 2% ROI at 5% # TC6: Stop-Loss triggers 2% Loss -tc6 = BTContainer(data=[ +tc5 = BTContainer(data=[ [0, 10000, 10050, 9950, 9975, 12345, 1, 0], [1, 10000, 10050, 9950, 9975, 12345, 0, 0], # enter trade (signal on last candle) [2, 9975, 10600, 9700, 10100, 12345, 0, 0], # Exit with stoploss [3, 9950, 10000, 9900, 9925, 12345, 0, 0], [4, 9925, 9975, 9945, 9900, 12345, 0, 0], [5, 9900, 9950, 9850, 9900, 12345, 0, 0]], - stop_loss=-0.02, roi=0.05, trades=1, profit_perc=-0.02, sell_r=SellType.STOP_LOSS) + stop_loss=-0.02, roi=0.05, profit_perc=-0.02, + trades=[BTrade(sell_r=SellType.STOP_LOSS, open_tick=1, close_tick=2)] +) # Test 7 - 6% Positive / 1% Negative / Close 1% Positve # Candle Data for test 7 # Set stop-loss at 2% ROI at 3% # TC7: ROI Triggers 3% Gain -tc7 = BTContainer(data=[ +tc6 = BTContainer(data=[ [0, 10000, 10050, 9950, 9975, 12345, 1, 0], [1, 10000, 10050, 9950, 9975, 12345, 0, 0], # enter trade (signal on last candle) [2, 9975, 10600, 9900, 10100, 12345, 0, 0], # ROI [3, 9950, 10000, 9900, 9925, 12345, 0, 0], [4, 9925, 9975, 9945, 9900, 12345, 0, 0], [5, 9900, 9950, 9850, 9900, 12345, 0, 0]], - stop_loss=-0.02, roi=0.03, trades=1, profit_perc=0.03, sell_r=SellType.ROI) + stop_loss=-0.02, roi=0.03, profit_perc=0.03, + trades=[BTrade(sell_r=SellType.ROI, open_tick=1, close_tick=2)] + ) TESTS = [ # tc_profit1, # tc_profit2, # tc_loss0, + tc0, tc1, tc2, tc3, tc4, tc5, tc6, - tc7, ] @@ -203,17 +205,22 @@ def test_backtest_results(default_conf, fee, mocker, caplog, data) -> None: ) print(results.T) - assert len(results) == data.trades + assert len(results) == len(data.trades) assert round(results["profit_percent"].sum(), 3) == round(data.profit_perc, 3) - if data.sell_r == SellType.STOP_LOSS: - assert log_has("Stop loss hit.", caplog.record_tuples) - else: - assert not log_has("Stop loss hit.", caplog.record_tuples) - log_test = (f'Force_selling still open trade UNITTEST/BTC with ' - f'{results.iloc[-1].profit_percent} perc - {results.iloc[-1].profit_abs}') - if data.sell_r == SellType.FORCE_SELL: - assert log_has(log_test, - caplog.record_tuples) - else: - assert not log_has(log_test, - caplog.record_tuples) + # if data.sell_r == SellType.STOP_LOSS: + # assert log_has("Stop loss hit.", caplog.record_tuples) + # else: + # assert not log_has("Stop loss hit.", caplog.record_tuples) + # log_test = (f'Force_selling still open trade UNITTEST/BTC with ' + # f'{results.iloc[-1].profit_percent} perc - {results.iloc[-1].profit_abs}') + # if data.sell_r == SellType.FORCE_SELL: + # assert log_has(log_test, + # caplog.record_tuples) + # else: + # assert not log_has(log_test, + # caplog.record_tuples) + for c, trade in enumerate(data.trades): + res = results.iloc[c] + assert res.sell_reason == trade.sell_r + assert res.open_time == _get_frame_time(trade.open_tick) + assert res.close_time == _get_frame_time(trade.close_tick) From fc3f8b436d1f1ba5da3332fd3be8ea72860e2655 Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 30 Oct 2018 19:36:19 +0100 Subject: [PATCH 206/699] some more cleanup --- .../tests/optimize/test_backtest_detail.py | 31 +++++++++---------- 1 file changed, 14 insertions(+), 17 deletions(-) diff --git a/freqtrade/tests/optimize/test_backtest_detail.py b/freqtrade/tests/optimize/test_backtest_detail.py index 705a803cc..e8b33d5f2 100644 --- a/freqtrade/tests/optimize/test_backtest_detail.py +++ b/freqtrade/tests/optimize/test_backtest_detail.py @@ -18,6 +18,9 @@ ticker_interval_in_minute = 60 class BTrade(NamedTuple): + """ + Minimalistic Trade result used for functional backtesting + """ sell_r: SellType open_tick: int close_tick: int @@ -25,7 +28,7 @@ class BTrade(NamedTuple): class BTContainer(NamedTuple): """ - NamedTuple Defining BacktestResults inputs. + Minimal BacktestContainer defining Backtest inputs and results. """ data: List[float] stop_loss: float @@ -50,14 +53,13 @@ def _build_dataframe(ticker_with_signals): return frame -# Test 1 Minus 8% Close -# Candle Data for test 1 – close at -8% (9200) +# Test 0 Minus 8% Close # Test with Stop-loss at 1% # TC1: Stop-Loss Triggered 1% loss tc0 = BTContainer(data=[ [0, 10000.0, 10050, 9950, 9975, 12345, 1, 0], [1, 10000, 10050, 9950, 9975, 12345, 0, 0], # enter trade (signal on last candle) - [2, 9975, 10025, 9200, 9200, 12345, 0, 0], # Exit with stoploss hit + [2, 9975, 10025, 9200, 9200, 12345, 0, 0], # exit with stoploss hit [3, 9950, 10000, 9960, 9955, 12345, 0, 0], [4, 9955, 9975, 9955, 9990, 12345, 0, 0], [5, 9990, 9990, 9990, 9900, 12345, 0, 0]], @@ -66,15 +68,14 @@ tc0 = BTContainer(data=[ ) -# Test 2 Minus 4% Low, minus 1% close -# Candle Data for test 2 +# Test 1 Minus 4% Low, minus 1% close # Test with Stop-Loss at 3% # TC2: Stop-Loss Triggered 3% Loss tc1 = BTContainer(data=[ [0, 10000, 10050, 9950, 9975, 12345, 1, 0], [1, 10000, 10050, 9950, 9975, 12345, 0, 0], # enter trade (signal on last candle) [2, 9975, 10025, 9925, 9950, 12345, 0, 0], - [3, 9950, 10000, 9600, 9925, 12345, 0, 0], # Exit with stoploss hit + [3, 9950, 10000, 9600, 9925, 12345, 0, 0], # exit with stoploss hit [4, 9925, 9975, 9875, 9900, 12345, 0, 0], [5, 9900, 9950, 9850, 9900, 12345, 0, 0]], stop_loss=-0.03, roi=1, profit_perc=-0.03, @@ -92,10 +93,10 @@ tc1 = BTContainer(data=[ tc2 = BTContainer(data=[ [0, 10000, 10050, 9950, 9975, 12345, 1, 0], [1, 10000, 10050, 9950, 9975, 12345, 0, 0], # enter trade (signal on last candle) - [2, 9975, 10025, 9600, 9950, 12345, 0, 0], + [2, 9975, 10025, 9600, 9950, 12345, 0, 0], # exit with stoploss hit [3, 9950, 10000, 9900, 9925, 12345, 1, 0], [4, 9950, 10000, 9900, 9925, 12345, 0, 0], # enter trade 2 (signal on last candle) - [5, 9925, 9975, 8000, 8000, 12345, 0, 0], + [5, 9925, 9975, 8000, 8000, 12345, 0, 0], # exit with stoploss hit [6, 9900, 9950, 9950, 9900, 12345, 0, 0]], stop_loss=-0.02, roi=1, profit_perc=-0.04, trades=[BTrade(sell_r=SellType.STOP_LOSS, open_tick=1, close_tick=2), @@ -103,22 +104,21 @@ tc2 = BTContainer(data=[ ) # Test 4 Minus 3% / recovery +15% -# Candle Data for test 4 – Candle drops 3% Closed 15% up +# Candle Data for test 3 – Candle drops 3% Closed 15% up # Test with Stop-loss at 2% ROI 6% # TC4: Stop-Loss Triggered 2% Loss tc3 = BTContainer(data=[ [0, 10000, 10050, 9950, 9975, 12345, 1, 0], [1, 10000, 10050, 9950, 9975, 12345, 0, 0], # enter trade (signal on last candle) - [2, 9975, 11500, 9700, 11500, 12345, 0, 0], + [2, 9975, 11500, 9700, 11500, 12345, 0, 0], # Exit with stoploss hit [3, 9950, 10000, 9900, 9925, 12345, 0, 0], - [4, 9925, 9975, 9875, 9900, 12345, 0, 0], # Exit with stoploss hit + [4, 9925, 9975, 9875, 9900, 12345, 0, 0], [5, 9900, 9950, 9850, 9900, 12345, 0, 0]], stop_loss=-0.02, roi=0.06, profit_perc=-0.02, trades=[BTrade(sell_r=SellType.STOP_LOSS, open_tick=1, close_tick=2)] ) -# Test 5 / Drops 0.5% Closes +20% -# Candle Data for test 5 +# Test 4 / Drops 0.5% Closes +20% # Set stop-loss at 1% ROI 3% # TC5: ROI triggers 3% Gain tc4 = BTContainer(data=[ @@ -163,9 +163,6 @@ tc6 = BTContainer(data=[ ) TESTS = [ - # tc_profit1, - # tc_profit2, - # tc_loss0, tc0, tc1, tc2, From 3679b0948a775a600dcad533acfaa03bbe7199a4 Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 30 Oct 2018 19:37:45 +0100 Subject: [PATCH 207/699] cleanup interface --- freqtrade/strategy/interface.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/freqtrade/strategy/interface.py b/freqtrade/strategy/interface.py index 21efa09d6..710a4e8f1 100644 --- a/freqtrade/strategy/interface.py +++ b/freqtrade/strategy/interface.py @@ -203,7 +203,7 @@ class IStrategy(ABC): return buy, sell def should_sell(self, trade: Trade, rate: float, date: datetime, buy: bool, - sell: bool, low: float=None, high: float=None) -> SellCheckTuple: + sell: bool, low: float = None, high: float = None) -> SellCheckTuple: """ This function evaluate if on the condition required to trigger a sell has been reached if the threshold is reached and updates the trade record. @@ -212,8 +212,8 @@ class IStrategy(ABC): # Set current rate to low for backtesting sell current_rate = rate if not low else low current_profit = trade.calc_profit_percent(current_rate) - stoplossflag = self.stop_loss_reached(current_rate=current_rate, trade=trade, current_time=date, - current_profit=current_profit) + stoplossflag = self.stop_loss_reached(current_rate=current_rate, trade=trade, + current_time=date, current_profit=current_profit) if stoplossflag.sell_flag: return stoplossflag # Set current rate to low for backtesting sell From 9798e881cbf1db8f2921a0f9a44ce6c1c77cacbd Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 30 Oct 2018 19:44:31 +0100 Subject: [PATCH 208/699] refactor sell_r to sell_reason --- .../tests/optimize/test_backtest_detail.py | 20 +++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/freqtrade/tests/optimize/test_backtest_detail.py b/freqtrade/tests/optimize/test_backtest_detail.py index e8b33d5f2..dc767210f 100644 --- a/freqtrade/tests/optimize/test_backtest_detail.py +++ b/freqtrade/tests/optimize/test_backtest_detail.py @@ -21,7 +21,7 @@ class BTrade(NamedTuple): """ Minimalistic Trade result used for functional backtesting """ - sell_r: SellType + sell_reason: SellType open_tick: int close_tick: int @@ -64,7 +64,7 @@ tc0 = BTContainer(data=[ [4, 9955, 9975, 9955, 9990, 12345, 0, 0], [5, 9990, 9990, 9990, 9900, 12345, 0, 0]], stop_loss=-0.01, roi=1, profit_perc=-0.01, - trades=[BTrade(sell_r=SellType.STOP_LOSS, open_tick=1, close_tick=2)] + trades=[BTrade(sell_reason=SellType.STOP_LOSS, open_tick=1, close_tick=2)] ) @@ -79,7 +79,7 @@ tc1 = BTContainer(data=[ [4, 9925, 9975, 9875, 9900, 12345, 0, 0], [5, 9900, 9950, 9850, 9900, 12345, 0, 0]], stop_loss=-0.03, roi=1, profit_perc=-0.03, - trades=[BTrade(sell_r=SellType.STOP_LOSS, open_tick=1, close_tick=3)] + trades=[BTrade(sell_reason=SellType.STOP_LOSS, open_tick=1, close_tick=3)] ) @@ -99,8 +99,8 @@ tc2 = BTContainer(data=[ [5, 9925, 9975, 8000, 8000, 12345, 0, 0], # exit with stoploss hit [6, 9900, 9950, 9950, 9900, 12345, 0, 0]], stop_loss=-0.02, roi=1, profit_perc=-0.04, - trades=[BTrade(sell_r=SellType.STOP_LOSS, open_tick=1, close_tick=2), - BTrade(sell_r=SellType.STOP_LOSS, open_tick=4, close_tick=5)] + trades=[BTrade(sell_reason=SellType.STOP_LOSS, open_tick=1, close_tick=2), + BTrade(sell_reason=SellType.STOP_LOSS, open_tick=4, close_tick=5)] ) # Test 4 Minus 3% / recovery +15% @@ -115,7 +115,7 @@ tc3 = BTContainer(data=[ [4, 9925, 9975, 9875, 9900, 12345, 0, 0], [5, 9900, 9950, 9850, 9900, 12345, 0, 0]], stop_loss=-0.02, roi=0.06, profit_perc=-0.02, - trades=[BTrade(sell_r=SellType.STOP_LOSS, open_tick=1, close_tick=2)] + trades=[BTrade(sell_reason=SellType.STOP_LOSS, open_tick=1, close_tick=2)] ) # Test 4 / Drops 0.5% Closes +20% @@ -129,7 +129,7 @@ tc4 = BTContainer(data=[ [4, 9925, 9975, 9945, 9900, 12345, 0, 0], [5, 9900, 9950, 9850, 9900, 12345, 0, 0]], stop_loss=-0.01, roi=0.03, profit_perc=0.03, - trades=[BTrade(sell_r=SellType.ROI, open_tick=1, close_tick=3)] + trades=[BTrade(sell_reason=SellType.ROI, open_tick=1, close_tick=3)] ) # Test 6 / Drops 3% / Recovers 6% Positive / Closes 1% positve @@ -144,7 +144,7 @@ tc5 = BTContainer(data=[ [4, 9925, 9975, 9945, 9900, 12345, 0, 0], [5, 9900, 9950, 9850, 9900, 12345, 0, 0]], stop_loss=-0.02, roi=0.05, profit_perc=-0.02, - trades=[BTrade(sell_r=SellType.STOP_LOSS, open_tick=1, close_tick=2)] + trades=[BTrade(sell_reason=SellType.STOP_LOSS, open_tick=1, close_tick=2)] ) # Test 7 - 6% Positive / 1% Negative / Close 1% Positve @@ -159,7 +159,7 @@ tc6 = BTContainer(data=[ [4, 9925, 9975, 9945, 9900, 12345, 0, 0], [5, 9900, 9950, 9850, 9900, 12345, 0, 0]], stop_loss=-0.02, roi=0.03, profit_perc=0.03, - trades=[BTrade(sell_r=SellType.ROI, open_tick=1, close_tick=2)] + trades=[BTrade(sell_reason=SellType.ROI, open_tick=1, close_tick=2)] ) TESTS = [ @@ -218,6 +218,6 @@ def test_backtest_results(default_conf, fee, mocker, caplog, data) -> None: # caplog.record_tuples) for c, trade in enumerate(data.trades): res = results.iloc[c] - assert res.sell_reason == trade.sell_r + assert res.sell_reason == trade.sell_reason assert res.open_time == _get_frame_time(trade.open_tick) assert res.close_time == _get_frame_time(trade.close_tick) From e442e22a20bdad7833c5ba6c402397cd3f2f5bbc Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 30 Oct 2018 19:58:06 +0100 Subject: [PATCH 209/699] refactorign --- freqtrade/tests/optimize/test_backtest_detail.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/freqtrade/tests/optimize/test_backtest_detail.py b/freqtrade/tests/optimize/test_backtest_detail.py index dc767210f..4b6bcd4ce 100644 --- a/freqtrade/tests/optimize/test_backtest_detail.py +++ b/freqtrade/tests/optimize/test_backtest_detail.py @@ -37,16 +37,16 @@ class BTContainer(NamedTuple): profit_perc: float -def _get_frame_time(offset): +def _get_frame_time_from_offset(offset): return ticker_start_time.shift( minutes=(offset * ticker_interval_in_minute)).datetime -def _build_dataframe(ticker_with_signals): +def _build_backtest_dataframe(ticker_with_signals): columns = ['date', 'open', 'high', 'low', 'close', 'volume', 'buy', 'sell'] frame = DataFrame.from_records(ticker_with_signals, columns=columns) - frame['date'] = frame['date'].apply(_get_frame_time) + frame['date'] = frame['date'].apply(_get_frame_time_from_offset) # Ensure floats are in place for column in ['open', 'high', 'low', 'close', 'volume']: frame[column] = frame[column].astype('float64') @@ -184,7 +184,7 @@ def test_backtest_results(default_conf, fee, mocker, caplog, data) -> None: # TODO: don't Mock fee to for now mocker.patch('freqtrade.exchange.Exchange.get_fee', MagicMock(return_value=0.0)) patch_exchange(mocker) - frame = _build_dataframe(data.data) + frame = _build_backtest_dataframe(data.data) backtesting = Backtesting(default_conf) backtesting.advise_buy = lambda a, m: frame backtesting.advise_sell = lambda a, m: frame @@ -219,5 +219,5 @@ def test_backtest_results(default_conf, fee, mocker, caplog, data) -> None: for c, trade in enumerate(data.trades): res = results.iloc[c] assert res.sell_reason == trade.sell_reason - assert res.open_time == _get_frame_time(trade.open_tick) - assert res.close_time == _get_frame_time(trade.close_tick) + assert res.open_time == _get_frame_time_from_offset(trade.open_tick) + assert res.close_time == _get_frame_time_from_offset(trade.close_tick) From 9e921d441011958ed06ae5953f09c36e71f763a9 Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 30 Oct 2018 20:02:01 +0100 Subject: [PATCH 210/699] refactor General bt-utils out of detailed backtest file --- freqtrade/tests/optimize/__init__.py | 45 +++++++++++++++++ .../tests/optimize/test_backtest_detail.py | 48 ++----------------- 2 files changed, 48 insertions(+), 45 deletions(-) diff --git a/freqtrade/tests/optimize/__init__.py b/freqtrade/tests/optimize/__init__.py index e69de29bb..2b7222e88 100644 --- a/freqtrade/tests/optimize/__init__.py +++ b/freqtrade/tests/optimize/__init__.py @@ -0,0 +1,45 @@ +from typing import NamedTuple, List + +import arrow +from pandas import DataFrame + +from freqtrade.strategy.interface import SellType + +ticker_start_time = arrow.get(2018, 10, 3) +ticker_interval_in_minute = 60 + + +class BTrade(NamedTuple): + """ + Minimalistic Trade result used for functional backtesting + """ + sell_reason: SellType + open_tick: int + close_tick: int + + +class BTContainer(NamedTuple): + """ + Minimal BacktestContainer defining Backtest inputs and results. + """ + data: List[float] + stop_loss: float + roi: float + trades: List[BTrade] + profit_perc: float + + +def _get_frame_time_from_offset(offset): + return ticker_start_time.shift( + minutes=(offset * ticker_interval_in_minute)).datetime + + +def _build_backtest_dataframe(ticker_with_signals): + columns = ['date', 'open', 'high', 'low', 'close', 'volume', 'buy', 'sell'] + + frame = DataFrame.from_records(ticker_with_signals, columns=columns) + frame['date'] = frame['date'].apply(_get_frame_time_from_offset) + # Ensure floats are in place + for column in ['open', 'high', 'low', 'close', 'volume']: + frame[column] = frame[column].astype('float64') + return frame diff --git a/freqtrade/tests/optimize/test_backtest_detail.py b/freqtrade/tests/optimize/test_backtest_detail.py index 4b6bcd4ce..323323683 100644 --- a/freqtrade/tests/optimize/test_backtest_detail.py +++ b/freqtrade/tests/optimize/test_backtest_detail.py @@ -1,56 +1,16 @@ # pragma pylint: disable=missing-docstring, W0212, line-too-long, C0103, unused-argument import logging from unittest.mock import MagicMock -from typing import NamedTuple, List from pandas import DataFrame import pytest -import arrow from freqtrade.optimize.backtesting import Backtesting from freqtrade.strategy.interface import SellType -from freqtrade.tests.conftest import patch_exchange, log_has - - -ticker_start_time = arrow.get(2018, 10, 3) -ticker_interval_in_minute = 60 - - -class BTrade(NamedTuple): - """ - Minimalistic Trade result used for functional backtesting - """ - sell_reason: SellType - open_tick: int - close_tick: int - - -class BTContainer(NamedTuple): - """ - Minimal BacktestContainer defining Backtest inputs and results. - """ - data: List[float] - stop_loss: float - roi: float - trades: List[BTrade] - profit_perc: float - - -def _get_frame_time_from_offset(offset): - return ticker_start_time.shift( - minutes=(offset * ticker_interval_in_minute)).datetime - - -def _build_backtest_dataframe(ticker_with_signals): - columns = ['date', 'open', 'high', 'low', 'close', 'volume', 'buy', 'sell'] - - frame = DataFrame.from_records(ticker_with_signals, columns=columns) - frame['date'] = frame['date'].apply(_get_frame_time_from_offset) - # Ensure floats are in place - for column in ['open', 'high', 'low', 'close', 'volume']: - frame[column] = frame[column].astype('float64') - return frame +from freqtrade.tests.optimize import (BTrade, BTContainer, _build_backtest_dataframe, + _get_frame_time_from_offset) +from freqtrade.tests.conftest import patch_exchange # Test 0 Minus 8% Close @@ -180,8 +140,6 @@ def test_backtest_results(default_conf, fee, mocker, caplog, data) -> None: """ default_conf["stoploss"] = data.stop_loss default_conf["minimal_roi"] = {"0": data.roi} - mocker.patch('freqtrade.exchange.Exchange.get_fee', fee) - # TODO: don't Mock fee to for now mocker.patch('freqtrade.exchange.Exchange.get_fee', MagicMock(return_value=0.0)) patch_exchange(mocker) frame = _build_backtest_dataframe(data.data) From f96f0cdea7b665435240c4bef07a9f01ba31f425 Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 30 Oct 2018 20:02:31 +0100 Subject: [PATCH 211/699] Add additional comment --- freqtrade/optimize/backtesting.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index 9678b6d09..b5c883d4f 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -208,7 +208,9 @@ class Backtesting(object): sell = self.strategy.should_sell(trade, sell_row.open, sell_row.date, buy_signal, sell_row.sell, low=sell_row.low, high=sell_row.high) if sell.sell_flag: + trade_dur = int((sell_row.date - buy_row.date).total_seconds() // 60) + # Special handling if high or low hit STOP_LOSS or ROI if sell.sell_type in (SellType.STOP_LOSS, SellType.TRAILING_STOP_LOSS): # Set close_rate to stoploss closerate = trade.stop_loss From fe2c158e59668eb31f54ecc8fee485a0b805bc3c Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 30 Oct 2018 20:13:56 +0100 Subject: [PATCH 212/699] Adjust sell-rate to new backtesting (respects roi/stoploss) --- freqtrade/tests/optimize/test_backtesting.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/tests/optimize/test_backtesting.py b/freqtrade/tests/optimize/test_backtesting.py index fc08eba89..2d3d49ab0 100644 --- a/freqtrade/tests/optimize/test_backtesting.py +++ b/freqtrade/tests/optimize/test_backtesting.py @@ -580,7 +580,7 @@ def test_processed(default_conf, mocker) -> None: def test_backtest_pricecontours(default_conf, fee, mocker) -> None: mocker.patch('freqtrade.exchange.Exchange.get_fee', fee) - tests = [['raise', 18], ['lower', 0], ['sine', 16]] + tests = [['raise', 18], ['lower', 0], ['sine', 19]] for [contour, numres] in tests: simple_backtest(default_conf, contour, numres, mocker) From 8c93760a6dda1eecdb4ec4276dea5cc04106eec2 Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 30 Oct 2018 20:23:31 +0100 Subject: [PATCH 213/699] simplify some code --- freqtrade/optimize/backtesting.py | 2 +- freqtrade/strategy/interface.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index b5c883d4f..1f02b9793 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -217,7 +217,7 @@ class Backtesting(object): elif sell.sell_type == (SellType.ROI): # get entry in min_roi >= to trade duration roi_entry = max(list(filter(lambda x: trade_dur >= x, - list(self.strategy.minimal_roi.keys())))) + self.strategy.minimal_roi.keys()))) # set close-rate to min-roi closerate = trade.open_rate + trade.open_rate * \ self.strategy.minimal_roi[roi_entry] diff --git a/freqtrade/strategy/interface.py b/freqtrade/strategy/interface.py index 710a4e8f1..27da6147c 100644 --- a/freqtrade/strategy/interface.py +++ b/freqtrade/strategy/interface.py @@ -210,14 +210,14 @@ class IStrategy(ABC): :return: True if trade should be sold, False otherwise """ # Set current rate to low for backtesting sell - current_rate = rate if not low else low + current_rate = low or rate current_profit = trade.calc_profit_percent(current_rate) stoplossflag = self.stop_loss_reached(current_rate=current_rate, trade=trade, current_time=date, current_profit=current_profit) if stoplossflag.sell_flag: return stoplossflag # Set current rate to low for backtesting sell - current_rate = rate if not high else high + current_rate = high or rate current_profit = trade.calc_profit_percent(current_rate) experimental = self.config.get('experimental', {}) From 79d1d63e6fd5a06cd77ee6682ce10315e73886bc Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 30 Oct 2018 20:42:34 +0100 Subject: [PATCH 214/699] Align data (by halfing all data) --- .../tests/optimize/test_backtest_detail.py | 86 +++++++++---------- 1 file changed, 43 insertions(+), 43 deletions(-) diff --git a/freqtrade/tests/optimize/test_backtest_detail.py b/freqtrade/tests/optimize/test_backtest_detail.py index 323323683..5ec9c02c2 100644 --- a/freqtrade/tests/optimize/test_backtest_detail.py +++ b/freqtrade/tests/optimize/test_backtest_detail.py @@ -17,12 +17,12 @@ from freqtrade.tests.conftest import patch_exchange # Test with Stop-loss at 1% # TC1: Stop-Loss Triggered 1% loss tc0 = BTContainer(data=[ - [0, 10000.0, 10050, 9950, 9975, 12345, 1, 0], - [1, 10000, 10050, 9950, 9975, 12345, 0, 0], # enter trade (signal on last candle) - [2, 9975, 10025, 9200, 9200, 12345, 0, 0], # exit with stoploss hit - [3, 9950, 10000, 9960, 9955, 12345, 0, 0], - [4, 9955, 9975, 9955, 9990, 12345, 0, 0], - [5, 9990, 9990, 9990, 9900, 12345, 0, 0]], + [0, 5000, 5025, 4975, 4987, 6172, 1, 0], + [1, 5000, 5025, 4975, 4987, 6172, 0, 0], # enter trade (signal on last candle) + [2, 4987, 5012, 4600, 4600, 6172, 0, 0], # exit with stoploss hit + [3, 4975, 5000, 4980, 4977, 6172, 0, 0], + [4, 4977, 4987, 4977, 4995, 6172, 0, 0], + [5, 4995, 4995, 4995, 4950, 6172, 0, 0]], stop_loss=-0.01, roi=1, profit_perc=-0.01, trades=[BTrade(sell_reason=SellType.STOP_LOSS, open_tick=1, close_tick=2)] ) @@ -32,12 +32,12 @@ tc0 = BTContainer(data=[ # Test with Stop-Loss at 3% # TC2: Stop-Loss Triggered 3% Loss tc1 = BTContainer(data=[ - [0, 10000, 10050, 9950, 9975, 12345, 1, 0], - [1, 10000, 10050, 9950, 9975, 12345, 0, 0], # enter trade (signal on last candle) - [2, 9975, 10025, 9925, 9950, 12345, 0, 0], - [3, 9950, 10000, 9600, 9925, 12345, 0, 0], # exit with stoploss hit - [4, 9925, 9975, 9875, 9900, 12345, 0, 0], - [5, 9900, 9950, 9850, 9900, 12345, 0, 0]], + [0, 5000, 5025, 4975, 4987, 6172, 1, 0], + [1, 5000, 5025, 4975, 4987, 6172, 0, 0], # enter trade (signal on last candle) + [2, 4987, 5012, 4962, 4975, 6172, 0, 0], + [3, 4975, 5000, 4800, 4962, 6172, 0, 0], # exit with stoploss hit + [4, 4962, 4987, 4937, 4950, 6172, 0, 0], + [5, 4950, 4975, 4925, 4950, 6172, 0, 0]], stop_loss=-0.03, roi=1, profit_perc=-0.03, trades=[BTrade(sell_reason=SellType.STOP_LOSS, open_tick=1, close_tick=3)] ) @@ -51,13 +51,13 @@ tc1 = BTContainer(data=[ # TC3: Trade-A: Stop-Loss Triggered 2% Loss # Trade-B: Stop-Loss Triggered 2% Loss tc2 = BTContainer(data=[ - [0, 10000, 10050, 9950, 9975, 12345, 1, 0], - [1, 10000, 10050, 9950, 9975, 12345, 0, 0], # enter trade (signal on last candle) - [2, 9975, 10025, 9600, 9950, 12345, 0, 0], # exit with stoploss hit - [3, 9950, 10000, 9900, 9925, 12345, 1, 0], - [4, 9950, 10000, 9900, 9925, 12345, 0, 0], # enter trade 2 (signal on last candle) - [5, 9925, 9975, 8000, 8000, 12345, 0, 0], # exit with stoploss hit - [6, 9900, 9950, 9950, 9900, 12345, 0, 0]], + [0, 5000, 5025, 4975, 4987, 6172, 1, 0], + [1, 5000, 5025, 4975, 4987, 6172, 0, 0], # enter trade (signal on last candle) + [2, 4987, 5012, 4800, 4975, 6172, 0, 0], # exit with stoploss hit + [3, 4975, 5000, 4950, 4962, 6172, 1, 0], + [4, 4975, 5000, 4950, 4962, 6172, 0, 0], # enter trade 2 (signal on last candle) + [5, 4962, 4987, 4000, 4000, 6172, 0, 0], # exit with stoploss hit + [6, 4950, 4975, 4975, 4950, 6172, 0, 0]], stop_loss=-0.02, roi=1, profit_perc=-0.04, trades=[BTrade(sell_reason=SellType.STOP_LOSS, open_tick=1, close_tick=2), BTrade(sell_reason=SellType.STOP_LOSS, open_tick=4, close_tick=5)] @@ -68,12 +68,12 @@ tc2 = BTContainer(data=[ # Test with Stop-loss at 2% ROI 6% # TC4: Stop-Loss Triggered 2% Loss tc3 = BTContainer(data=[ - [0, 10000, 10050, 9950, 9975, 12345, 1, 0], - [1, 10000, 10050, 9950, 9975, 12345, 0, 0], # enter trade (signal on last candle) - [2, 9975, 11500, 9700, 11500, 12345, 0, 0], # Exit with stoploss hit - [3, 9950, 10000, 9900, 9925, 12345, 0, 0], - [4, 9925, 9975, 9875, 9900, 12345, 0, 0], - [5, 9900, 9950, 9850, 9900, 12345, 0, 0]], + [0, 5000, 5025, 4975, 4987, 6172, 1, 0], + [1, 5000, 5025, 4975, 4987, 6172, 0, 0], # enter trade (signal on last candle) + [2, 4987, 5750, 4850, 5750, 6172, 0, 0], # Exit with stoploss hit + [3, 4975, 5000, 4950, 4962, 6172, 0, 0], + [4, 4962, 4987, 4937, 4950, 6172, 0, 0], + [5, 4950, 4975, 4925, 4950, 6172, 0, 0]], stop_loss=-0.02, roi=0.06, profit_perc=-0.02, trades=[BTrade(sell_reason=SellType.STOP_LOSS, open_tick=1, close_tick=2)] ) @@ -82,12 +82,12 @@ tc3 = BTContainer(data=[ # Set stop-loss at 1% ROI 3% # TC5: ROI triggers 3% Gain tc4 = BTContainer(data=[ - [0, 10000, 10050, 9960, 9975, 12345, 1, 0], - [1, 10000, 10050, 9960, 9975, 12345, 0, 0], # enter trade (signal on last candle) - [2, 9975, 10050, 9950, 9975, 12345, 0, 0], - [3, 9950, 12000, 9950, 12000, 12345, 0, 0], # ROI - [4, 9925, 9975, 9945, 9900, 12345, 0, 0], - [5, 9900, 9950, 9850, 9900, 12345, 0, 0]], + [0, 5000, 5025, 4980, 4987, 6172, 1, 0], + [1, 5000, 5025, 4980, 4987, 6172, 0, 0], # enter trade (signal on last candle) + [2, 4987, 5025, 4975, 4987, 6172, 0, 0], + [3, 4975, 6000, 4975, 6000, 6172, 0, 0], # ROI + [4, 4962, 4987, 4972, 4950, 6172, 0, 0], + [5, 4950, 4975, 4925, 4950, 6172, 0, 0]], stop_loss=-0.01, roi=0.03, profit_perc=0.03, trades=[BTrade(sell_reason=SellType.ROI, open_tick=1, close_tick=3)] ) @@ -97,12 +97,12 @@ tc4 = BTContainer(data=[ # Set stop-loss at 2% ROI at 5% # TC6: Stop-Loss triggers 2% Loss tc5 = BTContainer(data=[ - [0, 10000, 10050, 9950, 9975, 12345, 1, 0], - [1, 10000, 10050, 9950, 9975, 12345, 0, 0], # enter trade (signal on last candle) - [2, 9975, 10600, 9700, 10100, 12345, 0, 0], # Exit with stoploss - [3, 9950, 10000, 9900, 9925, 12345, 0, 0], - [4, 9925, 9975, 9945, 9900, 12345, 0, 0], - [5, 9900, 9950, 9850, 9900, 12345, 0, 0]], + [0, 5000, 5025, 4975, 4987, 6172, 1, 0], + [1, 5000, 5025, 4975, 4987, 6172, 0, 0], # enter trade (signal on last candle) + [2, 4987, 5300, 4850, 5050, 6172, 0, 0], # Exit with stoploss + [3, 4975, 5000, 4950, 4962, 6172, 0, 0], + [4, 4962, 4987, 4972, 4950, 6172, 0, 0], + [5, 4950, 4975, 4925, 4950, 6172, 0, 0]], stop_loss=-0.02, roi=0.05, profit_perc=-0.02, trades=[BTrade(sell_reason=SellType.STOP_LOSS, open_tick=1, close_tick=2)] ) @@ -112,12 +112,12 @@ tc5 = BTContainer(data=[ # Set stop-loss at 2% ROI at 3% # TC7: ROI Triggers 3% Gain tc6 = BTContainer(data=[ - [0, 10000, 10050, 9950, 9975, 12345, 1, 0], - [1, 10000, 10050, 9950, 9975, 12345, 0, 0], # enter trade (signal on last candle) - [2, 9975, 10600, 9900, 10100, 12345, 0, 0], # ROI - [3, 9950, 10000, 9900, 9925, 12345, 0, 0], - [4, 9925, 9975, 9945, 9900, 12345, 0, 0], - [5, 9900, 9950, 9850, 9900, 12345, 0, 0]], + [0, 5000, 5025, 4975, 4987, 6172, 1, 0], + [1, 5000, 5025, 4975, 4987, 6172, 0, 0], + [2, 4987, 5300, 4950, 5050, 6172, 0, 0], + [3, 4975, 5000, 4950, 4962, 6172, 0, 0], + [4, 4962, 4987, 4972, 4950, 6172, 0, 0], + [5, 4950, 4975, 4925, 4950, 6172, 0, 0]], stop_loss=-0.02, roi=0.03, profit_perc=0.03, trades=[BTrade(sell_reason=SellType.ROI, open_tick=1, close_tick=2)] ) From daa9863d0ba82c22f9fa41de8cc49434396868d0 Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 30 Oct 2018 20:45:32 +0100 Subject: [PATCH 215/699] Try adding headers --- freqtrade/tests/optimize/test_backtest_detail.py | 1 + 1 file changed, 1 insertion(+) diff --git a/freqtrade/tests/optimize/test_backtest_detail.py b/freqtrade/tests/optimize/test_backtest_detail.py index 5ec9c02c2..049a761b8 100644 --- a/freqtrade/tests/optimize/test_backtest_detail.py +++ b/freqtrade/tests/optimize/test_backtest_detail.py @@ -17,6 +17,7 @@ from freqtrade.tests.conftest import patch_exchange # Test with Stop-loss at 1% # TC1: Stop-Loss Triggered 1% loss tc0 = BTContainer(data=[ + # date, open, high, low, close, volume, buy, sell [0, 5000, 5025, 4975, 4987, 6172, 1, 0], [1, 5000, 5025, 4975, 4987, 6172, 0, 0], # enter trade (signal on last candle) [2, 4987, 5012, 4600, 4600, 6172, 0, 0], # exit with stoploss hit From a321d0a8203478f2eaebb1889490641463864d91 Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 30 Oct 2018 20:49:12 +0100 Subject: [PATCH 216/699] Short descriptors --- freqtrade/tests/optimize/test_backtest_detail.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/tests/optimize/test_backtest_detail.py b/freqtrade/tests/optimize/test_backtest_detail.py index 049a761b8..9efe2d573 100644 --- a/freqtrade/tests/optimize/test_backtest_detail.py +++ b/freqtrade/tests/optimize/test_backtest_detail.py @@ -17,7 +17,7 @@ from freqtrade.tests.conftest import patch_exchange # Test with Stop-loss at 1% # TC1: Stop-Loss Triggered 1% loss tc0 = BTContainer(data=[ - # date, open, high, low, close, volume, buy, sell + # D O H L C V B S [0, 5000, 5025, 4975, 4987, 6172, 1, 0], [1, 5000, 5025, 4975, 4987, 6172, 0, 0], # enter trade (signal on last candle) [2, 4987, 5012, 4600, 4600, 6172, 0, 0], # exit with stoploss hit From eab15e09f5d24444e8d5b1d31cada2b0e881f5c7 Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Wed, 31 Oct 2018 13:34:07 +0100 Subject: [PATCH 217/699] Update ccxt from 1.17.436 to 1.17.439 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 5244b54a8..26a2d7a97 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -ccxt==1.17.436 +ccxt==1.17.439 SQLAlchemy==1.2.12 python-telegram-bot==11.1.0 arrow==0.12.1 From c21b45647d4d06d4307385be0a49c69da5318f58 Mon Sep 17 00:00:00 2001 From: Matthias Date: Thu, 1 Nov 2018 13:05:57 +0100 Subject: [PATCH 218/699] Fix smoe comments in persistence --- freqtrade/persistence.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/freqtrade/persistence.py b/freqtrade/persistence.py index 02267ac21..51a8129fb 100644 --- a/freqtrade/persistence.py +++ b/freqtrade/persistence.py @@ -272,10 +272,10 @@ class Trade(_DECL_BASE): self, fee: Optional[float] = None) -> float: """ - Calculate the open_rate in BTC + Calculate the open_rate including fee. :param fee: fee to use on the open rate (optional). If rate is not set self.fee will be used - :return: Price in BTC of the open trade + :return: Price in of the open trade incl. Fees """ buy_trade = (Decimal(self.amount) * Decimal(self.open_rate)) @@ -287,7 +287,7 @@ class Trade(_DECL_BASE): rate: Optional[float] = None, fee: Optional[float] = None) -> float: """ - Calculate the close_rate in BTC + Calculate the close_rate including fee :param fee: fee to use on the close rate (optional). If rate is not set self.fee will be used :param rate: rate to compare with (optional). @@ -307,12 +307,12 @@ class Trade(_DECL_BASE): rate: Optional[float] = None, fee: Optional[float] = None) -> float: """ - Calculate the profit in BTC between Close and Open trade + Calculate the absolute profit in stake currency between Close and Open trade :param fee: fee to use on the close rate (optional). If rate is not set self.fee will be used :param rate: close rate to compare with (optional). If rate is not set self.close_rate will be used - :return: profit in BTC as float + :return: profit in stake currency as float """ open_trade_price = self.calc_open_trade_price() close_trade_price = self.calc_close_trade_price( From 95d271ca5d73be67ab8fc557a12b8e4a879a54a1 Mon Sep 17 00:00:00 2001 From: Matthias Date: Thu, 1 Nov 2018 13:14:59 +0100 Subject: [PATCH 219/699] Fix ROI close-rate calculation to work with fees - adjust tests --- freqtrade/optimize/backtesting.py | 8 +++++--- freqtrade/tests/optimize/test_backtesting.py | 20 +++++++++++--------- 2 files changed, 16 insertions(+), 12 deletions(-) diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index 1f02b9793..6fcde64fa 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -218,9 +218,11 @@ class Backtesting(object): # get entry in min_roi >= to trade duration roi_entry = max(list(filter(lambda x: trade_dur >= x, self.strategy.minimal_roi.keys()))) - # set close-rate to min-roi - closerate = trade.open_rate + trade.open_rate * \ - self.strategy.minimal_roi[roi_entry] + roi = self.strategy.minimal_roi[roi_entry] + + # - (Expected abs profit + open_rate + open_fee) / (fee_close -1) + closerate = - (trade.open_rate * roi + trade.open_rate * + (1 + trade.fee_open)) / (trade.fee_close - 1) else: closerate = sell_row.open diff --git a/freqtrade/tests/optimize/test_backtesting.py b/freqtrade/tests/optimize/test_backtesting.py index 2d3d49ab0..20f2a6582 100644 --- a/freqtrade/tests/optimize/test_backtesting.py +++ b/freqtrade/tests/optimize/test_backtesting.py @@ -518,18 +518,18 @@ def test_backtest(default_conf, fee, mocker) -> None: expected = pd.DataFrame( {'pair': [pair, pair], - 'profit_percent': [0.00029977, 0.00056716], - 'profit_abs': [1.49e-06, 7.6e-07], + 'profit_percent': [0.0, 0.0], + 'profit_abs': [0.0, 0.0], 'open_time': [Arrow(2018, 1, 29, 18, 40, 0).datetime, Arrow(2018, 1, 30, 3, 30, 0).datetime], - 'close_time': [Arrow(2018, 1, 29, 22, 40, 0).datetime, - Arrow(2018, 1, 30, 4, 20, 0).datetime], + 'close_time': [Arrow(2018, 1, 29, 22, 35, 0).datetime, + Arrow(2018, 1, 30, 4, 15, 0).datetime], 'open_index': [77, 183], - 'close_index': [125, 193], - 'trade_duration': [240, 50], + 'close_index': [124, 192], + 'trade_duration': [235, 45], 'open_at_end': [False, False], 'open_rate': [0.104445, 0.10302485], - 'close_rate': [0.105, 0.10359999], + 'close_rate': [0.104969, 0.103541], 'sell_reason': [SellType.ROI, SellType.ROI] }) pd.testing.assert_frame_equal(results, expected) @@ -539,9 +539,11 @@ def test_backtest(default_conf, fee, mocker) -> None: # Check open trade rate alignes to open rate assert ln is not None assert round(ln.iloc[0]["open"], 6) == round(t["open_rate"], 6) - # check close trade rate alignes to close rate + # check close trade rate alignes to close rate or is between high and low ln = data_pair.loc[data_pair["date"] == t["close_time"]] - assert round(ln.iloc[0]["open"], 6) == round(t["close_rate"], 6) + assert (round(ln.iloc[0]["open"], 6) == round(t["close_rate"], 6) or + round(ln.iloc[0]["low"], 6) < round( + t["close_rate"], 6) < round(ln.iloc[0]["high"], 6)) def test_backtest_1min_ticker_interval(default_conf, fee, mocker) -> None: From 8316acfa780c1e988f3aec46bc9503fdfeb6b8fb Mon Sep 17 00:00:00 2001 From: Matthias Date: Thu, 1 Nov 2018 13:16:10 +0100 Subject: [PATCH 220/699] Add column description to test-cases --- freqtrade/tests/optimize/test_backtest_detail.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/freqtrade/tests/optimize/test_backtest_detail.py b/freqtrade/tests/optimize/test_backtest_detail.py index 9efe2d573..806c136bc 100644 --- a/freqtrade/tests/optimize/test_backtest_detail.py +++ b/freqtrade/tests/optimize/test_backtest_detail.py @@ -33,6 +33,7 @@ tc0 = BTContainer(data=[ # Test with Stop-Loss at 3% # TC2: Stop-Loss Triggered 3% Loss tc1 = BTContainer(data=[ + # D O H L C V B S [0, 5000, 5025, 4975, 4987, 6172, 1, 0], [1, 5000, 5025, 4975, 4987, 6172, 0, 0], # enter trade (signal on last candle) [2, 4987, 5012, 4962, 4975, 6172, 0, 0], @@ -52,6 +53,7 @@ tc1 = BTContainer(data=[ # TC3: Trade-A: Stop-Loss Triggered 2% Loss # Trade-B: Stop-Loss Triggered 2% Loss tc2 = BTContainer(data=[ + # D O H L C V B S [0, 5000, 5025, 4975, 4987, 6172, 1, 0], [1, 5000, 5025, 4975, 4987, 6172, 0, 0], # enter trade (signal on last candle) [2, 4987, 5012, 4800, 4975, 6172, 0, 0], # exit with stoploss hit @@ -69,6 +71,7 @@ tc2 = BTContainer(data=[ # Test with Stop-loss at 2% ROI 6% # TC4: Stop-Loss Triggered 2% Loss tc3 = BTContainer(data=[ + # D O H L C V B S [0, 5000, 5025, 4975, 4987, 6172, 1, 0], [1, 5000, 5025, 4975, 4987, 6172, 0, 0], # enter trade (signal on last candle) [2, 4987, 5750, 4850, 5750, 6172, 0, 0], # Exit with stoploss hit @@ -83,6 +86,7 @@ tc3 = BTContainer(data=[ # Set stop-loss at 1% ROI 3% # TC5: ROI triggers 3% Gain tc4 = BTContainer(data=[ + # D O H L C V B S [0, 5000, 5025, 4980, 4987, 6172, 1, 0], [1, 5000, 5025, 4980, 4987, 6172, 0, 0], # enter trade (signal on last candle) [2, 4987, 5025, 4975, 4987, 6172, 0, 0], @@ -98,6 +102,7 @@ tc4 = BTContainer(data=[ # Set stop-loss at 2% ROI at 5% # TC6: Stop-Loss triggers 2% Loss tc5 = BTContainer(data=[ + # D O H L C V B S [0, 5000, 5025, 4975, 4987, 6172, 1, 0], [1, 5000, 5025, 4975, 4987, 6172, 0, 0], # enter trade (signal on last candle) [2, 4987, 5300, 4850, 5050, 6172, 0, 0], # Exit with stoploss @@ -113,6 +118,7 @@ tc5 = BTContainer(data=[ # Set stop-loss at 2% ROI at 3% # TC7: ROI Triggers 3% Gain tc6 = BTContainer(data=[ + # D O H L C V B S [0, 5000, 5025, 4975, 4987, 6172, 1, 0], [1, 5000, 5025, 4975, 4987, 6172, 0, 0], [2, 4987, 5300, 4950, 5050, 6172, 0, 0], From 92f9c828e6bc9322adda1036745cdd96e4d08697 Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Thu, 1 Nov 2018 13:34:07 +0100 Subject: [PATCH 221/699] Update ccxt from 1.17.439 to 1.17.448 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 26a2d7a97..8dffcd5c6 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -ccxt==1.17.439 +ccxt==1.17.448 SQLAlchemy==1.2.12 python-telegram-bot==11.1.0 arrow==0.12.1 From afc1329126652717e3299ce270185a53985063e0 Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Thu, 1 Nov 2018 13:34:08 +0100 Subject: [PATCH 222/699] Update sqlalchemy from 1.2.12 to 1.2.13 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 8dffcd5c6..c688e3a68 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,5 @@ ccxt==1.17.448 -SQLAlchemy==1.2.12 +SQLAlchemy==1.2.13 python-telegram-bot==11.1.0 arrow==0.12.1 cachetools==2.1.0 From 17895282a1f6ed8ebf03777e752183bf0775820d Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Fri, 2 Nov 2018 13:34:06 +0100 Subject: [PATCH 223/699] Update ccxt from 1.17.448 to 1.17.455 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index c688e3a68..57beabf49 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -ccxt==1.17.448 +ccxt==1.17.455 SQLAlchemy==1.2.13 python-telegram-bot==11.1.0 arrow==0.12.1 From 2f6aafe66c9d6ee9c1e74327d994f4369169d6b4 Mon Sep 17 00:00:00 2001 From: misagh Date: Fri, 2 Nov 2018 18:07:38 +0100 Subject: [PATCH 224/699] Edge calculation refactored: removing redundant calculations --- freqtrade/edge/__init__.py | 86 ++++++++++++++++---------------------- 1 file changed, 37 insertions(+), 49 deletions(-) diff --git a/freqtrade/edge/__init__.py b/freqtrade/edge/__init__.py index 5c074ceb1..ab36b8aa8 100644 --- a/freqtrade/edge/__init__.py +++ b/freqtrade/edge/__init__.py @@ -215,15 +215,11 @@ class Edge(): def _process_expectancy(self, results: DataFrame) -> list: """ - This is a temporary version of edge positioning calculation. - The function will be eventually moved to a plugin called Edge in order - to calculate necessary WR, RRR and - other indictaors related to money management periodically (each X minutes) - and keep it in a storage. + This calculates WinRate, Required Risk Reward, Risk Reward and Expectancy of all pairs The calulation will be done per pair and per strategy. """ # Removing pairs having less than min_trades_number - min_trades_number = self.edge_config.get('min_trade_number', 15) + min_trades_number = self.edge_config.get('min_trade_number', 10) results = results.groupby(['pair', 'stoploss']).filter(lambda x: len(x) > min_trades_number) ################################### @@ -242,61 +238,53 @@ class Edge(): results = results[results.trade_duration < max_trade_duration] ####################################################################### - # Win Rate is the number of profitable trades - # Divided by number of trades - def winrate(x): - x = x[x > 0].count() / x.count() - return x - ############################# - - # Risk Reward Ratio - # 1 / ((loss money / losing trades) / (gained money / winning trades)) - def risk_reward_ratio(x): - x = abs(1 / ((x[x < 0].sum() / x[x < 0].count()) / (x[x > 0].sum() / x[x > 0].count()))) - return x - ############################## - - # Required Risk Reward - # (1/(winrate - 1) - def required_risk_reward(x): - x = (1 / (x[x > 0].count() / x.count()) - 1) - return x - ############################## - - # Expectancy - # Tells you the interest percentage you should hope - # E.x. if expectancy is 0.35, on $1 trade you should expect a target of $1.35 - def expectancy(x): - average_win = float(x[x > 0].sum() / x[x > 0].count()) - average_loss = float(abs(x[x < 0].sum() / x[x < 0].count())) - winrate = float(x[x > 0].count() / x.count()) - x = ((1 + average_win / average_loss) * winrate) - 1 - return x - ############################## - if results.empty: return [] groupby_aggregator = { 'profit_abs': [ - winrate, - risk_reward_ratio, - required_risk_reward, - expectancy, - 'count'], - 'trade_duration': ['mean']} + ('nb_trades', 'count'), # number of all trades + ('profit_sum', lambda x: x[x > 0].sum()), # cumulative profit of all winning trades + ('loss_sum', lambda x: abs(x[x < 0].sum())), # cumulative loss of all losing trades + ('nb_win_trades', lambda x: x[x > 0].count()) # number of winning trades + ], + 'trade_duration': [('avg_trade_duration', 'mean')] + } - final = results.groupby(['pair', 'stoploss'])['profit_abs', 'trade_duration'].agg( + # Group by (pair and stoploss) the applying above aggregator + df = results.groupby(['pair', 'stoploss'])['profit_abs', 'trade_duration'].agg( groupby_aggregator).reset_index(col_level=1) - final.columns = final.columns.droplevel(0) - final = final.sort_values(by=['expectancy', 'stoploss'], ascending=False).groupby( + # Dropping level 0 as we don't need it + df.columns = df.columns.droplevel(0) + + # Calculating number of losing trades, average win and average loss + df['nb_loss_trades'] = df['nb_trades'] - df['nb_win_trades'] + df['average_win'] = df['profit_sum'] / df['nb_win_trades'] + df['average_loss'] = df['loss_sum'] / df['nb_loss_trades'] + + # Win rate = number of profitable trades / number of trades + df['winrate'] = df['nb_win_trades'] / df['nb_trades'] + + # risk_reward_ratio = 1 / (average loss / average win) + df['risk_reward_ratio'] = 1 / (df['average_loss'] / df['average_win']) + + # required_risk_reward = (1 / winrate) - 1 + df['required_risk_reward'] = (1 / df['winrate']) - 1 + + # expectancy = ((1 + average_win/average_loss) * winrate) - 1 + df['expectancy'] = ((1 + df['average_win'] / df['average_loss']) * df['winrate']) - 1 + + # sort by expectancy and stoploss + df = df.sort_values(by=['expectancy', 'stoploss'], ascending=False).groupby( 'pair').first().sort_values(by=['expectancy'], ascending=False).reset_index() - final.rename(columns={'mean': 'avg_duration(min)'}, inplace=True) + # dropping unecessary columns + df.drop(columns=['nb_loss_trades', 'nb_win_trades', 'average_win', 'average_loss', + 'profit_sum', 'loss_sum', 'avg_trade_duration', 'nb_trades'], inplace=True) # Returning an array of pairs in order of "expectancy" - return final.values + return df.values def _find_trades_for_stoploss_range(self, ticker_data, pair, stoploss_range): buy_column = ticker_data['buy'].values From 05b80104605791c199baaa8df6171f204994a4b7 Mon Sep 17 00:00:00 2001 From: misagh Date: Fri, 2 Nov 2018 18:10:03 +0100 Subject: [PATCH 225/699] removing unnecessary test cases --- freqtrade/tests/edge/test_edge.py | 13 ++----------- 1 file changed, 2 insertions(+), 11 deletions(-) diff --git a/freqtrade/tests/edge/test_edge.py b/freqtrade/tests/edge/test_edge.py index ec8af1200..4a2e06567 100644 --- a/freqtrade/tests/edge/test_edge.py +++ b/freqtrade/tests/edge/test_edge.py @@ -16,16 +16,7 @@ from unittest.mock import MagicMock # 4) Entered, sl 3%, candle drops 4%, recovers to 1% => Trade closed, 3% loss # 5) Entered, sl 2%, candle drops 4%, recovers to 1%, entry met, candle drops 20% => # Trade 1 closed: loss 2%, Trade 2 opened, Trade 2 closed: loss 2% -# 6) -################################################################### -# STOPLOSS: -# 6) Candle drops 8%, stoploss at 1%: Trade closed, 1% loss -# 7) Candle drops 4% but recovers to 1% loss, stoploss at 3%: Trade closed, 3% loss -# 8) Candle drops 4% recovers to 1% entry criteria are met, candle drops -# 20%, stoploss at 2%: Trade 1 closed, Loss 2%, Trade 2 opened, Trade 2 closed, Loss 2% -#################################################################### -# PRIORITY TO STOPLOSS: -# 9) Stoploss and sell are hit. should sell on stoploss +# 6) Stoploss and sell are hit. should sell on stoploss #################################################################### ticker_start_time = arrow.get(2018, 10, 3) @@ -179,7 +170,7 @@ def test_process_expectancy(mocker, default_conf): 'trade_duration': '', 'open_rate': 17, 'close_rate': 17, - 'exit_type': 'sell_signal'}, # sdfsdf + 'exit_type': 'sell_signal'}, {'pair': 'TEST/BTC', 'stoploss': -0.9, From 333d505b661b113853f674735ca301e67d28fdc5 Mon Sep 17 00:00:00 2001 From: misagh Date: Fri, 2 Nov 2018 19:01:37 +0100 Subject: [PATCH 226/699] OHLC validation corrected --- freqtrade/tests/edge/test_edge.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/tests/edge/test_edge.py b/freqtrade/tests/edge/test_edge.py index 4a2e06567..22e4f2352 100644 --- a/freqtrade/tests/edge/test_edge.py +++ b/freqtrade/tests/edge/test_edge.py @@ -56,7 +56,7 @@ def test_stoploss(mocker, default_conf): def _validate_ohlc(buy_ohlc_sell_matrice): for index, ohlc in enumerate(buy_ohlc_sell_matrice): # if not high < open < low or not high < close < low - if not ohlc[3] > ohlc[2] > ohlc[4] or not ohlc[3] > ohlc[5] > ohlc[4]: + if not ohlc[3] >= ohlc[2] >= ohlc[4] or not ohlc[3] >= ohlc[5] >= ohlc[4]: raise Exception('Line ' + str(index + 1) + ' of ohlc has invalid values!') return True From 7f3b4a97dd0d5f3cd39f8a340e3f2c55028c6434 Mon Sep 17 00:00:00 2001 From: Matthias Date: Fri, 2 Nov 2018 19:14:50 +0100 Subject: [PATCH 227/699] Reinstate df - which was removed in #1287 --- scripts/plot_dataframe.py | 381 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 381 insertions(+) create mode 100755 scripts/plot_dataframe.py diff --git a/scripts/plot_dataframe.py b/scripts/plot_dataframe.py new file mode 100755 index 000000000..68713f296 --- /dev/null +++ b/scripts/plot_dataframe.py @@ -0,0 +1,381 @@ +#!/usr/bin/env python3 +""" +Script to display when the bot will buy a specific pair + +Mandatory Cli parameters: +-p / --pair: pair to examine + +Option but recommended +-s / --strategy: strategy to use + + +Optional Cli parameters +-d / --datadir: path to pair backtest data +--timerange: specify what timerange of data to use. +-l / --live: Live, to download the latest ticker for the pair +-db / --db-url: Show trades stored in database + + +Indicators recommended +Row 1: sma, ema3, ema5, ema10, ema50 +Row 3: macd, rsi, fisher_rsi, mfi, slowd, slowk, fastd, fastk + +Example of usage: +> python3 scripts/plot_dataframe.py --pair BTC/EUR -d user_data/data/ --indicators1 sma,ema3 +--indicators2 fastk,fastd +""" +import json +import logging +import sys +from argparse import Namespace +from pathlib import Path +from typing import Dict, List, Any + +import pandas as pd +import plotly.graph_objs as go +import pytz + +from plotly import tools +from plotly.offline import plot + +import freqtrade.optimize as optimize +from freqtrade import persistence +from freqtrade.arguments import Arguments, TimeRange +from freqtrade.exchange import Exchange +from freqtrade.optimize.backtesting import setup_configuration +from freqtrade.persistence import Trade +from freqtrade.strategy.resolver import StrategyResolver + +logger = logging.getLogger(__name__) +_CONF: Dict[str, Any] = {} + +timeZone = pytz.UTC + + +def load_trades(args: Namespace, pair: str, timerange: TimeRange) -> pd.DataFrame: + trades: pd.DataFrame = pd.DataFrame() + if args.db_url: + persistence.init(_CONF) + columns = ["pair", "profit", "opents", "closets", "open_rate", "close_rate", "duration"] + + for x in Trade.query.all(): + print("date: {}".format(x.open_date)) + + trades = pd.DataFrame([(t.pair, t.calc_profit(), + t.open_date.replace(tzinfo=timeZone), + t.close_date.replace(tzinfo=timeZone) if t.close_date else None, + t.open_rate, t.close_rate, + t.close_date.timestamp() - t.open_date.timestamp() if t.close_date else None) + for t in Trade.query.filter(Trade.pair.is_(pair)).all()], + columns=columns) + + elif args.exportfilename: + file = Path(args.exportfilename) + # must align with columns in backtest.py + columns = ["pair", "profit", "opents", "closets", "index", "duration", + "open_rate", "close_rate", "open_at_end", "sell_reason"] + with file.open() as f: + data = json.load(f) + trades = pd.DataFrame(data, columns=columns) + trades = trades.loc[trades["pair"] == pair] + if timerange: + if timerange.starttype == 'date': + trades = trades.loc[trades["opents"] >= timerange.startts] + if timerange.stoptype == 'date': + trades = trades.loc[trades["opents"] <= timerange.stopts] + + trades['opents'] = pd.to_datetime(trades['opents'], + unit='s', + utc=True, + infer_datetime_format=True) + trades['closets'] = pd.to_datetime(trades['closets'], + unit='s', + utc=True, + infer_datetime_format=True) + return trades + + +def plot_analyzed_dataframe(args: Namespace) -> None: + """ + Calls analyze() and plots the returned dataframe + :return: None + """ + global _CONF + + # Load the configuration + _CONF.update(setup_configuration(args)) + + print(_CONF) + # Set the pair to audit + pair = args.pair + + if pair is None: + logger.critical('Parameter --pair mandatory;. E.g --pair ETH/BTC') + exit() + + if '/' not in pair: + logger.critical('--pair format must be XXX/YYY') + exit() + + # Set timerange to use + timerange = Arguments.parse_timerange(args.timerange) + + # Load the strategy + try: + strategy = StrategyResolver(_CONF).strategy + exchange = Exchange(_CONF) + except AttributeError: + logger.critical( + 'Impossible to load the strategy. Please check the file "user_data/strategies/%s.py"', + args.strategy + ) + exit() + + # Set the ticker to use + tick_interval = strategy.ticker_interval + + # Load pair tickers + tickers = {} + if args.live: + logger.info('Downloading pair.') + exchange.refresh_tickers([pair], tick_interval) + tickers[pair] = exchange.klines[pair] + else: + tickers = optimize.load_data( + datadir=_CONF.get("datadir"), + pairs=[pair], + ticker_interval=tick_interval, + refresh_pairs=_CONF.get('refresh_pairs', False), + timerange=timerange, + exchange=Exchange(_CONF) + ) + + # No ticker found, or impossible to download + if tickers == {}: + exit() + + # Get trades already made from the DB + trades = load_trades(args, pair, timerange) + + dataframes = strategy.tickerdata_to_dataframe(tickers) + + dataframe = dataframes[pair] + dataframe = strategy.advise_buy(dataframe, {'pair': pair}) + dataframe = strategy.advise_sell(dataframe, {'pair': pair}) + + if len(dataframe.index) > args.plot_limit: + logger.warning('Ticker contained more than %s candles as defined ' + 'with --plot-limit, clipping.', args.plot_limit) + dataframe = dataframe.tail(args.plot_limit) + + trades = trades.loc[trades['opents'] >= dataframe.iloc[0]['date']] + fig = generate_graph( + pair=pair, + trades=trades, + data=dataframe, + args=args + ) + + plot(fig, filename=str(Path('user_data').joinpath('freqtrade-plot.html'))) + + +def generate_graph(pair, trades: pd.DataFrame, data: pd.DataFrame, args) -> tools.make_subplots: + """ + Generate the graph from the data generated by Backtesting or from DB + :param pair: Pair to Display on the graph + :param trades: All trades created + :param data: Dataframe + :param args: sys.argv that contrains the two params indicators1, and indicators2 + :return: None + """ + + # Define the graph + fig = tools.make_subplots( + rows=3, + cols=1, + shared_xaxes=True, + row_width=[1, 1, 4], + vertical_spacing=0.0001, + ) + fig['layout'].update(title=pair) + fig['layout']['yaxis1'].update(title='Price') + fig['layout']['yaxis2'].update(title='Volume') + fig['layout']['yaxis3'].update(title='Other') + + # Common information + candles = go.Candlestick( + x=data.date, + open=data.open, + high=data.high, + low=data.low, + close=data.close, + name='Price' + ) + + df_buy = data[data['buy'] == 1] + buys = go.Scattergl( + x=df_buy.date, + y=df_buy.close, + mode='markers', + name='buy', + marker=dict( + symbol='triangle-up-dot', + size=9, + line=dict(width=1), + color='green', + ) + ) + df_sell = data[data['sell'] == 1] + sells = go.Scattergl( + x=df_sell.date, + y=df_sell.close, + mode='markers', + name='sell', + marker=dict( + symbol='triangle-down-dot', + size=9, + line=dict(width=1), + color='red', + ) + ) + + trade_buys = go.Scattergl( + x=trades["opents"], + y=trades["open_rate"], + mode='markers', + name='trade_buy', + marker=dict( + symbol='square-open', + size=11, + line=dict(width=2), + color='green' + ) + ) + trade_sells = go.Scattergl( + x=trades["closets"], + y=trades["close_rate"], + mode='markers', + name='trade_sell', + marker=dict( + symbol='square-open', + size=11, + line=dict(width=2), + color='red' + ) + ) + + # Row 1 + fig.append_trace(candles, 1, 1) + + if 'bb_lowerband' in data and 'bb_upperband' in data: + bb_lower = go.Scatter( + x=data.date, + y=data.bb_lowerband, + name='BB lower', + line={'color': 'rgba(255,255,255,0)'}, + ) + bb_upper = go.Scatter( + x=data.date, + y=data.bb_upperband, + name='BB upper', + fill="tonexty", + fillcolor="rgba(0,176,246,0.2)", + line={'color': 'rgba(255,255,255,0)'}, + ) + fig.append_trace(bb_lower, 1, 1) + fig.append_trace(bb_upper, 1, 1) + + fig = generate_row(fig=fig, row=1, raw_indicators=args.indicators1, data=data) + fig.append_trace(buys, 1, 1) + fig.append_trace(sells, 1, 1) + fig.append_trace(trade_buys, 1, 1) + fig.append_trace(trade_sells, 1, 1) + + # Row 2 + volume = go.Bar( + x=data['date'], + y=data['volume'], + name='Volume' + ) + fig.append_trace(volume, 2, 1) + + # Row 3 + fig = generate_row(fig=fig, row=3, raw_indicators=args.indicators2, data=data) + + return fig + + +def generate_row(fig, row, raw_indicators, data) -> tools.make_subplots: + """ + Generator all the indicator selected by the user for a specific row + """ + for indicator in raw_indicators.split(','): + if indicator in data: + scattergl = go.Scattergl( + x=data['date'], + y=data[indicator], + name=indicator + ) + fig.append_trace(scattergl, row, 1) + else: + logger.info( + 'Indicator "%s" ignored. Reason: This indicator is not found ' + 'in your strategy.', + indicator + ) + + return fig + + +def plot_parse_args(args: List[str]) -> Namespace: + """ + Parse args passed to the script + :param args: Cli arguments + :return: args: Array with all arguments + """ + arguments = Arguments(args, 'Graph dataframe') + arguments.scripts_options() + arguments.parser.add_argument( + '--indicators1', + help='Set indicators from your strategy you want in the first row of the graph. Separate ' + 'them with a coma. E.g: ema3,ema5 (default: %(default)s)', + type=str, + default='sma,ema3,ema5', + dest='indicators1', + ) + + arguments.parser.add_argument( + '--indicators2', + help='Set indicators from your strategy you want in the third row of the graph. Separate ' + 'them with a coma. E.g: fastd,fastk (default: %(default)s)', + type=str, + default='macd', + dest='indicators2', + ) + arguments.parser.add_argument( + '--plot-limit', + help='Specify tick limit for plotting - too high values cause huge files - ' + 'Default: %(default)s', + dest='plot_limit', + default=750, + type=int, + ) + arguments.common_args_parser() + arguments.optimizer_shared_options(arguments.parser) + arguments.backtesting_options(arguments.parser) + return arguments.parse_args() + + +def main(sysargv: List[str]) -> None: + """ + This function will initiate the bot and start the trading loop. + :return: None + """ + logger.info('Starting Plot Dataframe') + plot_analyzed_dataframe( + plot_parse_args(sysargv) + ) + + +if __name__ == '__main__': + main(sys.argv[1:]) From bb791eac7ea4fcb22c77b5be418012489f660d90 Mon Sep 17 00:00:00 2001 From: misagh Date: Fri, 2 Nov 2018 19:19:28 +0100 Subject: [PATCH 228/699] backtesting remove from import + whitespace removed --- freqtrade/edge/__init__.py | 1 - freqtrade/freqtradebot.py | 1 - 2 files changed, 2 deletions(-) diff --git a/freqtrade/edge/__init__.py b/freqtrade/edge/__init__.py index 67e0787e7..1b0229c61 100644 --- a/freqtrade/edge/__init__.py +++ b/freqtrade/edge/__init__.py @@ -13,7 +13,6 @@ from freqtrade.arguments import Arguments from freqtrade.arguments import TimeRange from freqtrade.strategy.interface import SellType from freqtrade.strategy.resolver import IStrategy, StrategyResolver -from freqtrade.optimize.backtesting import Backtesting import sys logger = logging.getLogger(__name__) diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index cade551d5..890f9e563 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -196,7 +196,6 @@ class FreqtradeBot(object): # Refreshing candles self.exchange.refresh_tickers(self.active_pair_whitelist, self.strategy.ticker_interval) - # Query trades from persistence layer trades = Trade.query.filter(Trade.is_open.is_(True)).all() From 3eeaa50fe5c8bcb2f6c3360a23d5119d4ec88b71 Mon Sep 17 00:00:00 2001 From: misagh Date: Fri, 2 Nov 2018 19:54:32 +0100 Subject: [PATCH 229/699] stoploss and sell signal tests done --- freqtrade/tests/edge/test_edge.py | 96 ++++++++++++++++++++++++++++++- 1 file changed, 93 insertions(+), 3 deletions(-) diff --git a/freqtrade/tests/edge/test_edge.py b/freqtrade/tests/edge/test_edge.py index 22e4f2352..b1946bdb1 100644 --- a/freqtrade/tests/edge/test_edge.py +++ b/freqtrade/tests/edge/test_edge.py @@ -14,9 +14,7 @@ from unittest.mock import MagicMock # 2) Two complete trades within dataframe (with sell hit for all) # 3) Entered, sl 1%, candle drops 8% => Trade closed, 1% loss # 4) Entered, sl 3%, candle drops 4%, recovers to 1% => Trade closed, 3% loss -# 5) Entered, sl 2%, candle drops 4%, recovers to 1%, entry met, candle drops 20% => -# Trade 1 closed: loss 2%, Trade 2 opened, Trade 2 closed: loss 2% -# 6) Stoploss and sell are hit. should sell on stoploss +# 5) Stoploss and sell are hit. should sell on stoploss #################################################################### ticker_start_time = arrow.get(2018, 10, 3) @@ -264,3 +262,95 @@ def test_two_complete_trades(mocker, default_conf): assert trades[1]['close_rate'] == ticker[6][_ohlc['open']] assert trades[1]['exit_type'] == SellType.SELL_SIGNAL ############################################################## + + +# 3) Entered, sl 1%, candle drops 8% => Trade closed, 1% loss +def test_case_3(mocker, default_conf): + exchange = get_patched_exchange(mocker, default_conf) + edge = Edge(default_conf, exchange) + + stoploss = -0.01 # we don't want stoploss to be hit in this test + ticker = [ + # D=Date, B=Buy, O=Open, H=High, L=Low, C=Close, S=Sell + #D, B, O, H, L, C, S + [0, 1, 15, 20, 12, 17, 0], # -> no action + [1, 0, 14, 15, 11, 12, 0], # -> enter to trade, stoploss hit + [2, 1, 12, 25, 11, 20, 0], # -> no action + ] + + ticker_df = _build_dataframe(ticker) + trades = edge._find_trades_for_stoploss_range(ticker_df, 'TEST/BTC', [stoploss]) + + # Two trades must have occured + assert len(trades) == 1 + + # First trade check + assert trades[0]['open_time'] == _time_on_candle(1) + assert trades[0]['close_time'] == _time_on_candle(1) + assert trades[0]['open_rate'] == ticker[1][_ohlc['open']] + assert trades[0]['close_rate'] == (stoploss + 1) * trades[0]['open_rate'] + assert trades[0]['exit_type'] == SellType.STOP_LOSS + ############################################################## + + +# 4) Entered, sl 3 %, candle drops 4%, recovers to 1 % = > Trade closed, 3 % loss +def test_case_4(mocker, default_conf): + exchange = get_patched_exchange(mocker, default_conf) + edge = Edge(default_conf, exchange) + + stoploss = -0.03 # we don't want stoploss to be hit in this test + ticker = [ + # D=Date, B=Buy, O=Open, H=High, L=Low, C=Close, S=Sell + #D, B, O, H, L, C, S + [0, 1, 15, 20, 12, 17, 0], # -> no action + [1, 0, 17, 22, 16.90, 17, 0], # -> enter to trade + [2, 0, 16, 17, 14.4, 15.5, 0], # -> stoploss hit + [3, 0, 17, 25, 16.9, 22, 0], # -> no action + ] + + ticker_df = _build_dataframe(ticker) + trades = edge._find_trades_for_stoploss_range(ticker_df, 'TEST/BTC', [stoploss]) + + # Two trades must have occured + assert len(trades) == 1 + + # First trade check + assert trades[0]['open_time'] == _time_on_candle(1) + assert trades[0]['close_time'] == _time_on_candle(2) + assert trades[0]['open_rate'] == ticker[1][_ohlc['open']] + assert trades[0]['close_rate'] == (stoploss + 1) * trades[0]['open_rate'] + assert trades[0]['exit_type'] == SellType.STOP_LOSS + ############################################################## + + +# 5) Stoploss and sell are hit. should sell on stoploss +def test_case_5(mocker, default_conf): + exchange = get_patched_exchange(mocker, default_conf) + edge = Edge(default_conf, exchange) + + exchange = get_patched_exchange(mocker, default_conf) + edge = Edge(default_conf, exchange) + + stoploss = -0.03 # we don't want stoploss to be hit in this test + ticker = [ + # D=Date, B=Buy, O=Open, H=High, L=Low, C=Close, S=Sell + #D, B, O, H, L, C, S + [0, 1, 15, 20, 12, 17, 0], # -> no action + [1, 0, 17, 22, 16.90, 17, 0], # -> enter to trade + [2, 0, 16, 17, 14.4, 15.5, 1], # -> stoploss hit and also sell signal + [3, 0, 17, 25, 16.9, 22, 0], # -> no action + ] + + ticker_df = _build_dataframe(ticker) + trades = edge._find_trades_for_stoploss_range(ticker_df, 'TEST/BTC', [stoploss]) + + # Two trades must have occured + assert len(trades) == 1 + + # First trade check + assert trades[0]['open_time'] == _time_on_candle(1) + assert trades[0]['close_time'] == _time_on_candle(2) + assert trades[0]['open_rate'] == ticker[1][_ohlc['open']] + assert trades[0]['close_rate'] == (stoploss + 1) * trades[0]['open_rate'] + assert trades[0]['exit_type'] == SellType.STOP_LOSS + ############################################################## From 2ef2754ffd731cd141ca776b36dda5a2376cd384 Mon Sep 17 00:00:00 2001 From: misagh Date: Fri, 2 Nov 2018 19:55:41 +0100 Subject: [PATCH 230/699] flake8 happiness satisfied --- freqtrade/tests/edge/test_edge.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/freqtrade/tests/edge/test_edge.py b/freqtrade/tests/edge/test_edge.py index b1946bdb1..bb2af9bf8 100644 --- a/freqtrade/tests/edge/test_edge.py +++ b/freqtrade/tests/edge/test_edge.py @@ -272,7 +272,7 @@ def test_case_3(mocker, default_conf): stoploss = -0.01 # we don't want stoploss to be hit in this test ticker = [ # D=Date, B=Buy, O=Open, H=High, L=Low, C=Close, S=Sell - #D, B, O, H, L, C, S + # D, B, O, H, L, C, S [0, 1, 15, 20, 12, 17, 0], # -> no action [1, 0, 14, 15, 11, 12, 0], # -> enter to trade, stoploss hit [2, 1, 12, 25, 11, 20, 0], # -> no action @@ -301,7 +301,7 @@ def test_case_4(mocker, default_conf): stoploss = -0.03 # we don't want stoploss to be hit in this test ticker = [ # D=Date, B=Buy, O=Open, H=High, L=Low, C=Close, S=Sell - #D, B, O, H, L, C, S + # D, B, O, H, L, C, S [0, 1, 15, 20, 12, 17, 0], # -> no action [1, 0, 17, 22, 16.90, 17, 0], # -> enter to trade [2, 0, 16, 17, 14.4, 15.5, 0], # -> stoploss hit @@ -334,7 +334,7 @@ def test_case_5(mocker, default_conf): stoploss = -0.03 # we don't want stoploss to be hit in this test ticker = [ # D=Date, B=Buy, O=Open, H=High, L=Low, C=Close, S=Sell - #D, B, O, H, L, C, S + # D, B, O, H, L, C, S [0, 1, 15, 20, 12, 17, 0], # -> no action [1, 0, 17, 22, 16.90, 17, 0], # -> enter to trade [2, 0, 16, 17, 14.4, 15.5, 1], # -> stoploss hit and also sell signal From 237233c3000e122319f238fa18616f40d150680a Mon Sep 17 00:00:00 2001 From: misagh Date: Fri, 2 Nov 2018 19:59:06 +0100 Subject: [PATCH 231/699] renaming tests --- freqtrade/tests/edge/test_edge.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/freqtrade/tests/edge/test_edge.py b/freqtrade/tests/edge/test_edge.py index bb2af9bf8..adb3ea00f 100644 --- a/freqtrade/tests/edge/test_edge.py +++ b/freqtrade/tests/edge/test_edge.py @@ -204,8 +204,8 @@ def test_process_expectancy(mocker, default_conf): # TODO: check expectancy + win rate etc - -def test_remove_open_trade_at_the_end(mocker, default_conf): +# 1) Open trade should be removed from the end +def test_case_1(mocker, default_conf): exchange = get_patched_exchange(mocker, default_conf) edge = Edge(default_conf, exchange) @@ -224,7 +224,8 @@ def test_remove_open_trade_at_the_end(mocker, default_conf): assert len(trades) == 0 -def test_two_complete_trades(mocker, default_conf): +# 2) Two complete trades within dataframe (with sell hit for all) +def test_case_2(mocker, default_conf): exchange = get_patched_exchange(mocker, default_conf) edge = Edge(default_conf, exchange) From d1ba994e5497603ad381317838216eea63fa0bbc Mon Sep 17 00:00:00 2001 From: misagh Date: Fri, 2 Nov 2018 20:07:45 +0100 Subject: [PATCH 232/699] expectancy test completed --- freqtrade/tests/edge/test_edge.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/freqtrade/tests/edge/test_edge.py b/freqtrade/tests/edge/test_edge.py index adb3ea00f..0d3bf44cf 100644 --- a/freqtrade/tests/edge/test_edge.py +++ b/freqtrade/tests/edge/test_edge.py @@ -202,7 +202,12 @@ def test_process_expectancy(mocker, default_conf): final = edge._process_expectancy(trades_df) assert len(final) == 1 - # TODO: check expectancy + win rate etc + assert final[0][0] == 'TEST/BTC' + assert final[0][1] == -0.9 + assert round(final[0][2], 10) == 0.3333333333 + assert round(final[0][3], 10) == 306.5384615384 + assert round(final[0][4], 10) == 2.0 + assert round(final[0][5], 10) == 101.5128205128 # 1) Open trade should be removed from the end def test_case_1(mocker, default_conf): From ece1c8a70210cb6041fc5039a7cd8415ac05578c Mon Sep 17 00:00:00 2001 From: misagh Date: Fri, 2 Nov 2018 20:12:48 +0100 Subject: [PATCH 233/699] flake8 again and again and again and again https://www.youtube.com/watch?v=MuSK3pDDYD4 --- freqtrade/tests/edge/test_edge.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/freqtrade/tests/edge/test_edge.py b/freqtrade/tests/edge/test_edge.py index 0d3bf44cf..510a79af1 100644 --- a/freqtrade/tests/edge/test_edge.py +++ b/freqtrade/tests/edge/test_edge.py @@ -202,13 +202,14 @@ def test_process_expectancy(mocker, default_conf): final = edge._process_expectancy(trades_df) assert len(final) == 1 - assert final[0][0] == 'TEST/BTC' + assert final[0][0] == 'TEST/BTC' assert final[0][1] == -0.9 assert round(final[0][2], 10) == 0.3333333333 assert round(final[0][3], 10) == 306.5384615384 assert round(final[0][4], 10) == 2.0 assert round(final[0][5], 10) == 101.5128205128 + # 1) Open trade should be removed from the end def test_case_1(mocker, default_conf): exchange = get_patched_exchange(mocker, default_conf) From 85768fcc51ce41e34812913c2323736b159dedb1 Mon Sep 17 00:00:00 2001 From: misagh Date: Fri, 2 Nov 2018 20:35:46 +0100 Subject: [PATCH 234/699] beginning of doc --- docs/index.md | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/docs/index.md b/docs/index.md index 730f1095e..b333e2db1 100644 --- a/docs/index.md +++ b/docs/index.md @@ -1,8 +1,8 @@ # freqtrade documentation Welcome to freqtrade documentation. Please feel free to contribute to -this documentation if you see it became outdated by sending us a -Pull-request. Do not hesitate to reach us on +this documentation if you see it became outdated by sending us a +Pull-request. Do not hesitate to reach us on [Slack](https://join.slack.com/t/highfrequencybot/shared_invite/enQtMjQ5NTM0OTYzMzY3LWMxYzE3M2MxNDdjMGM3ZTYwNzFjMGIwZGRjNTc3ZGU3MGE3NzdmZGMwNmU3NDM5ZTNmM2Y3NjRiNzk4NmM4OGE) if you do not find the answer to your questions. @@ -24,7 +24,10 @@ Pull-request. Do not hesitate to reach us on - [Bot Optimization](https://github.com/freqtrade/freqtrade/blob/develop/docs/bot-optimization.md) - [Change your strategy](https://github.com/freqtrade/freqtrade/blob/develop/docs/bot-optimization.md#change-your-strategy) - [Add more Indicator](https://github.com/freqtrade/freqtrade/blob/develop/docs/bot-optimization.md#add-more-indicator) - - [Test your strategy with Backtesting](https://github.com/freqtrade/freqtrade/blob/develop/docs/backtesting.md) + - [Test your strategy with Backtesting] + (https://github.com/freqtrade/freqtrade/blob/develop/docs/backtesting.md) + - [Edge positionning] + (https://github.com/freqtrade/freqtrade/blob/develop/docs/edge.md) - [Find optimal parameters with Hyperopt](https://github.com/freqtrade/freqtrade/blob/develop/docs/hyperopt.md) - [Control the bot with telegram](https://github.com/freqtrade/freqtrade/blob/develop/docs/telegram-usage.md) - [Receive notifications via webhook](https://github.com/freqtrade/freqtrade/blob/develop/docs/webhook-config.md) From 83b3323c569b607ab9411157330c01d98400f8e3 Mon Sep 17 00:00:00 2001 From: misagh Date: Fri, 2 Nov 2018 20:48:35 +0100 Subject: [PATCH 235/699] formating md --- docs/index.md | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/docs/index.md b/docs/index.md index b333e2db1..a4b172817 100644 --- a/docs/index.md +++ b/docs/index.md @@ -24,10 +24,8 @@ Pull-request. Do not hesitate to reach us on - [Bot Optimization](https://github.com/freqtrade/freqtrade/blob/develop/docs/bot-optimization.md) - [Change your strategy](https://github.com/freqtrade/freqtrade/blob/develop/docs/bot-optimization.md#change-your-strategy) - [Add more Indicator](https://github.com/freqtrade/freqtrade/blob/develop/docs/bot-optimization.md#add-more-indicator) - - [Test your strategy with Backtesting] - (https://github.com/freqtrade/freqtrade/blob/develop/docs/backtesting.md) - - [Edge positionning] - (https://github.com/freqtrade/freqtrade/blob/develop/docs/edge.md) + - [Test your strategy with Backtesting](https://github.com/freqtrade/freqtrade/blob/develop/docs/backtesting.md) + - [Edge positionning](https://github.com/freqtrade/freqtrade/blob/develop/docs/edge.md) - [Find optimal parameters with Hyperopt](https://github.com/freqtrade/freqtrade/blob/develop/docs/hyperopt.md) - [Control the bot with telegram](https://github.com/freqtrade/freqtrade/blob/develop/docs/telegram-usage.md) - [Receive notifications via webhook](https://github.com/freqtrade/freqtrade/blob/develop/docs/webhook-config.md) From b57ae20af495eb34cd058288bf0e6501f9171270 Mon Sep 17 00:00:00 2001 From: misagh Date: Fri, 2 Nov 2018 20:49:31 +0100 Subject: [PATCH 236/699] edge doc file added --- docs/edge.md | 1 + 1 file changed, 1 insertion(+) create mode 100644 docs/edge.md diff --git a/docs/edge.md b/docs/edge.md new file mode 100644 index 000000000..cbf11fc47 --- /dev/null +++ b/docs/edge.md @@ -0,0 +1 @@ +Something \ No newline at end of file From f77fa6b5925e77d4fab9ee63f5a05a2cb71d737b Mon Sep 17 00:00:00 2001 From: misagh Date: Fri, 2 Nov 2018 20:51:55 +0100 Subject: [PATCH 237/699] misharizing temporarily for doc --- docs/index.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/index.md b/docs/index.md index a4b172817..bd670b102 100644 --- a/docs/index.md +++ b/docs/index.md @@ -25,7 +25,7 @@ Pull-request. Do not hesitate to reach us on - [Change your strategy](https://github.com/freqtrade/freqtrade/blob/develop/docs/bot-optimization.md#change-your-strategy) - [Add more Indicator](https://github.com/freqtrade/freqtrade/blob/develop/docs/bot-optimization.md#add-more-indicator) - [Test your strategy with Backtesting](https://github.com/freqtrade/freqtrade/blob/develop/docs/backtesting.md) - - [Edge positionning](https://github.com/freqtrade/freqtrade/blob/develop/docs/edge.md) + - [Edge positionning](https://github.com/mishaker/freqtrade/blob/develop/docs/edge.md) - [Find optimal parameters with Hyperopt](https://github.com/freqtrade/freqtrade/blob/develop/docs/hyperopt.md) - [Control the bot with telegram](https://github.com/freqtrade/freqtrade/blob/develop/docs/telegram-usage.md) - [Receive notifications via webhook](https://github.com/freqtrade/freqtrade/blob/develop/docs/webhook-config.md) From 9cb660776cc9e4c398c84d89d5342ab8f6de583a Mon Sep 17 00:00:00 2001 From: misagh Date: Fri, 2 Nov 2018 20:52:46 +0100 Subject: [PATCH 238/699] money_mgt added --- docs/index.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/index.md b/docs/index.md index bd670b102..879ee4f80 100644 --- a/docs/index.md +++ b/docs/index.md @@ -25,7 +25,7 @@ Pull-request. Do not hesitate to reach us on - [Change your strategy](https://github.com/freqtrade/freqtrade/blob/develop/docs/bot-optimization.md#change-your-strategy) - [Add more Indicator](https://github.com/freqtrade/freqtrade/blob/develop/docs/bot-optimization.md#add-more-indicator) - [Test your strategy with Backtesting](https://github.com/freqtrade/freqtrade/blob/develop/docs/backtesting.md) - - [Edge positionning](https://github.com/mishaker/freqtrade/blob/develop/docs/edge.md) + - [Edge positionning](https://github.com/mishaker/freqtrade/blob/money_mgt/docs/edge.md) - [Find optimal parameters with Hyperopt](https://github.com/freqtrade/freqtrade/blob/develop/docs/hyperopt.md) - [Control the bot with telegram](https://github.com/freqtrade/freqtrade/blob/develop/docs/telegram-usage.md) - [Receive notifications via webhook](https://github.com/freqtrade/freqtrade/blob/develop/docs/webhook-config.md) From f6498bf5f7c4c441a45c81d5ec3c26df54c7bd0e Mon Sep 17 00:00:00 2001 From: misagh Date: Fri, 2 Nov 2018 22:13:38 +0100 Subject: [PATCH 239/699] beginning --- docs/edge.md | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/docs/edge.md b/docs/edge.md index cbf11fc47..d72422d53 100644 --- a/docs/edge.md +++ b/docs/edge.md @@ -1 +1,14 @@ -Something \ No newline at end of file +# Edge positionning + +## Introduction +Trading is all about probability. no one can claim having the strategy working all the time. you have to assume that sometimes you lose.

+But it doesn't mean there is no rule, it only means rules should work "most of the time". let's play a game: we toss a coin, heads: I give you 10$, tails: You give me 10$. is it an interetsing game ? no, it is quite boring, isn't it?

+But lets say the probabiliy that we have heads is 80%, and the probablilty that we have tails is 20%. now it is becoming interesting ... +That means 10$ x 80% versus 10$ x 20%. 8$ versus 2$. that means over time you will win 8$ risking only 2$ on each toss of coin.

+lets complicate it more: you win 80% of time but only 2$, I win 20% of time but if I win I win 8$. the calculation is: 80% * 2$ versus 20% * 8$. it is becoming boring again because overtime you win $1.6$ (80% x 2$) and me $1.6 (20% * 8$) too.

+The question is: how do you calculate that? how do you know if you wanna play with me? +The answer comes to two factors: +- Win Rate +- Risk Reward Ratio + +Win rate is quite self explanatory. means over X trades what is the perctange winning number of trades compared to total number of trades (note that we don't consider how much you gained but only If you won or not). \ No newline at end of file From 7155e5cfebb82e135843c7d19c9f373b81ee5a30 Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Sat, 3 Nov 2018 13:34:05 +0100 Subject: [PATCH 240/699] Update ccxt from 1.17.455 to 1.17.459 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 57beabf49..8cf18aebc 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -ccxt==1.17.455 +ccxt==1.17.459 SQLAlchemy==1.2.13 python-telegram-bot==11.1.0 arrow==0.12.1 From 4ab7a0fb5c22b277f7be4e19619441c17eb981a8 Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Sat, 3 Nov 2018 13:34:07 +0100 Subject: [PATCH 241/699] Update urllib3 from 1.24 to 1.24.1 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 8cf18aebc..0a790d23b 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,7 +4,7 @@ python-telegram-bot==11.1.0 arrow==0.12.1 cachetools==2.1.0 requests==2.20.0 -urllib3==1.24 +urllib3==1.24.1 wrapt==1.10.11 pandas==0.23.4 scikit-learn==0.20.0 From b6d4e11e88327a6883dfaf65d1faf48fab7a67ed Mon Sep 17 00:00:00 2001 From: misagh Date: Sat, 3 Nov 2018 14:31:34 +0100 Subject: [PATCH 242/699] added minimum win rate to config --- config.json.example | 13 ++++++------- config_full.json.example | 10 +++++----- freqtrade/constants.py | 2 +- freqtrade/edge/__init__.py | 14 +++++++++++--- 4 files changed, 23 insertions(+), 16 deletions(-) diff --git a/config.json.example b/config.json.example index 4ab4f2c72..3554260bb 100644 --- a/config.json.example +++ b/config.json.example @@ -55,19 +55,18 @@ }, "edge": { "enabled": false, - "process_throttle_secs": 1800, - "calculate_since_number_of_days": 14, + "process_throttle_secs": 3600, + "calculate_since_number_of_days": 2, "total_capital_in_stake_currency": 0.5, "allowed_risk": 0.01, "stoploss_range_min": -0.01, "stoploss_range_max": -0.1, - "stoploss_range_step": -0.001, - "maximum_winrate": 0.80, + "stoploss_range_step": -0.01, + "minimum_winrate": 0.60, "minimum_expectancy": 0.20, - "min_trade_number": 15, + "min_trade_number": 10, "max_trade_duration_minute": 1440, - "remove_pumps": true, - "minimum_delta": 1 + "remove_pumps": false }, "telegram": { "enabled": true, diff --git a/config_full.json.example b/config_full.json.example index e8a3417df..7182c1f85 100644 --- a/config_full.json.example +++ b/config_full.json.example @@ -61,18 +61,18 @@ }, "edge": { "enabled": false, - "process_throttle_secs": 1800, - "calculate_since_number_of_days": 14, + "process_throttle_secs": 3600, + "calculate_since_number_of_days": 2, "total_capital_in_stake_currency": 0.5, "allowed_risk": 0.01, "stoploss_range_min": -0.01, "stoploss_range_max": -0.1, "stoploss_range_step": -0.01, - "maximum_winrate": 0.80, + "minimum_winrate": 0.60, "minimum_expectancy": 0.20, - "min_trade_number": 15, + "min_trade_number": 10, "max_trade_duration_minute": 1440, - "remove_pumps": true + "remove_pumps": false }, "experimental": { "use_sell_signal": false, diff --git a/freqtrade/constants.py b/freqtrade/constants.py index 217855ecf..df98c30b8 100644 --- a/freqtrade/constants.py +++ b/freqtrade/constants.py @@ -182,7 +182,7 @@ CONF_SCHEMA = { "stoploss_range_min": {'type': 'number'}, "stoploss_range_max": {'type': 'number'}, "stoploss_range_step": {'type': 'number'}, - "maximum_winrate": {'type': 'number'}, + "minimum_winrate": {'type': 'number'}, "minimum_expectancy": {'type': 'number'}, "min_trade_number": {'type': 'number'}, "max_trade_duration_minute": {'type': 'integer'}, diff --git a/freqtrade/edge/__init__.py b/freqtrade/edge/__init__.py index 1b0229c61..f0ca6af06 100644 --- a/freqtrade/edge/__init__.py +++ b/freqtrade/edge/__init__.py @@ -145,10 +145,18 @@ class Edge(): def filter(self, pairs) -> list: # Filtering pairs acccording to the expectancy filtered_expectancy: list = [] + + # [pair, stoploss, winrate, risk reward ratio, required risk reward, expectancy] filtered_expectancy = [ - x[0] for x in self._cached_pairs if x[5] > float( - self.edge_config.get( - 'minimum_expectancy', 0.2))] + x[0] for x in self._cached_pairs if ( + (x[5] > float( + self.edge_config.get( + 'minimum_expectancy', + 0.2))) & ( + x[2] > float( + self.edge_config.get( + 'minimum_winrate', + 0.60))))] # Only return pairs which are included in "pairs" argument list final = [x for x in filtered_expectancy if x in pairs] From d7821acbf09548fec29afbd79a375135992043e2 Mon Sep 17 00:00:00 2001 From: misagh Date: Sat, 3 Nov 2018 14:33:17 +0100 Subject: [PATCH 243/699] refreshing pairs on each iteration --- freqtrade/edge/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/edge/__init__.py b/freqtrade/edge/__init__.py index f0ca6af06..2afb913bd 100644 --- a/freqtrade/edge/__init__.py +++ b/freqtrade/edge/__init__.py @@ -69,7 +69,7 @@ class Edge(): self.config['datadir'], pairs=pairs, ticker_interval=self.ticker_interval, - refresh_pairs=False, + refresh_pairs=True, exchange=self.exchange, timerange=self._timerange ) From 81f971f13e1524cfcfe4d5fa91c4b169d65ee5da Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Sun, 4 Nov 2018 13:34:07 +0100 Subject: [PATCH 244/699] Update ccxt from 1.17.459 to 1.17.464 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 0a790d23b..d57cbb7c4 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -ccxt==1.17.459 +ccxt==1.17.464 SQLAlchemy==1.2.13 python-telegram-bot==11.1.0 arrow==0.12.1 From 14bfd4b7ee3e4b9a42453ea3698891329e28876c Mon Sep 17 00:00:00 2001 From: misagh Date: Sun, 4 Nov 2018 18:11:58 +0100 Subject: [PATCH 245/699] using named tuples for keeping pairs data --- freqtrade/edge/__init__.py | 76 +++++++++++++++---------------- freqtrade/tests/edge/test_edge.py | 54 +++++++++++----------- 2 files changed, 64 insertions(+), 66 deletions(-) diff --git a/freqtrade/edge/__init__.py b/freqtrade/edge/__init__.py index 2afb913bd..e5334fb46 100644 --- a/freqtrade/edge/__init__.py +++ b/freqtrade/edge/__init__.py @@ -13,6 +13,7 @@ from freqtrade.arguments import Arguments from freqtrade.arguments import TimeRange from freqtrade.strategy.interface import SellType from freqtrade.strategy.resolver import IStrategy, StrategyResolver +from collections import namedtuple import sys logger = logging.getLogger(__name__) @@ -21,16 +22,11 @@ logger = logging.getLogger(__name__) class Edge(): config: Dict = {} - _last_updated: int # Timestamp of pairs last updated time - _cached_pairs: list = [] # Keeps an array of - # [pair, stoploss, winrate, risk reward ratio, required risk reward, expectancy] - - _total_capital: float - _allowed_risk: float - _since_number_of_days: int - _timerange: TimeRange + _cached_pairs: Dict[str, Any] = {} # Keeps a list of pairs def __init__(self, config: Dict[str, Any], exchange=None) -> None: + + # Increasing recursive limit as with need it for large datasets sys.setrecursionlimit(10000) self.config = config self.exchange = exchange @@ -42,13 +38,18 @@ class Edge(): self.advise_buy = self.strategy.advise_buy self.edge_config = self.config.get('edge', {}) - self._cached_pairs: list = [] - self._total_capital = self.edge_config.get('total_capital_in_stake_currency') - self._allowed_risk = self.edge_config.get('allowed_risk') - self._since_number_of_days = self.edge_config.get('calculate_since_number_of_days', 14) - self._last_updated = 0 - self._timerange = Arguments.parse_timerange("%s-" % arrow.now().shift( + # pair info data type + self._pair_info = namedtuple( + 'pair_info', 'stoploss, winrate, risk_reward_ratio, required_risk_reward, expectancy') + self._cached_pairs: Dict[str, Any] = {} # Keeps a list of pairs + + self._total_capital: float = self.edge_config.get('total_capital_in_stake_currency') + self._allowed_risk: float = self.edge_config.get('allowed_risk') + self._since_number_of_days: int = self.edge_config.get('calculate_since_number_of_days', 14) + self._last_updated: int = 0 # Timestamp of pairs last updated time + + self._timerange: TimeRange = Arguments.parse_timerange("%s-" % arrow.now().shift( days=-1 * self._since_number_of_days).format('YYYYMMDD')) self.fee = self.exchange.get_fee() @@ -132,34 +133,24 @@ class Edge(): return True def stake_amount(self, pair: str) -> float: - info = [x for x in self._cached_pairs if x[0] == pair][0] - stoploss = info[1] + stoploss = self._cached_pairs[pair].stoploss allowed_capital_at_risk = round(self._total_capital * self._allowed_risk, 5) position_size = abs(round((allowed_capital_at_risk / stoploss), 5)) return position_size def stoploss(self, pair: str) -> float: - info = [x for x in self._cached_pairs if x[0] == pair][0] - return info[1] + return self._cached_pairs[pair].stoploss def filter(self, pairs) -> list: - # Filtering pairs acccording to the expectancy - filtered_expectancy: list = [] - # [pair, stoploss, winrate, risk reward ratio, required risk reward, expectancy] - filtered_expectancy = [ - x[0] for x in self._cached_pairs if ( - (x[5] > float( - self.edge_config.get( - 'minimum_expectancy', - 0.2))) & ( - x[2] > float( - self.edge_config.get( - 'minimum_winrate', - 0.60))))] + final = [] + + for pair, info in self._cached_pairs.items(): + if info.expectancy > float(self.edge_config.get('minimum_expectancy', 0.2)) and \ + info.winrate > float(self.edge_config.get('minimum_winrate', 0.60)) and \ + pair in pairs: + final.append(pair) - # Only return pairs which are included in "pairs" argument list - final = [x for x in filtered_expectancy if x in pairs] if final: logger.info( 'Edge validated only %s', @@ -220,7 +211,7 @@ class Edge(): return result - def _process_expectancy(self, results: DataFrame) -> list: + def _process_expectancy(self, results: DataFrame) -> Dict[str, Any]: """ This calculates WinRate, Required Risk Reward, Risk Reward and Expectancy of all pairs The calulation will be done per pair and per strategy. @@ -246,7 +237,7 @@ class Edge(): ####################################################################### if results.empty: - return [] + return {} groupby_aggregator = { 'profit_abs': [ @@ -286,12 +277,17 @@ class Edge(): df = df.sort_values(by=['expectancy', 'stoploss'], ascending=False).groupby( 'pair').first().sort_values(by=['expectancy'], ascending=False).reset_index() - # dropping unecessary columns - df.drop(columns=['nb_loss_trades', 'nb_win_trades', 'average_win', 'average_loss', - 'profit_sum', 'loss_sum', 'avg_trade_duration', 'nb_trades'], inplace=True) + final = {} + for x in df.itertuples(): + final[x.pair] = self._pair_info( + x.stoploss, + x.winrate, + x.risk_reward_ratio, + x.required_risk_reward, + x.expectancy) - # Returning an array of pairs in order of "expectancy" - return df.values + # Returning a list of pairs in order of "expectancy" + return final def _find_trades_for_stoploss_range(self, ticker_data, pair, stoploss_range): buy_column = ticker_data['buy'].values diff --git a/freqtrade/tests/edge/test_edge.py b/freqtrade/tests/edge/test_edge.py index 510a79af1..710920e91 100644 --- a/freqtrade/tests/edge/test_edge.py +++ b/freqtrade/tests/edge/test_edge.py @@ -2,6 +2,7 @@ from freqtrade.tests.conftest import get_patched_exchange from freqtrade.edge import Edge from pandas import DataFrame, to_datetime from freqtrade.strategy.interface import SellType +from collections import namedtuple import arrow import numpy as np import math @@ -20,17 +21,19 @@ from unittest.mock import MagicMock ticker_start_time = arrow.get(2018, 10, 3) ticker_interval_in_minute = 60 _ohlc = {'date': 0, 'buy': 1, 'open': 2, 'high': 3, 'low': 4, 'close': 5, 'sell': 6, 'volume': 7} +_pair_info = namedtuple( + 'pair_info', 'stoploss, winrate, risk_reward_ratio, required_risk_reward, expectancy') def test_filter(mocker, default_conf): exchange = get_patched_exchange(mocker, default_conf) edge = Edge(default_conf, exchange) mocker.patch('freqtrade.edge.Edge._cached_pairs', mocker.PropertyMock( - return_value=[ - ['E/F', -0.01, 0.66, 3.71, 0.50, 1.71], - ['C/D', -0.01, 0.66, 3.71, 0.50, 1.71], - ['N/O', -0.01, 0.66, 3.71, 0.50, 1.71] - ] + return_value={ + 'E/F': _pair_info(-0.01, 0.66, 3.71, 0.50, 1.71), + 'C/D': _pair_info(-0.01, 0.66, 3.71, 0.50, 1.71), + 'N/O': _pair_info(-0.01, 0.66, 3.71, 0.50, 1.71) + } )) pairs = ['A/B', 'C/D', 'E/F', 'G/H'] @@ -41,11 +44,11 @@ def test_stoploss(mocker, default_conf): exchange = get_patched_exchange(mocker, default_conf) edge = Edge(default_conf, exchange) mocker.patch('freqtrade.edge.Edge._cached_pairs', mocker.PropertyMock( - return_value=[ - ['E/F', -0.01, 0.66, 3.71, 0.50, 1.71], - ['C/D', -0.01, 0.66, 3.71, 0.50, 1.71], - ['N/O', -0.01, 0.66, 3.71, 0.50, 1.71] - ] + return_value={ + 'E/F': _pair_info(-0.01, 0.66, 3.71, 0.50, 1.71), + 'C/D': _pair_info(-0.01, 0.66, 3.71, 0.50, 1.71), + 'N/O': _pair_info(-0.01, 0.66, 3.71, 0.50, 1.71) + } )) assert edge.stoploss('E/F') == -0.01 @@ -61,7 +64,7 @@ def _validate_ohlc(buy_ohlc_sell_matrice): def _build_dataframe(buy_ohlc_sell_matrice): _validate_ohlc(buy_ohlc_sell_matrice) - tickers = [] + tickers= [] for ohlc in buy_ohlc_sell_matrice: ticker = { 'date': ticker_start_time.shift( @@ -79,9 +82,9 @@ def _build_dataframe(buy_ohlc_sell_matrice): frame = DataFrame(tickers) frame['date'] = to_datetime(frame['date'], - unit='ms', - utc=True, - infer_datetime_format=True) + unit = 'ms', + utc = True, + infer_datetime_format = True) return frame @@ -92,17 +95,17 @@ def _time_on_candle(number): def test_edge_heartbeat_calculate(mocker, default_conf): - exchange = get_patched_exchange(mocker, default_conf) - edge = Edge(default_conf, exchange) - heartbeat = default_conf['edge']['process_throttle_secs'] + exchange=get_patched_exchange(mocker, default_conf) + edge=Edge(default_conf, exchange) + heartbeat=default_conf['edge']['process_throttle_secs'] # should not recalculate if heartbeat not reached - edge._last_updated = arrow.utcnow().timestamp - heartbeat + 1 + edge._last_updated=arrow.utcnow().timestamp - heartbeat + 1 assert edge.calculate() is False -def mocked_load_data(datadir, pairs=[], ticker_interval='0m', refresh_pairs=False, +def mocked_load_data(datadir, pairs = [], ticker_interval = '0m', refresh_pairs = False, timerange=None, exchange=None): hz = 0.1 base = 0.001 @@ -202,13 +205,12 @@ def test_process_expectancy(mocker, default_conf): final = edge._process_expectancy(trades_df) assert len(final) == 1 - assert final[0][0] == 'TEST/BTC' - assert final[0][1] == -0.9 - assert round(final[0][2], 10) == 0.3333333333 - assert round(final[0][3], 10) == 306.5384615384 - assert round(final[0][4], 10) == 2.0 - assert round(final[0][5], 10) == 101.5128205128 - + assert 'TEST/BTC' in final + assert final['TEST/BTC'].stoploss == -0.9 + assert round(final['TEST/BTC'].winrate, 10) == 0.3333333333 + assert round(final['TEST/BTC'].risk_reward_ratio, 10) == 306.5384615384 + assert round(final['TEST/BTC'].required_risk_reward, 10) == 2.0 + assert round(final['TEST/BTC'].expectancy, 10) == 101.5128205128 # 1) Open trade should be removed from the end def test_case_1(mocker, default_conf): From 120655d262b754de46fb4fdd30246266e7ab0ee1 Mon Sep 17 00:00:00 2001 From: misagh Date: Sun, 4 Nov 2018 18:43:57 +0100 Subject: [PATCH 246/699] fixing tests for namedtuple --- freqtrade/edge/__init__.py | 16 +++++++++------- freqtrade/tests/conftest.py | 24 +++++++++++++----------- freqtrade/tests/test_freqtradebot.py | 2 +- 3 files changed, 23 insertions(+), 19 deletions(-) diff --git a/freqtrade/edge/__init__.py b/freqtrade/edge/__init__.py index e5334fb46..2cf2d8b96 100644 --- a/freqtrade/edge/__init__.py +++ b/freqtrade/edge/__init__.py @@ -149,7 +149,7 @@ class Edge(): if info.expectancy > float(self.edge_config.get('minimum_expectancy', 0.2)) and \ info.winrate > float(self.edge_config.get('minimum_winrate', 0.60)) and \ pair in pairs: - final.append(pair) + final.append(pair) if final: logger.info( @@ -279,12 +279,14 @@ class Edge(): final = {} for x in df.itertuples(): - final[x.pair] = self._pair_info( - x.stoploss, - x.winrate, - x.risk_reward_ratio, - x.required_risk_reward, - x.expectancy) + info = { + 'stoploss': x.stoploss, + 'winrate': x.winrate, + 'risk_reward_ratio': x.risk_reward_ratio, + 'required_risk_reward': x.required_risk_reward, + 'expectancy': x.expectancy + } + final[x.pair] = self._pair_info(**info) # Returning a list of pairs in order of "expectancy" return final diff --git a/freqtrade/tests/conftest.py b/freqtrade/tests/conftest.py index 86e7e73c8..3dd7b1f2c 100644 --- a/freqtrade/tests/conftest.py +++ b/freqtrade/tests/conftest.py @@ -4,6 +4,7 @@ import logging from datetime import datetime from functools import reduce from typing import Dict, Optional +from collections import namedtuple from unittest.mock import MagicMock, PropertyMock import arrow @@ -48,19 +49,20 @@ def patch_edge(mocker) -> None: # "LTC/BTC", # "XRP/BTC", # "NEO/BTC" + pair_info = namedtuple('pair_info', 'stoploss, winrate, risk_reward_ratio, required_risk_reward, expectancy') mocker.patch('freqtrade.edge.Edge._cached_pairs', mocker.PropertyMock( - return_value=[ - ['NEO/BTC', -0.20, 0.66, 3.71, 0.50, 1.71], - ['LTC/BTC', -0.21, 0.66, 3.71, 0.50, 1.71], - ] + return_value={ + 'NEO/BTC': pair_info(-0.20, 0.66, 3.71, 0.50, 1.71), + 'LTC/BTC': pair_info(-0.21, 0.66, 3.71, 0.50, 1.71), + } )) - mocker.patch('freqtrade.edge.Edge.stoploss', MagicMock(return_value=-0.20)) - mocker.patch('freqtrade.edge.Edge.calculate', MagicMock(return_value=True)) + mocker.patch('freqtrade.edge.Edge.stoploss', MagicMock(return_value = -0.20)) + mocker.patch('freqtrade.edge.Edge.calculate', MagicMock(return_value = True)) def get_patched_edge(mocker, config) -> Edge: patch_edge(mocker) - edge = Edge(config) + edge=Edge(config) return edge # Functions for recurrent object patching @@ -84,15 +86,15 @@ def get_patched_freqtradebot(mocker, config) -> FreqtradeBot: return FreqtradeBot(config) -def patch_coinmarketcap(mocker, value: Optional[Dict[str, float]] = None) -> None: +def patch_coinmarketcap(mocker, value: Optional[Dict[str, float]]=None) -> None: """ Mocker to coinmarketcap to speed up tests :param mocker: mocker to patch coinmarketcap class :return: None """ - tickermock = MagicMock(return_value={'price_usd': 12345.0}) - listmock = MagicMock(return_value={'data': [{'id': 1, 'name': 'Bitcoin', 'symbol': 'BTC', + tickermock=MagicMock(return_value={'price_usd': 12345.0}) + listmock=MagicMock(return_value={'data': [{'id': 1, 'name': 'Bitcoin', 'symbol': 'BTC', 'website_slug': 'bitcoin'}, {'id': 1027, 'name': 'Ethereum', 'symbol': 'ETH', 'website_slug': 'ethereum'} @@ -108,7 +110,7 @@ def patch_coinmarketcap(mocker, value: Optional[Dict[str, float]] = None) -> Non @pytest.fixture(scope="function") def default_conf(): """ Returns validated configuration suitable for most tests """ - configuration = { + configuration={ "max_open_trades": 1, "stake_currency": "BTC", "stake_amount": 0.001, diff --git a/freqtrade/tests/test_freqtradebot.py b/freqtrade/tests/test_freqtradebot.py index 266ad82ee..217e438df 100644 --- a/freqtrade/tests/test_freqtradebot.py +++ b/freqtrade/tests/test_freqtradebot.py @@ -256,7 +256,7 @@ def test_edge_overrides_stake_amount(mocker, default_conf) -> None: # strategy stoploss should be ignored freqtrade.strategy.stoploss = -0.05 - with pytest.raises(IndexError): + with pytest.raises(KeyError): freqtrade._get_trade_stake_amount('ETH/BTC') assert freqtrade._get_trade_stake_amount('NEO/BTC') == 0.025 From 8ea9b3746bfbedc20983b32fcbb5ebdaa5661dc4 Mon Sep 17 00:00:00 2001 From: misagh Date: Sun, 4 Nov 2018 18:51:54 +0100 Subject: [PATCH 247/699] passing pair to get_trade_stake_amount --- freqtrade/rpc/rpc.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/rpc/rpc.py b/freqtrade/rpc/rpc.py index c3cbce2e7..7e7b60ebc 100644 --- a/freqtrade/rpc/rpc.py +++ b/freqtrade/rpc/rpc.py @@ -410,7 +410,7 @@ class RPC(object): raise RPCException(f'position for {pair} already open - id: {trade.id}') # gen stake amount - stakeamount = self._freqtrade._get_trade_stake_amount() + stakeamount = self._freqtrade._get_trade_stake_amount(pair) # execute buy if self._freqtrade.execute_buy(pair, stakeamount, price): From ed24d96a7944dcdf0c6547cf53f134053476c578 Mon Sep 17 00:00:00 2001 From: misagh Date: Sun, 4 Nov 2018 18:57:57 +0100 Subject: [PATCH 248/699] some formatting for flake8 --- freqtrade/tests/conftest.py | 18 ++++++++++-------- freqtrade/tests/edge/test_edge.py | 20 +++++++++++--------- 2 files changed, 21 insertions(+), 17 deletions(-) diff --git a/freqtrade/tests/conftest.py b/freqtrade/tests/conftest.py index 3dd7b1f2c..37109767b 100644 --- a/freqtrade/tests/conftest.py +++ b/freqtrade/tests/conftest.py @@ -49,20 +49,22 @@ def patch_edge(mocker) -> None: # "LTC/BTC", # "XRP/BTC", # "NEO/BTC" - pair_info = namedtuple('pair_info', 'stoploss, winrate, risk_reward_ratio, required_risk_reward, expectancy') + pair_info = namedtuple( + 'pair_info', + 'stoploss, winrate, risk_reward_ratio, required_risk_reward, expectancy') mocker.patch('freqtrade.edge.Edge._cached_pairs', mocker.PropertyMock( return_value={ 'NEO/BTC': pair_info(-0.20, 0.66, 3.71, 0.50, 1.71), 'LTC/BTC': pair_info(-0.21, 0.66, 3.71, 0.50, 1.71), } )) - mocker.patch('freqtrade.edge.Edge.stoploss', MagicMock(return_value = -0.20)) - mocker.patch('freqtrade.edge.Edge.calculate', MagicMock(return_value = True)) + mocker.patch('freqtrade.edge.Edge.stoploss', MagicMock(return_value=-0.20)) + mocker.patch('freqtrade.edge.Edge.calculate', MagicMock(return_value=True)) def get_patched_edge(mocker, config) -> Edge: patch_edge(mocker) - edge=Edge(config) + edge = Edge(config) return edge # Functions for recurrent object patching @@ -86,15 +88,15 @@ def get_patched_freqtradebot(mocker, config) -> FreqtradeBot: return FreqtradeBot(config) -def patch_coinmarketcap(mocker, value: Optional[Dict[str, float]]=None) -> None: +def patch_coinmarketcap(mocker, value: Optional[Dict[str, float]] = None) -> None: """ Mocker to coinmarketcap to speed up tests :param mocker: mocker to patch coinmarketcap class :return: None """ - tickermock=MagicMock(return_value={'price_usd': 12345.0}) - listmock=MagicMock(return_value={'data': [{'id': 1, 'name': 'Bitcoin', 'symbol': 'BTC', + tickermock = MagicMock(return_value={'price_usd': 12345.0}) + listmock = MagicMock(return_value={'data': [{'id': 1, 'name': 'Bitcoin', 'symbol': 'BTC', 'website_slug': 'bitcoin'}, {'id': 1027, 'name': 'Ethereum', 'symbol': 'ETH', 'website_slug': 'ethereum'} @@ -110,7 +112,7 @@ def patch_coinmarketcap(mocker, value: Optional[Dict[str, float]]=None) -> None: @pytest.fixture(scope="function") def default_conf(): """ Returns validated configuration suitable for most tests """ - configuration={ + configuration = { "max_open_trades": 1, "stake_currency": "BTC", "stake_amount": 0.001, diff --git a/freqtrade/tests/edge/test_edge.py b/freqtrade/tests/edge/test_edge.py index 710920e91..18e934352 100644 --- a/freqtrade/tests/edge/test_edge.py +++ b/freqtrade/tests/edge/test_edge.py @@ -64,7 +64,7 @@ def _validate_ohlc(buy_ohlc_sell_matrice): def _build_dataframe(buy_ohlc_sell_matrice): _validate_ohlc(buy_ohlc_sell_matrice) - tickers= [] + tickers = [] for ohlc in buy_ohlc_sell_matrice: ticker = { 'date': ticker_start_time.shift( @@ -82,9 +82,9 @@ def _build_dataframe(buy_ohlc_sell_matrice): frame = DataFrame(tickers) frame['date'] = to_datetime(frame['date'], - unit = 'ms', - utc = True, - infer_datetime_format = True) + unit='ms', + utc=True, + infer_datetime_format=True) return frame @@ -95,17 +95,17 @@ def _time_on_candle(number): def test_edge_heartbeat_calculate(mocker, default_conf): - exchange=get_patched_exchange(mocker, default_conf) - edge=Edge(default_conf, exchange) - heartbeat=default_conf['edge']['process_throttle_secs'] + exchange = get_patched_exchange(mocker, default_conf) + edge = Edge(default_conf, exchange) + heartbeat = default_conf['edge']['process_throttle_secs'] # should not recalculate if heartbeat not reached - edge._last_updated=arrow.utcnow().timestamp - heartbeat + 1 + edge._last_updated = arrow.utcnow().timestamp - heartbeat + 1 assert edge.calculate() is False -def mocked_load_data(datadir, pairs = [], ticker_interval = '0m', refresh_pairs = False, +def mocked_load_data(datadir, pairs=[], ticker_interval='0m', refresh_pairs=False, timerange=None, exchange=None): hz = 0.1 base = 0.001 @@ -213,6 +213,8 @@ def test_process_expectancy(mocker, default_conf): assert round(final['TEST/BTC'].expectancy, 10) == 101.5128205128 # 1) Open trade should be removed from the end + + def test_case_1(mocker, default_conf): exchange = get_patched_exchange(mocker, default_conf) edge = Edge(default_conf, exchange) From 3666e0139646eb667869a5e166eecdd28397932e Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Mon, 5 Nov 2018 13:34:07 +0100 Subject: [PATCH 249/699] Update ccxt from 1.17.464 to 1.17.469 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index d57cbb7c4..7da6ea321 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -ccxt==1.17.464 +ccxt==1.17.469 SQLAlchemy==1.2.13 python-telegram-bot==11.1.0 arrow==0.12.1 From 8a5e4c3f304aa6ff4a3e0722e1fcbb794c195e26 Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Mon, 5 Nov 2018 13:34:09 +0100 Subject: [PATCH 250/699] Update cachetools from 2.1.0 to 3.0.0 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 7da6ea321..2a505f4e0 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,7 +2,7 @@ ccxt==1.17.469 SQLAlchemy==1.2.13 python-telegram-bot==11.1.0 arrow==0.12.1 -cachetools==2.1.0 +cachetools==3.0.0 requests==2.20.0 urllib3==1.24.1 wrapt==1.10.11 From 9f03c26c9afccfecb74a3e715b11a9cf9531c0f7 Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Mon, 5 Nov 2018 13:34:10 +0100 Subject: [PATCH 251/699] Update numpy from 1.15.3 to 1.15.4 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 2a505f4e0..811334fb9 100644 --- a/requirements.txt +++ b/requirements.txt @@ -10,7 +10,7 @@ pandas==0.23.4 scikit-learn==0.20.0 scipy==1.1.0 jsonschema==2.6.0 -numpy==1.15.3 +numpy==1.15.4 TA-Lib==0.4.17 pytest==3.9.3 pytest-mock==1.10.0 From f92d229f2ea55c318cc84b18bd74d8a0393eb903 Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Mon, 5 Nov 2018 13:34:12 +0100 Subject: [PATCH 252/699] Update pytest from 3.9.3 to 3.10.0 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 811334fb9..02c3a54ff 100644 --- a/requirements.txt +++ b/requirements.txt @@ -12,7 +12,7 @@ scipy==1.1.0 jsonschema==2.6.0 numpy==1.15.4 TA-Lib==0.4.17 -pytest==3.9.3 +pytest==3.10.0 pytest-mock==1.10.0 pytest-asyncio==0.9.0 pytest-cov==2.6.0 From 2c0fc3c73564ec7eec9d309758a83eb0f6ea3ef9 Mon Sep 17 00:00:00 2001 From: misagh Date: Mon, 5 Nov 2018 14:36:10 +0100 Subject: [PATCH 253/699] Test latex images --- docs/edge.md | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/docs/edge.md b/docs/edge.md index d72422d53..40ecedc55 100644 --- a/docs/edge.md +++ b/docs/edge.md @@ -1,14 +1,22 @@ -# Edge positionning +# Edge positioning + +This page explains how to use Edge Positioning module in your bot in order to enter into a trade only of the trade has a reasonable win rate and risk reward ration, and consequently adjust your position size and stoploss. + +## Table of Contents + +- [Introduction](#introduction) ## Introduction -Trading is all about probability. no one can claim having the strategy working all the time. you have to assume that sometimes you lose.

+Trading is all about probability. no one can claim that he has the strategy working all the time. you have to assume that sometimes you lose.

But it doesn't mean there is no rule, it only means rules should work "most of the time". let's play a game: we toss a coin, heads: I give you 10$, tails: You give me 10$. is it an interetsing game ? no, it is quite boring, isn't it?

But lets say the probabiliy that we have heads is 80%, and the probablilty that we have tails is 20%. now it is becoming interesting ... That means 10$ x 80% versus 10$ x 20%. 8$ versus 2$. that means over time you will win 8$ risking only 2$ on each toss of coin.

-lets complicate it more: you win 80% of time but only 2$, I win 20% of time but if I win I win 8$. the calculation is: 80% * 2$ versus 20% * 8$. it is becoming boring again because overtime you win $1.6$ (80% x 2$) and me $1.6 (20% * 8$) too.

-The question is: how do you calculate that? how do you know if you wanna play with me? +lets complicate it more: you win 80% of the time but only 2$, I win 20% of the time but 8$. the calculation is: 80% * 2$ versus 20% * 8$. it is becoming boring again because overtime you win $1.6$ (80% x 2$) and me $1.6 (20% * 8$) too.

+The question is: how do you calculate that? how do you know if you wanna play? The answer comes to two factors: - Win Rate - Risk Reward Ratio -Win rate is quite self explanatory. means over X trades what is the perctange winning number of trades compared to total number of trades (note that we don't consider how much you gained but only If you won or not). \ No newline at end of file +Win rate means over X trades what is the perctange of winning trades to total number of trades (note that we don't consider how much you gained but only If you won or not). + + \ No newline at end of file From cc41317670f7aa695ebdbda4893143caea84a82c Mon Sep 17 00:00:00 2001 From: misagh Date: Mon, 5 Nov 2018 15:49:10 +0100 Subject: [PATCH 254/699] documentation WIP 1 --- docs/edge.md | 28 ++++++++++++++++++++++++++-- 1 file changed, 26 insertions(+), 2 deletions(-) diff --git a/docs/edge.md b/docs/edge.md index 40ecedc55..cab2de748 100644 --- a/docs/edge.md +++ b/docs/edge.md @@ -17,6 +17,30 @@ The answer comes to two factors: - Win Rate - Risk Reward Ratio -Win rate means over X trades what is the perctange of winning trades to total number of trades (note that we don't consider how much you gained but only If you won or not). - \ No newline at end of file +### Win Rate +Means over X trades what is the perctange of winning trades to total number of trades (note that we don't consider how much you gained but only If you won or not). + + + + +### Risk Reward Ratio +Risk Reward Ratio is a formula used to measure the expected gains of a given investment against the risk of loss. it is basically what you potentially win divided by what you potentially lose: + + + +Over time, on many trades, you can calculate your risk reward by dividing your average profit on winning trades by your average loss on losing trades: + + + +### Expectancy + +At this point we can combine W and R to create an expectancy ratio. This is a simple process of multiplying the risk reward ratio by the percentage of winning trades, and subtracting the percentage of losing trades, which is calculated as follows: + +Expectancy Ratio = (Risk Reward Ratio x Win Rate) – Loss Rate + +Superficially, this means that on average you expect this strategy’s trades to return .68 times the size of your losers. This is important for two reasons: First, it may seem obvious, but you know right away that you have a positive return. Second, you now have a number you can compare to other candidate systems to make decisions about which ones you employ. + +It is important to remember that any system with an expectancy greater than 0 is profitable using past data. The key is finding one that will be profitable in the future. + +You can also use this number to evaluate the effectiveness of modifications to this system. From 49d30ad0e250aa935a51b49e250c90ae3e297967 Mon Sep 17 00:00:00 2001 From: misagh Date: Mon, 5 Nov 2018 15:55:35 +0100 Subject: [PATCH 255/699] doc WIP 2 --- docs/edge.md | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/docs/edge.md b/docs/edge.md index cab2de748..d5648d9df 100644 --- a/docs/edge.md +++ b/docs/edge.md @@ -22,16 +22,20 @@ The answer comes to two factors: Means over X trades what is the perctange of winning trades to total number of trades (note that we don't consider how much you gained but only If you won or not). - +W = (Number of winning trades) / (Number of losing trades) ### Risk Reward Ratio Risk Reward Ratio is a formula used to measure the expected gains of a given investment against the risk of loss. it is basically what you potentially win divided by what you potentially lose: - +R = Profit / Loss Over time, on many trades, you can calculate your risk reward by dividing your average profit on winning trades by your average loss on losing trades: - +average profit = (Sum of profits) / (Number of winning trades) + +average loss = (Sum of losses) / (Number of losing trades) + +R = (average profit) / (average loss) ### Expectancy @@ -39,6 +43,10 @@ At this point we can combine W and R to create an expectancy ratio. This is a si Expectancy Ratio = (Risk Reward Ratio x Win Rate) – Loss Rate +So lets say your Win rate is 28% and your Risk Reward Ratio is 5: + +Expectancy = (5 * 0.28) - 0.72 = 0.68 + Superficially, this means that on average you expect this strategy’s trades to return .68 times the size of your losers. This is important for two reasons: First, it may seem obvious, but you know right away that you have a positive return. Second, you now have a number you can compare to other candidate systems to make decisions about which ones you employ. It is important to remember that any system with an expectancy greater than 0 is profitable using past data. The key is finding one that will be profitable in the future. From 4fbabd3b99f1cd8ed3ea9054dbc5c8bbb6ec711f Mon Sep 17 00:00:00 2001 From: misagh Date: Mon, 5 Nov 2018 17:24:11 +0100 Subject: [PATCH 256/699] Doc for Edge WIP 3 --- docs/edge.md | 85 ++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 85 insertions(+) diff --git a/docs/edge.md b/docs/edge.md index d5648d9df..07ccafd83 100644 --- a/docs/edge.md +++ b/docs/edge.md @@ -5,6 +5,8 @@ This page explains how to use Edge Positioning module in your bot in order to en ## Table of Contents - [Introduction](#introduction) +- [How does it work?](#how-does-it-work?) +- [Configurations](#configurations) ## Introduction Trading is all about probability. no one can claim that he has the strategy working all the time. you have to assume that sometimes you lose.

@@ -52,3 +54,86 @@ Superficially, this means that on average you expect this strategy’s trades to It is important to remember that any system with an expectancy greater than 0 is profitable using past data. The key is finding one that will be profitable in the future. You can also use this number to evaluate the effectiveness of modifications to this system. + +**NOTICE:** It's important to keep in mind that Edge is testing your expectancy using historical data , there's no guarantee that you will have a similar edge in the future. It's still vital to do this testing in order to build confidence in your methodology, but be wary of "curve-fitting" your approach to the historical data as things are unlikely to play out the exact same way for future trades. + +## How does it work? +If enabled in config, Edge will go through historical data with a range of stoplosses in order to find buy and sell/stoploss signals. it then calculates win rate and expectancy over X trades for each stoploss. here is an example: + +| Pair | Stoploss | Win Rate | Risk Reward Ratio | Expectancy | +|----------|:-------------:|-------------:|------------------:|-----------:| +| XZC/ETH | -0.03 | 0.52 |1.359670 | 0.228 | +| XZC/ETH | -0.01 | 0.50 |1.176384 | 0.088 | +| XZC/ETH | -0.02 | 0.51 |1.115941 | 0.079 | + +The goal here is to find the best stoploss for the strategy in order to have the maximum expectancy. in the above example stoploss at 3% leads to the maximum expectancy according to historical data. + +Edge then forces stoploss to your strategy dynamically. + +### Position size +Edge dictates the stake amount for each trade to the bot according to the following factors: + +- Allowed capital at risk +- Stoploss + +Alowed capital at risk is calculated as follows: + +**allowed capital at risk** = **total capital** X **allowed risk per trade** + +**Stoploss** is calculated as described above against historical data. + +Your position size then will be: + +**position size** = **allowed capital at risk** / **stoploss** + +## Configurations +Edge has following configurations: + +#### enabled +If true, then Edge will run periodically + +#### process_throttle_secs +How often should Edge run ? (in seconds) + +#### calculate_since_number_of_days +Number of days of data agaist which Edge calculates Win Rate, Risk Reward and Expectancy +Note that it downloads historical data so increasing this number would lead to slowing down the bot + +#### total_capital_in_stake_currency +This your total capital at risk. if edge is enabled then stake_amount is ignored in favor of this parameter + +#### allowed_risk +Percentage of allowed risk per trade + +#### stoploss_range_min +Minimum stoploss (default to -0.01) + +#### stoploss_range_max +Maximum stoploss (default to -0.10) + +#### stoploss_range_step +As an example if this is set to -0.01 then Edge will test the strategy for [-0.01, -0,02, -0,03 ..., -0.09, -0.10] ranges. +Note than having a smaller step means having a bigger range which could lead to slow calculation.
+if you set this parameter to -0.001, you then slow down the Edge calculatiob by a factor of 10 + +#### minimum_winrate +It filters pairs which don't have at least minimum_winrate (default to 0.60) +This comes handy if you want to be conservative and don't comprise win rate in favor of risk reward ratio. + +#### minimum_expectancy +It filters paris which have an expectancy lower than this number (default to 0.20) +Having an expectancy of 0.20 means if you put 10$ on a trade you expect a 12$ return. + +#### min_trade_number +When calulating W and R and E (expectancy) against histoical data, you always want to have a minimum number of trades. the more this number is the more Edge is reliable. having a win rate of 100% on a single trade doesn't mean anything at all. but having a win rate of 70% over past 100 trades means clearly something.
+ +Default to 10 (it is highly recommanded not to decrease this number) + +#### max_trade_duration_minute +Edge will filter out trades with long duration. if a trade is profitable after 1 month, it is hard to evaluate the stratgy based on it. but if most of trades are profitable and they have maximum duration of 30 minutes, then it is clearly a good sign.
+Default to 1 day (1440 = 60 * 24) + + +#### remove_pumps +Edge will remove sudden pumps in a given market while going through historical data. however, given that pumps happen very often in crypto markets, we recommand you keep this off.
+Default to false \ No newline at end of file From 8a25490146091c0f1de35a397a22153b0f89d531 Mon Sep 17 00:00:00 2001 From: misagh Date: Mon, 5 Nov 2018 17:28:07 +0100 Subject: [PATCH 257/699] Typo corrected --- docs/edge.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/edge.md b/docs/edge.md index 07ccafd83..f2756e831 100644 --- a/docs/edge.md +++ b/docs/edge.md @@ -100,10 +100,11 @@ Number of days of data agaist which Edge calculates Win Rate, Risk Reward and Ex Note that it downloads historical data so increasing this number would lead to slowing down the bot #### total_capital_in_stake_currency -This your total capital at risk. if edge is enabled then stake_amount is ignored in favor of this parameter +This your total capital at risk in your stake currency. if edge is enabled then stake_amount is ignored in favor of this parameter #### allowed_risk -Percentage of allowed risk per trade +Percentage of allowed risk per trade
+default to 1% #### stoploss_range_min Minimum stoploss (default to -0.01) @@ -114,7 +115,7 @@ Maximum stoploss (default to -0.10) #### stoploss_range_step As an example if this is set to -0.01 then Edge will test the strategy for [-0.01, -0,02, -0,03 ..., -0.09, -0.10] ranges. Note than having a smaller step means having a bigger range which could lead to slow calculation.
-if you set this parameter to -0.001, you then slow down the Edge calculatiob by a factor of 10 +if you set this parameter to -0.001, you then slow down the Edge calculation by a factor of 10 #### minimum_winrate It filters pairs which don't have at least minimum_winrate (default to 0.60) @@ -133,7 +134,6 @@ Default to 10 (it is highly recommanded not to decrease this number) Edge will filter out trades with long duration. if a trade is profitable after 1 month, it is hard to evaluate the stratgy based on it. but if most of trades are profitable and they have maximum duration of 30 minutes, then it is clearly a good sign.
Default to 1 day (1440 = 60 * 24) - #### remove_pumps Edge will remove sudden pumps in a given market while going through historical data. however, given that pumps happen very often in crypto markets, we recommand you keep this off.
Default to false \ No newline at end of file From 5754e5196019335d8a6032c20fe1900bcb8ccaa6 Mon Sep 17 00:00:00 2001 From: misagh Date: Mon, 5 Nov 2018 17:32:12 +0100 Subject: [PATCH 258/699] more typos --- docs/edge.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/edge.md b/docs/edge.md index f2756e831..808c4f008 100644 --- a/docs/edge.md +++ b/docs/edge.md @@ -1,6 +1,6 @@ # Edge positioning -This page explains how to use Edge Positioning module in your bot in order to enter into a trade only of the trade has a reasonable win rate and risk reward ration, and consequently adjust your position size and stoploss. +This page explains how to use Edge Positioning module in your bot in order to enter into a trade only if the trade has a reasonable win rate and risk reward ratio, and consequently adjust your position size and stoploss. ## Table of Contents @@ -9,7 +9,7 @@ This page explains how to use Edge Positioning module in your bot in order to en - [Configurations](#configurations) ## Introduction -Trading is all about probability. no one can claim that he has the strategy working all the time. you have to assume that sometimes you lose.

+Trading is all about probability. no one can claim that he has a strategy working all the time. you have to assume that sometimes you lose.

But it doesn't mean there is no rule, it only means rules should work "most of the time". let's play a game: we toss a coin, heads: I give you 10$, tails: You give me 10$. is it an interetsing game ? no, it is quite boring, isn't it?

But lets say the probabiliy that we have heads is 80%, and the probablilty that we have tails is 20%. now it is becoming interesting ... That means 10$ x 80% versus 10$ x 20%. 8$ versus 2$. that means over time you will win 8$ risking only 2$ on each toss of coin.

From 7278cdc7d5b5962673b97d036e0d9a5e2058d32f Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Tue, 6 Nov 2018 13:34:06 +0100 Subject: [PATCH 259/699] Update ccxt from 1.17.469 to 1.17.476 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 02c3a54ff..2671660ea 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -ccxt==1.17.469 +ccxt==1.17.476 SQLAlchemy==1.2.13 python-telegram-bot==11.1.0 arrow==0.12.1 From 5c38b92a75f6ac3e375d7a93043c4e7d59ad2b1b Mon Sep 17 00:00:00 2001 From: misagh Date: Tue, 6 Nov 2018 19:05:42 +0100 Subject: [PATCH 260/699] simplifying calculations to be more readable --- freqtrade/edge/__init__.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/freqtrade/edge/__init__.py b/freqtrade/edge/__init__.py index 2cf2d8b96..81d3d7c99 100644 --- a/freqtrade/edge/__init__.py +++ b/freqtrade/edge/__init__.py @@ -226,7 +226,7 @@ class Edge(): # Then every value more than (standard deviation + 2*average) is out (pump) # # Removing Pumps - if self.edge_config.get('remove_pumps', True): + if self.edge_config.get('remove_pumps', False): results = results.groupby(['pair', 'stoploss']).apply( lambda x: x[x['profit_abs'] < 2 * x['profit_abs'].std() + x['profit_abs'].mean()]) ########################################################################## @@ -249,7 +249,7 @@ class Edge(): 'trade_duration': [('avg_trade_duration', 'mean')] } - # Group by (pair and stoploss) the applying above aggregator + # Group by (pair and stoploss) by applying above aggregator df = results.groupby(['pair', 'stoploss'])['profit_abs', 'trade_duration'].agg( groupby_aggregator).reset_index(col_level=1) @@ -264,14 +264,14 @@ class Edge(): # Win rate = number of profitable trades / number of trades df['winrate'] = df['nb_win_trades'] / df['nb_trades'] - # risk_reward_ratio = 1 / (average loss / average win) - df['risk_reward_ratio'] = 1 / (df['average_loss'] / df['average_win']) + # risk_reward_ratio = average win / average loss + df['risk_reward_ratio'] = df['average_win'] / df['average_loss'] # required_risk_reward = (1 / winrate) - 1 df['required_risk_reward'] = (1 / df['winrate']) - 1 - # expectancy = ((1 + average_win/average_loss) * winrate) - 1 - df['expectancy'] = ((1 + df['average_win'] / df['average_loss']) * df['winrate']) - 1 + # expectancy = (risk_reward_ratio * winrate) - (lossrate) + df['expectancy'] = (df['risk_reward_ratio'] * df['winrate']) - (1 - df['winrate']) # sort by expectancy and stoploss df = df.sort_values(by=['expectancy', 'stoploss'], ascending=False).groupby( From 133ba5d6a1856d2bfdc1623c63f8b3ab4972cf1f Mon Sep 17 00:00:00 2001 From: misagh Date: Tue, 6 Nov 2018 19:16:20 +0100 Subject: [PATCH 261/699] =?UTF-8?q?moving=20stop=20loss=20range=20to=20ini?= =?UTF-8?q?t=20as=20it=20doesn=E2=80=99t=20need=20to=20be=20called=20on=20?= =?UTF-8?q?each=20iteration?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- freqtrade/edge/__init__.py | 25 +++++++++++++++++++------ 1 file changed, 19 insertions(+), 6 deletions(-) diff --git a/freqtrade/edge/__init__.py b/freqtrade/edge/__init__.py index 81d3d7c99..748f95a6e 100644 --- a/freqtrade/edge/__init__.py +++ b/freqtrade/edge/__init__.py @@ -20,6 +20,13 @@ logger = logging.getLogger(__name__) class Edge(): + """ + Calculates Win Rate, Risk Reward Ratio, Expectancy + against historical data for a give set of markets and a strategy + it then adjusts stoploss and position size accordingly + and force it into the strategy + Author: https://github.com/mishaker + """ config: Dict = {} _cached_pairs: Dict[str, Any] = {} # Keeps a list of pairs @@ -49,6 +56,17 @@ class Edge(): self._since_number_of_days: int = self.edge_config.get('calculate_since_number_of_days', 14) self._last_updated: int = 0 # Timestamp of pairs last updated time + self._stoploss_range_min = float(self.edge_config.get('stoploss_range_min', -0.01)) + self._stoploss_range_max = float(self.edge_config.get('stoploss_range_max', -0.05)) + self._stoploss_range_step = float(self.edge_config.get('stoploss_range_step', -0.001)) + + # calculating stoploss range + self._stoploss_range = np.arange( + self._stoploss_range_min, + self._stoploss_range_max, + self._stoploss_range_step + ) + self._timerange: TimeRange = Arguments.parse_timerange("%s-" % arrow.now().shift( days=-1 * self._since_number_of_days).format('YYYYMMDD')) @@ -91,11 +109,6 @@ class Edge(): ) headers = ['date', 'buy', 'open', 'close', 'sell', 'high', 'low'] - stoploss_range_min = float(self.edge_config.get('stoploss_range_min', -0.01)) - stoploss_range_max = float(self.edge_config.get('stoploss_range_max', -0.05)) - stoploss_range_step = float(self.edge_config.get('stoploss_range_step', -0.001)) - stoploss_range = np.arange(stoploss_range_min, stoploss_range_max, stoploss_range_step) - trades: list = [] for pair, pair_data in preprocessed.items(): # Sorting dataframe by date and reset index @@ -105,7 +118,7 @@ class Edge(): ticker_data = self.advise_sell( self.advise_buy(pair_data, {'pair': pair}), {'pair': pair})[headers].copy() - trades += self._find_trades_for_stoploss_range(ticker_data, pair, stoploss_range) + trades += self._find_trades_for_stoploss_range(ticker_data, pair, self._stoploss_range) # If no trade found then exit if len(trades) == 0: From bcecaa69d4cedc651de5887b59e158a5b8377752 Mon Sep 17 00:00:00 2001 From: misagh Date: Tue, 6 Nov 2018 19:41:46 +0100 Subject: [PATCH 262/699] removing global variable modification --- freqtrade/edge/__init__.py | 2 -- freqtrade/strategy/interface.py | 2 +- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/freqtrade/edge/__init__.py b/freqtrade/edge/__init__.py index 748f95a6e..648130e70 100644 --- a/freqtrade/edge/__init__.py +++ b/freqtrade/edge/__init__.py @@ -33,8 +33,6 @@ class Edge(): def __init__(self, config: Dict[str, Any], exchange=None) -> None: - # Increasing recursive limit as with need it for large datasets - sys.setrecursionlimit(10000) self.config = config self.exchange = exchange self.strategy: IStrategy = StrategyResolver(self.config).strategy diff --git a/freqtrade/strategy/interface.py b/freqtrade/strategy/interface.py index 4d613265a..de5fc1a60 100644 --- a/freqtrade/strategy/interface.py +++ b/freqtrade/strategy/interface.py @@ -203,7 +203,7 @@ class IStrategy(ABC): return buy, sell def should_sell(self, trade: Trade, rate: float, date: datetime, buy: bool, - sell: bool, force_stoploss=0) -> SellCheckTuple: + sell: bool, force_stoploss: float=0) -> SellCheckTuple: """ This function evaluate if on the condition required to trigger a sell has been reached if the threshold is reached and updates the trade record. From 1b457e902c1e168068f8bad396419108764bc2f4 Mon Sep 17 00:00:00 2001 From: misagh Date: Tue, 6 Nov 2018 19:45:41 +0100 Subject: [PATCH 263/699] config initializer refactored --- freqtrade/freqtradebot.py | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index 330306307..00dfa6cbc 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -58,8 +58,13 @@ class FreqtradeBot(object): self.exchange = Exchange(self.config) # Initializing Edge only if enabled - if self.config.get('edge', {}).get('enabled', False): - self.edge = Edge(self.config, self.exchange) + self.edge = Edge( + self.config, + self.exchange) if self.config.get( + 'edge', + {}).get( + 'enabled', + False) else None self.active_pair_whitelist: List[str] = self.config['exchange']['pair_whitelist'] self._init_modules() @@ -190,7 +195,7 @@ class FreqtradeBot(object): # Should be called before refresh_tickers # Otherwise it will override cached klines in exchange # with delta value (klines only from last refresh_pairs) - if self.config.get('edge', {}).get('enabled', False): + if self.edge: self.edge.calculate() # Refreshing candles @@ -332,7 +337,7 @@ class FreqtradeBot(object): for the stake currency :return: float: Stake Amount """ - if self.config['edge']['enabled']: + if self.edge: stake_amount = self.edge.stake_amount(pair) else: stake_amount = self.config['stake_amount'] @@ -354,7 +359,7 @@ class FreqtradeBot(object): stake_amount, self.config['stake_currency']) ) - return float(stake_amount) + return stake_amount def _get_min_pair_stake_amount(self, pair: str, price: float) -> Optional[float]: markets = self.exchange.get_markets() @@ -407,7 +412,7 @@ class FreqtradeBot(object): # running get_signal on historical data fetched # to find buy signals - if self.config['edge']['enabled']: + if self.edge: whitelist = self.edge.filter(whitelist) for _pair in whitelist: From 23d3a7f31e70c2afa5d0e94132608c4d0cd0e2e9 Mon Sep 17 00:00:00 2001 From: misagh Date: Tue, 6 Nov 2018 20:11:15 +0100 Subject: [PATCH 264/699] capital after dots and default values corrected --- config.json.example | 2 +- docs/edge.md | 33 +++++++++++++++++---------------- 2 files changed, 18 insertions(+), 17 deletions(-) diff --git a/config.json.example b/config.json.example index 57b97c61c..bbd9648da 100644 --- a/config.json.example +++ b/config.json.example @@ -56,7 +56,7 @@ "edge": { "enabled": false, "process_throttle_secs": 3600, - "calculate_since_number_of_days": 2, + "calculate_since_number_of_days": 7, "total_capital_in_stake_currency": 0.5, "allowed_risk": 0.01, "stoploss_range_min": -0.01, diff --git a/docs/edge.md b/docs/edge.md index 808c4f008..b083d1575 100644 --- a/docs/edge.md +++ b/docs/edge.md @@ -9,12 +9,12 @@ This page explains how to use Edge Positioning module in your bot in order to en - [Configurations](#configurations) ## Introduction -Trading is all about probability. no one can claim that he has a strategy working all the time. you have to assume that sometimes you lose.

-But it doesn't mean there is no rule, it only means rules should work "most of the time". let's play a game: we toss a coin, heads: I give you 10$, tails: You give me 10$. is it an interetsing game ? no, it is quite boring, isn't it?

-But lets say the probabiliy that we have heads is 80%, and the probablilty that we have tails is 20%. now it is becoming interesting ... -That means 10$ x 80% versus 10$ x 20%. 8$ versus 2$. that means over time you will win 8$ risking only 2$ on each toss of coin.

-lets complicate it more: you win 80% of the time but only 2$, I win 20% of the time but 8$. the calculation is: 80% * 2$ versus 20% * 8$. it is becoming boring again because overtime you win $1.6$ (80% x 2$) and me $1.6 (20% * 8$) too.

-The question is: how do you calculate that? how do you know if you wanna play? +Trading is all about probability. No one can claim that he has a strategy working all the time. You have to assume that sometimes you lose.

+But it doesn't mean there is no rule, it only means rules should work "most of the time". Let's play a game: we toss a coin, heads: I give you 10$, tails: You give me 10$. Is it an interetsing game ? no, it is quite boring, isn't it?

+But lets say the probabiliy that we have heads is 80%, and the probablilty that we have tails is 20%. Now it is becoming interesting ... +That means 10$ x 80% versus 10$ x 20%. 8$ versus 2$. That means over time you will win 8$ risking only 2$ on each toss of coin.

+Lets complicate it more: you win 80% of the time but only 2$, I win 20% of the time but 8$. The calculation is: 80% * 2$ versus 20% * 8$. It is becoming boring again because overtime you win $1.6$ (80% x 2$) and me $1.6 (20% * 8$) too.

+The question is: How do you calculate that? how do you know if you wanna play? The answer comes to two factors: - Win Rate - Risk Reward Ratio @@ -27,7 +27,7 @@ Means over X trades what is the perctange of winning trades to total number of t W = (Number of winning trades) / (Number of losing trades) ### Risk Reward Ratio -Risk Reward Ratio is a formula used to measure the expected gains of a given investment against the risk of loss. it is basically what you potentially win divided by what you potentially lose: +Risk Reward Ratio is a formula used to measure the expected gains of a given investment against the risk of loss. It is basically what you potentially win divided by what you potentially lose: R = Profit / Loss @@ -58,7 +58,7 @@ You can also use this number to evaluate the effectiveness of modifications to t **NOTICE:** It's important to keep in mind that Edge is testing your expectancy using historical data , there's no guarantee that you will have a similar edge in the future. It's still vital to do this testing in order to build confidence in your methodology, but be wary of "curve-fitting" your approach to the historical data as things are unlikely to play out the exact same way for future trades. ## How does it work? -If enabled in config, Edge will go through historical data with a range of stoplosses in order to find buy and sell/stoploss signals. it then calculates win rate and expectancy over X trades for each stoploss. here is an example: +If enabled in config, Edge will go through historical data with a range of stoplosses in order to find buy and sell/stoploss signals. It then calculates win rate and expectancy over X trades for each stoploss. Here is an example: | Pair | Stoploss | Win Rate | Risk Reward Ratio | Expectancy | |----------|:-------------:|-------------:|------------------:|-----------:| @@ -66,7 +66,7 @@ If enabled in config, Edge will go through historical data with a range of stopl | XZC/ETH | -0.01 | 0.50 |1.176384 | 0.088 | | XZC/ETH | -0.02 | 0.51 |1.115941 | 0.079 | -The goal here is to find the best stoploss for the strategy in order to have the maximum expectancy. in the above example stoploss at 3% leads to the maximum expectancy according to historical data. +The goal here is to find the best stoploss for the strategy in order to have the maximum expectancy. In the above example stoploss at 3% leads to the maximum expectancy according to historical data. Edge then forces stoploss to your strategy dynamically. @@ -93,18 +93,19 @@ Edge has following configurations: If true, then Edge will run periodically #### process_throttle_secs -How often should Edge run ? (in seconds) +How often should Edge run in seconds? (default to 3600 so one hour) #### calculate_since_number_of_days Number of days of data agaist which Edge calculates Win Rate, Risk Reward and Expectancy -Note that it downloads historical data so increasing this number would lead to slowing down the bot +Note that it downloads historical data so increasing this number would lead to slowing down the bot
+(default to 7) #### total_capital_in_stake_currency -This your total capital at risk in your stake currency. if edge is enabled then stake_amount is ignored in favor of this parameter +This your total capital at risk in your stake currency. If edge is enabled then stake_amount is ignored in favor of this parameter #### allowed_risk Percentage of allowed risk per trade
-default to 1% +(default to 1%) #### stoploss_range_min Minimum stoploss (default to -0.01) @@ -126,14 +127,14 @@ It filters paris which have an expectancy lower than this number (default to 0.2 Having an expectancy of 0.20 means if you put 10$ on a trade you expect a 12$ return. #### min_trade_number -When calulating W and R and E (expectancy) against histoical data, you always want to have a minimum number of trades. the more this number is the more Edge is reliable. having a win rate of 100% on a single trade doesn't mean anything at all. but having a win rate of 70% over past 100 trades means clearly something.
+When calulating W and R and E (expectancy) against histoical data, you always want to have a minimum number of trades. The more this number is the more Edge is reliable. Having a win rate of 100% on a single trade doesn't mean anything at all. But having a win rate of 70% over past 100 trades means clearly something.
Default to 10 (it is highly recommanded not to decrease this number) #### max_trade_duration_minute -Edge will filter out trades with long duration. if a trade is profitable after 1 month, it is hard to evaluate the stratgy based on it. but if most of trades are profitable and they have maximum duration of 30 minutes, then it is clearly a good sign.
+Edge will filter out trades with long duration. If a trade is profitable after 1 month, it is hard to evaluate the stratgy based on it. But if most of trades are profitable and they have maximum duration of 30 minutes, then it is clearly a good sign.
Default to 1 day (1440 = 60 * 24) #### remove_pumps -Edge will remove sudden pumps in a given market while going through historical data. however, given that pumps happen very often in crypto markets, we recommand you keep this off.
+Edge will remove sudden pumps in a given market while going through historical data. However, given that pumps happen very often in crypto markets, we recommand you keep this off.
Default to false \ No newline at end of file From 469db0d434e86965d2d85e7d37097740de3c4e24 Mon Sep 17 00:00:00 2001 From: Stephen Dade Date: Thu, 22 Mar 2018 19:27:13 +1100 Subject: [PATCH 265/699] Decoupled custom hyperopts from hyperopt.py --- freqtrade/arguments.py | 8 + freqtrade/configuration.py | 3 + freqtrade/constants.py | 1 + freqtrade/optimize/custom_hyperopt.py | 152 +++++++++++ freqtrade/optimize/default_hyperopt.py | 314 ++++++++++++++++++++++ freqtrade/optimize/hyperopt.py | 138 +--------- freqtrade/optimize/interface.py | 59 ++++ freqtrade/tests/optimize/test_hyperopt.py | 10 +- user_data/hyperopts/__init__.py | 0 user_data/hyperopts/test_hyperopt.py | 283 +++++++++++++++++++ 10 files changed, 839 insertions(+), 129 deletions(-) create mode 100644 freqtrade/optimize/custom_hyperopt.py create mode 100644 freqtrade/optimize/default_hyperopt.py create mode 100644 freqtrade/optimize/interface.py create mode 100644 user_data/hyperopts/__init__.py create mode 100644 user_data/hyperopts/test_hyperopt.py diff --git a/freqtrade/arguments.py b/freqtrade/arguments.py index bb571b4ea..a36b3e66a 100644 --- a/freqtrade/arguments.py +++ b/freqtrade/arguments.py @@ -104,6 +104,14 @@ class Arguments(object): type=str, metavar='PATH', ) + self.parser.add_argument( + '--hyperopt', + help='specify hyperopt file (default: %(default)s)', + dest='hyperopt', + default=Constants.DEFAULT_HYPEROPT, + type=str, + metavar='PATH', + ) self.parser.add_argument( '--dynamic-whitelist', help='dynamically generate and update whitelist' diff --git a/freqtrade/configuration.py b/freqtrade/configuration.py index e043525a7..7b43ff2f8 100644 --- a/freqtrade/configuration.py +++ b/freqtrade/configuration.py @@ -52,6 +52,9 @@ class Configuration(object): if self.args.strategy_path: config.update({'strategy_path': self.args.strategy_path}) + # Add the hyperopt file to use + config.update({'hyperopt': self.args.hyperopt}) + # Load Common configuration config = self._load_common_config(config) diff --git a/freqtrade/constants.py b/freqtrade/constants.py index 2b09aa6c9..a03efdc4b 100644 --- a/freqtrade/constants.py +++ b/freqtrade/constants.py @@ -9,6 +9,7 @@ TICKER_INTERVAL = 5 # min HYPEROPT_EPOCH = 100 # epochs RETRY_TIMEOUT = 30 # sec DEFAULT_STRATEGY = 'DefaultStrategy' +DEFAULT_HYPEROPT = 'default_hyperopt' DEFAULT_DB_PROD_URL = 'sqlite:///tradesv3.sqlite' DEFAULT_DB_DRYRUN_URL = 'sqlite://' UNLIMITED_STAKE_AMOUNT = 'unlimited' diff --git a/freqtrade/optimize/custom_hyperopt.py b/freqtrade/optimize/custom_hyperopt.py new file mode 100644 index 000000000..688d58df8 --- /dev/null +++ b/freqtrade/optimize/custom_hyperopt.py @@ -0,0 +1,152 @@ +# pragma pylint: disable=attribute-defined-outside-init + +""" +This module load custom hyperopts +""" +import importlib +import os +import sys +from typing import Dict, Any, Callable + +from pandas import DataFrame + +from freqtrade.constants import Constants +from freqtrade.logger import Logger +from freqtrade.optimize.interface import IHyperOpt + +sys.path.insert(0, r'../../user_data/hyperopts') + + +class CustomHyperOpt(object): + """ + This class contains all the logic to load custom hyperopt class + """ + def __init__(self, config: dict = {}) -> None: + """ + Load the custom class from config parameter + :param config: + :return: + """ + self.logger = Logger(name=__name__).get_logger() + + # Verify the hyperopt is in the configuration, otherwise fallback to the default hyperopt + if 'hyperopt' in config: + hyperopt = config['hyperopt'] + else: + hyperopt = Constants.DEFAULT_HYPEROPT + + # Load the hyperopt + self._load_hyperopt(hyperopt) + + def _load_hyperopt(self, hyperopt_name: str) -> None: + """ + Search and load the custom hyperopt. If no hyperopt found, fallback on the default hyperopt + Set the object into self.custom_hyperopt + :param hyperopt_name: name of the module to import + :return: None + """ + + try: + # Start by sanitizing the file name (remove any extensions) + hyperopt_name = self._sanitize_module_name(filename=hyperopt_name) + + # Search where can be the hyperopt file + path = self._search_hyperopt(filename=hyperopt_name) + + # Load the hyperopt + self.custom_hyperopt = self._load_class(path + hyperopt_name) + + # Fallback to the default hyperopt + except (ImportError, TypeError) as error: + self.logger.error( + "Impossible to load Hyperopt 'user_data/hyperopts/%s.py'. This file does not exist" + " or contains Python code errors", + hyperopt_name + ) + self.logger.error( + "The error is:\n%s.", + error + ) + + def _load_class(self, filename: str) -> IHyperOpt: + """ + Import a hyperopt as a module + :param filename: path to the hyperopt (path from freqtrade/optimize/) + :return: return the hyperopt class + """ + module = importlib.import_module(filename, __package__) + custom_hyperopt = getattr(module, module.class_name) + + self.logger.info("Load hyperopt class: %s (%s.py)", module.class_name, filename) + return custom_hyperopt() + + @staticmethod + def _sanitize_module_name(filename: str) -> str: + """ + Remove any extension from filename + :param filename: filename to sanatize + :return: return the filename without extensions + """ + filename = os.path.basename(filename) + filename = os.path.splitext(filename)[0] + return filename + + @staticmethod + def _search_hyperopt(filename: str) -> str: + """ + Search for the hyperopt file in different folder + 1. search into the user_data/hyperopts folder + 2. search into the freqtrade/optimize folder + 3. if nothing found, return None + :param hyperopt_name: module name to search + :return: module path where is the hyperopt + """ + pwd = os.path.dirname(os.path.realpath(__file__)) + '/' + user_data = os.path.join(pwd, '..', '..', 'user_data', 'hyperopts', filename + '.py') + hyperopt_folder = os.path.join(pwd, filename + '.py') + + path = None + if os.path.isfile(user_data): + path = 'user_data.hyperopts.' + elif os.path.isfile(hyperopt_folder): + path = '.' + + return path + + def populate_indicators(self, dataframe: DataFrame) -> DataFrame: + """ + Populate indicators that will be used in the Buy and Sell hyperopt + :param dataframe: Raw data from the exchange and parsed by parse_ticker_dataframe() + :return: a Dataframe with all mandatory indicators for the strategies + """ + return self.custom_hyperopt.populate_indicators(dataframe) + + def buy_strategy_generator(self, params: Dict[str, Any]) -> Callable: + """ + Create a buy strategy generator + """ + return self.custom_hyperopt.buy_strategy_generator(params) + + def indicator_space(self) -> Dict[str, Any]: + """ + Create an indicator space + """ + return self.custom_hyperopt.indicator_space() + + def generate_roi_table(self, params: Dict) -> Dict[int, float]: + """ + Create an roi table + """ + return self.custom_hyperopt.generate_roi_table(params) + + def stoploss_space(self) -> Dict[str, Any]: + """ + Create a stoploss space + """ + return self.custom_hyperopt.stoploss_space() + + def roi_space(self) -> Dict[str, Any]: + """ + Create a roi space + """ + return self.custom_hyperopt.roi_space() diff --git a/freqtrade/optimize/default_hyperopt.py b/freqtrade/optimize/default_hyperopt.py new file mode 100644 index 000000000..6ef95bb49 --- /dev/null +++ b/freqtrade/optimize/default_hyperopt.py @@ -0,0 +1,314 @@ +# pragma pylint: disable=missing-docstring, invalid-name, pointless-string-statement + +import talib.abstract as ta +from pandas import DataFrame +from typing import Dict, Any, Callable +from functools import reduce + +import numpy +from hyperopt import hp + +import freqtrade.vendor.qtpylib.indicators as qtpylib +from freqtrade.optimize.interface import IHyperOpt + +class_name = 'DefaultHyperOpts' + + +class DefaultHyperOpts(IHyperOpt): + """ + Default hyperopt provided by freqtrade bot. + You can override it with your own hyperopt + """ + + @staticmethod + def populate_indicators(dataframe: DataFrame) -> DataFrame: + """ + Adds several different TA indicators to the given DataFrame + """ + dataframe['adx'] = ta.ADX(dataframe) + dataframe['ao'] = qtpylib.awesome_oscillator(dataframe) + dataframe['cci'] = ta.CCI(dataframe) + macd = ta.MACD(dataframe) + dataframe['macd'] = macd['macd'] + dataframe['macdsignal'] = macd['macdsignal'] + dataframe['macdhist'] = macd['macdhist'] + dataframe['mfi'] = ta.MFI(dataframe) + dataframe['minus_dm'] = ta.MINUS_DM(dataframe) + dataframe['minus_di'] = ta.MINUS_DI(dataframe) + dataframe['plus_dm'] = ta.PLUS_DM(dataframe) + dataframe['plus_di'] = ta.PLUS_DI(dataframe) + dataframe['roc'] = ta.ROC(dataframe) + dataframe['rsi'] = ta.RSI(dataframe) + # Inverse Fisher transform on RSI, values [-1.0, 1.0] (https://goo.gl/2JGGoy) + rsi = 0.1 * (dataframe['rsi'] - 50) + dataframe['fisher_rsi'] = (numpy.exp(2 * rsi) - 1) / (numpy.exp(2 * rsi) + 1) + # Inverse Fisher transform on RSI normalized, value [0.0, 100.0] (https://goo.gl/2JGGoy) + dataframe['fisher_rsi_norma'] = 50 * (dataframe['fisher_rsi'] + 1) + # Stoch + stoch = ta.STOCH(dataframe) + dataframe['slowd'] = stoch['slowd'] + dataframe['slowk'] = stoch['slowk'] + # Stoch fast + stoch_fast = ta.STOCHF(dataframe) + dataframe['fastd'] = stoch_fast['fastd'] + dataframe['fastk'] = stoch_fast['fastk'] + # Stoch RSI + stoch_rsi = ta.STOCHRSI(dataframe) + dataframe['fastd_rsi'] = stoch_rsi['fastd'] + dataframe['fastk_rsi'] = stoch_rsi['fastk'] + # Bollinger bands + bollinger = qtpylib.bollinger_bands(qtpylib.typical_price(dataframe), window=20, stds=2) + dataframe['bb_lowerband'] = bollinger['lower'] + dataframe['bb_middleband'] = bollinger['mid'] + dataframe['bb_upperband'] = bollinger['upper'] + # EMA - Exponential Moving Average + dataframe['ema3'] = ta.EMA(dataframe, timeperiod=3) + dataframe['ema5'] = ta.EMA(dataframe, timeperiod=5) + dataframe['ema10'] = ta.EMA(dataframe, timeperiod=10) + dataframe['ema50'] = ta.EMA(dataframe, timeperiod=50) + dataframe['ema100'] = ta.EMA(dataframe, timeperiod=100) + # SAR Parabolic + dataframe['sar'] = ta.SAR(dataframe) + # SMA - Simple Moving Average + dataframe['sma'] = ta.SMA(dataframe, timeperiod=40) + # TEMA - Triple Exponential Moving Average + dataframe['tema'] = ta.TEMA(dataframe, timeperiod=9) + # Hilbert Transform Indicator - SineWave + hilbert = ta.HT_SINE(dataframe) + dataframe['htsine'] = hilbert['sine'] + dataframe['htleadsine'] = hilbert['leadsine'] + + # Pattern Recognition - Bullish candlestick patterns + # ------------------------------------ + """ + # Hammer: values [0, 100] + dataframe['CDLHAMMER'] = ta.CDLHAMMER(dataframe) + # Inverted Hammer: values [0, 100] + dataframe['CDLINVERTEDHAMMER'] = ta.CDLINVERTEDHAMMER(dataframe) + # Dragonfly Doji: values [0, 100] + dataframe['CDLDRAGONFLYDOJI'] = ta.CDLDRAGONFLYDOJI(dataframe) + # Piercing Line: values [0, 100] + dataframe['CDLPIERCING'] = ta.CDLPIERCING(dataframe) # values [0, 100] + # Morningstar: values [0, 100] + dataframe['CDLMORNINGSTAR'] = ta.CDLMORNINGSTAR(dataframe) # values [0, 100] + # Three White Soldiers: values [0, 100] + dataframe['CDL3WHITESOLDIERS'] = ta.CDL3WHITESOLDIERS(dataframe) # values [0, 100] + """ + + # Pattern Recognition - Bearish candlestick patterns + # ------------------------------------ + """ + # Hanging Man: values [0, 100] + dataframe['CDLHANGINGMAN'] = ta.CDLHANGINGMAN(dataframe) + # Shooting Star: values [0, 100] + dataframe['CDLSHOOTINGSTAR'] = ta.CDLSHOOTINGSTAR(dataframe) + # Gravestone Doji: values [0, 100] + dataframe['CDLGRAVESTONEDOJI'] = ta.CDLGRAVESTONEDOJI(dataframe) + # Dark Cloud Cover: values [0, 100] + dataframe['CDLDARKCLOUDCOVER'] = ta.CDLDARKCLOUDCOVER(dataframe) + # Evening Doji Star: values [0, 100] + dataframe['CDLEVENINGDOJISTAR'] = ta.CDLEVENINGDOJISTAR(dataframe) + # Evening Star: values [0, 100] + dataframe['CDLEVENINGSTAR'] = ta.CDLEVENINGSTAR(dataframe) + """ + + # Pattern Recognition - Bullish/Bearish candlestick patterns + # ------------------------------------ + """ + # Three Line Strike: values [0, -100, 100] + dataframe['CDL3LINESTRIKE'] = ta.CDL3LINESTRIKE(dataframe) + # Spinning Top: values [0, -100, 100] + dataframe['CDLSPINNINGTOP'] = ta.CDLSPINNINGTOP(dataframe) # values [0, -100, 100] + # Engulfing: values [0, -100, 100] + dataframe['CDLENGULFING'] = ta.CDLENGULFING(dataframe) # values [0, -100, 100] + # Harami: values [0, -100, 100] + dataframe['CDLHARAMI'] = ta.CDLHARAMI(dataframe) # values [0, -100, 100] + # Three Outside Up/Down: values [0, -100, 100] + dataframe['CDL3OUTSIDE'] = ta.CDL3OUTSIDE(dataframe) # values [0, -100, 100] + # Three Inside Up/Down: values [0, -100, 100] + dataframe['CDL3INSIDE'] = ta.CDL3INSIDE(dataframe) # values [0, -100, 100] + """ + + # Chart type + # ------------------------------------ + # Heikinashi stategy + heikinashi = qtpylib.heikinashi(dataframe) + dataframe['ha_open'] = heikinashi['open'] + dataframe['ha_close'] = heikinashi['close'] + dataframe['ha_high'] = heikinashi['high'] + dataframe['ha_low'] = heikinashi['low'] + + return dataframe + + @staticmethod + def buy_strategy_generator(params: Dict[str, Any]) -> Callable: + """ + Define the buy strategy parameters to be used by hyperopt + """ + def populate_buy_trend(dataframe: DataFrame) -> DataFrame: + """ + Buy strategy Hyperopt will build and use + """ + conditions = [] + # GUARDS AND TRENDS + if 'uptrend_long_ema' in params and params['uptrend_long_ema']['enabled']: + conditions.append(dataframe['ema50'] > dataframe['ema100']) + if 'macd_below_zero' in params and params['macd_below_zero']['enabled']: + conditions.append(dataframe['macd'] < 0) + if 'uptrend_short_ema' in params and params['uptrend_short_ema']['enabled']: + conditions.append(dataframe['ema5'] > dataframe['ema10']) + if 'mfi' in params and params['mfi']['enabled']: + conditions.append(dataframe['mfi'] < params['mfi']['value']) + if 'fastd' in params and params['fastd']['enabled']: + conditions.append(dataframe['fastd'] < params['fastd']['value']) + if 'adx' in params and params['adx']['enabled']: + conditions.append(dataframe['adx'] > params['adx']['value']) + if 'rsi' in params and params['rsi']['enabled']: + conditions.append(dataframe['rsi'] < params['rsi']['value']) + if 'over_sar' in params and params['over_sar']['enabled']: + conditions.append(dataframe['close'] > dataframe['sar']) + if 'green_candle' in params and params['green_candle']['enabled']: + conditions.append(dataframe['close'] > dataframe['open']) + if 'uptrend_sma' in params and params['uptrend_sma']['enabled']: + prevsma = dataframe['sma'].shift(1) + conditions.append(dataframe['sma'] > prevsma) + + # TRIGGERS + triggers = { + 'lower_bb': ( + dataframe['close'] < dataframe['bb_lowerband'] + ), + 'lower_bb_tema': ( + dataframe['tema'] < dataframe['bb_lowerband'] + ), + 'faststoch10': (qtpylib.crossed_above( + dataframe['fastd'], 10.0 + )), + 'ao_cross_zero': (qtpylib.crossed_above( + dataframe['ao'], 0.0 + )), + 'ema3_cross_ema10': (qtpylib.crossed_above( + dataframe['ema3'], dataframe['ema10'] + )), + 'macd_cross_signal': (qtpylib.crossed_above( + dataframe['macd'], dataframe['macdsignal'] + )), + 'sar_reversal': (qtpylib.crossed_above( + dataframe['close'], dataframe['sar'] + )), + 'ht_sine': (qtpylib.crossed_above( + dataframe['htleadsine'], dataframe['htsine'] + )), + 'heiken_reversal_bull': ( + (qtpylib.crossed_above(dataframe['ha_close'], dataframe['ha_open'])) & + (dataframe['ha_low'] == dataframe['ha_open']) + ), + 'di_cross': (qtpylib.crossed_above( + dataframe['plus_di'], dataframe['minus_di'] + )), + } + conditions.append(triggers.get(params['trigger']['type'])) + + dataframe.loc[ + reduce(lambda x, y: x & y, conditions), + 'buy'] = 1 + + return dataframe + + return populate_buy_trend + + @staticmethod + def indicator_space() -> Dict[str, Any]: + """ + Define your Hyperopt space for searching strategy parameters + """ + return { + 'macd_below_zero': hp.choice('macd_below_zero', [ + {'enabled': False}, + {'enabled': True} + ]), + 'mfi': hp.choice('mfi', [ + {'enabled': False}, + {'enabled': True, 'value': hp.quniform('mfi-value', 10, 25, 5)} + ]), + 'fastd': hp.choice('fastd', [ + {'enabled': False}, + {'enabled': True, 'value': hp.quniform('fastd-value', 15, 45, 5)} + ]), + 'adx': hp.choice('adx', [ + {'enabled': False}, + {'enabled': True, 'value': hp.quniform('adx-value', 20, 50, 5)} + ]), + 'rsi': hp.choice('rsi', [ + {'enabled': False}, + {'enabled': True, 'value': hp.quniform('rsi-value', 20, 40, 5)} + ]), + 'uptrend_long_ema': hp.choice('uptrend_long_ema', [ + {'enabled': False}, + {'enabled': True} + ]), + 'uptrend_short_ema': hp.choice('uptrend_short_ema', [ + {'enabled': False}, + {'enabled': True} + ]), + 'over_sar': hp.choice('over_sar', [ + {'enabled': False}, + {'enabled': True} + ]), + 'green_candle': hp.choice('green_candle', [ + {'enabled': False}, + {'enabled': True} + ]), + 'uptrend_sma': hp.choice('uptrend_sma', [ + {'enabled': False}, + {'enabled': True} + ]), + 'trigger': hp.choice('trigger', [ + {'type': 'lower_bb'}, + {'type': 'lower_bb_tema'}, + {'type': 'faststoch10'}, + {'type': 'ao_cross_zero'}, + {'type': 'ema3_cross_ema10'}, + {'type': 'macd_cross_signal'}, + {'type': 'sar_reversal'}, + {'type': 'ht_sine'}, + {'type': 'heiken_reversal_bull'}, + {'type': 'di_cross'}, + ]), + } + + @staticmethod + def generate_roi_table(params: Dict) -> Dict[int, float]: + """ + Generate the ROI table that will be used by Hyperopt + """ + roi_table = {} + roi_table[0] = params['roi_p1'] + params['roi_p2'] + params['roi_p3'] + roi_table[params['roi_t3']] = params['roi_p1'] + params['roi_p2'] + roi_table[params['roi_t3'] + params['roi_t2']] = params['roi_p1'] + roi_table[params['roi_t3'] + params['roi_t2'] + params['roi_t1']] = 0 + + return roi_table + + @staticmethod + def stoploss_space() -> Dict[str, Any]: + """ + Stoploss Value to search + """ + return { + 'stoploss': hp.quniform('stoploss', -0.5, -0.02, 0.02), + } + + @staticmethod + def roi_space() -> Dict[str, Any]: + """ + Values to search for each ROI steps + """ + return { + 'roi_t1': hp.quniform('roi_t1', 10, 120, 20), + 'roi_t2': hp.quniform('roi_t2', 10, 60, 15), + 'roi_t3': hp.quniform('roi_t3', 10, 40, 10), + 'roi_p1': hp.quniform('roi_p1', 0.01, 0.04, 0.01), + 'roi_p2': hp.quniform('roi_p2', 0.01, 0.07, 0.01), + 'roi_p3': hp.quniform('roi_p3', 0.01, 0.20, 0.01), + } diff --git a/freqtrade/optimize/hyperopt.py b/freqtrade/optimize/hyperopt.py index b2d05d603..5e23458d7 100644 --- a/freqtrade/optimize/hyperopt.py +++ b/freqtrade/optimize/hyperopt.py @@ -9,22 +9,20 @@ import multiprocessing import os import sys from argparse import Namespace -from functools import reduce from math import exp from operator import itemgetter -from typing import Any, Callable, Dict, List +from typing import Any, Dict, List -import talib.abstract as ta from pandas import DataFrame from sklearn.externals.joblib import Parallel, delayed, dump, load from skopt import Optimizer -from skopt.space import Categorical, Dimension, Integer, Real +from skopt.space import Dimension -import freqtrade.vendor.qtpylib.indicators as qtpylib from freqtrade.arguments import Arguments from freqtrade.configuration import Configuration from freqtrade.optimize import load_data from freqtrade.optimize.backtesting import Backtesting +from freqtrade.optimize.custom_hyperopt import CustomHyperOpt logger = logging.getLogger(__name__) @@ -42,6 +40,9 @@ class Hyperopt(Backtesting): """ def __init__(self, config: Dict[str, Any]) -> None: super().__init__(config) + + self.custom_hyperopt = CustomHyperOpt(self.config) + # set TARGET_TRADES to suit your number concurrent trades so its realistic # to the number of days self.target_trades = 600 @@ -74,24 +75,6 @@ class Hyperopt(Backtesting): arg_dict = {dim.name: value for dim, value in zip(dimensions, params)} return arg_dict - @staticmethod - def populate_indicators(dataframe: DataFrame, metadata: dict) -> DataFrame: - dataframe['adx'] = ta.ADX(dataframe) - macd = ta.MACD(dataframe) - dataframe['macd'] = macd['macd'] - dataframe['macdsignal'] = macd['macdsignal'] - dataframe['mfi'] = ta.MFI(dataframe) - dataframe['rsi'] = ta.RSI(dataframe) - stoch_fast = ta.STOCHF(dataframe) - dataframe['fastd'] = stoch_fast['fastd'] - dataframe['minus_di'] = ta.MINUS_DI(dataframe) - # Bollinger bands - bollinger = qtpylib.bollinger_bands(qtpylib.typical_price(dataframe), window=20, stds=2) - dataframe['bb_lowerband'] = bollinger['lower'] - dataframe['sar'] = ta.SAR(dataframe) - - return dataframe - def save_trials(self) -> None: """ Save hyperopt trials to file @@ -149,59 +132,6 @@ class Hyperopt(Backtesting): result = trade_loss + profit_loss + duration_loss return result - @staticmethod - def generate_roi_table(params: Dict) -> Dict[int, float]: - """ - Generate the ROI table that will be used by Hyperopt - """ - roi_table = {} - roi_table[0] = params['roi_p1'] + params['roi_p2'] + params['roi_p3'] - roi_table[params['roi_t3']] = params['roi_p1'] + params['roi_p2'] - roi_table[params['roi_t3'] + params['roi_t2']] = params['roi_p1'] - roi_table[params['roi_t3'] + params['roi_t2'] + params['roi_t1']] = 0 - - return roi_table - - @staticmethod - def roi_space() -> List[Dimension]: - """ - Values to search for each ROI steps - """ - return [ - Integer(10, 120, name='roi_t1'), - Integer(10, 60, name='roi_t2'), - Integer(10, 40, name='roi_t3'), - Real(0.01, 0.04, name='roi_p1'), - Real(0.01, 0.07, name='roi_p2'), - Real(0.01, 0.20, name='roi_p3'), - ] - - @staticmethod - def stoploss_space() -> List[Dimension]: - """ - Stoploss search space - """ - return [ - Real(-0.5, -0.02, name='stoploss'), - ] - - @staticmethod - def indicator_space() -> List[Dimension]: - """ - Define your Hyperopt space for searching strategy parameters - """ - return [ - Integer(10, 25, name='mfi-value'), - Integer(15, 45, name='fastd-value'), - Integer(20, 50, name='adx-value'), - Integer(20, 40, name='rsi-value'), - Categorical([True, False], name='mfi-enabled'), - Categorical([True, False], name='fastd-enabled'), - Categorical([True, False], name='adx-enabled'), - Categorical([True, False], name='rsi-enabled'), - Categorical(['bb_lower', 'macd_cross_signal', 'sar_reversal'], name='trigger') - ] - def has_space(self, space: str) -> bool: """ Tell if a space value is contained in the configuration @@ -216,61 +146,19 @@ class Hyperopt(Backtesting): """ spaces: List[Dimension] = [] if self.has_space('buy'): - spaces += Hyperopt.indicator_space() + spaces = {**spaces, **self.custom_hyperopt.indicator_space()} if self.has_space('roi'): - spaces += Hyperopt.roi_space() + spaces = {**spaces, **self.custom_hyperopt.roi_space()} if self.has_space('stoploss'): - spaces += Hyperopt.stoploss_space() + spaces = {**spaces, **self.custom_hyperopt.stoploss_space()} return spaces - @staticmethod - def buy_strategy_generator(params: Dict[str, Any]) -> Callable: - """ - Define the buy strategy parameters to be used by hyperopt - """ - def populate_buy_trend(dataframe: DataFrame, metadata: dict) -> DataFrame: - """ - Buy strategy Hyperopt will build and use - """ - conditions = [] - # GUARDS AND TRENDS - if 'mfi-enabled' in params and params['mfi-enabled']: - conditions.append(dataframe['mfi'] < params['mfi-value']) - if 'fastd-enabled' in params and params['fastd-enabled']: - conditions.append(dataframe['fastd'] < params['fastd-value']) - if 'adx-enabled' in params and params['adx-enabled']: - conditions.append(dataframe['adx'] > params['adx-value']) - if 'rsi-enabled' in params and params['rsi-enabled']: - conditions.append(dataframe['rsi'] < params['rsi-value']) - - # TRIGGERS - if params['trigger'] == 'bb_lower': - conditions.append(dataframe['close'] < dataframe['bb_lowerband']) - if params['trigger'] == 'macd_cross_signal': - conditions.append(qtpylib.crossed_above( - dataframe['macd'], dataframe['macdsignal'] - )) - if params['trigger'] == 'sar_reversal': - conditions.append(qtpylib.crossed_above( - dataframe['close'], dataframe['sar'] - )) - - dataframe.loc[ - reduce(lambda x, y: x & y, conditions), - 'buy'] = 1 - - return dataframe - - return populate_buy_trend - - def generate_optimizer(self, _params) -> Dict: - params = self.get_args(_params) - + def generate_optimizer(self, params: Dict) -> Dict: if self.has_space('roi'): - self.strategy.minimal_roi = self.generate_roi_table(params) + self.analyze.strategy.minimal_roi = self.custom_hyperopt.generate_roi_table(params) if self.has_space('buy'): - self.advise_buy = self.buy_strategy_generator(params) + self.populate_buy_trend = self.custom_hyperopt.buy_strategy_generator(params) if self.has_space('stoploss'): self.strategy.stoploss = params['stoploss'] @@ -351,7 +239,7 @@ class Hyperopt(Backtesting): ) if self.has_space('buy'): - self.strategy.advise_indicators = Hyperopt.populate_indicators # type: ignore + self.strategy.advise_indicators = self.custom_hyperopt.populate_indicators # type: ignore dump(self.strategy.tickerdata_to_dataframe(data), TICKERDATA_PICKLE) self.exchange = None # type: ignore self.load_previous_results() diff --git a/freqtrade/optimize/interface.py b/freqtrade/optimize/interface.py new file mode 100644 index 000000000..8bc3866b2 --- /dev/null +++ b/freqtrade/optimize/interface.py @@ -0,0 +1,59 @@ +""" +IHyperOpt interface +This module defines the interface to apply for hyperopts +""" + +from abc import ABC, abstractmethod +from typing import Dict, Any, Callable + +from pandas import DataFrame + + +class IHyperOpt(ABC): + """ + Interface for freqtrade hyperopts + Defines the mandatory structure must follow any custom strategies + + Attributes you can use: + minimal_roi -> Dict: Minimal ROI designed for the strategy + stoploss -> float: optimal stoploss designed for the strategy + ticker_interval -> int: value of the ticker interval to use for the strategy + """ + + @abstractmethod + def populate_indicators(self, dataframe: DataFrame) -> DataFrame: + """ + Populate indicators that will be used in the Buy and Sell strategy + :param dataframe: Raw data from the exchange and parsed by parse_ticker_dataframe() + :return: a Dataframe with all mandatory indicators for the strategies + """ + + @abstractmethod + def buy_strategy_generator(self, params: Dict[str, Any]) -> Callable: + """ + Create a buy strategy generator + """ + + @abstractmethod + def indicator_space(self) -> Dict[str, Any]: + """ + Create an indicator space + """ + + @abstractmethod + def generate_roi_table(self, params: Dict) -> Dict[int, float]: + """ + Create an roi table + """ + + @abstractmethod + def stoploss_space(self) -> Dict[str, Any]: + """ + Create a stoploss space + """ + + @abstractmethod + def roi_space(self) -> Dict[str, Any]: + """ + Create a roi space + """ diff --git a/freqtrade/tests/optimize/test_hyperopt.py b/freqtrade/tests/optimize/test_hyperopt.py index c93f2d316..85d140b6d 100644 --- a/freqtrade/tests/optimize/test_hyperopt.py +++ b/freqtrade/tests/optimize/test_hyperopt.py @@ -175,7 +175,7 @@ def test_roi_table_generation(hyperopt) -> None: 'roi_p3': 3, } - assert hyperopt.generate_roi_table(params) == {0: 6, 15: 3, 25: 1, 30: 0} + assert hyperopt.custom_hyperopt.generate_roi_table(params) == {0: 6, 15: 3, 25: 1, 30: 0} def test_start_calls_optimizer(mocker, default_conf, caplog) -> None: @@ -243,7 +243,8 @@ def test_populate_indicators(hyperopt) -> None: tick = load_tickerdata_file(None, 'UNITTEST/BTC', '1m') tickerlist = {'UNITTEST/BTC': tick} dataframes = hyperopt.strategy.tickerdata_to_dataframe(tickerlist) - dataframe = hyperopt.populate_indicators(dataframes['UNITTEST/BTC'], {'pair': 'UNITTEST/BTC'}) + dataframe = hyperopt.custom_hyperopt.populate_indicators(dataframes['UNITTEST/BTC'], + {'pair': 'UNITTEST/BTC'}) # Check if some indicators are generated. We will not test all of them assert 'adx' in dataframe @@ -255,9 +256,10 @@ def test_buy_strategy_generator(hyperopt) -> None: tick = load_tickerdata_file(None, 'UNITTEST/BTC', '1m') tickerlist = {'UNITTEST/BTC': tick} dataframes = hyperopt.strategy.tickerdata_to_dataframe(tickerlist) - dataframe = hyperopt.populate_indicators(dataframes['UNITTEST/BTC'], {'pair': 'UNITTEST/BTC'}) + dataframe = hyperopt.custom_hyperopt.populate_indicators(dataframes['UNITTEST/BTC'], + {'pair': 'UNITTEST/BTC'}) - populate_buy_trend = hyperopt.buy_strategy_generator( + populate_buy_trend = hyperopt.custom_hyperopt.buy_strategy_generator( { 'adx-value': 20, 'fastd-value': 20, diff --git a/user_data/hyperopts/__init__.py b/user_data/hyperopts/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/user_data/hyperopts/test_hyperopt.py b/user_data/hyperopts/test_hyperopt.py new file mode 100644 index 000000000..0ee43c64f --- /dev/null +++ b/user_data/hyperopts/test_hyperopt.py @@ -0,0 +1,283 @@ +# pragma pylint: disable=missing-docstring, invalid-name, pointless-string-statement + +import talib.abstract as ta +from pandas import DataFrame +from typing import Dict, Any, Callable +from functools import reduce +from math import exp + +import numpy +import talib.abstract as ta +from hyperopt import STATUS_FAIL, STATUS_OK, Trials, fmin, hp, space_eval, tpe + +import freqtrade.vendor.qtpylib.indicators as qtpylib +from freqtrade.indicator_helpers import fishers_inverse +from freqtrade.optimize.interface import IHyperOpt + + +# Update this variable if you change the class name +class_name = 'TestHyperOpt' + + +# This class is a sample. Feel free to customize it. +class TestHyperOpt(IHyperOpt): + """ + This is a test hyperopt to inspire you. + More information in https://github.com/gcarq/freqtrade/blob/develop/docs/hyperopt.md + + You can: + - Rename the class name (Do not forget to update class_name) + - Add any methods you want to build your hyperopt + - Add any lib you need to build your hyperopt + + You must keep: + - the prototype for the methods: populate_indicators, indicator_space, buy_strategy_generator, + roi_space, generate_roi_table, stoploss_space + """ + + @staticmethod + def populate_indicators(dataframe: DataFrame) -> DataFrame: + """ + Adds several different TA indicators to the given DataFrame + + Performance Note: For the best performance be frugal on the number of indicators + you are using. Let uncomment only the indicator you are using in your strategies + or your hyperopt configuration, otherwise you will waste your memory and CPU usage. + """ + + # Momentum Indicator + # ------------------------------------ + + # ADX + dataframe['adx'] = ta.ADX(dataframe) + + """ + # Awesome oscillator + dataframe['ao'] = qtpylib.awesome_oscillator(dataframe) + + # Commodity Channel Index: values Oversold:<-100, Overbought:>100 + dataframe['cci'] = ta.CCI(dataframe) + + # MACD + macd = ta.MACD(dataframe) + dataframe['macd'] = macd['macd'] + dataframe['macdsignal'] = macd['macdsignal'] + dataframe['macdhist'] = macd['macdhist'] + + # MFI + dataframe['mfi'] = ta.MFI(dataframe) + + # Minus Directional Indicator / Movement + dataframe['minus_dm'] = ta.MINUS_DM(dataframe) + dataframe['minus_di'] = ta.MINUS_DI(dataframe) + + # Plus Directional Indicator / Movement + dataframe['plus_dm'] = ta.PLUS_DM(dataframe) + dataframe['plus_di'] = ta.PLUS_DI(dataframe) + dataframe['minus_di'] = ta.MINUS_DI(dataframe) + + # ROC + dataframe['roc'] = ta.ROC(dataframe) + + # RSI + dataframe['rsi'] = ta.RSI(dataframe) + + # Inverse Fisher transform on RSI, values [-1.0, 1.0] (https://goo.gl/2JGGoy) + rsi = 0.1 * (dataframe['rsi'] - 50) + dataframe['fisher_rsi'] = (numpy.exp(2 * rsi) - 1) / (numpy.exp(2 * rsi) + 1) + + # Inverse Fisher transform on RSI normalized, value [0.0, 100.0] (https://goo.gl/2JGGoy) + dataframe['fisher_rsi_norma'] = 50 * (dataframe['fisher_rsi'] + 1) + + # Stoch + stoch = ta.STOCH(dataframe) + dataframe['slowd'] = stoch['slowd'] + dataframe['slowk'] = stoch['slowk'] + + # Stoch fast + stoch_fast = ta.STOCHF(dataframe) + dataframe['fastd'] = stoch_fast['fastd'] + dataframe['fastk'] = stoch_fast['fastk'] + + # Stoch RSI + stoch_rsi = ta.STOCHRSI(dataframe) + dataframe['fastd_rsi'] = stoch_rsi['fastd'] + dataframe['fastk_rsi'] = stoch_rsi['fastk'] + """ + + # Overlap Studies + # ------------------------------------ + + # Bollinger bands + bollinger = qtpylib.bollinger_bands(qtpylib.typical_price(dataframe), window=20, stds=2) + dataframe['bb_lowerband'] = bollinger['lower'] + dataframe['bb_middleband'] = bollinger['mid'] + dataframe['bb_upperband'] = bollinger['upper'] + + """ + # EMA - Exponential Moving Average + dataframe['ema3'] = ta.EMA(dataframe, timeperiod=3) + dataframe['ema5'] = ta.EMA(dataframe, timeperiod=5) + dataframe['ema10'] = ta.EMA(dataframe, timeperiod=10) + dataframe['ema50'] = ta.EMA(dataframe, timeperiod=50) + dataframe['ema100'] = ta.EMA(dataframe, timeperiod=100) + + # SAR Parabol + dataframe['sar'] = ta.SAR(dataframe) + + # SMA - Simple Moving Average + dataframe['sma'] = ta.SMA(dataframe, timeperiod=40) + """ + + # TEMA - Triple Exponential Moving Average + dataframe['tema'] = ta.TEMA(dataframe, timeperiod=9) + + # Cycle Indicator + # ------------------------------------ + # Hilbert Transform Indicator - SineWave + hilbert = ta.HT_SINE(dataframe) + dataframe['htsine'] = hilbert['sine'] + dataframe['htleadsine'] = hilbert['leadsine'] + + # Pattern Recognition - Bullish candlestick patterns + # ------------------------------------ + """ + # Hammer: values [0, 100] + dataframe['CDLHAMMER'] = ta.CDLHAMMER(dataframe) + # Inverted Hammer: values [0, 100] + dataframe['CDLINVERTEDHAMMER'] = ta.CDLINVERTEDHAMMER(dataframe) + # Dragonfly Doji: values [0, 100] + dataframe['CDLDRAGONFLYDOJI'] = ta.CDLDRAGONFLYDOJI(dataframe) + # Piercing Line: values [0, 100] + dataframe['CDLPIERCING'] = ta.CDLPIERCING(dataframe) # values [0, 100] + # Morningstar: values [0, 100] + dataframe['CDLMORNINGSTAR'] = ta.CDLMORNINGSTAR(dataframe) # values [0, 100] + # Three White Soldiers: values [0, 100] + dataframe['CDL3WHITESOLDIERS'] = ta.CDL3WHITESOLDIERS(dataframe) # values [0, 100] + """ + + # Pattern Recognition - Bearish candlestick patterns + # ------------------------------------ + """ + # Hanging Man: values [0, 100] + dataframe['CDLHANGINGMAN'] = ta.CDLHANGINGMAN(dataframe) + # Shooting Star: values [0, 100] + dataframe['CDLSHOOTINGSTAR'] = ta.CDLSHOOTINGSTAR(dataframe) + # Gravestone Doji: values [0, 100] + dataframe['CDLGRAVESTONEDOJI'] = ta.CDLGRAVESTONEDOJI(dataframe) + # Dark Cloud Cover: values [0, 100] + dataframe['CDLDARKCLOUDCOVER'] = ta.CDLDARKCLOUDCOVER(dataframe) + # Evening Doji Star: values [0, 100] + dataframe['CDLEVENINGDOJISTAR'] = ta.CDLEVENINGDOJISTAR(dataframe) + # Evening Star: values [0, 100] + dataframe['CDLEVENINGSTAR'] = ta.CDLEVENINGSTAR(dataframe) + """ + + # Pattern Recognition - Bullish/Bearish candlestick patterns + # ------------------------------------ + """ + # Three Line Strike: values [0, -100, 100] + dataframe['CDL3LINESTRIKE'] = ta.CDL3LINESTRIKE(dataframe) + # Spinning Top: values [0, -100, 100] + dataframe['CDLSPINNINGTOP'] = ta.CDLSPINNINGTOP(dataframe) # values [0, -100, 100] + # Engulfing: values [0, -100, 100] + dataframe['CDLENGULFING'] = ta.CDLENGULFING(dataframe) # values [0, -100, 100] + # Harami: values [0, -100, 100] + dataframe['CDLHARAMI'] = ta.CDLHARAMI(dataframe) # values [0, -100, 100] + # Three Outside Up/Down: values [0, -100, 100] + dataframe['CDL3OUTSIDE'] = ta.CDL3OUTSIDE(dataframe) # values [0, -100, 100] + # Three Inside Up/Down: values [0, -100, 100] + dataframe['CDL3INSIDE'] = ta.CDL3INSIDE(dataframe) # values [0, -100, 100] + """ + + # Chart type + # ------------------------------------ + """ + # Heikinashi stategy + heikinashi = qtpylib.heikinashi(dataframe) + dataframe['ha_open'] = heikinashi['open'] + dataframe['ha_close'] = heikinashi['close'] + dataframe['ha_high'] = heikinashi['high'] + dataframe['ha_low'] = heikinashi['low'] + """ + + return dataframe + + @staticmethod + def indicator_space() -> Dict[str, Any]: + """ + Define your Hyperopt space for searching strategy parameters + """ + return { + 'adx': hp.choice('adx', [ + {'enabled': False}, + {'enabled': True, 'value': hp.quniform('adx-value', 50, 80, 5)} + ]), + 'uptrend_tema': hp.choice('uptrend_tema', [ + {'enabled': False}, + {'enabled': True} + ]), + 'trigger': hp.choice('trigger', [ + {'type': 'middle_bb_tema'}, + ]), + } + + @staticmethod + def buy_strategy_generator(params: Dict[str, Any]) -> Callable: + """ + Define the buy strategy parameters to be used by hyperopt + """ + def populate_buy_trend(dataframe: DataFrame) -> DataFrame: + conditions = [] + # GUARDS AND TRENDS + if 'adx' in params and params['adx']['enabled']: + conditions.append(dataframe['adx'] > params['adx']['value']) + if 'uptrend_tema' in params and params['uptrend_tema']['enabled']: + prevtema = dataframe['tema'].shift(1) + conditions.append(dataframe['tema'] > prevtema) + + # TRIGGERS + triggers = { + 'middle_bb_tema': ( + dataframe['tema'] > dataframe['bb_middleband'] + ), + } + conditions.append(triggers.get(params['trigger']['type'])) + + dataframe.loc[ + reduce(lambda x, y: x & y, conditions), + 'buy'] = 1 + + return dataframe + + return populate_buy_trend + + @staticmethod + def roi_space() -> Dict[str, Any]: + return { + 'roi_t1': hp.quniform('roi_t1', 10, 120, 20), + 'roi_t2': hp.quniform('roi_t2', 10, 60, 15), + 'roi_t3': hp.quniform('roi_t3', 10, 40, 10), + 'roi_p1': hp.quniform('roi_p1', 0.01, 0.04, 0.01), + 'roi_p2': hp.quniform('roi_p2', 0.01, 0.07, 0.01), + 'roi_p3': hp.quniform('roi_p3', 0.01, 0.20, 0.01), + } + + @staticmethod + def generate_roi_table(params: Dict) -> Dict[int, float]: + """ + Generate the ROI table that will be used by Hyperopt + """ + roi_table = {} + roi_table[0] = params['roi_p1'] + params['roi_p2'] + params['roi_p3'] + roi_table[params['roi_t3']] = params['roi_p1'] + params['roi_p2'] + roi_table[params['roi_t3'] + params['roi_t2']] = params['roi_p1'] + roi_table[params['roi_t3'] + params['roi_t2'] + params['roi_t1']] = 0 + + return roi_table + + @staticmethod + def stoploss_space() -> Dict[str, Any]: + return { + 'stoploss': hp.quniform('stoploss', -0.5, -0.02, 0.02), + } \ No newline at end of file From 5816d1c1bdf54031b407574a1ff1198d4dc1562e Mon Sep 17 00:00:00 2001 From: Stephen Dade Date: Thu, 22 Mar 2018 21:07:22 +1100 Subject: [PATCH 266/699] Updated documentation for new hyperopt --- docs/hyperopt.md | 23 ++++++++++++++--------- 1 file changed, 14 insertions(+), 9 deletions(-) diff --git a/docs/hyperopt.md b/docs/hyperopt.md index e2dcf3e95..36e5d2d74 100644 --- a/docs/hyperopt.md +++ b/docs/hyperopt.md @@ -21,16 +21,19 @@ and still take a long time. We recommend you start by taking a look at `hyperopt.py` file located in [freqtrade/optimize](https://github.com/freqtrade/freqtrade/blob/develop/freqtrade/optimize/hyperopt.py) -### Configure your Guards and Triggers +### 1. Configure your Guards and Triggers -There are two places you need to change to add a new buy strategy for testing: -- Inside [populate_buy_trend()](https://github.com/freqtrade/freqtrade/blob/develop/freqtrade/optimize/hyperopt.py#L231-L264). -- Inside [hyperopt_space()](https://github.com/freqtrade/freqtrade/blob/develop/freqtrade/optimize/hyperopt.py#L213-L224) -and the associated methods `indicator_space`, `roi_space`, `stoploss_space`. +There are two places you need to change in your strategy file to add a +new buy strategy for testing: -There you have two different type of indicators: 1. `guards` and 2. `triggers`. -1. Guards are conditions like "never buy if ADX < 10", or "never buy if -current price is over EMA10". +- Inside [populate_buy_trend()](https://github.com/freqtrade/freqtrade/blob/develop/user_data/hyperopts/test_hyperopt.py#L230-L251). +- Inside [indicator_space()](https://github.com/freqtrade/freqtrade/blob/develop/user_data/hyperopts/test_hyperopt.py#L207-L223). + +There you have two different type of indicators: 1. `guards` and 2. +`triggers`. + +1. Guards are conditions like "never buy if ADX < 10", or never buy if +current price is over EMA10. 2. Triggers are ones that actually trigger buy in specific moment, like "buy when EMA5 crosses over EMA10" or "buy when close price touches lower bollinger band". @@ -124,9 +127,11 @@ Because hyperopt tries a lot of combinations to find the best parameters it will We strongly recommend to use `screen` or `tmux` to prevent any connection loss. ```bash -python3 ./freqtrade/main.py -c config.json hyperopt -e 5000 +python3 ./freqtrade/main.py -s --hyperopt -c config.json hyperopt -e 5000 ``` +Use `` and `` as the names of the custom strategy and custom hyperopt used. + The `-e` flag will set how many evaluations hyperopt will do. We recommend running at least several thousand evaluations. From 40368bd1b20655a7c77768e82fb194a9d0155958 Mon Sep 17 00:00:00 2001 From: Stephen Dade Date: Sat, 31 Mar 2018 16:49:06 +1100 Subject: [PATCH 267/699] Added more hyperopt documentation --- docs/bot-usage.md | 2 ++ docs/hyperopt.md | 27 ++++++++++++++++++--------- 2 files changed, 20 insertions(+), 9 deletions(-) diff --git a/docs/bot-usage.md b/docs/bot-usage.md index 83a8ee833..f6c15f26d 100644 --- a/docs/bot-usage.md +++ b/docs/bot-usage.md @@ -204,6 +204,8 @@ optional arguments: number) --timerange TIMERANGE specify what timerange of data to use. + --hyperopt PATH specify hyperopt file (default: + freqtrade/optimize/default_hyperopt.py) -e INT, --epochs INT specify number of epochs (default: 100) -s {all,buy,roi,stoploss} [{all,buy,roi,stoploss} ...], --spaces {all,buy,roi,stoploss} [{all,buy,roi,stoploss} ...] Specify which parameters to hyperopt. Space separate diff --git a/docs/hyperopt.md b/docs/hyperopt.md index 36e5d2d74..0484613c2 100644 --- a/docs/hyperopt.md +++ b/docs/hyperopt.md @@ -19,15 +19,23 @@ and still take a long time. ## Prepare Hyperopting -We recommend you start by taking a look at `hyperopt.py` file located in [freqtrade/optimize](https://github.com/freqtrade/freqtrade/blob/develop/freqtrade/optimize/hyperopt.py) +## Prepare Hyperopt +Before we start digging in Hyperopt, we recommend you to take a look at +an example hyperopt file located into [user_data/strategies/](https://github.com/gcarq/freqtrade/blob/develop/user_data/hyperopts/test_hyperopt.py) + +### 1. Install a Custom Hyperopt File +This is very simple. Put your hyperopt file into the folder +`user_data/hyperopts`. + +Let assume you want a hyperopt file `awesome_hyperopt.py`: +1. Copy the file `user_data/hyperopts/test_hyperopt.py` into `user_data/hyperopts/awesome_hyperopt.py` + -### 1. Configure your Guards and Triggers - -There are two places you need to change in your strategy file to add a -new buy strategy for testing: - -- Inside [populate_buy_trend()](https://github.com/freqtrade/freqtrade/blob/develop/user_data/hyperopts/test_hyperopt.py#L230-L251). -- Inside [indicator_space()](https://github.com/freqtrade/freqtrade/blob/develop/user_data/hyperopts/test_hyperopt.py#L207-L223). +### 2. Configure your Guards and Triggers +There are two places you need to change in your hyperopt file to add a +new buy hyperopt for testing: +- Inside [populate_buy_trend()](https://github.com/gcarq/freqtrade/blob/develop/user_data/hyperopts/test_hyperopt.py#L230-L251). +- Inside [indicator_space()](https://github.com/gcarq/freqtrade/blob/develop/user_data/hyperopts/test_hyperopt.py#L207-L223). There you have two different type of indicators: 1. `guards` and 2. `triggers`. @@ -130,7 +138,8 @@ We strongly recommend to use `screen` or `tmux` to prevent any connection loss. python3 ./freqtrade/main.py -s --hyperopt -c config.json hyperopt -e 5000 ``` -Use `` and `` as the names of the custom strategy and custom hyperopt used. +Use `` and `` as the names of the custom strategy +(only required for generating sells) and the custom hyperopt used. The `-e` flag will set how many evaluations hyperopt will do. We recommend running at least several thousand evaluations. From e0f420983e5b9b628a3af0437bea61efda4e27b0 Mon Sep 17 00:00:00 2001 From: Stephen Dade Date: Sat, 31 Mar 2018 17:21:15 +1100 Subject: [PATCH 268/699] Updated logger in custom_hyperopt --- freqtrade/optimize/custom_hyperopt.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/freqtrade/optimize/custom_hyperopt.py b/freqtrade/optimize/custom_hyperopt.py index 688d58df8..80168c46e 100644 --- a/freqtrade/optimize/custom_hyperopt.py +++ b/freqtrade/optimize/custom_hyperopt.py @@ -4,6 +4,7 @@ This module load custom hyperopts """ import importlib +import logging import os import sys from typing import Dict, Any, Callable @@ -11,11 +12,12 @@ from typing import Dict, Any, Callable from pandas import DataFrame from freqtrade.constants import Constants -from freqtrade.logger import Logger from freqtrade.optimize.interface import IHyperOpt sys.path.insert(0, r'../../user_data/hyperopts') +logger = logging.getLogger(__name__) + class CustomHyperOpt(object): """ @@ -27,7 +29,6 @@ class CustomHyperOpt(object): :param config: :return: """ - self.logger = Logger(name=__name__).get_logger() # Verify the hyperopt is in the configuration, otherwise fallback to the default hyperopt if 'hyperopt' in config: @@ -58,12 +59,12 @@ class CustomHyperOpt(object): # Fallback to the default hyperopt except (ImportError, TypeError) as error: - self.logger.error( + logger.error( "Impossible to load Hyperopt 'user_data/hyperopts/%s.py'. This file does not exist" " or contains Python code errors", hyperopt_name ) - self.logger.error( + logger.error( "The error is:\n%s.", error ) @@ -77,7 +78,7 @@ class CustomHyperOpt(object): module = importlib.import_module(filename, __package__) custom_hyperopt = getattr(module, module.class_name) - self.logger.info("Load hyperopt class: %s (%s.py)", module.class_name, filename) + logger.info("Load hyperopt class: %s (%s.py)", module.class_name, filename) return custom_hyperopt() @staticmethod From 477515c4b5904b81cd9d6a6c99aa93f59daf39a8 Mon Sep 17 00:00:00 2001 From: Stephen Dade Date: Sun, 1 Apr 2018 21:06:50 +1000 Subject: [PATCH 269/699] Now using resolver for custom hyperopts --- freqtrade/arguments.py | 6 +- freqtrade/constants.py | 2 +- freqtrade/optimize/custom_hyperopt.py | 153 -------------------------- freqtrade/optimize/hyperopt.py | 7 +- freqtrade/optimize/resolver.py | 104 +++++++++++++++++ user_data/hyperopts/test_hyperopt.py | 4 - 6 files changed, 112 insertions(+), 164 deletions(-) delete mode 100644 freqtrade/optimize/custom_hyperopt.py create mode 100644 freqtrade/optimize/resolver.py diff --git a/freqtrade/arguments.py b/freqtrade/arguments.py index a36b3e66a..429a7ca5a 100644 --- a/freqtrade/arguments.py +++ b/freqtrade/arguments.py @@ -105,12 +105,12 @@ class Arguments(object): metavar='PATH', ) self.parser.add_argument( - '--hyperopt', - help='specify hyperopt file (default: %(default)s)', + '--customhyperopt', + help='specify hyperopt class name (default: %(default)s)', dest='hyperopt', default=Constants.DEFAULT_HYPEROPT, type=str, - metavar='PATH', + metavar='NAME', ) self.parser.add_argument( '--dynamic-whitelist', diff --git a/freqtrade/constants.py b/freqtrade/constants.py index a03efdc4b..e76c54ba2 100644 --- a/freqtrade/constants.py +++ b/freqtrade/constants.py @@ -9,7 +9,7 @@ TICKER_INTERVAL = 5 # min HYPEROPT_EPOCH = 100 # epochs RETRY_TIMEOUT = 30 # sec DEFAULT_STRATEGY = 'DefaultStrategy' -DEFAULT_HYPEROPT = 'default_hyperopt' +DEFAULT_HYPEROPT = 'DefaultHyperOpts' DEFAULT_DB_PROD_URL = 'sqlite:///tradesv3.sqlite' DEFAULT_DB_DRYRUN_URL = 'sqlite://' UNLIMITED_STAKE_AMOUNT = 'unlimited' diff --git a/freqtrade/optimize/custom_hyperopt.py b/freqtrade/optimize/custom_hyperopt.py deleted file mode 100644 index 80168c46e..000000000 --- a/freqtrade/optimize/custom_hyperopt.py +++ /dev/null @@ -1,153 +0,0 @@ -# pragma pylint: disable=attribute-defined-outside-init - -""" -This module load custom hyperopts -""" -import importlib -import logging -import os -import sys -from typing import Dict, Any, Callable - -from pandas import DataFrame - -from freqtrade.constants import Constants -from freqtrade.optimize.interface import IHyperOpt - -sys.path.insert(0, r'../../user_data/hyperopts') - -logger = logging.getLogger(__name__) - - -class CustomHyperOpt(object): - """ - This class contains all the logic to load custom hyperopt class - """ - def __init__(self, config: dict = {}) -> None: - """ - Load the custom class from config parameter - :param config: - :return: - """ - - # Verify the hyperopt is in the configuration, otherwise fallback to the default hyperopt - if 'hyperopt' in config: - hyperopt = config['hyperopt'] - else: - hyperopt = Constants.DEFAULT_HYPEROPT - - # Load the hyperopt - self._load_hyperopt(hyperopt) - - def _load_hyperopt(self, hyperopt_name: str) -> None: - """ - Search and load the custom hyperopt. If no hyperopt found, fallback on the default hyperopt - Set the object into self.custom_hyperopt - :param hyperopt_name: name of the module to import - :return: None - """ - - try: - # Start by sanitizing the file name (remove any extensions) - hyperopt_name = self._sanitize_module_name(filename=hyperopt_name) - - # Search where can be the hyperopt file - path = self._search_hyperopt(filename=hyperopt_name) - - # Load the hyperopt - self.custom_hyperopt = self._load_class(path + hyperopt_name) - - # Fallback to the default hyperopt - except (ImportError, TypeError) as error: - logger.error( - "Impossible to load Hyperopt 'user_data/hyperopts/%s.py'. This file does not exist" - " or contains Python code errors", - hyperopt_name - ) - logger.error( - "The error is:\n%s.", - error - ) - - def _load_class(self, filename: str) -> IHyperOpt: - """ - Import a hyperopt as a module - :param filename: path to the hyperopt (path from freqtrade/optimize/) - :return: return the hyperopt class - """ - module = importlib.import_module(filename, __package__) - custom_hyperopt = getattr(module, module.class_name) - - logger.info("Load hyperopt class: %s (%s.py)", module.class_name, filename) - return custom_hyperopt() - - @staticmethod - def _sanitize_module_name(filename: str) -> str: - """ - Remove any extension from filename - :param filename: filename to sanatize - :return: return the filename without extensions - """ - filename = os.path.basename(filename) - filename = os.path.splitext(filename)[0] - return filename - - @staticmethod - def _search_hyperopt(filename: str) -> str: - """ - Search for the hyperopt file in different folder - 1. search into the user_data/hyperopts folder - 2. search into the freqtrade/optimize folder - 3. if nothing found, return None - :param hyperopt_name: module name to search - :return: module path where is the hyperopt - """ - pwd = os.path.dirname(os.path.realpath(__file__)) + '/' - user_data = os.path.join(pwd, '..', '..', 'user_data', 'hyperopts', filename + '.py') - hyperopt_folder = os.path.join(pwd, filename + '.py') - - path = None - if os.path.isfile(user_data): - path = 'user_data.hyperopts.' - elif os.path.isfile(hyperopt_folder): - path = '.' - - return path - - def populate_indicators(self, dataframe: DataFrame) -> DataFrame: - """ - Populate indicators that will be used in the Buy and Sell hyperopt - :param dataframe: Raw data from the exchange and parsed by parse_ticker_dataframe() - :return: a Dataframe with all mandatory indicators for the strategies - """ - return self.custom_hyperopt.populate_indicators(dataframe) - - def buy_strategy_generator(self, params: Dict[str, Any]) -> Callable: - """ - Create a buy strategy generator - """ - return self.custom_hyperopt.buy_strategy_generator(params) - - def indicator_space(self) -> Dict[str, Any]: - """ - Create an indicator space - """ - return self.custom_hyperopt.indicator_space() - - def generate_roi_table(self, params: Dict) -> Dict[int, float]: - """ - Create an roi table - """ - return self.custom_hyperopt.generate_roi_table(params) - - def stoploss_space(self) -> Dict[str, Any]: - """ - Create a stoploss space - """ - return self.custom_hyperopt.stoploss_space() - - def roi_space(self) -> Dict[str, Any]: - """ - Create a roi space - """ - return self.custom_hyperopt.roi_space() diff --git a/freqtrade/optimize/hyperopt.py b/freqtrade/optimize/hyperopt.py index 5e23458d7..07c24ff18 100644 --- a/freqtrade/optimize/hyperopt.py +++ b/freqtrade/optimize/hyperopt.py @@ -22,7 +22,8 @@ from freqtrade.arguments import Arguments from freqtrade.configuration import Configuration from freqtrade.optimize import load_data from freqtrade.optimize.backtesting import Backtesting -from freqtrade.optimize.custom_hyperopt import CustomHyperOpt +from freqtrade.optimize.resolver import HyperOptResolver + logger = logging.getLogger(__name__) @@ -40,8 +41,8 @@ class Hyperopt(Backtesting): """ def __init__(self, config: Dict[str, Any]) -> None: super().__init__(config) - - self.custom_hyperopt = CustomHyperOpt(self.config) + self.config = config + self.custom_hyperopt = HyperOptResolver(self.config).hyperopt # set TARGET_TRADES to suit your number concurrent trades so its realistic # to the number of days diff --git a/freqtrade/optimize/resolver.py b/freqtrade/optimize/resolver.py new file mode 100644 index 000000000..78637483b --- /dev/null +++ b/freqtrade/optimize/resolver.py @@ -0,0 +1,104 @@ +# pragma pylint: disable=attribute-defined-outside-init + +""" +This module load custom hyperopts +""" +import importlib.util +import inspect +import logging +import os +from typing import Optional, Dict, Type + +from freqtrade.constants import Constants +from freqtrade.optimize.interface import IHyperOpt + + +logger = logging.getLogger(__name__) + + +class HyperOptResolver(object): + """ + This class contains all the logic to load custom hyperopt class + """ + + __slots__ = ['hyperopt'] + + def __init__(self, config: Optional[Dict] = None) -> None: + """ + Load the custom class from config parameter + :param config: configuration dictionary or None + """ + config = config or {} + + # Verify the hyperopt is in the configuration, otherwise fallback to the default hyperopt + hyperopt_name = config.get('hyperopt') or Constants.DEFAULT_HYPEROPT + self.hyperopt = self._load_hyperopt(hyperopt_name, extra_dir=config.get('hyperopt_path')) + + def _load_hyperopt( + self, hyperopt_name: str, extra_dir: Optional[str] = None) -> Optional[IHyperOpt]: + """ + Search and loads the specified hyperopt. + :param hyperopt_name: name of the module to import + :param extra_dir: additional directory to search for the given hyperopt + :return: HyperOpt instance or None + """ + current_path = os.path.dirname(os.path.realpath(__file__)) + abs_paths = [ + os.path.join(current_path, '..', '..', 'user_data', 'hyperopts'), + current_path, + ] + + if extra_dir: + # Add extra hyperopt directory on top of search paths + abs_paths.insert(0, extra_dir) + + for path in abs_paths: + hyperopt = self._search_hyperopt(path, hyperopt_name) + if hyperopt: + logger.info('Using resolved hyperopt %s from \'%s\'', hyperopt_name, path) + return hyperopt + + raise ImportError( + "Impossible to load Hyperopt '{}'. This class does not exist" + " or contains Python code errors".format(hyperopt_name) + ) + + @staticmethod + def _get_valid_hyperopts(module_path: str, hyperopt_name: str) -> Optional[Type[IHyperOpt]]: + """ + Returns a list of all possible hyperopts for the given module_path + :param module_path: absolute path to the module + :param hyperopt_name: Class name of the hyperopt + :return: Tuple with (name, class) or None + """ + + # Generate spec based on absolute path + spec = importlib.util.spec_from_file_location('user_data.hyperopts', module_path) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + + valid_hyperopts_gen = ( + obj for name, obj in inspect.getmembers(module, inspect.isclass) + if hyperopt_name == name and IHyperOpt in obj.__bases__ + ) + return next(valid_hyperopts_gen, None) + + @staticmethod + def _search_hyperopt(directory: str, hyperopt_name: str) -> Optional[IHyperOpt]: + """ + Search for the hyperopt_name in the given directory + :param directory: relative or absolute directory path + :return: name of the hyperopt class + """ + logger.debug('Searching for hyperopt %s in \'%s\'', hyperopt_name, directory) + for entry in os.listdir(directory): + # Only consider python files + if not entry.endswith('.py'): + logger.debug('Ignoring %s', entry) + continue + hyperopt = HyperOptResolver._get_valid_hyperopts( + os.path.abspath(os.path.join(directory, entry)), hyperopt_name + ) + if hyperopt: + return hyperopt() + return None diff --git a/user_data/hyperopts/test_hyperopt.py b/user_data/hyperopts/test_hyperopt.py index 0ee43c64f..b742af6c9 100644 --- a/user_data/hyperopts/test_hyperopt.py +++ b/user_data/hyperopts/test_hyperopt.py @@ -15,10 +15,6 @@ from freqtrade.indicator_helpers import fishers_inverse from freqtrade.optimize.interface import IHyperOpt -# Update this variable if you change the class name -class_name = 'TestHyperOpt' - - # This class is a sample. Feel free to customize it. class TestHyperOpt(IHyperOpt): """ From 8044846d3741408731967410692f7cf51d916671 Mon Sep 17 00:00:00 2001 From: Matthias Date: Wed, 7 Nov 2018 07:05:40 +0100 Subject: [PATCH 270/699] Fix some refactoring problems --- freqtrade/arguments.py | 2 +- freqtrade/optimize/resolver.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/freqtrade/arguments.py b/freqtrade/arguments.py index 429a7ca5a..ceaf3e7e5 100644 --- a/freqtrade/arguments.py +++ b/freqtrade/arguments.py @@ -108,7 +108,7 @@ class Arguments(object): '--customhyperopt', help='specify hyperopt class name (default: %(default)s)', dest='hyperopt', - default=Constants.DEFAULT_HYPEROPT, + default=constants.DEFAULT_HYPEROPT, type=str, metavar='NAME', ) diff --git a/freqtrade/optimize/resolver.py b/freqtrade/optimize/resolver.py index 78637483b..1e0316c12 100644 --- a/freqtrade/optimize/resolver.py +++ b/freqtrade/optimize/resolver.py @@ -9,7 +9,7 @@ import logging import os from typing import Optional, Dict, Type -from freqtrade.constants import Constants +from freqtrade.constants import DEFAULT_HYPEROPT from freqtrade.optimize.interface import IHyperOpt @@ -31,7 +31,7 @@ class HyperOptResolver(object): config = config or {} # Verify the hyperopt is in the configuration, otherwise fallback to the default hyperopt - hyperopt_name = config.get('hyperopt') or Constants.DEFAULT_HYPEROPT + hyperopt_name = config.get('hyperopt') or DEFAULT_HYPEROPT self.hyperopt = self._load_hyperopt(hyperopt_name, extra_dir=config.get('hyperopt_path')) def _load_hyperopt( From f4b626eda34a5375fe02ca54f769160618347590 Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Wed, 7 Nov 2018 13:34:07 +0100 Subject: [PATCH 271/699] Update ccxt from 1.17.476 to 1.17.480 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 2671660ea..0d59236f9 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -ccxt==1.17.476 +ccxt==1.17.480 SQLAlchemy==1.2.13 python-telegram-bot==11.1.0 arrow==0.12.1 From 553e5656ac143619347f86ae86c071fb993bcaf0 Mon Sep 17 00:00:00 2001 From: misagh Date: Wed, 7 Nov 2018 18:12:46 +0100 Subject: [PATCH 272/699] forcestoploss refactored --- freqtrade/freqtradebot.py | 13 ++++--------- 1 file changed, 4 insertions(+), 9 deletions(-) diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index 00dfa6cbc..bff1b35ba 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -58,13 +58,8 @@ class FreqtradeBot(object): self.exchange = Exchange(self.config) # Initializing Edge only if enabled - self.edge = Edge( - self.config, - self.exchange) if self.config.get( - 'edge', - {}).get( - 'enabled', - False) else None + self.edge = Edge(self.config, self.exchange) if \ + self.config.get('edge', {}).get('enabled', False) else None self.active_pair_whitelist: List[str] = self.config['exchange']['pair_whitelist'] self._init_modules() @@ -653,10 +648,10 @@ class FreqtradeBot(object): return False def check_sell(self, trade: Trade, sell_rate: float, buy: bool, sell: bool) -> bool: - if (self.config['edge']['enabled']): + if self.edge: stoploss = self.edge.stoploss(trade.pair) should_sell = self.strategy.should_sell( - trade, sell_rate, datetime.utcnow(), buy, sell, stoploss) + trade, sell_rate, datetime.utcnow(), buy, sell, force_stoploss=stoploss) else: should_sell = self.strategy.should_sell(trade, sell_rate, datetime.utcnow(), buy, sell) From 6d63de1932f3c0d1d4c6a7ef99ef750ac224b4a6 Mon Sep 17 00:00:00 2001 From: misagh Date: Wed, 7 Nov 2018 18:15:04 +0100 Subject: [PATCH 273/699] removing unnecessary lib --- freqtrade/edge/__init__.py | 1 - freqtrade/strategy/interface.py | 3 ++- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/freqtrade/edge/__init__.py b/freqtrade/edge/__init__.py index 648130e70..0a56e8bfa 100644 --- a/freqtrade/edge/__init__.py +++ b/freqtrade/edge/__init__.py @@ -14,7 +14,6 @@ from freqtrade.arguments import TimeRange from freqtrade.strategy.interface import SellType from freqtrade.strategy.resolver import IStrategy, StrategyResolver from collections import namedtuple -import sys logger = logging.getLogger(__name__) diff --git a/freqtrade/strategy/interface.py b/freqtrade/strategy/interface.py index 27016b18c..212559c8c 100644 --- a/freqtrade/strategy/interface.py +++ b/freqtrade/strategy/interface.py @@ -203,7 +203,8 @@ class IStrategy(ABC): return buy, sell def should_sell(self, trade: Trade, rate: float, date: datetime, buy: bool, - sell: bool, low: float = None, high: float = None, force_stoploss: float = 0) -> SellCheckTuple: + sell: bool, low: float = None, high: float = None, + force_stoploss: float = 0) -> SellCheckTuple: """ This function evaluate if on the condition required to trigger a sell has been reached if the threshold is reached and updates the trade record. From bd1b05828ef0969d854c2608a94416df0081fda5 Mon Sep 17 00:00:00 2001 From: misagh Date: Wed, 7 Nov 2018 18:19:58 +0100 Subject: [PATCH 274/699] typos in documentation corrected --- docs/edge.md | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/docs/edge.md b/docs/edge.md index b083d1575..48461ec5b 100644 --- a/docs/edge.md +++ b/docs/edge.md @@ -10,10 +10,10 @@ This page explains how to use Edge Positioning module in your bot in order to en ## Introduction Trading is all about probability. No one can claim that he has a strategy working all the time. You have to assume that sometimes you lose.

-But it doesn't mean there is no rule, it only means rules should work "most of the time". Let's play a game: we toss a coin, heads: I give you 10$, tails: You give me 10$. Is it an interetsing game ? no, it is quite boring, isn't it?

-But lets say the probabiliy that we have heads is 80%, and the probablilty that we have tails is 20%. Now it is becoming interesting ... +But it doesn't mean there is no rule, it only means rules should work "most of the time". Let's play a game: we toss a coin, heads: I give you 10$, tails: You give me 10$. Is it an interesting game ? no, it is quite boring, isn't it?

+But let's say the probability that we have heads is 80%, and the probability that we have tails is 20%. Now it is becoming interesting ... That means 10$ x 80% versus 10$ x 20%. 8$ versus 2$. That means over time you will win 8$ risking only 2$ on each toss of coin.

-Lets complicate it more: you win 80% of the time but only 2$, I win 20% of the time but 8$. The calculation is: 80% * 2$ versus 20% * 8$. It is becoming boring again because overtime you win $1.6$ (80% x 2$) and me $1.6 (20% * 8$) too.

+Let's complicate it more: you win 80% of the time but only 2$, I win 20% of the time but 8$. The calculation is: 80% * 2$ versus 20% * 8$. It is becoming boring again because overtime you win $1.6$ (80% x 2$) and me $1.6 (20% * 8$) too.

The question is: How do you calculate that? how do you know if you wanna play? The answer comes to two factors: - Win Rate @@ -21,7 +21,7 @@ The answer comes to two factors: ### Win Rate -Means over X trades what is the perctange of winning trades to total number of trades (note that we don't consider how much you gained but only If you won or not). +Means over X trades what is the percentage of winning trades to total number of trades (note that we don't consider how much you gained but only If you won or not). W = (Number of winning trades) / (Number of losing trades) @@ -76,7 +76,7 @@ Edge dictates the stake amount for each trade to the bot according to the follow - Allowed capital at risk - Stoploss -Alowed capital at risk is calculated as follows: +Allowed capital at risk is calculated as follows: **allowed capital at risk** = **total capital** X **allowed risk per trade** @@ -96,7 +96,7 @@ If true, then Edge will run periodically How often should Edge run in seconds? (default to 3600 so one hour) #### calculate_since_number_of_days -Number of days of data agaist which Edge calculates Win Rate, Risk Reward and Expectancy +Number of days of data against which Edge calculates Win Rate, Risk Reward and Expectancy Note that it downloads historical data so increasing this number would lead to slowing down the bot
(default to 7) @@ -127,14 +127,14 @@ It filters paris which have an expectancy lower than this number (default to 0.2 Having an expectancy of 0.20 means if you put 10$ on a trade you expect a 12$ return. #### min_trade_number -When calulating W and R and E (expectancy) against histoical data, you always want to have a minimum number of trades. The more this number is the more Edge is reliable. Having a win rate of 100% on a single trade doesn't mean anything at all. But having a win rate of 70% over past 100 trades means clearly something.
+When calculating W and R and E (expectancy) against historical data, you always want to have a minimum number of trades. The more this number is the more Edge is reliable. Having a win rate of 100% on a single trade doesn't mean anything at all. But having a win rate of 70% over past 100 trades means clearly something.
-Default to 10 (it is highly recommanded not to decrease this number) +Default to 10 (it is highly recommended not to decrease this number) #### max_trade_duration_minute -Edge will filter out trades with long duration. If a trade is profitable after 1 month, it is hard to evaluate the stratgy based on it. But if most of trades are profitable and they have maximum duration of 30 minutes, then it is clearly a good sign.
+Edge will filter out trades with long duration. If a trade is profitable after 1 month, it is hard to evaluate the strategy based on it. But if most of trades are profitable and they have maximum duration of 30 minutes, then it is clearly a good sign.
Default to 1 day (1440 = 60 * 24) #### remove_pumps -Edge will remove sudden pumps in a given market while going through historical data. However, given that pumps happen very often in crypto markets, we recommand you keep this off.
+Edge will remove sudden pumps in a given market while going through historical data. However, given that pumps happen very often in crypto markets, we recommend you keep this off.
Default to false \ No newline at end of file From 5bd3bae5aff4aa9ac67af2af8e62fbc60c8fe566 Mon Sep 17 00:00:00 2001 From: misagh Date: Wed, 7 Nov 2018 18:24:13 +0100 Subject: [PATCH 275/699] unifying default value explanations --- docs/edge.md | 32 +++++++++++++++++++------------- 1 file changed, 19 insertions(+), 13 deletions(-) diff --git a/docs/edge.md b/docs/edge.md index 48461ec5b..427e37103 100644 --- a/docs/edge.md +++ b/docs/edge.md @@ -90,10 +90,12 @@ Your position size then will be: Edge has following configurations: #### enabled -If true, then Edge will run periodically +If true, then Edge will run periodically
+(default to false) #### process_throttle_secs -How often should Edge run in seconds? (default to 3600 so one hour) +How often should Edge run in seconds?
+(default to 3600 so one hour) #### calculate_since_number_of_days Number of days of data against which Edge calculates Win Rate, Risk Reward and Expectancy @@ -108,33 +110,37 @@ Percentage of allowed risk per trade
(default to 1%) #### stoploss_range_min -Minimum stoploss (default to -0.01) +Minimum stoploss
+(default to -0.01) #### stoploss_range_max -Maximum stoploss (default to -0.10) +Maximum stoploss
+(default to -0.10) #### stoploss_range_step As an example if this is set to -0.01 then Edge will test the strategy for [-0.01, -0,02, -0,03 ..., -0.09, -0.10] ranges. Note than having a smaller step means having a bigger range which could lead to slow calculation.
-if you set this parameter to -0.001, you then slow down the Edge calculation by a factor of 10 +if you set this parameter to -0.001, you then slow down the Edge calculation by a factor of 10.
+(default to -0.01) #### minimum_winrate -It filters pairs which don't have at least minimum_winrate (default to 0.60) -This comes handy if you want to be conservative and don't comprise win rate in favor of risk reward ratio. +It filters pairs which don't have at least minimum_winrate. +This comes handy if you want to be conservative and don't comprise win rate in favor of risk reward ratio.
+(default to 0.60) #### minimum_expectancy -It filters paris which have an expectancy lower than this number (default to 0.20) -Having an expectancy of 0.20 means if you put 10$ on a trade you expect a 12$ return. +It filters paris which have an expectancy lower than this number . +Having an expectancy of 0.20 means if you put 10$ on a trade you expect a 12$ return.
+(default to 0.20) #### min_trade_number When calculating W and R and E (expectancy) against historical data, you always want to have a minimum number of trades. The more this number is the more Edge is reliable. Having a win rate of 100% on a single trade doesn't mean anything at all. But having a win rate of 70% over past 100 trades means clearly something.
- -Default to 10 (it is highly recommended not to decrease this number) +(default to 10, it is highly recommended not to decrease this number) #### max_trade_duration_minute Edge will filter out trades with long duration. If a trade is profitable after 1 month, it is hard to evaluate the strategy based on it. But if most of trades are profitable and they have maximum duration of 30 minutes, then it is clearly a good sign.
-Default to 1 day (1440 = 60 * 24) +(default to 1 day, 1440 = 60 * 24) #### remove_pumps Edge will remove sudden pumps in a given market while going through historical data. However, given that pumps happen very often in crypto markets, we recommend you keep this off.
-Default to false \ No newline at end of file +(default to false) \ No newline at end of file From f75606d2955d9ac197e55ab2e1391aa53ec46829 Mon Sep 17 00:00:00 2001 From: misagh Date: Wed, 7 Nov 2018 18:27:10 +0100 Subject: [PATCH 276/699] formulas markdown style --- docs/edge.md | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/docs/edge.md b/docs/edge.md index 427e37103..21a5566ab 100644 --- a/docs/edge.md +++ b/docs/edge.md @@ -24,20 +24,20 @@ The answer comes to two factors: Means over X trades what is the percentage of winning trades to total number of trades (note that we don't consider how much you gained but only If you won or not). -W = (Number of winning trades) / (Number of losing trades) +`W = (Number of winning trades) / (Number of losing trades)` ### Risk Reward Ratio Risk Reward Ratio is a formula used to measure the expected gains of a given investment against the risk of loss. It is basically what you potentially win divided by what you potentially lose: -R = Profit / Loss +`R = Profit / Loss` Over time, on many trades, you can calculate your risk reward by dividing your average profit on winning trades by your average loss on losing trades: -average profit = (Sum of profits) / (Number of winning trades) +`Average profit = (Sum of profits) / (Number of winning trades)` -average loss = (Sum of losses) / (Number of losing trades) +`Average loss = (Sum of losses) / (Number of losing trades)` -R = (average profit) / (average loss) +`R = (Average profit) / (Average loss)` ### Expectancy @@ -47,7 +47,7 @@ Expectancy Ratio = (Risk Reward Ratio x Win Rate) – Loss Rate So lets say your Win rate is 28% and your Risk Reward Ratio is 5: -Expectancy = (5 * 0.28) - 0.72 = 0.68 +`Expectancy = (5 * 0.28) - 0.72 = 0.68` Superficially, this means that on average you expect this strategy’s trades to return .68 times the size of your losers. This is important for two reasons: First, it may seem obvious, but you know right away that you have a positive return. Second, you now have a number you can compare to other candidate systems to make decisions about which ones you employ. From b425cc3e3ba161a31ffbeaf2506f8d1d51eb466b Mon Sep 17 00:00:00 2001 From: misagh Date: Wed, 7 Nov 2018 18:33:35 +0100 Subject: [PATCH 277/699] adding explanation regarding max trade duration and interval --- docs/edge.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/edge.md b/docs/edge.md index 21a5566ab..8fd5607ec 100644 --- a/docs/edge.md +++ b/docs/edge.md @@ -139,6 +139,7 @@ When calculating W and R and E (expectancy) against historical data, you always #### max_trade_duration_minute Edge will filter out trades with long duration. If a trade is profitable after 1 month, it is hard to evaluate the strategy based on it. But if most of trades are profitable and they have maximum duration of 30 minutes, then it is clearly a good sign.
+**NOTICE:** While configuring this value, you should take into consideration your ticker interval. as an example filtering out trades having duration less than one day for a strategy which has 4h interval does not make sense. default value is set assuming your strategy interval is relatively small (1m or 5m, etc).
(default to 1 day, 1440 = 60 * 24) #### remove_pumps From 96a43327ca5c3093bffc1aa2a7d7d87cfc231800 Mon Sep 17 00:00:00 2001 From: misagh Date: Wed, 7 Nov 2018 18:52:15 +0100 Subject: [PATCH 278/699] _pair_info moved to class header for reusibility --- freqtrade/edge/__init__.py | 9 +++++---- freqtrade/tests/edge/test_edge.py | 15 ++++++--------- 2 files changed, 11 insertions(+), 13 deletions(-) diff --git a/freqtrade/edge/__init__.py b/freqtrade/edge/__init__.py index 0a56e8bfa..5d62c1415 100644 --- a/freqtrade/edge/__init__.py +++ b/freqtrade/edge/__init__.py @@ -30,6 +30,11 @@ class Edge(): config: Dict = {} _cached_pairs: Dict[str, Any] = {} # Keeps a list of pairs + # pair info data type + _pair_info = namedtuple( + 'pair_info', + ['stoploss', 'winrate', 'risk_reward_ratio', 'required_risk_reward', 'expectancy']) + def __init__(self, config: Dict[str, Any], exchange=None) -> None: self.config = config @@ -42,10 +47,6 @@ class Edge(): self.advise_buy = self.strategy.advise_buy self.edge_config = self.config.get('edge', {}) - - # pair info data type - self._pair_info = namedtuple( - 'pair_info', 'stoploss, winrate, risk_reward_ratio, required_risk_reward, expectancy') self._cached_pairs: Dict[str, Any] = {} # Keeps a list of pairs self._total_capital: float = self.edge_config.get('total_capital_in_stake_currency') diff --git a/freqtrade/tests/edge/test_edge.py b/freqtrade/tests/edge/test_edge.py index 18e934352..4dee004cb 100644 --- a/freqtrade/tests/edge/test_edge.py +++ b/freqtrade/tests/edge/test_edge.py @@ -2,7 +2,6 @@ from freqtrade.tests.conftest import get_patched_exchange from freqtrade.edge import Edge from pandas import DataFrame, to_datetime from freqtrade.strategy.interface import SellType -from collections import namedtuple import arrow import numpy as np import math @@ -21,8 +20,6 @@ from unittest.mock import MagicMock ticker_start_time = arrow.get(2018, 10, 3) ticker_interval_in_minute = 60 _ohlc = {'date': 0, 'buy': 1, 'open': 2, 'high': 3, 'low': 4, 'close': 5, 'sell': 6, 'volume': 7} -_pair_info = namedtuple( - 'pair_info', 'stoploss, winrate, risk_reward_ratio, required_risk_reward, expectancy') def test_filter(mocker, default_conf): @@ -30,9 +27,9 @@ def test_filter(mocker, default_conf): edge = Edge(default_conf, exchange) mocker.patch('freqtrade.edge.Edge._cached_pairs', mocker.PropertyMock( return_value={ - 'E/F': _pair_info(-0.01, 0.66, 3.71, 0.50, 1.71), - 'C/D': _pair_info(-0.01, 0.66, 3.71, 0.50, 1.71), - 'N/O': _pair_info(-0.01, 0.66, 3.71, 0.50, 1.71) + 'E/F': Edge._pair_info(-0.01, 0.66, 3.71, 0.50, 1.71), + 'C/D': Edge._pair_info(-0.01, 0.66, 3.71, 0.50, 1.71), + 'N/O': Edge._pair_info(-0.01, 0.66, 3.71, 0.50, 1.71) } )) @@ -45,9 +42,9 @@ def test_stoploss(mocker, default_conf): edge = Edge(default_conf, exchange) mocker.patch('freqtrade.edge.Edge._cached_pairs', mocker.PropertyMock( return_value={ - 'E/F': _pair_info(-0.01, 0.66, 3.71, 0.50, 1.71), - 'C/D': _pair_info(-0.01, 0.66, 3.71, 0.50, 1.71), - 'N/O': _pair_info(-0.01, 0.66, 3.71, 0.50, 1.71) + 'E/F': Edge._pair_info(-0.01, 0.66, 3.71, 0.50, 1.71), + 'C/D': Edge._pair_info(-0.01, 0.66, 3.71, 0.50, 1.71), + 'N/O': Edge._pair_info(-0.01, 0.66, 3.71, 0.50, 1.71) } )) From 934dd97eb2d160e83b7cc0e86501d605710cac24 Mon Sep 17 00:00:00 2001 From: misagh Date: Wed, 7 Nov 2018 18:54:21 +0100 Subject: [PATCH 279/699] adding init for edge test folder --- freqtrade/tests/edge/__init__.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 freqtrade/tests/edge/__init__.py diff --git a/freqtrade/tests/edge/__init__.py b/freqtrade/tests/edge/__init__.py new file mode 100644 index 000000000..e69de29bb From 7b8098553391e9297ecab09127996bfa40a25872 Mon Sep 17 00:00:00 2001 From: misagh Date: Wed, 7 Nov 2018 19:00:18 +0100 Subject: [PATCH 280/699] comments on recursive function + indentation of function declaration --- freqtrade/edge/__init__.py | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/freqtrade/edge/__init__.py b/freqtrade/edge/__init__.py index 5d62c1415..61cf6940e 100644 --- a/freqtrade/edge/__init__.py +++ b/freqtrade/edge/__init__.py @@ -316,15 +316,16 @@ class Edge(): return result - def _detect_next_stop_or_sell_point( - self, - buy_column, - sell_column, - date_column, - ohlc_columns, - stoploss, - pair, - start_point=0): + def _detect_next_stop_or_sell_point(self, buy_column, sell_column, date_column, + ohlc_columns, stoploss, pair, start_point=0): + """ + Iterate through ohlc_columns recursively in order to find the next trade + Next trade opens from the first buy signal noticed to + The sell or stoploss signal after it. + It then calls itself cutting OHLC, buy_column, sell_colum and date_column + Cut from (the exit trade index) + 1 + Author: https://github.com/mishaker + """ result: list = [] open_trade_index = utf1st.find_1st(buy_column, 1, utf1st.cmp_equal) From 3330d327ed97f7a099f2f02abdb771d8706572ce Mon Sep 17 00:00:00 2001 From: misagh Date: Wed, 7 Nov 2018 19:03:08 +0100 Subject: [PATCH 281/699] =?UTF-8?q?removing=20reserve=20keyword=20?= =?UTF-8?q?=E2=80=9Cfilter=E2=80=9D:=20replaced=20by=20=E2=80=9Cadjust?= =?UTF-8?q?=E2=80=9D?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- freqtrade/edge/__init__.py | 11 +++++------ freqtrade/freqtradebot.py | 2 +- 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/freqtrade/edge/__init__.py b/freqtrade/edge/__init__.py index 61cf6940e..c0df17909 100644 --- a/freqtrade/edge/__init__.py +++ b/freqtrade/edge/__init__.py @@ -152,10 +152,12 @@ class Edge(): def stoploss(self, pair: str) -> float: return self._cached_pairs[pair].stoploss - def filter(self, pairs) -> list: + def adjust(self, pairs) -> list: + """ + Filters out and sorts "pairs" according to Edge calculated pairs + """ final = [] - for pair, info in self._cached_pairs.items(): if info.expectancy > float(self.edge_config.get('minimum_expectancy', 0.2)) and \ info.winrate > float(self.edge_config.get('minimum_winrate', 0.60)) and \ @@ -163,10 +165,7 @@ class Edge(): final.append(pair) if final: - logger.info( - 'Edge validated only %s', - final - ) + logger.info('Edge validated only %s', final) else: logger.info('Edge removed all pairs as no pair with minimum expectancy was found !') diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index bff1b35ba..f82f6da59 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -408,7 +408,7 @@ class FreqtradeBot(object): # running get_signal on historical data fetched # to find buy signals if self.edge: - whitelist = self.edge.filter(whitelist) + whitelist = self.edge.adjust(whitelist) for _pair in whitelist: (buy, sell) = self.strategy.get_signal(_pair, interval, self.exchange.klines.get(_pair)) From 866da8aaa1d49f13ffa1b6fc169b234aa1c40cba Mon Sep 17 00:00:00 2001 From: misagh Date: Wed, 7 Nov 2018 19:24:53 +0100 Subject: [PATCH 282/699] reinitializing Edge calculated data in case of inability to download backtesting data --- freqtrade/edge/__init__.py | 2 ++ freqtrade/tests/edge/test_edge.py | 4 ++-- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/freqtrade/edge/__init__.py b/freqtrade/edge/__init__.py index c0df17909..5d1ce5f11 100644 --- a/freqtrade/edge/__init__.py +++ b/freqtrade/edge/__init__.py @@ -92,6 +92,8 @@ class Edge(): ) if not data: + # Reinitializing cached pairs + self._cached_pairs = {} logger.critical("No data found. Edge is stopped ...") return False diff --git a/freqtrade/tests/edge/test_edge.py b/freqtrade/tests/edge/test_edge.py index 4dee004cb..ed47a60eb 100644 --- a/freqtrade/tests/edge/test_edge.py +++ b/freqtrade/tests/edge/test_edge.py @@ -22,7 +22,7 @@ ticker_interval_in_minute = 60 _ohlc = {'date': 0, 'buy': 1, 'open': 2, 'high': 3, 'low': 4, 'close': 5, 'sell': 6, 'volume': 7} -def test_filter(mocker, default_conf): +def test_adjust(mocker, default_conf): exchange = get_patched_exchange(mocker, default_conf) edge = Edge(default_conf, exchange) mocker.patch('freqtrade.edge.Edge._cached_pairs', mocker.PropertyMock( @@ -34,7 +34,7 @@ def test_filter(mocker, default_conf): )) pairs = ['A/B', 'C/D', 'E/F', 'G/H'] - assert(edge.filter(pairs) == ['E/F', 'C/D']) + assert(edge.adjust(pairs) == ['E/F', 'C/D']) def test_stoploss(mocker, default_conf): From 7b62e71f235e2e4a5240fc1bf0b348db37b95040 Mon Sep 17 00:00:00 2001 From: Matthias Date: Wed, 7 Nov 2018 19:46:04 +0100 Subject: [PATCH 283/699] Fix some tests and rebase issues --- freqtrade/optimize/__init__.py | 1 + freqtrade/optimize/default_hyperopt.py | 278 +++++------------------- freqtrade/optimize/hyperopt.py | 15 +- user_data/hyperopts/sample_hyperopt.py | 138 ++++++++++++ user_data/hyperopts/test_hyperopt.py | 279 ------------------------- 5 files changed, 194 insertions(+), 517 deletions(-) create mode 100644 user_data/hyperopts/sample_hyperopt.py delete mode 100644 user_data/hyperopts/test_hyperopt.py diff --git a/freqtrade/optimize/__init__.py b/freqtrade/optimize/__init__.py index 52766f78e..b1407de18 100644 --- a/freqtrade/optimize/__init__.py +++ b/freqtrade/optimize/__init__.py @@ -20,6 +20,7 @@ from pandas import DataFrame from freqtrade import misc, constants, OperationalException from freqtrade.exchange import Exchange from freqtrade.arguments import TimeRange +from freqtrade.optimize.default_hyperopt import DefaultHyperOpts # noqa: F401 logger = logging.getLogger(__name__) diff --git a/freqtrade/optimize/default_hyperopt.py b/freqtrade/optimize/default_hyperopt.py index 6ef95bb49..e127fd6d8 100644 --- a/freqtrade/optimize/default_hyperopt.py +++ b/freqtrade/optimize/default_hyperopt.py @@ -2,11 +2,11 @@ import talib.abstract as ta from pandas import DataFrame -from typing import Dict, Any, Callable +from typing import Dict, Any, Callable, List from functools import reduce import numpy -from hyperopt import hp +from skopt.space import Categorical, Dimension, Integer, Real import freqtrade.vendor.qtpylib.indicators as qtpylib from freqtrade.optimize.interface import IHyperOpt @@ -21,193 +21,52 @@ class DefaultHyperOpts(IHyperOpt): """ @staticmethod - def populate_indicators(dataframe: DataFrame) -> DataFrame: - """ - Adds several different TA indicators to the given DataFrame - """ + def populate_indicators(dataframe: DataFrame, metadata: dict) -> DataFrame: dataframe['adx'] = ta.ADX(dataframe) - dataframe['ao'] = qtpylib.awesome_oscillator(dataframe) - dataframe['cci'] = ta.CCI(dataframe) macd = ta.MACD(dataframe) dataframe['macd'] = macd['macd'] dataframe['macdsignal'] = macd['macdsignal'] - dataframe['macdhist'] = macd['macdhist'] dataframe['mfi'] = ta.MFI(dataframe) - dataframe['minus_dm'] = ta.MINUS_DM(dataframe) - dataframe['minus_di'] = ta.MINUS_DI(dataframe) - dataframe['plus_dm'] = ta.PLUS_DM(dataframe) - dataframe['plus_di'] = ta.PLUS_DI(dataframe) - dataframe['roc'] = ta.ROC(dataframe) dataframe['rsi'] = ta.RSI(dataframe) - # Inverse Fisher transform on RSI, values [-1.0, 1.0] (https://goo.gl/2JGGoy) - rsi = 0.1 * (dataframe['rsi'] - 50) - dataframe['fisher_rsi'] = (numpy.exp(2 * rsi) - 1) / (numpy.exp(2 * rsi) + 1) - # Inverse Fisher transform on RSI normalized, value [0.0, 100.0] (https://goo.gl/2JGGoy) - dataframe['fisher_rsi_norma'] = 50 * (dataframe['fisher_rsi'] + 1) - # Stoch - stoch = ta.STOCH(dataframe) - dataframe['slowd'] = stoch['slowd'] - dataframe['slowk'] = stoch['slowk'] - # Stoch fast stoch_fast = ta.STOCHF(dataframe) dataframe['fastd'] = stoch_fast['fastd'] - dataframe['fastk'] = stoch_fast['fastk'] - # Stoch RSI - stoch_rsi = ta.STOCHRSI(dataframe) - dataframe['fastd_rsi'] = stoch_rsi['fastd'] - dataframe['fastk_rsi'] = stoch_rsi['fastk'] + dataframe['minus_di'] = ta.MINUS_DI(dataframe) # Bollinger bands bollinger = qtpylib.bollinger_bands(qtpylib.typical_price(dataframe), window=20, stds=2) dataframe['bb_lowerband'] = bollinger['lower'] - dataframe['bb_middleband'] = bollinger['mid'] - dataframe['bb_upperband'] = bollinger['upper'] - # EMA - Exponential Moving Average - dataframe['ema3'] = ta.EMA(dataframe, timeperiod=3) - dataframe['ema5'] = ta.EMA(dataframe, timeperiod=5) - dataframe['ema10'] = ta.EMA(dataframe, timeperiod=10) - dataframe['ema50'] = ta.EMA(dataframe, timeperiod=50) - dataframe['ema100'] = ta.EMA(dataframe, timeperiod=100) - # SAR Parabolic dataframe['sar'] = ta.SAR(dataframe) - # SMA - Simple Moving Average - dataframe['sma'] = ta.SMA(dataframe, timeperiod=40) - # TEMA - Triple Exponential Moving Average - dataframe['tema'] = ta.TEMA(dataframe, timeperiod=9) - # Hilbert Transform Indicator - SineWave - hilbert = ta.HT_SINE(dataframe) - dataframe['htsine'] = hilbert['sine'] - dataframe['htleadsine'] = hilbert['leadsine'] - - # Pattern Recognition - Bullish candlestick patterns - # ------------------------------------ - """ - # Hammer: values [0, 100] - dataframe['CDLHAMMER'] = ta.CDLHAMMER(dataframe) - # Inverted Hammer: values [0, 100] - dataframe['CDLINVERTEDHAMMER'] = ta.CDLINVERTEDHAMMER(dataframe) - # Dragonfly Doji: values [0, 100] - dataframe['CDLDRAGONFLYDOJI'] = ta.CDLDRAGONFLYDOJI(dataframe) - # Piercing Line: values [0, 100] - dataframe['CDLPIERCING'] = ta.CDLPIERCING(dataframe) # values [0, 100] - # Morningstar: values [0, 100] - dataframe['CDLMORNINGSTAR'] = ta.CDLMORNINGSTAR(dataframe) # values [0, 100] - # Three White Soldiers: values [0, 100] - dataframe['CDL3WHITESOLDIERS'] = ta.CDL3WHITESOLDIERS(dataframe) # values [0, 100] - """ - - # Pattern Recognition - Bearish candlestick patterns - # ------------------------------------ - """ - # Hanging Man: values [0, 100] - dataframe['CDLHANGINGMAN'] = ta.CDLHANGINGMAN(dataframe) - # Shooting Star: values [0, 100] - dataframe['CDLSHOOTINGSTAR'] = ta.CDLSHOOTINGSTAR(dataframe) - # Gravestone Doji: values [0, 100] - dataframe['CDLGRAVESTONEDOJI'] = ta.CDLGRAVESTONEDOJI(dataframe) - # Dark Cloud Cover: values [0, 100] - dataframe['CDLDARKCLOUDCOVER'] = ta.CDLDARKCLOUDCOVER(dataframe) - # Evening Doji Star: values [0, 100] - dataframe['CDLEVENINGDOJISTAR'] = ta.CDLEVENINGDOJISTAR(dataframe) - # Evening Star: values [0, 100] - dataframe['CDLEVENINGSTAR'] = ta.CDLEVENINGSTAR(dataframe) - """ - - # Pattern Recognition - Bullish/Bearish candlestick patterns - # ------------------------------------ - """ - # Three Line Strike: values [0, -100, 100] - dataframe['CDL3LINESTRIKE'] = ta.CDL3LINESTRIKE(dataframe) - # Spinning Top: values [0, -100, 100] - dataframe['CDLSPINNINGTOP'] = ta.CDLSPINNINGTOP(dataframe) # values [0, -100, 100] - # Engulfing: values [0, -100, 100] - dataframe['CDLENGULFING'] = ta.CDLENGULFING(dataframe) # values [0, -100, 100] - # Harami: values [0, -100, 100] - dataframe['CDLHARAMI'] = ta.CDLHARAMI(dataframe) # values [0, -100, 100] - # Three Outside Up/Down: values [0, -100, 100] - dataframe['CDL3OUTSIDE'] = ta.CDL3OUTSIDE(dataframe) # values [0, -100, 100] - # Three Inside Up/Down: values [0, -100, 100] - dataframe['CDL3INSIDE'] = ta.CDL3INSIDE(dataframe) # values [0, -100, 100] - """ - - # Chart type - # ------------------------------------ - # Heikinashi stategy - heikinashi = qtpylib.heikinashi(dataframe) - dataframe['ha_open'] = heikinashi['open'] - dataframe['ha_close'] = heikinashi['close'] - dataframe['ha_high'] = heikinashi['high'] - dataframe['ha_low'] = heikinashi['low'] - return dataframe - @staticmethod - def buy_strategy_generator(params: Dict[str, Any]) -> Callable: + def buy_strategy_generator(self, params: Dict[str, Any]) -> Callable: """ Define the buy strategy parameters to be used by hyperopt """ - def populate_buy_trend(dataframe: DataFrame) -> DataFrame: + def populate_buy_trend(dataframe: DataFrame, metadata: dict) -> DataFrame: """ Buy strategy Hyperopt will build and use """ conditions = [] # GUARDS AND TRENDS - if 'uptrend_long_ema' in params and params['uptrend_long_ema']['enabled']: - conditions.append(dataframe['ema50'] > dataframe['ema100']) - if 'macd_below_zero' in params and params['macd_below_zero']['enabled']: - conditions.append(dataframe['macd'] < 0) - if 'uptrend_short_ema' in params and params['uptrend_short_ema']['enabled']: - conditions.append(dataframe['ema5'] > dataframe['ema10']) - if 'mfi' in params and params['mfi']['enabled']: - conditions.append(dataframe['mfi'] < params['mfi']['value']) - if 'fastd' in params and params['fastd']['enabled']: - conditions.append(dataframe['fastd'] < params['fastd']['value']) - if 'adx' in params and params['adx']['enabled']: - conditions.append(dataframe['adx'] > params['adx']['value']) - if 'rsi' in params and params['rsi']['enabled']: - conditions.append(dataframe['rsi'] < params['rsi']['value']) - if 'over_sar' in params and params['over_sar']['enabled']: - conditions.append(dataframe['close'] > dataframe['sar']) - if 'green_candle' in params and params['green_candle']['enabled']: - conditions.append(dataframe['close'] > dataframe['open']) - if 'uptrend_sma' in params and params['uptrend_sma']['enabled']: - prevsma = dataframe['sma'].shift(1) - conditions.append(dataframe['sma'] > prevsma) + if 'mfi-enabled' in params and params['mfi-enabled']: + conditions.append(dataframe['mfi'] < params['mfi-value']) + if 'fastd-enabled' in params and params['fastd-enabled']: + conditions.append(dataframe['fastd'] < params['fastd-value']) + if 'adx-enabled' in params and params['adx-enabled']: + conditions.append(dataframe['adx'] > params['adx-value']) + if 'rsi-enabled' in params and params['rsi-enabled']: + conditions.append(dataframe['rsi'] < params['rsi-value']) # TRIGGERS - triggers = { - 'lower_bb': ( - dataframe['close'] < dataframe['bb_lowerband'] - ), - 'lower_bb_tema': ( - dataframe['tema'] < dataframe['bb_lowerband'] - ), - 'faststoch10': (qtpylib.crossed_above( - dataframe['fastd'], 10.0 - )), - 'ao_cross_zero': (qtpylib.crossed_above( - dataframe['ao'], 0.0 - )), - 'ema3_cross_ema10': (qtpylib.crossed_above( - dataframe['ema3'], dataframe['ema10'] - )), - 'macd_cross_signal': (qtpylib.crossed_above( + if params['trigger'] == 'bb_lower': + conditions.append(dataframe['close'] < dataframe['bb_lowerband']) + if params['trigger'] == 'macd_cross_signal': + conditions.append(qtpylib.crossed_above( dataframe['macd'], dataframe['macdsignal'] - )), - 'sar_reversal': (qtpylib.crossed_above( + )) + if params['trigger'] == 'sar_reversal': + conditions.append(qtpylib.crossed_above( dataframe['close'], dataframe['sar'] - )), - 'ht_sine': (qtpylib.crossed_above( - dataframe['htleadsine'], dataframe['htsine'] - )), - 'heiken_reversal_bull': ( - (qtpylib.crossed_above(dataframe['ha_close'], dataframe['ha_open'])) & - (dataframe['ha_low'] == dataframe['ha_open']) - ), - 'di_cross': (qtpylib.crossed_above( - dataframe['plus_di'], dataframe['minus_di'] - )), - } - conditions.append(triggers.get(params['trigger']['type'])) + )) dataframe.loc[ reduce(lambda x, y: x & y, conditions), @@ -218,64 +77,21 @@ class DefaultHyperOpts(IHyperOpt): return populate_buy_trend @staticmethod - def indicator_space() -> Dict[str, Any]: + def indicator_space() -> List[Dimension]: """ Define your Hyperopt space for searching strategy parameters """ - return { - 'macd_below_zero': hp.choice('macd_below_zero', [ - {'enabled': False}, - {'enabled': True} - ]), - 'mfi': hp.choice('mfi', [ - {'enabled': False}, - {'enabled': True, 'value': hp.quniform('mfi-value', 10, 25, 5)} - ]), - 'fastd': hp.choice('fastd', [ - {'enabled': False}, - {'enabled': True, 'value': hp.quniform('fastd-value', 15, 45, 5)} - ]), - 'adx': hp.choice('adx', [ - {'enabled': False}, - {'enabled': True, 'value': hp.quniform('adx-value', 20, 50, 5)} - ]), - 'rsi': hp.choice('rsi', [ - {'enabled': False}, - {'enabled': True, 'value': hp.quniform('rsi-value', 20, 40, 5)} - ]), - 'uptrend_long_ema': hp.choice('uptrend_long_ema', [ - {'enabled': False}, - {'enabled': True} - ]), - 'uptrend_short_ema': hp.choice('uptrend_short_ema', [ - {'enabled': False}, - {'enabled': True} - ]), - 'over_sar': hp.choice('over_sar', [ - {'enabled': False}, - {'enabled': True} - ]), - 'green_candle': hp.choice('green_candle', [ - {'enabled': False}, - {'enabled': True} - ]), - 'uptrend_sma': hp.choice('uptrend_sma', [ - {'enabled': False}, - {'enabled': True} - ]), - 'trigger': hp.choice('trigger', [ - {'type': 'lower_bb'}, - {'type': 'lower_bb_tema'}, - {'type': 'faststoch10'}, - {'type': 'ao_cross_zero'}, - {'type': 'ema3_cross_ema10'}, - {'type': 'macd_cross_signal'}, - {'type': 'sar_reversal'}, - {'type': 'ht_sine'}, - {'type': 'heiken_reversal_bull'}, - {'type': 'di_cross'}, - ]), - } + return [ + Integer(10, 25, name='mfi-value'), + Integer(15, 45, name='fastd-value'), + Integer(20, 50, name='adx-value'), + Integer(20, 40, name='rsi-value'), + Categorical([True, False], name='mfi-enabled'), + Categorical([True, False], name='fastd-enabled'), + Categorical([True, False], name='adx-enabled'), + Categorical([True, False], name='rsi-enabled'), + Categorical(['bb_lower', 'macd_cross_signal', 'sar_reversal'], name='trigger') + ] @staticmethod def generate_roi_table(params: Dict) -> Dict[int, float]: @@ -291,24 +107,24 @@ class DefaultHyperOpts(IHyperOpt): return roi_table @staticmethod - def stoploss_space() -> Dict[str, Any]: + def stoploss_space() -> List[Dimension]: """ Stoploss Value to search """ - return { - 'stoploss': hp.quniform('stoploss', -0.5, -0.02, 0.02), - } + return [ + Real(-0.5, -0.02, name='stoploss'), + ] @staticmethod - def roi_space() -> Dict[str, Any]: + def roi_space() -> List[Dimension]: """ Values to search for each ROI steps """ - return { - 'roi_t1': hp.quniform('roi_t1', 10, 120, 20), - 'roi_t2': hp.quniform('roi_t2', 10, 60, 15), - 'roi_t3': hp.quniform('roi_t3', 10, 40, 10), - 'roi_p1': hp.quniform('roi_p1', 0.01, 0.04, 0.01), - 'roi_p2': hp.quniform('roi_p2', 0.01, 0.07, 0.01), - 'roi_p3': hp.quniform('roi_p3', 0.01, 0.20, 0.01), - } + return [ + Integer(10, 120, name='roi_t1'), + Integer(10, 60, name='roi_t2'), + Integer(10, 40, name='roi_t3'), + Real(0.01, 0.04, name='roi_p1'), + Real(0.01, 0.07, name='roi_p2'), + Real(0.01, 0.20, name='roi_p3'), + ] diff --git a/freqtrade/optimize/hyperopt.py b/freqtrade/optimize/hyperopt.py index 07c24ff18..f5f4222d6 100644 --- a/freqtrade/optimize/hyperopt.py +++ b/freqtrade/optimize/hyperopt.py @@ -105,7 +105,7 @@ class Hyperopt(Backtesting): best_result['params'] ) if 'roi_t1' in best_result['params']: - logger.info('ROI table:\n%s', self.generate_roi_table(best_result['params'])) + logger.info('ROI table:\n%s', self.custom_hyperopt.generate_roi_table(best_result['params'])) def log_results(self, results) -> None: """ @@ -147,19 +147,20 @@ class Hyperopt(Backtesting): """ spaces: List[Dimension] = [] if self.has_space('buy'): - spaces = {**spaces, **self.custom_hyperopt.indicator_space()} + spaces += self.custom_hyperopt.indicator_space() if self.has_space('roi'): - spaces = {**spaces, **self.custom_hyperopt.roi_space()} + spaces += self.custom_hyperopt.roi_space() if self.has_space('stoploss'): - spaces = {**spaces, **self.custom_hyperopt.stoploss_space()} + spaces += self.custom_hyperopt.stoploss_space() return spaces - def generate_optimizer(self, params: Dict) -> Dict: + def generate_optimizer(self, _params: Dict) -> Dict: + params = self.get_args(_params) if self.has_space('roi'): - self.analyze.strategy.minimal_roi = self.custom_hyperopt.generate_roi_table(params) + self.strategy.minimal_roi = self.custom_hyperopt.generate_roi_table(params) if self.has_space('buy'): - self.populate_buy_trend = self.custom_hyperopt.buy_strategy_generator(params) + self.advise_buy = self.custom_hyperopt.buy_strategy_generator(params) if self.has_space('stoploss'): self.strategy.stoploss = params['stoploss'] diff --git a/user_data/hyperopts/sample_hyperopt.py b/user_data/hyperopts/sample_hyperopt.py new file mode 100644 index 000000000..77e3b1696 --- /dev/null +++ b/user_data/hyperopts/sample_hyperopt.py @@ -0,0 +1,138 @@ +# pragma pylint: disable=missing-docstring, invalid-name, pointless-string-statement + +import talib.abstract as ta +from pandas import DataFrame +from typing import Dict, Any, Callable, List +from functools import reduce + +import numpy +from skopt.space import Categorical, Dimension, Integer, Real + +import freqtrade.vendor.qtpylib.indicators as qtpylib +from freqtrade.optimize.interface import IHyperOpt + +class_name = 'SampleHyperOpts' + + +# This class is a sample. Feel free to customize it. +class SampleHyperOpts(IHyperOpt): + """ + This is a test hyperopt to inspire you. + More information in https://github.com/freqtrade/freqtrade/blob/develop/docs/hyperopt.md + You can: + - Rename the class name (Do not forget to update class_name) + - Add any methods you want to build your hyperopt + - Add any lib you need to build your hyperopt + You must keep: + - the prototype for the methods: populate_indicators, indicator_space, buy_strategy_generator, + roi_space, generate_roi_table, stoploss_space + """ + + @staticmethod + def populate_indicators(dataframe: DataFrame, metadata: dict) -> DataFrame: + dataframe['adx'] = ta.ADX(dataframe) + macd = ta.MACD(dataframe) + dataframe['macd'] = macd['macd'] + dataframe['macdsignal'] = macd['macdsignal'] + dataframe['mfi'] = ta.MFI(dataframe) + dataframe['rsi'] = ta.RSI(dataframe) + stoch_fast = ta.STOCHF(dataframe) + dataframe['fastd'] = stoch_fast['fastd'] + dataframe['minus_di'] = ta.MINUS_DI(dataframe) + # Bollinger bands + bollinger = qtpylib.bollinger_bands(qtpylib.typical_price(dataframe), window=20, stds=2) + dataframe['bb_lowerband'] = bollinger['lower'] + dataframe['sar'] = ta.SAR(dataframe) + return dataframe + + def buy_strategy_generator(self, params: Dict[str, Any]) -> Callable: + """ + Define the buy strategy parameters to be used by hyperopt + """ + def populate_buy_trend(dataframe: DataFrame, metadata: dict) -> DataFrame: + """ + Buy strategy Hyperopt will build and use + """ + conditions = [] + # GUARDS AND TRENDS + if 'mfi-enabled' in params and params['mfi-enabled']: + conditions.append(dataframe['mfi'] < params['mfi-value']) + if 'fastd-enabled' in params and params['fastd-enabled']: + conditions.append(dataframe['fastd'] < params['fastd-value']) + if 'adx-enabled' in params and params['adx-enabled']: + conditions.append(dataframe['adx'] > params['adx-value']) + if 'rsi-enabled' in params and params['rsi-enabled']: + conditions.append(dataframe['rsi'] < params['rsi-value']) + + # TRIGGERS + if params['trigger'] == 'bb_lower': + conditions.append(dataframe['close'] < dataframe['bb_lowerband']) + if params['trigger'] == 'macd_cross_signal': + conditions.append(qtpylib.crossed_above( + dataframe['macd'], dataframe['macdsignal'] + )) + if params['trigger'] == 'sar_reversal': + conditions.append(qtpylib.crossed_above( + dataframe['close'], dataframe['sar'] + )) + + dataframe.loc[ + reduce(lambda x, y: x & y, conditions), + 'buy'] = 1 + + return dataframe + + return populate_buy_trend + + @staticmethod + def indicator_space() -> List[Dimension]: + """ + Define your Hyperopt space for searching strategy parameters + """ + return [ + Integer(10, 25, name='mfi-value'), + Integer(15, 45, name='fastd-value'), + Integer(20, 50, name='adx-value'), + Integer(20, 40, name='rsi-value'), + Categorical([True, False], name='mfi-enabled'), + Categorical([True, False], name='fastd-enabled'), + Categorical([True, False], name='adx-enabled'), + Categorical([True, False], name='rsi-enabled'), + Categorical(['bb_lower', 'macd_cross_signal', 'sar_reversal'], name='trigger') + ] + + @staticmethod + def generate_roi_table(params: Dict) -> Dict[int, float]: + """ + Generate the ROI table that will be used by Hyperopt + """ + roi_table = {} + roi_table[0] = params['roi_p1'] + params['roi_p2'] + params['roi_p3'] + roi_table[params['roi_t3']] = params['roi_p1'] + params['roi_p2'] + roi_table[params['roi_t3'] + params['roi_t2']] = params['roi_p1'] + roi_table[params['roi_t3'] + params['roi_t2'] + params['roi_t1']] = 0 + + return roi_table + + @staticmethod + def stoploss_space() -> List[Dimension]: + """ + Stoploss Value to search + """ + return [ + Real(-0.5, -0.02, name='stoploss'), + ] + + @staticmethod + def roi_space() -> List[Dimension]: + """ + Values to search for each ROI steps + """ + return [ + Integer(10, 120, name='roi_t1'), + Integer(10, 60, name='roi_t2'), + Integer(10, 40, name='roi_t3'), + Real(0.01, 0.04, name='roi_p1'), + Real(0.01, 0.07, name='roi_p2'), + Real(0.01, 0.20, name='roi_p3'), + ] diff --git a/user_data/hyperopts/test_hyperopt.py b/user_data/hyperopts/test_hyperopt.py deleted file mode 100644 index b742af6c9..000000000 --- a/user_data/hyperopts/test_hyperopt.py +++ /dev/null @@ -1,279 +0,0 @@ -# pragma pylint: disable=missing-docstring, invalid-name, pointless-string-statement - -import talib.abstract as ta -from pandas import DataFrame -from typing import Dict, Any, Callable -from functools import reduce -from math import exp - -import numpy -import talib.abstract as ta -from hyperopt import STATUS_FAIL, STATUS_OK, Trials, fmin, hp, space_eval, tpe - -import freqtrade.vendor.qtpylib.indicators as qtpylib -from freqtrade.indicator_helpers import fishers_inverse -from freqtrade.optimize.interface import IHyperOpt - - -# This class is a sample. Feel free to customize it. -class TestHyperOpt(IHyperOpt): - """ - This is a test hyperopt to inspire you. - More information in https://github.com/gcarq/freqtrade/blob/develop/docs/hyperopt.md - - You can: - - Rename the class name (Do not forget to update class_name) - - Add any methods you want to build your hyperopt - - Add any lib you need to build your hyperopt - - You must keep: - - the prototype for the methods: populate_indicators, indicator_space, buy_strategy_generator, - roi_space, generate_roi_table, stoploss_space - """ - - @staticmethod - def populate_indicators(dataframe: DataFrame) -> DataFrame: - """ - Adds several different TA indicators to the given DataFrame - - Performance Note: For the best performance be frugal on the number of indicators - you are using. Let uncomment only the indicator you are using in your strategies - or your hyperopt configuration, otherwise you will waste your memory and CPU usage. - """ - - # Momentum Indicator - # ------------------------------------ - - # ADX - dataframe['adx'] = ta.ADX(dataframe) - - """ - # Awesome oscillator - dataframe['ao'] = qtpylib.awesome_oscillator(dataframe) - - # Commodity Channel Index: values Oversold:<-100, Overbought:>100 - dataframe['cci'] = ta.CCI(dataframe) - - # MACD - macd = ta.MACD(dataframe) - dataframe['macd'] = macd['macd'] - dataframe['macdsignal'] = macd['macdsignal'] - dataframe['macdhist'] = macd['macdhist'] - - # MFI - dataframe['mfi'] = ta.MFI(dataframe) - - # Minus Directional Indicator / Movement - dataframe['minus_dm'] = ta.MINUS_DM(dataframe) - dataframe['minus_di'] = ta.MINUS_DI(dataframe) - - # Plus Directional Indicator / Movement - dataframe['plus_dm'] = ta.PLUS_DM(dataframe) - dataframe['plus_di'] = ta.PLUS_DI(dataframe) - dataframe['minus_di'] = ta.MINUS_DI(dataframe) - - # ROC - dataframe['roc'] = ta.ROC(dataframe) - - # RSI - dataframe['rsi'] = ta.RSI(dataframe) - - # Inverse Fisher transform on RSI, values [-1.0, 1.0] (https://goo.gl/2JGGoy) - rsi = 0.1 * (dataframe['rsi'] - 50) - dataframe['fisher_rsi'] = (numpy.exp(2 * rsi) - 1) / (numpy.exp(2 * rsi) + 1) - - # Inverse Fisher transform on RSI normalized, value [0.0, 100.0] (https://goo.gl/2JGGoy) - dataframe['fisher_rsi_norma'] = 50 * (dataframe['fisher_rsi'] + 1) - - # Stoch - stoch = ta.STOCH(dataframe) - dataframe['slowd'] = stoch['slowd'] - dataframe['slowk'] = stoch['slowk'] - - # Stoch fast - stoch_fast = ta.STOCHF(dataframe) - dataframe['fastd'] = stoch_fast['fastd'] - dataframe['fastk'] = stoch_fast['fastk'] - - # Stoch RSI - stoch_rsi = ta.STOCHRSI(dataframe) - dataframe['fastd_rsi'] = stoch_rsi['fastd'] - dataframe['fastk_rsi'] = stoch_rsi['fastk'] - """ - - # Overlap Studies - # ------------------------------------ - - # Bollinger bands - bollinger = qtpylib.bollinger_bands(qtpylib.typical_price(dataframe), window=20, stds=2) - dataframe['bb_lowerband'] = bollinger['lower'] - dataframe['bb_middleband'] = bollinger['mid'] - dataframe['bb_upperband'] = bollinger['upper'] - - """ - # EMA - Exponential Moving Average - dataframe['ema3'] = ta.EMA(dataframe, timeperiod=3) - dataframe['ema5'] = ta.EMA(dataframe, timeperiod=5) - dataframe['ema10'] = ta.EMA(dataframe, timeperiod=10) - dataframe['ema50'] = ta.EMA(dataframe, timeperiod=50) - dataframe['ema100'] = ta.EMA(dataframe, timeperiod=100) - - # SAR Parabol - dataframe['sar'] = ta.SAR(dataframe) - - # SMA - Simple Moving Average - dataframe['sma'] = ta.SMA(dataframe, timeperiod=40) - """ - - # TEMA - Triple Exponential Moving Average - dataframe['tema'] = ta.TEMA(dataframe, timeperiod=9) - - # Cycle Indicator - # ------------------------------------ - # Hilbert Transform Indicator - SineWave - hilbert = ta.HT_SINE(dataframe) - dataframe['htsine'] = hilbert['sine'] - dataframe['htleadsine'] = hilbert['leadsine'] - - # Pattern Recognition - Bullish candlestick patterns - # ------------------------------------ - """ - # Hammer: values [0, 100] - dataframe['CDLHAMMER'] = ta.CDLHAMMER(dataframe) - # Inverted Hammer: values [0, 100] - dataframe['CDLINVERTEDHAMMER'] = ta.CDLINVERTEDHAMMER(dataframe) - # Dragonfly Doji: values [0, 100] - dataframe['CDLDRAGONFLYDOJI'] = ta.CDLDRAGONFLYDOJI(dataframe) - # Piercing Line: values [0, 100] - dataframe['CDLPIERCING'] = ta.CDLPIERCING(dataframe) # values [0, 100] - # Morningstar: values [0, 100] - dataframe['CDLMORNINGSTAR'] = ta.CDLMORNINGSTAR(dataframe) # values [0, 100] - # Three White Soldiers: values [0, 100] - dataframe['CDL3WHITESOLDIERS'] = ta.CDL3WHITESOLDIERS(dataframe) # values [0, 100] - """ - - # Pattern Recognition - Bearish candlestick patterns - # ------------------------------------ - """ - # Hanging Man: values [0, 100] - dataframe['CDLHANGINGMAN'] = ta.CDLHANGINGMAN(dataframe) - # Shooting Star: values [0, 100] - dataframe['CDLSHOOTINGSTAR'] = ta.CDLSHOOTINGSTAR(dataframe) - # Gravestone Doji: values [0, 100] - dataframe['CDLGRAVESTONEDOJI'] = ta.CDLGRAVESTONEDOJI(dataframe) - # Dark Cloud Cover: values [0, 100] - dataframe['CDLDARKCLOUDCOVER'] = ta.CDLDARKCLOUDCOVER(dataframe) - # Evening Doji Star: values [0, 100] - dataframe['CDLEVENINGDOJISTAR'] = ta.CDLEVENINGDOJISTAR(dataframe) - # Evening Star: values [0, 100] - dataframe['CDLEVENINGSTAR'] = ta.CDLEVENINGSTAR(dataframe) - """ - - # Pattern Recognition - Bullish/Bearish candlestick patterns - # ------------------------------------ - """ - # Three Line Strike: values [0, -100, 100] - dataframe['CDL3LINESTRIKE'] = ta.CDL3LINESTRIKE(dataframe) - # Spinning Top: values [0, -100, 100] - dataframe['CDLSPINNINGTOP'] = ta.CDLSPINNINGTOP(dataframe) # values [0, -100, 100] - # Engulfing: values [0, -100, 100] - dataframe['CDLENGULFING'] = ta.CDLENGULFING(dataframe) # values [0, -100, 100] - # Harami: values [0, -100, 100] - dataframe['CDLHARAMI'] = ta.CDLHARAMI(dataframe) # values [0, -100, 100] - # Three Outside Up/Down: values [0, -100, 100] - dataframe['CDL3OUTSIDE'] = ta.CDL3OUTSIDE(dataframe) # values [0, -100, 100] - # Three Inside Up/Down: values [0, -100, 100] - dataframe['CDL3INSIDE'] = ta.CDL3INSIDE(dataframe) # values [0, -100, 100] - """ - - # Chart type - # ------------------------------------ - """ - # Heikinashi stategy - heikinashi = qtpylib.heikinashi(dataframe) - dataframe['ha_open'] = heikinashi['open'] - dataframe['ha_close'] = heikinashi['close'] - dataframe['ha_high'] = heikinashi['high'] - dataframe['ha_low'] = heikinashi['low'] - """ - - return dataframe - - @staticmethod - def indicator_space() -> Dict[str, Any]: - """ - Define your Hyperopt space for searching strategy parameters - """ - return { - 'adx': hp.choice('adx', [ - {'enabled': False}, - {'enabled': True, 'value': hp.quniform('adx-value', 50, 80, 5)} - ]), - 'uptrend_tema': hp.choice('uptrend_tema', [ - {'enabled': False}, - {'enabled': True} - ]), - 'trigger': hp.choice('trigger', [ - {'type': 'middle_bb_tema'}, - ]), - } - - @staticmethod - def buy_strategy_generator(params: Dict[str, Any]) -> Callable: - """ - Define the buy strategy parameters to be used by hyperopt - """ - def populate_buy_trend(dataframe: DataFrame) -> DataFrame: - conditions = [] - # GUARDS AND TRENDS - if 'adx' in params and params['adx']['enabled']: - conditions.append(dataframe['adx'] > params['adx']['value']) - if 'uptrend_tema' in params and params['uptrend_tema']['enabled']: - prevtema = dataframe['tema'].shift(1) - conditions.append(dataframe['tema'] > prevtema) - - # TRIGGERS - triggers = { - 'middle_bb_tema': ( - dataframe['tema'] > dataframe['bb_middleband'] - ), - } - conditions.append(triggers.get(params['trigger']['type'])) - - dataframe.loc[ - reduce(lambda x, y: x & y, conditions), - 'buy'] = 1 - - return dataframe - - return populate_buy_trend - - @staticmethod - def roi_space() -> Dict[str, Any]: - return { - 'roi_t1': hp.quniform('roi_t1', 10, 120, 20), - 'roi_t2': hp.quniform('roi_t2', 10, 60, 15), - 'roi_t3': hp.quniform('roi_t3', 10, 40, 10), - 'roi_p1': hp.quniform('roi_p1', 0.01, 0.04, 0.01), - 'roi_p2': hp.quniform('roi_p2', 0.01, 0.07, 0.01), - 'roi_p3': hp.quniform('roi_p3', 0.01, 0.20, 0.01), - } - - @staticmethod - def generate_roi_table(params: Dict) -> Dict[int, float]: - """ - Generate the ROI table that will be used by Hyperopt - """ - roi_table = {} - roi_table[0] = params['roi_p1'] + params['roi_p2'] + params['roi_p3'] - roi_table[params['roi_t3']] = params['roi_p1'] + params['roi_p2'] - roi_table[params['roi_t3'] + params['roi_t2']] = params['roi_p1'] - roi_table[params['roi_t3'] + params['roi_t2'] + params['roi_t1']] = 0 - - return roi_table - - @staticmethod - def stoploss_space() -> Dict[str, Any]: - return { - 'stoploss': hp.quniform('stoploss', -0.5, -0.02, 0.02), - } \ No newline at end of file From e5c6499706ba9c7523b57b3c91c72ca61147ad06 Mon Sep 17 00:00:00 2001 From: misagh Date: Thu, 8 Nov 2018 00:22:46 +0100 Subject: [PATCH 284/699] assigning strategy to edge from FreqtradeBot --- freqtrade/edge/__init__.py | 8 ++--- freqtrade/freqtradebot.py | 2 +- freqtrade/tests/edge/test_edge.py | 52 ++++++++++++++++--------------- 3 files changed, 32 insertions(+), 30 deletions(-) diff --git a/freqtrade/edge/__init__.py b/freqtrade/edge/__init__.py index 5d1ce5f11..a468a76a3 100644 --- a/freqtrade/edge/__init__.py +++ b/freqtrade/edge/__init__.py @@ -2,6 +2,7 @@ """ Edge positioning package """ import logging from typing import Any, Dict +from collections import namedtuple import arrow import numpy as np @@ -12,8 +13,7 @@ import freqtrade.optimize as optimize from freqtrade.arguments import Arguments from freqtrade.arguments import TimeRange from freqtrade.strategy.interface import SellType -from freqtrade.strategy.resolver import IStrategy, StrategyResolver -from collections import namedtuple + logger = logging.getLogger(__name__) @@ -35,11 +35,11 @@ class Edge(): 'pair_info', ['stoploss', 'winrate', 'risk_reward_ratio', 'required_risk_reward', 'expectancy']) - def __init__(self, config: Dict[str, Any], exchange=None) -> None: + def __init__(self, config: Dict[str, Any], exchange, strategy) -> None: self.config = config self.exchange = exchange - self.strategy: IStrategy = StrategyResolver(self.config).strategy + self.strategy = strategy self.ticker_interval = self.strategy.ticker_interval self.tickerdata_to_dataframe = self.strategy.tickerdata_to_dataframe self.get_timeframe = optimize.get_timeframe diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index f82f6da59..babe0a1da 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -58,7 +58,7 @@ class FreqtradeBot(object): self.exchange = Exchange(self.config) # Initializing Edge only if enabled - self.edge = Edge(self.config, self.exchange) if \ + self.edge = Edge(self.config, self.exchange, self.strategy) if \ self.config.get('edge', {}).get('enabled', False) else None self.active_pair_whitelist: List[str] = self.config['exchange']['pair_whitelist'] diff --git a/freqtrade/tests/edge/test_edge.py b/freqtrade/tests/edge/test_edge.py index ed47a60eb..1ac5e62c4 100644 --- a/freqtrade/tests/edge/test_edge.py +++ b/freqtrade/tests/edge/test_edge.py @@ -1,4 +1,7 @@ -from freqtrade.tests.conftest import get_patched_exchange +# pragma pylint: disable=missing-docstring, C0103 +# pragma pylint: disable=protected-access, too-many-lines, invalid-name, too-many-arguments + +from freqtrade.tests.conftest import get_patched_freqtradebot from freqtrade.edge import Edge from pandas import DataFrame, to_datetime from freqtrade.strategy.interface import SellType @@ -23,8 +26,8 @@ _ohlc = {'date': 0, 'buy': 1, 'open': 2, 'high': 3, 'low': 4, 'close': 5, 'sell' def test_adjust(mocker, default_conf): - exchange = get_patched_exchange(mocker, default_conf) - edge = Edge(default_conf, exchange) + freqtrade = get_patched_freqtradebot(mocker, default_conf) + edge = Edge(default_conf, freqtrade.exchange, freqtrade.strategy) mocker.patch('freqtrade.edge.Edge._cached_pairs', mocker.PropertyMock( return_value={ 'E/F': Edge._pair_info(-0.01, 0.66, 3.71, 0.50, 1.71), @@ -38,8 +41,8 @@ def test_adjust(mocker, default_conf): def test_stoploss(mocker, default_conf): - exchange = get_patched_exchange(mocker, default_conf) - edge = Edge(default_conf, exchange) + freqtrade = get_patched_freqtradebot(mocker, default_conf) + edge = Edge(default_conf, freqtrade.exchange, freqtrade.strategy) mocker.patch('freqtrade.edge.Edge._cached_pairs', mocker.PropertyMock( return_value={ 'E/F': Edge._pair_info(-0.01, 0.66, 3.71, 0.50, 1.71), @@ -92,8 +95,8 @@ def _time_on_candle(number): def test_edge_heartbeat_calculate(mocker, default_conf): - exchange = get_patched_exchange(mocker, default_conf) - edge = Edge(default_conf, exchange) + freqtrade = get_patched_freqtradebot(mocker, default_conf) + edge = Edge(default_conf, freqtrade.exchange, freqtrade.strategy) heartbeat = default_conf['edge']['process_throttle_secs'] # should not recalculate if heartbeat not reached @@ -135,11 +138,10 @@ def mocked_load_data(datadir, pairs=[], ticker_interval='0m', refresh_pairs=Fals def test_edge_process_downloaded_data(mocker, default_conf): default_conf['datadir'] = None - exchange = get_patched_exchange(mocker, default_conf) + freqtrade = get_patched_freqtradebot(mocker, default_conf) mocker.patch('freqtrade.exchange.Exchange.get_fee', MagicMock(return_value=0.001)) mocker.patch('freqtrade.optimize.load_data', mocked_load_data) - mocker.patch('freqtrade.exchange.Exchange.refresh_tickers', MagicMock()) - edge = Edge(default_conf, exchange) + edge = Edge(default_conf, freqtrade.exchange, freqtrade.strategy) assert edge.calculate() assert len(edge._cached_pairs) == 2 @@ -148,13 +150,13 @@ def test_edge_process_downloaded_data(mocker, default_conf): def test_process_expectancy(mocker, default_conf): default_conf['edge']['min_trade_number'] = 2 - exchange = get_patched_exchange(mocker, default_conf) + freqtrade = get_patched_freqtradebot(mocker, default_conf) def get_fee(): return 0.001 - exchange.get_fee = get_fee - edge = Edge(default_conf, exchange) + freqtrade.exchange.get_fee = get_fee + edge = Edge(default_conf, freqtrade.exchange, freqtrade.strategy) trades = [ {'pair': 'TEST/BTC', @@ -213,8 +215,8 @@ def test_process_expectancy(mocker, default_conf): def test_case_1(mocker, default_conf): - exchange = get_patched_exchange(mocker, default_conf) - edge = Edge(default_conf, exchange) + freqtrade = get_patched_freqtradebot(mocker, default_conf) + edge = Edge(default_conf, freqtrade.exchange, freqtrade.strategy) stoploss = -0.99 # we don't want stoploss to be hit in this test ticker = [ @@ -233,8 +235,8 @@ def test_case_1(mocker, default_conf): # 2) Two complete trades within dataframe (with sell hit for all) def test_case_2(mocker, default_conf): - exchange = get_patched_exchange(mocker, default_conf) - edge = Edge(default_conf, exchange) + freqtrade = get_patched_freqtradebot(mocker, default_conf) + edge = Edge(default_conf, freqtrade.exchange, freqtrade.strategy) stoploss = -0.99 # we don't want stoploss to be hit in this test ticker = [ @@ -274,8 +276,8 @@ def test_case_2(mocker, default_conf): # 3) Entered, sl 1%, candle drops 8% => Trade closed, 1% loss def test_case_3(mocker, default_conf): - exchange = get_patched_exchange(mocker, default_conf) - edge = Edge(default_conf, exchange) + freqtrade = get_patched_freqtradebot(mocker, default_conf) + edge = Edge(default_conf, freqtrade.exchange, freqtrade.strategy) stoploss = -0.01 # we don't want stoploss to be hit in this test ticker = [ @@ -303,8 +305,8 @@ def test_case_3(mocker, default_conf): # 4) Entered, sl 3 %, candle drops 4%, recovers to 1 % = > Trade closed, 3 % loss def test_case_4(mocker, default_conf): - exchange = get_patched_exchange(mocker, default_conf) - edge = Edge(default_conf, exchange) + freqtrade = get_patched_freqtradebot(mocker, default_conf) + edge = Edge(default_conf, freqtrade.exchange, freqtrade.strategy) stoploss = -0.03 # we don't want stoploss to be hit in this test ticker = [ @@ -333,11 +335,11 @@ def test_case_4(mocker, default_conf): # 5) Stoploss and sell are hit. should sell on stoploss def test_case_5(mocker, default_conf): - exchange = get_patched_exchange(mocker, default_conf) - edge = Edge(default_conf, exchange) + freqtrade = get_patched_freqtradebot(mocker, default_conf) + edge = Edge(default_conf, freqtrade.exchange, freqtrade.strategy) - exchange = get_patched_exchange(mocker, default_conf) - edge = Edge(default_conf, exchange) + freqtrade = get_patched_freqtradebot(mocker, default_conf) + edge = Edge(default_conf, freqtrade.exchange, freqtrade.strategy) stoploss = -0.03 # we don't want stoploss to be hit in this test ticker = [ From 6d80c038773252d9ae065adc4c729983844f5dfc Mon Sep 17 00:00:00 2001 From: misagh Date: Thu, 8 Nov 2018 00:28:20 +0100 Subject: [PATCH 285/699] removing raise KeyError in test --- freqtrade/tests/test_freqtradebot.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/freqtrade/tests/test_freqtradebot.py b/freqtrade/tests/test_freqtradebot.py index 1afb86024..9c72f9177 100644 --- a/freqtrade/tests/test_freqtradebot.py +++ b/freqtrade/tests/test_freqtradebot.py @@ -256,9 +256,7 @@ def test_edge_overrides_stake_amount(mocker, default_conf) -> None: # strategy stoploss should be ignored freqtrade.strategy.stoploss = -0.05 - with pytest.raises(KeyError): - freqtrade._get_trade_stake_amount('ETH/BTC') - + assert 'ETH/BTC' not in freqtrade.edge._cached_pairs assert freqtrade._get_trade_stake_amount('NEO/BTC') == 0.025 assert freqtrade._get_trade_stake_amount('LTC/BTC') == 0.02381 From cca371c573ffb9b5cf19a0d0a0797113ac37af0e Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Thu, 8 Nov 2018 13:34:06 +0100 Subject: [PATCH 286/699] Update ccxt from 1.17.480 to 1.17.481 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 0d59236f9..8ba831d7e 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -ccxt==1.17.480 +ccxt==1.17.481 SQLAlchemy==1.2.13 python-telegram-bot==11.1.0 arrow==0.12.1 From 5d850825f52bd7e5ee30da90745ce27d22ee3a87 Mon Sep 17 00:00:00 2001 From: misagh Date: Thu, 8 Nov 2018 14:10:52 +0100 Subject: [PATCH 287/699] adding a notice about the incompatibility of Edge with Dynamic whitelist --- docs/edge.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/edge.md b/docs/edge.md index 8fd5607ec..5cace4ea3 100644 --- a/docs/edge.md +++ b/docs/edge.md @@ -2,6 +2,8 @@ This page explains how to use Edge Positioning module in your bot in order to enter into a trade only if the trade has a reasonable win rate and risk reward ratio, and consequently adjust your position size and stoploss. +**NOTICE:** Edge positioning is not compatible with dynamic whitelist. it overrides dynamic whitelist. + ## Table of Contents - [Introduction](#introduction) From a7dc8f5f4f1782db3e7518ffcffd64654ec3b482 Mon Sep 17 00:00:00 2001 From: misagh Date: Thu, 8 Nov 2018 14:16:46 +0100 Subject: [PATCH 288/699] adding edge configuration to configuration.md and removed whitespaces --- docs/configuration.md | 21 +++++++++++---------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/docs/configuration.md b/docs/configuration.md index 15ba4b48d..372cd5bb3 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -13,7 +13,7 @@ This page explains how to configure your `config.json` file. We recommend to copy and use the `config.json.example` as a template for your bot configuration. -The table below will list all configuration parameters. +The table below will list all configuration parameters. | Command | Default | Mandatory | Description | |----------|---------|----------|-------------| @@ -21,11 +21,11 @@ The table below will list all configuration parameters. | `stake_currency` | BTC | Yes | Crypto-currency used for trading. | `stake_amount` | 0.05 | Yes | Amount of crypto-currency your bot will use for each trade. Per default, the bot will use (0.05 BTC x 3) = 0.15 BTC in total will be always engaged. Set it to 'unlimited' to allow the bot to use all avaliable balance. | `ticker_interval` | [1m, 5m, 30m, 1h, 1d] | No | The ticker interval to use (1min, 5 min, 30 min, 1 hour or 1 day). Default is 5 minutes -| `fiat_display_currency` | USD | Yes | Fiat currency used to show your profits. More information below. +| `fiat_display_currency` | USD | Yes | Fiat currency used to show your profits. More information below. | `dry_run` | true | Yes | Define if the bot must be in Dry-run or production mode. -| `process_only_new_candles` | false | No | If set to true indicators are processed only once a new candle arrives. If false each loop populates the indicators, this will mean the same candle is processed many times creating system load but can be useful of your strategy depends on tick data not only candle. Can be set either in Configuration or in the strategy. -| `minimal_roi` | See below | No | Set the threshold in percent the bot will use to sell a trade. More information below. If set, this parameter will override `minimal_roi` from your strategy file. -| `stoploss` | -0.10 | No | Value of the stoploss in percent used by the bot. More information below. If set, this parameter will override `stoploss` from your strategy file. +| `process_only_new_candles` | false | No | If set to true indicators are processed only once a new candle arrives. If false each loop populates the indicators, this will mean the same candle is processed many times creating system load but can be useful of your strategy depends on tick data not only candle. Can be set either in Configuration or in the strategy. +| `minimal_roi` | See below | No | Set the threshold in percent the bot will use to sell a trade. More information below. If set, this parameter will override `minimal_roi` from your strategy file. +| `stoploss` | -0.10 | No | Value of the stoploss in percent used by the bot. More information below. If set, this parameter will override `stoploss` from your strategy file. | `trailing_stop` | false | No | Enables trailing stop-loss (based on `stoploss` in either configuration or strategy file). | `trailing_stop_positve` | 0 | No | Changes stop-loss once profit has been reached. | `trailing_stop_positve_offset` | 0 | No | Offset on when to apply `trailing_stop_positive`. Percentage value which should be positive. @@ -47,6 +47,7 @@ The table below will list all configuration parameters. | `exchange.ccxt_rate_limit` | True | No | DEPRECATED!! Have CCXT handle Exchange rate limits. Depending on the exchange, having this to false can lead to temporary bans from the exchange. | `exchange.ccxt_config` | None | No | Additional CCXT parameters passed to the regular ccxt instance. Parameters may differ from exchange to exchange and are documented in the [ccxt documentation](https://ccxt.readthedocs.io/en/latest/manual.html#instantiation) | `exchange.ccxt_async_config` | None | No | Additional CCXT parameters passed to the async ccxt instance. Parameters may differ from exchange to exchange and are documented in the [ccxt documentation](https://ccxt.readthedocs.io/en/latest/manual.html#instantiation) +| `edge` | false | No | Please refer to [edge configuration document](edge.md) for detailed explanation | `experimental.use_sell_signal` | false | No | Use your sell strategy in addition of the `minimal_roi`. | `experimental.sell_profit_only` | false | No | waits until you have made a positive profit before taking a sell decision. | `experimental.ignore_roi_if_buy_signal` | false | No | Does not sell if the buy-signal is still active. Takes preference over `minimal_roi` and `use_sell_signal` @@ -70,7 +71,7 @@ The definition of each config parameters is in [misc.py](https://github.com/freq ### Understand stake_amount `stake_amount` is an amount of crypto-currency your bot will use for each trade. -The minimal value is 0.0005. If there is not enough crypto-currency in +The minimal value is 0.0005. If there is not enough crypto-currency in the account an exception is generated. To allow the bot to trade all the avaliable `stake_currency` in your account set `stake_amount` = `unlimited`. In this case a trade amount is calclulated as `currency_balanse / (max_open_trades - current_open_trades)`. @@ -186,13 +187,13 @@ creating trades. } ``` -Once you will be happy with your bot performance, you can switch it to +Once you will be happy with your bot performance, you can switch it to production mode. ## Switch to production mode -In production mode, the bot will engage your money. Be careful a wrong -strategy can lose all your money. Be aware of what you are doing when +In production mode, the bot will engage your money. Be careful a wrong +strategy can lose all your money. Be aware of what you are doing when you run it in production mode. ### To switch your bot in production mode: @@ -242,7 +243,7 @@ freqtrade ### Embedding Strategies -FreqTrade provides you with with an easy way to embed the strategy into your configuration file. +FreqTrade provides you with with an easy way to embed the strategy into your configuration file. This is done by utilizing BASE64 encoding and providing this string at the strategy configuration field, in your chosen config file. From aefc20738af5714cdab91602ce1d3aa1ccd2f8be Mon Sep 17 00:00:00 2001 From: misagh Date: Thu, 8 Nov 2018 14:18:07 +0100 Subject: [PATCH 289/699] adding dot to the end of the phrase. --- docs/configuration.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/configuration.md b/docs/configuration.md index 372cd5bb3..d70a47b38 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -47,7 +47,7 @@ The table below will list all configuration parameters. | `exchange.ccxt_rate_limit` | True | No | DEPRECATED!! Have CCXT handle Exchange rate limits. Depending on the exchange, having this to false can lead to temporary bans from the exchange. | `exchange.ccxt_config` | None | No | Additional CCXT parameters passed to the regular ccxt instance. Parameters may differ from exchange to exchange and are documented in the [ccxt documentation](https://ccxt.readthedocs.io/en/latest/manual.html#instantiation) | `exchange.ccxt_async_config` | None | No | Additional CCXT parameters passed to the async ccxt instance. Parameters may differ from exchange to exchange and are documented in the [ccxt documentation](https://ccxt.readthedocs.io/en/latest/manual.html#instantiation) -| `edge` | false | No | Please refer to [edge configuration document](edge.md) for detailed explanation +| `edge` | false | No | Please refer to [edge configuration document](edge.md) for detailed explanation. | `experimental.use_sell_signal` | false | No | Use your sell strategy in addition of the `minimal_roi`. | `experimental.sell_profit_only` | false | No | waits until you have made a positive profit before taking a sell decision. | `experimental.ignore_roi_if_buy_signal` | false | No | Does not sell if the buy-signal is still active. Takes preference over `minimal_roi` and `use_sell_signal` From e94da7ca41c817ac96373232f9da3448c235a330 Mon Sep 17 00:00:00 2001 From: Matthias Date: Mon, 15 Oct 2018 22:02:23 +0200 Subject: [PATCH 290/699] inverse backtest logic to loop over time - not pairs (more realistic) --- freqtrade/optimize/backtesting.py | 40 +++++++++++++++++++++++++------ 1 file changed, 33 insertions(+), 7 deletions(-) diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index 6fcde64fa..212c675fd 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -66,6 +66,7 @@ class Backtesting(object): if self.config.get('strategy_list', None): # Force one interval self.ticker_interval = str(self.config.get('ticker_interval')) + self.ticker_interval_mins = constants.TICKER_INTERVAL_MINUTES[self.ticker_interval] for strat in list(self.config['strategy_list']): stratconf = deepcopy(self.config) stratconf['strategy'] = strat @@ -86,6 +87,8 @@ class Backtesting(object): """ self.strategy = strategy self.ticker_interval = self.config.get('ticker_interval') + self.ticker_interval_mins = constants.TICKER_INTERVAL_MINUTES[self.ticker_interval] + self.tickerdata_to_dataframe = strategy.tickerdata_to_dataframe self.advise_buy = strategy.advise_buy self.advise_sell = strategy.advise_sell @@ -280,8 +283,13 @@ class Backtesting(object): processed = args['processed'] max_open_trades = args.get('max_open_trades', 0) position_stacking = args.get('position_stacking', False) + start_date = args.get('start_date') + end_date = args.get('end_date') trades = [] trade_count_lock: Dict = {} + ticker: Dict = {} + pairs = [] + # Create ticker dict for pair, pair_data in processed.items(): pair_data['buy'], pair_data['sell'] = 0, 0 # cleanup from previous run @@ -296,15 +304,29 @@ class Backtesting(object): # Convert from Pandas to list for performance reasons # (Looping Pandas is slow.) - ticker = [x for x in ticker_data.itertuples()] + ticker[pair] = [x for x in ticker_data.itertuples()] + pairs.append(pair) + + lock_pair_until: Dict = {} + tmp = start_date + timedelta(minutes=self.ticker_interval_mins) + index = 0 + # Loop timerange and test per pair + while tmp < end_date: + # print(f"time: {tmp}") + for i, pair in enumerate(ticker): + try: + row = ticker[pair][index] + except IndexError: + # missing Data for one pair ... + # TODO:howto handle this + # logger.warning(f"i: {index} - {tmp} did not exist for {pair}") + continue - lock_pair_until = None - for index, row in enumerate(ticker): if row.buy == 0 or row.sell == 1: continue # skip rows where no buy signal or that would immediately sell off if not position_stacking: - if lock_pair_until is not None and row.date <= lock_pair_until: + if pair in lock_pair_until and row.date <= lock_pair_until[pair]: continue if max_open_trades > 0: # Check if max_open_trades has already been reached for the given date @@ -313,17 +335,19 @@ class Backtesting(object): trade_count_lock[row.date] = trade_count_lock.get(row.date, 0) + 1 - trade_entry = self._get_sell_trade_entry(pair, row, ticker[index + 1:], + trade_entry = self._get_sell_trade_entry(pair, row, ticker[pair][index + 1:], trade_count_lock, args) if trade_entry: - lock_pair_until = trade_entry.close_time + lock_pair_until[pair] = trade_entry.close_time trades.append(trade_entry) else: # Set lock_pair_until to end of testing period if trade could not be closed # This happens only if the buy-signal was with the last candle - lock_pair_until = ticker_data.iloc[-1].date + lock_pair_until[pair] = end_date + tmp += timedelta(minutes=self.ticker_interval_mins) + index += 1 return DataFrame.from_records(trades, columns=BacktestResult._fields) def start(self) -> None: @@ -390,6 +414,8 @@ class Backtesting(object): 'processed': preprocessed, 'max_open_trades': max_open_trades, 'position_stacking': self.config.get('position_stacking', False), + 'start_date': min_date, + 'end_date': max_date, } ) From 96efd12a312c14afae0034c4dae2d99972059e6f Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 16 Oct 2018 19:35:16 +0200 Subject: [PATCH 291/699] add new options to hyperopt --- freqtrade/optimize/hyperopt.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/freqtrade/optimize/hyperopt.py b/freqtrade/optimize/hyperopt.py index b2d05d603..9766f5acb 100644 --- a/freqtrade/optimize/hyperopt.py +++ b/freqtrade/optimize/hyperopt.py @@ -276,11 +276,14 @@ class Hyperopt(Backtesting): self.strategy.stoploss = params['stoploss'] processed = load(TICKERDATA_PICKLE) + min_date, max_date = Backtesting.get_timeframe(processed) results = self.backtest( { 'stake_amount': self.config['stake_amount'], 'processed': processed, 'position_stacking': self.config.get('position_stacking', True), + 'start_date': min_date, + 'end_date': max_date, } ) result_explanation = self.format_results(results) From 6729dfa6d3ed8fec31f371f936f475184256b5dd Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 16 Oct 2018 20:32:33 +0200 Subject: [PATCH 292/699] Add get_timeframe mock for hyperopt --- freqtrade/tests/optimize/test_hyperopt.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/freqtrade/tests/optimize/test_hyperopt.py b/freqtrade/tests/optimize/test_hyperopt.py index c93f2d316..3ea374bf4 100644 --- a/freqtrade/tests/optimize/test_hyperopt.py +++ b/freqtrade/tests/optimize/test_hyperopt.py @@ -1,4 +1,5 @@ # pragma pylint: disable=missing-docstring,W0212,C0103 +from datetime import datetime import os from unittest.mock import MagicMock @@ -291,6 +292,10 @@ def test_generate_optimizer(mocker, default_conf) -> None: 'freqtrade.optimize.hyperopt.Hyperopt.backtest', MagicMock(return_value=backtest_result) ) + mocker.patch( + 'freqtrade.optimize.backtesting.Backtesting.get_timeframe', + MagicMock(return_value=(datetime(2017, 12, 10), datetime(2017, 12, 13))) + ) patch_exchange(mocker) mocker.patch('freqtrade.optimize.hyperopt.load', MagicMock()) From 03cda8e23eccbeccf8f66d618605d2413b3c33f1 Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 16 Oct 2018 20:09:12 +0200 Subject: [PATCH 293/699] remove meaningless backtesting test --- freqtrade/tests/optimize/test_backtesting.py | 17 +---------------- 1 file changed, 1 insertion(+), 16 deletions(-) diff --git a/freqtrade/tests/optimize/test_backtesting.py b/freqtrade/tests/optimize/test_backtesting.py index 20f2a6582..2d2b2d4a7 100644 --- a/freqtrade/tests/optimize/test_backtesting.py +++ b/freqtrade/tests/optimize/test_backtesting.py @@ -449,7 +449,7 @@ def test_backtesting_start(default_conf, mocker, caplog) -> None: ) default_conf['exchange']['pair_whitelist'] = ['UNITTEST/BTC'] - default_conf['ticker_interval'] = "1m" + default_conf['ticker_interval'] = '1m' default_conf['live'] = False default_conf['datadir'] = None default_conf['export'] = None @@ -587,21 +587,6 @@ def test_backtest_pricecontours(default_conf, fee, mocker) -> None: simple_backtest(default_conf, contour, numres, mocker) -# Test backtest using offline data (testdata directory) -def test_backtest_ticks(default_conf, fee, mocker): - mocker.patch('freqtrade.exchange.Exchange.get_fee', fee) - patch_exchange(mocker) - ticks = [1, 5] - fun = Backtesting(default_conf).advise_buy - for _ in ticks: - backtest_conf = _make_backtest_conf(mocker, conf=default_conf) - backtesting = Backtesting(default_conf) - backtesting.advise_buy = fun # Override - backtesting.advise_sell = fun # Override - results = backtesting.backtest(backtest_conf) - assert not results.empty - - def test_backtest_clash_buy_sell(mocker, default_conf): # Override the default buy trend function in our default_strategy def fun(dataframe=None, pair=None): From db17ccef2b9e89982d20cbfe11623f47a702095d Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 16 Oct 2018 20:34:20 +0200 Subject: [PATCH 294/699] Adapt backtesting-tests to new backtest-logic --- freqtrade/tests/optimize/test_backtesting.py | 32 +++++++++++++++----- 1 file changed, 25 insertions(+), 7 deletions(-) diff --git a/freqtrade/tests/optimize/test_backtesting.py b/freqtrade/tests/optimize/test_backtesting.py index 2d2b2d4a7..e3e078d72 100644 --- a/freqtrade/tests/optimize/test_backtesting.py +++ b/freqtrade/tests/optimize/test_backtesting.py @@ -86,17 +86,21 @@ def load_data_test(what): def simple_backtest(config, contour, num_results, mocker) -> None: patch_exchange(mocker) + config['ticker_interval'] = '1m' backtesting = Backtesting(config) data = load_data_test(contour) processed = backtesting.strategy.tickerdata_to_dataframe(data) + min_date, max_date = Backtesting.get_timeframe(processed) assert isinstance(processed, dict) results = backtesting.backtest( { 'stake_amount': config['stake_amount'], 'processed': processed, 'max_open_trades': 1, - 'position_stacking': False + 'position_stacking': False, + 'start_date': min_date, + 'end_date': max_date, } ) # results :: @@ -123,12 +127,16 @@ def _make_backtest_conf(mocker, conf=None, pair='UNITTEST/BTC', record=None): data = trim_dictlist(data, -201) patch_exchange(mocker) backtesting = Backtesting(conf) + processed = backtesting.strategy.tickerdata_to_dataframe(data) + min_date, max_date = Backtesting.get_timeframe(processed) return { 'stake_amount': conf['stake_amount'], - 'processed': backtesting.strategy.tickerdata_to_dataframe(data), + 'processed': processed, 'max_open_trades': 10, 'position_stacking': False, - 'record': record + 'record': record, + 'start_date': min_date, + 'end_date': max_date, } @@ -505,12 +513,15 @@ def test_backtest(default_conf, fee, mocker) -> None: data = optimize.load_data(None, ticker_interval='5m', pairs=['UNITTEST/BTC']) data = trim_dictlist(data, -200) data_processed = backtesting.strategy.tickerdata_to_dataframe(data) + min_date, max_date = Backtesting.get_timeframe(data_processed) results = backtesting.backtest( { 'stake_amount': default_conf['stake_amount'], 'processed': data_processed, 'max_open_trades': 10, - 'position_stacking': False + 'position_stacking': False, + 'start_date': min_date, + 'end_date': max_date, } ) assert not results.empty @@ -554,12 +565,16 @@ def test_backtest_1min_ticker_interval(default_conf, fee, mocker) -> None: # Run a backtesting for an exiting 5min ticker_interval data = optimize.load_data(None, ticker_interval='1m', pairs=['UNITTEST/BTC']) data = trim_dictlist(data, -200) + processed = backtesting.strategy.tickerdata_to_dataframe(data) + min_date, max_date = Backtesting.get_timeframe(processed) results = backtesting.backtest( { 'stake_amount': default_conf['stake_amount'], - 'processed': backtesting.strategy.tickerdata_to_dataframe(data), + 'processed': processed, 'max_open_trades': 1, - 'position_stacking': False + 'position_stacking': False, + 'start_date': min_date, + 'end_date': max_date, } ) assert not results.empty @@ -582,7 +597,10 @@ def test_processed(default_conf, mocker) -> None: def test_backtest_pricecontours(default_conf, fee, mocker) -> None: mocker.patch('freqtrade.exchange.Exchange.get_fee', fee) - tests = [['raise', 18], ['lower', 0], ['sine', 19]] + tests = [['raise', 18], ['lower', 0], ['sine', 16]] + # We need to enable sell-signal - otherwise it sells on ROI!! + default_conf['experimental'] = {"use_sell_signal": True} + for [contour, numres] in tests: simple_backtest(default_conf, contour, numres, mocker) From 83a8d79115b1056c1469338da4965817243e33f2 Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 16 Oct 2018 20:35:08 +0200 Subject: [PATCH 295/699] Fix alternate buy/sell (this should respect the sell signal!) --- freqtrade/tests/optimize/test_backtesting.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/freqtrade/tests/optimize/test_backtesting.py b/freqtrade/tests/optimize/test_backtesting.py index e3e078d72..20c117a8e 100644 --- a/freqtrade/tests/optimize/test_backtesting.py +++ b/freqtrade/tests/optimize/test_backtesting.py @@ -638,14 +638,16 @@ def test_backtest_only_sell(mocker, default_conf): def test_backtest_alternate_buy_sell(default_conf, fee, mocker): mocker.patch('freqtrade.exchange.Exchange.get_fee', fee) backtest_conf = _make_backtest_conf(mocker, conf=default_conf, pair='UNITTEST/BTC') + # We need to enable sell-signal - otherwise it sells on ROI!! + default_conf['experimental'] = {"use_sell_signal": True} backtesting = Backtesting(default_conf) backtesting.advise_buy = _trend_alternate # Override backtesting.advise_sell = _trend_alternate # Override results = backtesting.backtest(backtest_conf) backtesting._store_backtest_result("test_.json", results) - assert len(results) == 4 + assert len(results) == 21 # One trade was force-closed at the end - assert len(results.loc[results.open_at_end]) == 1 + assert len(results.loc[results.open_at_end]) == 0 def test_backtest_record(default_conf, fee, mocker): From 66487f2a133decf3ef887f732fc58b1927c9079a Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 16 Oct 2018 20:36:24 +0200 Subject: [PATCH 296/699] require start/end-date argument in backtest --- freqtrade/optimize/backtesting.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index 212c675fd..49f6375f7 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -283,8 +283,8 @@ class Backtesting(object): processed = args['processed'] max_open_trades = args.get('max_open_trades', 0) position_stacking = args.get('position_stacking', False) - start_date = args.get('start_date') - end_date = args.get('end_date') + start_date = args['start_date'] + end_date = args['end_date'] trades = [] trade_count_lock: Dict = {} ticker: Dict = {} From 2371d1e696b3a6491cbc2883b0fa523b491b9896 Mon Sep 17 00:00:00 2001 From: Matthias Date: Wed, 17 Oct 2018 06:54:07 +0200 Subject: [PATCH 297/699] Fix backtest test (don't use 8m file if we use 1m tickers) --- freqtrade/tests/optimize/test_backtesting.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/freqtrade/tests/optimize/test_backtesting.py b/freqtrade/tests/optimize/test_backtesting.py index 20c117a8e..d0c8a88aa 100644 --- a/freqtrade/tests/optimize/test_backtesting.py +++ b/freqtrade/tests/optimize/test_backtesting.py @@ -640,12 +640,16 @@ def test_backtest_alternate_buy_sell(default_conf, fee, mocker): backtest_conf = _make_backtest_conf(mocker, conf=default_conf, pair='UNITTEST/BTC') # We need to enable sell-signal - otherwise it sells on ROI!! default_conf['experimental'] = {"use_sell_signal": True} + default_conf['ticker_interval'] = '1m' backtesting = Backtesting(default_conf) backtesting.advise_buy = _trend_alternate # Override backtesting.advise_sell = _trend_alternate # Override results = backtesting.backtest(backtest_conf) backtesting._store_backtest_result("test_.json", results) - assert len(results) == 21 + # 200 candles in backtest data + # won't buy on first (shifted by 1) + # 100 buys signals + assert len(results) == 99 # One trade was force-closed at the end assert len(results.loc[results.open_at_end]) == 0 From fa4c199aa63d13837688ed0b7f3fa2d0a7d627a3 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 4 Nov 2018 13:23:59 +0100 Subject: [PATCH 298/699] fix some mismatches after rebase --- freqtrade/tests/optimize/test_backtesting.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/freqtrade/tests/optimize/test_backtesting.py b/freqtrade/tests/optimize/test_backtesting.py index d0c8a88aa..b0d4f2cbd 100644 --- a/freqtrade/tests/optimize/test_backtesting.py +++ b/freqtrade/tests/optimize/test_backtesting.py @@ -13,6 +13,7 @@ from arrow import Arrow from freqtrade import DependencyException, constants, optimize from freqtrade.arguments import Arguments, TimeRange +from freqtrade.optimize import get_timeframe from freqtrade.optimize.backtesting import (Backtesting, setup_configuration, start) from freqtrade.tests.conftest import log_has, patch_exchange @@ -91,7 +92,7 @@ def simple_backtest(config, contour, num_results, mocker) -> None: data = load_data_test(contour) processed = backtesting.strategy.tickerdata_to_dataframe(data) - min_date, max_date = Backtesting.get_timeframe(processed) + min_date, max_date = get_timeframe(processed) assert isinstance(processed, dict) results = backtesting.backtest( { @@ -128,7 +129,7 @@ def _make_backtest_conf(mocker, conf=None, pair='UNITTEST/BTC', record=None): patch_exchange(mocker) backtesting = Backtesting(conf) processed = backtesting.strategy.tickerdata_to_dataframe(data) - min_date, max_date = Backtesting.get_timeframe(processed) + min_date, max_date = get_timeframe(processed) return { 'stake_amount': conf['stake_amount'], 'processed': processed, @@ -513,7 +514,7 @@ def test_backtest(default_conf, fee, mocker) -> None: data = optimize.load_data(None, ticker_interval='5m', pairs=['UNITTEST/BTC']) data = trim_dictlist(data, -200) data_processed = backtesting.strategy.tickerdata_to_dataframe(data) - min_date, max_date = Backtesting.get_timeframe(data_processed) + min_date, max_date = get_timeframe(data_processed) results = backtesting.backtest( { 'stake_amount': default_conf['stake_amount'], @@ -566,7 +567,7 @@ def test_backtest_1min_ticker_interval(default_conf, fee, mocker) -> None: data = optimize.load_data(None, ticker_interval='1m', pairs=['UNITTEST/BTC']) data = trim_dictlist(data, -200) processed = backtesting.strategy.tickerdata_to_dataframe(data) - min_date, max_date = Backtesting.get_timeframe(processed) + min_date, max_date = get_timeframe(processed) results = backtesting.backtest( { 'stake_amount': default_conf['stake_amount'], From 9cd2ed5a16ae7ed5c617dd345ade46d7b8d6d18c Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 4 Nov 2018 13:43:09 +0100 Subject: [PATCH 299/699] fix hyperopt get_timeframe mock --- freqtrade/optimize/hyperopt.py | 4 ++-- freqtrade/tests/optimize/test_hyperopt.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/freqtrade/optimize/hyperopt.py b/freqtrade/optimize/hyperopt.py index 9766f5acb..edf73fcf3 100644 --- a/freqtrade/optimize/hyperopt.py +++ b/freqtrade/optimize/hyperopt.py @@ -23,7 +23,7 @@ from skopt.space import Categorical, Dimension, Integer, Real import freqtrade.vendor.qtpylib.indicators as qtpylib from freqtrade.arguments import Arguments from freqtrade.configuration import Configuration -from freqtrade.optimize import load_data +from freqtrade.optimize import load_data, get_timeframe from freqtrade.optimize.backtesting import Backtesting logger = logging.getLogger(__name__) @@ -276,7 +276,7 @@ class Hyperopt(Backtesting): self.strategy.stoploss = params['stoploss'] processed = load(TICKERDATA_PICKLE) - min_date, max_date = Backtesting.get_timeframe(processed) + min_date, max_date = get_timeframe(processed) results = self.backtest( { 'stake_amount': self.config['stake_amount'], diff --git a/freqtrade/tests/optimize/test_hyperopt.py b/freqtrade/tests/optimize/test_hyperopt.py index 3ea374bf4..703b88fc1 100644 --- a/freqtrade/tests/optimize/test_hyperopt.py +++ b/freqtrade/tests/optimize/test_hyperopt.py @@ -6,7 +6,7 @@ from unittest.mock import MagicMock import pandas as pd import pytest -from freqtrade.optimize.__init__ import load_tickerdata_file +from freqtrade.optimize import load_tickerdata_file from freqtrade.optimize.hyperopt import Hyperopt, start from freqtrade.strategy.resolver import StrategyResolver from freqtrade.tests.conftest import log_has, patch_exchange @@ -293,7 +293,7 @@ def test_generate_optimizer(mocker, default_conf) -> None: MagicMock(return_value=backtest_result) ) mocker.patch( - 'freqtrade.optimize.backtesting.Backtesting.get_timeframe', + 'freqtrade.optimize.hyperopt.get_timeframe', MagicMock(return_value=(datetime(2017, 12, 10), datetime(2017, 12, 13))) ) patch_exchange(mocker) From 93429a58b2d605215b4157cb8409cb845ded97ed Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 4 Nov 2018 13:45:22 +0100 Subject: [PATCH 300/699] remove TODO --- freqtrade/optimize/backtesting.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index 49f6375f7..4dc3119d3 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -318,8 +318,7 @@ class Backtesting(object): row = ticker[pair][index] except IndexError: # missing Data for one pair ... - # TODO:howto handle this - # logger.warning(f"i: {index} - {tmp} did not exist for {pair}") + # Warnings for this are shown by `validate_backtest_data` continue if row.buy == 0 or row.sell == 1: From 56dcf080a906645df2a2872618c3666c129d8973 Mon Sep 17 00:00:00 2001 From: Matthias Date: Mon, 5 Nov 2018 20:03:04 +0100 Subject: [PATCH 301/699] Add explicit test for parallel trades --- freqtrade/tests/optimize/test_backtesting.py | 71 ++++++++++++++++++++ 1 file changed, 71 insertions(+) diff --git a/freqtrade/tests/optimize/test_backtesting.py b/freqtrade/tests/optimize/test_backtesting.py index b0d4f2cbd..e03e011f3 100644 --- a/freqtrade/tests/optimize/test_backtesting.py +++ b/freqtrade/tests/optimize/test_backtesting.py @@ -655,6 +655,77 @@ def test_backtest_alternate_buy_sell(default_conf, fee, mocker): assert len(results.loc[results.open_at_end]) == 0 +def test_backtest_multi_pair(default_conf, fee, mocker): + + def evaluate_result_multi(results, freq, max_open_trades): + # Find overlapping trades by expanding each trade once per period + # and then counting overlaps + dates = [pd.Series(pd.date_range(row[1].open_time, row[1].close_time, freq=freq)) + for row in results[['open_time', 'close_time']].iterrows()] + deltas = [len(x) for x in dates] + dates = pd.Series(pd.concat(dates).values, name='date') + df2 = pd.DataFrame(np.repeat(results.values, deltas, axis=0), columns=results.columns) + + df2 = df2.astype(dtype={"open_time": "datetime64", "close_time": "datetime64"}) + df2 = pd.concat([dates, df2], axis=1) + df2 = df2.set_index('date') + df_final = df2.resample(freq)[['pair']].count() + return df_final[df_final['pair'] > max_open_trades] + + def _trend_alternate_hold(dataframe=None, metadata=None): + """ + Buy every 8th candle - sell every other 8th -2 (hold on to pairs a bit) + """ + multi = 8 + dataframe['buy'] = np.where(dataframe.index % multi == 0, 1, 0) + dataframe['sell'] = np.where((dataframe.index + multi - 2) % multi == 0, 1, 0) + if metadata['pair'] in('ETH/BTC', 'LTC/BTC'): + dataframe['buy'] = dataframe['buy'].shift(-4) + dataframe['sell'] = dataframe['sell'].shift(-4) + return dataframe + + mocker.patch('freqtrade.exchange.Exchange.get_fee', fee) + pairs = ['ADA/BTC', 'DASH/BTC', 'ETH/BTC', 'LTC/BTC', 'NXT/BTC'] + data = optimize.load_data(None, ticker_interval='5m', pairs=pairs) + data = trim_dictlist(data, -500) + # We need to enable sell-signal - otherwise it sells on ROI!! + default_conf['experimental'] = {"use_sell_signal": True} + default_conf['ticker_interval'] = '5m' + + backtesting = Backtesting(default_conf) + backtesting.advise_buy = _trend_alternate_hold # Override + backtesting.advise_sell = _trend_alternate_hold # Override + + data_processed = backtesting.strategy.tickerdata_to_dataframe(data) + min_date, max_date = get_timeframe(data_processed) + backtest_conf = { + 'stake_amount': default_conf['stake_amount'], + 'processed': data_processed, + 'max_open_trades': 3, + 'position_stacking': False, + 'start_date': min_date, + 'end_date': max_date, + } + + results = backtesting.backtest(backtest_conf) + + # Make sure we have parallel trades + assert len(evaluate_result_multi(results, '5min', 2)) > 0 + # make sure we don't have trades with more than configured max_open_trades + assert len(evaluate_result_multi(results, '5min', 3)) == 0 + + backtest_conf = { + 'stake_amount': default_conf['stake_amount'], + 'processed': data_processed, + 'max_open_trades': 1, + 'position_stacking': False, + 'start_date': min_date, + 'end_date': max_date, + } + results = backtesting.backtest(backtest_conf) + assert len(evaluate_result_multi(results, '5min', 1)) == 0 + + def test_backtest_record(default_conf, fee, mocker): names = [] records = [] From 272ff51d512d0c06832812f985744a7cbea72719 Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 6 Nov 2018 20:37:59 +0100 Subject: [PATCH 302/699] correctly patch exchange --- freqtrade/tests/optimize/test_backtesting.py | 1 + 1 file changed, 1 insertion(+) diff --git a/freqtrade/tests/optimize/test_backtesting.py b/freqtrade/tests/optimize/test_backtesting.py index e03e011f3..aa1f144cd 100644 --- a/freqtrade/tests/optimize/test_backtesting.py +++ b/freqtrade/tests/optimize/test_backtesting.py @@ -685,6 +685,7 @@ def test_backtest_multi_pair(default_conf, fee, mocker): return dataframe mocker.patch('freqtrade.exchange.Exchange.get_fee', fee) + patch_exchange(mocker) pairs = ['ADA/BTC', 'DASH/BTC', 'ETH/BTC', 'LTC/BTC', 'NXT/BTC'] data = optimize.load_data(None, ticker_interval='5m', pairs=pairs) data = trim_dictlist(data, -500) From 5c5fe4c13a4b92be65e0e52903bb804f0e397f09 Mon Sep 17 00:00:00 2001 From: Matthias Date: Fri, 9 Nov 2018 07:14:43 +0100 Subject: [PATCH 303/699] Fix test --- freqtrade/tests/optimize/test_backtesting.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/tests/optimize/test_backtesting.py b/freqtrade/tests/optimize/test_backtesting.py index aa1f144cd..03d874f5f 100644 --- a/freqtrade/tests/optimize/test_backtesting.py +++ b/freqtrade/tests/optimize/test_backtesting.py @@ -598,7 +598,7 @@ def test_processed(default_conf, mocker) -> None: def test_backtest_pricecontours(default_conf, fee, mocker) -> None: mocker.patch('freqtrade.exchange.Exchange.get_fee', fee) - tests = [['raise', 18], ['lower', 0], ['sine', 16]] + tests = [['raise', 18], ['lower', 0], ['sine', 19]] # We need to enable sell-signal - otherwise it sells on ROI!! default_conf['experimental'] = {"use_sell_signal": True} From 1db9169cfcaa81f54e78df7ba3240144e9505f55 Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Fri, 9 Nov 2018 13:34:07 +0100 Subject: [PATCH 304/699] Update ccxt from 1.17.481 to 1.17.485 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 8ba831d7e..b46d3de31 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -ccxt==1.17.481 +ccxt==1.17.485 SQLAlchemy==1.2.13 python-telegram-bot==11.1.0 arrow==0.12.1 From 1840695a1c6a06afbd1c24ff13c42da8e99178c3 Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Fri, 9 Nov 2018 13:34:08 +0100 Subject: [PATCH 305/699] Update requests from 2.20.0 to 2.20.1 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index b46d3de31..4e38f66de 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,7 +3,7 @@ SQLAlchemy==1.2.13 python-telegram-bot==11.1.0 arrow==0.12.1 cachetools==3.0.0 -requests==2.20.0 +requests==2.20.1 urllib3==1.24.1 wrapt==1.10.11 pandas==0.23.4 From 292962d64dfb88d9d0b4b55aab9f5883577867b3 Mon Sep 17 00:00:00 2001 From: Matthias Date: Fri, 9 Nov 2018 19:34:18 +0100 Subject: [PATCH 306/699] Fix tests --- freqtrade/tests/optimize/__init__.py | 5 +++-- freqtrade/tests/optimize/test_backtest_detail.py | 7 ++++++- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/freqtrade/tests/optimize/__init__.py b/freqtrade/tests/optimize/__init__.py index 2b7222e88..58ea7c343 100644 --- a/freqtrade/tests/optimize/__init__.py +++ b/freqtrade/tests/optimize/__init__.py @@ -4,9 +4,10 @@ import arrow from pandas import DataFrame from freqtrade.strategy.interface import SellType +from freqtrade.constants import TICKER_INTERVAL_MINUTES ticker_start_time = arrow.get(2018, 10, 3) -ticker_interval_in_minute = 60 +tests_ticker_interval = "1h" class BTrade(NamedTuple): @@ -31,7 +32,7 @@ class BTContainer(NamedTuple): def _get_frame_time_from_offset(offset): return ticker_start_time.shift( - minutes=(offset * ticker_interval_in_minute)).datetime + minutes=(offset * TICKER_INTERVAL_MINUTES[tests_ticker_interval])).datetime def _build_backtest_dataframe(ticker_with_signals): diff --git a/freqtrade/tests/optimize/test_backtest_detail.py b/freqtrade/tests/optimize/test_backtest_detail.py index 806c136bc..7db6913f3 100644 --- a/freqtrade/tests/optimize/test_backtest_detail.py +++ b/freqtrade/tests/optimize/test_backtest_detail.py @@ -6,10 +6,11 @@ from pandas import DataFrame import pytest +from freqtrade.optimize import get_timeframe from freqtrade.optimize.backtesting import Backtesting from freqtrade.strategy.interface import SellType from freqtrade.tests.optimize import (BTrade, BTContainer, _build_backtest_dataframe, - _get_frame_time_from_offset) + _get_frame_time_from_offset, tests_ticker_interval) from freqtrade.tests.conftest import patch_exchange @@ -147,6 +148,7 @@ def test_backtest_results(default_conf, fee, mocker, caplog, data) -> None: """ default_conf["stoploss"] = data.stop_loss default_conf["minimal_roi"] = {"0": data.roi} + default_conf['ticker_interval'] = tests_ticker_interval mocker.patch('freqtrade.exchange.Exchange.get_fee', MagicMock(return_value=0.0)) patch_exchange(mocker) frame = _build_backtest_dataframe(data.data) @@ -158,11 +160,14 @@ def test_backtest_results(default_conf, fee, mocker, caplog, data) -> None: pair = 'UNITTEST/BTC' # Dummy data as we mock the analyze functions data_processed = {pair: DataFrame()} + min_date, max_date = get_timeframe({pair: frame}) results = backtesting.backtest( { 'stake_amount': default_conf['stake_amount'], 'processed': data_processed, 'max_open_trades': 10, + 'start_date': min_date, + 'end_date': max_date, } ) print(results.T) From 59cd4fe0ef735bb18d2d3ab484616921670176c0 Mon Sep 17 00:00:00 2001 From: Matthias Date: Fri, 9 Nov 2018 19:34:46 +0100 Subject: [PATCH 307/699] Remove boilerplate comments --- freqtrade/tests/optimize/test_backtest_detail.py | 13 +------------ 1 file changed, 1 insertion(+), 12 deletions(-) diff --git a/freqtrade/tests/optimize/test_backtest_detail.py b/freqtrade/tests/optimize/test_backtest_detail.py index 7db6913f3..e5849319e 100644 --- a/freqtrade/tests/optimize/test_backtest_detail.py +++ b/freqtrade/tests/optimize/test_backtest_detail.py @@ -174,18 +174,7 @@ def test_backtest_results(default_conf, fee, mocker, caplog, data) -> None: assert len(results) == len(data.trades) assert round(results["profit_percent"].sum(), 3) == round(data.profit_perc, 3) - # if data.sell_r == SellType.STOP_LOSS: - # assert log_has("Stop loss hit.", caplog.record_tuples) - # else: - # assert not log_has("Stop loss hit.", caplog.record_tuples) - # log_test = (f'Force_selling still open trade UNITTEST/BTC with ' - # f'{results.iloc[-1].profit_percent} perc - {results.iloc[-1].profit_abs}') - # if data.sell_r == SellType.FORCE_SELL: - # assert log_has(log_test, - # caplog.record_tuples) - # else: - # assert not log_has(log_test, - # caplog.record_tuples) + for c, trade in enumerate(data.trades): res = results.iloc[c] assert res.sell_reason == trade.sell_reason From b41633cfe362e0fd6cf847ee55f816c1c182216e Mon Sep 17 00:00:00 2001 From: Matthias Date: Fri, 9 Nov 2018 20:26:10 +0100 Subject: [PATCH 308/699] point out "good first issue" label --- CONTRIBUTING.md | 32 +++++++++++++++++--------------- README.md | 8 +++++++- 2 files changed, 24 insertions(+), 16 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 1c840fa31..58185b27c 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -2,50 +2,52 @@ ## Contribute to freqtrade -Feel like our bot is missing a feature? We welcome your pull requests! Few pointers for contributions: +Feel like our bot is missing a feature? We welcome your pull requests! + +Issues labeled [good first issue](https://github.com/freqtrade/freqtrade/labels/good%20first%20issue) can be good first contributions, and will help get you familiar with the codebase. + +Few pointers for contributions: - Create your PR against the `develop` branch, not `master`. -- New features need to contain unit tests and must be PEP8 - -conformant (max-line-length = 100). +- New features need to contain unit tests and must be PEP8 conformant (max-line-length = 100). If you are unsure, discuss the feature on our [Slack](https://join.slack.com/t/highfrequencybot/shared_invite/enQtMjQ5NTM0OTYzMzY3LWMxYzE3M2MxNDdjMGM3ZTYwNzFjMGIwZGRjNTc3ZGU3MGE3NzdmZGMwNmU3NDM5ZTNmM2Y3NjRiNzk4NmM4OGE) or in a [issue](https://github.com/freqtrade/freqtrade/issues) before a PR. -**Before sending the PR:** +## Before sending the PR: -## 1. Run unit tests +### 1. Run unit tests All unit tests must pass. If a unit test is broken, change your code to make it pass. It means you have introduced a regression. -### Test the whole project +#### Test the whole project ```bash pytest freqtrade ``` -### Test only one file +#### Test only one file ```bash pytest freqtrade/tests/test_.py ``` -### Test only one method from one file +#### Test only one method from one file ```bash pytest freqtrade/tests/test_.py::test_ ``` -## 2. Test if your code is PEP8 compliant +### 2. Test if your code is PEP8 compliant -### Install packages +#### Install packages ```bash pip3.6 install flake8 coveralls ``` -### Run Flake8 +#### Run Flake8 ```bash flake8 freqtrade @@ -56,15 +58,15 @@ To help with that, we encourage you to install the git pre-commit hook that will warn you when you try to commit code that fails these checks. Guide for installing them is [here](http://flake8.pycqa.org/en/latest/user/using-hooks.html). -## 3. Test if all type-hints are correct +### 3. Test if all type-hints are correct -### Install packages +#### Install packages ``` bash pip3.6 install mypy ``` -### Run mypy +#### Run mypy ``` bash mypy freqtrade diff --git a/README.md b/README.md index a46c1530f..571709e3b 100644 --- a/README.md +++ b/README.md @@ -62,6 +62,7 @@ hesitate to read the source code and understand the mechanism of this bot. - [Requirements](#requirements) - [Min hardware required](#min-hardware-required) - [Software requirements](#software-requirements) +- [Wanna help?] ## Quick start @@ -189,11 +190,15 @@ in the bug reports. ### [Pull Requests](https://github.com/freqtrade/freqtrade/pulls) -Feel like our bot is missing a feature? We welcome your pull requests! +Feel like our bot is missing a feature? We welcome your pull requests! + Please read our [Contributing document](https://github.com/freqtrade/freqtrade/blob/develop/CONTRIBUTING.md) to understand the requirements before sending your pull-requests. +Coding is not a neccessity to contribute - maybe start with improving our documentation? +Issues labeled [good first issue](https://github.com/freqtrade/freqtrade/labels/good%20first%20issue) can be good first contributions, and will help get you familiar with the codebase. + **Note** before starting any major new feature work, *please open an issue describing what you are planning to do* or talk to us on [Slack](https://join.slack.com/t/highfrequencybot/shared_invite/enQtMjQ5NTM0OTYzMzY3LWMxYzE3M2MxNDdjMGM3ZTYwNzFjMGIwZGRjNTc3ZGU3MGE3NzdmZGMwNmU3NDM5ZTNmM2Y3NjRiNzk4NmM4OGE). This will ensure that interested parties can give valuable feedback on the feature, and let others know that you are working on it. **Important:** Always create your PR against the `develop` branch, not `master`. @@ -218,3 +223,4 @@ To run this bot we recommend you a cloud instance with a minimum of: - [TA-Lib](https://mrjbq7.github.io/ta-lib/install.html) - [virtualenv](https://virtualenv.pypa.io/en/stable/installation/) (Recommended) - [Docker](https://www.docker.com/products/docker) (Recommended) + From 12e735e8316fc681ee0970f45b72f61fa17360fc Mon Sep 17 00:00:00 2001 From: misagh Date: Fri, 9 Nov 2018 20:51:15 +0100 Subject: [PATCH 309/699] 1) extracting edge_conf to a fixture 2) test cased adjusted to Backtesting 3) Formatted backtesting_details a bit --- freqtrade/tests/conftest.py | 35 ++- freqtrade/tests/edge/test_edge.py | 272 +++++++----------- .../tests/optimize/test_backtest_detail.py | 18 +- 3 files changed, 137 insertions(+), 188 deletions(-) diff --git a/freqtrade/tests/conftest.py b/freqtrade/tests/conftest.py index 37109767b..cbaca6c04 100644 --- a/freqtrade/tests/conftest.py +++ b/freqtrade/tests/conftest.py @@ -156,21 +156,6 @@ def default_conf(): "NEO/BTC" ] }, - "edge": { - "enabled": False, - "process_throttle_secs": 1800, - "calculate_since_number_of_days": 14, - "total_capital_in_stake_currency": 0.5, - "allowed_risk": 0.01, - "stoploss_range_min": -0.01, - "stoploss_range_max": -0.1, - "stoploss_range_step": -0.01, - "maximum_winrate": 0.80, - "minimum_expectancy": 0.20, - "min_trade_number": 15, - "max_trade_duration_minute": 1440, - "remove_pumps": True - }, "telegram": { "enabled": True, "token": "token", @@ -794,3 +779,23 @@ def buy_order_fee(): 'status': 'closed', 'fee': None } + +@pytest.fixture(scope="function") +def edge_conf(default_conf): + default_conf['edge'] = { + "enabled": False, + "process_throttle_secs": 1800, + "calculate_since_number_of_days": 14, + "total_capital_in_stake_currency": 0.5, + "allowed_risk": 0.01, + "stoploss_range_min": -0.01, + "stoploss_range_max": -0.1, + "stoploss_range_step": -0.01, + "maximum_winrate": 0.80, + "minimum_expectancy": 0.20, + "min_trade_number": 15, + "max_trade_duration_minute": 1440, + "remove_pumps": False + } + + return default_conf diff --git a/freqtrade/tests/edge/test_edge.py b/freqtrade/tests/edge/test_edge.py index 1ac5e62c4..aa45ddfff 100644 --- a/freqtrade/tests/edge/test_edge.py +++ b/freqtrade/tests/edge/test_edge.py @@ -1,17 +1,20 @@ -# pragma pylint: disable=missing-docstring, C0103 +# pragma pylint: disable=missing-docstring, C0103, C0330 # pragma pylint: disable=protected-access, too-many-lines, invalid-name, too-many-arguments +import pytest +import logging from freqtrade.tests.conftest import get_patched_freqtradebot from freqtrade.edge import Edge from pandas import DataFrame, to_datetime from freqtrade.strategy.interface import SellType +from freqtrade.tests.optimize import (BTrade, BTContainer, _build_backtest_dataframe, + _get_frame_time_from_offset) import arrow import numpy as np import math from unittest.mock import MagicMock - # Cases to be tested: # 1) Open trade should be removed from the end # 2) Two complete trades within dataframe (with sell hit for all) @@ -25,6 +28,101 @@ ticker_interval_in_minute = 60 _ohlc = {'date': 0, 'buy': 1, 'open': 2, 'high': 3, 'low': 4, 'close': 5, 'sell': 6, 'volume': 7} +# Open trade should be removed from the end +tc0 = BTContainer(data=[ + # D O H L C V B S + [0, 5000, 5025, 4975, 4987, 6172, 1, 0], + [1, 5000, 5025, 4975, 4987, 6172, 0, 1]], # enter trade (signal on last candle) + stop_loss=-0.99, roi=float('inf'), profit_perc=0.00, + trades=[] +) + +# Two complete trades within dataframe(with sell hit for all) +tc1 = BTContainer(data=[ + # D O H L C V B S + [0, 5000, 5025, 4975, 4987, 6172, 1, 0], + [1, 5000, 5025, 4975, 4987, 6172, 0, 1], # enter trade (signal on last candle) + [2, 5000, 5025, 4975, 4987, 6172, 0, 0], # exit at open + [3, 5000, 5025, 4975, 4987, 6172, 1, 0], # no action + [4, 5000, 5025, 4975, 4987, 6172, 0, 0], # should enter the trade + [5, 5000, 5025, 4975, 4987, 6172, 0, 1], # no action + [6, 5000, 5025, 4975, 4987, 6172, 0, 0], # should sell +], + stop_loss=-0.99, roi=float('inf'), profit_perc=0.00, + trades=[BTrade(sell_reason=SellType.SELL_SIGNAL, open_tick=1, close_tick=2), + BTrade(sell_reason=SellType.SELL_SIGNAL, open_tick=4, close_tick=6)] +) + +# 3) Entered, sl 1%, candle drops 8% => Trade closed, 1% loss +tc2 = BTContainer(data=[ + # D O H L C V B S + [0, 5000, 5025, 4975, 4987, 6172, 1, 0], + [1, 5000, 5025, 4600, 4987, 6172, 0, 0], # enter trade, stoploss hit + [2, 5000, 5025, 4975, 4987, 6172, 0, 0], +], + stop_loss=-0.01, roi=float('inf'), profit_perc=-0.01, + trades=[BTrade(sell_reason=SellType.STOP_LOSS, open_tick=1, close_tick=1)] +) + +# 4) Entered, sl 3 %, candle drops 4%, recovers to 1 % = > Trade closed, 3 % loss +tc3 = BTContainer(data=[ + # D O H L C V B S + [0, 5000, 5025, 4975, 4987, 6172, 1, 0], + [1, 5000, 5025, 4800, 4987, 6172, 0, 0], # enter trade, stoploss hit + [2, 5000, 5025, 4975, 4987, 6172, 0, 0], +], + stop_loss=-0.03, roi=float('inf'), profit_perc=-0.03, + trades=[BTrade(sell_reason=SellType.STOP_LOSS, open_tick=1, close_tick=1)] +) + +#5) Stoploss and sell are hit. should sell on stoploss +tc4=BTContainer(data = [ + # D O H L C V B S + [0, 5000, 5025, 4975, 4987, 6172, 1, 0], + [1, 5000, 5025, 4800, 4987, 6172, 0, 1], # enter trade, stoploss hit, sell signal + [2, 5000, 5025, 4975, 4987, 6172, 0, 0], +], + stop_loss = -0.03, roi = float('inf'), profit_perc = -0.03, + trades = [BTrade(sell_reason=SellType.STOP_LOSS, open_tick=1, close_tick=1)] +) + +TESTS = [ + tc0, + tc1, + tc2, + tc3, + tc4 +] + + +@pytest.mark.parametrize("data", TESTS) +def test_edge_results(edge_conf, mocker, caplog, data) -> None: + """ + run functional tests + """ + freqtrade = get_patched_freqtradebot(mocker, edge_conf) + edge = Edge(edge_conf, freqtrade.exchange, freqtrade.strategy) + frame = _build_backtest_dataframe(data.data) + caplog.set_level(logging.DEBUG) + edge.fee = 0 + + trades = edge._find_trades_for_stoploss_range(frame, 'TEST/BTC', [data.stop_loss]) + results = edge._fill_calculable_fields(DataFrame(trades)) if trades else DataFrame() + + print(results) + + assert len(trades) == len(data.trades) + + if not results.empty: + assert round(results["profit_percent"].sum(), 3) == round(data.profit_perc, 3) + + for c, trade in enumerate(data.trades): + res = results.iloc[c] + assert res.exit_type == trade.sell_reason + assert res.open_time == _get_frame_time_from_offset(trade.open_tick) + assert res.close_time == _get_frame_time_from_offset(trade.close_tick) + + def test_adjust(mocker, default_conf): freqtrade = get_patched_freqtradebot(mocker, default_conf) edge = Edge(default_conf, freqtrade.exchange, freqtrade.strategy) @@ -94,10 +192,10 @@ def _time_on_candle(number): minutes=(number * ticker_interval_in_minute)).timestamp * 1000, 'ms') -def test_edge_heartbeat_calculate(mocker, default_conf): - freqtrade = get_patched_freqtradebot(mocker, default_conf) - edge = Edge(default_conf, freqtrade.exchange, freqtrade.strategy) - heartbeat = default_conf['edge']['process_throttle_secs'] +def test_edge_heartbeat_calculate(mocker, edge_conf): + freqtrade = get_patched_freqtradebot(mocker, edge_conf) + edge = Edge(edge_conf, freqtrade.exchange, freqtrade.strategy) + heartbeat = edge_conf['edge']['process_throttle_secs'] # should not recalculate if heartbeat not reached edge._last_updated = arrow.utcnow().timestamp - heartbeat + 1 @@ -148,15 +246,15 @@ def test_edge_process_downloaded_data(mocker, default_conf): assert edge._last_updated <= arrow.utcnow().timestamp + 2 -def test_process_expectancy(mocker, default_conf): - default_conf['edge']['min_trade_number'] = 2 - freqtrade = get_patched_freqtradebot(mocker, default_conf) +def test_process_expectancy(mocker, edge_conf): + edge_conf['edge']['min_trade_number'] = 2 + freqtrade = get_patched_freqtradebot(mocker, edge_conf) def get_fee(): return 0.001 freqtrade.exchange.get_fee = get_fee - edge = Edge(default_conf, freqtrade.exchange, freqtrade.strategy) + edge = Edge(edge_conf, freqtrade.exchange, freqtrade.strategy) trades = [ {'pair': 'TEST/BTC', @@ -210,157 +308,3 @@ def test_process_expectancy(mocker, default_conf): assert round(final['TEST/BTC'].risk_reward_ratio, 10) == 306.5384615384 assert round(final['TEST/BTC'].required_risk_reward, 10) == 2.0 assert round(final['TEST/BTC'].expectancy, 10) == 101.5128205128 - -# 1) Open trade should be removed from the end - - -def test_case_1(mocker, default_conf): - freqtrade = get_patched_freqtradebot(mocker, default_conf) - edge = Edge(default_conf, freqtrade.exchange, freqtrade.strategy) - - stoploss = -0.99 # we don't want stoploss to be hit in this test - ticker = [ - # D=Date, B=Buy, O=Open, H=High, L=Low, C=Close, S=Sell - # D, B, O, H, L, C, S - [3, 1, 12, 25, 11, 20, 0], # -> - [4, 0, 20, 30, 19, 25, 1], # -> should enter the trade - ] - - ticker_df = _build_dataframe(ticker) - trades = edge._find_trades_for_stoploss_range(ticker_df, 'TEST/BTC', [stoploss]) - - # No trade should be found - assert len(trades) == 0 - - -# 2) Two complete trades within dataframe (with sell hit for all) -def test_case_2(mocker, default_conf): - freqtrade = get_patched_freqtradebot(mocker, default_conf) - edge = Edge(default_conf, freqtrade.exchange, freqtrade.strategy) - - stoploss = -0.99 # we don't want stoploss to be hit in this test - ticker = [ - # D=Date, B=Buy, O=Open, H=High, L=Low, C=Close, S=Sell - # D, B, O, H, L, C, S - [0, 1, 15, 20, 12, 17, 0], # -> no action - [1, 0, 17, 18, 13, 14, 1], # -> should enter the trade as B signal recieved on last candle - [2, 0, 14, 15, 11, 12, 0], # -> exit the trade as the sell signal recieved on last candle - [3, 1, 12, 25, 11, 20, 0], # -> no action - [4, 0, 20, 30, 19, 25, 0], # -> should enter the trade - [5, 0, 25, 27, 22, 26, 1], # -> no action - [6, 0, 26, 36, 25, 35, 0], # -> should sell - ] - - ticker_df = _build_dataframe(ticker) - trades = edge._find_trades_for_stoploss_range(ticker_df, 'TEST/BTC', [stoploss]) - - # Two trades must have occured - assert len(trades) == 2 - - # First trade check - assert trades[0]['open_time'] == _time_on_candle(1) - assert trades[0]['close_time'] == _time_on_candle(2) - assert trades[0]['open_rate'] == ticker[1][_ohlc['open']] - assert trades[0]['close_rate'] == ticker[2][_ohlc['open']] - assert trades[0]['exit_type'] == SellType.SELL_SIGNAL - ############################################################## - - # Second trade check - assert trades[1]['open_time'] == _time_on_candle(4) - assert trades[1]['close_time'] == _time_on_candle(6) - assert trades[1]['open_rate'] == ticker[4][_ohlc['open']] - assert trades[1]['close_rate'] == ticker[6][_ohlc['open']] - assert trades[1]['exit_type'] == SellType.SELL_SIGNAL - ############################################################## - - -# 3) Entered, sl 1%, candle drops 8% => Trade closed, 1% loss -def test_case_3(mocker, default_conf): - freqtrade = get_patched_freqtradebot(mocker, default_conf) - edge = Edge(default_conf, freqtrade.exchange, freqtrade.strategy) - - stoploss = -0.01 # we don't want stoploss to be hit in this test - ticker = [ - # D=Date, B=Buy, O=Open, H=High, L=Low, C=Close, S=Sell - # D, B, O, H, L, C, S - [0, 1, 15, 20, 12, 17, 0], # -> no action - [1, 0, 14, 15, 11, 12, 0], # -> enter to trade, stoploss hit - [2, 1, 12, 25, 11, 20, 0], # -> no action - ] - - ticker_df = _build_dataframe(ticker) - trades = edge._find_trades_for_stoploss_range(ticker_df, 'TEST/BTC', [stoploss]) - - # Two trades must have occured - assert len(trades) == 1 - - # First trade check - assert trades[0]['open_time'] == _time_on_candle(1) - assert trades[0]['close_time'] == _time_on_candle(1) - assert trades[0]['open_rate'] == ticker[1][_ohlc['open']] - assert trades[0]['close_rate'] == (stoploss + 1) * trades[0]['open_rate'] - assert trades[0]['exit_type'] == SellType.STOP_LOSS - ############################################################## - - -# 4) Entered, sl 3 %, candle drops 4%, recovers to 1 % = > Trade closed, 3 % loss -def test_case_4(mocker, default_conf): - freqtrade = get_patched_freqtradebot(mocker, default_conf) - edge = Edge(default_conf, freqtrade.exchange, freqtrade.strategy) - - stoploss = -0.03 # we don't want stoploss to be hit in this test - ticker = [ - # D=Date, B=Buy, O=Open, H=High, L=Low, C=Close, S=Sell - # D, B, O, H, L, C, S - [0, 1, 15, 20, 12, 17, 0], # -> no action - [1, 0, 17, 22, 16.90, 17, 0], # -> enter to trade - [2, 0, 16, 17, 14.4, 15.5, 0], # -> stoploss hit - [3, 0, 17, 25, 16.9, 22, 0], # -> no action - ] - - ticker_df = _build_dataframe(ticker) - trades = edge._find_trades_for_stoploss_range(ticker_df, 'TEST/BTC', [stoploss]) - - # Two trades must have occured - assert len(trades) == 1 - - # First trade check - assert trades[0]['open_time'] == _time_on_candle(1) - assert trades[0]['close_time'] == _time_on_candle(2) - assert trades[0]['open_rate'] == ticker[1][_ohlc['open']] - assert trades[0]['close_rate'] == (stoploss + 1) * trades[0]['open_rate'] - assert trades[0]['exit_type'] == SellType.STOP_LOSS - ############################################################## - - -# 5) Stoploss and sell are hit. should sell on stoploss -def test_case_5(mocker, default_conf): - freqtrade = get_patched_freqtradebot(mocker, default_conf) - edge = Edge(default_conf, freqtrade.exchange, freqtrade.strategy) - - freqtrade = get_patched_freqtradebot(mocker, default_conf) - edge = Edge(default_conf, freqtrade.exchange, freqtrade.strategy) - - stoploss = -0.03 # we don't want stoploss to be hit in this test - ticker = [ - # D=Date, B=Buy, O=Open, H=High, L=Low, C=Close, S=Sell - # D, B, O, H, L, C, S - [0, 1, 15, 20, 12, 17, 0], # -> no action - [1, 0, 17, 22, 16.90, 17, 0], # -> enter to trade - [2, 0, 16, 17, 14.4, 15.5, 1], # -> stoploss hit and also sell signal - [3, 0, 17, 25, 16.9, 22, 0], # -> no action - ] - - ticker_df = _build_dataframe(ticker) - trades = edge._find_trades_for_stoploss_range(ticker_df, 'TEST/BTC', [stoploss]) - - # Two trades must have occured - assert len(trades) == 1 - - # First trade check - assert trades[0]['open_time'] == _time_on_candle(1) - assert trades[0]['close_time'] == _time_on_candle(2) - assert trades[0]['open_rate'] == ticker[1][_ohlc['open']] - assert trades[0]['close_rate'] == (stoploss + 1) * trades[0]['open_rate'] - assert trades[0]['exit_type'] == SellType.STOP_LOSS - ############################################################## diff --git a/freqtrade/tests/optimize/test_backtest_detail.py b/freqtrade/tests/optimize/test_backtest_detail.py index 806c136bc..eaec3bf49 100644 --- a/freqtrade/tests/optimize/test_backtest_detail.py +++ b/freqtrade/tests/optimize/test_backtest_detail.py @@ -1,4 +1,4 @@ -# pragma pylint: disable=missing-docstring, W0212, line-too-long, C0103, unused-argument +# pragma pylint: disable=missing-docstring, W0212, line-too-long, C0103, C0330, unused-argument import logging from unittest.mock import MagicMock @@ -34,15 +34,15 @@ tc0 = BTContainer(data=[ # TC2: Stop-Loss Triggered 3% Loss tc1 = BTContainer(data=[ # D O H L C V B S - [0, 5000, 5025, 4975, 4987, 6172, 1, 0], - [1, 5000, 5025, 4975, 4987, 6172, 0, 0], # enter trade (signal on last candle) - [2, 4987, 5012, 4962, 4975, 6172, 0, 0], - [3, 4975, 5000, 4800, 4962, 6172, 0, 0], # exit with stoploss hit - [4, 4962, 4987, 4937, 4950, 6172, 0, 0], - [5, 4950, 4975, 4925, 4950, 6172, 0, 0]], + [0, 5000, 5025, 4975, 4987, 6172, 1, 0], + [1, 5000, 5025, 4975, 4987, 6172, 0, 0], # enter trade (signal on last candle) + [2, 4987, 5012, 4962, 4975, 6172, 0, 0], + [3, 4975, 5000, 4800, 4962, 6172, 0, 0], # exit with stoploss hit + [4, 4962, 4987, 4937, 4950, 6172, 0, 0], + [5, 4950, 4975, 4925, 4950, 6172, 0, 0]], stop_loss=-0.03, roi=1, profit_perc=-0.03, trades=[BTrade(sell_reason=SellType.STOP_LOSS, open_tick=1, close_tick=3)] - ) +) # Test 3 Candle drops 4%, Recovers 1%. @@ -127,7 +127,7 @@ tc6 = BTContainer(data=[ [5, 4950, 4975, 4925, 4950, 6172, 0, 0]], stop_loss=-0.02, roi=0.03, profit_perc=0.03, trades=[BTrade(sell_reason=SellType.ROI, open_tick=1, close_tick=2)] - ) +) TESTS = [ tc0, From 617a58402f82b4cb19623a6d7fe7260a901eadfb Mon Sep 17 00:00:00 2001 From: misagh Date: Fri, 9 Nov 2018 20:52:03 +0100 Subject: [PATCH 310/699] putting edge adjust function in _process not in create_trade --- freqtrade/freqtradebot.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index babe0a1da..4a17e889e 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -192,6 +192,7 @@ class FreqtradeBot(object): # with delta value (klines only from last refresh_pairs) if self.edge: self.edge.calculate() + self.active_pair_whitelist = self.edge.adjust(self.active_pair_whitelist) # Refreshing candles self.exchange.refresh_tickers(self.active_pair_whitelist, self.strategy.ticker_interval) @@ -406,10 +407,6 @@ class FreqtradeBot(object): raise DependencyException('No currency pairs in whitelist') # running get_signal on historical data fetched - # to find buy signals - if self.edge: - whitelist = self.edge.adjust(whitelist) - for _pair in whitelist: (buy, sell) = self.strategy.get_signal(_pair, interval, self.exchange.klines.get(_pair)) if buy and not sell: From 4dcd15da1d7e393f593b78548f525511df6986c8 Mon Sep 17 00:00:00 2001 From: misagh Date: Fri, 9 Nov 2018 20:59:28 +0100 Subject: [PATCH 311/699] improving documentation for positioning --- docs/edge.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/docs/edge.md b/docs/edge.md index 5cace4ea3..dc4fda72a 100644 --- a/docs/edge.md +++ b/docs/edge.md @@ -88,6 +88,10 @@ Your position size then will be: **position size** = **allowed capital at risk** / **stoploss** +Example: +Let's say your total capital is 3 ETH, you would allow 1% of risk for each trade. thus your allowed capital at risk would be **3 x 0.01 = 0.03 ETH**. Let's assume Edge has calculated that for **XLM/ETH** market your stoploss should be at 2%. So your position size will be **0.03 / 0.02= 1.5ETH**.
+**Notice:** if Edge is enabled, the stake_amount config is overriden by total_capital_in_stake_currency config explained below (see configuration part) + ## Configurations Edge has following configurations: From 0f2ddbbef2f521ec0c791e55884d1d761f5f30c5 Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Sat, 10 Nov 2018 13:34:06 +0100 Subject: [PATCH 312/699] Update ccxt from 1.17.485 to 1.17.488 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 4e38f66de..4007cc0ac 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -ccxt==1.17.485 +ccxt==1.17.488 SQLAlchemy==1.2.13 python-telegram-bot==11.1.0 arrow==0.12.1 From 523a9a603cedb0df6e5672937cc2d135b4dbf379 Mon Sep 17 00:00:00 2001 From: misagh Date: Sat, 10 Nov 2018 17:20:11 +0100 Subject: [PATCH 313/699] fix tests --- freqtrade/tests/conftest.py | 3 ++- freqtrade/tests/edge/test_edge.py | 8 ++++---- freqtrade/tests/optimize/__init__.py | 2 +- freqtrade/tests/test_freqtradebot.py | 27 ++++++++++----------------- 4 files changed, 17 insertions(+), 23 deletions(-) diff --git a/freqtrade/tests/conftest.py b/freqtrade/tests/conftest.py index cbaca6c04..b93140860 100644 --- a/freqtrade/tests/conftest.py +++ b/freqtrade/tests/conftest.py @@ -780,10 +780,11 @@ def buy_order_fee(): 'fee': None } + @pytest.fixture(scope="function") def edge_conf(default_conf): default_conf['edge'] = { - "enabled": False, + "enabled": True, "process_throttle_secs": 1800, "calculate_since_number_of_days": 14, "total_capital_in_stake_currency": 0.5, diff --git a/freqtrade/tests/edge/test_edge.py b/freqtrade/tests/edge/test_edge.py index aa45ddfff..a7b1882a5 100644 --- a/freqtrade/tests/edge/test_edge.py +++ b/freqtrade/tests/edge/test_edge.py @@ -75,15 +75,15 @@ tc3 = BTContainer(data=[ trades=[BTrade(sell_reason=SellType.STOP_LOSS, open_tick=1, close_tick=1)] ) -#5) Stoploss and sell are hit. should sell on stoploss -tc4=BTContainer(data = [ +# 5) Stoploss and sell are hit. should sell on stoploss +tc4 = BTContainer(data=[ # D O H L C V B S [0, 5000, 5025, 4975, 4987, 6172, 1, 0], [1, 5000, 5025, 4800, 4987, 6172, 0, 1], # enter trade, stoploss hit, sell signal [2, 5000, 5025, 4975, 4987, 6172, 0, 0], ], - stop_loss = -0.03, roi = float('inf'), profit_perc = -0.03, - trades = [BTrade(sell_reason=SellType.STOP_LOSS, open_tick=1, close_tick=1)] + stop_loss=-0.03, roi=float('inf'), profit_perc=-0.03, + trades=[BTrade(sell_reason=SellType.STOP_LOSS, open_tick=1, close_tick=1)] ) TESTS = [ diff --git a/freqtrade/tests/optimize/__init__.py b/freqtrade/tests/optimize/__init__.py index 2b7222e88..3d3066950 100644 --- a/freqtrade/tests/optimize/__init__.py +++ b/freqtrade/tests/optimize/__init__.py @@ -31,7 +31,7 @@ class BTContainer(NamedTuple): def _get_frame_time_from_offset(offset): return ticker_start_time.shift( - minutes=(offset * ticker_interval_in_minute)).datetime + minutes=(offset * ticker_interval_in_minute)).datetime.replace(tzinfo=None) def _build_backtest_dataframe(ticker_with_signals): diff --git a/freqtrade/tests/test_freqtradebot.py b/freqtrade/tests/test_freqtradebot.py index 9c72f9177..907d228d6 100644 --- a/freqtrade/tests/test_freqtradebot.py +++ b/freqtrade/tests/test_freqtradebot.py @@ -246,12 +246,11 @@ def test_get_trade_stake_amount_unlimited_amount(default_conf, assert result is None -def test_edge_overrides_stake_amount(mocker, default_conf) -> None: - default_conf['edge']['enabled'] = True +def test_edge_overrides_stake_amount(mocker, edge_conf) -> None: patch_RPCManager(mocker) patch_exchange(mocker) patch_edge(mocker) - freqtrade = FreqtradeBot(default_conf) + freqtrade = FreqtradeBot(edge_conf) # strategy stoploss should be ignored freqtrade.strategy.stoploss = -0.05 @@ -261,14 +260,8 @@ def test_edge_overrides_stake_amount(mocker, default_conf) -> None: assert freqtrade._get_trade_stake_amount('LTC/BTC') == 0.02381 -def test_edge_overrides_stoploss( - limit_buy_order, - fee, - markets, - caplog, - mocker, - default_conf) -> None: - default_conf['edge']['enabled'] = True +def test_edge_overrides_stoploss(limit_buy_order, fee, markets, caplog, mocker, edge_conf) -> None: + patch_RPCManager(mocker) patch_exchange(mocker) patch_edge(mocker) @@ -292,7 +285,8 @@ def test_edge_overrides_stoploss( ############################################# # Create a trade with "limit_buy_order" price - freqtrade = FreqtradeBot(default_conf) + freqtrade = FreqtradeBot(edge_conf) + freqtrade.active_pair_whitelist = ['NEO/BTC'] patch_get_signal(freqtrade) freqtrade.strategy.min_roi_reached = lambda trade, current_profit, current_time: False freqtrade.create_trade() @@ -302,14 +296,12 @@ def test_edge_overrides_stoploss( # stoploss shoud be hit assert freqtrade.handle_trade(trade) is True - assert log_has('executed sell, reason: SellType.STOP_LOSS', caplog.record_tuples) assert trade.sell_reason == SellType.STOP_LOSS.value def test_edge_should_ignore_strategy_stoploss(limit_buy_order, fee, markets, - mocker, default_conf) -> None: - default_conf['edge']['enabled'] = True + mocker, edge_conf) -> None: patch_RPCManager(mocker) patch_exchange(mocker) patch_edge(mocker) @@ -333,7 +325,8 @@ def test_edge_should_ignore_strategy_stoploss(limit_buy_order, fee, markets, ############################################# # Create a trade with "limit_buy_order" price - freqtrade = FreqtradeBot(default_conf) + freqtrade = FreqtradeBot(edge_conf) + freqtrade.active_pair_whitelist = ['NEO/BTC'] patch_get_signal(freqtrade) freqtrade.strategy.min_roi_reached = lambda trade, current_profit, current_time: False freqtrade.create_trade() @@ -341,7 +334,7 @@ def test_edge_should_ignore_strategy_stoploss(limit_buy_order, fee, markets, trade.update(limit_buy_order) ############################################# - # stoploss shoud be hit + # stoploss shoud not be hit assert freqtrade.handle_trade(trade) is False From 97fd33d75267941102faa7a2ebb75defedb99cae Mon Sep 17 00:00:00 2001 From: misagh Date: Sat, 10 Nov 2018 18:03:46 +0100 Subject: [PATCH 314/699] adding test for process --- freqtrade/tests/test_freqtradebot.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/freqtrade/tests/test_freqtradebot.py b/freqtrade/tests/test_freqtradebot.py index 907d228d6..e83163a30 100644 --- a/freqtrade/tests/test_freqtradebot.py +++ b/freqtrade/tests/test_freqtradebot.py @@ -246,6 +246,20 @@ def test_get_trade_stake_amount_unlimited_amount(default_conf, assert result is None +def test_edge_called_in_process(mocker, edge_conf) -> None: + patch_RPCManager(mocker) + patch_edge(mocker) + def _refresh_whitelist(list): + return ['ETH/BTC', 'LTC/BTC', 'XRP/BTC', 'NEO/BTC'] + + patch_exchange(mocker) + freqtrade = FreqtradeBot(edge_conf) + freqtrade._refresh_whitelist = _refresh_whitelist + patch_get_signal(freqtrade) + freqtrade._process() + assert freqtrade.active_pair_whitelist == ['NEO/BTC', 'LTC/BTC'] + + def test_edge_overrides_stake_amount(mocker, edge_conf) -> None: patch_RPCManager(mocker) patch_exchange(mocker) From 7dd74c374a5bc4ca6897768edff6c7bee5dd8e1f Mon Sep 17 00:00:00 2001 From: misagh Date: Sat, 10 Nov 2018 18:09:32 +0100 Subject: [PATCH 315/699] flake happiness provided --- freqtrade/tests/test_freqtradebot.py | 1 + 1 file changed, 1 insertion(+) diff --git a/freqtrade/tests/test_freqtradebot.py b/freqtrade/tests/test_freqtradebot.py index e83163a30..66c5d6a1c 100644 --- a/freqtrade/tests/test_freqtradebot.py +++ b/freqtrade/tests/test_freqtradebot.py @@ -249,6 +249,7 @@ def test_get_trade_stake_amount_unlimited_amount(default_conf, def test_edge_called_in_process(mocker, edge_conf) -> None: patch_RPCManager(mocker) patch_edge(mocker) + def _refresh_whitelist(list): return ['ETH/BTC', 'LTC/BTC', 'XRP/BTC', 'NEO/BTC'] From d61355330679a8e45949c141b739261ba426440b Mon Sep 17 00:00:00 2001 From: misagh Date: Sat, 10 Nov 2018 18:22:34 +0100 Subject: [PATCH 316/699] base position on stake amount instead of total capital --- freqtrade/edge/__init__.py | 2 +- freqtrade/tests/test_freqtradebot.py | 8 ++------ 2 files changed, 3 insertions(+), 7 deletions(-) diff --git a/freqtrade/edge/__init__.py b/freqtrade/edge/__init__.py index a468a76a3..31d70add1 100644 --- a/freqtrade/edge/__init__.py +++ b/freqtrade/edge/__init__.py @@ -49,7 +49,7 @@ class Edge(): self.edge_config = self.config.get('edge', {}) self._cached_pairs: Dict[str, Any] = {} # Keeps a list of pairs - self._total_capital: float = self.edge_config.get('total_capital_in_stake_currency') + self._total_capital: float = self.config.get('stake_amount') self._allowed_risk: float = self.edge_config.get('allowed_risk') self._since_number_of_days: int = self.edge_config.get('calculate_since_number_of_days', 14) self._last_updated: int = 0 # Timestamp of pairs last updated time diff --git a/freqtrade/tests/test_freqtradebot.py b/freqtrade/tests/test_freqtradebot.py index 66c5d6a1c..4cba9e308 100644 --- a/freqtrade/tests/test_freqtradebot.py +++ b/freqtrade/tests/test_freqtradebot.py @@ -267,12 +267,8 @@ def test_edge_overrides_stake_amount(mocker, edge_conf) -> None: patch_edge(mocker) freqtrade = FreqtradeBot(edge_conf) - # strategy stoploss should be ignored - freqtrade.strategy.stoploss = -0.05 - - assert 'ETH/BTC' not in freqtrade.edge._cached_pairs - assert freqtrade._get_trade_stake_amount('NEO/BTC') == 0.025 - assert freqtrade._get_trade_stake_amount('LTC/BTC') == 0.02381 + assert freqtrade._get_trade_stake_amount('NEO/BTC') == (0.001 * 0.01) / 0.20 + assert freqtrade._get_trade_stake_amount('LTC/BTC') == (0.001 * 0.01) / 0.20 def test_edge_overrides_stoploss(limit_buy_order, fee, markets, caplog, mocker, edge_conf) -> None: From aacc1d5004004e47627de66739167a963418cecc Mon Sep 17 00:00:00 2001 From: misagh Date: Sat, 10 Nov 2018 18:28:05 +0100 Subject: [PATCH 317/699] removing total capital in favour of stake amount --- config_full.json.example | 1 - docs/edge.md | 8 +++----- freqtrade/constants.py | 1 - freqtrade/tests/conftest.py | 2 +- 4 files changed, 4 insertions(+), 8 deletions(-) diff --git a/config_full.json.example b/config_full.json.example index a7fdd5bab..9dba8f539 100644 --- a/config_full.json.example +++ b/config_full.json.example @@ -63,7 +63,6 @@ "enabled": false, "process_throttle_secs": 3600, "calculate_since_number_of_days": 2, - "total_capital_in_stake_currency": 0.5, "allowed_risk": 0.01, "stoploss_range_min": -0.01, "stoploss_range_max": -0.1, diff --git a/docs/edge.md b/docs/edge.md index dc4fda72a..cfb95b936 100644 --- a/docs/edge.md +++ b/docs/edge.md @@ -82,6 +82,8 @@ Allowed capital at risk is calculated as follows: **allowed capital at risk** = **total capital** X **allowed risk per trade** +**total capital** is your stake amount. + **Stoploss** is calculated as described above against historical data. Your position size then will be: @@ -89,8 +91,7 @@ Your position size then will be: **position size** = **allowed capital at risk** / **stoploss** Example: -Let's say your total capital is 3 ETH, you would allow 1% of risk for each trade. thus your allowed capital at risk would be **3 x 0.01 = 0.03 ETH**. Let's assume Edge has calculated that for **XLM/ETH** market your stoploss should be at 2%. So your position size will be **0.03 / 0.02= 1.5ETH**.
-**Notice:** if Edge is enabled, the stake_amount config is overriden by total_capital_in_stake_currency config explained below (see configuration part) +Let's say your stake amount is 3 ETH, you would allow 1% of risk for each trade. thus your allowed capital at risk would be **3 x 0.01 = 0.03 ETH**. Let's assume Edge has calculated that for **XLM/ETH** market your stoploss should be at 2%. So your position size will be **0.03 / 0.02= 1.5ETH**.
## Configurations Edge has following configurations: @@ -108,9 +109,6 @@ Number of days of data against which Edge calculates Win Rate, Risk Reward and E Note that it downloads historical data so increasing this number would lead to slowing down the bot
(default to 7) -#### total_capital_in_stake_currency -This your total capital at risk in your stake currency. If edge is enabled then stake_amount is ignored in favor of this parameter - #### allowed_risk Percentage of allowed risk per trade
(default to 1%) diff --git a/freqtrade/constants.py b/freqtrade/constants.py index d37e78687..b7c069c45 100644 --- a/freqtrade/constants.py +++ b/freqtrade/constants.py @@ -178,7 +178,6 @@ CONF_SCHEMA = { "enabled": {'type': 'boolean'}, "process_throttle_secs": {'type': 'integer', 'minimum': 600}, "calculate_since_number_of_days": {'type': 'integer'}, - "total_capital_in_stake_currency": {'type': 'number'}, "allowed_risk": {'type': 'number'}, "stoploss_range_min": {'type': 'number'}, "stoploss_range_max": {'type': 'number'}, diff --git a/freqtrade/tests/conftest.py b/freqtrade/tests/conftest.py index b93140860..487d0d150 100644 --- a/freqtrade/tests/conftest.py +++ b/freqtrade/tests/conftest.py @@ -787,7 +787,7 @@ def edge_conf(default_conf): "enabled": True, "process_throttle_secs": 1800, "calculate_since_number_of_days": 14, - "total_capital_in_stake_currency": 0.5, + #"total_capital_in_stake_currency": 0.5, "allowed_risk": 0.01, "stoploss_range_min": -0.01, "stoploss_range_max": -0.1, From 9bbaeb4e6f59aa89f9c314c75114ef1722c32a71 Mon Sep 17 00:00:00 2001 From: misagh Date: Sat, 10 Nov 2018 18:39:49 +0100 Subject: [PATCH 318/699] mypy expression --- freqtrade/edge/__init__.py | 2 +- freqtrade/tests/conftest.py | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/freqtrade/edge/__init__.py b/freqtrade/edge/__init__.py index 31d70add1..7b25a8306 100644 --- a/freqtrade/edge/__init__.py +++ b/freqtrade/edge/__init__.py @@ -49,7 +49,7 @@ class Edge(): self.edge_config = self.config.get('edge', {}) self._cached_pairs: Dict[str, Any] = {} # Keeps a list of pairs - self._total_capital: float = self.config.get('stake_amount') + self._total_capital: float = self.config['stake_amount'] self._allowed_risk: float = self.edge_config.get('allowed_risk') self._since_number_of_days: int = self.edge_config.get('calculate_since_number_of_days', 14) self._last_updated: int = 0 # Timestamp of pairs last updated time diff --git a/freqtrade/tests/conftest.py b/freqtrade/tests/conftest.py index 487d0d150..8a497725f 100644 --- a/freqtrade/tests/conftest.py +++ b/freqtrade/tests/conftest.py @@ -787,7 +787,6 @@ def edge_conf(default_conf): "enabled": True, "process_throttle_secs": 1800, "calculate_since_number_of_days": 14, - #"total_capital_in_stake_currency": 0.5, "allowed_risk": 0.01, "stoploss_range_min": -0.01, "stoploss_range_max": -0.1, From 02527eeea404f5b461b657238a34b3e47712e3ae Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 10 Nov 2018 20:07:09 +0100 Subject: [PATCH 319/699] Add rpc_whitelist call --- freqtrade/rpc/rpc.py | 7 +++++++ freqtrade/tests/rpc/test_rpc.py | 25 +++++++++++++++++++++++++ 2 files changed, 32 insertions(+) diff --git a/freqtrade/rpc/rpc.py b/freqtrade/rpc/rpc.py index c3cbce2e7..cf283c6b1 100644 --- a/freqtrade/rpc/rpc.py +++ b/freqtrade/rpc/rpc.py @@ -443,3 +443,10 @@ class RPC(object): raise RPCException('trader is not running') return Trade.query.filter(Trade.is_open.is_(True)).all() + + def _rpc_whitelist(self) -> Dict: + """ Returns the currently active whitelist""" + res = {'method': self._freqtrade.config.get('dynamic_whitelist', 0) or 'static', + 'whitelist': self._freqtrade.active_pair_whitelist + } + return res diff --git a/freqtrade/tests/rpc/test_rpc.py b/freqtrade/tests/rpc/test_rpc.py index 19692db50..ff72ef634 100644 --- a/freqtrade/tests/rpc/test_rpc.py +++ b/freqtrade/tests/rpc/test_rpc.py @@ -645,3 +645,28 @@ def test_rpcforcebuy_disabled(mocker, default_conf) -> None: pair = 'ETH/BTC' with pytest.raises(RPCException, match=r'Forcebuy not enabled.'): rpc._rpc_forcebuy(pair, None) + + +def test_rpc_whitelist(mocker, default_conf) -> None: + patch_coinmarketcap(mocker) + patch_exchange(mocker) + mocker.patch('freqtrade.rpc.telegram.Telegram', MagicMock()) + + freqtradebot = FreqtradeBot(default_conf) + rpc = RPC(freqtradebot) + ret = rpc._rpc_whitelist() + assert ret['method'] == 'static' + assert ret['whitelist'] == default_conf['exchange']['pair_whitelist'] + + +def test_rpc_whitelist_dynamic(mocker, default_conf) -> None: + patch_coinmarketcap(mocker) + patch_exchange(mocker) + default_conf['dynamic_whitelist'] = 4 + mocker.patch('freqtrade.rpc.telegram.Telegram', MagicMock()) + + freqtradebot = FreqtradeBot(default_conf) + rpc = RPC(freqtradebot) + ret = rpc._rpc_whitelist() + assert ret['method'] == 4 + assert ret['whitelist'] == default_conf['exchange']['pair_whitelist'] From 62402351b38a24964aa82a9481b4433825289ba0 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 10 Nov 2018 20:14:46 +0100 Subject: [PATCH 320/699] Clarify volume selection for dynamic whitelist --- freqtrade/freqtradebot.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index 51160332d..b63329ac5 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -133,7 +133,7 @@ class FreqtradeBot(object): f'*Strategy:* `{strategy_name}`' }) if self.config.get('dynamic_whitelist', False): - top_pairs = 'top ' + str(self.config.get('dynamic_whitelist', 20)) + top_pairs = 'top volume ' + str(self.config.get('dynamic_whitelist', 20)) specific_pairs = '' else: top_pairs = 'whitelisted' From 08ef2730a94595597e059843bcb573dea79f1b0f Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 10 Nov 2018 20:15:06 +0100 Subject: [PATCH 321/699] Add /whitelist call to telegram --- freqtrade/rpc/telegram.py | 20 ++++++++++++ freqtrade/tests/rpc/test_rpc_telegram.py | 40 +++++++++++++++++++++++- 2 files changed, 59 insertions(+), 1 deletion(-) diff --git a/freqtrade/rpc/telegram.py b/freqtrade/rpc/telegram.py index eaabd35c6..43aabf1dc 100644 --- a/freqtrade/rpc/telegram.py +++ b/freqtrade/rpc/telegram.py @@ -91,6 +91,7 @@ class Telegram(RPC): CommandHandler('daily', self._daily), CommandHandler('count', self._count), CommandHandler('reload_conf', self._reload_conf), + CommandHandler('whitelist', self._whitelist), CommandHandler('help', self._help), CommandHandler('version', self._version), ] @@ -438,6 +439,25 @@ class Telegram(RPC): except RPCException as e: self._send_msg(str(e), bot=bot) + @authorized_only + def _whitelist(self, bot: Bot, update: Update) -> None: + """ + Handler for /whitelist + Shows the currently active whitelist + """ + try: + whitelist = self._rpc_whitelist() + if whitelist['method'] == 'static': + message = f"Using static whitelist with `{len(whitelist['whitelist'])}` pairs \n" + else: + message = f"Dynamic whitelist with `{whitelist['method']}` pairs\n" + message += f"`{', '.join(whitelist['whitelist'])}`" + + logger.debug(message) + self._send_msg(message) + except RPCException as e: + self._send_msg(str(e), bot=bot) + @authorized_only def _help(self, bot: Bot, update: Update) -> None: """ diff --git a/freqtrade/tests/rpc/test_rpc_telegram.py b/freqtrade/tests/rpc/test_rpc_telegram.py index 097fc1ff2..a8e8bf003 100644 --- a/freqtrade/tests/rpc/test_rpc_telegram.py +++ b/freqtrade/tests/rpc/test_rpc_telegram.py @@ -72,7 +72,8 @@ def test_init(default_conf, mocker, caplog) -> None: message_str = "rpc.telegram is listening for following commands: [['status'], ['profit'], " \ "['balance'], ['start'], ['stop'], ['forcesell'], ['forcebuy'], " \ - "['performance'], ['daily'], ['count'], ['reload_conf'], ['help'], ['version']]" + "['performance'], ['daily'], ['count'], ['reload_conf'], " \ + "['whitelist'], ['help'], ['version']]" assert log_has(message_str, caplog.record_tuples) @@ -1006,6 +1007,43 @@ def test_count_handle(default_conf, update, ticker, fee, markets, mocker) -> Non assert msg in msg_mock.call_args_list[0][0][0] +def test_whitelist_static(default_conf, update, mocker) -> None: + patch_coinmarketcap(mocker) + msg_mock = MagicMock() + mocker.patch.multiple( + 'freqtrade.rpc.telegram.Telegram', + _init=MagicMock(), + _send_msg=msg_mock + ) + freqtradebot = get_patched_freqtradebot(mocker, default_conf) + + telegram = Telegram(freqtradebot) + + telegram._whitelist(bot=MagicMock(), update=update) + assert msg_mock.call_count == 1 + assert ('Using static whitelist with `4` pairs \n`ETH/BTC, LTC/BTC, XRP/BTC, NEO/BTC`' + in msg_mock.call_args_list[0][0][0]) + + +def test_whitelist_dynamic(default_conf, update, mocker) -> None: + patch_coinmarketcap(mocker) + msg_mock = MagicMock() + mocker.patch.multiple( + 'freqtrade.rpc.telegram.Telegram', + _init=MagicMock(), + _send_msg=msg_mock + ) + default_conf['dynamic_whitelist'] = 4 + freqtradebot = get_patched_freqtradebot(mocker, default_conf) + + telegram = Telegram(freqtradebot) + + telegram._whitelist(bot=MagicMock(), update=update) + assert msg_mock.call_count == 1 + assert ('Dynamic whitelist with `4` pairs\n`ETH/BTC, LTC/BTC, XRP/BTC, NEO/BTC`' + in msg_mock.call_args_list[0][0][0]) + + def test_help_handle(default_conf, update, mocker) -> None: patch_coinmarketcap(mocker) msg_mock = MagicMock() From 060a1b3fbcf94269bf7d2d12b4a960a7b704bc59 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 10 Nov 2018 20:16:20 +0100 Subject: [PATCH 322/699] Add /whitelist to help message --- freqtrade/rpc/telegram.py | 1 + 1 file changed, 1 insertion(+) diff --git a/freqtrade/rpc/telegram.py b/freqtrade/rpc/telegram.py index 43aabf1dc..55c5abef2 100644 --- a/freqtrade/rpc/telegram.py +++ b/freqtrade/rpc/telegram.py @@ -480,6 +480,7 @@ class Telegram(RPC): "\n" \ "*/balance:* `Show account balance per currency`\n" \ "*/reload_conf:* `Reload configuration file` \n" \ + "*/whitelist:* `Show current whitelist` \n" \ "*/help:* `This help message`\n" \ "*/version:* `Show version`" From ef2c31b5434d0822af6a7f7219fb046c96fa5905 Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Sun, 11 Nov 2018 13:34:06 +0100 Subject: [PATCH 323/699] Update ccxt from 1.17.488 to 1.17.489 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 4007cc0ac..ecc6cb682 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -ccxt==1.17.488 +ccxt==1.17.489 SQLAlchemy==1.2.13 python-telegram-bot==11.1.0 arrow==0.12.1 From 261cd7746bb21b9c4071669ce24ad00c5786e39a Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Sun, 11 Nov 2018 13:34:07 +0100 Subject: [PATCH 324/699] Update sqlalchemy from 1.2.13 to 1.2.14 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index ecc6cb682..f0fe0e05d 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,5 @@ ccxt==1.17.489 -SQLAlchemy==1.2.13 +SQLAlchemy==1.2.14 python-telegram-bot==11.1.0 arrow==0.12.1 cachetools==3.0.0 From c29543dd6c74bee082c3da42d82d67e93244ec85 Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Mon, 12 Nov 2018 13:34:07 +0100 Subject: [PATCH 325/699] Update ccxt from 1.17.489 to 1.17.491 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index f0fe0e05d..b48679838 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -ccxt==1.17.489 +ccxt==1.17.491 SQLAlchemy==1.2.14 python-telegram-bot==11.1.0 arrow==0.12.1 From 028139fa3aea004a5af5b97410280ef382b90383 Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Mon, 12 Nov 2018 13:34:08 +0100 Subject: [PATCH 326/699] Update pytest from 3.10.0 to 3.10.1 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index b48679838..f79e425b6 100644 --- a/requirements.txt +++ b/requirements.txt @@ -12,7 +12,7 @@ scipy==1.1.0 jsonschema==2.6.0 numpy==1.15.4 TA-Lib==0.4.17 -pytest==3.10.0 +pytest==3.10.1 pytest-mock==1.10.0 pytest-asyncio==0.9.0 pytest-cov==2.6.0 From bdba6186d8e9d4408fbd7155922cf969e3e4f678 Mon Sep 17 00:00:00 2001 From: Matthias Date: Mon, 12 Nov 2018 19:43:20 +0100 Subject: [PATCH 327/699] Fix doc-typos --- docs/edge.md | 4 ++-- docs/index.md | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/edge.md b/docs/edge.md index cfb95b936..f74f8fdcc 100644 --- a/docs/edge.md +++ b/docs/edge.md @@ -111,7 +111,7 @@ Note that it downloads historical data so increasing this number would lead to s #### allowed_risk Percentage of allowed risk per trade
-(default to 1%) +(default to 0.01 [1%]) #### stoploss_range_min Minimum stoploss
@@ -148,4 +148,4 @@ Edge will filter out trades with long duration. If a trade is profitable after 1 #### remove_pumps Edge will remove sudden pumps in a given market while going through historical data. However, given that pumps happen very often in crypto markets, we recommend you keep this off.
-(default to false) \ No newline at end of file +(default to false) diff --git a/docs/index.md b/docs/index.md index 879ee4f80..e6e643ba7 100644 --- a/docs/index.md +++ b/docs/index.md @@ -25,7 +25,7 @@ Pull-request. Do not hesitate to reach us on - [Change your strategy](https://github.com/freqtrade/freqtrade/blob/develop/docs/bot-optimization.md#change-your-strategy) - [Add more Indicator](https://github.com/freqtrade/freqtrade/blob/develop/docs/bot-optimization.md#add-more-indicator) - [Test your strategy with Backtesting](https://github.com/freqtrade/freqtrade/blob/develop/docs/backtesting.md) - - [Edge positionning](https://github.com/mishaker/freqtrade/blob/money_mgt/docs/edge.md) + - [Edge positioning](https://github.com/mishaker/freqtrade/blob/money_mgt/docs/edge.md) - [Find optimal parameters with Hyperopt](https://github.com/freqtrade/freqtrade/blob/develop/docs/hyperopt.md) - [Control the bot with telegram](https://github.com/freqtrade/freqtrade/blob/develop/docs/telegram-usage.md) - [Receive notifications via webhook](https://github.com/freqtrade/freqtrade/blob/develop/docs/webhook-config.md) From 4e64bc3d29e6925c68c10e9b76a7ddd48c430937 Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Tue, 13 Nov 2018 13:34:07 +0100 Subject: [PATCH 328/699] Update ccxt from 1.17.491 to 1.17.492 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index f79e425b6..2c0011aea 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -ccxt==1.17.491 +ccxt==1.17.492 SQLAlchemy==1.2.14 python-telegram-bot==11.1.0 arrow==0.12.1 From 51dfd2bf470d46623068cef50dd8b61c8380cd76 Mon Sep 17 00:00:00 2001 From: misagh Date: Wed, 14 Nov 2018 11:37:53 +0100 Subject: [PATCH 329/699] If max_open_trade=-1 means it should be ignored. --- docs/configuration.md | 2 +- freqtrade/configuration.py | 5 +++++ freqtrade/constants.py | 2 +- freqtrade/tests/test_configuration.py | 16 ++++++++++++++++ 4 files changed, 23 insertions(+), 2 deletions(-) diff --git a/docs/configuration.md b/docs/configuration.md index d70a47b38..f40c2c338 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -17,7 +17,7 @@ The table below will list all configuration parameters. | Command | Default | Mandatory | Description | |----------|---------|----------|-------------| -| `max_open_trades` | 3 | Yes | Number of trades open your bot will have. +| `max_open_trades` | 3 | Yes | Number of trades open your bot will have. If -1 then it is ignored (i.e. potentially unlimited open trades) | `stake_currency` | BTC | Yes | Crypto-currency used for trading. | `stake_amount` | 0.05 | Yes | Amount of crypto-currency your bot will use for each trade. Per default, the bot will use (0.05 BTC x 3) = 0.15 BTC in total will be always engaged. Set it to 'unlimited' to allow the bot to use all avaliable balance. | `ticker_interval` | [1m, 5m, 30m, 1h, 1d] | No | The ticker interval to use (1min, 5 min, 30 min, 1 hour or 1 day). Default is 5 minutes diff --git a/freqtrade/configuration.py b/freqtrade/configuration.py index e043525a7..b2daa5f1a 100644 --- a/freqtrade/configuration.py +++ b/freqtrade/configuration.py @@ -33,6 +33,7 @@ class Configuration(object): Class to read and init the bot configuration Reuse this class for the bot, backtesting, hyperopt and every script that required configuration """ + def __init__(self, args: Namespace) -> None: self.args = args self.config: Optional[Dict[str, Any]] = None @@ -130,6 +131,10 @@ class Configuration(object): if config.get('forcebuy_enable', False): logger.warning('`forcebuy` RPC message enabled.') + # Setting max_open_trades to infinite if -1 + if config.get('max_open_trades') == -1: + config['max_open_trades'] = float('inf') + logger.info(f'Using DB: "{config["db_url"]}"') # Check if the exchange set by the user is supported diff --git a/freqtrade/constants.py b/freqtrade/constants.py index b7c069c45..2c753190e 100644 --- a/freqtrade/constants.py +++ b/freqtrade/constants.py @@ -43,7 +43,7 @@ SUPPORTED_FIAT = [ CONF_SCHEMA = { 'type': 'object', 'properties': { - 'max_open_trades': {'type': 'integer', 'minimum': 0}, + 'max_open_trades': {'type': 'integer', 'minimum': -1}, 'ticker_interval': {'type': 'string', 'enum': list(TICKER_INTERVAL_MINUTES.keys())}, 'stake_currency': {'type': 'string', 'enum': ['BTC', 'XBT', 'ETH', 'USDT', 'EUR', 'USD']}, 'stake_amount': { diff --git a/freqtrade/tests/test_configuration.py b/freqtrade/tests/test_configuration.py index daaaec090..23fefd3cd 100644 --- a/freqtrade/tests/test_configuration.py +++ b/freqtrade/tests/test_configuration.py @@ -64,6 +64,22 @@ def test_load_config_max_open_trades_zero(default_conf, mocker, caplog) -> None: assert log_has('Validating configuration ...', caplog.record_tuples) +def test_load_config_max_open_trades_minus_one(default_conf, mocker, caplog) -> None: + default_conf['max_open_trades'] = -1 + mocker.patch('freqtrade.configuration.open', mocker.mock_open( + read_data=json.dumps(default_conf) + )) + + args = Arguments([], '').get_parsed_arg() + configuration = Configuration(args) + validated_conf = configuration.load_config() + print(validated_conf) + + assert validated_conf['max_open_trades'] > 999999999 + assert validated_conf['max_open_trades'] == float('inf') + assert log_has('Validating configuration ...', caplog.record_tuples) + + def test_load_config_file_exception(mocker) -> None: mocker.patch( 'freqtrade.configuration.open', From cf974168e97a2e3c279a77acc764e7f18919c05d Mon Sep 17 00:00:00 2001 From: misagh Date: Wed, 14 Nov 2018 12:37:15 +0100 Subject: [PATCH 330/699] Edge cli drafted --- freqtrade/arguments.py | 84 ++++++++++++++++++++++++++++++------- freqtrade/edge/__init__.py | 2 +- freqtrade/optimize/edge.py | 86 ++++++++++++++++++++++++++++++++++++++ 3 files changed, 156 insertions(+), 16 deletions(-) create mode 100644 freqtrade/optimize/edge.py diff --git a/freqtrade/arguments.py b/freqtrade/arguments.py index bb571b4ea..2b369561a 100644 --- a/freqtrade/arguments.py +++ b/freqtrade/arguments.py @@ -128,6 +128,22 @@ class Arguments(object): """ Parses given arguments for Backtesting scripts. """ + parser.add_argument( + '--eps', '--enable-position-stacking', + help='Allow buying the same pair multiple times (position stacking)', + action='store_true', + dest='position_stacking', + default=False + ) + + parser.add_argument( + '--dmmp', '--disable-max-market-positions', + help='Disable applying `max_open_trades` during backtest ' + '(same as setting `max_open_trades` to a very high number)', + action='store_false', + dest='use_max_market_positions', + default=True + ) parser.add_argument( '-l', '--live', help='using live data', @@ -171,6 +187,38 @@ class Arguments(object): metavar='PATH', ) + @staticmethod + def edge_options(parser: argparse.ArgumentParser) -> None: + """ + Parses given arguments for Backtesting scripts. + """ + parser.add_argument( + '-r', '--refresh-pairs-cached', + help='refresh the pairs files in tests/testdata with the latest data from the ' + 'exchange. Use it if you want to run your backtesting with up-to-date data.', + action='store_true', + dest='refresh_pairs', + ) + parser.add_argument( + '--export', + help='export backtest results, argument are: trades\ + Example --export=trades', + type=str, + default=None, + dest='export', + ) + parser.add_argument( + '--export-filename', + help='Save backtest results to this filename \ + requires --export to be set as well\ + Example --export-filename=user_data/backtest_data/backtest_today.json\ + (default: %(default)s)', + type=str, + default=os.path.join('user_data', 'backtest_data', 'backtest-result.json'), + dest='exportfilename', + metavar='PATH', + ) + @staticmethod def optimizer_shared_options(parser: argparse.ArgumentParser) -> None: """ @@ -184,6 +232,20 @@ class Arguments(object): dest='ticker_interval', type=str, ) + + parser.add_argument( + '--timerange', + help='specify what timerange of data to use.', + default=None, + type=str, + dest='timerange', + ) + + @staticmethod + def hyperopt_options(parser: argparse.ArgumentParser) -> None: + """ + Parses given arguments for Hyperopt scripts. + """ parser.add_argument( '--eps', '--enable-position-stacking', help='Allow buying the same pair multiple times (position stacking)', @@ -200,20 +262,6 @@ class Arguments(object): dest='use_max_market_positions', default=True ) - - parser.add_argument( - '--timerange', - help='specify what timerange of data to use.', - default=None, - type=str, - dest='timerange', - ) - - @staticmethod - def hyperopt_options(parser: argparse.ArgumentParser) -> None: - """ - Parses given arguments for Hyperopt scripts. - """ parser.add_argument( '-e', '--epochs', help='specify number of epochs (default: %(default)d)', @@ -237,7 +285,7 @@ class Arguments(object): Builds and attaches all subcommands :return: None """ - from freqtrade.optimize import backtesting, hyperopt + from freqtrade.optimize import backtesting, hyperopt, edge subparsers = self.parser.add_subparsers(dest='subparser') @@ -247,6 +295,12 @@ class Arguments(object): self.optimizer_shared_options(backtesting_cmd) self.backtesting_options(backtesting_cmd) + # Add edge subcommand + edge_cmd = subparsers.add_parser('edge', help='edge module') + edge_cmd.set_defaults(func=edge.start) + self.optimizer_shared_options(edge_cmd) + self.edge_options(edge_cmd) + # Add hyperopt subcommand hyperopt_cmd = subparsers.add_parser('hyperopt', help='hyperopt module') hyperopt_cmd.set_defaults(func=hyperopt.start) diff --git a/freqtrade/edge/__init__.py b/freqtrade/edge/__init__.py index 7b25a8306..bb1cd118f 100644 --- a/freqtrade/edge/__init__.py +++ b/freqtrade/edge/__init__.py @@ -35,7 +35,7 @@ class Edge(): 'pair_info', ['stoploss', 'winrate', 'risk_reward_ratio', 'required_risk_reward', 'expectancy']) - def __init__(self, config: Dict[str, Any], exchange, strategy) -> None: + def __init__(self, config: Dict[str, Any], exchange, strategy, refresh_pairs=True) -> None: self.config = config self.exchange = exchange diff --git a/freqtrade/optimize/edge.py b/freqtrade/optimize/edge.py new file mode 100644 index 000000000..49c58987b --- /dev/null +++ b/freqtrade/optimize/edge.py @@ -0,0 +1,86 @@ +# pragma pylint: disable=missing-docstring, W0212, too-many-arguments + +""" +This module contains the backtesting logic +""" +import logging +import operator +from argparse import Namespace +from copy import deepcopy +from datetime import datetime, timedelta +from pathlib import Path +from typing import Any, Dict, List, NamedTuple, Optional, Tuple +from freqtrade.edge import Edge + +import freqtrade.optimize as optimize +from freqtrade import DependencyException, constants +from freqtrade.arguments import Arguments +from freqtrade.configuration import Configuration +from freqtrade.exchange import Exchange +from freqtrade.misc import file_dump_json +from freqtrade.persistence import Trade +from freqtrade.strategy.interface import SellType +from freqtrade.strategy.resolver import IStrategy, StrategyResolver +import pdb + +logger = logging.getLogger(__name__) + + +class EdgeCli(object): + """ + Backtesting class, this class contains all the logic to run a backtest + + To run a backtest: + backtesting = Backtesting(config) + backtesting.start() + """ + + def __init__(self, config: Dict[str, Any]) -> None: + self.config = config + + # Reset keys for edge + self.config['exchange']['key'] = '' + self.config['exchange']['secret'] = '' + self.config['exchange']['password'] = '' + self.config['exchange']['uid'] = '' + self.config['dry_run'] = True + self.exchange = Exchange(self.config) + self.strategy = StrategyResolver(self.config).strategy + + self.edge = Edge(config, self.exchange, self.strategy) + + def start(self) -> None: + self.edge.calculate() + # pair_info(stoploss=-0.01, winrate=0.08333333333333333, risk_reward_ratio=0.6399436929988924, required_risk_reward=11.0, expectancy=-0.8633380255834255) + pdb.set_trace() + + +def setup_configuration(args: Namespace) -> Dict[str, Any]: + """ + Prepare the configuration for the backtesting + :param args: Cli args from Arguments() + :return: Configuration + """ + configuration = Configuration(args) + config = configuration.get_config() + + # Ensure we do not use Exchange credentials + config['exchange']['key'] = '' + config['exchange']['secret'] = '' + + return config + + +def start(args: Namespace) -> None: + """ + Start Edge script + :param args: Cli args from Arguments() + :return: None + """ + # Initialize configuration + config = setup_configuration(args) + logger.info('Starting freqtrade in Edge mode') + print('edge talking here...') + # Initialize Edge object + edge_cli = EdgeCli(config) + edge_cli.start() From 95cbbf1cb5e1baa9b7657b631fa4c29c9e50a987 Mon Sep 17 00:00:00 2001 From: misagh Date: Wed, 14 Nov 2018 12:53:20 +0100 Subject: [PATCH 331/699] adding edge configuration to cli --- freqtrade/configuration.py | 42 ++++++++++++++++++++++++++++++++++++++ freqtrade/edge/__init__.py | 5 +++-- freqtrade/optimize/edge.py | 3 ++- 3 files changed, 47 insertions(+), 3 deletions(-) diff --git a/freqtrade/configuration.py b/freqtrade/configuration.py index e043525a7..163581f84 100644 --- a/freqtrade/configuration.py +++ b/freqtrade/configuration.py @@ -58,6 +58,9 @@ class Configuration(object): # Load Backtesting config = self._load_backtesting_config(config) + # Load Edge + config = self._load_edge_config(config) + # Load Hyperopt config = self._load_hyperopt_config(config) @@ -213,6 +216,45 @@ class Configuration(object): return config + def _load_edge_config(self, config: Dict[str, Any]) -> Dict[str, Any]: + """ + Extract information for sys.argv and load Edge configuration + :return: configuration as dictionary + """ + + # If --timerange is used we add it to the configuration + if 'timerange' in self.args and self.args.timerange: + config.update({'timerange': self.args.timerange}) + logger.info('Parameter --timerange detected: %s ...', self.args.timerange) + + # If --datadir is used we add it to the configuration + if 'datadir' in self.args and self.args.datadir: + config.update({'datadir': self.args.datadir}) + else: + config.update({'datadir': self._create_default_datadir(config)}) + logger.info('Using data folder: %s ...', config.get('datadir')) + + # If -r/--refresh-pairs-cached is used we add it to the configuration + if 'refresh_pairs' in self.args and self.args.refresh_pairs: + config.update({'refresh_pairs': True}) + logger.info('Parameter -r/--refresh-pairs-cached detected ...') + + if 'ticker_interval' in self.args and self.args.ticker_interval: + config.update({'ticker_interval': self.args.ticker_interval}) + logger.info('Overriding ticker interval with Command line argument') + + # If --export is used we add it to the configuration + if 'export' in self.args and self.args.export: + config.update({'export': self.args.export}) + logger.info('Parameter --export detected: %s ...', self.args.export) + + # If --export-filename is used we add it to the configuration + if 'export' in config and 'exportfilename' in self.args and self.args.exportfilename: + config.update({'exportfilename': self.args.exportfilename}) + logger.info('Storing backtest results to %s ...', self.args.exportfilename) + + return config + def _load_hyperopt_config(self, config: Dict[str, Any]) -> Dict[str, Any]: """ Extract information for sys.argv and load Hyperopt configuration diff --git a/freqtrade/edge/__init__.py b/freqtrade/edge/__init__.py index bb1cd118f..55c59b644 100644 --- a/freqtrade/edge/__init__.py +++ b/freqtrade/edge/__init__.py @@ -35,7 +35,7 @@ class Edge(): 'pair_info', ['stoploss', 'winrate', 'risk_reward_ratio', 'required_risk_reward', 'expectancy']) - def __init__(self, config: Dict[str, Any], exchange, strategy, refresh_pairs=True) -> None: + def __init__(self, config: Dict[str, Any], exchange, strategy) -> None: self.config = config self.exchange = exchange @@ -53,6 +53,7 @@ class Edge(): self._allowed_risk: float = self.edge_config.get('allowed_risk') self._since_number_of_days: int = self.edge_config.get('calculate_since_number_of_days', 14) self._last_updated: int = 0 # Timestamp of pairs last updated time + self._refresh_pairs = True self._stoploss_range_min = float(self.edge_config.get('stoploss_range_min', -0.01)) self._stoploss_range_max = float(self.edge_config.get('stoploss_range_max', -0.05)) @@ -86,7 +87,7 @@ class Edge(): self.config['datadir'], pairs=pairs, ticker_interval=self.ticker_interval, - refresh_pairs=True, + refresh_pairs=self._refresh_pairs, exchange=self.exchange, timerange=self._timerange ) diff --git a/freqtrade/optimize/edge.py b/freqtrade/optimize/edge.py index 49c58987b..585c397fd 100644 --- a/freqtrade/optimize/edge.py +++ b/freqtrade/optimize/edge.py @@ -48,6 +48,7 @@ class EdgeCli(object): self.strategy = StrategyResolver(self.config).strategy self.edge = Edge(config, self.exchange, self.strategy) + self.edge._refresh_pairs = self.config.get('refresh_pairs', False) def start(self) -> None: self.edge.calculate() @@ -80,7 +81,7 @@ def start(args: Namespace) -> None: # Initialize configuration config = setup_configuration(args) logger.info('Starting freqtrade in Edge mode') - print('edge talking here...') + # Initialize Edge object edge_cli = EdgeCli(config) edge_cli.start() From 5de3f1d9dd837b90d2d7b2571580b290e4a884ee Mon Sep 17 00:00:00 2001 From: misagh Date: Wed, 14 Nov 2018 13:25:44 +0100 Subject: [PATCH 332/699] showing result in tabular --- freqtrade/optimize/edge.py | 23 +++++++++++++++++++++-- 1 file changed, 21 insertions(+), 2 deletions(-) diff --git a/freqtrade/optimize/edge.py b/freqtrade/optimize/edge.py index 585c397fd..2c276687b 100644 --- a/freqtrade/optimize/edge.py +++ b/freqtrade/optimize/edge.py @@ -11,6 +11,7 @@ from datetime import datetime, timedelta from pathlib import Path from typing import Any, Dict, List, NamedTuple, Optional, Tuple from freqtrade.edge import Edge +from tabulate import tabulate import freqtrade.optimize as optimize from freqtrade import DependencyException, constants @@ -50,10 +51,28 @@ class EdgeCli(object): self.edge = Edge(config, self.exchange, self.strategy) self.edge._refresh_pairs = self.config.get('refresh_pairs', False) + def _generate_edge_table(self, results: dict) -> str: + + floatfmt = ('s', '.2f', '.2f', '.2f', '.2f', '.2f') + tabular_data = [] + headers = ['pair', 'stoploss', 'win rate', 'risk reward ratio', + 'required risk reward', 'expectancy'] + + for result in results.items(): + tabular_data.append([ + result[0], + result[1].stoploss, + result[1].winrate, + result[1].risk_reward_ratio, + result[1].required_risk_reward, + result[1].expectancy + ]) + + return tabulate(tabular_data, headers=headers, floatfmt=floatfmt, tablefmt="pipe") + def start(self) -> None: self.edge.calculate() - # pair_info(stoploss=-0.01, winrate=0.08333333333333333, risk_reward_ratio=0.6399436929988924, required_risk_reward=11.0, expectancy=-0.8633380255834255) - pdb.set_trace() + print(self._generate_edge_table(self.edge._cached_pairs)) def setup_configuration(args: Namespace) -> Dict[str, Any]: From 9baf228e8d2377d95951522a5e79f658ce4976c9 Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Wed, 14 Nov 2018 13:34:08 +0100 Subject: [PATCH 333/699] Update ccxt from 1.17.492 to 1.17.494 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 374d6fce0..a76bf9996 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -ccxt==1.17.492 +ccxt==1.17.494 SQLAlchemy==1.2.14 python-telegram-bot==11.1.0 arrow==0.12.1 From 7fb8ae3e1b75f8f95d5b827b781c90a6eb14de0b Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Wed, 14 Nov 2018 13:34:09 +0100 Subject: [PATCH 334/699] Update py_find_1st from 1.1.2 to 1.1.3 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index a76bf9996..cfcffa07c 100644 --- a/requirements.txt +++ b/requirements.txt @@ -26,7 +26,7 @@ scikit-optimize==0.5.2 #plotly==3.1.1 # find first, C search in arrays -py_find_1st==1.1.2 +py_find_1st==1.1.3 #Load ticker files 30% faster ujson==1.35 From ac0c93149231daa8d3f10d6d9a7be16c9c44b428 Mon Sep 17 00:00:00 2001 From: misagh Date: Wed, 14 Nov 2018 13:38:04 +0100 Subject: [PATCH 335/699] adding number of trades + average trade duration to edge info --- freqtrade/edge/__init__.py | 8 ++++++-- freqtrade/tests/edge/test_edge.py | 12 ++++++------ 2 files changed, 12 insertions(+), 8 deletions(-) diff --git a/freqtrade/edge/__init__.py b/freqtrade/edge/__init__.py index 55c59b644..dedaa19a3 100644 --- a/freqtrade/edge/__init__.py +++ b/freqtrade/edge/__init__.py @@ -33,7 +33,9 @@ class Edge(): # pair info data type _pair_info = namedtuple( 'pair_info', - ['stoploss', 'winrate', 'risk_reward_ratio', 'required_risk_reward', 'expectancy']) + ['stoploss', 'winrate', 'risk_reward_ratio', 'required_risk_reward', 'expectancy', + 'nb_trades', 'avg_trade_duration'] + ) def __init__(self, config: Dict[str, Any], exchange, strategy) -> None: @@ -297,7 +299,9 @@ class Edge(): 'winrate': x.winrate, 'risk_reward_ratio': x.risk_reward_ratio, 'required_risk_reward': x.required_risk_reward, - 'expectancy': x.expectancy + 'expectancy': x.expectancy, + 'nb_trades': x.nb_trades, + 'avg_trade_duration': x.avg_trade_duration } final[x.pair] = self._pair_info(**info) diff --git a/freqtrade/tests/edge/test_edge.py b/freqtrade/tests/edge/test_edge.py index a7b1882a5..14c9114c3 100644 --- a/freqtrade/tests/edge/test_edge.py +++ b/freqtrade/tests/edge/test_edge.py @@ -128,9 +128,9 @@ def test_adjust(mocker, default_conf): edge = Edge(default_conf, freqtrade.exchange, freqtrade.strategy) mocker.patch('freqtrade.edge.Edge._cached_pairs', mocker.PropertyMock( return_value={ - 'E/F': Edge._pair_info(-0.01, 0.66, 3.71, 0.50, 1.71), - 'C/D': Edge._pair_info(-0.01, 0.66, 3.71, 0.50, 1.71), - 'N/O': Edge._pair_info(-0.01, 0.66, 3.71, 0.50, 1.71) + 'E/F': Edge._pair_info(-0.01, 0.66, 3.71, 0.50, 1.71, 10, 60), + 'C/D': Edge._pair_info(-0.01, 0.66, 3.71, 0.50, 1.71, 10, 60), + 'N/O': Edge._pair_info(-0.01, 0.66, 3.71, 0.50, 1.71, 10, 60) } )) @@ -143,9 +143,9 @@ def test_stoploss(mocker, default_conf): edge = Edge(default_conf, freqtrade.exchange, freqtrade.strategy) mocker.patch('freqtrade.edge.Edge._cached_pairs', mocker.PropertyMock( return_value={ - 'E/F': Edge._pair_info(-0.01, 0.66, 3.71, 0.50, 1.71), - 'C/D': Edge._pair_info(-0.01, 0.66, 3.71, 0.50, 1.71), - 'N/O': Edge._pair_info(-0.01, 0.66, 3.71, 0.50, 1.71) + 'E/F': Edge._pair_info(-0.01, 0.66, 3.71, 0.50, 1.71, 10, 60), + 'C/D': Edge._pair_info(-0.01, 0.66, 3.71, 0.50, 1.71, 10, 60), + 'N/O': Edge._pair_info(-0.01, 0.66, 3.71, 0.50, 1.71, 10, 60) } )) From 36030176bb21ab247e8e27bd1a62a79ce48940ba Mon Sep 17 00:00:00 2001 From: misagh Date: Wed, 14 Nov 2018 13:38:23 +0100 Subject: [PATCH 336/699] nb_trades and avg_trade_duration added to cli --- freqtrade/optimize/edge.py | 23 +++++++++++++---------- 1 file changed, 13 insertions(+), 10 deletions(-) diff --git a/freqtrade/optimize/edge.py b/freqtrade/optimize/edge.py index 2c276687b..0c4d4c29f 100644 --- a/freqtrade/optimize/edge.py +++ b/freqtrade/optimize/edge.py @@ -53,20 +53,23 @@ class EdgeCli(object): def _generate_edge_table(self, results: dict) -> str: - floatfmt = ('s', '.2f', '.2f', '.2f', '.2f', '.2f') + floatfmt = ('s', '.2f', '.2f', '.2f', '.2f', '.2f', 'd', '.d') tabular_data = [] headers = ['pair', 'stoploss', 'win rate', 'risk reward ratio', - 'required risk reward', 'expectancy'] + 'required risk reward', 'expectancy', 'total number of trades', 'average duration (min)'] for result in results.items(): - tabular_data.append([ - result[0], - result[1].stoploss, - result[1].winrate, - result[1].risk_reward_ratio, - result[1].required_risk_reward, - result[1].expectancy - ]) + if result[1].nb_trades > 0: + tabular_data.append([ + result[0], + result[1].stoploss, + result[1].winrate, + result[1].risk_reward_ratio, + result[1].required_risk_reward, + result[1].expectancy, + result[1].nb_trades, + round(result[1].avg_trade_duration) + ]) return tabulate(tabular_data, headers=headers, floatfmt=floatfmt, tablefmt="pipe") From b0e4aa8eff128b2cec7164f08bd48e8fa546037a Mon Sep 17 00:00:00 2001 From: misagh Date: Wed, 14 Nov 2018 16:31:23 +0100 Subject: [PATCH 337/699] stop loss range added to args --- freqtrade/arguments.py | 7 +++++++ freqtrade/configuration.py | 8 ++++++++ freqtrade/optimize/edge.py | 2 +- 3 files changed, 16 insertions(+), 1 deletion(-) diff --git a/freqtrade/arguments.py b/freqtrade/arguments.py index 2b369561a..d8a8bf472 100644 --- a/freqtrade/arguments.py +++ b/freqtrade/arguments.py @@ -199,6 +199,13 @@ class Arguments(object): action='store_true', dest='refresh_pairs', ) + parser.add_argument( + '--stoplosses', + help='defines a range of stoploss against which edge will assess the strategy' + 'the format is "min, max, step". example: -0.01, -0.1, -0.001', + type=str, + dest='stoploss_range', + ) parser.add_argument( '--export', help='export backtest results, argument are: trades\ diff --git a/freqtrade/configuration.py b/freqtrade/configuration.py index 163581f84..760057c7d 100644 --- a/freqtrade/configuration.py +++ b/freqtrade/configuration.py @@ -227,6 +227,14 @@ class Configuration(object): config.update({'timerange': self.args.timerange}) logger.info('Parameter --timerange detected: %s ...', self.args.timerange) + # If --timerange is used we add it to the configuration + if 'stoploss_range' in self.args and self.args.stoploss_range: + txt_range = eval(self.args.stoploss_range) + config['edge'].update({'stoploss_range_min': txt_range[0]}) + config['edge'].update({'stoploss_range_max': txt_range[1]}) + config['edge'].update({'stoploss_range_step': txt_range[2]}) + logger.info('Parameter --stoplosses detected: %s ...', self.args.stoploss_range) + # If --datadir is used we add it to the configuration if 'datadir' in self.args and self.args.datadir: config.update({'datadir': self.args.datadir}) diff --git a/freqtrade/optimize/edge.py b/freqtrade/optimize/edge.py index 0c4d4c29f..c64456131 100644 --- a/freqtrade/optimize/edge.py +++ b/freqtrade/optimize/edge.py @@ -53,7 +53,7 @@ class EdgeCli(object): def _generate_edge_table(self, results: dict) -> str: - floatfmt = ('s', '.2f', '.2f', '.2f', '.2f', '.2f', 'd', '.d') + floatfmt = ('s', '.10g', '.2f', '.2f', '.2f', '.2f', 'd', '.d') tabular_data = [] headers = ['pair', 'stoploss', 'win rate', 'risk reward ratio', 'required risk reward', 'expectancy', 'total number of trades', 'average duration (min)'] From dd47d7adb407d048273482f95bcfeb927845d15d Mon Sep 17 00:00:00 2001 From: misagh Date: Wed, 14 Nov 2018 16:37:26 +0100 Subject: [PATCH 338/699] cli blank line added to readability --- freqtrade/optimize/edge.py | 1 + 1 file changed, 1 insertion(+) diff --git a/freqtrade/optimize/edge.py b/freqtrade/optimize/edge.py index c64456131..6f3fe7940 100644 --- a/freqtrade/optimize/edge.py +++ b/freqtrade/optimize/edge.py @@ -75,6 +75,7 @@ class EdgeCli(object): def start(self) -> None: self.edge.calculate() + print('') # blank like for readability print(self._generate_edge_table(self.edge._cached_pairs)) From 0767718a179aac64f2ef059cc96f62f8974d4b2d Mon Sep 17 00:00:00 2001 From: misagh Date: Wed, 14 Nov 2018 16:38:55 +0100 Subject: [PATCH 339/699] clear help added to stop losses arg --- freqtrade/arguments.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/arguments.py b/freqtrade/arguments.py index d8a8bf472..594a01e78 100644 --- a/freqtrade/arguments.py +++ b/freqtrade/arguments.py @@ -202,7 +202,7 @@ class Arguments(object): parser.add_argument( '--stoplosses', help='defines a range of stoploss against which edge will assess the strategy' - 'the format is "min, max, step". example: -0.01, -0.1, -0.001', + 'the format is "min,max,step" (without any space). example: --stoplosses=-0.01,-0.1,-0.001', type=str, dest='stoploss_range', ) From 5d73b303fe28cfa89fb0f2d2fb337dc15a137287 Mon Sep 17 00:00:00 2001 From: misagh Date: Wed, 14 Nov 2018 16:49:16 +0100 Subject: [PATCH 340/699] unnecessary libraries removed + arg help enriched --- freqtrade/arguments.py | 3 ++- freqtrade/optimize/edge.py | 20 +++++--------------- 2 files changed, 7 insertions(+), 16 deletions(-) diff --git a/freqtrade/arguments.py b/freqtrade/arguments.py index 594a01e78..a071f0dd6 100644 --- a/freqtrade/arguments.py +++ b/freqtrade/arguments.py @@ -202,7 +202,8 @@ class Arguments(object): parser.add_argument( '--stoplosses', help='defines a range of stoploss against which edge will assess the strategy' - 'the format is "min,max,step" (without any space). example: --stoplosses=-0.01,-0.1,-0.001', + 'the format is "min,max,step" (without any space).' + 'example: --stoplosses=-0.01,-0.1,-0.001', type=str, dest='stoploss_range', ) diff --git a/freqtrade/optimize/edge.py b/freqtrade/optimize/edge.py index 6f3fe7940..f2b75770b 100644 --- a/freqtrade/optimize/edge.py +++ b/freqtrade/optimize/edge.py @@ -4,25 +4,14 @@ This module contains the backtesting logic """ import logging -import operator from argparse import Namespace -from copy import deepcopy -from datetime import datetime, timedelta -from pathlib import Path -from typing import Any, Dict, List, NamedTuple, Optional, Tuple -from freqtrade.edge import Edge +from typing import Dict, Any from tabulate import tabulate +from freqtrade.edge import Edge -import freqtrade.optimize as optimize -from freqtrade import DependencyException, constants -from freqtrade.arguments import Arguments from freqtrade.configuration import Configuration from freqtrade.exchange import Exchange -from freqtrade.misc import file_dump_json -from freqtrade.persistence import Trade -from freqtrade.strategy.interface import SellType -from freqtrade.strategy.resolver import IStrategy, StrategyResolver -import pdb +from freqtrade.strategy.resolver import StrategyResolver logger = logging.getLogger(__name__) @@ -56,7 +45,8 @@ class EdgeCli(object): floatfmt = ('s', '.10g', '.2f', '.2f', '.2f', '.2f', 'd', '.d') tabular_data = [] headers = ['pair', 'stoploss', 'win rate', 'risk reward ratio', - 'required risk reward', 'expectancy', 'total number of trades', 'average duration (min)'] + 'required risk reward', 'expectancy', 'total number of trades', + 'average duration (min)'] for result in results.items(): if result[1].nb_trades > 0: From ca22a116ad6db58d71d7db095e12bc430a695c34 Mon Sep 17 00:00:00 2001 From: misagh Date: Wed, 14 Nov 2018 17:14:37 +0100 Subject: [PATCH 341/699] timerange added to args --- freqtrade/optimize/edge.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/freqtrade/optimize/edge.py b/freqtrade/optimize/edge.py index f2b75770b..59bcbc098 100644 --- a/freqtrade/optimize/edge.py +++ b/freqtrade/optimize/edge.py @@ -10,6 +10,7 @@ from tabulate import tabulate from freqtrade.edge import Edge from freqtrade.configuration import Configuration +from freqtrade.arguments import Arguments from freqtrade.exchange import Exchange from freqtrade.strategy.resolver import StrategyResolver @@ -40,6 +41,11 @@ class EdgeCli(object): self.edge = Edge(config, self.exchange, self.strategy) self.edge._refresh_pairs = self.config.get('refresh_pairs', False) + self.timerange = Arguments.parse_timerange(None if self.config.get( + 'timerange') is None else str(self.config.get('timerange'))) + + self.edge._timerange = self.timerange + def _generate_edge_table(self, results: dict) -> str: floatfmt = ('s', '.10g', '.2f', '.2f', '.2f', '.2f', 'd', '.d') From 9698eee93408c8684492e974026681eeaabc6408 Mon Sep 17 00:00:00 2001 From: misagh Date: Wed, 14 Nov 2018 17:14:44 +0100 Subject: [PATCH 342/699] documentation added --- docs/edge.md | 55 ++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 55 insertions(+) diff --git a/docs/edge.md b/docs/edge.md index f74f8fdcc..94b6c65ac 100644 --- a/docs/edge.md +++ b/docs/edge.md @@ -9,6 +9,7 @@ This page explains how to use Edge Positioning module in your bot in order to en - [Introduction](#introduction) - [How does it work?](#how-does-it-work?) - [Configurations](#configurations) +- [Running Edge independently](#running-edge-independently) ## Introduction Trading is all about probability. No one can claim that he has a strategy working all the time. You have to assume that sometimes you lose.

@@ -149,3 +150,57 @@ Edge will filter out trades with long duration. If a trade is profitable after 1 #### remove_pumps Edge will remove sudden pumps in a given market while going through historical data. However, given that pumps happen very often in crypto markets, we recommend you keep this off.
(default to false) + + +## Running Edge independently +You can run Edge independently in order to see in details the result. Here is an example: +```bash +python3 ./freqtrade/main.py edge +``` + +An example of its output: + +| pair | stoploss | win rate | risk reward ratio | required risk reward | expectancy | total number of trades | average duration (min) | +|:----------|-----------:|-----------:|--------------------:|-----------------------:|-------------:|-------------------------:|-------------------------:| +| AGI/BTC | -0.02 | 0.64 | 5.86 | 0.56 | 3.41 | 14 | 54 | +| NXS/BTC | -0.03 | 0.64 | 2.99 | 0.57 | 1.54 | 11 | 26 | +| LEND/BTC | -0.02 | 0.82 | 2.05 | 0.22 | 1.50 | 11 | 36 | +| VIA/BTC | -0.01 | 0.55 | 3.01 | 0.83 | 1.19 | 11 | 48 | +| MTH/BTC | -0.09 | 0.56 | 2.82 | 0.80 | 1.12 | 18 | 52 | +| ARDR/BTC | -0.04 | 0.42 | 3.14 | 1.40 | 0.73 | 12 | 42 | +| BCPT/BTC | -0.01 | 0.71 | 1.34 | 0.40 | 0.67 | 14 | 30 | +| WINGS/BTC | -0.02 | 0.56 | 1.97 | 0.80 | 0.65 | 27 | 42 | +| VIBE/BTC | -0.02 | 0.83 | 0.91 | 0.20 | 0.59 | 12 | 35 | +| MCO/BTC | -0.02 | 0.79 | 0.97 | 0.27 | 0.55 | 14 | 31 | +| GNT/BTC | -0.02 | 0.50 | 2.06 | 1.00 | 0.53 | 18 | 24 | +| HOT/BTC | -0.01 | 0.17 | 7.72 | 4.81 | 0.50 | 209 | 7 | +| SNM/BTC | -0.03 | 0.71 | 1.06 | 0.42 | 0.45 | 17 | 38 | +| APPC/BTC | -0.02 | 0.44 | 2.28 | 1.27 | 0.44 | 25 | 43 | +| NEBL/BTC | -0.03 | 0.63 | 1.29 | 0.58 | 0.44 | 19 | 59 | + +### Update cached pairs with the latest data +```bash +python3 ./freqtrade/main.py edge --refresh-pairs-cached +``` + +### Precising stoploss range +```bash +python3 ./freqtrade/main.py edge --stoplosses=-0.01,-0.1,-0.001 #min,max,step +``` + +### Advanced use of timerange +```bash +python3 ./freqtrade/main.py edge --timerange=20181110-20181113 +``` + +Doing --timerange=-200 will get the last 200 timeframes from your inputdata. You can also specify specific dates, or a range span indexed by start and stop. + +The full timerange specification: + +* Use last 123 tickframes of data: --timerange=-123 +* Use first 123 tickframes of data: --timerange=123- +* Use tickframes from line 123 through 456: --timerange=123-456 +* Use tickframes till 2018/01/31: --timerange=-20180131 +* Use tickframes since 2018/01/31: --timerange=20180131- +* Use tickframes since 2018/01/31 till 2018/03/01 : --timerange=20180131-20180301 +* Use tickframes between POSIX timestamps 1527595200 1527618600: --timerange=1527595200-1527618600 \ No newline at end of file From bb9a1e5f9f1a324099114a58ce31b51db1b73b52 Mon Sep 17 00:00:00 2001 From: misagh Date: Wed, 14 Nov 2018 19:14:34 +0100 Subject: [PATCH 343/699] edge cli tests added --- freqtrade/tests/edge/test_edge_cli.py | 140 ++++++++++++++++++++++++++ 1 file changed, 140 insertions(+) create mode 100644 freqtrade/tests/edge/test_edge_cli.py diff --git a/freqtrade/tests/edge/test_edge_cli.py b/freqtrade/tests/edge/test_edge_cli.py new file mode 100644 index 000000000..e73507a55 --- /dev/null +++ b/freqtrade/tests/edge/test_edge_cli.py @@ -0,0 +1,140 @@ +# pragma pylint: disable=missing-docstring, C0103, C0330 +# pragma pylint: disable=protected-access, too-many-lines, invalid-name, too-many-arguments + +from unittest.mock import MagicMock +import json +from typing import List +from freqtrade.edge import Edge +from freqtrade.arguments import Arguments +from freqtrade.optimize.edge import (EdgeCli, setup_configuration, start) +from freqtrade.tests.conftest import log_has, patch_exchange + + +def get_args(args) -> List[str]: + return Arguments(args, '').get_parsed_arg() + + +def test_setup_configuration_without_arguments(mocker, default_conf, caplog) -> None: + mocker.patch('freqtrade.configuration.open', mocker.mock_open( + read_data=json.dumps(default_conf) + )) + + args = [ + '--config', 'config.json', + '--strategy', 'DefaultStrategy', + 'edge' + ] + + config = setup_configuration(get_args(args)) + assert 'max_open_trades' in config + assert 'stake_currency' in config + assert 'stake_amount' in config + assert 'exchange' in config + assert 'pair_whitelist' in config['exchange'] + assert 'datadir' in config + assert log_has( + 'Using data folder: {} ...'.format(config['datadir']), + caplog.record_tuples + ) + assert 'ticker_interval' in config + assert not log_has('Parameter -i/--ticker-interval detected ...', caplog.record_tuples) + + assert 'refresh_pairs' not in config + assert not log_has('Parameter -r/--refresh-pairs-cached detected ...', caplog.record_tuples) + + assert 'timerange' not in config + assert 'stoploss_range' not in config + + +def test_setup_configuration_with_arguments(mocker, edge_conf, caplog) -> None: + mocker.patch('freqtrade.configuration.open', mocker.mock_open( + read_data=json.dumps(edge_conf) + )) + + args = [ + '--config', 'config.json', + '--strategy', 'DefaultStrategy', + '--datadir', '/foo/bar', + 'edge', + '--ticker-interval', '1m', + '--refresh-pairs-cached', + '--timerange', ':100', + '--stoplosses=-0.01,-0.10,-0.001' + ] + + config = setup_configuration(get_args(args)) + assert 'max_open_trades' in config + assert 'stake_currency' in config + assert 'stake_amount' in config + assert 'exchange' in config + assert 'pair_whitelist' in config['exchange'] + assert 'datadir' in config + assert log_has( + 'Using data folder: {} ...'.format(config['datadir']), + caplog.record_tuples + ) + assert 'ticker_interval' in config + assert log_has('Parameter -i/--ticker-interval detected ...', caplog.record_tuples) + assert log_has( + 'Using ticker_interval: 1m ...', + caplog.record_tuples + ) + + assert 'refresh_pairs' in config + assert log_has('Parameter -r/--refresh-pairs-cached detected ...', caplog.record_tuples) + assert 'timerange' in config + assert log_has( + 'Parameter --timerange detected: {} ...'.format(config['timerange']), + caplog.record_tuples + ) + + +def test_start(mocker, fee, edge_conf, caplog) -> None: + start_mock = MagicMock() + mocker.patch('freqtrade.exchange.Exchange.get_fee', fee) + patch_exchange(mocker) + mocker.patch('freqtrade.optimize.edge.EdgeCli.start', start_mock) + mocker.patch('freqtrade.configuration.open', mocker.mock_open( + read_data=json.dumps(edge_conf) + )) + args = [ + '--config', 'config.json', + '--strategy', 'DefaultStrategy', + 'edge' + ] + args = get_args(args) + start(args) + assert log_has( + 'Starting freqtrade in Edge mode', + caplog.record_tuples + ) + assert start_mock.call_count == 1 + + +def test_edge_init(mocker, edge_conf) -> None: + patch_exchange(mocker) + edge_cli = EdgeCli(edge_conf) + assert edge_cli.config == edge_conf + assert callable(edge_cli.edge.calculate) + + +def test_generate_edge_table(edge_conf, mocker): + patch_exchange(mocker) + edge_cli = EdgeCli(edge_conf) + + results = {} + info = { + 'stoploss': -0.01, + 'winrate': 0.60, + 'risk_reward_ratio': 2, + 'required_risk_reward': 1, + 'expectancy': 3, + 'nb_trades': 10, + 'avg_trade_duration': 60 + } + + results['ETH/BTC'] = Edge._pair_info(**info) + assert edge_cli._generate_edge_table(results).count(':|') == 7 + assert edge_cli._generate_edge_table(results).count('| ETH/BTC |') == 1 + assert edge_cli._generate_edge_table(results).count( + '| risk reward ratio | required risk reward | expectancy |') == 1 From 4f800bfbc88335df5047292c00cc5b19c7d2b08c Mon Sep 17 00:00:00 2001 From: Matthias Date: Wed, 14 Nov 2018 20:25:43 +0100 Subject: [PATCH 344/699] Fix pickling-error --- freqtrade/optimize/hyperopt.py | 4 ++-- requirements.txt | 1 + setup.py | 1 + 3 files changed, 4 insertions(+), 2 deletions(-) diff --git a/freqtrade/optimize/hyperopt.py b/freqtrade/optimize/hyperopt.py index f5f4222d6..f9a2924a4 100644 --- a/freqtrade/optimize/hyperopt.py +++ b/freqtrade/optimize/hyperopt.py @@ -14,7 +14,7 @@ from operator import itemgetter from typing import Any, Dict, List from pandas import DataFrame -from sklearn.externals.joblib import Parallel, delayed, dump, load +from joblib import Parallel, delayed, dump, load, wrap_non_picklable_objects from skopt import Optimizer from skopt.space import Dimension @@ -219,7 +219,7 @@ class Hyperopt(Backtesting): ) def run_optimizer_parallel(self, parallel, asked) -> List: - return parallel(delayed(self.generate_optimizer)(v) for v in asked) + return parallel(delayed(wrap_non_picklable_objects(self.generate_optimizer))(v) for v in asked) def load_previous_results(self): """ read trials file if we have one """ diff --git a/requirements.txt b/requirements.txt index 2671660ea..11d1bea6b 100644 --- a/requirements.txt +++ b/requirements.txt @@ -8,6 +8,7 @@ urllib3==1.24.1 wrapt==1.10.11 pandas==0.23.4 scikit-learn==0.20.0 +joblib==0.13.0 scipy==1.1.0 jsonschema==2.6.0 numpy==1.15.4 diff --git a/setup.py b/setup.py index 8853ef7f8..119ad03db 100644 --- a/setup.py +++ b/setup.py @@ -30,6 +30,7 @@ setup(name='freqtrade', 'wrapt', 'pandas', 'scikit-learn', + 'joblib' 'scipy', 'jsonschema', 'TA-Lib', From 6a71f80a9e9aebf770227048c5efa9abef8714cf Mon Sep 17 00:00:00 2001 From: Matthias Date: Thu, 15 Nov 2018 06:58:24 +0100 Subject: [PATCH 345/699] Add support for different order types --- freqtrade/exchange/__init__.py | 8 ++++---- freqtrade/freqtradebot.py | 9 +++++++-- freqtrade/strategy/default_strategy.py | 7 +++++++ freqtrade/strategy/interface.py | 7 +++++++ freqtrade/tests/exchange/test_exchange.py | 20 ++++++++++---------- 5 files changed, 35 insertions(+), 16 deletions(-) diff --git a/freqtrade/exchange/__init__.py b/freqtrade/exchange/__init__.py index 4af9db6db..8e0116368 100644 --- a/freqtrade/exchange/__init__.py +++ b/freqtrade/exchange/__init__.py @@ -249,7 +249,7 @@ class Exchange(object): price = ceil(big_price) / pow(10, symbol_prec) return price - def buy(self, pair: str, rate: float, amount: float) -> Dict: + def buy(self, pair: str, rate: float, amount: float, ordertype: str = 'limt') -> Dict: if self._conf['dry_run']: order_id = f'dry_run_buy_{randint(0, 10**6)}' self._dry_run_open_orders[order_id] = { @@ -270,7 +270,7 @@ class Exchange(object): amount = self.symbol_amount_prec(pair, amount) rate = self.symbol_price_prec(pair, rate) - return self._api.create_limit_buy_order(pair, amount, rate) + return self._api.create_order(pair, ordertype, 'buy', amount, rate) except ccxt.InsufficientFunds as e: raise DependencyException( f'Insufficient funds to create limit buy order on market {pair}.' @@ -287,7 +287,7 @@ class Exchange(object): except ccxt.BaseError as e: raise OperationalException(e) - def sell(self, pair: str, rate: float, amount: float) -> Dict: + def sell(self, pair: str, rate: float, amount: float, ordertype: str = 'limt') -> Dict: if self._conf['dry_run']: order_id = f'dry_run_sell_{randint(0, 10**6)}' self._dry_run_open_orders[order_id] = { @@ -307,7 +307,7 @@ class Exchange(object): amount = self.symbol_amount_prec(pair, amount) rate = self.symbol_price_prec(pair, rate) - return self._api.create_limit_sell_order(pair, amount, rate) + return self._api.create_order(pair, ordertype, 'sell', amount, rate) except ccxt.InsufficientFunds as e: raise DependencyException( f'Insufficient funds to create limit sell order on market {pair}.' diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index eb92375ec..75f935c0c 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -475,7 +475,8 @@ class FreqtradeBot(object): amount = stake_amount / buy_limit - order_id = self.exchange.buy(pair, buy_limit, amount)['id'] + order_id = self.exchange.buy(pair=pair, rate=buy_limit, amount=amount, + ordertype=self.strategy.order_types['buy'])['id'] self.rpc.send_msg({ 'type': RPCMessageType.BUY_NOTIFICATION, @@ -762,8 +763,12 @@ class FreqtradeBot(object): :param sellreason: Reason the sell was triggered :return: None """ + sell_type = 'sell' + if sell_reason in (SellType.STOP_LOSS, SellType.TRAILING_STOP_LOSS): + sell_type = 'stoploss' # Execute sell and update trade record - order_id = self.exchange.sell(str(trade.pair), limit, trade.amount)['id'] + order_id = self.exchange.sell(pair=str(trade.pair), rate=limit, amount=trade.amount, + ordertype=self.strategy.order_types[sell_type])['id'] trade.open_order_id = order_id trade.close_rate_requested = limit trade.sell_reason = sell_reason.value diff --git a/freqtrade/strategy/default_strategy.py b/freqtrade/strategy/default_strategy.py index f1646779b..458847636 100644 --- a/freqtrade/strategy/default_strategy.py +++ b/freqtrade/strategy/default_strategy.py @@ -28,6 +28,13 @@ class DefaultStrategy(IStrategy): # Optimal ticker interval for the strategy ticker_interval = '5m' + # Optional order types + order_types = { + 'buy': 'limit', + 'sell': 'limit', + 'stoploss': 'market' + } + def populate_indicators(self, dataframe: DataFrame, metadata: dict) -> DataFrame: """ Adds several different TA indicators to the given DataFrame diff --git a/freqtrade/strategy/interface.py b/freqtrade/strategy/interface.py index 212559c8c..9d6c5f098 100644 --- a/freqtrade/strategy/interface.py +++ b/freqtrade/strategy/interface.py @@ -70,6 +70,13 @@ class IStrategy(ABC): # associated ticker interval ticker_interval: str + # Optional order types + order_types: Dict = { + 'buy': 'limit', + 'sell': 'limit', + 'stoploss': 'market' + } + # run "populate_indicators" only for new candle process_only_new_candles: bool = False diff --git a/freqtrade/tests/exchange/test_exchange.py b/freqtrade/tests/exchange/test_exchange.py index 788ef4518..fe37c6ac1 100644 --- a/freqtrade/tests/exchange/test_exchange.py +++ b/freqtrade/tests/exchange/test_exchange.py @@ -381,7 +381,7 @@ def test_buy_dry_run(default_conf, mocker): def test_buy_prod(default_conf, mocker): api_mock = MagicMock() order_id = 'test_prod_buy_{}'.format(randint(0, 10 ** 6)) - api_mock.create_limit_buy_order = MagicMock(return_value={ + api_mock.create_order = MagicMock(return_value={ 'id': order_id, 'info': { 'foo': 'bar' @@ -397,22 +397,22 @@ def test_buy_prod(default_conf, mocker): # test exception handling with pytest.raises(DependencyException): - api_mock.create_limit_buy_order = MagicMock(side_effect=ccxt.InsufficientFunds) + api_mock.create_order = MagicMock(side_effect=ccxt.InsufficientFunds) exchange = get_patched_exchange(mocker, default_conf, api_mock) exchange.buy(pair='ETH/BTC', rate=200, amount=1) with pytest.raises(DependencyException): - api_mock.create_limit_buy_order = MagicMock(side_effect=ccxt.InvalidOrder) + api_mock.create_order = MagicMock(side_effect=ccxt.InvalidOrder) exchange = get_patched_exchange(mocker, default_conf, api_mock) exchange.buy(pair='ETH/BTC', rate=200, amount=1) with pytest.raises(TemporaryError): - api_mock.create_limit_buy_order = MagicMock(side_effect=ccxt.NetworkError) + api_mock.create_order = MagicMock(side_effect=ccxt.NetworkError) exchange = get_patched_exchange(mocker, default_conf, api_mock) exchange.buy(pair='ETH/BTC', rate=200, amount=1) with pytest.raises(OperationalException): - api_mock.create_limit_buy_order = MagicMock(side_effect=ccxt.BaseError) + api_mock.create_order = MagicMock(side_effect=ccxt.BaseError) exchange = get_patched_exchange(mocker, default_conf, api_mock) exchange.buy(pair='ETH/BTC', rate=200, amount=1) @@ -429,7 +429,7 @@ def test_sell_dry_run(default_conf, mocker): def test_sell_prod(default_conf, mocker): api_mock = MagicMock() order_id = 'test_prod_sell_{}'.format(randint(0, 10 ** 6)) - api_mock.create_limit_sell_order = MagicMock(return_value={ + api_mock.create_order = MagicMock(return_value={ 'id': order_id, 'info': { 'foo': 'bar' @@ -446,22 +446,22 @@ def test_sell_prod(default_conf, mocker): # test exception handling with pytest.raises(DependencyException): - api_mock.create_limit_sell_order = MagicMock(side_effect=ccxt.InsufficientFunds) + api_mock.create_order = MagicMock(side_effect=ccxt.InsufficientFunds) exchange = get_patched_exchange(mocker, default_conf, api_mock) exchange.sell(pair='ETH/BTC', rate=200, amount=1) with pytest.raises(DependencyException): - api_mock.create_limit_sell_order = MagicMock(side_effect=ccxt.InvalidOrder) + api_mock.create_order = MagicMock(side_effect=ccxt.InvalidOrder) exchange = get_patched_exchange(mocker, default_conf, api_mock) exchange.sell(pair='ETH/BTC', rate=200, amount=1) with pytest.raises(TemporaryError): - api_mock.create_limit_sell_order = MagicMock(side_effect=ccxt.NetworkError) + api_mock.create_order = MagicMock(side_effect=ccxt.NetworkError) exchange = get_patched_exchange(mocker, default_conf, api_mock) exchange.sell(pair='ETH/BTC', rate=200, amount=1) with pytest.raises(OperationalException): - api_mock.create_limit_sell_order = MagicMock(side_effect=ccxt.BaseError) + api_mock.create_order = MagicMock(side_effect=ccxt.BaseError) exchange = get_patched_exchange(mocker, default_conf, api_mock) exchange.sell(pair='ETH/BTC', rate=200, amount=1) From f666d1596ba468d26edd941e04e939908cea1bc7 Mon Sep 17 00:00:00 2001 From: misagh Date: Thu, 15 Nov 2018 10:31:56 +0100 Subject: [PATCH 346/699] renaming edge to edge_cli for command line version --- freqtrade/arguments.py | 4 ++-- freqtrade/optimize/{edge.py => edge_cli.py} | 0 freqtrade/tests/{edge => optimize}/test_edge_cli.py | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) rename freqtrade/optimize/{edge.py => edge_cli.py} (100%) rename freqtrade/tests/{edge => optimize}/test_edge_cli.py (96%) diff --git a/freqtrade/arguments.py b/freqtrade/arguments.py index a071f0dd6..db5e4cb5e 100644 --- a/freqtrade/arguments.py +++ b/freqtrade/arguments.py @@ -293,7 +293,7 @@ class Arguments(object): Builds and attaches all subcommands :return: None """ - from freqtrade.optimize import backtesting, hyperopt, edge + from freqtrade.optimize import backtesting, hyperopt, edge_cli subparsers = self.parser.add_subparsers(dest='subparser') @@ -305,7 +305,7 @@ class Arguments(object): # Add edge subcommand edge_cmd = subparsers.add_parser('edge', help='edge module') - edge_cmd.set_defaults(func=edge.start) + edge_cmd.set_defaults(func=edge_cli.start) self.optimizer_shared_options(edge_cmd) self.edge_options(edge_cmd) diff --git a/freqtrade/optimize/edge.py b/freqtrade/optimize/edge_cli.py similarity index 100% rename from freqtrade/optimize/edge.py rename to freqtrade/optimize/edge_cli.py diff --git a/freqtrade/tests/edge/test_edge_cli.py b/freqtrade/tests/optimize/test_edge_cli.py similarity index 96% rename from freqtrade/tests/edge/test_edge_cli.py rename to freqtrade/tests/optimize/test_edge_cli.py index e73507a55..f8db3dec4 100644 --- a/freqtrade/tests/edge/test_edge_cli.py +++ b/freqtrade/tests/optimize/test_edge_cli.py @@ -6,7 +6,7 @@ import json from typing import List from freqtrade.edge import Edge from freqtrade.arguments import Arguments -from freqtrade.optimize.edge import (EdgeCli, setup_configuration, start) +from freqtrade.optimize.edge_cli import (EdgeCli, setup_configuration, start) from freqtrade.tests.conftest import log_has, patch_exchange @@ -93,7 +93,7 @@ def test_start(mocker, fee, edge_conf, caplog) -> None: start_mock = MagicMock() mocker.patch('freqtrade.exchange.Exchange.get_fee', fee) patch_exchange(mocker) - mocker.patch('freqtrade.optimize.edge.EdgeCli.start', start_mock) + mocker.patch('freqtrade.optimize.edge_cli.EdgeCli.start', start_mock) mocker.patch('freqtrade.configuration.open', mocker.mock_open( read_data=json.dumps(edge_conf) )) From 1cfd19aee34f3eb5602b876b60225088817bd0ee Mon Sep 17 00:00:00 2001 From: misagh Date: Thu, 15 Nov 2018 10:44:33 +0100 Subject: [PATCH 347/699] removing unnecessary args for edge --- freqtrade/arguments.py | 21 +-------------------- 1 file changed, 1 insertion(+), 20 deletions(-) diff --git a/freqtrade/arguments.py b/freqtrade/arguments.py index db5e4cb5e..3d1f61dbe 100644 --- a/freqtrade/arguments.py +++ b/freqtrade/arguments.py @@ -195,7 +195,7 @@ class Arguments(object): parser.add_argument( '-r', '--refresh-pairs-cached', help='refresh the pairs files in tests/testdata with the latest data from the ' - 'exchange. Use it if you want to run your backtesting with up-to-date data.', + 'exchange. Use it if you want to run your edge with up-to-date data.', action='store_true', dest='refresh_pairs', ) @@ -207,25 +207,6 @@ class Arguments(object): type=str, dest='stoploss_range', ) - parser.add_argument( - '--export', - help='export backtest results, argument are: trades\ - Example --export=trades', - type=str, - default=None, - dest='export', - ) - parser.add_argument( - '--export-filename', - help='Save backtest results to this filename \ - requires --export to be set as well\ - Example --export-filename=user_data/backtest_data/backtest_today.json\ - (default: %(default)s)', - type=str, - default=os.path.join('user_data', 'backtest_data', 'backtest-result.json'), - dest='exportfilename', - metavar='PATH', - ) @staticmethod def optimizer_shared_options(parser: argparse.ArgumentParser) -> None: From 03e6caa501460781711dbc7b47de382e53409008 Mon Sep 17 00:00:00 2001 From: misagh Date: Thu, 15 Nov 2018 10:46:36 +0100 Subject: [PATCH 348/699] adding notice about Edge ignoring ROI and TSL in doc --- docs/edge.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/edge.md b/docs/edge.md index 94b6c65ac..e5575554b 100644 --- a/docs/edge.md +++ b/docs/edge.md @@ -3,6 +3,7 @@ This page explains how to use Edge Positioning module in your bot in order to enter into a trade only if the trade has a reasonable win rate and risk reward ratio, and consequently adjust your position size and stoploss. **NOTICE:** Edge positioning is not compatible with dynamic whitelist. it overrides dynamic whitelist. +**NOTICE2:** Edge won't consider anything else than buy/sell/stoploss signals. So trailing stoploss, ROI, and everything else will be ignored in its calculation. ## Table of Contents From 69619030f3596757c6f5a6b0e52c4bd583a236ac Mon Sep 17 00:00:00 2001 From: misagh Date: Thu, 15 Nov 2018 10:50:40 +0100 Subject: [PATCH 349/699] removing unnecessary args from config --- freqtrade/configuration.py | 21 --------------------- 1 file changed, 21 deletions(-) diff --git a/freqtrade/configuration.py b/freqtrade/configuration.py index 760057c7d..ab70a5fb9 100644 --- a/freqtrade/configuration.py +++ b/freqtrade/configuration.py @@ -235,32 +235,11 @@ class Configuration(object): config['edge'].update({'stoploss_range_step': txt_range[2]}) logger.info('Parameter --stoplosses detected: %s ...', self.args.stoploss_range) - # If --datadir is used we add it to the configuration - if 'datadir' in self.args and self.args.datadir: - config.update({'datadir': self.args.datadir}) - else: - config.update({'datadir': self._create_default_datadir(config)}) - logger.info('Using data folder: %s ...', config.get('datadir')) - # If -r/--refresh-pairs-cached is used we add it to the configuration if 'refresh_pairs' in self.args and self.args.refresh_pairs: config.update({'refresh_pairs': True}) logger.info('Parameter -r/--refresh-pairs-cached detected ...') - if 'ticker_interval' in self.args and self.args.ticker_interval: - config.update({'ticker_interval': self.args.ticker_interval}) - logger.info('Overriding ticker interval with Command line argument') - - # If --export is used we add it to the configuration - if 'export' in self.args and self.args.export: - config.update({'export': self.args.export}) - logger.info('Parameter --export detected: %s ...', self.args.export) - - # If --export-filename is used we add it to the configuration - if 'export' in config and 'exportfilename' in self.args and self.args.exportfilename: - config.update({'exportfilename': self.args.exportfilename}) - logger.info('Storing backtest results to %s ...', self.args.exportfilename) - return config def _load_hyperopt_config(self, config: Dict[str, Any]) -> Dict[str, Any]: From 23295514f6029b21558ad937b0232f697169262b Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Thu, 15 Nov 2018 13:34:07 +0100 Subject: [PATCH 350/699] Update ccxt from 1.17.494 to 1.17.498 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index cfcffa07c..eda4b0811 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -ccxt==1.17.494 +ccxt==1.17.498 SQLAlchemy==1.2.14 python-telegram-bot==11.1.0 arrow==0.12.1 From 52f4d700ca5492ab451465478cadbbd325b1fd60 Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Thu, 15 Nov 2018 13:34:08 +0100 Subject: [PATCH 351/699] Update pytest from 3.10.1 to 4.0.0 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index eda4b0811..66593c264 100644 --- a/requirements.txt +++ b/requirements.txt @@ -12,7 +12,7 @@ scipy==1.1.0 jsonschema==2.6.0 numpy==1.15.4 TA-Lib==0.4.17 -pytest==3.10.1 +pytest==4.0.0 pytest-mock==1.10.0 pytest-asyncio==0.9.0 pytest-cov==2.6.0 From e6baa9ccf2edad645f45bc10bc4b5d4673918a8e Mon Sep 17 00:00:00 2001 From: Matthias Date: Thu, 15 Nov 2018 19:31:24 +0100 Subject: [PATCH 352/699] Switch tests to kwarguments --- freqtrade/tests/test_freqtradebot.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/freqtrade/tests/test_freqtradebot.py b/freqtrade/tests/test_freqtradebot.py index 4cba9e308..cef89c250 100644 --- a/freqtrade/tests/test_freqtradebot.py +++ b/freqtrade/tests/test_freqtradebot.py @@ -553,7 +553,7 @@ def test_create_trade_minimal_amount(default_conf, ticker, limit_buy_order, patch_get_signal(freqtrade) freqtrade.create_trade() - rate, amount = buy_mock.call_args[0][1], buy_mock.call_args[0][2] + rate, amount = buy_mock.call_args[1]['rate'], buy_mock.call_args[1]['amount'] assert rate * amount >= default_conf['stake_amount'] @@ -863,10 +863,10 @@ def test_execute_buy(mocker, default_conf, fee, markets, limit_buy_order) -> Non assert freqtrade.execute_buy(pair, stake_amount) assert get_bid.call_count == 1 assert buy_mm.call_count == 1 - call_args = buy_mm.call_args_list[0][0] - assert call_args[0] == pair - assert call_args[1] == bid - assert call_args[2] == stake_amount / bid + call_args = buy_mm.call_args_list[0][1] + assert call_args['pair'] == pair + assert call_args['rate'] == bid + assert call_args['amount'] == stake_amount / bid # Test calling with price fix_price = 0.06 @@ -875,10 +875,10 @@ def test_execute_buy(mocker, default_conf, fee, markets, limit_buy_order) -> Non assert get_bid.call_count == 1 assert buy_mm.call_count == 2 - call_args = buy_mm.call_args_list[1][0] - assert call_args[0] == pair - assert call_args[1] == fix_price - assert call_args[2] == stake_amount / fix_price + call_args = buy_mm.call_args_list[1][1] + assert call_args['pair'] == pair + assert call_args['rate'] == fix_price + assert call_args['amount'] == stake_amount / fix_price def test_process_maybe_execute_buy(mocker, default_conf) -> None: From dcf9930858d92ffed54671b1e864a3bd6fd6f27f Mon Sep 17 00:00:00 2001 From: Matthias Date: Thu, 15 Nov 2018 19:36:04 +0100 Subject: [PATCH 353/699] improve hyperopt documentation (links) --- docs/hyperopt.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/hyperopt.md b/docs/hyperopt.md index 0484613c2..8df864c29 100644 --- a/docs/hyperopt.md +++ b/docs/hyperopt.md @@ -21,21 +21,21 @@ and still take a long time. ## Prepare Hyperopt Before we start digging in Hyperopt, we recommend you to take a look at -an example hyperopt file located into [user_data/strategies/](https://github.com/gcarq/freqtrade/blob/develop/user_data/hyperopts/test_hyperopt.py) +an example hyperopt file located into [user_data/hyperopts/](https://github.com/gcarq/freqtrade/blob/develop/user_data/hyperopts/test_hyperopt.py) ### 1. Install a Custom Hyperopt File This is very simple. Put your hyperopt file into the folder `user_data/hyperopts`. Let assume you want a hyperopt file `awesome_hyperopt.py`: -1. Copy the file `user_data/hyperopts/test_hyperopt.py` into `user_data/hyperopts/awesome_hyperopt.py` +1. Copy the file `user_data/hyperopts/sample_hyperopt.py` into `user_data/hyperopts/awesome_hyperopt.py` ### 2. Configure your Guards and Triggers There are two places you need to change in your hyperopt file to add a new buy hyperopt for testing: -- Inside [populate_buy_trend()](https://github.com/gcarq/freqtrade/blob/develop/user_data/hyperopts/test_hyperopt.py#L230-L251). -- Inside [indicator_space()](https://github.com/gcarq/freqtrade/blob/develop/user_data/hyperopts/test_hyperopt.py#L207-L223). +- Inside [populate_buy_trend()](https://github.com/freqtrade/freqtrade/blob/develop/user_data/hyperopts/test_hyperopt.py#L230-L251). +- Inside [indicator_space()](https://github.com/freqtrade/freqtrade/blob/develop/user_data/hyperopts/test_hyperopt.py#L207-L223). There you have two different type of indicators: 1. `guards` and 2. `triggers`. From d05c671a7e5b506cf598d1194fd704cf60596e54 Mon Sep 17 00:00:00 2001 From: misagh Date: Thu, 15 Nov 2018 19:54:17 +0100 Subject: [PATCH 354/699] adding edge args to bot-usage --- docs/bot-usage.md | 63 +++++++++++++++++++++++++++++++++-------------- docs/index.md | 1 + 2 files changed, 45 insertions(+), 19 deletions(-) diff --git a/docs/bot-usage.md b/docs/bot-usage.md index 83a8ee833..31fea17fc 100644 --- a/docs/bot-usage.md +++ b/docs/bot-usage.md @@ -44,11 +44,11 @@ optional arguments: ### How to use a different config file? -The bot allows you to select which config file you want to use. Per +The bot allows you to select which config file you want to use. Per default, the bot will load the file `./config.json` ```bash -python3 ./freqtrade/main.py -c path/far/far/away/config.json +python3 ./freqtrade/main.py -c path/far/far/away/config.json ``` ### How to use --strategy? @@ -61,7 +61,7 @@ The bot will search your strategy file within `user_data/strategies` and `freqtr To load a strategy, simply pass the class name (e.g.: `CustomStrategy`) in this parameter. -**Example:** +**Example:** In `user_data/strategies` you have a file `my_awesome_strategy.py` which has a strategy class called `AwesomeStrategy` to load it: @@ -69,7 +69,7 @@ a strategy class called `AwesomeStrategy` to load it: python3 ./freqtrade/main.py --strategy AwesomeStrategy ``` -If the bot does not find your strategy file, it will display in an error +If the bot does not find your strategy file, it will display in an error message the reason (File not found, or errors in your code). Learn more about strategy file in [optimize your bot](https://github.com/freqtrade/freqtrade/blob/develop/docs/bot-optimization.md). @@ -84,37 +84,37 @@ python3 ./freqtrade/main.py --strategy AwesomeStrategy --strategy-path /some/fol #### How to install a strategy? -This is very simple. Copy paste your strategy file into the folder +This is very simple. Copy paste your strategy file into the folder `user_data/strategies` or use `--strategy-path`. And voila, the bot is ready to use it. ### How to use --dynamic-whitelist? -Per default `--dynamic-whitelist` will retrieve the 20 currencies based +Per default `--dynamic-whitelist` will retrieve the 20 currencies based on BaseVolume. This value can be changed when you run the script. -**By Default** -Get the 20 currencies based on BaseVolume. +**By Default** +Get the 20 currencies based on BaseVolume. ```bash python3 ./freqtrade/main.py --dynamic-whitelist ``` -**Customize the number of currencies to retrieve** -Get the 30 currencies based on BaseVolume. +**Customize the number of currencies to retrieve** +Get the 30 currencies based on BaseVolume. ```bash python3 ./freqtrade/main.py --dynamic-whitelist 30 ``` -**Exception** +**Exception** `--dynamic-whitelist` must be greater than 0. If you enter 0 or a negative value (e.g -2), `--dynamic-whitelist` will use the default value (20). ### How to use --db-url? -When you run the bot in Dry-run mode, per default no transactions are -stored in a database. If you want to store your bot actions in a DB +When you run the bot in Dry-run mode, per default no transactions are +stored in a database. If you want to store your bot actions in a DB using `--db-url`. This can also be used to specify a custom database in production mode. Example command: @@ -170,15 +170,15 @@ optional arguments: ### How to use --refresh-pairs-cached parameter? -The first time your run Backtesting, it will take the pairs you have -set in your config file and download data from Bittrex. +The first time your run Backtesting, it will take the pairs you have +set in your config file and download data from Bittrex. -If for any reason you want to update your data set, you use -`--refresh-pairs-cached` to force Backtesting to update the data it has. +If for any reason you want to update your data set, you use +`--refresh-pairs-cached` to force Backtesting to update the data it has. **Use it only if you want to update your data set. You will not be able to come back to the previous version.** -To test your strategy with latest data, we recommend continuing using +To test your strategy with latest data, we recommend continuing using the parameter `-l` or `--live`. ## Hyperopt commands @@ -211,6 +211,31 @@ optional arguments: ``` +## Edge commands + +To know your trade expectacny and winrate against historical data, you can use Edge. + +``` +usage: main.py edge [-h] [-i TICKER_INTERVAL] [--timerange TIMERANGE] [-r] + [--stoplosses STOPLOSS_RANGE] + +optional arguments: + -h, --help show this help message and exit + -i TICKER_INTERVAL, --ticker-interval TICKER_INTERVAL + specify ticker interval (1m, 5m, 30m, 1h, 1d) + --timerange TIMERANGE + specify what timerange of data to use. + -r, --refresh-pairs-cached + refresh the pairs files in tests/testdata with the + latest data from the exchange. Use it if you want to + run your edge with up-to-date data. + --stoplosses STOPLOSS_RANGE + defines a range of stoploss against which edge will + assess the strategythe format is "min,max,step" + (without any space).example: + --stoplosses=-0.01,-0.1,-0.001 +``` + ## A parameter missing in the configuration? All parameters for `main.py`, `backtesting`, `hyperopt` are referenced @@ -218,5 +243,5 @@ in [misc.py](https://github.com/freqtrade/freqtrade/blob/develop/freqtrade/misc. ## Next step -The optimal strategy of the bot will change with time depending of the market trends. The next step is to +The optimal strategy of the bot will change with time depending of the market trends. The next step is to [optimize your bot](https://github.com/freqtrade/freqtrade/blob/develop/docs/bot-optimization.md). diff --git a/docs/index.md b/docs/index.md index e6e643ba7..c64b9c188 100644 --- a/docs/index.md +++ b/docs/index.md @@ -21,6 +21,7 @@ Pull-request. Do not hesitate to reach us on - [Bot commands](https://github.com/freqtrade/freqtrade/blob/develop/docs/bot-usage.md#bot-commands) - [Backtesting commands](https://github.com/freqtrade/freqtrade/blob/develop/docs/bot-usage.md#backtesting-commands) - [Hyperopt commands](https://github.com/freqtrade/freqtrade/blob/develop/docs/bot-usage.md#hyperopt-commands) + - [Edge commands](https://github.com/mishaker/freqtrade/blob/develop/docs/bot-usage.md#edge-commands) - [Bot Optimization](https://github.com/freqtrade/freqtrade/blob/develop/docs/bot-optimization.md) - [Change your strategy](https://github.com/freqtrade/freqtrade/blob/develop/docs/bot-optimization.md#change-your-strategy) - [Add more Indicator](https://github.com/freqtrade/freqtrade/blob/develop/docs/bot-optimization.md#add-more-indicator) From db8c8ea4a42d6d90a00c7b63c401e73b8bd25b1f Mon Sep 17 00:00:00 2001 From: misagh Date: Thu, 15 Nov 2018 20:02:07 +0100 Subject: [PATCH 355/699] added a space in help --- freqtrade/arguments.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/arguments.py b/freqtrade/arguments.py index 3d1f61dbe..8e26752fe 100644 --- a/freqtrade/arguments.py +++ b/freqtrade/arguments.py @@ -201,7 +201,7 @@ class Arguments(object): ) parser.add_argument( '--stoplosses', - help='defines a range of stoploss against which edge will assess the strategy' + help='defines a range of stoploss against which edge will assess the strategy ' 'the format is "min,max,step" (without any space).' 'example: --stoplosses=-0.01,-0.1,-0.001', type=str, From 98df3c810308e07a524e4ce97d46a9499fa93d32 Mon Sep 17 00:00:00 2001 From: Matthias Date: Thu, 15 Nov 2018 20:02:48 +0100 Subject: [PATCH 356/699] Fix missing mock in backtesting --- freqtrade/tests/optimize/test_backtesting.py | 1 + 1 file changed, 1 insertion(+) diff --git a/freqtrade/tests/optimize/test_backtesting.py b/freqtrade/tests/optimize/test_backtesting.py index 20f2a6582..e6e0a1c5d 100644 --- a/freqtrade/tests/optimize/test_backtesting.py +++ b/freqtrade/tests/optimize/test_backtesting.py @@ -634,6 +634,7 @@ def test_backtest_only_sell(mocker, default_conf): def test_backtest_alternate_buy_sell(default_conf, fee, mocker): mocker.patch('freqtrade.exchange.Exchange.get_fee', fee) + mocker.patch('freqtrade.optimize.backtesting.file_dump_json', MagicMock()) backtest_conf = _make_backtest_conf(mocker, conf=default_conf, pair='UNITTEST/BTC') backtesting = Backtesting(default_conf) backtesting.advise_buy = _trend_alternate # Override From b7abf7dda92bf78da1e075fc31a3b7e42327d5a0 Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Fri, 16 Nov 2018 13:34:08 +0100 Subject: [PATCH 357/699] Update ccxt from 1.17.498 to 1.17.500 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 66593c264..6b356cd9e 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -ccxt==1.17.498 +ccxt==1.17.500 SQLAlchemy==1.2.14 python-telegram-bot==11.1.0 arrow==0.12.1 From 24ed9a8b7d93f502ca2a12a3ef8db62628691a3e Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 17 Nov 2018 10:14:18 +0100 Subject: [PATCH 358/699] Add loading order_types from config file --- config_full.json.example | 5 +++++ freqtrade/strategy/resolver.py | 9 ++++++++ freqtrade/tests/strategy/test_strategy.py | 26 +++++++++++++++++++++++ 3 files changed, 40 insertions(+) diff --git a/config_full.json.example b/config_full.json.example index 9dba8f539..b0719bcc6 100644 --- a/config_full.json.example +++ b/config_full.json.example @@ -33,6 +33,11 @@ "order_book_min": 1, "order_book_max": 9 }, + "order_types": { + "buy": "limit", + "sell": "limit", + "stoploss": "market" + }, "exchange": { "name": "bittrex", "key": "your_exchange_key", diff --git a/freqtrade/strategy/resolver.py b/freqtrade/strategy/resolver.py index aee47580c..31bd21ec8 100644 --- a/freqtrade/strategy/resolver.py +++ b/freqtrade/strategy/resolver.py @@ -75,6 +75,15 @@ class StrategyResolver(object): else: config['process_only_new_candles'] = self.strategy.process_only_new_candles + if 'order_types' in config: + self.strategy.order_types = config['order_types'] + logger.info( + "Override strategy 'order_types' with value in config file: %s.", + config['order_types'] + ) + else: + config['order_types'] = self.strategy.order_types + # Sort and apply type conversions self.strategy.minimal_roi = OrderedDict(sorted( {int(key): value for (key, value) in self.strategy.minimal_roi.items()}.items(), diff --git a/freqtrade/tests/strategy/test_strategy.py b/freqtrade/tests/strategy/test_strategy.py index abc531689..e6204b5f5 100644 --- a/freqtrade/tests/strategy/test_strategy.py +++ b/freqtrade/tests/strategy/test_strategy.py @@ -182,6 +182,32 @@ def test_strategy_override_process_only_new_candles(caplog): ) in caplog.record_tuples +def test_strategy_override_order_types(caplog): + caplog.set_level(logging.INFO) + + order_types = { + 'buy': 'market', + 'sell': 'limit', + 'stoploss': 'limit' + } + + config = { + 'strategy': 'DefaultStrategy', + 'order_types': order_types + } + resolver = StrategyResolver(config) + + assert resolver.strategy.order_types + for method in ['buy', 'sell', 'stoploss']: + assert resolver.strategy.order_types[method] == order_types[method] + + assert ('freqtrade.strategy.resolver', + logging.INFO, + "Override strategy 'order_types' with value in config file:" + " {'buy': 'market', 'sell': 'limit', 'stoploss': 'limit'}." + ) in caplog.record_tuples + + def test_deprecate_populate_indicators(result): default_location = path.join(path.dirname(path.realpath(__file__))) resolver = StrategyResolver({'strategy': 'TestStrategyLegacy', From 6e78efd9719f188f5d47d14db82309b9dddf02fe Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 17 Nov 2018 10:24:42 +0100 Subject: [PATCH 359/699] Document "order_types" setting --- docs/configuration.md | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/docs/configuration.md b/docs/configuration.md index d70a47b38..5a3f2b371 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -39,6 +39,7 @@ The table below will list all configuration parameters. | `ask_strategy.use_order_book` | false | No | Allows selling of open traded pair using the rates in Order Book Asks. | `ask_strategy.order_book_min` | 0 | No | Bot will scan from the top min to max Order Book Asks searching for a profitable rate. | `ask_strategy.order_book_max` | 0 | No | Bot will scan from the top min to max Order Book Asks searching for a profitable rate. +| `order_types` | None | No | Configure order-types depending on the action (`"buy"`, `"sell"`, `"stoploss"`). | `exchange.name` | bittrex | Yes | Name of the exchange class to use. [List below](#user-content-what-values-for-exchangename). | `exchange.key` | key | No | API key to use for the exchange. Only required when you are in production mode. | `exchange.secret` | secret | No | API secret to use for the exchange. Only required when you are in production mode. @@ -138,6 +139,22 @@ use the `last` price and values between those interpolate between ask and last price. Using `ask` price will guarantee quick success in bid, but bot will also end up paying more then would probably have been necessary. +### Understand order_types + +`order_types` contains a dict mapping order-types to market-types. This allows to buy using limit orders, sell using limit-orders, and create stoploss orders using market. +This can be set in the configuration or in the strategy. Configuration overwrites strategy configurations. + +If this is configured, all 3 values (`"buy"`, `"sell"` and `"stoploss"`) need to be present, otherwise the bot warn about it and will fail to start. +The below is the default which is used if this is not configured in either Strategy or configuration. + +``` json + "order_types": { + "buy": "limit", + "sell": "limit", + "stoploss": "market" + }, +``` + ### What values for exchange.name? Freqtrade is based on [CCXT library](https://github.com/ccxt/ccxt) that supports 115 cryptocurrency From 3ab0cf49af865f31d74db03e883b8c78dbb5532b Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 17 Nov 2018 10:26:15 +0100 Subject: [PATCH 360/699] Add order_types to sample strategy --- freqtrade/strategy/default_strategy.py | 2 +- user_data/strategies/test_strategy.py | 7 +++++++ 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/freqtrade/strategy/default_strategy.py b/freqtrade/strategy/default_strategy.py index 458847636..59e280b6e 100644 --- a/freqtrade/strategy/default_strategy.py +++ b/freqtrade/strategy/default_strategy.py @@ -28,7 +28,7 @@ class DefaultStrategy(IStrategy): # Optimal ticker interval for the strategy ticker_interval = '5m' - # Optional order types + # Optional order type mapping order_types = { 'buy': 'limit', 'sell': 'limit', diff --git a/user_data/strategies/test_strategy.py b/user_data/strategies/test_strategy.py index 7c3892b77..fd2e9ab75 100644 --- a/user_data/strategies/test_strategy.py +++ b/user_data/strategies/test_strategy.py @@ -48,6 +48,13 @@ class TestStrategy(IStrategy): # run "populate_indicators" only for new candle ta_on_candle = False + # Optional order type mapping + order_types = { + 'buy': 'limit', + 'sell': 'limit', + 'stoploss': 'market' + } + def populate_indicators(self, dataframe: DataFrame, metadata: dict) -> DataFrame: """ Adds several different TA indicators to the given DataFrame From 54a86d72f28550e5904981c7b1fe909383dc91dc Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 17 Nov 2018 12:59:16 +0100 Subject: [PATCH 361/699] Raise error if one of the required ordertypes is not present --- freqtrade/constants.py | 1 + freqtrade/strategy/resolver.py | 4 ++++ 2 files changed, 5 insertions(+) diff --git a/freqtrade/constants.py b/freqtrade/constants.py index b7c069c45..d5c23fe1d 100644 --- a/freqtrade/constants.py +++ b/freqtrade/constants.py @@ -12,6 +12,7 @@ DEFAULT_STRATEGY = 'DefaultStrategy' DEFAULT_DB_PROD_URL = 'sqlite:///tradesv3.sqlite' DEFAULT_DB_DRYRUN_URL = 'sqlite://' UNLIMITED_STAKE_AMOUNT = 'unlimited' +REQUIRED_ORDERTYPES = ['buy', 'sell', 'stoploss'] TICKER_INTERVAL_MINUTES = { diff --git a/freqtrade/strategy/resolver.py b/freqtrade/strategy/resolver.py index 31bd21ec8..3f25e4838 100644 --- a/freqtrade/strategy/resolver.py +++ b/freqtrade/strategy/resolver.py @@ -84,6 +84,10 @@ class StrategyResolver(object): else: config['order_types'] = self.strategy.order_types + if not all(k in self.strategy.order_types for k in constants.REQUIRED_ORDERTYPES): + raise ImportError(f"Impossible to load Strategy '{self.strategy.__class__.__name__}'. " + f"Order-types mapping is incomplete.") + # Sort and apply type conversions self.strategy.minimal_roi = OrderedDict(sorted( {int(key): value for (key, value) in self.strategy.minimal_roi.items()}.items(), From 9ba281c141eb2735798fd5298b654369f0b7876f Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 17 Nov 2018 13:05:35 +0100 Subject: [PATCH 362/699] add supported limit values --- freqtrade/constants.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/freqtrade/constants.py b/freqtrade/constants.py index d5c23fe1d..fdbff886e 100644 --- a/freqtrade/constants.py +++ b/freqtrade/constants.py @@ -13,6 +13,7 @@ DEFAULT_DB_PROD_URL = 'sqlite:///tradesv3.sqlite' DEFAULT_DB_DRYRUN_URL = 'sqlite://' UNLIMITED_STAKE_AMOUNT = 'unlimited' REQUIRED_ORDERTYPES = ['buy', 'sell', 'stoploss'] +ORDERTYPE_POSSIBILITIES = ['limit', 'market'] TICKER_INTERVAL_MINUTES = { @@ -102,6 +103,15 @@ CONF_SCHEMA = { 'order_book_max': {'type': 'number', 'minimum': 1, 'maximum': 50} } }, + 'order_types': { + 'type': 'object', + 'properties': { + 'buy': {'type': 'string', 'enum': ORDERTYPE_POSSIBILITIES}, + 'sell': {'type': 'string', 'enum': ORDERTYPE_POSSIBILITIES}, + 'stoploss': {'type': 'string', 'enum': ORDERTYPE_POSSIBILITIES} + }, + 'required': ['buy', 'sell', 'stoploss'] + }, 'exchange': {'$ref': '#/definitions/exchange'}, 'edge': {'$ref': '#/definitions/edge'}, 'experimental': { From e485aff597d68d9f649109976a8dd85517bf8235 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 17 Nov 2018 13:12:11 +0100 Subject: [PATCH 363/699] Test failed load on invalid ordertypes --- freqtrade/tests/strategy/test_strategy.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/freqtrade/tests/strategy/test_strategy.py b/freqtrade/tests/strategy/test_strategy.py index e6204b5f5..d1a87ecfa 100644 --- a/freqtrade/tests/strategy/test_strategy.py +++ b/freqtrade/tests/strategy/test_strategy.py @@ -207,6 +207,16 @@ def test_strategy_override_order_types(caplog): " {'buy': 'market', 'sell': 'limit', 'stoploss': 'limit'}." ) in caplog.record_tuples + config = { + 'strategy': 'DefaultStrategy', + 'order_types': {'buy': 'market'} + } + # Raise error for invalid configuration + with pytest.raises(ImportError, + match=r"Impossible to load Strategy 'DefaultStrategy'. " + r"Order-types mapping is incomplete."): + StrategyResolver(config) + def test_deprecate_populate_indicators(result): default_location = path.join(path.dirname(path.realpath(__file__))) From 543873263a711ccf54f90c3d0f8ed7f827b2adf1 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 17 Nov 2018 13:13:16 +0100 Subject: [PATCH 364/699] remove need for escaping quote --- freqtrade/tests/strategy/test_strategy.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/freqtrade/tests/strategy/test_strategy.py b/freqtrade/tests/strategy/test_strategy.py index d1a87ecfa..a38050f24 100644 --- a/freqtrade/tests/strategy/test_strategy.py +++ b/freqtrade/tests/strategy/test_strategy.py @@ -88,8 +88,8 @@ def test_load_strategy_invalid_directory(result, caplog): def test_load_not_found_strategy(): strategy = StrategyResolver() with pytest.raises(ImportError, - match=r'Impossible to load Strategy \'NotFoundStrategy\'.' - r' This class does not exist or contains Python code errors'): + match=r"Impossible to load Strategy 'NotFoundStrategy'." + r" This class does not exist or contains Python code errors"): strategy._load_strategy(strategy_name='NotFoundStrategy', config={}) From ef1e20bfe8d2e3a009bad9bf46c12f41afe9af80 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 17 Nov 2018 13:23:13 +0100 Subject: [PATCH 365/699] Don't add default value for ordertype sort parameters to align with ccxt --- freqtrade/exchange/__init__.py | 8 ++++---- freqtrade/freqtradebot.py | 9 +++++---- 2 files changed, 9 insertions(+), 8 deletions(-) diff --git a/freqtrade/exchange/__init__.py b/freqtrade/exchange/__init__.py index 8e0116368..57db4a125 100644 --- a/freqtrade/exchange/__init__.py +++ b/freqtrade/exchange/__init__.py @@ -249,14 +249,14 @@ class Exchange(object): price = ceil(big_price) / pow(10, symbol_prec) return price - def buy(self, pair: str, rate: float, amount: float, ordertype: str = 'limt') -> Dict: + def buy(self, pair: str, ordertype: str, amount: float, rate: float) -> Dict: if self._conf['dry_run']: order_id = f'dry_run_buy_{randint(0, 10**6)}' self._dry_run_open_orders[order_id] = { 'pair': pair, 'price': rate, 'amount': amount, - 'type': 'limit', + 'type': ordertype, 'side': 'buy', 'remaining': 0.0, 'datetime': arrow.utcnow().isoformat(), @@ -287,14 +287,14 @@ class Exchange(object): except ccxt.BaseError as e: raise OperationalException(e) - def sell(self, pair: str, rate: float, amount: float, ordertype: str = 'limt') -> Dict: + def sell(self, pair: str, ordertype: str, amount: float, rate: float) -> Dict: if self._conf['dry_run']: order_id = f'dry_run_sell_{randint(0, 10**6)}' self._dry_run_open_orders[order_id] = { 'pair': pair, 'price': rate, 'amount': amount, - 'type': 'limit', + 'type': ordertype, 'side': 'sell', 'remaining': 0.0, 'datetime': arrow.utcnow().isoformat(), diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index 75f935c0c..c7532ead7 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -475,8 +475,8 @@ class FreqtradeBot(object): amount = stake_amount / buy_limit - order_id = self.exchange.buy(pair=pair, rate=buy_limit, amount=amount, - ordertype=self.strategy.order_types['buy'])['id'] + order_id = self.exchange.buy(pair=pair, ordertype=self.strategy.order_types['buy'], + amount=amount, rate=buy_limit)['id'] self.rpc.send_msg({ 'type': RPCMessageType.BUY_NOTIFICATION, @@ -767,8 +767,9 @@ class FreqtradeBot(object): if sell_reason in (SellType.STOP_LOSS, SellType.TRAILING_STOP_LOSS): sell_type = 'stoploss' # Execute sell and update trade record - order_id = self.exchange.sell(pair=str(trade.pair), rate=limit, amount=trade.amount, - ordertype=self.strategy.order_types[sell_type])['id'] + order_id = self.exchange.sell(pair=str(trade.pair), + ordertype=self.strategy.order_types[sell_type], + amount=trade.amount, rate=limit)['id'] trade.open_order_id = order_id trade.close_rate_requested = limit trade.sell_reason = sell_reason.value From a9a157af0fa58e1a0994747aeefd2985df9ac317 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 17 Nov 2018 13:29:42 +0100 Subject: [PATCH 366/699] Align tests and test if ordertype is passed to ccxt correctly --- freqtrade/tests/exchange/test_exchange.py | 37 +++++++++++++++-------- 1 file changed, 25 insertions(+), 12 deletions(-) diff --git a/freqtrade/tests/exchange/test_exchange.py b/freqtrade/tests/exchange/test_exchange.py index fe37c6ac1..12a8fbcf4 100644 --- a/freqtrade/tests/exchange/test_exchange.py +++ b/freqtrade/tests/exchange/test_exchange.py @@ -373,7 +373,7 @@ def test_buy_dry_run(default_conf, mocker): default_conf['dry_run'] = True exchange = get_patched_exchange(mocker, default_conf) - order = exchange.buy(pair='ETH/BTC', rate=200, amount=1) + order = exchange.buy(pair='ETH/BTC', ordertype='limit', amount=1, rate=200) assert 'id' in order assert 'dry_run_buy_' in order['id'] @@ -381,6 +381,7 @@ def test_buy_dry_run(default_conf, mocker): def test_buy_prod(default_conf, mocker): api_mock = MagicMock() order_id = 'test_prod_buy_{}'.format(randint(0, 10 ** 6)) + order_type = 'market' api_mock.create_order = MagicMock(return_value={ 'id': order_id, 'info': { @@ -390,38 +391,43 @@ def test_buy_prod(default_conf, mocker): default_conf['dry_run'] = False exchange = get_patched_exchange(mocker, default_conf, api_mock) - order = exchange.buy(pair='ETH/BTC', rate=200, amount=1) + order = exchange.buy(pair='ETH/BTC', ordertype=order_type, amount=1, rate=200) assert 'id' in order assert 'info' in order assert order['id'] == order_id + assert api_mock.create_order.call_args[0][1] == order_type + order_type = 'limit' + api_mock.create_order.reset_mock() + order = exchange.buy(pair='ETH/BTC', ordertype=order_type, amount=1, rate=200) + assert api_mock.create_order.call_args[0][1] == order_type # test exception handling with pytest.raises(DependencyException): api_mock.create_order = MagicMock(side_effect=ccxt.InsufficientFunds) exchange = get_patched_exchange(mocker, default_conf, api_mock) - exchange.buy(pair='ETH/BTC', rate=200, amount=1) + exchange.buy(pair='ETH/BTC', ordertype=order_type, amount=1, rate=200) with pytest.raises(DependencyException): api_mock.create_order = MagicMock(side_effect=ccxt.InvalidOrder) exchange = get_patched_exchange(mocker, default_conf, api_mock) - exchange.buy(pair='ETH/BTC', rate=200, amount=1) + exchange.buy(pair='ETH/BTC', ordertype=order_type, amount=1, rate=200) with pytest.raises(TemporaryError): api_mock.create_order = MagicMock(side_effect=ccxt.NetworkError) exchange = get_patched_exchange(mocker, default_conf, api_mock) - exchange.buy(pair='ETH/BTC', rate=200, amount=1) + exchange.buy(pair='ETH/BTC', ordertype=order_type, amount=1, rate=200) with pytest.raises(OperationalException): api_mock.create_order = MagicMock(side_effect=ccxt.BaseError) exchange = get_patched_exchange(mocker, default_conf, api_mock) - exchange.buy(pair='ETH/BTC', rate=200, amount=1) + exchange.buy(pair='ETH/BTC', ordertype=order_type, amount=1, rate=200) def test_sell_dry_run(default_conf, mocker): default_conf['dry_run'] = True exchange = get_patched_exchange(mocker, default_conf) - order = exchange.sell(pair='ETH/BTC', rate=200, amount=1) + order = exchange.sell(pair='ETH/BTC', ordertype='limit', amount=1, rate=200) assert 'id' in order assert 'dry_run_sell_' in order['id'] @@ -429,6 +435,7 @@ def test_sell_dry_run(default_conf, mocker): def test_sell_prod(default_conf, mocker): api_mock = MagicMock() order_id = 'test_prod_sell_{}'.format(randint(0, 10 ** 6)) + order_type = 'market' api_mock.create_order = MagicMock(return_value={ 'id': order_id, 'info': { @@ -439,31 +446,37 @@ def test_sell_prod(default_conf, mocker): exchange = get_patched_exchange(mocker, default_conf, api_mock) - order = exchange.sell(pair='ETH/BTC', rate=200, amount=1) + order = exchange.sell(pair='ETH/BTC', ordertype=order_type, amount=1, rate=200) assert 'id' in order assert 'info' in order assert order['id'] == order_id + assert api_mock.create_order.call_args[0][1] == order_type + + order_type = 'limit' + api_mock.create_order.reset_mock() + order = exchange.sell(pair='ETH/BTC', ordertype=order_type, amount=1, rate=200) + assert api_mock.create_order.call_args[0][1] == order_type # test exception handling with pytest.raises(DependencyException): api_mock.create_order = MagicMock(side_effect=ccxt.InsufficientFunds) exchange = get_patched_exchange(mocker, default_conf, api_mock) - exchange.sell(pair='ETH/BTC', rate=200, amount=1) + exchange.sell(pair='ETH/BTC', ordertype=order_type, amount=1, rate=200) with pytest.raises(DependencyException): api_mock.create_order = MagicMock(side_effect=ccxt.InvalidOrder) exchange = get_patched_exchange(mocker, default_conf, api_mock) - exchange.sell(pair='ETH/BTC', rate=200, amount=1) + exchange.sell(pair='ETH/BTC', ordertype=order_type, amount=1, rate=200) with pytest.raises(TemporaryError): api_mock.create_order = MagicMock(side_effect=ccxt.NetworkError) exchange = get_patched_exchange(mocker, default_conf, api_mock) - exchange.sell(pair='ETH/BTC', rate=200, amount=1) + exchange.sell(pair='ETH/BTC', ordertype=order_type, amount=1, rate=200) with pytest.raises(OperationalException): api_mock.create_order = MagicMock(side_effect=ccxt.BaseError) exchange = get_patched_exchange(mocker, default_conf, api_mock) - exchange.sell(pair='ETH/BTC', rate=200, amount=1) + exchange.sell(pair='ETH/BTC', ordertype=order_type, amount=1, rate=200) def test_get_balance_dry_run(default_conf, mocker): From 681659f2d2f0f5fd7310f749e3fb28d762577987 Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Sat, 17 Nov 2018 13:34:06 +0100 Subject: [PATCH 367/699] Update ccxt from 1.17.500 to 1.17.502 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 6b356cd9e..56e0d080a 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -ccxt==1.17.500 +ccxt==1.17.502 SQLAlchemy==1.2.14 python-telegram-bot==11.1.0 arrow==0.12.1 From 492868a966e352251dc15d74eaed4b133c8da475 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 17 Nov 2018 13:34:23 +0100 Subject: [PATCH 368/699] Seperate different tests within one test clearer --- freqtrade/tests/exchange/test_exchange.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/freqtrade/tests/exchange/test_exchange.py b/freqtrade/tests/exchange/test_exchange.py index 12a8fbcf4..fd4512bd2 100644 --- a/freqtrade/tests/exchange/test_exchange.py +++ b/freqtrade/tests/exchange/test_exchange.py @@ -396,8 +396,9 @@ def test_buy_prod(default_conf, mocker): assert 'info' in order assert order['id'] == order_id assert api_mock.create_order.call_args[0][1] == order_type - order_type = 'limit' + api_mock.create_order.reset_mock() + order_type = 'limit' order = exchange.buy(pair='ETH/BTC', ordertype=order_type, amount=1, rate=200) assert api_mock.create_order.call_args[0][1] == order_type @@ -452,8 +453,8 @@ def test_sell_prod(default_conf, mocker): assert order['id'] == order_id assert api_mock.create_order.call_args[0][1] == order_type - order_type = 'limit' api_mock.create_order.reset_mock() + order_type = 'limit' order = exchange.sell(pair='ETH/BTC', ordertype=order_type, amount=1, rate=200) assert api_mock.create_order.call_args[0][1] == order_type From 69dd56b237a8c964dddd42ec3eed70220830097d Mon Sep 17 00:00:00 2001 From: misagh Date: Sat, 17 Nov 2018 18:47:13 +0100 Subject: [PATCH 369/699] wallet sync drafted --- freqtrade/freqtradebot.py | 16 +++++++++++++++- freqtrade/persistence.py | 30 +++++++++++++++++++++++++++++- 2 files changed, 44 insertions(+), 2 deletions(-) diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index eb92375ec..428e5a726 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -18,7 +18,7 @@ from freqtrade import (DependencyException, OperationalException, TemporaryError, __version__, constants, persistence) from freqtrade.exchange import Exchange from freqtrade.edge import Edge -from freqtrade.persistence import Trade +from freqtrade.persistence import Trade, Wallet from freqtrade.rpc import RPCManager, RPCMessageType from freqtrade.state import State from freqtrade.strategy.interface import SellType @@ -800,3 +800,17 @@ class FreqtradeBot(object): # Send the message self.rpc.send_msg(msg) Trade.session.flush() + + def update_wallets(self) -> bool: + wallets = self.exchange.get_balances() + + for currency in wallets: + wallet = Wallet( + exchange=self.exchange._api.id, + currency=currency, + free=wallets[currency]['free'], + used=wallets[currency]['used'], + total=wallets[currency]['total'] + ) + Wallet.session.add(wallet) + Wallet.session.flush() diff --git a/freqtrade/persistence.py b/freqtrade/persistence.py index 51a8129fb..ac7833d79 100644 --- a/freqtrade/persistence.py +++ b/freqtrade/persistence.py @@ -9,7 +9,7 @@ from typing import Any, Dict, Optional import arrow from sqlalchemy import (Boolean, Column, DateTime, Float, Integer, String, - create_engine, inspect) + create_engine, inspect, PrimaryKeyConstraint) from sqlalchemy.exc import NoSuchModuleError from sqlalchemy.ext.declarative import declarative_base from sqlalchemy.orm.scoping import scoped_session @@ -50,8 +50,13 @@ def init(config: Dict) -> None: f'is no valid database URL! (See {_SQL_DOCS_URL})') session = scoped_session(sessionmaker(bind=engine, autoflush=True, autocommit=True)) + Trade.session = session() Trade.query = session.query_property() + + Wallet.session = session() + Wallet.query = session.query_property() + _DECL_BASE.metadata.create_all(engine) check_migrate(engine) @@ -341,3 +346,26 @@ class Trade(_DECL_BASE): ) profit_percent = (close_trade_price / open_trade_price) - 1 return float(f"{profit_percent:.8f}") + + +class Wallet(_DECL_BASE): + """ + Class for wallet structure + It is a mirror of wallets on an exchange + """ + __tablename__ = 'wallets' + + exchange = Column(String, nullable=False, primary_key=True, index=True) + currency = Column(String, nullable=False, primary_key=True, index=True) + + free = Column(Float, index=True) + used = Column(Float) + total = Column(Float) + base = Column(Boolean, index=True, default=False) + quote = Column(Boolean, index=True, default=False) + + __table_args__ = ( + PrimaryKeyConstraint( + exchange, + currency), + {}) From 968184ef0db70040eedb7be2b1734baa11e06e3c Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 17 Nov 2018 19:40:22 +0100 Subject: [PATCH 370/699] Swap default mode to all limit (defaults to how it was before) --- freqtrade/strategy/default_strategy.py | 2 +- freqtrade/strategy/interface.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/freqtrade/strategy/default_strategy.py b/freqtrade/strategy/default_strategy.py index 59e280b6e..b282a5938 100644 --- a/freqtrade/strategy/default_strategy.py +++ b/freqtrade/strategy/default_strategy.py @@ -32,7 +32,7 @@ class DefaultStrategy(IStrategy): order_types = { 'buy': 'limit', 'sell': 'limit', - 'stoploss': 'market' + 'stoploss': 'limit' } def populate_indicators(self, dataframe: DataFrame, metadata: dict) -> DataFrame: diff --git a/freqtrade/strategy/interface.py b/freqtrade/strategy/interface.py index 9d6c5f098..139bcd8be 100644 --- a/freqtrade/strategy/interface.py +++ b/freqtrade/strategy/interface.py @@ -74,7 +74,7 @@ class IStrategy(ABC): order_types: Dict = { 'buy': 'limit', 'sell': 'limit', - 'stoploss': 'market' + 'stoploss': 'limit' } # run "populate_indicators" only for new candle From c11984d943af8a59a548ddbee7a78d49fcb8bdde Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 17 Nov 2018 19:54:55 +0100 Subject: [PATCH 371/699] Check if exchange supports all configured market types --- freqtrade/exchange/__init__.py | 11 ++++++++- freqtrade/tests/conftest.py | 1 + freqtrade/tests/exchange/test_exchange.py | 30 +++++++++++++++++++++++ 3 files changed, 41 insertions(+), 1 deletion(-) diff --git a/freqtrade/exchange/__init__.py b/freqtrade/exchange/__init__.py index 57db4a125..11836ed4e 100644 --- a/freqtrade/exchange/__init__.py +++ b/freqtrade/exchange/__init__.py @@ -102,7 +102,7 @@ class Exchange(object): self.markets = self._load_markets() # Check if all pairs are available self.validate_pairs(config['exchange']['pair_whitelist']) - + self.validate_ordertypes(config.get('order_types', {})) if config.get('ticker_interval'): # Check if timeframe is available self.validate_timeframes(config['ticker_interval']) @@ -218,6 +218,15 @@ class Exchange(object): raise OperationalException( f'Invalid ticker {timeframe}, this Exchange supports {timeframes}') + def validate_ordertypes(self, order_types: Dict) -> None: + """ + Checks if order-types configured in strategy/config are supported + """ + if any(v == 'market' for k, v in order_types.items()): + if not self.exchange_has('createMarketOrder'): + raise OperationalException( + f'Exchange {self.name} does not support market orders.') + def exchange_has(self, endpoint: str) -> bool: """ Checks if exchange implements a specific API endpoint. diff --git a/freqtrade/tests/conftest.py b/freqtrade/tests/conftest.py index 8a497725f..b6c022b45 100644 --- a/freqtrade/tests/conftest.py +++ b/freqtrade/tests/conftest.py @@ -30,6 +30,7 @@ def log_has(line, logs): def patch_exchange(mocker, api_mock=None) -> None: mocker.patch('freqtrade.exchange.Exchange._load_markets', MagicMock(return_value={})) mocker.patch('freqtrade.exchange.Exchange.validate_timeframes', MagicMock()) + mocker.patch('freqtrade.exchange.Exchange.validate_ordertypes', MagicMock()) mocker.patch('freqtrade.exchange.Exchange.name', PropertyMock(return_value="Bittrex")) mocker.patch('freqtrade.exchange.Exchange.id', PropertyMock(return_value="bittrex")) if api_mock: diff --git a/freqtrade/tests/exchange/test_exchange.py b/freqtrade/tests/exchange/test_exchange.py index fd4512bd2..10644a9be 100644 --- a/freqtrade/tests/exchange/test_exchange.py +++ b/freqtrade/tests/exchange/test_exchange.py @@ -355,6 +355,36 @@ def test_validate_timeframes_not_in_config(default_conf, mocker): Exchange(default_conf) +def test_validate_order_types(default_conf, mocker): + api_mock = MagicMock() + + type(api_mock).has = PropertyMock(return_value={'createMarketOrder': True}) + mocker.patch('freqtrade.exchange.Exchange._init_ccxt', MagicMock(return_value=api_mock)) + mocker.patch('freqtrade.exchange.Exchange._load_markets', MagicMock(return_value={})) + mocker.patch('freqtrade.exchange.Exchange.validate_timeframes', MagicMock()) + default_conf['order_types'] = {'buy': 'limit', 'sell': 'limit', 'stoploss': 'market'} + Exchange(default_conf) + + type(api_mock).has = PropertyMock(return_value={'createMarketOrder': False}) + mocker.patch('freqtrade.exchange.Exchange._init_ccxt', MagicMock(return_value=api_mock)) + + default_conf['order_types'] = {'buy': 'limit', 'sell': 'limit', 'stoploss': 'market'} + + with pytest.raises(OperationalException, + match=r'Exchange .* does not support market orders.'): + Exchange(default_conf) + + +def test_validate_order_types_not_in_config(default_conf, mocker): + api_mock = MagicMock() + mocker.patch('freqtrade.exchange.Exchange._init_ccxt', MagicMock(return_value=api_mock)) + mocker.patch('freqtrade.exchange.Exchange._load_markets', MagicMock(return_value={})) + mocker.patch('freqtrade.exchange.Exchange.validate_timeframes', MagicMock()) + + conf = copy.deepcopy(default_conf) + Exchange(conf) + + def test_exchange_has(default_conf, mocker): exchange = get_patched_exchange(mocker, default_conf) assert not exchange.exchange_has('ASDFASDF') From b3e08831f7a233e9fccd67d6d7749d5577896e54 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 17 Nov 2018 20:09:05 +0100 Subject: [PATCH 372/699] Remove rate for market orders --- freqtrade/exchange/__init__.py | 4 ++-- freqtrade/tests/exchange/test_exchange.py | 20 ++++++++++++++++++++ 2 files changed, 22 insertions(+), 2 deletions(-) diff --git a/freqtrade/exchange/__init__.py b/freqtrade/exchange/__init__.py index 11836ed4e..ae07e36e9 100644 --- a/freqtrade/exchange/__init__.py +++ b/freqtrade/exchange/__init__.py @@ -277,7 +277,7 @@ class Exchange(object): try: # Set the precision for amount and price(rate) as accepted by the exchange amount = self.symbol_amount_prec(pair, amount) - rate = self.symbol_price_prec(pair, rate) + rate = self.symbol_price_prec(pair, rate) if ordertype != 'market' else None return self._api.create_order(pair, ordertype, 'buy', amount, rate) except ccxt.InsufficientFunds as e: @@ -314,7 +314,7 @@ class Exchange(object): try: # Set the precision for amount and price(rate) as accepted by the exchange amount = self.symbol_amount_prec(pair, amount) - rate = self.symbol_price_prec(pair, rate) + rate = self.symbol_price_prec(pair, rate) if ordertype != 'market' else None return self._api.create_order(pair, ordertype, 'sell', amount, rate) except ccxt.InsufficientFunds as e: diff --git a/freqtrade/tests/exchange/test_exchange.py b/freqtrade/tests/exchange/test_exchange.py index 10644a9be..207f14efe 100644 --- a/freqtrade/tests/exchange/test_exchange.py +++ b/freqtrade/tests/exchange/test_exchange.py @@ -419,18 +419,28 @@ def test_buy_prod(default_conf, mocker): } }) default_conf['dry_run'] = False + mocker.patch('freqtrade.exchange.Exchange.symbol_amount_prec', lambda s, x, y: y) + mocker.patch('freqtrade.exchange.Exchange.symbol_price_prec', lambda s, x, y: y) exchange = get_patched_exchange(mocker, default_conf, api_mock) order = exchange.buy(pair='ETH/BTC', ordertype=order_type, amount=1, rate=200) assert 'id' in order assert 'info' in order assert order['id'] == order_id + assert api_mock.create_order.call_args[0][0] == 'ETH/BTC' assert api_mock.create_order.call_args[0][1] == order_type + assert api_mock.create_order.call_args[0][2] == 'buy' + assert api_mock.create_order.call_args[0][3] == 1 + assert api_mock.create_order.call_args[0][4] is None api_mock.create_order.reset_mock() order_type = 'limit' order = exchange.buy(pair='ETH/BTC', ordertype=order_type, amount=1, rate=200) + assert api_mock.create_order.call_args[0][0] == 'ETH/BTC' assert api_mock.create_order.call_args[0][1] == order_type + assert api_mock.create_order.call_args[0][2] == 'buy' + assert api_mock.create_order.call_args[0][3] == 1 + assert api_mock.create_order.call_args[0][4] == 200 # test exception handling with pytest.raises(DependencyException): @@ -476,17 +486,27 @@ def test_sell_prod(default_conf, mocker): default_conf['dry_run'] = False exchange = get_patched_exchange(mocker, default_conf, api_mock) + mocker.patch('freqtrade.exchange.Exchange.symbol_amount_prec', lambda s, x, y: y) + mocker.patch('freqtrade.exchange.Exchange.symbol_price_prec', lambda s, x, y: y) order = exchange.sell(pair='ETH/BTC', ordertype=order_type, amount=1, rate=200) assert 'id' in order assert 'info' in order assert order['id'] == order_id + assert api_mock.create_order.call_args[0][0] == 'ETH/BTC' assert api_mock.create_order.call_args[0][1] == order_type + assert api_mock.create_order.call_args[0][2] == 'sell' + assert api_mock.create_order.call_args[0][3] == 1 + assert api_mock.create_order.call_args[0][4] is None api_mock.create_order.reset_mock() order_type = 'limit' order = exchange.sell(pair='ETH/BTC', ordertype=order_type, amount=1, rate=200) + assert api_mock.create_order.call_args[0][0] == 'ETH/BTC' assert api_mock.create_order.call_args[0][1] == order_type + assert api_mock.create_order.call_args[0][2] == 'sell' + assert api_mock.create_order.call_args[0][3] == 1 + assert api_mock.create_order.call_args[0][4] == 200 # test exception handling with pytest.raises(DependencyException): From 82cb0e4d95fdd663381977e9720797f96cb6bd31 Mon Sep 17 00:00:00 2001 From: misagh Date: Sat, 17 Nov 2018 21:16:32 +0100 Subject: [PATCH 373/699] =?UTF-8?q?putting=20wallets=20into=20a=20class=20?= =?UTF-8?q?(doesn=E2=80=99t=20need=20to=20be=20in=20persistence)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- freqtrade/finance/wallets.py | 39 ++++++++++++++++++++++++++++++++++++ freqtrade/persistence.py | 23 --------------------- 2 files changed, 39 insertions(+), 23 deletions(-) create mode 100644 freqtrade/finance/wallets.py diff --git a/freqtrade/finance/wallets.py b/freqtrade/finance/wallets.py new file mode 100644 index 000000000..c8d8bab2f --- /dev/null +++ b/freqtrade/finance/wallets.py @@ -0,0 +1,39 @@ +# pragma pylint: disable=W0603 +""" Wallet """ +import logging +from typing import Dict +from collections import namedtuple +from freqtrade.exchange import Exchange + +logger = logging.getLogger(__name__) + + +class Wallets(object): + + # wallet data structure + wallet = namedtuple( + 'wallet', + ['exchange', 'currency', 'free', 'used', 'total'] + ) + + def __init__(self, exchange: Exchange) -> None: + self.exchange = exchange + self.wallets: Dict[str, self.wallet] = {} + + def _update_wallets(self) -> None: + balances = self.exchange.get_balances() + + for currency in balances: + info = { + 'exchange': self.exchange.id, + 'currency': currency, + 'free': balances[currency['free']], + 'used': balances[currency['used']], + 'total': balances[currency['total']] + } + self.wallets[currency] = self.wallet(**info) + + logger.info('Wallets synced ...') + + def update(self) -> None: + self._update_wallets() diff --git a/freqtrade/persistence.py b/freqtrade/persistence.py index ac7833d79..82daa0b74 100644 --- a/freqtrade/persistence.py +++ b/freqtrade/persistence.py @@ -346,26 +346,3 @@ class Trade(_DECL_BASE): ) profit_percent = (close_trade_price / open_trade_price) - 1 return float(f"{profit_percent:.8f}") - - -class Wallet(_DECL_BASE): - """ - Class for wallet structure - It is a mirror of wallets on an exchange - """ - __tablename__ = 'wallets' - - exchange = Column(String, nullable=False, primary_key=True, index=True) - currency = Column(String, nullable=False, primary_key=True, index=True) - - free = Column(Float, index=True) - used = Column(Float) - total = Column(Float) - base = Column(Boolean, index=True, default=False) - quote = Column(Boolean, index=True, default=False) - - __table_args__ = ( - PrimaryKeyConstraint( - exchange, - currency), - {}) From afe52efc8ac1f05a0f9210252fa4937bb9b1f5eb Mon Sep 17 00:00:00 2001 From: misagh Date: Sat, 17 Nov 2018 21:17:39 +0100 Subject: [PATCH 374/699] removing wallet from freq --- freqtrade/freqtradebot.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index 428e5a726..570b925cc 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -18,7 +18,7 @@ from freqtrade import (DependencyException, OperationalException, TemporaryError, __version__, constants, persistence) from freqtrade.exchange import Exchange from freqtrade.edge import Edge -from freqtrade.persistence import Trade, Wallet +from freqtrade.persistence import Trade from freqtrade.rpc import RPCManager, RPCMessageType from freqtrade.state import State from freqtrade.strategy.interface import SellType From b815c8fe2dd37385708b9313cf689b9179449117 Mon Sep 17 00:00:00 2001 From: misagh Date: Sat, 17 Nov 2018 21:22:54 +0100 Subject: [PATCH 375/699] updating wallets whenever a trade happens --- freqtrade/freqtradebot.py | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index 570b925cc..d9f17b999 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -17,6 +17,7 @@ from cachetools import TTLCache, cached from freqtrade import (DependencyException, OperationalException, TemporaryError, __version__, constants, persistence) from freqtrade.exchange import Exchange +from freqtrade.finance.wallets import Wallets from freqtrade.edge import Edge from freqtrade.persistence import Trade from freqtrade.rpc import RPCManager, RPCMessageType @@ -56,6 +57,7 @@ class FreqtradeBot(object): self.rpc: RPCManager = RPCManager(self) self.persistence = None self.exchange = Exchange(self.config) + self.wallets = Wallets(self.exchange) # Initializing Edge only if enabled self.edge = Edge(self.config, self.exchange, self.strategy) if \ @@ -505,6 +507,10 @@ class FreqtradeBot(object): ) Trade.session.add(trade) Trade.session.flush() + + # Updating wallets + self.wallets.update() + return True def process_maybe_execute_buy(self) -> bool: @@ -549,7 +555,11 @@ class FreqtradeBot(object): if trade.is_open and trade.open_order_id is None: # Check if we can sell our current pair - return self.handle_trade(trade) + result = self.handle_trade(trade) + if result: + self.wallets.update() + return result + except DependencyException as exception: logger.warning('Unable to sell trade: %s', exception) return False From 12f07ee126145b105623b0e7d28a711845bf5265 Mon Sep 17 00:00:00 2001 From: misagh Date: Sat, 17 Nov 2018 21:26:41 +0100 Subject: [PATCH 376/699] space removed --- freqtrade/freqtradebot.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index d9f17b999..631f4d3b1 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -555,7 +555,7 @@ class FreqtradeBot(object): if trade.is_open and trade.open_order_id is None: # Check if we can sell our current pair - result = self.handle_trade(trade) + result = self.handle_trade(trade) if result: self.wallets.update() return result From a0658bb50448b65c5997ea19c96c10a0d7aefe38 Mon Sep 17 00:00:00 2001 From: misagh Date: Sat, 17 Nov 2018 21:27:42 +0100 Subject: [PATCH 377/699] comments added --- freqtrade/freqtradebot.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index 631f4d3b1..568769fff 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -556,8 +556,11 @@ class FreqtradeBot(object): if trade.is_open and trade.open_order_id is None: # Check if we can sell our current pair result = self.handle_trade(trade) + + # Updating wallets if any trade occured if result: self.wallets.update() + return result except DependencyException as exception: From d5b47abe98af09f150176c904b87b51c2e7d1f25 Mon Sep 17 00:00:00 2001 From: misagh Date: Sat, 17 Nov 2018 21:31:06 +0100 Subject: [PATCH 378/699] Wallet table removed --- freqtrade/persistence.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/freqtrade/persistence.py b/freqtrade/persistence.py index 82daa0b74..2936819a6 100644 --- a/freqtrade/persistence.py +++ b/freqtrade/persistence.py @@ -54,9 +54,6 @@ def init(config: Dict) -> None: Trade.session = session() Trade.query = session.query_property() - Wallet.session = session() - Wallet.query = session.query_property() - _DECL_BASE.metadata.create_all(engine) check_migrate(engine) From f4bb203782eb54f8d8557edd2b3d27a020ddc9db Mon Sep 17 00:00:00 2001 From: misagh Date: Sat, 17 Nov 2018 21:59:21 +0100 Subject: [PATCH 379/699] removing persistence update --- freqtrade/freqtradebot.py | 14 -------------- freqtrade/persistence.py | 2 +- 2 files changed, 1 insertion(+), 15 deletions(-) diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index 568769fff..0a75ce3d7 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -813,17 +813,3 @@ class FreqtradeBot(object): # Send the message self.rpc.send_msg(msg) Trade.session.flush() - - def update_wallets(self) -> bool: - wallets = self.exchange.get_balances() - - for currency in wallets: - wallet = Wallet( - exchange=self.exchange._api.id, - currency=currency, - free=wallets[currency]['free'], - used=wallets[currency]['used'], - total=wallets[currency]['total'] - ) - Wallet.session.add(wallet) - Wallet.session.flush() diff --git a/freqtrade/persistence.py b/freqtrade/persistence.py index 2936819a6..aa380d20b 100644 --- a/freqtrade/persistence.py +++ b/freqtrade/persistence.py @@ -9,7 +9,7 @@ from typing import Any, Dict, Optional import arrow from sqlalchemy import (Boolean, Column, DateTime, Float, Integer, String, - create_engine, inspect, PrimaryKeyConstraint) + create_engine, inspect) from sqlalchemy.exc import NoSuchModuleError from sqlalchemy.ext.declarative import declarative_base from sqlalchemy.orm.scoping import scoped_session From 606e41d5743594bc69a30e548de1046e3480a1ea Mon Sep 17 00:00:00 2001 From: misagh Date: Sat, 17 Nov 2018 22:58:27 +0100 Subject: [PATCH 380/699] wallet tests added --- freqtrade/tests/finance/test_wallets.py | 32 +++++++++++++++++++++++++ 1 file changed, 32 insertions(+) create mode 100644 freqtrade/tests/finance/test_wallets.py diff --git a/freqtrade/tests/finance/test_wallets.py b/freqtrade/tests/finance/test_wallets.py new file mode 100644 index 000000000..e226d117f --- /dev/null +++ b/freqtrade/tests/finance/test_wallets.py @@ -0,0 +1,32 @@ +# pragma pylint: disable=missing-docstring +from freqtrade.tests.conftest import get_patched_freqtradebot +from unittest.mock import MagicMock + + +def test_sync_wallet_at_boot(mocker, default_conf): + mocker.patch.multiple( + 'freqtrade.exchange.Exchange', + get_balances=MagicMock(return_value={ + "BNT": { + "free": 1.0, + "used": 2.0, + "total": 3.0 + }, + "GAS": { + "free": 0.260739, + "used": 0.0, + "total": 0.260739 + }, + }) + ) + + freqtrade = get_patched_freqtradebot(mocker, default_conf) + + assert len(freqtrade.wallets.wallets) == 2 + assert freqtrade.wallets.wallets['BNT'].free == 1.0 + assert freqtrade.wallets.wallets['BNT'].used == 2.0 + assert freqtrade.wallets.wallets['BNT'].total == 3.0 + + assert freqtrade.wallets.wallets['GAS'].free == 0.260739 + assert freqtrade.wallets.wallets['GAS'].used == 0.0 + assert freqtrade.wallets.wallets['GAS'].total == 0.260739 From 7cb8b28f5885dfeca3ba99723e377668f385bddf Mon Sep 17 00:00:00 2001 From: misagh Date: Sat, 17 Nov 2018 23:03:07 +0100 Subject: [PATCH 381/699] wallet sync added --- freqtrade/finance/wallets.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/freqtrade/finance/wallets.py b/freqtrade/finance/wallets.py index c8d8bab2f..7155f90ac 100644 --- a/freqtrade/finance/wallets.py +++ b/freqtrade/finance/wallets.py @@ -19,6 +19,7 @@ class Wallets(object): def __init__(self, exchange: Exchange) -> None: self.exchange = exchange self.wallets: Dict[str, self.wallet] = {} + self._update_wallets() def _update_wallets(self) -> None: balances = self.exchange.get_balances() @@ -27,10 +28,11 @@ class Wallets(object): info = { 'exchange': self.exchange.id, 'currency': currency, - 'free': balances[currency['free']], - 'used': balances[currency['used']], - 'total': balances[currency['total']] + 'free': balances[currency]['free'], + 'used': balances[currency]['used'], + 'total': balances[currency]['total'] } + self.wallets[currency] = self.wallet(**info) logger.info('Wallets synced ...') From a92619f18ca9d56361960d68c213098ada3e23a9 Mon Sep 17 00:00:00 2001 From: misagh Date: Sun, 18 Nov 2018 14:34:31 +0100 Subject: [PATCH 382/699] Added empty lines related to last commit removed --- freqtrade/persistence.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/freqtrade/persistence.py b/freqtrade/persistence.py index aa380d20b..51a8129fb 100644 --- a/freqtrade/persistence.py +++ b/freqtrade/persistence.py @@ -50,10 +50,8 @@ def init(config: Dict) -> None: f'is no valid database URL! (See {_SQL_DOCS_URL})') session = scoped_session(sessionmaker(bind=engine, autoflush=True, autocommit=True)) - Trade.session = session() Trade.query = session.query_property() - _DECL_BASE.metadata.create_all(engine) check_migrate(engine) From 608ce98e1a5c67ce4bd0556cfdccf4d0375c6096 Mon Sep 17 00:00:00 2001 From: misagh Date: Sun, 18 Nov 2018 14:38:31 +0100 Subject: [PATCH 383/699] moving wallets to root --- freqtrade/freqtradebot.py | 2 +- freqtrade/{finance => }/wallets.py | 0 2 files changed, 1 insertion(+), 1 deletion(-) rename freqtrade/{finance => }/wallets.py (100%) diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index 0a75ce3d7..83b5b85e5 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -17,7 +17,7 @@ from cachetools import TTLCache, cached from freqtrade import (DependencyException, OperationalException, TemporaryError, __version__, constants, persistence) from freqtrade.exchange import Exchange -from freqtrade.finance.wallets import Wallets +from freqtrade.wallets import Wallets from freqtrade.edge import Edge from freqtrade.persistence import Trade from freqtrade.rpc import RPCManager, RPCMessageType diff --git a/freqtrade/finance/wallets.py b/freqtrade/wallets.py similarity index 100% rename from freqtrade/finance/wallets.py rename to freqtrade/wallets.py From 9c549f451390822e22c6b4e141aa239234c9b85d Mon Sep 17 00:00:00 2001 From: misagh Date: Sun, 18 Nov 2018 14:39:31 +0100 Subject: [PATCH 384/699] removing unnecessary private function --- freqtrade/wallets.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/freqtrade/wallets.py b/freqtrade/wallets.py index 7155f90ac..fc2769831 100644 --- a/freqtrade/wallets.py +++ b/freqtrade/wallets.py @@ -19,9 +19,9 @@ class Wallets(object): def __init__(self, exchange: Exchange) -> None: self.exchange = exchange self.wallets: Dict[str, self.wallet] = {} - self._update_wallets() + self.update() - def _update_wallets(self) -> None: + def update(self) -> None: balances = self.exchange.get_balances() for currency in balances: @@ -36,6 +36,3 @@ class Wallets(object): self.wallets[currency] = self.wallet(**info) logger.info('Wallets synced ...') - - def update(self) -> None: - self._update_wallets() From c03337804808914afed09bcaafdd0e50d0cba135 Mon Sep 17 00:00:00 2001 From: misagh Date: Sun, 18 Nov 2018 14:57:03 +0100 Subject: [PATCH 385/699] change dict type to Any --- freqtrade/wallets.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/freqtrade/wallets.py b/freqtrade/wallets.py index fc2769831..b4b89bc59 100644 --- a/freqtrade/wallets.py +++ b/freqtrade/wallets.py @@ -1,7 +1,7 @@ # pragma pylint: disable=W0603 """ Wallet """ import logging -from typing import Dict +from typing import Dict, Any from collections import namedtuple from freqtrade.exchange import Exchange @@ -18,7 +18,7 @@ class Wallets(object): def __init__(self, exchange: Exchange) -> None: self.exchange = exchange - self.wallets: Dict[str, self.wallet] = {} + self.wallets: Dict[str, Any] = {} self.update() def update(self) -> None: From b680681b34e25d0641d2c65ca959c395403b9d2f Mon Sep 17 00:00:00 2001 From: misagh Date: Mon, 19 Nov 2018 11:16:07 +0100 Subject: [PATCH 386/699] updating wallet at handle timeout functions too --- freqtrade/freqtradebot.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index 83b5b85e5..570a806be 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -340,7 +340,9 @@ class FreqtradeBot(object): else: stake_amount = self.config['stake_amount'] + # TODO: should come from the wallet avaliable_amount = self.exchange.get_balance(self.config['stake_currency']) + #avaliable_amount = self.wallets.wallets[self.config['stake_currency']].free if stake_amount == constants.UNLIMITED_STAKE_AMOUNT: open_trades = len(Trade.query.filter(Trade.is_open.is_(True)).all()) @@ -707,8 +709,10 @@ class FreqtradeBot(object): if order['status'] == 'open': if order['side'] == 'buy' and ordertime < buy_timeoutthreashold: self.handle_timedout_limit_buy(trade, order) + self.wallets.update() elif order['side'] == 'sell' and ordertime < sell_timeoutthreashold: self.handle_timedout_limit_sell(trade, order) + self.wallets.update() # FIX: 20180110, why is cancel.order unconditionally here, whereas # it is conditionally called in the From 003480ad9031012f99a56c0d03af6b4a5415549a Mon Sep 17 00:00:00 2001 From: misagh Date: Mon, 19 Nov 2018 13:01:17 +0100 Subject: [PATCH 387/699] flake indentation --- freqtrade/freqtradebot.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index 570a806be..2a2a05c84 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -342,7 +342,7 @@ class FreqtradeBot(object): # TODO: should come from the wallet avaliable_amount = self.exchange.get_balance(self.config['stake_currency']) - #avaliable_amount = self.wallets.wallets[self.config['stake_currency']].free + # avaliable_amount = self.wallets.wallets[self.config['stake_currency']].free if stake_amount == constants.UNLIMITED_STAKE_AMOUNT: open_trades = len(Trade.query.filter(Trade.is_open.is_(True)).all()) From cf2d68501c02c0cc96e95102f0ce5a78798fba79 Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Mon, 19 Nov 2018 13:34:07 +0100 Subject: [PATCH 388/699] Update ccxt from 1.17.502 to 1.17.513 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 56e0d080a..9cb83d517 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -ccxt==1.17.502 +ccxt==1.17.513 SQLAlchemy==1.2.14 python-telegram-bot==11.1.0 arrow==0.12.1 From b50250139ec3171309ad62dd784b159e965245b9 Mon Sep 17 00:00:00 2001 From: misagh Date: Mon, 19 Nov 2018 20:02:26 +0100 Subject: [PATCH 389/699] Drafting stoploss on exchange --- freqtrade/strategy/interface.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/freqtrade/strategy/interface.py b/freqtrade/strategy/interface.py index 212559c8c..d1d4703a4 100644 --- a/freqtrade/strategy/interface.py +++ b/freqtrade/strategy/interface.py @@ -67,6 +67,11 @@ class IStrategy(ABC): # associated stoploss stoploss: float + # if the stoploss should be on exchange. + # if this is True then a stoploss order will be placed + # immediately after a successful buy order. + stoploss_on_exchange: bool = False + # associated ticker interval ticker_interval: str @@ -214,7 +219,11 @@ class IStrategy(ABC): # Set current rate to low for backtesting sell current_rate = low or rate current_profit = trade.calc_profit_percent(current_rate) - stoplossflag = self.stop_loss_reached(current_rate=current_rate, trade=trade, + + if self.stoploss_on_exchange: + stoplossflag = False + else: + stoplossflag = self.stop_loss_reached(current_rate=current_rate, trade=trade, current_time=date, current_profit=current_profit, force_stoploss=force_stoploss) if stoplossflag.sell_flag: From e69f9439118c07a9abb2f5b1467ec36181cc1bf2 Mon Sep 17 00:00:00 2001 From: Matthias Date: Thu, 15 Nov 2018 19:53:16 +0100 Subject: [PATCH 390/699] Add missing semicolon --- docs/hyperopt.md | 1 - setup.py | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/docs/hyperopt.md b/docs/hyperopt.md index 8df864c29..9fd11e5de 100644 --- a/docs/hyperopt.md +++ b/docs/hyperopt.md @@ -19,7 +19,6 @@ and still take a long time. ## Prepare Hyperopting -## Prepare Hyperopt Before we start digging in Hyperopt, we recommend you to take a look at an example hyperopt file located into [user_data/hyperopts/](https://github.com/gcarq/freqtrade/blob/develop/user_data/hyperopts/test_hyperopt.py) diff --git a/setup.py b/setup.py index 119ad03db..d2a9b97f9 100644 --- a/setup.py +++ b/setup.py @@ -30,7 +30,7 @@ setup(name='freqtrade', 'wrapt', 'pandas', 'scikit-learn', - 'joblib' + 'joblib', 'scipy', 'jsonschema', 'TA-Lib', From ce092742dab519130c69e73ac85980a0868fec1c Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Tue, 20 Nov 2018 13:34:07 +0100 Subject: [PATCH 391/699] Update ccxt from 1.17.513 to 1.17.518 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 9cb83d517..a845ddeb9 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -ccxt==1.17.513 +ccxt==1.17.518 SQLAlchemy==1.2.14 python-telegram-bot==11.1.0 arrow==0.12.1 From 5a550ef2af5227ae217a6fa5f57ff05a4697b2ec Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 20 Nov 2018 17:36:17 +0100 Subject: [PATCH 392/699] Fix docs typo in hyperopt --- docs/hyperopt.md | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/docs/hyperopt.md b/docs/hyperopt.md index 9fd11e5de..dffe84d1d 100644 --- a/docs/hyperopt.md +++ b/docs/hyperopt.md @@ -36,8 +36,7 @@ new buy hyperopt for testing: - Inside [populate_buy_trend()](https://github.com/freqtrade/freqtrade/blob/develop/user_data/hyperopts/test_hyperopt.py#L230-L251). - Inside [indicator_space()](https://github.com/freqtrade/freqtrade/blob/develop/user_data/hyperopts/test_hyperopt.py#L207-L223). -There you have two different type of indicators: 1. `guards` and 2. -`triggers`. +There you have two different types of indicators: 1. `guards` and 2. `triggers`. 1. Guards are conditions like "never buy if ADX < 10", or never buy if current price is over EMA10. From 5dd013c3b19dbb1fa1680d391aa9c068526418c8 Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 20 Nov 2018 17:40:45 +0100 Subject: [PATCH 393/699] Rename hyperopt interface and resolver --- freqtrade/optimize/default_hyperopt.py | 3 +-- freqtrade/optimize/hyperopt.py | 2 +- freqtrade/optimize/{interface.py => hyperopt_interface.py} | 0 freqtrade/optimize/{resolver.py => hyperopt_resolver.py} | 2 +- user_data/hyperopts/sample_hyperopt.py | 2 +- 5 files changed, 4 insertions(+), 5 deletions(-) rename freqtrade/optimize/{interface.py => hyperopt_interface.py} (100%) rename freqtrade/optimize/{resolver.py => hyperopt_resolver.py} (98%) diff --git a/freqtrade/optimize/default_hyperopt.py b/freqtrade/optimize/default_hyperopt.py index e127fd6d8..723dfde17 100644 --- a/freqtrade/optimize/default_hyperopt.py +++ b/freqtrade/optimize/default_hyperopt.py @@ -5,11 +5,10 @@ from pandas import DataFrame from typing import Dict, Any, Callable, List from functools import reduce -import numpy from skopt.space import Categorical, Dimension, Integer, Real import freqtrade.vendor.qtpylib.indicators as qtpylib -from freqtrade.optimize.interface import IHyperOpt +from freqtrade.optimize.hyperopt_interface import IHyperOpt class_name = 'DefaultHyperOpts' diff --git a/freqtrade/optimize/hyperopt.py b/freqtrade/optimize/hyperopt.py index f9a2924a4..bd6e43daa 100644 --- a/freqtrade/optimize/hyperopt.py +++ b/freqtrade/optimize/hyperopt.py @@ -22,7 +22,7 @@ from freqtrade.arguments import Arguments from freqtrade.configuration import Configuration from freqtrade.optimize import load_data from freqtrade.optimize.backtesting import Backtesting -from freqtrade.optimize.resolver import HyperOptResolver +from freqtrade.optimize.hyperopt_resolver import HyperOptResolver logger = logging.getLogger(__name__) diff --git a/freqtrade/optimize/interface.py b/freqtrade/optimize/hyperopt_interface.py similarity index 100% rename from freqtrade/optimize/interface.py rename to freqtrade/optimize/hyperopt_interface.py diff --git a/freqtrade/optimize/resolver.py b/freqtrade/optimize/hyperopt_resolver.py similarity index 98% rename from freqtrade/optimize/resolver.py rename to freqtrade/optimize/hyperopt_resolver.py index 1e0316c12..1c3a9d577 100644 --- a/freqtrade/optimize/resolver.py +++ b/freqtrade/optimize/hyperopt_resolver.py @@ -10,7 +10,7 @@ import os from typing import Optional, Dict, Type from freqtrade.constants import DEFAULT_HYPEROPT -from freqtrade.optimize.interface import IHyperOpt +from freqtrade.optimize.hyperopt_interface import IHyperOpt logger = logging.getLogger(__name__) diff --git a/user_data/hyperopts/sample_hyperopt.py b/user_data/hyperopts/sample_hyperopt.py index 77e3b1696..83dc5f71a 100644 --- a/user_data/hyperopts/sample_hyperopt.py +++ b/user_data/hyperopts/sample_hyperopt.py @@ -9,7 +9,7 @@ import numpy from skopt.space import Categorical, Dimension, Integer, Real import freqtrade.vendor.qtpylib.indicators as qtpylib -from freqtrade.optimize.interface import IHyperOpt +from freqtrade.optimize.hyperopt_interface import IHyperOpt class_name = 'SampleHyperOpts' From 7757c53b0668d53fac0c9b178e2def511c4101da Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 20 Nov 2018 17:43:49 +0100 Subject: [PATCH 394/699] Small fixes --- freqtrade/optimize/hyperopt.py | 9 ++++++--- setup.py | 2 +- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/freqtrade/optimize/hyperopt.py b/freqtrade/optimize/hyperopt.py index bd6e43daa..70d20673c 100644 --- a/freqtrade/optimize/hyperopt.py +++ b/freqtrade/optimize/hyperopt.py @@ -105,7 +105,8 @@ class Hyperopt(Backtesting): best_result['params'] ) if 'roi_t1' in best_result['params']: - logger.info('ROI table:\n%s', self.custom_hyperopt.generate_roi_table(best_result['params'])) + logger.info('ROI table:\n%s', + self.custom_hyperopt.generate_roi_table(best_result['params'])) def log_results(self, results) -> None: """ @@ -219,7 +220,8 @@ class Hyperopt(Backtesting): ) def run_optimizer_parallel(self, parallel, asked) -> List: - return parallel(delayed(wrap_non_picklable_objects(self.generate_optimizer))(v) for v in asked) + return parallel(delayed( + wrap_non_picklable_objects(self.generate_optimizer))(v) for v in asked) def load_previous_results(self): """ read trials file if we have one """ @@ -241,7 +243,8 @@ class Hyperopt(Backtesting): ) if self.has_space('buy'): - self.strategy.advise_indicators = self.custom_hyperopt.populate_indicators # type: ignore + self.strategy.advise_indicators = \ + self.custom_hyperopt.populate_indicators # type: ignore dump(self.strategy.tickerdata_to_dataframe(data), TICKERDATA_PICKLE) self.exchange = None # type: ignore self.load_previous_results() diff --git a/setup.py b/setup.py index d2a9b97f9..34d4c8755 100644 --- a/setup.py +++ b/setup.py @@ -30,8 +30,8 @@ setup(name='freqtrade', 'wrapt', 'pandas', 'scikit-learn', - 'joblib', 'scipy', + 'joblib', 'jsonschema', 'TA-Lib', 'tabulate', From a3b600411570eaf23443c19bb3322b2c36d14cc6 Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 20 Nov 2018 19:41:07 +0100 Subject: [PATCH 395/699] IHyperopt: all methods static, somef ixes for mypy --- freqtrade/optimize/default_hyperopt.py | 3 ++- freqtrade/optimize/hyperopt_interface.py | 21 ++++++++++++++------- freqtrade/optimize/hyperopt_resolver.py | 4 ++-- user_data/hyperopts/sample_hyperopt.py | 3 ++- 4 files changed, 20 insertions(+), 11 deletions(-) diff --git a/freqtrade/optimize/default_hyperopt.py b/freqtrade/optimize/default_hyperopt.py index 723dfde17..6139f8140 100644 --- a/freqtrade/optimize/default_hyperopt.py +++ b/freqtrade/optimize/default_hyperopt.py @@ -36,7 +36,8 @@ class DefaultHyperOpts(IHyperOpt): dataframe['sar'] = ta.SAR(dataframe) return dataframe - def buy_strategy_generator(self, params: Dict[str, Any]) -> Callable: + @staticmethod + def buy_strategy_generator(params: Dict[str, Any]) -> Callable: """ Define the buy strategy parameters to be used by hyperopt """ diff --git a/freqtrade/optimize/hyperopt_interface.py b/freqtrade/optimize/hyperopt_interface.py index 8bc3866b2..d42206658 100644 --- a/freqtrade/optimize/hyperopt_interface.py +++ b/freqtrade/optimize/hyperopt_interface.py @@ -4,9 +4,10 @@ This module defines the interface to apply for hyperopts """ from abc import ABC, abstractmethod -from typing import Dict, Any, Callable +from typing import Dict, Any, Callable, List from pandas import DataFrame +from skopt.space import Dimension class IHyperOpt(ABC): @@ -20,40 +21,46 @@ class IHyperOpt(ABC): ticker_interval -> int: value of the ticker interval to use for the strategy """ + @staticmethod @abstractmethod - def populate_indicators(self, dataframe: DataFrame) -> DataFrame: + def populate_indicators(dataframe: DataFrame, metadata: dict) -> DataFrame: """ Populate indicators that will be used in the Buy and Sell strategy :param dataframe: Raw data from the exchange and parsed by parse_ticker_dataframe() :return: a Dataframe with all mandatory indicators for the strategies """ + @staticmethod @abstractmethod - def buy_strategy_generator(self, params: Dict[str, Any]) -> Callable: + def buy_strategy_generator(params: Dict[str, Any]) -> Callable: """ Create a buy strategy generator """ + @staticmethod @abstractmethod - def indicator_space(self) -> Dict[str, Any]: + def indicator_space() -> List[Dimension]: """ Create an indicator space """ + @staticmethod @abstractmethod - def generate_roi_table(self, params: Dict) -> Dict[int, float]: + def generate_roi_table(params: Dict) -> Dict[int, float]: """ Create an roi table """ + @staticmethod @abstractmethod - def stoploss_space(self) -> Dict[str, Any]: + def stoploss_space() -> List[Dimension]: """ Create a stoploss space """ + @staticmethod @abstractmethod - def roi_space(self) -> Dict[str, Any]: + def roi_space() -> List[Dimension]: """ Create a roi space """ diff --git a/freqtrade/optimize/hyperopt_resolver.py b/freqtrade/optimize/hyperopt_resolver.py index 1c3a9d577..3d019e8df 100644 --- a/freqtrade/optimize/hyperopt_resolver.py +++ b/freqtrade/optimize/hyperopt_resolver.py @@ -35,7 +35,7 @@ class HyperOptResolver(object): self.hyperopt = self._load_hyperopt(hyperopt_name, extra_dir=config.get('hyperopt_path')) def _load_hyperopt( - self, hyperopt_name: str, extra_dir: Optional[str] = None) -> Optional[IHyperOpt]: + self, hyperopt_name: str, extra_dir: Optional[str] = None) -> IHyperOpt: """ Search and loads the specified hyperopt. :param hyperopt_name: name of the module to import @@ -75,7 +75,7 @@ class HyperOptResolver(object): # Generate spec based on absolute path spec = importlib.util.spec_from_file_location('user_data.hyperopts', module_path) module = importlib.util.module_from_spec(spec) - spec.loader.exec_module(module) + spec.loader.exec_module(module) # type: ignore # importlib does not use typehints valid_hyperopts_gen = ( obj for name, obj in inspect.getmembers(module, inspect.isclass) diff --git a/user_data/hyperopts/sample_hyperopt.py b/user_data/hyperopts/sample_hyperopt.py index 83dc5f71a..f11236a82 100644 --- a/user_data/hyperopts/sample_hyperopt.py +++ b/user_data/hyperopts/sample_hyperopt.py @@ -45,7 +45,8 @@ class SampleHyperOpts(IHyperOpt): dataframe['sar'] = ta.SAR(dataframe) return dataframe - def buy_strategy_generator(self, params: Dict[str, Any]) -> Callable: + @staticmethod + def buy_strategy_generator(params: Dict[str, Any]) -> Callable: """ Define the buy strategy parameters to be used by hyperopt """ From d745e577b4229803dad1ccf1cd5072620197f9ce Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Wed, 21 Nov 2018 13:34:06 +0100 Subject: [PATCH 396/699] Update ccxt from 1.17.518 to 1.17.522 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index a845ddeb9..821ff8b4f 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -ccxt==1.17.518 +ccxt==1.17.522 SQLAlchemy==1.2.14 python-telegram-bot==11.1.0 arrow==0.12.1 From 64129897f9fe696e991af1456f75ddf035f3b799 Mon Sep 17 00:00:00 2001 From: misagh Date: Wed, 21 Nov 2018 14:00:15 +0100 Subject: [PATCH 397/699] refresh_ticker should be called just once per iteration. --- freqtrade/freqtradebot.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index eb92375ec..5ab78ab35 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -194,9 +194,6 @@ class FreqtradeBot(object): self.edge.calculate() self.active_pair_whitelist = self.edge.adjust(self.active_pair_whitelist) - # Refreshing candles - self.exchange.refresh_tickers(self.active_pair_whitelist, self.strategy.ticker_interval) - # Query trades from persistence layer trades = Trade.query.filter(Trade.is_open.is_(True)).all() From 68f81aa2afb61509bca05a80da16d2d9ee1d5387 Mon Sep 17 00:00:00 2001 From: misagh Date: Wed, 21 Nov 2018 17:27:45 +0100 Subject: [PATCH 398/699] test wallets moved to tests folder --- freqtrade/tests/{finance => }/test_wallets.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename freqtrade/tests/{finance => }/test_wallets.py (100%) diff --git a/freqtrade/tests/finance/test_wallets.py b/freqtrade/tests/test_wallets.py similarity index 100% rename from freqtrade/tests/finance/test_wallets.py rename to freqtrade/tests/test_wallets.py From 5b689402130d2a45ccdc8a65bbac669b5e247bdb Mon Sep 17 00:00:00 2001 From: misagh Date: Wed, 21 Nov 2018 17:48:53 +0100 Subject: [PATCH 399/699] update wallet in casse order remaining is zero --- freqtrade/freqtradebot.py | 1 + 1 file changed, 1 insertion(+) diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index 2a2a05c84..7d6fdd261 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -703,6 +703,7 @@ class FreqtradeBot(object): # Check if trade is still actually open if int(order['remaining']) == 0: + self.wallets.update() continue # Check if trade is still actually open From aeb372c2f04ac992e5d6c82dca34ddb770100424 Mon Sep 17 00:00:00 2001 From: misagh Date: Wed, 21 Nov 2018 17:54:14 +0100 Subject: [PATCH 400/699] test wallet when api return changes --- freqtrade/tests/test_wallets.py | 27 ++++++++++++++++++++++++++- 1 file changed, 26 insertions(+), 1 deletion(-) diff --git a/freqtrade/tests/test_wallets.py b/freqtrade/tests/test_wallets.py index e226d117f..a5295579b 100644 --- a/freqtrade/tests/test_wallets.py +++ b/freqtrade/tests/test_wallets.py @@ -26,7 +26,32 @@ def test_sync_wallet_at_boot(mocker, default_conf): assert freqtrade.wallets.wallets['BNT'].free == 1.0 assert freqtrade.wallets.wallets['BNT'].used == 2.0 assert freqtrade.wallets.wallets['BNT'].total == 3.0 - assert freqtrade.wallets.wallets['GAS'].free == 0.260739 assert freqtrade.wallets.wallets['GAS'].used == 0.0 assert freqtrade.wallets.wallets['GAS'].total == 0.260739 + + mocker.patch.multiple( + 'freqtrade.exchange.Exchange', + get_balances=MagicMock(return_value={ + "BNT": { + "free": 1.2, + "used": 1.9, + "total": 3.5 + }, + "GAS": { + "free": 0.270739, + "used": 0.1, + "total": 0.260439 + }, + }) + ) + + freqtrade.wallets.update() + + assert len(freqtrade.wallets.wallets) == 2 + assert freqtrade.wallets.wallets['BNT'].free == 1.2 + assert freqtrade.wallets.wallets['BNT'].used == 1.9 + assert freqtrade.wallets.wallets['BNT'].total == 3.5 + assert freqtrade.wallets.wallets['GAS'].free == 0.270739 + assert freqtrade.wallets.wallets['GAS'].used == 0.1 + assert freqtrade.wallets.wallets['GAS'].total == 0.260439 From cb3cf960d734373618f878bde2632317a366dd25 Mon Sep 17 00:00:00 2001 From: misagh Date: Wed, 21 Nov 2018 19:47:28 +0100 Subject: [PATCH 401/699] tests added in case of missing data --- freqtrade/tests/test_wallets.py | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/freqtrade/tests/test_wallets.py b/freqtrade/tests/test_wallets.py index a5295579b..cc10d665c 100644 --- a/freqtrade/tests/test_wallets.py +++ b/freqtrade/tests/test_wallets.py @@ -55,3 +55,30 @@ def test_sync_wallet_at_boot(mocker, default_conf): assert freqtrade.wallets.wallets['GAS'].free == 0.270739 assert freqtrade.wallets.wallets['GAS'].used == 0.1 assert freqtrade.wallets.wallets['GAS'].total == 0.260439 + + +def test_sync_wallet_missing_data(mocker, default_conf): + mocker.patch.multiple( + 'freqtrade.exchange.Exchange', + get_balances=MagicMock(return_value={ + "BNT": { + "free": 1.0, + "used": 2.0, + "total": 3.0 + }, + "GAS": { + "free": 0.260739, + "total": 0.260739 + }, + }) + ) + + freqtrade = get_patched_freqtradebot(mocker, default_conf) + + assert len(freqtrade.wallets.wallets) == 2 + assert freqtrade.wallets.wallets['BNT'].free == 1.0 + assert freqtrade.wallets.wallets['BNT'].used == 2.0 + assert freqtrade.wallets.wallets['BNT'].total == 3.0 + assert freqtrade.wallets.wallets['GAS'].free == 0.260739 + assert freqtrade.wallets.wallets['GAS'].used is None + assert freqtrade.wallets.wallets['GAS'].total == 0.260739 From 88f61581d924138b0926a5459a634459c517cf68 Mon Sep 17 00:00:00 2001 From: misagh Date: Wed, 21 Nov 2018 19:47:51 +0100 Subject: [PATCH 402/699] 1) NamedTuple refactored 2) Missing data handled --- freqtrade/wallets.py | 26 ++++++++++++++++---------- 1 file changed, 16 insertions(+), 10 deletions(-) diff --git a/freqtrade/wallets.py b/freqtrade/wallets.py index b4b89bc59..5ee33f41e 100644 --- a/freqtrade/wallets.py +++ b/freqtrade/wallets.py @@ -1,13 +1,21 @@ # pragma pylint: disable=W0603 """ Wallet """ import logging -from typing import Dict, Any +from typing import Dict, Any, NamedTuple from collections import namedtuple from freqtrade.exchange import Exchange logger = logging.getLogger(__name__) +class Wallet(NamedTuple): + exchange: str = None + currency: str = None + free: float = 0 + used: float = 0 + total: float = 0 + + class Wallets(object): # wallet data structure @@ -25,14 +33,12 @@ class Wallets(object): balances = self.exchange.get_balances() for currency in balances: - info = { - 'exchange': self.exchange.id, - 'currency': currency, - 'free': balances[currency]['free'], - 'used': balances[currency]['used'], - 'total': balances[currency]['total'] - } - - self.wallets[currency] = self.wallet(**info) + self.wallets[currency] = Wallet( + self.exchange.id, + currency, + balances[currency].get('free', None), + balances[currency].get('used', None), + balances[currency].get('total', None) + ) logger.info('Wallets synced ...') From b129750f4dbff4daca892b6ebefdf1648637caff Mon Sep 17 00:00:00 2001 From: misagh Date: Wed, 21 Nov 2018 19:58:28 +0100 Subject: [PATCH 403/699] =?UTF-8?q?adding=20=E2=80=9Coptional=E2=80=9D=20i?= =?UTF-8?q?n=20str?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- freqtrade/wallets.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/freqtrade/wallets.py b/freqtrade/wallets.py index 5ee33f41e..531e83a28 100644 --- a/freqtrade/wallets.py +++ b/freqtrade/wallets.py @@ -1,7 +1,7 @@ # pragma pylint: disable=W0603 """ Wallet """ import logging -from typing import Dict, Any, NamedTuple +from typing import Dict, Any, NamedTuple, Optional from collections import namedtuple from freqtrade.exchange import Exchange @@ -9,8 +9,8 @@ logger = logging.getLogger(__name__) class Wallet(NamedTuple): - exchange: str = None - currency: str = None + exchange: Optional[str] = None + currency: Optional[str] = None free: float = 0 used: float = 0 total: float = 0 From 4d75e9059c0f3c10a8c910bea83ff74292ea7367 Mon Sep 17 00:00:00 2001 From: misagh Date: Wed, 21 Nov 2018 21:05:20 +0100 Subject: [PATCH 404/699] None ripped off for optional as wallet must have exchange and currency --- freqtrade/wallets.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/freqtrade/wallets.py b/freqtrade/wallets.py index 531e83a28..3b4152b8d 100644 --- a/freqtrade/wallets.py +++ b/freqtrade/wallets.py @@ -9,8 +9,8 @@ logger = logging.getLogger(__name__) class Wallet(NamedTuple): - exchange: Optional[str] = None - currency: Optional[str] = None + exchange: Optional[str] + currency: Optional[str] free: float = 0 used: float = 0 total: float = 0 From 3a2134db24d21285cafa8b65c6257cc0e380b928 Mon Sep 17 00:00:00 2001 From: misagh Date: Wed, 21 Nov 2018 23:35:44 +0100 Subject: [PATCH 405/699] removed Optional --- freqtrade/wallets.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/freqtrade/wallets.py b/freqtrade/wallets.py index 3b4152b8d..65a5e880f 100644 --- a/freqtrade/wallets.py +++ b/freqtrade/wallets.py @@ -9,8 +9,8 @@ logger = logging.getLogger(__name__) class Wallet(NamedTuple): - exchange: Optional[str] - currency: Optional[str] + exchange: str + currency: str free: float = 0 used: float = 0 total: float = 0 From 4b86b2b7e3fa89e16d5a40625f9af12cf488151c Mon Sep 17 00:00:00 2001 From: misagh Date: Wed, 21 Nov 2018 23:36:48 +0100 Subject: [PATCH 406/699] Happy flake8 ! --- freqtrade/wallets.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/wallets.py b/freqtrade/wallets.py index 65a5e880f..82f527d2c 100644 --- a/freqtrade/wallets.py +++ b/freqtrade/wallets.py @@ -1,7 +1,7 @@ # pragma pylint: disable=W0603 """ Wallet """ import logging -from typing import Dict, Any, NamedTuple, Optional +from typing import Dict, Any, NamedTuple from collections import namedtuple from freqtrade.exchange import Exchange From eb53281434df691ef954dcb8b964ad7c4eeba74e Mon Sep 17 00:00:00 2001 From: misagh Date: Thu, 22 Nov 2018 00:04:20 +0100 Subject: [PATCH 407/699] python beginner problem resolved --- freqtrade/edge/__init__.py | 39 ++++++++++++----------- freqtrade/tests/edge/test_edge.py | 14 ++++---- freqtrade/tests/optimize/test_edge_cli.py | 13 ++------ 3 files changed, 29 insertions(+), 37 deletions(-) diff --git a/freqtrade/edge/__init__.py b/freqtrade/edge/__init__.py index dedaa19a3..009b80664 100644 --- a/freqtrade/edge/__init__.py +++ b/freqtrade/edge/__init__.py @@ -1,8 +1,7 @@ # pragma pylint: disable=W0603 """ Edge positioning package """ import logging -from typing import Any, Dict -from collections import namedtuple +from typing import Any, Dict, NamedTuple import arrow import numpy as np @@ -18,6 +17,16 @@ from freqtrade.strategy.interface import SellType logger = logging.getLogger(__name__) +class PairInfo(NamedTuple): + stoploss: float + winrate: float + risk_reward_ratio: float + required_risk_reward: float + expectancy: float + nb_trades: int + avg_trade_duration: float + + class Edge(): """ Calculates Win Rate, Risk Reward Ratio, Expectancy @@ -30,13 +39,6 @@ class Edge(): config: Dict = {} _cached_pairs: Dict[str, Any] = {} # Keeps a list of pairs - # pair info data type - _pair_info = namedtuple( - 'pair_info', - ['stoploss', 'winrate', 'risk_reward_ratio', 'required_risk_reward', 'expectancy', - 'nb_trades', 'avg_trade_duration'] - ) - def __init__(self, config: Dict[str, Any], exchange, strategy) -> None: self.config = config @@ -294,16 +296,15 @@ class Edge(): final = {} for x in df.itertuples(): - info = { - 'stoploss': x.stoploss, - 'winrate': x.winrate, - 'risk_reward_ratio': x.risk_reward_ratio, - 'required_risk_reward': x.required_risk_reward, - 'expectancy': x.expectancy, - 'nb_trades': x.nb_trades, - 'avg_trade_duration': x.avg_trade_duration - } - final[x.pair] = self._pair_info(**info) + final[x.pair] = PairInfo( + x.stoploss, + x.winrate, + x.risk_reward_ratio, + x.required_risk_reward, + x.expectancy, + x.nb_trades, + x.avg_trade_duration + ) # Returning a list of pairs in order of "expectancy" return final diff --git a/freqtrade/tests/edge/test_edge.py b/freqtrade/tests/edge/test_edge.py index 14c9114c3..fac055c17 100644 --- a/freqtrade/tests/edge/test_edge.py +++ b/freqtrade/tests/edge/test_edge.py @@ -4,7 +4,7 @@ import pytest import logging from freqtrade.tests.conftest import get_patched_freqtradebot -from freqtrade.edge import Edge +from freqtrade.edge import Edge, PairInfo from pandas import DataFrame, to_datetime from freqtrade.strategy.interface import SellType from freqtrade.tests.optimize import (BTrade, BTContainer, _build_backtest_dataframe, @@ -128,9 +128,9 @@ def test_adjust(mocker, default_conf): edge = Edge(default_conf, freqtrade.exchange, freqtrade.strategy) mocker.patch('freqtrade.edge.Edge._cached_pairs', mocker.PropertyMock( return_value={ - 'E/F': Edge._pair_info(-0.01, 0.66, 3.71, 0.50, 1.71, 10, 60), - 'C/D': Edge._pair_info(-0.01, 0.66, 3.71, 0.50, 1.71, 10, 60), - 'N/O': Edge._pair_info(-0.01, 0.66, 3.71, 0.50, 1.71, 10, 60) + 'E/F': PairInfo(-0.01, 0.66, 3.71, 0.50, 1.71, 10, 60), + 'C/D': PairInfo(-0.01, 0.66, 3.71, 0.50, 1.71, 10, 60), + 'N/O': PairInfo(-0.01, 0.66, 3.71, 0.50, 1.71, 10, 60) } )) @@ -143,9 +143,9 @@ def test_stoploss(mocker, default_conf): edge = Edge(default_conf, freqtrade.exchange, freqtrade.strategy) mocker.patch('freqtrade.edge.Edge._cached_pairs', mocker.PropertyMock( return_value={ - 'E/F': Edge._pair_info(-0.01, 0.66, 3.71, 0.50, 1.71, 10, 60), - 'C/D': Edge._pair_info(-0.01, 0.66, 3.71, 0.50, 1.71, 10, 60), - 'N/O': Edge._pair_info(-0.01, 0.66, 3.71, 0.50, 1.71, 10, 60) + 'E/F': PairInfo(-0.01, 0.66, 3.71, 0.50, 1.71, 10, 60), + 'C/D': PairInfo(-0.01, 0.66, 3.71, 0.50, 1.71, 10, 60), + 'N/O': PairInfo(-0.01, 0.66, 3.71, 0.50, 1.71, 10, 60) } )) diff --git a/freqtrade/tests/optimize/test_edge_cli.py b/freqtrade/tests/optimize/test_edge_cli.py index f8db3dec4..0d0f64e0c 100644 --- a/freqtrade/tests/optimize/test_edge_cli.py +++ b/freqtrade/tests/optimize/test_edge_cli.py @@ -4,7 +4,7 @@ from unittest.mock import MagicMock import json from typing import List -from freqtrade.edge import Edge +from freqtrade.edge import PairInfo from freqtrade.arguments import Arguments from freqtrade.optimize.edge_cli import (EdgeCli, setup_configuration, start) from freqtrade.tests.conftest import log_has, patch_exchange @@ -123,17 +123,8 @@ def test_generate_edge_table(edge_conf, mocker): edge_cli = EdgeCli(edge_conf) results = {} - info = { - 'stoploss': -0.01, - 'winrate': 0.60, - 'risk_reward_ratio': 2, - 'required_risk_reward': 1, - 'expectancy': 3, - 'nb_trades': 10, - 'avg_trade_duration': 60 - } + results['ETH/BTC'] = PairInfo(-0.01, 0.60, 2, 1, 3, 10, 60) - results['ETH/BTC'] = Edge._pair_info(**info) assert edge_cli._generate_edge_table(results).count(':|') == 7 assert edge_cli._generate_edge_table(results).count('| ETH/BTC |') == 1 assert edge_cli._generate_edge_table(results).count( From f73a18c56c8a66e109a2b47339c66e395c829121 Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Thu, 22 Nov 2018 13:34:06 +0100 Subject: [PATCH 408/699] Update ccxt from 1.17.522 to 1.17.529 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 1b271e09e..34a9b477e 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -ccxt==1.17.522 +ccxt==1.17.529 SQLAlchemy==1.2.14 python-telegram-bot==11.1.0 arrow==0.12.1 From bfbdddff26bd9296da0b046f260db1ce5778b703 Mon Sep 17 00:00:00 2001 From: misagh Date: Thu, 22 Nov 2018 16:24:40 +0100 Subject: [PATCH 409/699] stoploss limit order added to exchange --- freqtrade/exchange/__init__.py | 18 ++++++++++++++++++ freqtrade/strategy/interface.py | 2 ++ 2 files changed, 20 insertions(+) diff --git a/freqtrade/exchange/__init__.py b/freqtrade/exchange/__init__.py index ae07e36e9..59a5da23e 100644 --- a/freqtrade/exchange/__init__.py +++ b/freqtrade/exchange/__init__.py @@ -333,6 +333,24 @@ class Exchange(object): except ccxt.BaseError as e: raise OperationalException(e) + def stoploss_limit(self, pair: str, amount: float, stop_price: float, rate: float) -> Dict: + # Only binance is supported + if not self._api.name == 'Binance': + raise NotImplementedError( + 'Stoploss limit orders are implemented only for binance as of now.') + + # Set the precision for amount and price(rate) as accepted by the exchange + amount = self.symbol_amount_prec(pair, amount) + rate = self.symbol_price_prec(pair, rate) + stop_price = self.symbol_price_prec(pair, stop_price) + + # Ensure rate is less than stop price + if stop_price >= rate: + raise OperationalException( + 'In stoploss limit order, stop price should be more than limit price') + + return self._api.create_order(pair, 'stop_loss', 'sell', amount, rate, {'stopPrice': stop_price}) + @retrier def get_balance(self, currency: str) -> float: if self._conf['dry_run']: diff --git a/freqtrade/strategy/interface.py b/freqtrade/strategy/interface.py index 9b7b180cc..df0e3cf72 100644 --- a/freqtrade/strategy/interface.py +++ b/freqtrade/strategy/interface.py @@ -233,8 +233,10 @@ class IStrategy(ABC): stoplossflag = self.stop_loss_reached(current_rate=current_rate, trade=trade, current_time=date, current_profit=current_profit, force_stoploss=force_stoploss) + if stoplossflag.sell_flag: return stoplossflag + # Set current rate to low for backtesting sell current_rate = high or rate current_profit = trade.calc_profit_percent(current_rate) From 3b7e05e07b15c1df1bfe62bfbf264feac9877784 Mon Sep 17 00:00:00 2001 From: misagh Date: Thu, 22 Nov 2018 16:26:24 +0100 Subject: [PATCH 410/699] stop loss order added right after a buy order is executued --- freqtrade/freqtradebot.py | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index 8a2db84a9..f1aae3c3f 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -508,6 +508,37 @@ class FreqtradeBot(object): Trade.session.add(trade) Trade.session.flush() + # Check if stoploss should be added on exchange + # If True then here immediately after buy we should + # Add the stoploss order + if self.strategy.stoploss_on_exchange: + stoploss = self.edge.stoploss if self.edge else self.strategy.stoploss + stop_price = buy_limit * (1 + stoploss) + + # limit price should be less than stop price. + # 0.98 is arbitrary here. + limit_price = stop_price * 0.98 + + order_id = self.exchange.stoploss_limit(pair=pair, amount=amount, + stop_price=stop_price, rate=limit_price)['id'] + + trade = Trade( + pair=pair, + stake_amount=stake_amount, + amount=amount, + fee_open=fee, + fee_close=fee, + stoploss=stop_price, + open_date=datetime.utcnow(), + exchange=self.exchange.id, + open_order_id=order_id, + strategy=self.strategy.get_strategy_name(), + ticker_interval=constants.TICKER_INTERVAL_MINUTES[self.config['ticker_interval']] + ) + + Trade.session.add(trade) + Trade.session.flush() + # Updating wallets self.wallets.update() From bb37b56dea02fdf7e5b51e54ae7144e808ab2e08 Mon Sep 17 00:00:00 2001 From: misagh Date: Thu, 22 Nov 2018 16:47:52 +0100 Subject: [PATCH 411/699] adding stop loss order id to Trade --- freqtrade/persistence.py | 1 + 1 file changed, 1 insertion(+) diff --git a/freqtrade/persistence.py b/freqtrade/persistence.py index 51a8129fb..db6d526c7 100644 --- a/freqtrade/persistence.py +++ b/freqtrade/persistence.py @@ -178,6 +178,7 @@ class Trade(_DECL_BASE): # absolute value of the initial stop loss initial_stop_loss = Column(Float, nullable=True, default=0.0) # absolute value of the highest reached price + stoploss_order_id = Column(Integer, nullable=True, index=True) max_rate = Column(Float, nullable=True, default=0.0) sell_reason = Column(String, nullable=True) strategy = Column(String, nullable=True) From fad75939356188b1bf2d8ca7eb82fa5e4201ba4a Mon Sep 17 00:00:00 2001 From: misagh Date: Thu, 22 Nov 2018 16:53:50 +0100 Subject: [PATCH 412/699] =?UTF-8?q?doesn=E2=80=99t=20have=20to=20create=20?= =?UTF-8?q?another=20Trade=20for=20SL.=20can=20be=20cumulated=20into=20the?= =?UTF-8?q?=20same.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- freqtrade/freqtradebot.py | 48 ++++++++++++++------------------------- 1 file changed, 17 insertions(+), 31 deletions(-) diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index f1aae3c3f..f3537f2ab 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -479,6 +479,22 @@ class FreqtradeBot(object): order_id = self.exchange.buy(pair=pair, ordertype=self.strategy.order_types['buy'], amount=amount, rate=buy_limit)['id'] + stoploss_order_id: int = None + + # Check if stoploss should be added on exchange + # If True then here immediately after buy we should + # Add the stoploss order + if self.strategy.stoploss_on_exchange: + stoploss = self.edge.stoploss if self.edge else self.strategy.stoploss + stop_price = buy_limit * (1 + stoploss) + + # limit price should be less than stop price. + # 0.98 is arbitrary here. + limit_price = stop_price * 0.98 + + stoploss_order_id = self.exchange.stoploss_limit(pair=pair, amount=amount, + stop_price=stop_price, rate=limit_price)['id'] + self.rpc.send_msg({ 'type': RPCMessageType.BUY_NOTIFICATION, 'exchange': self.exchange.name.capitalize(), @@ -502,43 +518,13 @@ class FreqtradeBot(object): open_date=datetime.utcnow(), exchange=self.exchange.id, open_order_id=order_id, + stoploss_order_id=stoploss_order_id, strategy=self.strategy.get_strategy_name(), ticker_interval=constants.TICKER_INTERVAL_MINUTES[self.config['ticker_interval']] ) Trade.session.add(trade) Trade.session.flush() - # Check if stoploss should be added on exchange - # If True then here immediately after buy we should - # Add the stoploss order - if self.strategy.stoploss_on_exchange: - stoploss = self.edge.stoploss if self.edge else self.strategy.stoploss - stop_price = buy_limit * (1 + stoploss) - - # limit price should be less than stop price. - # 0.98 is arbitrary here. - limit_price = stop_price * 0.98 - - order_id = self.exchange.stoploss_limit(pair=pair, amount=amount, - stop_price=stop_price, rate=limit_price)['id'] - - trade = Trade( - pair=pair, - stake_amount=stake_amount, - amount=amount, - fee_open=fee, - fee_close=fee, - stoploss=stop_price, - open_date=datetime.utcnow(), - exchange=self.exchange.id, - open_order_id=order_id, - strategy=self.strategy.get_strategy_name(), - ticker_interval=constants.TICKER_INTERVAL_MINUTES[self.config['ticker_interval']] - ) - - Trade.session.add(trade) - Trade.session.flush() - # Updating wallets self.wallets.update() From da5617624c97389b0a1aabd3e95e18258e6b6348 Mon Sep 17 00:00:00 2001 From: misagh Date: Thu, 22 Nov 2018 17:02:02 +0100 Subject: [PATCH 413/699] cancelling stop loss order before selling --- freqtrade/freqtradebot.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index f3537f2ab..744f92156 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -505,6 +505,7 @@ class FreqtradeBot(object): 'stake_currency': stake_currency, 'fiat_currency': fiat_currency }) + # Fee is applied twice because we make a LIMIT_BUY and LIMIT_SELL fee = self.exchange.get_fee(symbol=pair, taker_or_maker='maker') trade = Trade( @@ -522,6 +523,7 @@ class FreqtradeBot(object): strategy=self.strategy.get_strategy_name(), ticker_interval=constants.TICKER_INTERVAL_MINUTES[self.config['ticker_interval']] ) + Trade.session.add(trade) Trade.session.flush() @@ -798,6 +800,11 @@ class FreqtradeBot(object): sell_type = 'sell' if sell_reason in (SellType.STOP_LOSS, SellType.TRAILING_STOP_LOSS): sell_type = 'stoploss' + + # First cancelling stoploss on exchange ... + if self.strategy.stoploss_on_exchange and trade.stoploss_order_id: + self.exchange.cancel_order(trade.stoploss_order_id, trade.pair) + # Execute sell and update trade record order_id = self.exchange.sell(pair=str(trade.pair), ordertype=self.strategy.order_types[sell_type], From bbe8e4e49456b8ec5de23ed22901dfb1c4c82561 Mon Sep 17 00:00:00 2001 From: misagh Date: Thu, 22 Nov 2018 17:07:37 +0100 Subject: [PATCH 414/699] flake8 --- freqtrade/exchange/__init__.py | 3 ++- freqtrade/freqtradebot.py | 4 ++-- freqtrade/strategy/interface.py | 4 ++-- 3 files changed, 6 insertions(+), 5 deletions(-) diff --git a/freqtrade/exchange/__init__.py b/freqtrade/exchange/__init__.py index 59a5da23e..53ae6c2d7 100644 --- a/freqtrade/exchange/__init__.py +++ b/freqtrade/exchange/__init__.py @@ -349,7 +349,8 @@ class Exchange(object): raise OperationalException( 'In stoploss limit order, stop price should be more than limit price') - return self._api.create_order(pair, 'stop_loss', 'sell', amount, rate, {'stopPrice': stop_price}) + return self._api.create_order(pair, 'stop_loss', 'sell', + amount, rate, {'stopPrice': stop_price}) @retrier def get_balance(self, currency: str) -> float: diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index 744f92156..40734f385 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -492,8 +492,8 @@ class FreqtradeBot(object): # 0.98 is arbitrary here. limit_price = stop_price * 0.98 - stoploss_order_id = self.exchange.stoploss_limit(pair=pair, amount=amount, - stop_price=stop_price, rate=limit_price)['id'] + stoploss_order_id = self.exchange.stoploss_limit( + pair=pair, amount=amount, stop_price=stop_price, rate=limit_price)['id'] self.rpc.send_msg({ 'type': RPCMessageType.BUY_NOTIFICATION, diff --git a/freqtrade/strategy/interface.py b/freqtrade/strategy/interface.py index df0e3cf72..9047d8807 100644 --- a/freqtrade/strategy/interface.py +++ b/freqtrade/strategy/interface.py @@ -231,8 +231,8 @@ class IStrategy(ABC): stoplossflag = False else: stoplossflag = self.stop_loss_reached(current_rate=current_rate, trade=trade, - current_time=date, current_profit=current_profit, - force_stoploss=force_stoploss) + current_time=date, current_profit=current_profit, + force_stoploss=force_stoploss) if stoplossflag.sell_flag: return stoplossflag From 3a1c378325dc97b48bb1ad767fc5ff8281bdf88c Mon Sep 17 00:00:00 2001 From: misagh Date: Thu, 22 Nov 2018 17:14:22 +0100 Subject: [PATCH 415/699] typing bugs --- freqtrade/freqtradebot.py | 4 ++-- freqtrade/strategy/interface.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index 40734f385..ac9fd758c 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -479,13 +479,13 @@ class FreqtradeBot(object): order_id = self.exchange.buy(pair=pair, ordertype=self.strategy.order_types['buy'], amount=amount, rate=buy_limit)['id'] - stoploss_order_id: int = None + stoploss_order_id = None # Check if stoploss should be added on exchange # If True then here immediately after buy we should # Add the stoploss order if self.strategy.stoploss_on_exchange: - stoploss = self.edge.stoploss if self.edge else self.strategy.stoploss + stoploss = self.edge.stoploss(pair=pair) if self.edge else self.strategy.stoploss stop_price = buy_limit * (1 + stoploss) # limit price should be less than stop price. diff --git a/freqtrade/strategy/interface.py b/freqtrade/strategy/interface.py index 9047d8807..30fc62f42 100644 --- a/freqtrade/strategy/interface.py +++ b/freqtrade/strategy/interface.py @@ -228,7 +228,7 @@ class IStrategy(ABC): current_profit = trade.calc_profit_percent(current_rate) if self.stoploss_on_exchange: - stoplossflag = False + stoplossflag = SellCheckTuple(sell_flag=False, sell_type=SellType.NONE) else: stoplossflag = self.stop_loss_reached(current_rate=current_rate, trade=trade, current_time=date, current_profit=current_profit, From 2461d86c8d2d0dd692edfabf4fd79162eb87fbd4 Mon Sep 17 00:00:00 2001 From: misagh Date: Thu, 22 Nov 2018 17:24:45 +0100 Subject: [PATCH 416/699] dry run should consider stop loss is hit on limit price --- freqtrade/freqtradebot.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index ac9fd758c..281b22bc5 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -805,6 +805,11 @@ class FreqtradeBot(object): if self.strategy.stoploss_on_exchange and trade.stoploss_order_id: self.exchange.cancel_order(trade.stoploss_order_id, trade.pair) + # Dry-run should consider stoploss is executed at the limit price + # So overriding limit in case of dry-run + if self.config['dry_run']: + limit = trade.stop_loss + # Execute sell and update trade record order_id = self.exchange.sell(pair=str(trade.pair), ordertype=self.strategy.order_types[sell_type], From 24df093a85ea65293b6baa653ab515049e4e0e56 Mon Sep 17 00:00:00 2001 From: misagh Date: Thu, 22 Nov 2018 17:41:01 +0100 Subject: [PATCH 417/699] test: only implemented for binance --- freqtrade/tests/exchange/test_exchange.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/freqtrade/tests/exchange/test_exchange.py b/freqtrade/tests/exchange/test_exchange.py index 207f14efe..5cbe5b42e 100644 --- a/freqtrade/tests/exchange/test_exchange.py +++ b/freqtrade/tests/exchange/test_exchange.py @@ -1171,3 +1171,10 @@ def test_get_fee(default_conf, mocker): ccxt_exceptionhandlers(mocker, default_conf, api_mock, 'get_fee', 'calculate_fee') + +def test_stoploss_limit_available_only_for_binance(default_conf, mocker): + api_mock = MagicMock() + exchange = get_patched_exchange(mocker, default_conf, api_mock) + with pytest.raises(NotImplementedError): + exchange.stoploss_limit('BTC/ETH', 1, 0.8, 0.79) + From 3418592908a8e01699e485993c1d31ffb0d4c291 Mon Sep 17 00:00:00 2001 From: misagh Date: Thu, 22 Nov 2018 19:25:26 +0100 Subject: [PATCH 418/699] freqtradebot test added for orders on exchange --- freqtrade/exchange/__init__.py | 2 +- freqtrade/tests/conftest.py | 7 ++-- freqtrade/tests/test_freqtradebot.py | 50 ++++++++++++++++++++++++++++ 3 files changed, 55 insertions(+), 4 deletions(-) diff --git a/freqtrade/exchange/__init__.py b/freqtrade/exchange/__init__.py index 53ae6c2d7..90660c9aa 100644 --- a/freqtrade/exchange/__init__.py +++ b/freqtrade/exchange/__init__.py @@ -335,7 +335,7 @@ class Exchange(object): def stoploss_limit(self, pair: str, amount: float, stop_price: float, rate: float) -> Dict: # Only binance is supported - if not self._api.name == 'Binance': + if not self.name == 'Binance': raise NotImplementedError( 'Stoploss limit orders are implemented only for binance as of now.') diff --git a/freqtrade/tests/conftest.py b/freqtrade/tests/conftest.py index b6c022b45..3453b4ddf 100644 --- a/freqtrade/tests/conftest.py +++ b/freqtrade/tests/conftest.py @@ -27,12 +27,13 @@ def log_has(line, logs): False) -def patch_exchange(mocker, api_mock=None) -> None: +def patch_exchange(mocker, api_mock=None, id='bittrex') -> None: mocker.patch('freqtrade.exchange.Exchange._load_markets', MagicMock(return_value={})) mocker.patch('freqtrade.exchange.Exchange.validate_timeframes', MagicMock()) mocker.patch('freqtrade.exchange.Exchange.validate_ordertypes', MagicMock()) - mocker.patch('freqtrade.exchange.Exchange.name', PropertyMock(return_value="Bittrex")) - mocker.patch('freqtrade.exchange.Exchange.id', PropertyMock(return_value="bittrex")) + mocker.patch('freqtrade.exchange.Exchange.id', PropertyMock(return_value=id)) + mocker.patch('freqtrade.exchange.Exchange.name', PropertyMock(return_value=id.title())) + if api_mock: mocker.patch('freqtrade.exchange.Exchange._init_ccxt', MagicMock(return_value=api_mock)) else: diff --git a/freqtrade/tests/test_freqtradebot.py b/freqtrade/tests/test_freqtradebot.py index cef89c250..af4071591 100644 --- a/freqtrade/tests/test_freqtradebot.py +++ b/freqtrade/tests/test_freqtradebot.py @@ -880,6 +880,56 @@ def test_execute_buy(mocker, default_conf, fee, markets, limit_buy_order) -> Non assert call_args['rate'] == fix_price assert call_args['amount'] == stake_amount / fix_price +def test_execute_buy_with_stoploss_on_exchange(mocker, default_conf, fee, markets, limit_buy_order) -> None: + default_conf['exchange']['name'] = 'binance' + patch_RPCManager(mocker) + patch_exchange(mocker) + freqtrade = FreqtradeBot(default_conf) + freqtrade.strategy.stoploss_on_exchange = True + freqtrade.strategy.stoploss = -0.05 + stake_amount = 2 + bid = 0.11 + get_bid = MagicMock(return_value=bid) + mocker.patch.multiple( + 'freqtrade.freqtradebot.FreqtradeBot', + get_target_bid=get_bid, + _get_min_pair_stake_amount=MagicMock(return_value=1) + ) + buy_mm = MagicMock(return_value={'id': limit_buy_order['id']}) + stoploss_limit = MagicMock(return_value={'id': 13434334}) + mocker.patch.multiple( + 'freqtrade.exchange.Exchange', + get_ticker=MagicMock(return_value={ + 'bid': 0.00001172, + 'ask': 0.00001173, + 'last': 0.00001172 + }), + buy=buy_mm, + get_fee=fee, + get_markets=markets, + stoploss_limit=stoploss_limit + ) + pair = 'ETH/BTC' + print(buy_mm.call_args_list) + + assert freqtrade.execute_buy(pair, stake_amount) + assert stoploss_limit.call_count == 1 + assert get_bid.call_count == 1 + assert buy_mm.call_count == 1 + call_args = buy_mm.call_args_list[0][1] + assert call_args['pair'] == pair + assert call_args['rate'] == bid + assert call_args['amount'] == stake_amount / bid + + call_args = stoploss_limit.call_args_list[0][1] + assert call_args['pair'] == pair + assert call_args['amount'] == stake_amount / bid + assert call_args['stop_price'] == 0.11 * 0.95 + assert call_args['rate'] == 0.11 * 0.95 * 0.98 + + trade = Trade.query.first() + assert trade.is_open is True + assert trade.stoploss_order_id == 13434334 def test_process_maybe_execute_buy(mocker, default_conf) -> None: freqtrade = get_patched_freqtradebot(mocker, default_conf) From cc1422d448f898f5570224dda448c3995dc9d600 Mon Sep 17 00:00:00 2001 From: misagh Date: Thu, 22 Nov 2018 19:27:32 +0100 Subject: [PATCH 419/699] flake8 --- freqtrade/tests/test_freqtradebot.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/freqtrade/tests/test_freqtradebot.py b/freqtrade/tests/test_freqtradebot.py index af4071591..03bc68025 100644 --- a/freqtrade/tests/test_freqtradebot.py +++ b/freqtrade/tests/test_freqtradebot.py @@ -880,7 +880,9 @@ def test_execute_buy(mocker, default_conf, fee, markets, limit_buy_order) -> Non assert call_args['rate'] == fix_price assert call_args['amount'] == stake_amount / fix_price -def test_execute_buy_with_stoploss_on_exchange(mocker, default_conf, fee, markets, limit_buy_order) -> None: + +def test_execute_buy_with_stoploss_on_exchange(mocker, default_conf, + fee, markets, limit_buy_order) -> None: default_conf['exchange']['name'] = 'binance' patch_RPCManager(mocker) patch_exchange(mocker) @@ -931,6 +933,7 @@ def test_execute_buy_with_stoploss_on_exchange(mocker, default_conf, fee, market assert trade.is_open is True assert trade.stoploss_order_id == 13434334 + def test_process_maybe_execute_buy(mocker, default_conf) -> None: freqtrade = get_patched_freqtradebot(mocker, default_conf) From ecb2c4dca384d08cc9ebf6a112ebb98cefe0b795 Mon Sep 17 00:00:00 2001 From: misagh Date: Thu, 22 Nov 2018 19:38:20 +0100 Subject: [PATCH 420/699] bloody flake8 --- freqtrade/tests/exchange/test_exchange.py | 2 +- freqtrade/tests/test_freqtradebot.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/freqtrade/tests/exchange/test_exchange.py b/freqtrade/tests/exchange/test_exchange.py index 5cbe5b42e..9dbc50a66 100644 --- a/freqtrade/tests/exchange/test_exchange.py +++ b/freqtrade/tests/exchange/test_exchange.py @@ -1172,9 +1172,9 @@ def test_get_fee(default_conf, mocker): ccxt_exceptionhandlers(mocker, default_conf, api_mock, 'get_fee', 'calculate_fee') + def test_stoploss_limit_available_only_for_binance(default_conf, mocker): api_mock = MagicMock() exchange = get_patched_exchange(mocker, default_conf, api_mock) with pytest.raises(NotImplementedError): exchange.stoploss_limit('BTC/ETH', 1, 0.8, 0.79) - diff --git a/freqtrade/tests/test_freqtradebot.py b/freqtrade/tests/test_freqtradebot.py index 03bc68025..571c89bc0 100644 --- a/freqtrade/tests/test_freqtradebot.py +++ b/freqtrade/tests/test_freqtradebot.py @@ -885,7 +885,7 @@ def test_execute_buy_with_stoploss_on_exchange(mocker, default_conf, fee, markets, limit_buy_order) -> None: default_conf['exchange']['name'] = 'binance' patch_RPCManager(mocker) - patch_exchange(mocker) + patch_exchange(mocker, id='binance') freqtrade = FreqtradeBot(default_conf) freqtrade.strategy.stoploss_on_exchange = True freqtrade.strategy.stoploss = -0.05 From 07ac9024512e7f96185ee447243ee8d6ccd188f4 Mon Sep 17 00:00:00 2001 From: misagh Date: Thu, 22 Nov 2018 20:30:31 +0100 Subject: [PATCH 421/699] test exchange added --- freqtrade/exchange/__init__.py | 2 +- freqtrade/tests/conftest.py | 4 +-- freqtrade/tests/exchange/test_exchange.py | 36 +++++++++++++++++++++++ 3 files changed, 39 insertions(+), 3 deletions(-) diff --git a/freqtrade/exchange/__init__.py b/freqtrade/exchange/__init__.py index 90660c9aa..a88fb6ee8 100644 --- a/freqtrade/exchange/__init__.py +++ b/freqtrade/exchange/__init__.py @@ -345,7 +345,7 @@ class Exchange(object): stop_price = self.symbol_price_prec(pair, stop_price) # Ensure rate is less than stop price - if stop_price >= rate: + if stop_price <= rate: raise OperationalException( 'In stoploss limit order, stop price should be more than limit price') diff --git a/freqtrade/tests/conftest.py b/freqtrade/tests/conftest.py index 3453b4ddf..f4c263959 100644 --- a/freqtrade/tests/conftest.py +++ b/freqtrade/tests/conftest.py @@ -40,8 +40,8 @@ def patch_exchange(mocker, api_mock=None, id='bittrex') -> None: mocker.patch('freqtrade.exchange.Exchange._init_ccxt', MagicMock()) -def get_patched_exchange(mocker, config, api_mock=None) -> Exchange: - patch_exchange(mocker, api_mock) +def get_patched_exchange(mocker, config, api_mock=None, id='bittrex') -> Exchange: + patch_exchange(mocker, api_mock, id) exchange = Exchange(config) return exchange diff --git a/freqtrade/tests/exchange/test_exchange.py b/freqtrade/tests/exchange/test_exchange.py index 9dbc50a66..53e77d7b6 100644 --- a/freqtrade/tests/exchange/test_exchange.py +++ b/freqtrade/tests/exchange/test_exchange.py @@ -1178,3 +1178,39 @@ def test_stoploss_limit_available_only_for_binance(default_conf, mocker): exchange = get_patched_exchange(mocker, default_conf, api_mock) with pytest.raises(NotImplementedError): exchange.stoploss_limit('BTC/ETH', 1, 0.8, 0.79) + + +def test_stoploss_limit_order(default_conf, mocker): + api_mock = MagicMock() + order_id = 'test_prod_buy_{}'.format(randint(0, 10 ** 6)) + order_type = 'stop_loss' + + api_mock.create_order = MagicMock(return_value={ + 'id': order_id, + 'info': { + 'foo': 'bar' + } + }) + + default_conf['dry_run'] = False + mocker.patch('freqtrade.exchange.Exchange.symbol_amount_prec', lambda s, x, y: y) + mocker.patch('freqtrade.exchange.Exchange.symbol_price_prec', lambda s, x, y: y) + + exchange = get_patched_exchange(mocker, default_conf, api_mock, 'binance') + + with pytest.raises(OperationalException): + order = exchange.stoploss_limit(pair='ETH/BTC', amount=1, stop_price=190, rate=200) + + api_mock.create_order.reset_mock() + + order = exchange.stoploss_limit(pair='ETH/BTC', amount=1, stop_price=220, rate=200) + + assert 'id' in order + assert 'info' in order + assert order['id'] == order_id + assert api_mock.create_order.call_args[0][0] == 'ETH/BTC' + assert api_mock.create_order.call_args[0][1] == order_type + assert api_mock.create_order.call_args[0][2] == 'sell' + assert api_mock.create_order.call_args[0][3] == 1 + assert api_mock.create_order.call_args[0][4] == 200 + assert api_mock.create_order.call_args[0][5] == {'stopPrice': 220} From 7faafea8a2949a94725b883e89d81a7e854ec7f5 Mon Sep 17 00:00:00 2001 From: misagh Date: Thu, 22 Nov 2018 21:01:39 +0100 Subject: [PATCH 422/699] added test for cancelling stop loss before sell --- freqtrade/tests/test_freqtradebot.py | 50 ++++++++++++++++++++++++++++ 1 file changed, 50 insertions(+) diff --git a/freqtrade/tests/test_freqtradebot.py b/freqtrade/tests/test_freqtradebot.py index 571c89bc0..333b4d51f 100644 --- a/freqtrade/tests/test_freqtradebot.py +++ b/freqtrade/tests/test_freqtradebot.py @@ -1528,6 +1528,56 @@ def test_execute_sell_down(default_conf, ticker, fee, ticker_sell_down, markets, } == last_msg +def test_execute_sell_with_stoploss_on_exchange(default_conf, + ticker, fee, ticker_sell_up, + markets, mocker) -> None: + + default_conf['exchange']['name'] = 'binance' + mocker.patch.multiple( + 'freqtrade.exchange.Exchange', + _load_markets=MagicMock(return_value={}), + get_ticker=ticker, + get_fee=fee, + get_markets=markets + ) + + stoploss_limit = MagicMock(return_value={ + 'id': 123, + 'info': { + 'foo': 'bar' + } + }) + + cancel_order = MagicMock(return_value=True) + + mocker.patch('freqtrade.exchange.Exchange.symbol_amount_prec', lambda s, x, y: y) + mocker.patch('freqtrade.exchange.Exchange.symbol_price_prec', lambda s, x, y: y) + mocker.patch('freqtrade.exchange.Exchange.stoploss_limit', stoploss_limit) + mocker.patch('freqtrade.exchange.Exchange.cancel_order', cancel_order) + + freqtrade = FreqtradeBot(default_conf) + freqtrade.strategy.stoploss_on_exchange = True + patch_get_signal(freqtrade) + + # Create some test data + freqtrade.create_trade() + + trade = Trade.query.first() + assert trade + + # Increase the price and sell it + mocker.patch.multiple( + 'freqtrade.exchange.Exchange', + get_ticker=ticker_sell_up + ) + + freqtrade.execute_sell(trade=trade, limit=ticker_sell_up()['bid'], sell_reason=SellType.ROI) + + trade = Trade.query.first() + assert trade + assert cancel_order.call_count == 1 + + def test_execute_sell_without_conf_sell_up(default_conf, ticker, fee, ticker_sell_up, markets, mocker) -> None: rpc_mock = patch_RPCManager(mocker) From 6f0025c6de194c9f3bf3dd3ea43c135c63abef04 Mon Sep 17 00:00:00 2001 From: misagh Date: Thu, 22 Nov 2018 21:07:33 +0100 Subject: [PATCH 423/699] documentation written --- docs/configuration.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/configuration.md b/docs/configuration.md index 62559a41e..64f0a2ea6 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -26,6 +26,7 @@ The table below will list all configuration parameters. | `process_only_new_candles` | false | No | If set to true indicators are processed only once a new candle arrives. If false each loop populates the indicators, this will mean the same candle is processed many times creating system load but can be useful of your strategy depends on tick data not only candle. Can be set either in Configuration or in the strategy. | `minimal_roi` | See below | No | Set the threshold in percent the bot will use to sell a trade. More information below. If set, this parameter will override `minimal_roi` from your strategy file. | `stoploss` | -0.10 | No | Value of the stoploss in percent used by the bot. More information below. If set, this parameter will override `stoploss` from your strategy file. +| `stoploss_on_exchange` | false | No | Only for binance users for now: If this parameter is on then stoploss limit order is executed immediately after buy order is done on binance. | `trailing_stop` | false | No | Enables trailing stop-loss (based on `stoploss` in either configuration or strategy file). | `trailing_stop_positve` | 0 | No | Changes stop-loss once profit has been reached. | `trailing_stop_positve_offset` | 0 | No | Offset on when to apply `trailing_stop_positive`. Percentage value which should be positive. From 1dde56790c4650fa7be00108eb807e9c0e806645 Mon Sep 17 00:00:00 2001 From: misagh Date: Thu, 22 Nov 2018 21:12:49 +0100 Subject: [PATCH 424/699] final broken test fixed --- freqtrade/tests/test_freqtradebot.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/freqtrade/tests/test_freqtradebot.py b/freqtrade/tests/test_freqtradebot.py index 333b4d51f..aad5c371f 100644 --- a/freqtrade/tests/test_freqtradebot.py +++ b/freqtrade/tests/test_freqtradebot.py @@ -1533,6 +1533,7 @@ def test_execute_sell_with_stoploss_on_exchange(default_conf, markets, mocker) -> None: default_conf['exchange']['name'] = 'binance' + rpc_mock = patch_RPCManager(mocker) mocker.patch.multiple( 'freqtrade.exchange.Exchange', _load_markets=MagicMock(return_value={}), @@ -1576,6 +1577,7 @@ def test_execute_sell_with_stoploss_on_exchange(default_conf, trade = Trade.query.first() assert trade assert cancel_order.call_count == 1 + assert rpc_mock.call_count == 2 def test_execute_sell_without_conf_sell_up(default_conf, ticker, fee, From 27a6dcf3fc13821006903573d330c06cecf9b169 Mon Sep 17 00:00:00 2001 From: misagh Date: Thu, 22 Nov 2018 21:23:35 +0100 Subject: [PATCH 425/699] getting available balance from wallet instead of API call. --- freqtrade/freqtradebot.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index 8a2db84a9..49a4959c1 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -338,8 +338,8 @@ class FreqtradeBot(object): stake_amount = self.config['stake_amount'] # TODO: should come from the wallet - avaliable_amount = self.exchange.get_balance(self.config['stake_currency']) - # avaliable_amount = self.wallets.wallets[self.config['stake_currency']].free + #avaliable_amount = self.exchange.get_balance(self.config['stake_currency']) + avaliable_amount = self.wallets.wallets[self.config['stake_currency']].free if stake_amount == constants.UNLIMITED_STAKE_AMOUNT: open_trades = len(Trade.query.filter(Trade.is_open.is_(True)).all()) From a9f04609d3c2cf2ed623c15ab223bc5f912a7b1e Mon Sep 17 00:00:00 2001 From: misagh Date: Fri, 23 Nov 2018 10:17:10 +0100 Subject: [PATCH 426/699] tests fixed --- freqtrade/freqtradebot.py | 2 -- freqtrade/tests/conftest.py | 7 +++++++ freqtrade/tests/test_freqtradebot.py | 11 ++++------- freqtrade/tests/test_wallets.py | 2 ++ freqtrade/wallets.py | 5 ++++- 5 files changed, 17 insertions(+), 10 deletions(-) diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index 49a4959c1..c93811cae 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -337,8 +337,6 @@ class FreqtradeBot(object): else: stake_amount = self.config['stake_amount'] - # TODO: should come from the wallet - #avaliable_amount = self.exchange.get_balance(self.config['stake_currency']) avaliable_amount = self.wallets.wallets[self.config['stake_currency']].free if stake_amount == constants.UNLIMITED_STAKE_AMOUNT: diff --git a/freqtrade/tests/conftest.py b/freqtrade/tests/conftest.py index b6c022b45..cdd71fc9a 100644 --- a/freqtrade/tests/conftest.py +++ b/freqtrade/tests/conftest.py @@ -14,6 +14,7 @@ from telegram import Chat, Message, Update from freqtrade.exchange.exchange_helpers import parse_ticker_dataframe from freqtrade.exchange import Exchange from freqtrade.edge import Edge +from freqtrade.wallets import Wallet from freqtrade.freqtradebot import FreqtradeBot logging.getLogger('').setLevel(logging.INFO) @@ -45,6 +46,12 @@ def get_patched_exchange(mocker, config, api_mock=None) -> Exchange: return exchange +def patch_wallet(mocker, currency='BTC', free=999.9) -> None: + mocker.patch('freqtrade.wallets.Wallet', MagicMock( + return_value=Wallet('bittrex', currency, free, 100, 1000) + )) + + def patch_edge(mocker) -> None: # "ETH/BTC", # "LTC/BTC", diff --git a/freqtrade/tests/test_freqtradebot.py b/freqtrade/tests/test_freqtradebot.py index cef89c250..bcd914d0d 100644 --- a/freqtrade/tests/test_freqtradebot.py +++ b/freqtrade/tests/test_freqtradebot.py @@ -18,7 +18,7 @@ from freqtrade.persistence import Trade from freqtrade.rpc import RPCMessageType from freqtrade.state import State from freqtrade.strategy.interface import SellType, SellCheckTuple -from freqtrade.tests.conftest import log_has, patch_exchange, patch_edge +from freqtrade.tests.conftest import log_has, patch_exchange, patch_edge, patch_wallet # Functions for recurrent object patching @@ -188,10 +188,7 @@ def test_get_trade_stake_amount_no_stake_amount(default_conf, mocker) -> None: patch_RPCManager(mocker) patch_exchange(mocker) - mocker.patch.multiple( - 'freqtrade.exchange.Exchange', - get_balance=MagicMock(return_value=default_conf['stake_amount'] * 0.5) - ) + patch_wallet(mocker, free=default_conf['stake_amount'] * 0.5) freqtrade = FreqtradeBot(default_conf) with pytest.raises(DependencyException, match=r'.*stake amount.*'): @@ -206,12 +203,12 @@ def test_get_trade_stake_amount_unlimited_amount(default_conf, mocker) -> None: patch_RPCManager(mocker) patch_exchange(mocker) + patch_wallet(mocker, free=default_conf['stake_amount']) mocker.patch.multiple( 'freqtrade.exchange.Exchange', validate_pairs=MagicMock(), get_ticker=ticker, buy=MagicMock(return_value={'id': limit_buy_order['id']}), - get_balance=MagicMock(return_value=default_conf['stake_amount']), get_fee=fee, get_markets=markets ) @@ -521,11 +518,11 @@ def test_create_trade_no_stake_amount(default_conf, ticker, limit_buy_order, fee, markets, mocker) -> None: patch_RPCManager(mocker) patch_exchange(mocker) + patch_wallet(mocker, free=default_conf['stake_amount'] * 0.5) mocker.patch.multiple( 'freqtrade.exchange.Exchange', get_ticker=ticker, buy=MagicMock(return_value={'id': limit_buy_order['id']}), - get_balance=MagicMock(return_value=default_conf['stake_amount'] * 0.5), get_fee=fee, get_markets=markets ) diff --git a/freqtrade/tests/test_wallets.py b/freqtrade/tests/test_wallets.py index cc10d665c..e6a17ecbf 100644 --- a/freqtrade/tests/test_wallets.py +++ b/freqtrade/tests/test_wallets.py @@ -4,6 +4,7 @@ from unittest.mock import MagicMock def test_sync_wallet_at_boot(mocker, default_conf): + default_conf['dry_run'] = False mocker.patch.multiple( 'freqtrade.exchange.Exchange', get_balances=MagicMock(return_value={ @@ -58,6 +59,7 @@ def test_sync_wallet_at_boot(mocker, default_conf): def test_sync_wallet_missing_data(mocker, default_conf): + default_conf['dry_run'] = False mocker.patch.multiple( 'freqtrade.exchange.Exchange', get_balances=MagicMock(return_value={ diff --git a/freqtrade/wallets.py b/freqtrade/wallets.py index 82f527d2c..4415478d3 100644 --- a/freqtrade/wallets.py +++ b/freqtrade/wallets.py @@ -26,7 +26,10 @@ class Wallets(object): def __init__(self, exchange: Exchange) -> None: self.exchange = exchange - self.wallets: Dict[str, Any] = {} + if self.exchange._conf['dry_run']: + self.wallets: Dict[str, Any] = {'BTC': Wallet('Bittrex', 'BTC', 999.99, 100, 1000)} + else: + self.wallets: Dict[str, Any] = {} self.update() def update(self) -> None: From 270624c0c518fa96cba53670fb93eea6d2932280 Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Fri, 23 Nov 2018 13:34:08 +0100 Subject: [PATCH 427/699] Update ccxt from 1.17.529 to 1.17.533 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 34a9b477e..00026af8d 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -ccxt==1.17.529 +ccxt==1.17.533 SQLAlchemy==1.2.14 python-telegram-bot==11.1.0 arrow==0.12.1 From 605211dbaf5957d54ea8fdc04e79640305cb7726 Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Fri, 23 Nov 2018 13:34:09 +0100 Subject: [PATCH 428/699] Update scikit-learn from 0.20.0 to 0.20.1 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 00026af8d..b01dbdbbd 100644 --- a/requirements.txt +++ b/requirements.txt @@ -7,7 +7,7 @@ requests==2.20.1 urllib3==1.24.1 wrapt==1.10.11 pandas==0.23.4 -scikit-learn==0.20.0 +scikit-learn==0.20.1 joblib==0.13.0 scipy==1.1.0 jsonschema==2.6.0 From fea77824d09a0ebc9c3f1acf09273b799aa39e23 Mon Sep 17 00:00:00 2001 From: misagh Date: Fri, 23 Nov 2018 15:17:36 +0100 Subject: [PATCH 429/699] handle stop loss on exchange added --- freqtrade/freqtradebot.py | 24 +++++++++ freqtrade/persistence.py | 5 +- freqtrade/strategy/interface.py | 1 + freqtrade/tests/test_freqtradebot.py | 75 +++++++++++++++++++++++++++- 4 files changed, 102 insertions(+), 3 deletions(-) diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index 281b22bc5..1e5dfd175 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -572,6 +572,17 @@ class FreqtradeBot(object): trade.update(order) + # Check if stoploss on exchnage is hit first + if self.strategy.stoploss_on_exchange and trade.stoploss_order_id: + # Check if stoploss is hit + result = self.handle_stoploss_on_exchage(trade) + + # Updating wallets if stoploss is hit + if result: + self.wallets.update() + + return result + if trade.is_open and trade.open_order_id is None: # Check if we can sell our current pair result = self.handle_trade(trade) @@ -676,6 +687,19 @@ class FreqtradeBot(object): logger.info('Found no sell signals for whitelisted currencies. Trying again..') return False + def handle_stoploss_on_exchage(self, trade: Trade) -> bool: + if not trade.is_open: + raise ValueError(f'attempt to handle stoploss on exchnage for a closed trade: {trade}') + + logger.debug('Handling stoploss on exchange %s ...', trade) + order = self.exchange.get_order(trade.stoploss_order_id, trade.pair) + if order['status'] == 'closed': + trade.sell_reason = SellType.STOPLOSS_ON_EXCHNAGE.value + trade.update(order) + return True + else: + return False + def check_sell(self, trade: Trade, sell_rate: float, buy: bool, sell: bool) -> bool: if self.edge: stoploss = self.edge.stoploss(trade.pair) diff --git a/freqtrade/persistence.py b/freqtrade/persistence.py index db6d526c7..02caeeccd 100644 --- a/freqtrade/persistence.py +++ b/freqtrade/persistence.py @@ -178,7 +178,7 @@ class Trade(_DECL_BASE): # absolute value of the initial stop loss initial_stop_loss = Column(Float, nullable=True, default=0.0) # absolute value of the highest reached price - stoploss_order_id = Column(Integer, nullable=True, index=True) + stoploss_order_id = Column(String, nullable=True, index=True) max_rate = Column(Float, nullable=True, default=0.0) sell_reason = Column(String, nullable=True) strategy = Column(String, nullable=True) @@ -250,6 +250,9 @@ class Trade(_DECL_BASE): self.open_order_id = None elif order_type == 'limit' and order['side'] == 'sell': self.close(order['price']) + elif order_type == 'stop_loss_limit': + self.stoploss_order_id = None + self.close(order['price']) else: raise ValueError(f'Unknown order type: {order_type}') cleanup() diff --git a/freqtrade/strategy/interface.py b/freqtrade/strategy/interface.py index 30fc62f42..d1e22850c 100644 --- a/freqtrade/strategy/interface.py +++ b/freqtrade/strategy/interface.py @@ -33,6 +33,7 @@ class SellType(Enum): """ ROI = "roi" STOP_LOSS = "stop_loss" + STOPLOSS_ON_EXCHNAGE = "stoploss_on_exchange" TRAILING_STOP_LOSS = "trailing_stop_loss" SELL_SIGNAL = "sell_signal" FORCE_SELL = "force_sell" diff --git a/freqtrade/tests/test_freqtradebot.py b/freqtrade/tests/test_freqtradebot.py index aad5c371f..48918645d 100644 --- a/freqtrade/tests/test_freqtradebot.py +++ b/freqtrade/tests/test_freqtradebot.py @@ -931,7 +931,7 @@ def test_execute_buy_with_stoploss_on_exchange(mocker, default_conf, trade = Trade.query.first() assert trade.is_open is True - assert trade.stoploss_order_id == 13434334 + assert trade.stoploss_order_id == '13434334' def test_process_maybe_execute_buy(mocker, default_conf) -> None: @@ -1572,7 +1572,8 @@ def test_execute_sell_with_stoploss_on_exchange(default_conf, get_ticker=ticker_sell_up ) - freqtrade.execute_sell(trade=trade, limit=ticker_sell_up()['bid'], sell_reason=SellType.ROI) + freqtrade.execute_sell(trade=trade, limit=ticker_sell_up()['bid'], + sell_reason=SellType.SELL_SIGNAL) trade = Trade.query.first() assert trade @@ -1580,6 +1581,76 @@ def test_execute_sell_with_stoploss_on_exchange(default_conf, assert rpc_mock.call_count == 2 +def test_may_execute_sell_after_stoploss_on_exchange_hit(default_conf, + ticker, fee, + limit_buy_order, + markets, mocker) -> None: + default_conf['exchange']['name'] = 'binance' + rpc_mock = patch_RPCManager(mocker) + mocker.patch.multiple( + 'freqtrade.exchange.Exchange', + _load_markets=MagicMock(return_value={}), + get_ticker=ticker, + get_fee=fee, + get_markets=markets + ) + + stoploss_limit = MagicMock(return_value={ + 'id': 123, + 'info': { + 'foo': 'bar' + } + }) + + mocker.patch('freqtrade.exchange.Exchange.symbol_amount_prec', lambda s, x, y: y) + mocker.patch('freqtrade.exchange.Exchange.symbol_price_prec', lambda s, x, y: y) + mocker.patch('freqtrade.exchange.Exchange.stoploss_limit', stoploss_limit) + + freqtrade = FreqtradeBot(default_conf) + freqtrade.strategy.stoploss_on_exchange = True + patch_get_signal(freqtrade) + + # Create some test data + freqtrade.create_trade() + + trade = Trade.query.first() + assert trade + assert trade.stoploss_order_id == '123' + assert trade.open_order_id is not None + + trade.update(limit_buy_order) + + # Assuming stoploss on exchnage is hit + # stoploss_order_id should become None + # and trade should be sold at the price of stoploss + stoploss_limit_executed = MagicMock(return_value={ + "id": "123", + "timestamp": 1542707426845, + "datetime": "2018-11-20T09:50:26.845Z", + "lastTradeTimestamp": None, + "symbol": "BTC/USDT", + "type": "stop_loss_limit", + "side": "sell", + "price": 1.08801, + "amount": 90.99181074, + "cost": 99.0000000032274, + "average": 1.08801, + "filled": 90.99181074, + "remaining": 0.0, + "status": "closed", + "fee": None, + "trades": None + }) + mocker.patch('freqtrade.exchange.Exchange.get_order', stoploss_limit_executed) + + freqtrade.process_maybe_execute_sell(trade) + assert trade.stoploss_order_id is None + assert trade.is_open is False + print(trade.sell_reason) + assert trade.sell_reason == SellType.STOPLOSS_ON_EXCHNAGE.value + assert rpc_mock.call_count == 1 + + def test_execute_sell_without_conf_sell_up(default_conf, ticker, fee, ticker_sell_up, markets, mocker) -> None: rpc_mock = patch_RPCManager(mocker) From 5ee2faa182fd8c5fd470e50febf266648fbbaf21 Mon Sep 17 00:00:00 2001 From: misagh Date: Fri, 23 Nov 2018 19:17:36 +0100 Subject: [PATCH 430/699] adding stop loss on exchange after the buy order is fulfilled not before. --- freqtrade/freqtradebot.py | 41 +++++++++++++++++++-------------------- 1 file changed, 20 insertions(+), 21 deletions(-) diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index 1e5dfd175..c2ef0e406 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -479,22 +479,6 @@ class FreqtradeBot(object): order_id = self.exchange.buy(pair=pair, ordertype=self.strategy.order_types['buy'], amount=amount, rate=buy_limit)['id'] - stoploss_order_id = None - - # Check if stoploss should be added on exchange - # If True then here immediately after buy we should - # Add the stoploss order - if self.strategy.stoploss_on_exchange: - stoploss = self.edge.stoploss(pair=pair) if self.edge else self.strategy.stoploss - stop_price = buy_limit * (1 + stoploss) - - # limit price should be less than stop price. - # 0.98 is arbitrary here. - limit_price = stop_price * 0.98 - - stoploss_order_id = self.exchange.stoploss_limit( - pair=pair, amount=amount, stop_price=stop_price, rate=limit_price)['id'] - self.rpc.send_msg({ 'type': RPCMessageType.BUY_NOTIFICATION, 'exchange': self.exchange.name.capitalize(), @@ -519,7 +503,6 @@ class FreqtradeBot(object): open_date=datetime.utcnow(), exchange=self.exchange.id, open_order_id=order_id, - stoploss_order_id=stoploss_order_id, strategy=self.strategy.get_strategy_name(), ticker_interval=constants.TICKER_INTERVAL_MINUTES[self.config['ticker_interval']] ) @@ -572,16 +555,32 @@ class FreqtradeBot(object): trade.update(order) - # Check if stoploss on exchnage is hit first - if self.strategy.stoploss_on_exchange and trade.stoploss_order_id: + # Check uf trade is fulfulled in which case the stoploss + # on exchange should be added immediately if stoploss on exchnage + # is on + if self.strategy.stoploss_on_exchange and trade.is_open and \ + trade.open_order_id is None and trade.stoploss_order_id is None: + + stoploss = self.edge.stoploss(pair=trade.pair) if self.edge else self.strategy.stoploss + stop_price = trade.open_rate * (1 + stoploss) + + # limit price should be less than stop price. + # 0.98 is arbitrary here. + limit_price = stop_price * 0.98 + + stoploss_order_id = self.exchange.stoploss_limit( + pair=trade.pair, amount=trade.amount, stop_price=stop_price, rate=limit_price)['id'] + trade.stoploss_order_id = stoploss_order_id + + # Or Check if there is a stoploss on exchnage and it is hit + elif self.strategy.stoploss_on_exchange and trade.stoploss_order_id: # Check if stoploss is hit result = self.handle_stoploss_on_exchage(trade) # Updating wallets if stoploss is hit if result: self.wallets.update() - - return result + return result if trade.is_open and trade.open_order_id is None: # Check if we can sell our current pair From 9144a8f79df3c3839e294e4d2285a2a123aa3993 Mon Sep 17 00:00:00 2001 From: misagh Date: Fri, 23 Nov 2018 20:28:01 +0100 Subject: [PATCH 431/699] tests fixed --- freqtrade/freqtradebot.py | 14 ++++-- freqtrade/tests/test_freqtradebot.py | 70 +++++++++------------------- 2 files changed, 31 insertions(+), 53 deletions(-) diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index c2ef0e406..d4ea83d25 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -559,9 +559,13 @@ class FreqtradeBot(object): # on exchange should be added immediately if stoploss on exchnage # is on if self.strategy.stoploss_on_exchange and trade.is_open and \ - trade.open_order_id is None and trade.stoploss_order_id is None: + trade.open_order_id is None and trade.stoploss_order_id is None: + + if self.edge: + stoploss = self.edge.stoploss(pair=trade.pair) + else: + stoploss = self.strategy.stoploss - stoploss = self.edge.stoploss(pair=trade.pair) if self.edge else self.strategy.stoploss stop_price = trade.open_rate * (1 + stoploss) # limit price should be less than stop price. @@ -569,8 +573,10 @@ class FreqtradeBot(object): limit_price = stop_price * 0.98 stoploss_order_id = self.exchange.stoploss_limit( - pair=trade.pair, amount=trade.amount, stop_price=stop_price, rate=limit_price)['id'] - trade.stoploss_order_id = stoploss_order_id + pair=trade.pair, amount=trade.amount, stop_price=stop_price, rate=limit_price + )['id'] + + trade.stoploss_order_id = str(stoploss_order_id) # Or Check if there is a stoploss on exchnage and it is hit elif self.strategy.stoploss_on_exchange and trade.stoploss_order_id: diff --git a/freqtrade/tests/test_freqtradebot.py b/freqtrade/tests/test_freqtradebot.py index 48918645d..64c8e9765 100644 --- a/freqtrade/tests/test_freqtradebot.py +++ b/freqtrade/tests/test_freqtradebot.py @@ -881,57 +881,29 @@ def test_execute_buy(mocker, default_conf, fee, markets, limit_buy_order) -> Non assert call_args['amount'] == stake_amount / fix_price -def test_execute_buy_with_stoploss_on_exchange(mocker, default_conf, - fee, markets, limit_buy_order) -> None: - default_conf['exchange']['name'] = 'binance' +def test_add_stoploss_on_exchange(mocker, default_conf, limit_buy_order) -> None: patch_RPCManager(mocker) - patch_exchange(mocker, id='binance') + mocker.patch('freqtrade.freqtradebot.FreqtradeBot.handle_trade', MagicMock(return_value=True)) + mocker.patch('freqtrade.exchange.Exchange.get_order', return_value=limit_buy_order) + mocker.patch('freqtrade.exchange.Exchange.get_trades_for_order', return_value=[]) + mocker.patch('freqtrade.freqtradebot.FreqtradeBot.get_real_amount', + return_value=limit_buy_order['amount']) + + stoploss_limit = MagicMock(return_value={'id': 13434334}) + mocker.patch('freqtrade.exchange.Exchange.stoploss_limit', stoploss_limit) + freqtrade = FreqtradeBot(default_conf) freqtrade.strategy.stoploss_on_exchange = True - freqtrade.strategy.stoploss = -0.05 - stake_amount = 2 - bid = 0.11 - get_bid = MagicMock(return_value=bid) - mocker.patch.multiple( - 'freqtrade.freqtradebot.FreqtradeBot', - get_target_bid=get_bid, - _get_min_pair_stake_amount=MagicMock(return_value=1) - ) - buy_mm = MagicMock(return_value={'id': limit_buy_order['id']}) - stoploss_limit = MagicMock(return_value={'id': 13434334}) - mocker.patch.multiple( - 'freqtrade.exchange.Exchange', - get_ticker=MagicMock(return_value={ - 'bid': 0.00001172, - 'ask': 0.00001173, - 'last': 0.00001172 - }), - buy=buy_mm, - get_fee=fee, - get_markets=markets, - stoploss_limit=stoploss_limit - ) - pair = 'ETH/BTC' - print(buy_mm.call_args_list) - assert freqtrade.execute_buy(pair, stake_amount) - assert stoploss_limit.call_count == 1 - assert get_bid.call_count == 1 - assert buy_mm.call_count == 1 - call_args = buy_mm.call_args_list[0][1] - assert call_args['pair'] == pair - assert call_args['rate'] == bid - assert call_args['amount'] == stake_amount / bid + trade = MagicMock() + trade.open_order_id = None + trade.stoploss_order_id = None + trade.is_open = True - call_args = stoploss_limit.call_args_list[0][1] - assert call_args['pair'] == pair - assert call_args['amount'] == stake_amount / bid - assert call_args['stop_price'] == 0.11 * 0.95 - assert call_args['rate'] == 0.11 * 0.95 * 0.98 - - trade = Trade.query.first() - assert trade.is_open is True + freqtrade.process_maybe_execute_sell(trade) assert trade.stoploss_order_id == '13434334' + assert stoploss_limit.call_count == 1 + assert trade.is_open is True def test_process_maybe_execute_buy(mocker, default_conf) -> None: @@ -1566,6 +1538,8 @@ def test_execute_sell_with_stoploss_on_exchange(default_conf, trade = Trade.query.first() assert trade + freqtrade.process_maybe_execute_sell(trade) + # Increase the price and sell it mocker.patch.multiple( 'freqtrade.exchange.Exchange', @@ -1612,13 +1586,11 @@ def test_may_execute_sell_after_stoploss_on_exchange_hit(default_conf, # Create some test data freqtrade.create_trade() - trade = Trade.query.first() + freqtrade.process_maybe_execute_sell(trade) assert trade assert trade.stoploss_order_id == '123' - assert trade.open_order_id is not None - - trade.update(limit_buy_order) + assert trade.open_order_id is None # Assuming stoploss on exchnage is hit # stoploss_order_id should become None From 1c2c19b12cfa60951e2169c77277452e412374b0 Mon Sep 17 00:00:00 2001 From: misagh Date: Fri, 23 Nov 2018 20:47:17 +0100 Subject: [PATCH 432/699] the complex in the life of flake8 resolved --- freqtrade/freqtradebot.py | 69 ++++++++++++++++++--------------------- 1 file changed, 32 insertions(+), 37 deletions(-) diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index d4ea83d25..3d52ffffa 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -555,35 +555,8 @@ class FreqtradeBot(object): trade.update(order) - # Check uf trade is fulfulled in which case the stoploss - # on exchange should be added immediately if stoploss on exchnage - # is on - if self.strategy.stoploss_on_exchange and trade.is_open and \ - trade.open_order_id is None and trade.stoploss_order_id is None: - - if self.edge: - stoploss = self.edge.stoploss(pair=trade.pair) - else: - stoploss = self.strategy.stoploss - - stop_price = trade.open_rate * (1 + stoploss) - - # limit price should be less than stop price. - # 0.98 is arbitrary here. - limit_price = stop_price * 0.98 - - stoploss_order_id = self.exchange.stoploss_limit( - pair=trade.pair, amount=trade.amount, stop_price=stop_price, rate=limit_price - )['id'] - - trade.stoploss_order_id = str(stoploss_order_id) - - # Or Check if there is a stoploss on exchnage and it is hit - elif self.strategy.stoploss_on_exchange and trade.stoploss_order_id: - # Check if stoploss is hit + if self.strategy.stoploss_on_exchange: result = self.handle_stoploss_on_exchage(trade) - - # Updating wallets if stoploss is hit if result: self.wallets.update() return result @@ -693,18 +666,40 @@ class FreqtradeBot(object): return False def handle_stoploss_on_exchage(self, trade: Trade) -> bool: - if not trade.is_open: - raise ValueError(f'attempt to handle stoploss on exchnage for a closed trade: {trade}') + # Check uf trade is fulfulled in which case the stoploss + # on exchange should be added immediately if stoploss on exchnage + # is on + if trade.is_open and trade.open_order_id is None and trade.stoploss_order_id is None: + if self.edge: + stoploss = self.edge.stoploss(pair=trade.pair) + else: + stoploss = self.strategy.stoploss - logger.debug('Handling stoploss on exchange %s ...', trade) - order = self.exchange.get_order(trade.stoploss_order_id, trade.pair) - if order['status'] == 'closed': - trade.sell_reason = SellType.STOPLOSS_ON_EXCHNAGE.value - trade.update(order) - return True - else: + stop_price = trade.open_rate * (1 + stoploss) + + # limit price should be less than stop price. + # 0.98 is arbitrary here. + limit_price = stop_price * 0.98 + + stoploss_order_id = self.exchange.stoploss_limit( + pair=trade.pair, amount=trade.amount, stop_price=stop_price, rate=limit_price + )['id'] + + trade.stoploss_order_id = str(stoploss_order_id) return False + # Or Check if there is a stoploss on exchnage and it is hit + elif trade.stoploss_order_id: + logger.debug('Handling stoploss on exchange %s ...', trade) + order = self.exchange.get_order(trade.stoploss_order_id, trade.pair) + if order['status'] == 'closed': + trade.sell_reason = SellType.STOPLOSS_ON_EXCHNAGE.value + trade.update(order) + return True + else: + return False + + def check_sell(self, trade: Trade, sell_rate: float, buy: bool, sell: bool) -> bool: if self.edge: stoploss = self.edge.stoploss(trade.pair) From 89eb3d9f36d960d8862bf4fb66b02f2e12a4d7c4 Mon Sep 17 00:00:00 2001 From: misagh Date: Fri, 23 Nov 2018 20:49:00 +0100 Subject: [PATCH 433/699] blank line removed --- freqtrade/freqtradebot.py | 1 - 1 file changed, 1 deletion(-) diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index 3d52ffffa..bd4e1b9e7 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -699,7 +699,6 @@ class FreqtradeBot(object): else: return False - def check_sell(self, trade: Trade, sell_rate: float, buy: bool, sell: bool) -> bool: if self.edge: stoploss = self.edge.stoploss(trade.pair) From dedf1ff70340df99647d0f30cc556dcaedc3c291 Mon Sep 17 00:00:00 2001 From: misagh Date: Fri, 23 Nov 2018 20:51:23 +0100 Subject: [PATCH 434/699] refactoring --- freqtrade/freqtradebot.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index bd4e1b9e7..335a0f76e 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -669,6 +669,7 @@ class FreqtradeBot(object): # Check uf trade is fulfulled in which case the stoploss # on exchange should be added immediately if stoploss on exchnage # is on + result = False if trade.is_open and trade.open_order_id is None and trade.stoploss_order_id is None: if self.edge: stoploss = self.edge.stoploss(pair=trade.pair) @@ -684,9 +685,7 @@ class FreqtradeBot(object): stoploss_order_id = self.exchange.stoploss_limit( pair=trade.pair, amount=trade.amount, stop_price=stop_price, rate=limit_price )['id'] - trade.stoploss_order_id = str(stoploss_order_id) - return False # Or Check if there is a stoploss on exchnage and it is hit elif trade.stoploss_order_id: @@ -695,9 +694,10 @@ class FreqtradeBot(object): if order['status'] == 'closed': trade.sell_reason = SellType.STOPLOSS_ON_EXCHNAGE.value trade.update(order) - return True + result = True else: - return False + result = False + return result def check_sell(self, trade: Trade, sell_rate: float, buy: bool, sell: bool) -> bool: if self.edge: From 3e8de28b51482b67a78ba3b333cc766437618ce6 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 24 Nov 2018 13:24:14 +0100 Subject: [PATCH 435/699] Add Note about order types support --- docs/configuration.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/docs/configuration.md b/docs/configuration.md index 62559a41e..1e144e5af 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -155,6 +155,9 @@ The below is the default which is used if this is not configured in either Strat }, ``` +**NOTE**: Not all exchanges support "market" orders. +The following message will be shown if your exchange does not support market orders: `"Exchange does not support market orders."` + ### What values for exchange.name? Freqtrade is based on [CCXT library](https://github.com/ccxt/ccxt) that supports 115 cryptocurrency From 412a627d9e377309912e86eac4487c5b9eab1c69 Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Sat, 24 Nov 2018 13:34:05 +0100 Subject: [PATCH 436/699] Update ccxt from 1.17.533 to 1.17.535 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index b01dbdbbd..b3791ccfe 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -ccxt==1.17.533 +ccxt==1.17.535 SQLAlchemy==1.2.14 python-telegram-bot==11.1.0 arrow==0.12.1 From 29a4c99d1d4d10f6fb8d0657f5eb3d35040d1af7 Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Sat, 24 Nov 2018 13:34:07 +0100 Subject: [PATCH 437/699] Update pytest from 4.0.0 to 4.0.1 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index b3791ccfe..44cfe4a07 100644 --- a/requirements.txt +++ b/requirements.txt @@ -13,7 +13,7 @@ scipy==1.1.0 jsonschema==2.6.0 numpy==1.15.4 TA-Lib==0.4.17 -pytest==4.0.0 +pytest==4.0.1 pytest-mock==1.10.0 pytest-asyncio==0.9.0 pytest-cov==2.6.0 From 29347a693187cbce26f2d75babca002dfec9ddda Mon Sep 17 00:00:00 2001 From: misagh Date: Sat, 24 Nov 2018 16:37:28 +0100 Subject: [PATCH 438/699] adding get_free to wallet --- freqtrade/freqtradebot.py | 2 +- freqtrade/tests/conftest.py | 7 +++---- freqtrade/wallets.py | 16 ++++++++++++---- 3 files changed, 16 insertions(+), 9 deletions(-) diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index c93811cae..0e77a31c0 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -337,7 +337,7 @@ class FreqtradeBot(object): else: stake_amount = self.config['stake_amount'] - avaliable_amount = self.wallets.wallets[self.config['stake_currency']].free + avaliable_amount = self.wallets.get_free(self.config['stake_currency']) if stake_amount == constants.UNLIMITED_STAKE_AMOUNT: open_trades = len(Trade.query.filter(Trade.is_open.is_(True)).all()) diff --git a/freqtrade/tests/conftest.py b/freqtrade/tests/conftest.py index cdd71fc9a..63655126c 100644 --- a/freqtrade/tests/conftest.py +++ b/freqtrade/tests/conftest.py @@ -14,7 +14,6 @@ from telegram import Chat, Message, Update from freqtrade.exchange.exchange_helpers import parse_ticker_dataframe from freqtrade.exchange import Exchange from freqtrade.edge import Edge -from freqtrade.wallets import Wallet from freqtrade.freqtradebot import FreqtradeBot logging.getLogger('').setLevel(logging.INFO) @@ -46,9 +45,9 @@ def get_patched_exchange(mocker, config, api_mock=None) -> Exchange: return exchange -def patch_wallet(mocker, currency='BTC', free=999.9) -> None: - mocker.patch('freqtrade.wallets.Wallet', MagicMock( - return_value=Wallet('bittrex', currency, free, 100, 1000) +def patch_wallet(mocker, free=999.9) -> None: + mocker.patch('freqtrade.wallets.Wallets.get_free', MagicMock( + return_value=free )) diff --git a/freqtrade/wallets.py b/freqtrade/wallets.py index 4415478d3..b8b37907d 100644 --- a/freqtrade/wallets.py +++ b/freqtrade/wallets.py @@ -26,12 +26,20 @@ class Wallets(object): def __init__(self, exchange: Exchange) -> None: self.exchange = exchange - if self.exchange._conf['dry_run']: - self.wallets: Dict[str, Any] = {'BTC': Wallet('Bittrex', 'BTC', 999.99, 100, 1000)} - else: - self.wallets: Dict[str, Any] = {} + self.wallets: Dict[str, Any] = {} self.update() + def get_free(self, currency) -> float: + + if self.exchange._conf['dry_run']: + return 999.9 + + balance = self.wallets.get(currency) + if balance and balance['free']: + return balance['free'] + else: + return 0 + def update(self) -> None: balances = self.exchange.get_balances() From 63c2ea110acb9fe04338be56055a824566de4bee Mon Sep 17 00:00:00 2001 From: misagh Date: Sat, 24 Nov 2018 16:41:17 +0100 Subject: [PATCH 439/699] Not sure why those arguments were there ! --- freqtrade/tests/test_freqtradebot.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/freqtrade/tests/test_freqtradebot.py b/freqtrade/tests/test_freqtradebot.py index bcd914d0d..a36ae2cd8 100644 --- a/freqtrade/tests/test_freqtradebot.py +++ b/freqtrade/tests/test_freqtradebot.py @@ -181,11 +181,7 @@ def test_get_trade_stake_amount(default_conf, ticker, limit_buy_order, fee, mock assert result == default_conf['stake_amount'] -def test_get_trade_stake_amount_no_stake_amount(default_conf, - ticker, - limit_buy_order, - fee, - mocker) -> None: +def test_get_trade_stake_amount_no_stake_amount(default_conf, mocker) -> None: patch_RPCManager(mocker) patch_exchange(mocker) patch_wallet(mocker, free=default_conf['stake_amount'] * 0.5) From 1a8e9ebc0f8e60239015fc17dbde7ed7bcd0f77d Mon Sep 17 00:00:00 2001 From: misagh Date: Sat, 24 Nov 2018 16:53:10 +0100 Subject: [PATCH 440/699] stoploss_order_id added to migration script --- freqtrade/persistence.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/freqtrade/persistence.py b/freqtrade/persistence.py index 02caeeccd..364af06ce 100644 --- a/freqtrade/persistence.py +++ b/freqtrade/persistence.py @@ -91,6 +91,7 @@ def check_migrate(engine) -> None: close_rate_requested = get_column_def(cols, 'close_rate_requested', 'null') stop_loss = get_column_def(cols, 'stop_loss', '0.0') initial_stop_loss = get_column_def(cols, 'initial_stop_loss', '0.0') + stoploss_order_id = get_column_def(cols, 'stoploss_order_id', 'null') max_rate = get_column_def(cols, 'max_rate', '0.0') sell_reason = get_column_def(cols, 'sell_reason', 'null') strategy = get_column_def(cols, 'strategy', 'null') @@ -106,7 +107,7 @@ def check_migrate(engine) -> None: (id, exchange, pair, is_open, fee_open, fee_close, open_rate, open_rate_requested, close_rate, close_rate_requested, close_profit, stake_amount, amount, open_date, close_date, open_order_id, - stop_loss, initial_stop_loss, max_rate, sell_reason, strategy, + stop_loss, initial_stop_loss, stoploss_order_id, max_rate, sell_reason, strategy, ticker_interval ) select id, lower(exchange), @@ -122,7 +123,8 @@ def check_migrate(engine) -> None: {close_rate_requested} close_rate_requested, close_profit, stake_amount, amount, open_date, close_date, open_order_id, {stop_loss} stop_loss, {initial_stop_loss} initial_stop_loss, - {max_rate} max_rate, {sell_reason} sell_reason, {strategy} strategy, + {stoploss_order_id} stoploss_order_id, {max_rate} max_rate, + {sell_reason} sell_reason, {strategy} strategy, {ticker_interval} ticker_interval from {table_back_name} """) @@ -177,8 +179,9 @@ class Trade(_DECL_BASE): stop_loss = Column(Float, nullable=True, default=0.0) # absolute value of the initial stop loss initial_stop_loss = Column(Float, nullable=True, default=0.0) - # absolute value of the highest reached price + # stoploss order id which is on exchange stoploss_order_id = Column(String, nullable=True, index=True) + # absolute value of the highest reached price max_rate = Column(Float, nullable=True, default=0.0) sell_reason = Column(String, nullable=True) strategy = Column(String, nullable=True) From a9ec5c66993a00bb8e7d6ec57b3a65079d419144 Mon Sep 17 00:00:00 2001 From: misagh Date: Sat, 24 Nov 2018 17:07:35 +0100 Subject: [PATCH 441/699] simplifying if conditions --- freqtrade/freqtradebot.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index 335a0f76e..f197d9117 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -670,7 +670,7 @@ class FreqtradeBot(object): # on exchange should be added immediately if stoploss on exchnage # is on result = False - if trade.is_open and trade.open_order_id is None and trade.stoploss_order_id is None: + if trade.is_open and not trade.open_order_id and not trade.stoploss_order_id: if self.edge: stoploss = self.edge.stoploss(pair=trade.pair) else: From afd0a054b2a6bfdac724d61f6e4a23c80cd0bc71 Mon Sep 17 00:00:00 2001 From: misagh Date: Sat, 24 Nov 2018 17:08:12 +0100 Subject: [PATCH 442/699] typo corrected --- freqtrade/freqtradebot.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index f197d9117..a8736d6f1 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -556,7 +556,7 @@ class FreqtradeBot(object): trade.update(order) if self.strategy.stoploss_on_exchange: - result = self.handle_stoploss_on_exchage(trade) + result = self.handle_stoploss_on_exchange(trade) if result: self.wallets.update() return result @@ -665,7 +665,7 @@ class FreqtradeBot(object): logger.info('Found no sell signals for whitelisted currencies. Trying again..') return False - def handle_stoploss_on_exchage(self, trade: Trade) -> bool: + def handle_stoploss_on_exchange(self, trade: Trade) -> bool: # Check uf trade is fulfulled in which case the stoploss # on exchange should be added immediately if stoploss on exchnage # is on From 531d9ecd0c8f15f9bf7721d6d0b691486a240740 Mon Sep 17 00:00:00 2001 From: misagh Date: Sat, 24 Nov 2018 17:10:51 +0100 Subject: [PATCH 443/699] docstring added --- freqtrade/freqtradebot.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index a8736d6f1..369bed173 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -666,9 +666,12 @@ class FreqtradeBot(object): return False def handle_stoploss_on_exchange(self, trade: Trade) -> bool: - # Check uf trade is fulfulled in which case the stoploss - # on exchange should be added immediately if stoploss on exchnage - # is on + """ + Check if trade is fulfilled in which case the stoploss + on exchange should be added immediately if stoploss on exchnage + is enabled. + """ + result = False if trade.is_open and not trade.open_order_id and not trade.stoploss_order_id: if self.edge: From 870631f324eb785fb0faf12c4544b742dc59cb09 Mon Sep 17 00:00:00 2001 From: misagh Date: Sat, 24 Nov 2018 17:32:25 +0100 Subject: [PATCH 444/699] 1) comments added to handle_sl 2) dry-run force price removed --- freqtrade/freqtradebot.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index 369bed173..cf22ee52a 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -673,6 +673,9 @@ class FreqtradeBot(object): """ result = False + + # If trade is open and the buy order is fulfilled but there is no stoploss, + # then we add a stoploss on exchange if trade.is_open and not trade.open_order_id and not trade.stoploss_order_id: if self.edge: stoploss = self.edge.stoploss(pair=trade.pair) @@ -690,7 +693,8 @@ class FreqtradeBot(object): )['id'] trade.stoploss_order_id = str(stoploss_order_id) - # Or Check if there is a stoploss on exchnage and it is hit + # Or there is already a stoploss on exchnage. + # so we check if it is hit ... elif trade.stoploss_order_id: logger.debug('Handling stoploss on exchange %s ...', trade) order = self.exchange.get_order(trade.stoploss_order_id, trade.pair) @@ -831,11 +835,6 @@ class FreqtradeBot(object): if self.strategy.stoploss_on_exchange and trade.stoploss_order_id: self.exchange.cancel_order(trade.stoploss_order_id, trade.pair) - # Dry-run should consider stoploss is executed at the limit price - # So overriding limit in case of dry-run - if self.config['dry_run']: - limit = trade.stop_loss - # Execute sell and update trade record order_id = self.exchange.sell(pair=str(trade.pair), ordertype=self.strategy.order_types[sell_type], From 000711b0256e541bf8dada406ae6ad8d77c24701 Mon Sep 17 00:00:00 2001 From: misagh Date: Sat, 24 Nov 2018 18:08:11 +0100 Subject: [PATCH 445/699] added stoploss_limit_order for dry-run --- freqtrade/tests/exchange/test_exchange.py | 36 +++++++++++++++++++++++ 1 file changed, 36 insertions(+) diff --git a/freqtrade/tests/exchange/test_exchange.py b/freqtrade/tests/exchange/test_exchange.py index 53e77d7b6..402596da8 100644 --- a/freqtrade/tests/exchange/test_exchange.py +++ b/freqtrade/tests/exchange/test_exchange.py @@ -1214,3 +1214,39 @@ def test_stoploss_limit_order(default_conf, mocker): assert api_mock.create_order.call_args[0][3] == 1 assert api_mock.create_order.call_args[0][4] == 200 assert api_mock.create_order.call_args[0][5] == {'stopPrice': 220} + + +def test_stoploss_limit_order_dry_run(default_conf, mocker): + api_mock = MagicMock() + order_id = 'test_prod_buy_{}'.format(randint(0, 10 ** 6)) + order_type = 'stop_loss' + + api_mock.create_order = MagicMock(return_value={ + 'id': order_id, + 'info': { + 'foo': 'bar' + } + }) + + default_conf['dry_run'] = True + mocker.patch('freqtrade.exchange.Exchange.symbol_amount_prec', lambda s, x, y: y) + mocker.patch('freqtrade.exchange.Exchange.symbol_price_prec', lambda s, x, y: y) + + exchange = get_patched_exchange(mocker, default_conf, api_mock, 'binance') + + with pytest.raises(OperationalException): + order = exchange.stoploss_limit(pair='ETH/BTC', amount=1, stop_price=190, rate=200) + + api_mock.create_order.reset_mock() + + order = exchange.stoploss_limit(pair='ETH/BTC', amount=1, stop_price=220, rate=200) + + assert 'id' in order + assert 'info' in order + assert order['id'] == order_id + assert api_mock.create_order.call_args[0][0] == 'ETH/BTC' + assert api_mock.create_order.call_args[0][1] == order_type + assert api_mock.create_order.call_args[0][2] == 'sell' + assert api_mock.create_order.call_args[0][3] == 1 + assert api_mock.create_order.call_args[0][4] == 200 + assert api_mock.create_order.call_args[0][5] == {'stopPrice': 220} From b2c0b20a58c161961d13bb1fa34494516f93332e Mon Sep 17 00:00:00 2001 From: misagh Date: Sat, 24 Nov 2018 18:26:04 +0100 Subject: [PATCH 446/699] added real tests for stop on exchange in dry-run --- freqtrade/exchange/__init__.py | 17 +++++++++++++++++ freqtrade/tests/exchange/test_exchange.py | 23 ++++++----------------- 2 files changed, 23 insertions(+), 17 deletions(-) diff --git a/freqtrade/exchange/__init__.py b/freqtrade/exchange/__init__.py index a88fb6ee8..fc28516f4 100644 --- a/freqtrade/exchange/__init__.py +++ b/freqtrade/exchange/__init__.py @@ -349,6 +349,23 @@ class Exchange(object): raise OperationalException( 'In stoploss limit order, stop price should be more than limit price') + if self._conf['dry_run']: + order_id = f'dry_run_buy_{randint(0, 10**6)}' + self._dry_run_open_orders[order_id] = { + 'info': {}, + 'id': order_id, + 'pair': pair, + 'price': stop_price, + 'amount': amount, + 'type': 'stop_loss_limit', + 'side': 'sell', + 'remaining': amount, + 'datetime': arrow.utcnow().isoformat(), + 'status': 'open', + 'fee': None + } + return self._dry_run_open_orders[order_id] + return self._api.create_order(pair, 'stop_loss', 'sell', amount, rate, {'stopPrice': stop_price}) diff --git a/freqtrade/tests/exchange/test_exchange.py b/freqtrade/tests/exchange/test_exchange.py index 402596da8..5ae6a031a 100644 --- a/freqtrade/tests/exchange/test_exchange.py +++ b/freqtrade/tests/exchange/test_exchange.py @@ -1218,16 +1218,7 @@ def test_stoploss_limit_order(default_conf, mocker): def test_stoploss_limit_order_dry_run(default_conf, mocker): api_mock = MagicMock() - order_id = 'test_prod_buy_{}'.format(randint(0, 10 ** 6)) - order_type = 'stop_loss' - - api_mock.create_order = MagicMock(return_value={ - 'id': order_id, - 'info': { - 'foo': 'bar' - } - }) - + order_type = 'stop_loss_limit' default_conf['dry_run'] = True mocker.patch('freqtrade.exchange.Exchange.symbol_amount_prec', lambda s, x, y: y) mocker.patch('freqtrade.exchange.Exchange.symbol_price_prec', lambda s, x, y: y) @@ -1243,10 +1234,8 @@ def test_stoploss_limit_order_dry_run(default_conf, mocker): assert 'id' in order assert 'info' in order - assert order['id'] == order_id - assert api_mock.create_order.call_args[0][0] == 'ETH/BTC' - assert api_mock.create_order.call_args[0][1] == order_type - assert api_mock.create_order.call_args[0][2] == 'sell' - assert api_mock.create_order.call_args[0][3] == 1 - assert api_mock.create_order.call_args[0][4] == 200 - assert api_mock.create_order.call_args[0][5] == {'stopPrice': 220} + assert 'type' in order + + assert order['type'] == order_type + assert order['price'] == 220 + assert order['amount'] == 1 From fe8927136c6427c3351017256489c9d18a2c2094 Mon Sep 17 00:00:00 2001 From: misagh Date: Sat, 24 Nov 2018 18:36:07 +0100 Subject: [PATCH 447/699] typo --- freqtrade/freqtradebot.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index cf22ee52a..126cc485f 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -693,7 +693,7 @@ class FreqtradeBot(object): )['id'] trade.stoploss_order_id = str(stoploss_order_id) - # Or there is already a stoploss on exchnage. + # Or there is already a stoploss on exchange. # so we check if it is hit ... elif trade.stoploss_order_id: logger.debug('Handling stoploss on exchange %s ...', trade) From b5192880df34ef09e6eb8b25dce0d51a64a4a777 Mon Sep 17 00:00:00 2001 From: misagh Date: Sat, 24 Nov 2018 19:00:59 +0100 Subject: [PATCH 448/699] [WIP] adding tests for handle_stoploss_on_exchange. --- freqtrade/tests/test_freqtradebot.py | 76 ++++++++++++++++++++++++++++ 1 file changed, 76 insertions(+) diff --git a/freqtrade/tests/test_freqtradebot.py b/freqtrade/tests/test_freqtradebot.py index 64c8e9765..4ecd287d6 100644 --- a/freqtrade/tests/test_freqtradebot.py +++ b/freqtrade/tests/test_freqtradebot.py @@ -906,6 +906,82 @@ def test_add_stoploss_on_exchange(mocker, default_conf, limit_buy_order) -> None assert trade.is_open is True +def test_handle_stoploss_on_exchange(mocker, default_conf, fee, + markets, limit_buy_order, limit_sell_order) -> None: + + freqtrade = get_patched_freqtradebot(mocker, default_conf) + stoploss_limit = MagicMock(return_value={'id': 13434334}) + mocker.patch('freqtrade.exchange.Exchange.stoploss_limit', stoploss_limit) + + # First case: when stoploss is not yet set but the order is open + # should get the stoploss order id immediately + # and should return false as no trade actually happened + trade = MagicMock() + trade.is_open = True + trade.open_order_id = None + trade.stoploss_order_id = None + + assert freqtrade.handle_stoploss_on_exchange(trade) is False + assert stoploss_limit.call_count == 1 + assert trade.stoploss_order_id == "13434334" + + trade.reset_mock() + + # Second case: when stoploss is set but it is not yet hit + # should do nothing and return false + trade.is_open = True + trade.open_order_id = None + trade.stoploss_order_id = 100 + + hanging_stoploss_order = MagicMock(return_value={'status': 'open'}) + mocker.patch('freqtrade.exchange.Exchange.get_order', hanging_stoploss_order) + + assert freqtrade.handle_stoploss_on_exchange(trade) is False + assert trade.stoploss_order_id == 100 + + trade.reset_mock() + + # Third case: when stoploss is set and it is hit + # should unset stoploss_order_id and return true + # as a trade actually happened + patch_RPCManager(mocker) + patch_exchange(mocker) + mocker.patch.multiple( + 'freqtrade.exchange.Exchange', + get_ticker=MagicMock(return_value={ + 'bid': 0.00001172, + 'ask': 0.00001173, + 'last': 0.00001172 + }), + buy=MagicMock(return_value={'id': limit_buy_order['id']}), + sell=MagicMock(return_value={'id': limit_sell_order['id']}), + get_fee=fee, + get_markets=markets + ) + freqtrade = FreqtradeBot(default_conf) + patch_get_signal(freqtrade) + + freqtrade.create_trade() + + trade = Trade.query.first() + assert trade + + # trade = freqtrade.create + # trade.is_open = True + # trade.open_order_id = None + # trade.stoploss_order_id = 100 + + # stoploss_order_hit = MagicMock(return_value={'status': 'closed'}) + # mocker.patch('freqtrade.exchange.Exchange.get_order', stoploss_order_hit) + + # # trade = Trade.query.first() + # # assert trade + # assert freqtrade.handle_stoploss_on_exchange(trade) is True + # time.sleep(0.01) # Race condition fix + # assert trade.is_open is True + # assert trade.stoploss_order_id is None + + def test_process_maybe_execute_buy(mocker, default_conf) -> None: freqtrade = get_patched_freqtradebot(mocker, default_conf) From c8a0956e1bc34fbb418de13d08eac686b3cd6581 Mon Sep 17 00:00:00 2001 From: misagh Date: Sat, 24 Nov 2018 19:12:00 +0100 Subject: [PATCH 449/699] fixed test handle_stoploss_on_exchange --- freqtrade/tests/test_freqtradebot.py | 68 ++++++++++++---------------- 1 file changed, 29 insertions(+), 39 deletions(-) diff --git a/freqtrade/tests/test_freqtradebot.py b/freqtrade/tests/test_freqtradebot.py index 4ecd287d6..9b1f40ccf 100644 --- a/freqtrade/tests/test_freqtradebot.py +++ b/freqtrade/tests/test_freqtradebot.py @@ -908,10 +908,24 @@ def test_add_stoploss_on_exchange(mocker, default_conf, limit_buy_order) -> None def test_handle_stoploss_on_exchange(mocker, default_conf, fee, markets, limit_buy_order, limit_sell_order) -> None: - - freqtrade = get_patched_freqtradebot(mocker, default_conf) stoploss_limit = MagicMock(return_value={'id': 13434334}) - mocker.patch('freqtrade.exchange.Exchange.stoploss_limit', stoploss_limit) + patch_RPCManager(mocker) + patch_exchange(mocker) + mocker.patch.multiple( + 'freqtrade.exchange.Exchange', + get_ticker=MagicMock(return_value={ + 'bid': 0.00001172, + 'ask': 0.00001173, + 'last': 0.00001172 + }), + buy=MagicMock(return_value={'id': limit_buy_order['id']}), + sell=MagicMock(return_value={'id': limit_sell_order['id']}), + get_fee=fee, + get_markets=markets, + stoploss_limit=stoploss_limit + ) + freqtrade = FreqtradeBot(default_conf) + patch_get_signal(freqtrade) # First case: when stoploss is not yet set but the order is open # should get the stoploss order id immediately @@ -925,8 +939,6 @@ def test_handle_stoploss_on_exchange(mocker, default_conf, fee, assert stoploss_limit.call_count == 1 assert trade.stoploss_order_id == "13434334" - trade.reset_mock() - # Second case: when stoploss is set but it is not yet hit # should do nothing and return false trade.is_open = True @@ -939,47 +951,25 @@ def test_handle_stoploss_on_exchange(mocker, default_conf, fee, assert freqtrade.handle_stoploss_on_exchange(trade) is False assert trade.stoploss_order_id == 100 - trade.reset_mock() - # Third case: when stoploss is set and it is hit # should unset stoploss_order_id and return true # as a trade actually happened - patch_RPCManager(mocker) - patch_exchange(mocker) - mocker.patch.multiple( - 'freqtrade.exchange.Exchange', - get_ticker=MagicMock(return_value={ - 'bid': 0.00001172, - 'ask': 0.00001173, - 'last': 0.00001172 - }), - buy=MagicMock(return_value={'id': limit_buy_order['id']}), - sell=MagicMock(return_value={'id': limit_sell_order['id']}), - get_fee=fee, - get_markets=markets - ) - freqtrade = FreqtradeBot(default_conf) - patch_get_signal(freqtrade) - freqtrade.create_trade() - trade = Trade.query.first() + trade.is_open = True + trade.open_order_id = None + trade.stoploss_order_id = 100 assert trade - # trade = freqtrade.create - # trade.is_open = True - # trade.open_order_id = None - # trade.stoploss_order_id = 100 - - # stoploss_order_hit = MagicMock(return_value={'status': 'closed'}) - # mocker.patch('freqtrade.exchange.Exchange.get_order', stoploss_order_hit) - - # # trade = Trade.query.first() - # # assert trade - # assert freqtrade.handle_stoploss_on_exchange(trade) is True - # time.sleep(0.01) # Race condition fix - # assert trade.is_open is True - # assert trade.stoploss_order_id is None + stoploss_order_hit = MagicMock(return_value={ + 'status': 'closed', + 'type': 'stop_loss_limit', + 'price': 2 + }) + mocker.patch('freqtrade.exchange.Exchange.get_order', stoploss_order_hit) + assert freqtrade.handle_stoploss_on_exchange(trade) is True + assert trade.stoploss_order_id is None + assert trade.is_open is False def test_process_maybe_execute_buy(mocker, default_conf) -> None: From 21a093bcdbe64d3fa9c20dde208169c12d03ca3a Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 24 Nov 2018 20:00:02 +0100 Subject: [PATCH 450/699] extract resolvers to IResolvers and it's own package --- freqtrade/freqtradebot.py | 4 +- freqtrade/optimize/backtesting.py | 4 +- freqtrade/optimize/edge_cli.py | 2 +- freqtrade/resolvers/__init__.py | 2 + freqtrade/resolvers/iresolver.py | 72 +++++++++++++++++++ .../strategyresolver.py} | 52 ++------------ freqtrade/tests/optimize/test_hyperopt.py | 2 +- freqtrade/tests/strategy/test_strategy.py | 32 +++++---- freqtrade/tests/test_dataframe.py | 2 +- scripts/plot_dataframe.py | 2 +- scripts/plot_profit.py | 2 +- 11 files changed, 107 insertions(+), 69 deletions(-) create mode 100644 freqtrade/resolvers/__init__.py create mode 100644 freqtrade/resolvers/iresolver.py rename freqtrade/{strategy/resolver.py => resolvers/strategyresolver.py} (74%) diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index 8a2db84a9..0526ef425 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -21,9 +21,9 @@ from freqtrade.wallets import Wallets from freqtrade.edge import Edge from freqtrade.persistence import Trade from freqtrade.rpc import RPCManager, RPCMessageType +from freqtrade.resolvers import StrategyResolver from freqtrade.state import State -from freqtrade.strategy.interface import SellType -from freqtrade.strategy.resolver import IStrategy, StrategyResolver +from freqtrade.strategy.interface import SellType, IStrategy from freqtrade.exchange.exchange_helpers import order_book_to_dataframe diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index 6fcde64fa..c6cf7276f 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -20,8 +20,8 @@ from freqtrade.configuration import Configuration from freqtrade.exchange import Exchange from freqtrade.misc import file_dump_json from freqtrade.persistence import Trade -from freqtrade.strategy.interface import SellType -from freqtrade.strategy.resolver import IStrategy, StrategyResolver +from freqtrade.resolvers import StrategyResolver +from freqtrade.strategy.interface import SellType, IStrategy logger = logging.getLogger(__name__) diff --git a/freqtrade/optimize/edge_cli.py b/freqtrade/optimize/edge_cli.py index 59bcbc098..a2189f6c1 100644 --- a/freqtrade/optimize/edge_cli.py +++ b/freqtrade/optimize/edge_cli.py @@ -12,7 +12,7 @@ from freqtrade.edge import Edge from freqtrade.configuration import Configuration from freqtrade.arguments import Arguments from freqtrade.exchange import Exchange -from freqtrade.strategy.resolver import StrategyResolver +from freqtrade.resolvers import StrategyResolver logger = logging.getLogger(__name__) diff --git a/freqtrade/resolvers/__init__.py b/freqtrade/resolvers/__init__.py new file mode 100644 index 000000000..fe81b7712 --- /dev/null +++ b/freqtrade/resolvers/__init__.py @@ -0,0 +1,2 @@ +from freqtrade.resolvers.iresolver import IResolver # noqa: F401 +from freqtrade.resolvers.strategyresolver import StrategyResolver # noqa: F401 diff --git a/freqtrade/resolvers/iresolver.py b/freqtrade/resolvers/iresolver.py new file mode 100644 index 000000000..ea38b30c2 --- /dev/null +++ b/freqtrade/resolvers/iresolver.py @@ -0,0 +1,72 @@ +# pragma pylint: disable=attribute-defined-outside-init + +""" +This module load custom hyperopts +""" +import importlib.util +import inspect +import logging +import os +from typing import Optional, Dict, Type, Any + +from freqtrade.constants import DEFAULT_HYPEROPT +from freqtrade.optimize.hyperopt_interface import IHyperOpt + + +logger = logging.getLogger(__name__) + + +class IResolver(object): + """ + This class contains all the logic to load custom hyperopt class + """ + + def __init__(self, object_type, config: Optional[Dict] = None) -> None: + """ + Load the custom class from config parameter + :param config: configuration dictionary or None + """ + config = config or {} + + @staticmethod + def _get_valid_objects(object_type, module_path: str, + object_name: str) -> Optional[Type[Any]]: + """ + Returns a list of all possible objects for the given module_path of type oject_type + :param object_type: object_type (class) + :param module_path: absolute path to the module + :param object_name: Class name of the object + :return: Tuple with (name, class) or None + """ + + # Generate spec based on absolute path + spec = importlib.util.spec_from_file_location('unknown', module_path) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) # type: ignore # importlib does not use typehints + + valid_objects_gen = ( + obj for name, obj in inspect.getmembers(module, inspect.isclass) + if object_name == name and object_type in obj.__bases__ + ) + return next(valid_objects_gen, None) + + @staticmethod + def _search_object(directory: str, object_type, object_name: str, + kwargs: dict) -> Optional[Any]: + """ + Search for the objectname in the given directory + :param directory: relative or absolute directory path + :return: object instance + """ + logger.debug('Searching for %s %s in \'%s\'', object_type.__name__, object_name, directory) + for entry in os.listdir(directory): + # Only consider python files + if not entry.endswith('.py'): + logger.debug('Ignoring %s', entry) + continue + obj = IResolver._get_valid_objects( + object_type, os.path.abspath(os.path.join(directory, entry)), object_name + ) + if obj: + return obj(**kwargs) + return None diff --git a/freqtrade/strategy/resolver.py b/freqtrade/resolvers/strategyresolver.py similarity index 74% rename from freqtrade/strategy/resolver.py rename to freqtrade/resolvers/strategyresolver.py index 3f25e4838..a7f3b2c25 100644 --- a/freqtrade/strategy/resolver.py +++ b/freqtrade/resolvers/strategyresolver.py @@ -3,7 +3,6 @@ """ This module load custom strategies """ -import importlib.util import inspect import logging import os @@ -11,16 +10,17 @@ import tempfile from base64 import urlsafe_b64decode from collections import OrderedDict from pathlib import Path -from typing import Dict, Optional, Type +from typing import Dict, Optional from freqtrade import constants +from freqtrade.resolvers import IResolver from freqtrade.strategy import import_strategy from freqtrade.strategy.interface import IStrategy logger = logging.getLogger(__name__) -class StrategyResolver(object): +class StrategyResolver(IResolver): """ This class contains all the logic to load custom strategy class """ @@ -103,7 +103,8 @@ class StrategyResolver(object): :param extra_dir: additional directory to search for the given strategy :return: Strategy instance or None """ - current_path = os.path.dirname(os.path.realpath(__file__)) + current_path = os.path.join(os.path.dirname(os.path.dirname(os.path.realpath(__file__))), 'strategy') + abs_paths = [ os.path.join(os.getcwd(), 'user_data', 'strategies'), current_path, @@ -131,7 +132,8 @@ class StrategyResolver(object): for path in abs_paths: try: - strategy = self._search_strategy(path, strategy_name=strategy_name, config=config) + strategy = self._search_object(directory=path, object_type=IStrategy, + object_name=strategy_name, kwargs={'config': config}) if strategy: logger.info('Using resolved strategy %s from \'%s\'', strategy_name, path) strategy._populate_fun_len = len( @@ -149,43 +151,3 @@ class StrategyResolver(object): "Impossible to load Strategy '{}'. This class does not exist" " or contains Python code errors".format(strategy_name) ) - - @staticmethod - def _get_valid_strategies(module_path: str, strategy_name: str) -> Optional[Type[IStrategy]]: - """ - Returns a list of all possible strategies for the given module_path - :param module_path: absolute path to the module - :param strategy_name: Class name of the strategy - :return: Tuple with (name, class) or None - """ - - # Generate spec based on absolute path - spec = importlib.util.spec_from_file_location('unknown', module_path) - module = importlib.util.module_from_spec(spec) - spec.loader.exec_module(module) # type: ignore # importlib does not use typehints - - valid_strategies_gen = ( - obj for name, obj in inspect.getmembers(module, inspect.isclass) - if strategy_name == name and IStrategy in obj.__bases__ - ) - return next(valid_strategies_gen, None) - - @staticmethod - def _search_strategy(directory: str, strategy_name: str, config: dict) -> Optional[IStrategy]: - """ - Search for the strategy_name in the given directory - :param directory: relative or absolute directory path - :return: name of the strategy class - """ - logger.debug('Searching for strategy %s in \'%s\'', strategy_name, directory) - for entry in os.listdir(directory): - # Only consider python files - if not entry.endswith('.py'): - logger.debug('Ignoring %s', entry) - continue - strategy = StrategyResolver._get_valid_strategies( - os.path.abspath(os.path.join(directory, entry)), strategy_name - ) - if strategy: - return strategy(config) - return None diff --git a/freqtrade/tests/optimize/test_hyperopt.py b/freqtrade/tests/optimize/test_hyperopt.py index 85d140b6d..01d2e8b17 100644 --- a/freqtrade/tests/optimize/test_hyperopt.py +++ b/freqtrade/tests/optimize/test_hyperopt.py @@ -7,7 +7,7 @@ import pytest from freqtrade.optimize.__init__ import load_tickerdata_file from freqtrade.optimize.hyperopt import Hyperopt, start -from freqtrade.strategy.resolver import StrategyResolver +from freqtrade.resolvers import StrategyResolver from freqtrade.tests.conftest import log_has, patch_exchange from freqtrade.tests.optimize.test_backtesting import get_args diff --git a/freqtrade/tests/strategy/test_strategy.py b/freqtrade/tests/strategy/test_strategy.py index a38050f24..ac20c9cab 100644 --- a/freqtrade/tests/strategy/test_strategy.py +++ b/freqtrade/tests/strategy/test_strategy.py @@ -10,7 +10,7 @@ from pandas import DataFrame from freqtrade.strategy import import_strategy from freqtrade.strategy.default_strategy import DefaultStrategy from freqtrade.strategy.interface import IStrategy -from freqtrade.strategy.resolver import StrategyResolver +from freqtrade.resolvers import StrategyResolver def test_import_strategy(caplog): @@ -44,17 +44,19 @@ def test_search_strategy(): path.realpath(__file__)), '..', '..', 'strategy' ) assert isinstance( - StrategyResolver._search_strategy( - default_location, - config=default_config, - strategy_name='DefaultStrategy' + StrategyResolver._search_object( + directory=default_location, + object_type=IStrategy, + kwargs={'config': default_config}, + object_name='DefaultStrategy' ), IStrategy ) - assert StrategyResolver._search_strategy( - default_location, - config=default_config, - strategy_name='NotFoundStrategy' + assert StrategyResolver._search_object( + directory=default_location, + object_type=IStrategy, + kwargs={'config': default_config}, + object_name='NotFoundStrategy' ) is None @@ -77,7 +79,7 @@ def test_load_strategy_invalid_directory(result, caplog): resolver._load_strategy('TestStrategy', config={}, extra_dir=extra_dir) assert ( - 'freqtrade.strategy.resolver', + 'freqtrade.resolvers.strategyresolver', logging.WARNING, 'Path "{}" does not exist'.format(extra_dir), ) in caplog.record_tuples @@ -128,7 +130,7 @@ def test_strategy_override_minimal_roi(caplog): resolver = StrategyResolver(config) assert resolver.strategy.minimal_roi[0] == 0.5 - assert ('freqtrade.strategy.resolver', + assert ('freqtrade.resolvers.strategyresolver', logging.INFO, "Override strategy 'minimal_roi' with value in config file: {'0': 0.5}." ) in caplog.record_tuples @@ -143,7 +145,7 @@ def test_strategy_override_stoploss(caplog): resolver = StrategyResolver(config) assert resolver.strategy.stoploss == -0.5 - assert ('freqtrade.strategy.resolver', + assert ('freqtrade.resolvers.strategyresolver', logging.INFO, "Override strategy 'stoploss' with value in config file: -0.5." ) in caplog.record_tuples @@ -159,7 +161,7 @@ def test_strategy_override_ticker_interval(caplog): resolver = StrategyResolver(config) assert resolver.strategy.ticker_interval == 60 - assert ('freqtrade.strategy.resolver', + assert ('freqtrade.resolvers.strategyresolver', logging.INFO, "Override strategy 'ticker_interval' with value in config file: 60." ) in caplog.record_tuples @@ -175,7 +177,7 @@ def test_strategy_override_process_only_new_candles(caplog): resolver = StrategyResolver(config) assert resolver.strategy.process_only_new_candles - assert ('freqtrade.strategy.resolver', + assert ('freqtrade.resolvers.strategyresolver', logging.INFO, "Override process_only_new_candles 'process_only_new_candles' " "with value in config file: True." @@ -201,7 +203,7 @@ def test_strategy_override_order_types(caplog): for method in ['buy', 'sell', 'stoploss']: assert resolver.strategy.order_types[method] == order_types[method] - assert ('freqtrade.strategy.resolver', + assert ('freqtrade.resolvers.strategyresolver', logging.INFO, "Override strategy 'order_types' with value in config file:" " {'buy': 'market', 'sell': 'limit', 'stoploss': 'limit'}." diff --git a/freqtrade/tests/test_dataframe.py b/freqtrade/tests/test_dataframe.py index dc030d630..6afb83a3f 100644 --- a/freqtrade/tests/test_dataframe.py +++ b/freqtrade/tests/test_dataframe.py @@ -3,7 +3,7 @@ import pandas from freqtrade.optimize import load_data -from freqtrade.strategy.resolver import StrategyResolver +from freqtrade.resolvers import StrategyResolver _pairs = ['ETH/BTC'] diff --git a/scripts/plot_dataframe.py b/scripts/plot_dataframe.py index 68713f296..8fd3a43bd 100755 --- a/scripts/plot_dataframe.py +++ b/scripts/plot_dataframe.py @@ -44,7 +44,7 @@ from freqtrade.arguments import Arguments, TimeRange from freqtrade.exchange import Exchange from freqtrade.optimize.backtesting import setup_configuration from freqtrade.persistence import Trade -from freqtrade.strategy.resolver import StrategyResolver +from freqtrade.resolvers import StrategyResolver logger = logging.getLogger(__name__) _CONF: Dict[str, Any] = {} diff --git a/scripts/plot_profit.py b/scripts/plot_profit.py index 9c3468c74..53f14ca3c 100755 --- a/scripts/plot_profit.py +++ b/scripts/plot_profit.py @@ -27,7 +27,7 @@ import plotly.graph_objs as go from freqtrade.arguments import Arguments from freqtrade.configuration import Configuration from freqtrade import constants -from freqtrade.strategy.resolver import StrategyResolver +from freqtrade.resolvers import StrategyResolver import freqtrade.optimize as optimize import freqtrade.misc as misc From 2c0d0946e6e9a953ef47ad617f8d921ede54685e Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 24 Nov 2018 20:02:29 +0100 Subject: [PATCH 451/699] Small stylistic improvements to strategyresolver --- freqtrade/resolvers/iresolver.py | 8 ++------ freqtrade/resolvers/strategyresolver.py | 12 ++++++------ 2 files changed, 8 insertions(+), 12 deletions(-) diff --git a/freqtrade/resolvers/iresolver.py b/freqtrade/resolvers/iresolver.py index ea38b30c2..37230537e 100644 --- a/freqtrade/resolvers/iresolver.py +++ b/freqtrade/resolvers/iresolver.py @@ -1,7 +1,7 @@ # pragma pylint: disable=attribute-defined-outside-init """ -This module load custom hyperopts +This module load custom objects """ import importlib.util import inspect @@ -9,10 +9,6 @@ import logging import os from typing import Optional, Dict, Type, Any -from freqtrade.constants import DEFAULT_HYPEROPT -from freqtrade.optimize.hyperopt_interface import IHyperOpt - - logger = logging.getLogger(__name__) @@ -51,7 +47,7 @@ class IResolver(object): return next(valid_objects_gen, None) @staticmethod - def _search_object(directory: str, object_type, object_name: str, + def _search_object(directory: str, object_type, object_name: str, kwargs: dict) -> Optional[Any]: """ Search for the objectname in the given directory diff --git a/freqtrade/resolvers/strategyresolver.py b/freqtrade/resolvers/strategyresolver.py index a7f3b2c25..273effe2d 100644 --- a/freqtrade/resolvers/strategyresolver.py +++ b/freqtrade/resolvers/strategyresolver.py @@ -5,7 +5,7 @@ This module load custom strategies """ import inspect import logging -import os +from os import getcwd, path import tempfile from base64 import urlsafe_b64decode from collections import OrderedDict @@ -103,10 +103,10 @@ class StrategyResolver(IResolver): :param extra_dir: additional directory to search for the given strategy :return: Strategy instance or None """ - current_path = os.path.join(os.path.dirname(os.path.dirname(os.path.realpath(__file__))), 'strategy') + current_path = path.join(path.dirname(path.dirname(path.realpath(__file__))), 'strategy') abs_paths = [ - os.path.join(os.getcwd(), 'user_data', 'strategies'), + path.join(getcwd(), 'user_data', 'strategies'), current_path, ] @@ -125,14 +125,14 @@ class StrategyResolver(IResolver): temp.joinpath(name).write_text(urlsafe_b64decode(strat[1]).decode('utf-8')) temp.joinpath("__init__.py").touch() - strategy_name = os.path.splitext(name)[0] + strategy_name = path.splitext(name)[0] # register temp path with the bot abs_paths.insert(0, str(temp.resolve())) - for path in abs_paths: + for _path in abs_paths: try: - strategy = self._search_object(directory=path, object_type=IStrategy, + strategy = self._search_object(directory=_path, object_type=IStrategy, object_name=strategy_name, kwargs={'config': config}) if strategy: logger.info('Using resolved strategy %s from \'%s\'', strategy_name, path) From 519b1f00e2932d034afb76ee999f4273b8c7467a Mon Sep 17 00:00:00 2001 From: misagh Date: Sat, 24 Nov 2018 20:12:50 +0100 Subject: [PATCH 452/699] adding strategy config consistency function --- freqtrade/freqtradebot.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index 126cc485f..dc1189bb9 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -54,11 +54,14 @@ class FreqtradeBot(object): # Init objects self.config = config self.strategy: IStrategy = StrategyResolver(self.config).strategy + self.check_strategy_config_consistency(config, self.strategy) + self.rpc: RPCManager = RPCManager(self) self.persistence = None self.exchange = Exchange(self.config) self.wallets = Wallets(self.exchange) + # Initializing Edge only if enabled self.edge = Edge(self.config, self.exchange, self.strategy) if \ self.config.get('edge', {}).get('enabled', False) else None @@ -66,6 +69,16 @@ class FreqtradeBot(object): self.active_pair_whitelist: List[str] = self.config['exchange']['pair_whitelist'] self._init_modules() + def check_strategy_config_consistency(self, config, strategy: IStrategy) -> None: + """ + checks if config is compatible with the given strategy + """ + + # Stoploss on exchange is only implemented for binance + if strategy.stoploss_on_exchange and config.get('exchange') is not 'binance': + raise OperationalException( + 'Stoploss limit orders are implemented only for binance as of now.') + def _init_modules(self) -> None: """ Initializes all modules and updates the config From cc7b8209783c123c9c42ed9a9df855fb7478d807 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 24 Nov 2018 20:14:08 +0100 Subject: [PATCH 453/699] Move hyperoptresolver to resolvers package --- freqtrade/optimize/hyperopt.py | 2 +- freqtrade/optimize/hyperopt_resolver.py | 104 ------------------ freqtrade/resolvers/__init__.py | 3 +- freqtrade/resolvers/hyperopt_resolver.py | 64 +++++++++++ freqtrade/resolvers/iresolver.py | 2 +- ...rategyresolver.py => strategy_resolver.py} | 2 +- freqtrade/tests/strategy/test_strategy.py | 12 +- 7 files changed, 75 insertions(+), 114 deletions(-) delete mode 100644 freqtrade/optimize/hyperopt_resolver.py create mode 100644 freqtrade/resolvers/hyperopt_resolver.py rename freqtrade/resolvers/{strategyresolver.py => strategy_resolver.py} (98%) diff --git a/freqtrade/optimize/hyperopt.py b/freqtrade/optimize/hyperopt.py index 70d20673c..fcf35acfe 100644 --- a/freqtrade/optimize/hyperopt.py +++ b/freqtrade/optimize/hyperopt.py @@ -22,7 +22,7 @@ from freqtrade.arguments import Arguments from freqtrade.configuration import Configuration from freqtrade.optimize import load_data from freqtrade.optimize.backtesting import Backtesting -from freqtrade.optimize.hyperopt_resolver import HyperOptResolver +from freqtrade.resolvers import HyperOptResolver logger = logging.getLogger(__name__) diff --git a/freqtrade/optimize/hyperopt_resolver.py b/freqtrade/optimize/hyperopt_resolver.py deleted file mode 100644 index 3d019e8df..000000000 --- a/freqtrade/optimize/hyperopt_resolver.py +++ /dev/null @@ -1,104 +0,0 @@ -# pragma pylint: disable=attribute-defined-outside-init - -""" -This module load custom hyperopts -""" -import importlib.util -import inspect -import logging -import os -from typing import Optional, Dict, Type - -from freqtrade.constants import DEFAULT_HYPEROPT -from freqtrade.optimize.hyperopt_interface import IHyperOpt - - -logger = logging.getLogger(__name__) - - -class HyperOptResolver(object): - """ - This class contains all the logic to load custom hyperopt class - """ - - __slots__ = ['hyperopt'] - - def __init__(self, config: Optional[Dict] = None) -> None: - """ - Load the custom class from config parameter - :param config: configuration dictionary or None - """ - config = config or {} - - # Verify the hyperopt is in the configuration, otherwise fallback to the default hyperopt - hyperopt_name = config.get('hyperopt') or DEFAULT_HYPEROPT - self.hyperopt = self._load_hyperopt(hyperopt_name, extra_dir=config.get('hyperopt_path')) - - def _load_hyperopt( - self, hyperopt_name: str, extra_dir: Optional[str] = None) -> IHyperOpt: - """ - Search and loads the specified hyperopt. - :param hyperopt_name: name of the module to import - :param extra_dir: additional directory to search for the given hyperopt - :return: HyperOpt instance or None - """ - current_path = os.path.dirname(os.path.realpath(__file__)) - abs_paths = [ - os.path.join(current_path, '..', '..', 'user_data', 'hyperopts'), - current_path, - ] - - if extra_dir: - # Add extra hyperopt directory on top of search paths - abs_paths.insert(0, extra_dir) - - for path in abs_paths: - hyperopt = self._search_hyperopt(path, hyperopt_name) - if hyperopt: - logger.info('Using resolved hyperopt %s from \'%s\'', hyperopt_name, path) - return hyperopt - - raise ImportError( - "Impossible to load Hyperopt '{}'. This class does not exist" - " or contains Python code errors".format(hyperopt_name) - ) - - @staticmethod - def _get_valid_hyperopts(module_path: str, hyperopt_name: str) -> Optional[Type[IHyperOpt]]: - """ - Returns a list of all possible hyperopts for the given module_path - :param module_path: absolute path to the module - :param hyperopt_name: Class name of the hyperopt - :return: Tuple with (name, class) or None - """ - - # Generate spec based on absolute path - spec = importlib.util.spec_from_file_location('user_data.hyperopts', module_path) - module = importlib.util.module_from_spec(spec) - spec.loader.exec_module(module) # type: ignore # importlib does not use typehints - - valid_hyperopts_gen = ( - obj for name, obj in inspect.getmembers(module, inspect.isclass) - if hyperopt_name == name and IHyperOpt in obj.__bases__ - ) - return next(valid_hyperopts_gen, None) - - @staticmethod - def _search_hyperopt(directory: str, hyperopt_name: str) -> Optional[IHyperOpt]: - """ - Search for the hyperopt_name in the given directory - :param directory: relative or absolute directory path - :return: name of the hyperopt class - """ - logger.debug('Searching for hyperopt %s in \'%s\'', hyperopt_name, directory) - for entry in os.listdir(directory): - # Only consider python files - if not entry.endswith('.py'): - logger.debug('Ignoring %s', entry) - continue - hyperopt = HyperOptResolver._get_valid_hyperopts( - os.path.abspath(os.path.join(directory, entry)), hyperopt_name - ) - if hyperopt: - return hyperopt() - return None diff --git a/freqtrade/resolvers/__init__.py b/freqtrade/resolvers/__init__.py index fe81b7712..84e3bcdcd 100644 --- a/freqtrade/resolvers/__init__.py +++ b/freqtrade/resolvers/__init__.py @@ -1,2 +1,3 @@ from freqtrade.resolvers.iresolver import IResolver # noqa: F401 -from freqtrade.resolvers.strategyresolver import StrategyResolver # noqa: F401 +from freqtrade.resolvers.hyperopt_resolver import HyperOptResolver # noqa: F401 +from freqtrade.resolvers.strategy_resolver import StrategyResolver # noqa: F401 diff --git a/freqtrade/resolvers/hyperopt_resolver.py b/freqtrade/resolvers/hyperopt_resolver.py new file mode 100644 index 000000000..38cb683c9 --- /dev/null +++ b/freqtrade/resolvers/hyperopt_resolver.py @@ -0,0 +1,64 @@ +# pragma pylint: disable=attribute-defined-outside-init + +""" +This module load custom hyperopts +""" +import logging +from os import path +from typing import Optional, Dict + +from freqtrade.constants import DEFAULT_HYPEROPT +from freqtrade.optimize.hyperopt_interface import IHyperOpt +from freqtrade.resolvers import IResolver + +logger = logging.getLogger(__name__) + + +class HyperOptResolver(IResolver): + """ + This class contains all the logic to load custom hyperopt class + """ + + __slots__ = ['hyperopt'] + + def __init__(self, config: Optional[Dict] = None) -> None: + """ + Load the custom class from config parameter + :param config: configuration dictionary or None + """ + config = config or {} + + # Verify the hyperopt is in the configuration, otherwise fallback to the default hyperopt + hyperopt_name = config.get('hyperopt') or DEFAULT_HYPEROPT + self.hyperopt = self._load_hyperopt(hyperopt_name, extra_dir=config.get('hyperopt_path')) + + def _load_hyperopt( + self, hyperopt_name: str, extra_dir: Optional[str] = None) -> IHyperOpt: + """ + Search and loads the specified hyperopt. + :param hyperopt_name: name of the module to import + :param extra_dir: additional directory to search for the given hyperopt + :return: HyperOpt instance or None + """ + current_path = path.join(path.dirname(path.dirname(path.realpath(__file__))), 'optimize') + + abs_paths = [ + path.join(current_path, '..', '..', 'user_data', 'hyperopts'), + current_path, + ] + + if extra_dir: + # Add extra hyperopt directory on top of search paths + abs_paths.insert(0, extra_dir) + + for _path in abs_paths: + hyperopt = self._search_object(directory=_path, object_type=IHyperOpt, + object_name=hyperopt_name) + if hyperopt: + logger.info('Using resolved hyperopt %s from \'%s\'', hyperopt_name, _path) + return hyperopt + + raise ImportError( + "Impossible to load Hyperopt '{}'. This class does not exist" + " or contains Python code errors".format(hyperopt_name) + ) diff --git a/freqtrade/resolvers/iresolver.py b/freqtrade/resolvers/iresolver.py index 37230537e..5cf6a1bc2 100644 --- a/freqtrade/resolvers/iresolver.py +++ b/freqtrade/resolvers/iresolver.py @@ -48,7 +48,7 @@ class IResolver(object): @staticmethod def _search_object(directory: str, object_type, object_name: str, - kwargs: dict) -> Optional[Any]: + kwargs: dict = {}) -> Optional[Any]: """ Search for the objectname in the given directory :param directory: relative or absolute directory path diff --git a/freqtrade/resolvers/strategyresolver.py b/freqtrade/resolvers/strategy_resolver.py similarity index 98% rename from freqtrade/resolvers/strategyresolver.py rename to freqtrade/resolvers/strategy_resolver.py index 273effe2d..f950a6e41 100644 --- a/freqtrade/resolvers/strategyresolver.py +++ b/freqtrade/resolvers/strategy_resolver.py @@ -145,7 +145,7 @@ class StrategyResolver(IResolver): return import_strategy(strategy, config=config) except FileNotFoundError: - logger.warning('Path "%s" does not exist', path) + logger.warning('Path "%s" does not exist', _path) raise ImportError( "Impossible to load Strategy '{}'. This class does not exist" diff --git a/freqtrade/tests/strategy/test_strategy.py b/freqtrade/tests/strategy/test_strategy.py index ac20c9cab..230ec7ab7 100644 --- a/freqtrade/tests/strategy/test_strategy.py +++ b/freqtrade/tests/strategy/test_strategy.py @@ -79,7 +79,7 @@ def test_load_strategy_invalid_directory(result, caplog): resolver._load_strategy('TestStrategy', config={}, extra_dir=extra_dir) assert ( - 'freqtrade.resolvers.strategyresolver', + 'freqtrade.resolvers.strategy_resolver', logging.WARNING, 'Path "{}" does not exist'.format(extra_dir), ) in caplog.record_tuples @@ -130,7 +130,7 @@ def test_strategy_override_minimal_roi(caplog): resolver = StrategyResolver(config) assert resolver.strategy.minimal_roi[0] == 0.5 - assert ('freqtrade.resolvers.strategyresolver', + assert ('freqtrade.resolvers.strategy_resolver', logging.INFO, "Override strategy 'minimal_roi' with value in config file: {'0': 0.5}." ) in caplog.record_tuples @@ -145,7 +145,7 @@ def test_strategy_override_stoploss(caplog): resolver = StrategyResolver(config) assert resolver.strategy.stoploss == -0.5 - assert ('freqtrade.resolvers.strategyresolver', + assert ('freqtrade.resolvers.strategy_resolver', logging.INFO, "Override strategy 'stoploss' with value in config file: -0.5." ) in caplog.record_tuples @@ -161,7 +161,7 @@ def test_strategy_override_ticker_interval(caplog): resolver = StrategyResolver(config) assert resolver.strategy.ticker_interval == 60 - assert ('freqtrade.resolvers.strategyresolver', + assert ('freqtrade.resolvers.strategy_resolver', logging.INFO, "Override strategy 'ticker_interval' with value in config file: 60." ) in caplog.record_tuples @@ -177,7 +177,7 @@ def test_strategy_override_process_only_new_candles(caplog): resolver = StrategyResolver(config) assert resolver.strategy.process_only_new_candles - assert ('freqtrade.resolvers.strategyresolver', + assert ('freqtrade.resolvers.strategy_resolver', logging.INFO, "Override process_only_new_candles 'process_only_new_candles' " "with value in config file: True." @@ -203,7 +203,7 @@ def test_strategy_override_order_types(caplog): for method in ['buy', 'sell', 'stoploss']: assert resolver.strategy.order_types[method] == order_types[method] - assert ('freqtrade.resolvers.strategyresolver', + assert ('freqtrade.resolvers.strategy_resolver', logging.INFO, "Override strategy 'order_types' with value in config file:" " {'buy': 'market', 'sell': 'limit', 'stoploss': 'limit'}." From 20de8c82e4c25aec91aa12fee76070dabc4d5f13 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 24 Nov 2018 20:39:16 +0100 Subject: [PATCH 454/699] Convert to Pathlib --- freqtrade/resolvers/hyperopt_resolver.py | 8 ++++---- freqtrade/resolvers/iresolver.py | 14 +++++++------- freqtrade/resolvers/strategy_resolver.py | 15 +++++++-------- freqtrade/tests/strategy/test_strategy.py | 5 ++--- 4 files changed, 20 insertions(+), 22 deletions(-) diff --git a/freqtrade/resolvers/hyperopt_resolver.py b/freqtrade/resolvers/hyperopt_resolver.py index 38cb683c9..da7b65648 100644 --- a/freqtrade/resolvers/hyperopt_resolver.py +++ b/freqtrade/resolvers/hyperopt_resolver.py @@ -4,7 +4,7 @@ This module load custom hyperopts """ import logging -from os import path +from pathlib import Path from typing import Optional, Dict from freqtrade.constants import DEFAULT_HYPEROPT @@ -40,16 +40,16 @@ class HyperOptResolver(IResolver): :param extra_dir: additional directory to search for the given hyperopt :return: HyperOpt instance or None """ - current_path = path.join(path.dirname(path.dirname(path.realpath(__file__))), 'optimize') + current_path = Path(__file__).parent.parent.joinpath('optimize').resolve() abs_paths = [ - path.join(current_path, '..', '..', 'user_data', 'hyperopts'), + current_path.parent.parent.joinpath('user_data/hyperopts'), current_path, ] if extra_dir: # Add extra hyperopt directory on top of search paths - abs_paths.insert(0, extra_dir) + abs_paths.insert(0, Path(extra_dir)) for _path in abs_paths: hyperopt = self._search_object(directory=_path, object_type=IHyperOpt, diff --git a/freqtrade/resolvers/iresolver.py b/freqtrade/resolvers/iresolver.py index 5cf6a1bc2..78df32e89 100644 --- a/freqtrade/resolvers/iresolver.py +++ b/freqtrade/resolvers/iresolver.py @@ -6,7 +6,7 @@ This module load custom objects import importlib.util import inspect import logging -import os +from pathlib import Path from typing import Optional, Dict, Type, Any logger = logging.getLogger(__name__) @@ -25,7 +25,7 @@ class IResolver(object): config = config or {} @staticmethod - def _get_valid_objects(object_type, module_path: str, + def _get_valid_objects(object_type, module_path: Path, object_name: str) -> Optional[Type[Any]]: """ Returns a list of all possible objects for the given module_path of type oject_type @@ -36,7 +36,7 @@ class IResolver(object): """ # Generate spec based on absolute path - spec = importlib.util.spec_from_file_location('unknown', module_path) + spec = importlib.util.spec_from_file_location('unknown', str(module_path)) module = importlib.util.module_from_spec(spec) spec.loader.exec_module(module) # type: ignore # importlib does not use typehints @@ -47,7 +47,7 @@ class IResolver(object): return next(valid_objects_gen, None) @staticmethod - def _search_object(directory: str, object_type, object_name: str, + def _search_object(directory: Path, object_type, object_name: str, kwargs: dict = {}) -> Optional[Any]: """ Search for the objectname in the given directory @@ -55,13 +55,13 @@ class IResolver(object): :return: object instance """ logger.debug('Searching for %s %s in \'%s\'', object_type.__name__, object_name, directory) - for entry in os.listdir(directory): + for entry in directory.iterdir(): # Only consider python files - if not entry.endswith('.py'): + if not str(entry).endswith('.py'): logger.debug('Ignoring %s', entry) continue obj = IResolver._get_valid_objects( - object_type, os.path.abspath(os.path.join(directory, entry)), object_name + object_type, Path.resolve(directory.joinpath(entry)), object_name ) if obj: return obj(**kwargs) diff --git a/freqtrade/resolvers/strategy_resolver.py b/freqtrade/resolvers/strategy_resolver.py index f950a6e41..4576d0ec8 100644 --- a/freqtrade/resolvers/strategy_resolver.py +++ b/freqtrade/resolvers/strategy_resolver.py @@ -5,7 +5,6 @@ This module load custom strategies """ import inspect import logging -from os import getcwd, path import tempfile from base64 import urlsafe_b64decode from collections import OrderedDict @@ -103,16 +102,16 @@ class StrategyResolver(IResolver): :param extra_dir: additional directory to search for the given strategy :return: Strategy instance or None """ - current_path = path.join(path.dirname(path.dirname(path.realpath(__file__))), 'strategy') + current_path = Path(__file__).parent.parent.joinpath('strategy').resolve() abs_paths = [ - path.join(getcwd(), 'user_data', 'strategies'), + Path.cwd().joinpath('user_data/strategies'), current_path, ] if extra_dir: # Add extra strategy directory on top of search paths - abs_paths.insert(0, extra_dir) + abs_paths.insert(0, Path(extra_dir).resolve()) if ":" in strategy_name: logger.info("loading base64 endocded strategy") @@ -125,17 +124,17 @@ class StrategyResolver(IResolver): temp.joinpath(name).write_text(urlsafe_b64decode(strat[1]).decode('utf-8')) temp.joinpath("__init__.py").touch() - strategy_name = path.splitext(name)[0] + strategy_name = strat[0] # register temp path with the bot - abs_paths.insert(0, str(temp.resolve())) + abs_paths.insert(0, temp.resolve()) for _path in abs_paths: try: strategy = self._search_object(directory=_path, object_type=IStrategy, object_name=strategy_name, kwargs={'config': config}) if strategy: - logger.info('Using resolved strategy %s from \'%s\'', strategy_name, path) + logger.info('Using resolved strategy %s from \'%s\'', strategy_name, _path) strategy._populate_fun_len = len( inspect.getfullargspec(strategy.populate_indicators).args) strategy._buy_fun_len = len( @@ -145,7 +144,7 @@ class StrategyResolver(IResolver): return import_strategy(strategy, config=config) except FileNotFoundError: - logger.warning('Path "%s" does not exist', _path) + logger.warning('Path "%s" does not exist', _path.relative_to(Path.cwd())) raise ImportError( "Impossible to load Strategy '{}'. This class does not exist" diff --git a/freqtrade/tests/strategy/test_strategy.py b/freqtrade/tests/strategy/test_strategy.py index 230ec7ab7..80bd9e120 100644 --- a/freqtrade/tests/strategy/test_strategy.py +++ b/freqtrade/tests/strategy/test_strategy.py @@ -2,6 +2,7 @@ import logging from base64 import urlsafe_b64encode from os import path +from pathlib import Path import warnings import pytest @@ -40,9 +41,7 @@ def test_import_strategy(caplog): def test_search_strategy(): default_config = {} - default_location = path.join(path.dirname( - path.realpath(__file__)), '..', '..', 'strategy' - ) + default_location = Path(__file__).parent.parent.joinpath('strategy').resolve() assert isinstance( StrategyResolver._search_object( directory=default_location, From 266bd7b9b62d095d7f5d703e2cd8f27439102abd Mon Sep 17 00:00:00 2001 From: misagh Date: Sat, 24 Nov 2018 21:42:15 +0100 Subject: [PATCH 455/699] error message improved --- freqtrade/freqtradebot.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index dc1189bb9..fbabb7d90 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -77,7 +77,7 @@ class FreqtradeBot(object): # Stoploss on exchange is only implemented for binance if strategy.stoploss_on_exchange and config.get('exchange') is not 'binance': raise OperationalException( - 'Stoploss limit orders are implemented only for binance as of now.') + 'On exchange stoploss is not supported for %s.' % config.get('exchange')) def _init_modules(self) -> None: """ From a3477e07eb223217bee5b244c60e82216540008b Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 25 Nov 2018 09:55:36 +0100 Subject: [PATCH 456/699] Remove constructor, it's not needed in the baseclass --- freqtrade/resolvers/iresolver.py | 11 ++--------- 1 file changed, 2 insertions(+), 9 deletions(-) diff --git a/freqtrade/resolvers/iresolver.py b/freqtrade/resolvers/iresolver.py index 78df32e89..4bc2159fe 100644 --- a/freqtrade/resolvers/iresolver.py +++ b/freqtrade/resolvers/iresolver.py @@ -14,16 +14,9 @@ logger = logging.getLogger(__name__) class IResolver(object): """ - This class contains all the logic to load custom hyperopt class + This class contains all the logic to load custom classes """ - def __init__(self, object_type, config: Optional[Dict] = None) -> None: - """ - Load the custom class from config parameter - :param config: configuration dictionary or None - """ - config = config or {} - @staticmethod def _get_valid_objects(object_type, module_path: Path, object_name: str) -> Optional[Type[Any]]: @@ -54,7 +47,7 @@ class IResolver(object): :param directory: relative or absolute directory path :return: object instance """ - logger.debug('Searching for %s %s in \'%s\'', object_type.__name__, object_name, directory) + logger.debug('Searching for %s %s in \'%s\'', object_type.__name__, object_name, directory) for entry in directory.iterdir(): # Only consider python files if not str(entry).endswith('.py'): From 1d35428c8d530567916bf98a0b30f3a15ccf225a Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 25 Nov 2018 10:07:06 +0100 Subject: [PATCH 457/699] Rename get_valid_objects to get_valid object it only ever returns one object ... --- freqtrade/resolvers/iresolver.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/freqtrade/resolvers/iresolver.py b/freqtrade/resolvers/iresolver.py index 4bc2159fe..aee292926 100644 --- a/freqtrade/resolvers/iresolver.py +++ b/freqtrade/resolvers/iresolver.py @@ -7,7 +7,7 @@ import importlib.util import inspect import logging from pathlib import Path -from typing import Optional, Dict, Type, Any +from typing import Optional, Type, Any logger = logging.getLogger(__name__) @@ -18,14 +18,14 @@ class IResolver(object): """ @staticmethod - def _get_valid_objects(object_type, module_path: Path, - object_name: str) -> Optional[Type[Any]]: + def _get_valid_object(object_type, module_path: Path, + object_name: str) -> Optional[Type[Any]]: """ - Returns a list of all possible objects for the given module_path of type oject_type + Returns the first object with matching object_type and object_name in the path given. :param object_type: object_type (class) :param module_path: absolute path to the module :param object_name: Class name of the object - :return: Tuple with (name, class) or None + :return: class or None """ # Generate spec based on absolute path @@ -53,7 +53,7 @@ class IResolver(object): if not str(entry).endswith('.py'): logger.debug('Ignoring %s', entry) continue - obj = IResolver._get_valid_objects( + obj = IResolver._get_valid_object( object_type, Path.resolve(directory.joinpath(entry)), object_name ) if obj: From 664b96173eabe2e6472e6a812e631aa6378080eb Mon Sep 17 00:00:00 2001 From: misagh Date: Sun, 25 Nov 2018 10:54:36 +0100 Subject: [PATCH 458/699] removing NotImplementedError from stoploss_limit --- freqtrade/exchange/__init__.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/freqtrade/exchange/__init__.py b/freqtrade/exchange/__init__.py index fc28516f4..3ccd2369a 100644 --- a/freqtrade/exchange/__init__.py +++ b/freqtrade/exchange/__init__.py @@ -334,10 +334,10 @@ class Exchange(object): raise OperationalException(e) def stoploss_limit(self, pair: str, amount: float, stop_price: float, rate: float) -> Dict: - # Only binance is supported - if not self.name == 'Binance': - raise NotImplementedError( - 'Stoploss limit orders are implemented only for binance as of now.') + """ + creates a stoploss limit order. + NOTICE: it is not supported by all exchanges. only binance is tested for now. + """ # Set the precision for amount and price(rate) as accepted by the exchange amount = self.symbol_amount_prec(pair, amount) From dcae3a26440a07cadb362999003a2d81c6486157 Mon Sep 17 00:00:00 2001 From: misagh Date: Sun, 25 Nov 2018 11:29:04 +0100 Subject: [PATCH 459/699] test of check_consistency added --- freqtrade/freqtradebot.py | 5 ++--- freqtrade/tests/exchange/test_exchange.py | 7 ------- freqtrade/tests/test_freqtradebot.py | 6 ++++++ 3 files changed, 8 insertions(+), 10 deletions(-) diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index fbabb7d90..005f698dd 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -61,7 +61,6 @@ class FreqtradeBot(object): self.exchange = Exchange(self.config) self.wallets = Wallets(self.exchange) - # Initializing Edge only if enabled self.edge = Edge(self.config, self.exchange, self.strategy) if \ self.config.get('edge', {}).get('enabled', False) else None @@ -73,11 +72,11 @@ class FreqtradeBot(object): """ checks if config is compatible with the given strategy """ - # Stoploss on exchange is only implemented for binance if strategy.stoploss_on_exchange and config.get('exchange') is not 'binance': raise OperationalException( - 'On exchange stoploss is not supported for %s.' % config.get('exchange')) + 'On exchange stoploss is not supported for %s.' % config['exchange']['name'] + ) def _init_modules(self) -> None: """ diff --git a/freqtrade/tests/exchange/test_exchange.py b/freqtrade/tests/exchange/test_exchange.py index 5ae6a031a..57be54262 100644 --- a/freqtrade/tests/exchange/test_exchange.py +++ b/freqtrade/tests/exchange/test_exchange.py @@ -1173,13 +1173,6 @@ def test_get_fee(default_conf, mocker): 'get_fee', 'calculate_fee') -def test_stoploss_limit_available_only_for_binance(default_conf, mocker): - api_mock = MagicMock() - exchange = get_patched_exchange(mocker, default_conf, api_mock) - with pytest.raises(NotImplementedError): - exchange.stoploss_limit('BTC/ETH', 1, 0.8, 0.79) - - def test_stoploss_limit_order(default_conf, mocker): api_mock = MagicMock() order_id = 'test_prod_buy_{}'.format(randint(0, 10 ** 6)) diff --git a/freqtrade/tests/test_freqtradebot.py b/freqtrade/tests/test_freqtradebot.py index 9b1f40ccf..5acf4fdcb 100644 --- a/freqtrade/tests/test_freqtradebot.py +++ b/freqtrade/tests/test_freqtradebot.py @@ -2516,3 +2516,9 @@ def test_startup_messages(default_conf, mocker): default_conf['dynamic_whitelist'] = 20 freqtrade = get_patched_freqtradebot(mocker, default_conf) assert freqtrade.state is State.RUNNING + + +def test_check_consistency(default_conf, mocker, caplog): + mocker.patch('freqtrade.freqtradebot.IStrategy.stoploss_on_exchange', True) + with pytest.raises(OperationalException): + FreqtradeBot(default_conf) From e4744c1ba4cb428fc40da67c9b960e57b0c275b8 Mon Sep 17 00:00:00 2001 From: misagh Date: Sun, 25 Nov 2018 11:31:30 +0100 Subject: [PATCH 460/699] stop loss on exchanged removed from doc --- docs/configuration.md | 1 - 1 file changed, 1 deletion(-) diff --git a/docs/configuration.md b/docs/configuration.md index 64f0a2ea6..62559a41e 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -26,7 +26,6 @@ The table below will list all configuration parameters. | `process_only_new_candles` | false | No | If set to true indicators are processed only once a new candle arrives. If false each loop populates the indicators, this will mean the same candle is processed many times creating system load but can be useful of your strategy depends on tick data not only candle. Can be set either in Configuration or in the strategy. | `minimal_roi` | See below | No | Set the threshold in percent the bot will use to sell a trade. More information below. If set, this parameter will override `minimal_roi` from your strategy file. | `stoploss` | -0.10 | No | Value of the stoploss in percent used by the bot. More information below. If set, this parameter will override `stoploss` from your strategy file. -| `stoploss_on_exchange` | false | No | Only for binance users for now: If this parameter is on then stoploss limit order is executed immediately after buy order is done on binance. | `trailing_stop` | false | No | Enables trailing stop-loss (based on `stoploss` in either configuration or strategy file). | `trailing_stop_positve` | 0 | No | Changes stop-loss once profit has been reached. | `trailing_stop_positve_offset` | 0 | No | Offset on when to apply `trailing_stop_positive`. Percentage value which should be positive. From e89df448e878a208a384fcafc2c58b94752548e7 Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Sun, 25 Nov 2018 13:34:08 +0100 Subject: [PATCH 461/699] Update ccxt from 1.17.535 to 1.17.536 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 44cfe4a07..fa550195d 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -ccxt==1.17.535 +ccxt==1.17.536 SQLAlchemy==1.2.14 python-telegram-bot==11.1.0 arrow==0.12.1 From fd7184718baa99cd328c4f0dacac98c7f79cd3c0 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 25 Nov 2018 14:31:46 +0100 Subject: [PATCH 462/699] replace lambda with Magicmock in test --- freqtrade/tests/test_freqtradebot.py | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/freqtrade/tests/test_freqtradebot.py b/freqtrade/tests/test_freqtradebot.py index a36ae2cd8..eb5336c61 100644 --- a/freqtrade/tests/test_freqtradebot.py +++ b/freqtrade/tests/test_freqtradebot.py @@ -292,7 +292,7 @@ def test_edge_overrides_stoploss(limit_buy_order, fee, markets, caplog, mocker, freqtrade = FreqtradeBot(edge_conf) freqtrade.active_pair_whitelist = ['NEO/BTC'] patch_get_signal(freqtrade) - freqtrade.strategy.min_roi_reached = lambda trade, current_profit, current_time: False + freqtrade.strategy.min_roi_reached = MagicMock(return_value=False) freqtrade.create_trade() trade = Trade.query.first() trade.update(limit_buy_order) @@ -332,7 +332,7 @@ def test_edge_should_ignore_strategy_stoploss(limit_buy_order, fee, markets, freqtrade = FreqtradeBot(edge_conf) freqtrade.active_pair_whitelist = ['NEO/BTC'] patch_get_signal(freqtrade) - freqtrade.strategy.min_roi_reached = lambda trade, current_profit, current_time: False + freqtrade.strategy.min_roi_reached = MagicMock(return_value=False) freqtrade.create_trade() trade = Trade.query.first() trade.update(limit_buy_order) @@ -1010,7 +1010,7 @@ def test_handle_overlpapping_signals(default_conf, ticker, limit_buy_order, freqtrade = FreqtradeBot(default_conf) patch_get_signal(freqtrade, value=(True, True)) - freqtrade.strategy.min_roi_reached = lambda trade, current_profit, current_time: False + freqtrade.strategy.min_roi_reached = MagicMock(return_value=False) freqtrade.create_trade() @@ -1066,7 +1066,7 @@ def test_handle_trade_roi(default_conf, ticker, limit_buy_order, freqtrade = FreqtradeBot(default_conf) patch_get_signal(freqtrade, value=(True, False)) - freqtrade.strategy.min_roi_reached = lambda trade, current_profit, current_time: True + freqtrade.strategy.min_roi_reached = MagicMock(return_value=True) freqtrade.create_trade() @@ -1099,7 +1099,7 @@ def test_handle_trade_experimental( freqtrade = FreqtradeBot(default_conf) patch_get_signal(freqtrade) - freqtrade.strategy.min_roi_reached = lambda trade, current_profit, current_time: False + freqtrade.strategy.min_roi_reached = MagicMock(return_value=False) freqtrade.create_trade() trade = Trade.query.first() @@ -1580,7 +1580,7 @@ def test_sell_profit_only_enable_profit(default_conf, limit_buy_order, } freqtrade = FreqtradeBot(default_conf) patch_get_signal(freqtrade) - freqtrade.strategy.min_roi_reached = lambda trade, current_profit, current_time: False + freqtrade.strategy.min_roi_reached = MagicMock(return_value=False) freqtrade.create_trade() @@ -1612,7 +1612,7 @@ def test_sell_profit_only_disable_profit(default_conf, limit_buy_order, } freqtrade = FreqtradeBot(default_conf) patch_get_signal(freqtrade) - freqtrade.strategy.min_roi_reached = lambda trade, current_profit, current_time: False + freqtrade.strategy.min_roi_reached = MagicMock(return_value=False) freqtrade.create_trade() trade = Trade.query.first() @@ -1674,7 +1674,7 @@ def test_sell_profit_only_disable_loss(default_conf, limit_buy_order, fee, marke freqtrade = FreqtradeBot(default_conf) patch_get_signal(freqtrade) - freqtrade.strategy.min_roi_reached = lambda trade, current_profit, current_time: False + freqtrade.strategy.min_roi_reached = MagicMock(return_value=False) freqtrade.create_trade() @@ -1704,7 +1704,7 @@ def test_ignore_roi_if_buy_signal(default_conf, limit_buy_order, fee, markets, m } freqtrade = FreqtradeBot(default_conf) patch_get_signal(freqtrade) - freqtrade.strategy.min_roi_reached = lambda trade, current_profit, current_time: True + freqtrade.strategy.min_roi_reached = MagicMock(return_value=True) freqtrade.create_trade() @@ -1736,7 +1736,7 @@ def test_trailing_stop_loss(default_conf, limit_buy_order, fee, markets, caplog, default_conf['trailing_stop'] = True freqtrade = FreqtradeBot(default_conf) patch_get_signal(freqtrade) - freqtrade.strategy.min_roi_reached = lambda trade, current_profit, current_time: False + freqtrade.strategy.min_roi_reached = MagicMock(return_value=False) freqtrade.create_trade() @@ -1771,7 +1771,7 @@ def test_trailing_stop_loss_positive(default_conf, limit_buy_order, fee, markets default_conf['trailing_stop_positive'] = 0.01 freqtrade = FreqtradeBot(default_conf) patch_get_signal(freqtrade) - freqtrade.strategy.min_roi_reached = lambda trade, current_profit, current_time: False + freqtrade.strategy.min_roi_reached = MagicMock(return_value=False) freqtrade.create_trade() trade = Trade.query.first() @@ -1831,7 +1831,7 @@ def test_trailing_stop_loss_offset(default_conf, limit_buy_order, fee, default_conf['trailing_stop_positive_offset'] = 0.011 freqtrade = FreqtradeBot(default_conf) patch_get_signal(freqtrade) - freqtrade.strategy.min_roi_reached = lambda trade, current_profit, current_time: False + freqtrade.strategy.min_roi_reached = MagicMock(return_value=False) freqtrade.create_trade() trade = Trade.query.first() @@ -1890,7 +1890,7 @@ def test_disable_ignore_roi_if_buy_signal(default_conf, limit_buy_order, } freqtrade = FreqtradeBot(default_conf) patch_get_signal(freqtrade) - freqtrade.strategy.min_roi_reached = lambda trade, current_profit, current_time: True + freqtrade.strategy.min_roi_reached = MagicMock(return_value=True) freqtrade.create_trade() From 317eba2139f7fcc352b97c1de333a76ef18d27eb Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 25 Nov 2018 14:36:02 +0100 Subject: [PATCH 463/699] Remove dual instanciation of pairinfo named tuple --- freqtrade/tests/conftest.py | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/freqtrade/tests/conftest.py b/freqtrade/tests/conftest.py index 63655126c..f7fe697b8 100644 --- a/freqtrade/tests/conftest.py +++ b/freqtrade/tests/conftest.py @@ -4,7 +4,6 @@ import logging from datetime import datetime from functools import reduce from typing import Dict, Optional -from collections import namedtuple from unittest.mock import MagicMock, PropertyMock import arrow @@ -13,7 +12,7 @@ from telegram import Chat, Message, Update from freqtrade.exchange.exchange_helpers import parse_ticker_dataframe from freqtrade.exchange import Exchange -from freqtrade.edge import Edge +from freqtrade.edge import Edge, PairInfo from freqtrade.freqtradebot import FreqtradeBot logging.getLogger('').setLevel(logging.INFO) @@ -56,13 +55,11 @@ def patch_edge(mocker) -> None: # "LTC/BTC", # "XRP/BTC", # "NEO/BTC" - pair_info = namedtuple( - 'pair_info', - 'stoploss, winrate, risk_reward_ratio, required_risk_reward, expectancy') + mocker.patch('freqtrade.edge.Edge._cached_pairs', mocker.PropertyMock( return_value={ - 'NEO/BTC': pair_info(-0.20, 0.66, 3.71, 0.50, 1.71), - 'LTC/BTC': pair_info(-0.21, 0.66, 3.71, 0.50, 1.71), + 'NEO/BTC': PairInfo(-0.20, 0.66, 3.71, 0.50, 1.71, 10, 25), + 'LTC/BTC': PairInfo(-0.21, 0.66, 3.71, 0.50, 1.71, 11, 20), } )) mocker.patch('freqtrade.edge.Edge.stoploss', MagicMock(return_value=-0.20)) From 745a5177958ad16e220dd0ba2d8dc982fddfc016 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 25 Nov 2018 14:40:21 +0100 Subject: [PATCH 464/699] Fix comment pointing to wrong column --- freqtrade/exchange/exchange_helpers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/exchange/exchange_helpers.py b/freqtrade/exchange/exchange_helpers.py index 8f4b03daf..84e68d4bb 100644 --- a/freqtrade/exchange/exchange_helpers.py +++ b/freqtrade/exchange/exchange_helpers.py @@ -11,7 +11,7 @@ logger = logging.getLogger(__name__) def parse_ticker_dataframe(ticker: list) -> DataFrame: """ Analyses the trend for the given ticker history - :param ticker: See exchange.get_candle_history + :param ticker: ticker list, as returned by exchange.async_get_candle_history :return: DataFrame """ cols = ['date', 'open', 'high', 'low', 'close', 'volume'] From 8a4361199266c3f558e6eb6b98947155ace77d57 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 25 Nov 2018 14:48:15 +0100 Subject: [PATCH 465/699] Remove get_candle_history (it's now async) convert sort-test to async --- freqtrade/exchange/__init__.py | 45 ------------- freqtrade/tests/exchange/test_exchange.py | 78 ++++------------------- 2 files changed, 12 insertions(+), 111 deletions(-) diff --git a/freqtrade/exchange/__init__.py b/freqtrade/exchange/__init__.py index ae07e36e9..0f8d8895c 100644 --- a/freqtrade/exchange/__init__.py +++ b/freqtrade/exchange/__init__.py @@ -500,51 +500,6 @@ class Exchange(object): except ccxt.BaseError as e: raise OperationalException(f'Could not fetch ticker data. Msg: {e}') - @retrier - def get_candle_history(self, pair: str, tick_interval: str, - since_ms: Optional[int] = None) -> List[Dict]: - try: - # last item should be in the time interval [now - tick_interval, now] - till_time_ms = arrow.utcnow().shift( - minutes=-constants.TICKER_INTERVAL_MINUTES[tick_interval] - ).timestamp * 1000 - # it looks as if some exchanges return cached data - # and they update it one in several minute, so 10 mins interval - # is necessary to skeep downloading of an empty array when all - # chached data was already downloaded - till_time_ms = min(till_time_ms, arrow.utcnow().shift(minutes=-10).timestamp * 1000) - - data: List[Dict[Any, Any]] = [] - while not since_ms or since_ms < till_time_ms: - data_part = self._api.fetch_ohlcv(pair, timeframe=tick_interval, since=since_ms) - - # Because some exchange sort Tickers ASC and other DESC. - # Ex: Bittrex returns a list of tickers ASC (oldest first, newest last) - # when GDAX returns a list of tickers DESC (newest first, oldest last) - data_part = sorted(data_part, key=lambda x: x[0]) - - if not data_part: - break - - logger.debug('Downloaded data for %s time range [%s, %s]', - pair, - arrow.get(data_part[0][0] / 1000).format(), - arrow.get(data_part[-1][0] / 1000).format()) - - data.extend(data_part) - since_ms = data[-1][0] + 1 - - return data - except ccxt.NotSupported as e: - raise OperationalException( - f'Exchange {self._api.name} does not support fetching historical candlestick data.' - f'Message: {e}') - except (ccxt.NetworkError, ccxt.ExchangeError) as e: - raise TemporaryError( - f'Could not load ticker history due to {e.__class__.__name__}. Message: {e}') - except ccxt.BaseError as e: - raise OperationalException(f'Could not fetch ticker data. Msg: {e}') - @retrier def cancel_order(self, order_id: str, pair: str) -> None: if self._conf['dry_run']: diff --git a/freqtrade/tests/exchange/test_exchange.py b/freqtrade/tests/exchange/test_exchange.py index 207f14efe..7fadb4cc1 100644 --- a/freqtrade/tests/exchange/test_exchange.py +++ b/freqtrade/tests/exchange/test_exchange.py @@ -875,64 +875,8 @@ def make_fetch_ohlcv_mock(data): return fetch_ohlcv_mock -def test_get_candle_history(default_conf, mocker): - api_mock = MagicMock() - tick = [ - [ - 1511686200000, # unix timestamp ms - 1, # open - 2, # high - 3, # low - 4, # close - 5, # volume (in quote currency) - ] - ] - type(api_mock).has = PropertyMock(return_value={'fetchOHLCV': True}) - api_mock.fetch_ohlcv = MagicMock(side_effect=make_fetch_ohlcv_mock(tick)) - exchange = get_patched_exchange(mocker, default_conf, api_mock) - - # retrieve original ticker - ticks = exchange.get_candle_history('ETH/BTC', default_conf['ticker_interval']) - assert ticks[0][0] == 1511686200000 - assert ticks[0][1] == 1 - assert ticks[0][2] == 2 - assert ticks[0][3] == 3 - assert ticks[0][4] == 4 - assert ticks[0][5] == 5 - - # change ticker and ensure tick changes - new_tick = [ - [ - 1511686210000, # unix timestamp ms - 6, # open - 7, # high - 8, # low - 9, # close - 10, # volume (in quote currency) - ] - ] - api_mock.fetch_ohlcv = MagicMock(side_effect=make_fetch_ohlcv_mock(new_tick)) - exchange = get_patched_exchange(mocker, default_conf, api_mock) - - ticks = exchange.get_candle_history('ETH/BTC', default_conf['ticker_interval']) - assert ticks[0][0] == 1511686210000 - assert ticks[0][1] == 6 - assert ticks[0][2] == 7 - assert ticks[0][3] == 8 - assert ticks[0][4] == 9 - assert ticks[0][5] == 10 - - ccxt_exceptionhandlers(mocker, default_conf, api_mock, - "get_candle_history", "fetch_ohlcv", - pair='ABCD/BTC', tick_interval=default_conf['ticker_interval']) - - with pytest.raises(OperationalException, match=r'Exchange .* does not support.*'): - api_mock.fetch_ohlcv = MagicMock(side_effect=ccxt.NotSupported) - exchange = get_patched_exchange(mocker, default_conf, api_mock) - exchange.get_candle_history(pair='ABCD/BTC', tick_interval=default_conf['ticker_interval']) - - -def test_get_candle_history_sort(default_conf, mocker): +@pytest.mark.asyncio +async def test___async_get_candle_history_sort(default_conf, mocker): api_mock = MagicMock() # GDAX use-case (real data from GDAX) @@ -949,13 +893,14 @@ def test_get_candle_history_sort(default_conf, mocker): [1527830700000, 0.07652, 0.07652, 0.07651, 0.07652, 10.04822687], [1527830400000, 0.07649, 0.07651, 0.07649, 0.07651, 2.5734867] ] - type(api_mock).has = PropertyMock(return_value={'fetchOHLCV': True}) api_mock.fetch_ohlcv = MagicMock(side_effect=make_fetch_ohlcv_mock(tick)) - - exchange = get_patched_exchange(mocker, default_conf, api_mock) + exchange = get_patched_exchange(mocker, default_conf) + exchange._api_async.fetch_ohlcv = get_mock_coro(tick) # Test the ticker history sort - ticks = exchange.get_candle_history('ETH/BTC', default_conf['ticker_interval']) + res = await exchange._async_get_candle_history('ETH/BTC', default_conf['ticker_interval']) + assert res[0] == 'ETH/BTC' + ticks = res[1] assert ticks[0][0] == 1527830400000 assert ticks[0][1] == 0.07649 assert ticks[0][2] == 0.07651 @@ -984,11 +929,12 @@ def test_get_candle_history_sort(default_conf, mocker): [1527830100000, 0.076695, 0.07671, 0.07624171, 0.07671, 1.80689244], [1527830400000, 0.07671, 0.07674399, 0.07629216, 0.07655213, 2.31452783] ] - type(api_mock).has = PropertyMock(return_value={'fetchOHLCV': True}) - api_mock.fetch_ohlcv = MagicMock(side_effect=make_fetch_ohlcv_mock(tick)) - exchange = get_patched_exchange(mocker, default_conf, api_mock) + exchange._api_async.fetch_ohlcv = get_mock_coro(tick) # Test the ticker history sort - ticks = exchange.get_candle_history('ETH/BTC', default_conf['ticker_interval']) + res = await exchange._async_get_candle_history('ETH/BTC', default_conf['ticker_interval']) + assert res[0] == 'ETH/BTC' + ticks = res[1] + assert ticks[0][0] == 1527827700000 assert ticks[0][1] == 0.07659999 assert ticks[0][2] == 0.0766 From ebaf58b0fe465be049d46ef891c57746946ed4a0 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 25 Nov 2018 15:00:50 +0100 Subject: [PATCH 466/699] Only sort data if necessary --- freqtrade/exchange/__init__.py | 4 +++- freqtrade/tests/exchange/test_exchange.py | 13 +++++++++---- 2 files changed, 12 insertions(+), 5 deletions(-) diff --git a/freqtrade/exchange/__init__.py b/freqtrade/exchange/__init__.py index 0f8d8895c..350c730a4 100644 --- a/freqtrade/exchange/__init__.py +++ b/freqtrade/exchange/__init__.py @@ -478,7 +478,9 @@ class Exchange(object): # Because some exchange sort Tickers ASC and other DESC. # Ex: Bittrex returns a list of tickers ASC (oldest first, newest last) # when GDAX returns a list of tickers DESC (newest first, oldest last) - data = sorted(data, key=lambda x: x[0]) + # Only sort if necessary to save computing time + if data and data[0][0] > data[-1][0]: + data = sorted(data, key=lambda x: x[0]) # keeping last candle time as last refreshed time of the pair if data: diff --git a/freqtrade/tests/exchange/test_exchange.py b/freqtrade/tests/exchange/test_exchange.py index 7fadb4cc1..dbb8d4ec2 100644 --- a/freqtrade/tests/exchange/test_exchange.py +++ b/freqtrade/tests/exchange/test_exchange.py @@ -877,7 +877,8 @@ def make_fetch_ohlcv_mock(data): @pytest.mark.asyncio async def test___async_get_candle_history_sort(default_conf, mocker): - api_mock = MagicMock() + def sort_data(data, key): + return sorted(data, key=key) # GDAX use-case (real data from GDAX) # This ticker history is ordered DESC (newest first, oldest last) @@ -893,14 +894,15 @@ async def test___async_get_candle_history_sort(default_conf, mocker): [1527830700000, 0.07652, 0.07652, 0.07651, 0.07652, 10.04822687], [1527830400000, 0.07649, 0.07651, 0.07649, 0.07651, 2.5734867] ] - api_mock.fetch_ohlcv = MagicMock(side_effect=make_fetch_ohlcv_mock(tick)) exchange = get_patched_exchange(mocker, default_conf) exchange._api_async.fetch_ohlcv = get_mock_coro(tick) - + sort_mock = mocker.patch('freqtrade.exchange.sorted', MagicMock(side_effect=sort_data)) # Test the ticker history sort res = await exchange._async_get_candle_history('ETH/BTC', default_conf['ticker_interval']) assert res[0] == 'ETH/BTC' ticks = res[1] + + assert sort_mock.call_count == 1 assert ticks[0][0] == 1527830400000 assert ticks[0][1] == 0.07649 assert ticks[0][2] == 0.07651 @@ -930,11 +932,14 @@ async def test___async_get_candle_history_sort(default_conf, mocker): [1527830400000, 0.07671, 0.07674399, 0.07629216, 0.07655213, 2.31452783] ] exchange._api_async.fetch_ohlcv = get_mock_coro(tick) + # Reset sort mock + sort_mock = mocker.patch('freqtrade.exchange.sorted', MagicMock(side_effect=sort_data)) # Test the ticker history sort res = await exchange._async_get_candle_history('ETH/BTC', default_conf['ticker_interval']) assert res[0] == 'ETH/BTC' ticks = res[1] - + # Sorted not called again - data is already in order + assert sort_mock.call_count == 0 assert ticks[0][0] == 1527827700000 assert ticks[0][1] == 0.07659999 assert ticks[0][2] == 0.0766 From 3e29fbb17a49544c77b06e46776f53dcd6f674bc Mon Sep 17 00:00:00 2001 From: misagh Date: Sun, 25 Nov 2018 17:22:56 +0100 Subject: [PATCH 467/699] stoploss on exchange added as a parameter to order_types --- freqtrade/constants.py | 3 ++- freqtrade/exchange/__init__.py | 6 ++++++ freqtrade/freqtradebot.py | 15 ++------------- freqtrade/strategy/default_strategy.py | 3 ++- freqtrade/strategy/interface.py | 10 +++------- freqtrade/tests/exchange/test_exchange.py | 20 +++++++++++++++++++- freqtrade/tests/test_freqtradebot.py | 12 +++--------- 7 files changed, 37 insertions(+), 32 deletions(-) diff --git a/freqtrade/constants.py b/freqtrade/constants.py index 055fee3b2..481a219d6 100644 --- a/freqtrade/constants.py +++ b/freqtrade/constants.py @@ -109,7 +109,8 @@ CONF_SCHEMA = { 'properties': { 'buy': {'type': 'string', 'enum': ORDERTYPE_POSSIBILITIES}, 'sell': {'type': 'string', 'enum': ORDERTYPE_POSSIBILITIES}, - 'stoploss': {'type': 'string', 'enum': ORDERTYPE_POSSIBILITIES} + 'stoploss': {'type': 'string', 'enum': ORDERTYPE_POSSIBILITIES}, + 'stoploss_on_exchange': {'type': 'boolean'} }, 'required': ['buy', 'sell', 'stoploss'] }, diff --git a/freqtrade/exchange/__init__.py b/freqtrade/exchange/__init__.py index 3ccd2369a..6e826794a 100644 --- a/freqtrade/exchange/__init__.py +++ b/freqtrade/exchange/__init__.py @@ -227,6 +227,12 @@ class Exchange(object): raise OperationalException( f'Exchange {self.name} does not support market orders.') + if order_types.get('stoploss_on_exchange', False): + if self.name is not 'Binance': + raise OperationalException( + 'On exchange stoploss is not supported for %s.' % self.name + ) + def exchange_has(self, endpoint: str) -> bool: """ Checks if exchange implements a specific API endpoint. diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index 005f698dd..d9a15f56a 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -54,7 +54,6 @@ class FreqtradeBot(object): # Init objects self.config = config self.strategy: IStrategy = StrategyResolver(self.config).strategy - self.check_strategy_config_consistency(config, self.strategy) self.rpc: RPCManager = RPCManager(self) self.persistence = None @@ -68,16 +67,6 @@ class FreqtradeBot(object): self.active_pair_whitelist: List[str] = self.config['exchange']['pair_whitelist'] self._init_modules() - def check_strategy_config_consistency(self, config, strategy: IStrategy) -> None: - """ - checks if config is compatible with the given strategy - """ - # Stoploss on exchange is only implemented for binance - if strategy.stoploss_on_exchange and config.get('exchange') is not 'binance': - raise OperationalException( - 'On exchange stoploss is not supported for %s.' % config['exchange']['name'] - ) - def _init_modules(self) -> None: """ Initializes all modules and updates the config @@ -567,7 +556,7 @@ class FreqtradeBot(object): trade.update(order) - if self.strategy.stoploss_on_exchange: + if self.strategy.order_types.get('stoploss_on_exchange'): result = self.handle_stoploss_on_exchange(trade) if result: self.wallets.update() @@ -844,7 +833,7 @@ class FreqtradeBot(object): sell_type = 'stoploss' # First cancelling stoploss on exchange ... - if self.strategy.stoploss_on_exchange and trade.stoploss_order_id: + if self.strategy.order_types.get('stoploss_on_exchange') and trade.stoploss_order_id: self.exchange.cancel_order(trade.stoploss_order_id, trade.pair) # Execute sell and update trade record diff --git a/freqtrade/strategy/default_strategy.py b/freqtrade/strategy/default_strategy.py index b282a5938..9c850a8be 100644 --- a/freqtrade/strategy/default_strategy.py +++ b/freqtrade/strategy/default_strategy.py @@ -32,7 +32,8 @@ class DefaultStrategy(IStrategy): order_types = { 'buy': 'limit', 'sell': 'limit', - 'stoploss': 'limit' + 'stoploss': 'limit', + 'stoploss_on_exchange': False } def populate_indicators(self, dataframe: DataFrame, metadata: dict) -> DataFrame: diff --git a/freqtrade/strategy/interface.py b/freqtrade/strategy/interface.py index d1e22850c..1073f8028 100644 --- a/freqtrade/strategy/interface.py +++ b/freqtrade/strategy/interface.py @@ -68,11 +68,6 @@ class IStrategy(ABC): # associated stoploss stoploss: float - # if the stoploss should be on exchange. - # if this is True then a stoploss order will be placed - # immediately after a successful buy order. - stoploss_on_exchange: bool = False - # associated ticker interval ticker_interval: str @@ -80,7 +75,8 @@ class IStrategy(ABC): order_types: Dict = { 'buy': 'limit', 'sell': 'limit', - 'stoploss': 'limit' + 'stoploss': 'limit', + 'stoploss_on_exchange': False } # run "populate_indicators" only for new candle @@ -228,7 +224,7 @@ class IStrategy(ABC): current_rate = low or rate current_profit = trade.calc_profit_percent(current_rate) - if self.stoploss_on_exchange: + if self.order_types.get('stoploss_on_exchange'): stoplossflag = SellCheckTuple(sell_flag=False, sell_type=SellType.NONE) else: stoplossflag = self.stop_loss_reached(current_rate=current_rate, trade=trade, diff --git a/freqtrade/tests/exchange/test_exchange.py b/freqtrade/tests/exchange/test_exchange.py index 57be54262..1a46ff001 100644 --- a/freqtrade/tests/exchange/test_exchange.py +++ b/freqtrade/tests/exchange/test_exchange.py @@ -362,7 +362,14 @@ def test_validate_order_types(default_conf, mocker): mocker.patch('freqtrade.exchange.Exchange._init_ccxt', MagicMock(return_value=api_mock)) mocker.patch('freqtrade.exchange.Exchange._load_markets', MagicMock(return_value={})) mocker.patch('freqtrade.exchange.Exchange.validate_timeframes', MagicMock()) - default_conf['order_types'] = {'buy': 'limit', 'sell': 'limit', 'stoploss': 'market'} + mocker.patch('freqtrade.exchange.Exchange.name', 'Bittrex') + default_conf['order_types'] = { + 'buy': 'limit', + 'sell': 'limit', + 'stoploss': 'market', + 'stoploss_on_exchange': False + } + Exchange(default_conf) type(api_mock).has = PropertyMock(return_value={'createMarketOrder': False}) @@ -374,6 +381,17 @@ def test_validate_order_types(default_conf, mocker): match=r'Exchange .* does not support market orders.'): Exchange(default_conf) + default_conf['order_types'] = { + 'buy': 'limit', + 'sell': 'limit', + 'stoploss': 'limit', + 'stoploss_on_exchange': True + } + + with pytest.raises(OperationalException, + match=r'On exchange stoploss is not supported for .*'): + Exchange(default_conf) + def test_validate_order_types_not_in_config(default_conf, mocker): api_mock = MagicMock() diff --git a/freqtrade/tests/test_freqtradebot.py b/freqtrade/tests/test_freqtradebot.py index 5acf4fdcb..b6b42d1da 100644 --- a/freqtrade/tests/test_freqtradebot.py +++ b/freqtrade/tests/test_freqtradebot.py @@ -893,7 +893,7 @@ def test_add_stoploss_on_exchange(mocker, default_conf, limit_buy_order) -> None mocker.patch('freqtrade.exchange.Exchange.stoploss_limit', stoploss_limit) freqtrade = FreqtradeBot(default_conf) - freqtrade.strategy.stoploss_on_exchange = True + freqtrade.strategy.order_types['stoploss_on_exchange'] = True trade = MagicMock() trade.open_order_id = None @@ -1595,7 +1595,7 @@ def test_execute_sell_with_stoploss_on_exchange(default_conf, mocker.patch('freqtrade.exchange.Exchange.cancel_order', cancel_order) freqtrade = FreqtradeBot(default_conf) - freqtrade.strategy.stoploss_on_exchange = True + freqtrade.strategy.order_types['stoploss_on_exchange'] = True patch_get_signal(freqtrade) # Create some test data @@ -1647,7 +1647,7 @@ def test_may_execute_sell_after_stoploss_on_exchange_hit(default_conf, mocker.patch('freqtrade.exchange.Exchange.stoploss_limit', stoploss_limit) freqtrade = FreqtradeBot(default_conf) - freqtrade.strategy.stoploss_on_exchange = True + freqtrade.strategy.order_types['stoploss_on_exchange'] = True patch_get_signal(freqtrade) # Create some test data @@ -2516,9 +2516,3 @@ def test_startup_messages(default_conf, mocker): default_conf['dynamic_whitelist'] = 20 freqtrade = get_patched_freqtradebot(mocker, default_conf) assert freqtrade.state is State.RUNNING - - -def test_check_consistency(default_conf, mocker, caplog): - mocker.patch('freqtrade.freqtradebot.IStrategy.stoploss_on_exchange', True) - with pytest.raises(OperationalException): - FreqtradeBot(default_conf) From 5e1fb11124cf67df36e29876698450714132a51a Mon Sep 17 00:00:00 2001 From: misagh Date: Sun, 25 Nov 2018 17:30:06 +0100 Subject: [PATCH 468/699] documentation added for stop loss on exchange --- config_full.json.example | 3 ++- docs/configuration.md | 9 +++++---- freqtrade/constants.py | 2 +- freqtrade/exchange/__init__.py | 2 +- 4 files changed, 9 insertions(+), 7 deletions(-) diff --git a/config_full.json.example b/config_full.json.example index b0719bcc6..6134e9cad 100644 --- a/config_full.json.example +++ b/config_full.json.example @@ -36,7 +36,8 @@ "order_types": { "buy": "limit", "sell": "limit", - "stoploss": "market" + "stoploss": "market", + "stoploss_on_exchange": "false" }, "exchange": { "name": "bittrex", diff --git a/docs/configuration.md b/docs/configuration.md index 62559a41e..03059e261 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -39,7 +39,7 @@ The table below will list all configuration parameters. | `ask_strategy.use_order_book` | false | No | Allows selling of open traded pair using the rates in Order Book Asks. | `ask_strategy.order_book_min` | 0 | No | Bot will scan from the top min to max Order Book Asks searching for a profitable rate. | `ask_strategy.order_book_max` | 0 | No | Bot will scan from the top min to max Order Book Asks searching for a profitable rate. -| `order_types` | None | No | Configure order-types depending on the action (`"buy"`, `"sell"`, `"stoploss"`). +| `order_types` | None | No | Configure order-types depending on the action (`"buy"`, `"sell"`, `"stoploss"`, `"stoploss_on_exchange"`). | `exchange.name` | bittrex | Yes | Name of the exchange class to use. [List below](#user-content-what-values-for-exchangename). | `exchange.key` | key | No | API key to use for the exchange. Only required when you are in production mode. | `exchange.secret` | secret | No | API secret to use for the exchange. Only required when you are in production mode. @@ -141,17 +141,18 @@ end up paying more then would probably have been necessary. ### Understand order_types -`order_types` contains a dict mapping order-types to market-types. This allows to buy using limit orders, sell using limit-orders, and create stoploss orders using market. +`order_types` contains a dict mapping order-types to market-types as well as stoploss on or off exchange type. This allows to buy using limit orders, sell using limit-orders, and create stoploss orders using market. It also allows to set the stoploss "on exchange" which means stoploss order would be placed immediately once the buy order is fulfilled. This can be set in the configuration or in the strategy. Configuration overwrites strategy configurations. -If this is configured, all 3 values (`"buy"`, `"sell"` and `"stoploss"`) need to be present, otherwise the bot warn about it and will fail to start. +If this is configured, all 4 values (`"buy"`, `"sell"`, `"stoploss"`, `"stoploss_on_exchange"`) need to be present, otherwise the bot warn about it and will fail to start. The below is the default which is used if this is not configured in either Strategy or configuration. ``` json "order_types": { "buy": "limit", "sell": "limit", - "stoploss": "market" + "stoploss": "market", + "stoploss_on_exchange": "false" }, ``` diff --git a/freqtrade/constants.py b/freqtrade/constants.py index 481a219d6..86067d395 100644 --- a/freqtrade/constants.py +++ b/freqtrade/constants.py @@ -13,7 +13,7 @@ DEFAULT_HYPEROPT = 'DefaultHyperOpts' DEFAULT_DB_PROD_URL = 'sqlite:///tradesv3.sqlite' DEFAULT_DB_DRYRUN_URL = 'sqlite://' UNLIMITED_STAKE_AMOUNT = 'unlimited' -REQUIRED_ORDERTYPES = ['buy', 'sell', 'stoploss'] +REQUIRED_ORDERTYPES = ['buy', 'sell', 'stoploss', 'stoploss_on_exchange'] ORDERTYPE_POSSIBILITIES = ['limit', 'market'] diff --git a/freqtrade/exchange/__init__.py b/freqtrade/exchange/__init__.py index 6e826794a..a311fd666 100644 --- a/freqtrade/exchange/__init__.py +++ b/freqtrade/exchange/__init__.py @@ -227,7 +227,7 @@ class Exchange(object): raise OperationalException( f'Exchange {self.name} does not support market orders.') - if order_types.get('stoploss_on_exchange', False): + if order_types.get('stoploss_on_exchange'): if self.name is not 'Binance': raise OperationalException( 'On exchange stoploss is not supported for %s.' % self.name From 92930b2343494c7c2aa99b6511fa91466f03fccd Mon Sep 17 00:00:00 2001 From: misagh Date: Sun, 25 Nov 2018 19:03:28 +0100 Subject: [PATCH 469/699] test fixed --- freqtrade/constants.py | 2 +- freqtrade/tests/exchange/test_exchange.py | 2 +- freqtrade/tests/strategy/test_strategy.py | 13 +++++++------ user_data/strategies/test_strategy.py | 5 +++-- 4 files changed, 12 insertions(+), 10 deletions(-) diff --git a/freqtrade/constants.py b/freqtrade/constants.py index 86067d395..f8fb91240 100644 --- a/freqtrade/constants.py +++ b/freqtrade/constants.py @@ -112,7 +112,7 @@ CONF_SCHEMA = { 'stoploss': {'type': 'string', 'enum': ORDERTYPE_POSSIBILITIES}, 'stoploss_on_exchange': {'type': 'boolean'} }, - 'required': ['buy', 'sell', 'stoploss'] + 'required': ['buy', 'sell', 'stoploss', 'stoploss_on_exchange'] }, 'exchange': {'$ref': '#/definitions/exchange'}, 'edge': {'$ref': '#/definitions/edge'}, diff --git a/freqtrade/tests/exchange/test_exchange.py b/freqtrade/tests/exchange/test_exchange.py index 1a46ff001..6ad84585c 100644 --- a/freqtrade/tests/exchange/test_exchange.py +++ b/freqtrade/tests/exchange/test_exchange.py @@ -375,7 +375,7 @@ def test_validate_order_types(default_conf, mocker): type(api_mock).has = PropertyMock(return_value={'createMarketOrder': False}) mocker.patch('freqtrade.exchange.Exchange._init_ccxt', MagicMock(return_value=api_mock)) - default_conf['order_types'] = {'buy': 'limit', 'sell': 'limit', 'stoploss': 'market'} + default_conf['order_types'] = {'buy': 'limit', 'sell': 'limit', 'stoploss': 'market', 'stoploss_on_exchange': 'false'} with pytest.raises(OperationalException, match=r'Exchange .* does not support market orders.'): diff --git a/freqtrade/tests/strategy/test_strategy.py b/freqtrade/tests/strategy/test_strategy.py index a38050f24..66d988075 100644 --- a/freqtrade/tests/strategy/test_strategy.py +++ b/freqtrade/tests/strategy/test_strategy.py @@ -188,7 +188,8 @@ def test_strategy_override_order_types(caplog): order_types = { 'buy': 'market', 'sell': 'limit', - 'stoploss': 'limit' + 'stoploss': 'limit', + 'stoploss_on_exchange': True, } config = { @@ -198,13 +199,13 @@ def test_strategy_override_order_types(caplog): resolver = StrategyResolver(config) assert resolver.strategy.order_types - for method in ['buy', 'sell', 'stoploss']: + for method in ['buy', 'sell', 'stoploss', 'stoploss_on_exchange']: assert resolver.strategy.order_types[method] == order_types[method] assert ('freqtrade.strategy.resolver', logging.INFO, "Override strategy 'order_types' with value in config file:" - " {'buy': 'market', 'sell': 'limit', 'stoploss': 'limit'}." + " {'buy': 'market', 'sell': 'limit', 'stoploss': 'limit', 'stoploss_on_exchange': True}." ) in caplog.record_tuples config = { @@ -262,13 +263,13 @@ def test_call_deprecated_function(result, monkeypatch): assert resolver.strategy._sell_fun_len == 2 indicator_df = resolver.strategy.advise_indicators(result, metadata=metadata) - assert type(indicator_df) is DataFrame + assert isinstance(indicator_df, DataFrame) assert 'adx' in indicator_df.columns buydf = resolver.strategy.advise_buy(result, metadata=metadata) - assert type(buydf) is DataFrame + assert isinstance(buydf, DataFrame) assert 'buy' in buydf.columns selldf = resolver.strategy.advise_sell(result, metadata=metadata) - assert type(selldf) is DataFrame + assert isinstance(selldf, DataFrame) assert 'sell' in selldf diff --git a/user_data/strategies/test_strategy.py b/user_data/strategies/test_strategy.py index fd2e9ab75..e7804e683 100644 --- a/user_data/strategies/test_strategy.py +++ b/user_data/strategies/test_strategy.py @@ -7,7 +7,7 @@ from pandas import DataFrame # Add your lib to import here import talib.abstract as ta import freqtrade.vendor.qtpylib.indicators as qtpylib -import numpy # noqa +import numpy # noqa # This class is a sample. Feel free to customize it. @@ -52,7 +52,8 @@ class TestStrategy(IStrategy): order_types = { 'buy': 'limit', 'sell': 'limit', - 'stoploss': 'market' + 'stoploss': 'market', + 'stoploss_on_exchange': False } def populate_indicators(self, dataframe: DataFrame, metadata: dict) -> DataFrame: From a80c984323ef8dec3aa5468ef572eca0ae966260 Mon Sep 17 00:00:00 2001 From: misagh Date: Sun, 25 Nov 2018 19:09:11 +0100 Subject: [PATCH 470/699] flake8 --- freqtrade/tests/exchange/test_exchange.py | 7 ++++++- freqtrade/tests/strategy/test_strategy.py | 3 ++- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/freqtrade/tests/exchange/test_exchange.py b/freqtrade/tests/exchange/test_exchange.py index 6ad84585c..d7f70d477 100644 --- a/freqtrade/tests/exchange/test_exchange.py +++ b/freqtrade/tests/exchange/test_exchange.py @@ -375,7 +375,12 @@ def test_validate_order_types(default_conf, mocker): type(api_mock).has = PropertyMock(return_value={'createMarketOrder': False}) mocker.patch('freqtrade.exchange.Exchange._init_ccxt', MagicMock(return_value=api_mock)) - default_conf['order_types'] = {'buy': 'limit', 'sell': 'limit', 'stoploss': 'market', 'stoploss_on_exchange': 'false'} + default_conf['order_types'] = { + 'buy': 'limit', + 'sell': 'limit', + 'stoploss': 'market', + 'stoploss_on_exchange': 'false' + } with pytest.raises(OperationalException, match=r'Exchange .* does not support market orders.'): diff --git a/freqtrade/tests/strategy/test_strategy.py b/freqtrade/tests/strategy/test_strategy.py index 66d988075..1ad774ffa 100644 --- a/freqtrade/tests/strategy/test_strategy.py +++ b/freqtrade/tests/strategy/test_strategy.py @@ -205,7 +205,8 @@ def test_strategy_override_order_types(caplog): assert ('freqtrade.strategy.resolver', logging.INFO, "Override strategy 'order_types' with value in config file:" - " {'buy': 'market', 'sell': 'limit', 'stoploss': 'limit', 'stoploss_on_exchange': True}." + " {'buy': 'market', 'sell': 'limit', 'stoploss': 'limit'," + " 'stoploss_on_exchange': True}." ) in caplog.record_tuples config = { From 1ad5ccdfb0462e451da44089076b6c8b82739440 Mon Sep 17 00:00:00 2001 From: misagh Date: Sun, 25 Nov 2018 19:48:46 +0100 Subject: [PATCH 471/699] dry run condition when sell occurs --- freqtrade/freqtradebot.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index 8a2db84a9..343036d19 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -781,6 +781,10 @@ class FreqtradeBot(object): sell_type = 'sell' if sell_reason in (SellType.STOP_LOSS, SellType.TRAILING_STOP_LOSS): sell_type = 'stoploss' + + if self.config.get('dry_run', False) and sell_type == 'stoploss': + limit = trade.stop_loss + # Execute sell and update trade record order_id = self.exchange.sell(pair=str(trade.pair), ordertype=self.strategy.order_types[sell_type], From 5c257730a8b104e3d9a0d9b7c92d6ecf36337f3c Mon Sep 17 00:00:00 2001 From: misagh Date: Sun, 25 Nov 2018 20:16:53 +0100 Subject: [PATCH 472/699] test added for dry run stop loss sell --- freqtrade/tests/test_freqtradebot.py | 53 +++++++++++++++++++++++++++- 1 file changed, 52 insertions(+), 1 deletion(-) diff --git a/freqtrade/tests/test_freqtradebot.py b/freqtrade/tests/test_freqtradebot.py index a36ae2cd8..547a048bf 100644 --- a/freqtrade/tests/test_freqtradebot.py +++ b/freqtrade/tests/test_freqtradebot.py @@ -1422,7 +1422,7 @@ def test_execute_sell_up(default_conf, ticker, fee, ticker_sell_up, markets, moc } == last_msg -def test_execute_sell_down(default_conf, ticker, fee, ticker_sell_down, markets, mocker) -> None: +def test_execute_sell_down_live(default_conf, ticker, fee, ticker_sell_down, markets, mocker) -> None: rpc_mock = patch_RPCManager(mocker) mocker.patch.multiple( 'freqtrade.exchange.Exchange', @@ -1468,6 +1468,57 @@ def test_execute_sell_down(default_conf, ticker, fee, ticker_sell_down, markets, } == last_msg +def test_execute_sell_down_dry_run(default_conf, ticker, fee, ticker_sell_down, markets, mocker) -> None: + rpc_mock = patch_RPCManager(mocker) + mocker.patch.multiple( + 'freqtrade.exchange.Exchange', + _load_markets=MagicMock(return_value={}), + get_ticker=ticker, + get_fee=fee, + get_markets=markets + ) + freqtrade = FreqtradeBot(default_conf) + patch_get_signal(freqtrade) + + # Create some test data + freqtrade.create_trade() + + trade = Trade.query.first() + assert trade + + # Decrease the price and sell it + mocker.patch.multiple( + 'freqtrade.exchange.Exchange', + get_ticker=ticker_sell_down + ) + + default_conf['dry_run'] = True + + # Setting trade stoploss to 0.01 + trade.stop_loss = 0.00001099 * 0.99 + + freqtrade.execute_sell(trade=trade, limit=ticker_sell_down()['bid'], + sell_reason=SellType.STOP_LOSS) + + assert rpc_mock.call_count == 2 + last_msg = rpc_mock.call_args_list[-1][0][0] + assert { + 'type': RPCMessageType.SELL_NOTIFICATION, + 'exchange': 'Bittrex', + 'pair': 'ETH/BTC', + 'gain': 'loss', + 'market_url': 'https://bittrex.com/Market/Index?MarketName=BTC-ETH', + 'limit': 1.08801e-05, + 'amount': 90.99181073703367, + 'open_rate': 1.099e-05, + 'current_rate': 1.044e-05, + 'profit_amount': -1.498e-05, + 'profit_percent': -0.01493766, + 'stake_currency': 'BTC', + 'fiat_currency': 'USD', + } == last_msg + + def test_execute_sell_without_conf_sell_up(default_conf, ticker, fee, ticker_sell_up, markets, mocker) -> None: rpc_mock = patch_RPCManager(mocker) From b57976861852fe75f45e479ed604b2acf184c8eb Mon Sep 17 00:00:00 2001 From: misagh Date: Sun, 25 Nov 2018 20:20:11 +0100 Subject: [PATCH 473/699] dry run set explicitly to False for live stop loss --- freqtrade/tests/test_freqtradebot.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/tests/test_freqtradebot.py b/freqtrade/tests/test_freqtradebot.py index 547a048bf..a418e3e45 100644 --- a/freqtrade/tests/test_freqtradebot.py +++ b/freqtrade/tests/test_freqtradebot.py @@ -1445,7 +1445,7 @@ def test_execute_sell_down_live(default_conf, ticker, fee, ticker_sell_down, mar 'freqtrade.exchange.Exchange', get_ticker=ticker_sell_down ) - + default_conf['dry_run'] = False freqtrade.execute_sell(trade=trade, limit=ticker_sell_down()['bid'], sell_reason=SellType.STOP_LOSS) From 6c38bde24aca4d0b293b52a00994515aa64b5eeb Mon Sep 17 00:00:00 2001 From: misagh Date: Sun, 25 Nov 2018 20:21:50 +0100 Subject: [PATCH 474/699] some formatting fixed --- freqtrade/tests/test_freqtradebot.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/freqtrade/tests/test_freqtradebot.py b/freqtrade/tests/test_freqtradebot.py index a418e3e45..6dd709dbf 100644 --- a/freqtrade/tests/test_freqtradebot.py +++ b/freqtrade/tests/test_freqtradebot.py @@ -1422,7 +1422,8 @@ def test_execute_sell_up(default_conf, ticker, fee, ticker_sell_up, markets, moc } == last_msg -def test_execute_sell_down_live(default_conf, ticker, fee, ticker_sell_down, markets, mocker) -> None: +def test_execute_sell_down_live(default_conf, ticker, fee, + ticker_sell_down, markets, mocker) -> None: rpc_mock = patch_RPCManager(mocker) mocker.patch.multiple( 'freqtrade.exchange.Exchange', @@ -1468,7 +1469,8 @@ def test_execute_sell_down_live(default_conf, ticker, fee, ticker_sell_down, mar } == last_msg -def test_execute_sell_down_dry_run(default_conf, ticker, fee, ticker_sell_down, markets, mocker) -> None: +def test_execute_sell_down_dry_run(default_conf, ticker, fee, + ticker_sell_down, markets, mocker) -> None: rpc_mock = patch_RPCManager(mocker) mocker.patch.multiple( 'freqtrade.exchange.Exchange', From fb7b65c9094dae1a004601d45abc6220d0f11dc9 Mon Sep 17 00:00:00 2001 From: misagh Date: Sun, 25 Nov 2018 20:44:40 +0100 Subject: [PATCH 475/699] time in force drafted time in force drafted --- freqtrade/strategy/default_strategy.py | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/freqtrade/strategy/default_strategy.py b/freqtrade/strategy/default_strategy.py index b282a5938..024b531f4 100644 --- a/freqtrade/strategy/default_strategy.py +++ b/freqtrade/strategy/default_strategy.py @@ -16,10 +16,10 @@ class DefaultStrategy(IStrategy): # Minimal ROI designed for the strategy minimal_roi = { - "40": 0.0, - "30": 0.01, - "20": 0.02, - "0": 0.04 + "40": 0.0, + "30": 0.01, + "20": 0.02, + "0": 0.04 } # Optimal stoploss designed for the strategy @@ -35,6 +35,12 @@ class DefaultStrategy(IStrategy): 'stoploss': 'limit' } + # Optional time in force for orders + order_time_in_force = { + 'buy': 'gtc', + 'sell': 'gtc', + } + def populate_indicators(self, dataframe: DataFrame, metadata: dict) -> DataFrame: """ Adds several different TA indicators to the given DataFrame From 890cef88ab0e9b07ab37550fdc5a0b0257bcab3a Mon Sep 17 00:00:00 2001 From: misagh Date: Sun, 25 Nov 2018 21:02:58 +0100 Subject: [PATCH 476/699] oops, lost in git :/ --- freqtrade/freqtradebot.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index 602c4ae2f..7a974d385 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -779,10 +779,6 @@ class FreqtradeBot(object): sell_type = 'sell' if sell_reason in (SellType.STOP_LOSS, SellType.TRAILING_STOP_LOSS): sell_type = 'stoploss' - - if self.config.get('dry_run', False) and sell_type == 'stoploss': - limit = trade.stop_loss - # Execute sell and update trade record order_id = self.exchange.sell(pair=str(trade.pair), ordertype=self.strategy.order_types[sell_type], From ba20b1b5c7b259587541a890052219e184df5ec8 Mon Sep 17 00:00:00 2001 From: misagh Date: Sun, 25 Nov 2018 21:05:25 +0100 Subject: [PATCH 477/699] TIF added to constants and json full --- config_full.json.example | 4 ++++ freqtrade/constants.py | 9 +++++++++ 2 files changed, 13 insertions(+) diff --git a/config_full.json.example b/config_full.json.example index b0719bcc6..3e7f4bc3b 100644 --- a/config_full.json.example +++ b/config_full.json.example @@ -38,6 +38,10 @@ "sell": "limit", "stoploss": "market" }, + "order_time_in_force": { + "buy": "gtc", + "sell": "gtc", + }, "exchange": { "name": "bittrex", "key": "your_exchange_key", diff --git a/freqtrade/constants.py b/freqtrade/constants.py index 055fee3b2..c6b8519c8 100644 --- a/freqtrade/constants.py +++ b/freqtrade/constants.py @@ -15,6 +15,7 @@ DEFAULT_DB_DRYRUN_URL = 'sqlite://' UNLIMITED_STAKE_AMOUNT = 'unlimited' REQUIRED_ORDERTYPES = ['buy', 'sell', 'stoploss'] ORDERTYPE_POSSIBILITIES = ['limit', 'market'] +ORDERTIF_POSSIBILITIES = ['gtc', 'aon', 'fok', 'ioc'] TICKER_INTERVAL_MINUTES = { @@ -113,6 +114,14 @@ CONF_SCHEMA = { }, 'required': ['buy', 'sell', 'stoploss'] }, + 'order_time_in_force': { + 'type': 'object', + 'properties': { + 'buy': {'type': 'string', 'enum': ORDERTIF_POSSIBILITIES}, + 'sell': {'type': 'string', 'enum': ORDERTIF_POSSIBILITIES} + }, + 'required': ['buy', 'sell'] + }, 'exchange': {'$ref': '#/definitions/exchange'}, 'edge': {'$ref': '#/definitions/edge'}, 'experimental': { From 181424e8ea1d044d8696837291778472319cf8a9 Mon Sep 17 00:00:00 2001 From: misagh Date: Sun, 25 Nov 2018 21:09:35 +0100 Subject: [PATCH 478/699] time in force validator added --- freqtrade/exchange/__init__.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/freqtrade/exchange/__init__.py b/freqtrade/exchange/__init__.py index ae07e36e9..05d6e9c1a 100644 --- a/freqtrade/exchange/__init__.py +++ b/freqtrade/exchange/__init__.py @@ -103,6 +103,7 @@ class Exchange(object): # Check if all pairs are available self.validate_pairs(config['exchange']['pair_whitelist']) self.validate_ordertypes(config.get('order_types', {})) + self.validate_order_time_in_force(config.get('order_time_in_force', {})) if config.get('ticker_interval'): # Check if timeframe is available self.validate_timeframes(config['ticker_interval']) @@ -227,6 +228,15 @@ class Exchange(object): raise OperationalException( f'Exchange {self.name} does not support market orders.') + def validate_order_time_in_force(self, order_time_in_force: Dict) -> None: + """ + Checks if order time in force configured in strategy/config are supported + """ + if any(v != 'gtc' for k, v in order_time_in_force.items()): + if not self.name == 'Binance': + raise OperationalException( + f'Time in force policies are not supporetd for {self.name} yet.') + def exchange_has(self, endpoint: str) -> bool: """ Checks if exchange implements a specific API endpoint. From 29c23e3136db159cfb00ac96b27c918e5400802b Mon Sep 17 00:00:00 2001 From: misagh Date: Sun, 25 Nov 2018 21:38:11 +0100 Subject: [PATCH 479/699] added time in force in buy and sell functions --- freqtrade/exchange/__init__.py | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/freqtrade/exchange/__init__.py b/freqtrade/exchange/__init__.py index 05d6e9c1a..f7801fd02 100644 --- a/freqtrade/exchange/__init__.py +++ b/freqtrade/exchange/__init__.py @@ -268,7 +268,7 @@ class Exchange(object): price = ceil(big_price) / pow(10, symbol_prec) return price - def buy(self, pair: str, ordertype: str, amount: float, rate: float) -> Dict: + def buy(self, pair: str, ordertype: str, amount: float, rate: float, time_in_force='gtc') -> Dict: if self._conf['dry_run']: order_id = f'dry_run_buy_{randint(0, 10**6)}' self._dry_run_open_orders[order_id] = { @@ -289,7 +289,12 @@ class Exchange(object): amount = self.symbol_amount_prec(pair, amount) rate = self.symbol_price_prec(pair, rate) if ordertype != 'market' else None - return self._api.create_order(pair, ordertype, 'buy', amount, rate) + if time_in_force == 'gtc': + return self._api.create_order(pair, ordertype, 'buy', amount, rate) + else: + return self._api.create_order(pair, ordertype, 'buy', + amount, rate, {'timeInForce': time_in_force}) + except ccxt.InsufficientFunds as e: raise DependencyException( f'Insufficient funds to create limit buy order on market {pair}.' @@ -306,7 +311,8 @@ class Exchange(object): except ccxt.BaseError as e: raise OperationalException(e) - def sell(self, pair: str, ordertype: str, amount: float, rate: float) -> Dict: + def sell(self, pair: str, ordertype: str, amount: float, + rate: float, time_in_force='gtc') -> Dict: if self._conf['dry_run']: order_id = f'dry_run_sell_{randint(0, 10**6)}' self._dry_run_open_orders[order_id] = { @@ -326,7 +332,12 @@ class Exchange(object): amount = self.symbol_amount_prec(pair, amount) rate = self.symbol_price_prec(pair, rate) if ordertype != 'market' else None - return self._api.create_order(pair, ordertype, 'sell', amount, rate) + if time_in_force == 'gtc': + return self._api.create_order(pair, ordertype, 'sell', amount, rate) + else: + return self._api.create_order(pair, ordertype, 'sell', + amount, rate, {'timeInForce': time_in_force}) + except ccxt.InsufficientFunds as e: raise DependencyException( f'Insufficient funds to create limit sell order on market {pair}.' From 962b02b0797895f149b5f3e56d1439b45cbba63f Mon Sep 17 00:00:00 2001 From: misagh Date: Sun, 25 Nov 2018 22:02:59 +0100 Subject: [PATCH 480/699] one last step before tests --- freqtrade/constants.py | 1 + freqtrade/freqtradebot.py | 8 ++++++-- freqtrade/resolvers/strategy_resolver.py | 13 +++++++++++++ freqtrade/strategy/interface.py | 6 ++++++ user_data/strategies/test_strategy.py | 8 +++++++- 5 files changed, 33 insertions(+), 3 deletions(-) diff --git a/freqtrade/constants.py b/freqtrade/constants.py index c6b8519c8..0b12343e6 100644 --- a/freqtrade/constants.py +++ b/freqtrade/constants.py @@ -14,6 +14,7 @@ DEFAULT_DB_PROD_URL = 'sqlite:///tradesv3.sqlite' DEFAULT_DB_DRYRUN_URL = 'sqlite://' UNLIMITED_STAKE_AMOUNT = 'unlimited' REQUIRED_ORDERTYPES = ['buy', 'sell', 'stoploss'] +REQUIRED_ORDERTIF = ['buy', 'sell'] ORDERTYPE_POSSIBILITIES = ['limit', 'market'] ORDERTIF_POSSIBILITIES = ['gtc', 'aon', 'fok', 'ioc'] diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index 7a974d385..7258413f1 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -475,7 +475,8 @@ class FreqtradeBot(object): amount = stake_amount / buy_limit order_id = self.exchange.buy(pair=pair, ordertype=self.strategy.order_types['buy'], - amount=amount, rate=buy_limit)['id'] + amount=amount, rate=buy_limit, + time_in_force=self.strategy.order_time_in_force['buy'])['id'] self.rpc.send_msg({ 'type': RPCMessageType.BUY_NOTIFICATION, @@ -782,7 +783,10 @@ class FreqtradeBot(object): # Execute sell and update trade record order_id = self.exchange.sell(pair=str(trade.pair), ordertype=self.strategy.order_types[sell_type], - amount=trade.amount, rate=limit)['id'] + amount=trade.amount, rate=limit, + time_in_force=self.strategy.order_time_in_force['sell'] + )['id'] + trade.open_order_id = order_id trade.close_rate_requested = limit trade.sell_reason = sell_reason.value diff --git a/freqtrade/resolvers/strategy_resolver.py b/freqtrade/resolvers/strategy_resolver.py index 4576d0ec8..c1d967383 100644 --- a/freqtrade/resolvers/strategy_resolver.py +++ b/freqtrade/resolvers/strategy_resolver.py @@ -83,10 +83,23 @@ class StrategyResolver(IResolver): else: config['order_types'] = self.strategy.order_types + if 'order_time_in_force' in config: + self.strategy.order_time_in_force = config['order_time_in_force'] + logger.info( + "Override strategy 'order_time_in_force' with value in config file: %s.", + config['order_time_in_force'] + ) + else: + config['order_time_in_force'] = self.strategy.order_time_in_force + if not all(k in self.strategy.order_types for k in constants.REQUIRED_ORDERTYPES): raise ImportError(f"Impossible to load Strategy '{self.strategy.__class__.__name__}'. " f"Order-types mapping is incomplete.") + if not all(k in self.strategy.order_time_in_force for k in constants.REQUIRED_ORDERTIF): + raise ImportError(f"Impossible to load Strategy '{self.strategy.__class__.__name__}'. " + f"Order-time-in-force mapping is incomplete.") + # Sort and apply type conversions self.strategy.minimal_roi = OrderedDict(sorted( {int(key): value for (key, value) in self.strategy.minimal_roi.items()}.items(), diff --git a/freqtrade/strategy/interface.py b/freqtrade/strategy/interface.py index 139bcd8be..77abb6a61 100644 --- a/freqtrade/strategy/interface.py +++ b/freqtrade/strategy/interface.py @@ -77,6 +77,12 @@ class IStrategy(ABC): 'stoploss': 'limit' } + # Optional time in force + order_time_in_force: Dict = { + 'buy': 'gtc', + 'sell': 'gtc', + } + # run "populate_indicators" only for new candle process_only_new_candles: bool = False diff --git a/user_data/strategies/test_strategy.py b/user_data/strategies/test_strategy.py index fd2e9ab75..5e0f40d18 100644 --- a/user_data/strategies/test_strategy.py +++ b/user_data/strategies/test_strategy.py @@ -7,7 +7,7 @@ from pandas import DataFrame # Add your lib to import here import talib.abstract as ta import freqtrade.vendor.qtpylib.indicators as qtpylib -import numpy # noqa +import numpy # noqa # This class is a sample. Feel free to customize it. @@ -55,6 +55,12 @@ class TestStrategy(IStrategy): 'stoploss': 'market' } + # Optional order time in force + order_types = { + 'buy': 'gtc', + 'sell': 'gtc' + } + def populate_indicators(self, dataframe: DataFrame, metadata: dict) -> DataFrame: """ Adds several different TA indicators to the given DataFrame From 9f26022ce52044c5d8ec3d228ce1d7a7fea048b7 Mon Sep 17 00:00:00 2001 From: misagh Date: Sun, 25 Nov 2018 22:08:42 +0100 Subject: [PATCH 481/699] copy/paste corrected --- user_data/strategies/test_strategy.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/user_data/strategies/test_strategy.py b/user_data/strategies/test_strategy.py index 5e0f40d18..c4db26a5d 100644 --- a/user_data/strategies/test_strategy.py +++ b/user_data/strategies/test_strategy.py @@ -56,7 +56,7 @@ class TestStrategy(IStrategy): } # Optional order time in force - order_types = { + order_time_in_force = { 'buy': 'gtc', 'sell': 'gtc' } From 16eec078d78fcdc8e5948d45998dac2536e9a902 Mon Sep 17 00:00:00 2001 From: Pan Long Date: Mon, 26 Nov 2018 09:18:29 +0800 Subject: [PATCH 482/699] Use dot to access attribute in NamedTuple This should fix the crash in #1359 --- freqtrade/wallets.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/freqtrade/wallets.py b/freqtrade/wallets.py index b8b37907d..bf6f8b027 100644 --- a/freqtrade/wallets.py +++ b/freqtrade/wallets.py @@ -35,8 +35,8 @@ class Wallets(object): return 999.9 balance = self.wallets.get(currency) - if balance and balance['free']: - return balance['free'] + if balance and balance.free: + return balance.free else: return 0 From ad8592f3168c6d4fa908e3d9fae615f9490f9ae8 Mon Sep 17 00:00:00 2001 From: Matthias Date: Mon, 26 Nov 2018 06:22:53 +0100 Subject: [PATCH 483/699] Test live mode of get_free --- freqtrade/tests/test_wallets.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/freqtrade/tests/test_wallets.py b/freqtrade/tests/test_wallets.py index e6a17ecbf..88366a869 100644 --- a/freqtrade/tests/test_wallets.py +++ b/freqtrade/tests/test_wallets.py @@ -30,6 +30,7 @@ def test_sync_wallet_at_boot(mocker, default_conf): assert freqtrade.wallets.wallets['GAS'].free == 0.260739 assert freqtrade.wallets.wallets['GAS'].used == 0.0 assert freqtrade.wallets.wallets['GAS'].total == 0.260739 + assert freqtrade.wallets.get_free('BNT') == 1.0 mocker.patch.multiple( 'freqtrade.exchange.Exchange', @@ -56,6 +57,7 @@ def test_sync_wallet_at_boot(mocker, default_conf): assert freqtrade.wallets.wallets['GAS'].free == 0.270739 assert freqtrade.wallets.wallets['GAS'].used == 0.1 assert freqtrade.wallets.wallets['GAS'].total == 0.260439 + assert freqtrade.wallets.get_free('GAS') == 0.270739 def test_sync_wallet_missing_data(mocker, default_conf): @@ -84,3 +86,4 @@ def test_sync_wallet_missing_data(mocker, default_conf): assert freqtrade.wallets.wallets['GAS'].free == 0.260739 assert freqtrade.wallets.wallets['GAS'].used is None assert freqtrade.wallets.wallets['GAS'].total == 0.260739 + assert freqtrade.wallets.get_free('GAS') == 0.260739 From d3712c6e4035552e62490e31d8e422944a36abc6 Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Mon, 26 Nov 2018 13:34:05 +0100 Subject: [PATCH 484/699] Update ccxt from 1.17.536 to 1.17.539 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index fa550195d..6f969be06 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -ccxt==1.17.536 +ccxt==1.17.539 SQLAlchemy==1.2.14 python-telegram-bot==11.1.0 arrow==0.12.1 From b2634e8e085d8510740bf16c1293067a8e8a6b73 Mon Sep 17 00:00:00 2001 From: misagh Date: Mon, 26 Nov 2018 18:28:13 +0100 Subject: [PATCH 485/699] typo corrected --- freqtrade/freqtradebot.py | 2 +- freqtrade/strategy/interface.py | 2 +- freqtrade/tests/test_freqtradebot.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index d9a15f56a..5425e8736 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -700,7 +700,7 @@ class FreqtradeBot(object): logger.debug('Handling stoploss on exchange %s ...', trade) order = self.exchange.get_order(trade.stoploss_order_id, trade.pair) if order['status'] == 'closed': - trade.sell_reason = SellType.STOPLOSS_ON_EXCHNAGE.value + trade.sell_reason = SellType.STOPLOSS_ON_EXCHANGE.value trade.update(order) result = True else: diff --git a/freqtrade/strategy/interface.py b/freqtrade/strategy/interface.py index 1073f8028..141dd996c 100644 --- a/freqtrade/strategy/interface.py +++ b/freqtrade/strategy/interface.py @@ -33,7 +33,7 @@ class SellType(Enum): """ ROI = "roi" STOP_LOSS = "stop_loss" - STOPLOSS_ON_EXCHNAGE = "stoploss_on_exchange" + STOPLOSS_ON_EXCHANGE = "stoploss_on_exchange" TRAILING_STOP_LOSS = "trailing_stop_loss" SELL_SIGNAL = "sell_signal" FORCE_SELL = "force_sell" diff --git a/freqtrade/tests/test_freqtradebot.py b/freqtrade/tests/test_freqtradebot.py index b6b42d1da..131d7df99 100644 --- a/freqtrade/tests/test_freqtradebot.py +++ b/freqtrade/tests/test_freqtradebot.py @@ -1685,7 +1685,7 @@ def test_may_execute_sell_after_stoploss_on_exchange_hit(default_conf, assert trade.stoploss_order_id is None assert trade.is_open is False print(trade.sell_reason) - assert trade.sell_reason == SellType.STOPLOSS_ON_EXCHNAGE.value + assert trade.sell_reason == SellType.STOPLOSS_ON_EXCHANGE.value assert rpc_mock.call_count == 1 From 17004a5a72a232b3c80389f09de97fe107439d98 Mon Sep 17 00:00:00 2001 From: misagh Date: Mon, 26 Nov 2018 18:29:41 +0100 Subject: [PATCH 486/699] documentation corrected --- docs/configuration.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/configuration.md b/docs/configuration.md index 03059e261..e05405aed 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -152,7 +152,7 @@ The below is the default which is used if this is not configured in either Strat "buy": "limit", "sell": "limit", "stoploss": "market", - "stoploss_on_exchange": "false" + "stoploss_on_exchange": False }, ``` From 1f1770ad5a21e5c994fd46041b183fab6db2cc34 Mon Sep 17 00:00:00 2001 From: misagh Date: Mon, 26 Nov 2018 18:46:59 +0100 Subject: [PATCH 487/699] migration script and and error handling on stop loss order --- freqtrade/exchange/__init__.py | 19 ++++++++++++++++++- freqtrade/persistence.py | 2 +- freqtrade/tests/test_persistence.py | 1 + 3 files changed, 20 insertions(+), 2 deletions(-) diff --git a/freqtrade/exchange/__init__.py b/freqtrade/exchange/__init__.py index a311fd666..38e2fa317 100644 --- a/freqtrade/exchange/__init__.py +++ b/freqtrade/exchange/__init__.py @@ -372,9 +372,26 @@ class Exchange(object): } return self._dry_run_open_orders[order_id] - return self._api.create_order(pair, 'stop_loss', 'sell', + try: + return self._api.create_order(pair, 'stop_loss', 'sell', amount, rate, {'stopPrice': stop_price}) + except ccxt.InsufficientFunds as e: + raise DependencyException( + f'Insufficient funds to place stoploss limit order on market {pair}.' + f'Tried to put a stoploss amount {amount} at rate {rate} (total {rate*amount}).' + f'Message: {e}') + except ccxt.InvalidOrder as e: + raise DependencyException( + f'Could not place stoploss limit order on market {pair}.' + f'Tried to place stoploss amount {amount} at rate {rate} (total {rate*amount}).' + f'Message: {e}') + except (ccxt.NetworkError, ccxt.ExchangeError) as e: + raise TemporaryError( + f'Could not place stoploss limit order due to {e.__class__.__name__}. Message: {e}') + except ccxt.BaseError as e: + raise OperationalException(e) + @retrier def get_balance(self, currency: str) -> float: if self._conf['dry_run']: diff --git a/freqtrade/persistence.py b/freqtrade/persistence.py index 364af06ce..26b0d9d93 100644 --- a/freqtrade/persistence.py +++ b/freqtrade/persistence.py @@ -82,7 +82,7 @@ def check_migrate(engine) -> None: logger.debug(f'trying {table_back_name}') # Check for latest column - if not has_column(cols, 'ticker_interval'): + if not has_column(cols, 'stoploss_order_id'): logger.info(f'Running database migration - backup available as {table_back_name}') fee_open = get_column_def(cols, 'fee_open', 'fee') diff --git a/freqtrade/tests/test_persistence.py b/freqtrade/tests/test_persistence.py index 5e0647dff..cdfdef6e6 100644 --- a/freqtrade/tests/test_persistence.py +++ b/freqtrade/tests/test_persistence.py @@ -426,6 +426,7 @@ def test_migrate_new(mocker, default_conf, fee, caplog): max_rate FLOAT, sell_reason VARCHAR, strategy VARCHAR, + ticker_interval INTEGER, PRIMARY KEY (id), CHECK (is_open IN (0, 1)) );""" From b63535083e66a404724968c5edf6c5841c65b87e Mon Sep 17 00:00:00 2001 From: misagh Date: Mon, 26 Nov 2018 18:47:32 +0100 Subject: [PATCH 488/699] flake8 --- freqtrade/exchange/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/exchange/__init__.py b/freqtrade/exchange/__init__.py index 38e2fa317..64875af87 100644 --- a/freqtrade/exchange/__init__.py +++ b/freqtrade/exchange/__init__.py @@ -374,7 +374,7 @@ class Exchange(object): try: return self._api.create_order(pair, 'stop_loss', 'sell', - amount, rate, {'stopPrice': stop_price}) + amount, rate, {'stopPrice': stop_price}) except ccxt.InsufficientFunds as e: raise DependencyException( From 7f6fc7e90f4edde9757d3f794d396916d935da47 Mon Sep 17 00:00:00 2001 From: misagh Date: Mon, 26 Nov 2018 19:13:36 +0100 Subject: [PATCH 489/699] Lost in git ! --- freqtrade/main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/main.py b/freqtrade/main.py index 7f4022d1f..3ed478ec3 100755 --- a/freqtrade/main.py +++ b/freqtrade/main.py @@ -45,7 +45,7 @@ def main(sysargv: List[str]) -> None: freqtrade = FreqtradeBot(config) state = None - while True: + while 1: state = freqtrade.worker(old_state=state) if state == State.RELOAD_CONF: freqtrade = reconfigure(freqtrade, args) From 6351fe7a7f927763aa79e28d941d60ab3e9a7fea Mon Sep 17 00:00:00 2001 From: misagh Date: Mon, 26 Nov 2018 20:24:13 +0100 Subject: [PATCH 490/699] test added: stoploss_order_id should be null after migration --- freqtrade/tests/test_persistence.py | 1 + 1 file changed, 1 insertion(+) diff --git a/freqtrade/tests/test_persistence.py b/freqtrade/tests/test_persistence.py index cdfdef6e6..d0a209f40 100644 --- a/freqtrade/tests/test_persistence.py +++ b/freqtrade/tests/test_persistence.py @@ -472,6 +472,7 @@ def test_migrate_new(mocker, default_conf, fee, caplog): assert trade.sell_reason is None assert trade.strategy is None assert trade.ticker_interval is None + assert trade.stoploss_order_id is None assert log_has("trying trades_bak1", caplog.record_tuples) assert log_has("trying trades_bak2", caplog.record_tuples) assert log_has("Running database migration - backup available as trades_bak2", From f5a70750f0f19eb1867d52950c27971cf5caba91 Mon Sep 17 00:00:00 2001 From: misagh Date: Mon, 26 Nov 2018 21:06:32 +0100 Subject: [PATCH 491/699] edge real position sizing drafted --- freqtrade/edge/__init__.py | 16 +++++++++++++--- freqtrade/freqtradebot.py | 4 +++- 2 files changed, 16 insertions(+), 4 deletions(-) diff --git a/freqtrade/edge/__init__.py b/freqtrade/edge/__init__.py index 009b80664..6be09b19d 100644 --- a/freqtrade/edge/__init__.py +++ b/freqtrade/edge/__init__.py @@ -9,6 +9,7 @@ import utils_find_1st as utf1st from pandas import DataFrame import freqtrade.optimize as optimize +from freqtrade import constants, OperationalException from freqtrade.arguments import Arguments from freqtrade.arguments import TimeRange from freqtrade.strategy.interface import SellType @@ -53,7 +54,15 @@ class Edge(): self.edge_config = self.config.get('edge', {}) self._cached_pairs: Dict[str, Any] = {} # Keeps a list of pairs - self._total_capital: float = self.config['stake_amount'] + # checking max_open_trades. it should be -1 as with Edge + # the number of trades is determined by position size + if self.config['max_open_trades'] != -1: + logger.critical('max_open_trades should be -1 in config !') + + if self.config['stake_amount'] != constants.UNLIMITED_STAKE_AMOUNT: + raise OperationalException('Edge works only with unlimited stake amount') + + self._capital_percentage: float = self.edge_config.get('capital_available_percentage') self._allowed_risk: float = self.edge_config.get('allowed_risk') self._since_number_of_days: int = self.edge_config.get('calculate_since_number_of_days', 14) self._last_updated: int = 0 # Timestamp of pairs last updated time @@ -150,9 +159,10 @@ class Edge(): return True - def stake_amount(self, pair: str) -> float: + def stake_amount(self, pair: str, capital: float) -> float: stoploss = self._cached_pairs[pair].stoploss - allowed_capital_at_risk = round(self._total_capital * self._allowed_risk, 5) + available_capital = capital * self._capital_percentage + allowed_capital_at_risk = round(available_capital * self._allowed_risk, 5) position_size = abs(round((allowed_capital_at_risk / stoploss), 5)) return position_size diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index 602c4ae2f..a6d2df954 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -333,7 +333,9 @@ class FreqtradeBot(object): :return: float: Stake Amount """ if self.edge: - stake_amount = self.edge.stake_amount(pair) + stake_amount = self.edge.stake_amount( + pair, self.wallets.get_free(self.config['stake_currency']) + ) else: stake_amount = self.config['stake_amount'] From 7832fe7074a403fa7417f7cc38c5156ed1fa64a4 Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Tue, 27 Nov 2018 13:34:08 +0100 Subject: [PATCH 492/699] Update ccxt from 1.17.539 to 1.17.543 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 6f969be06..12c6f5cda 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -ccxt==1.17.539 +ccxt==1.17.543 SQLAlchemy==1.2.14 python-telegram-bot==11.1.0 arrow==0.12.1 From 159ac6e6573a9d44a244482739cf199eff1155d4 Mon Sep 17 00:00:00 2001 From: misagh Date: Tue, 27 Nov 2018 14:02:34 +0100 Subject: [PATCH 493/699] edge tests fixed for position sizing --- freqtrade/edge/__init__.py | 6 +++--- freqtrade/freqtradebot.py | 4 +--- freqtrade/tests/conftest.py | 4 ++++ freqtrade/tests/edge/test_edge.py | 20 ++++++++++---------- freqtrade/tests/test_freqtradebot.py | 4 ++-- 5 files changed, 20 insertions(+), 18 deletions(-) diff --git a/freqtrade/edge/__init__.py b/freqtrade/edge/__init__.py index 6be09b19d..f5961d9b4 100644 --- a/freqtrade/edge/__init__.py +++ b/freqtrade/edge/__init__.py @@ -162,8 +162,8 @@ class Edge(): def stake_amount(self, pair: str, capital: float) -> float: stoploss = self._cached_pairs[pair].stoploss available_capital = capital * self._capital_percentage - allowed_capital_at_risk = round(available_capital * self._allowed_risk, 5) - position_size = abs(round((allowed_capital_at_risk / stoploss), 5)) + allowed_capital_at_risk = round(available_capital * self._allowed_risk, 15) + position_size = abs(round((allowed_capital_at_risk / stoploss), 15)) return position_size def stoploss(self, pair: str) -> float: @@ -207,7 +207,7 @@ class Edge(): # 0.05% is 0.0005 # fee = 0.001 - stake = self.config.get('stake_amount') + stake = 0.015 fee = self.fee open_fee = fee / 2 diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index a6d2df954..937fa0d3f 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -336,6 +336,7 @@ class FreqtradeBot(object): stake_amount = self.edge.stake_amount( pair, self.wallets.get_free(self.config['stake_currency']) ) + return stake_amount else: stake_amount = self.config['stake_amount'] @@ -782,9 +783,6 @@ class FreqtradeBot(object): if sell_reason in (SellType.STOP_LOSS, SellType.TRAILING_STOP_LOSS): sell_type = 'stoploss' - if self.config.get('dry_run', False) and sell_type == 'stoploss': - limit = trade.stop_loss - # Execute sell and update trade record order_id = self.exchange.sell(pair=str(trade.pair), ordertype=self.strategy.order_types[sell_type], diff --git a/freqtrade/tests/conftest.py b/freqtrade/tests/conftest.py index f7fe697b8..75c8032f1 100644 --- a/freqtrade/tests/conftest.py +++ b/freqtrade/tests/conftest.py @@ -10,6 +10,7 @@ import arrow import pytest from telegram import Chat, Message, Update +from freqtrade import constants from freqtrade.exchange.exchange_helpers import parse_ticker_dataframe from freqtrade.exchange import Exchange from freqtrade.edge import Edge, PairInfo @@ -787,10 +788,13 @@ def buy_order_fee(): @pytest.fixture(scope="function") def edge_conf(default_conf): + default_conf['max_open_trades'] = -1 + default_conf['stake_amount'] = constants.UNLIMITED_STAKE_AMOUNT default_conf['edge'] = { "enabled": True, "process_throttle_secs": 1800, "calculate_since_number_of_days": 14, + "capital_available_percentage": 0.5, "allowed_risk": 0.01, "stoploss_range_min": -0.01, "stoploss_range_max": -0.1, diff --git a/freqtrade/tests/edge/test_edge.py b/freqtrade/tests/edge/test_edge.py index fac055c17..40bb35209 100644 --- a/freqtrade/tests/edge/test_edge.py +++ b/freqtrade/tests/edge/test_edge.py @@ -123,9 +123,9 @@ def test_edge_results(edge_conf, mocker, caplog, data) -> None: assert res.close_time == _get_frame_time_from_offset(trade.close_tick) -def test_adjust(mocker, default_conf): - freqtrade = get_patched_freqtradebot(mocker, default_conf) - edge = Edge(default_conf, freqtrade.exchange, freqtrade.strategy) +def test_adjust(mocker, edge_conf): + freqtrade = get_patched_freqtradebot(mocker, edge_conf) + edge = Edge(edge_conf, freqtrade.exchange, freqtrade.strategy) mocker.patch('freqtrade.edge.Edge._cached_pairs', mocker.PropertyMock( return_value={ 'E/F': PairInfo(-0.01, 0.66, 3.71, 0.50, 1.71, 10, 60), @@ -138,9 +138,9 @@ def test_adjust(mocker, default_conf): assert(edge.adjust(pairs) == ['E/F', 'C/D']) -def test_stoploss(mocker, default_conf): - freqtrade = get_patched_freqtradebot(mocker, default_conf) - edge = Edge(default_conf, freqtrade.exchange, freqtrade.strategy) +def test_stoploss(mocker, edge_conf): + freqtrade = get_patched_freqtradebot(mocker, edge_conf) + edge = Edge(edge_conf, freqtrade.exchange, freqtrade.strategy) mocker.patch('freqtrade.edge.Edge._cached_pairs', mocker.PropertyMock( return_value={ 'E/F': PairInfo(-0.01, 0.66, 3.71, 0.50, 1.71, 10, 60), @@ -234,12 +234,12 @@ def mocked_load_data(datadir, pairs=[], ticker_interval='0m', refresh_pairs=Fals return pairdata -def test_edge_process_downloaded_data(mocker, default_conf): - default_conf['datadir'] = None - freqtrade = get_patched_freqtradebot(mocker, default_conf) +def test_edge_process_downloaded_data(mocker, edge_conf): + edge_conf['datadir'] = None + freqtrade = get_patched_freqtradebot(mocker, edge_conf) mocker.patch('freqtrade.exchange.Exchange.get_fee', MagicMock(return_value=0.001)) mocker.patch('freqtrade.optimize.load_data', mocked_load_data) - edge = Edge(default_conf, freqtrade.exchange, freqtrade.strategy) + edge = Edge(edge_conf, freqtrade.exchange, freqtrade.strategy) assert edge.calculate() assert len(edge._cached_pairs) == 2 diff --git a/freqtrade/tests/test_freqtradebot.py b/freqtrade/tests/test_freqtradebot.py index eb5336c61..04d00d00a 100644 --- a/freqtrade/tests/test_freqtradebot.py +++ b/freqtrade/tests/test_freqtradebot.py @@ -260,8 +260,8 @@ def test_edge_overrides_stake_amount(mocker, edge_conf) -> None: patch_edge(mocker) freqtrade = FreqtradeBot(edge_conf) - assert freqtrade._get_trade_stake_amount('NEO/BTC') == (0.001 * 0.01) / 0.20 - assert freqtrade._get_trade_stake_amount('LTC/BTC') == (0.001 * 0.01) / 0.20 + assert freqtrade._get_trade_stake_amount('NEO/BTC') == (999.9 * 0.5 * 0.01) / 0.20 + assert freqtrade._get_trade_stake_amount('LTC/BTC') == (999.9 * 0.5 * 0.01) / 0.21 def test_edge_overrides_stoploss(limit_buy_order, fee, markets, caplog, mocker, edge_conf) -> None: From 7dbf0fed684e9af10da8c2fda45a03b9f1a3bdcb Mon Sep 17 00:00:00 2001 From: misagh Date: Tue, 27 Nov 2018 17:09:51 +0100 Subject: [PATCH 494/699] stop loss limit order type corrected --- freqtrade/exchange/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/exchange/__init__.py b/freqtrade/exchange/__init__.py index a2857a12a..2480dbe32 100644 --- a/freqtrade/exchange/__init__.py +++ b/freqtrade/exchange/__init__.py @@ -373,7 +373,7 @@ class Exchange(object): return self._dry_run_open_orders[order_id] try: - return self._api.create_order(pair, 'stop_loss', 'sell', + return self._api.create_order(pair, 'stop_loss_limit', 'sell', amount, rate, {'stopPrice': stop_price}) except ccxt.InsufficientFunds as e: From 29f680ec5d6f5bd837788b935e7b27dc3eee77f7 Mon Sep 17 00:00:00 2001 From: misagh Date: Tue, 27 Nov 2018 17:26:06 +0100 Subject: [PATCH 495/699] fix order type test --- freqtrade/tests/exchange/test_exchange.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/tests/exchange/test_exchange.py b/freqtrade/tests/exchange/test_exchange.py index 63f1008b9..ec7c2acae 100644 --- a/freqtrade/tests/exchange/test_exchange.py +++ b/freqtrade/tests/exchange/test_exchange.py @@ -1150,7 +1150,7 @@ def test_get_fee(default_conf, mocker): def test_stoploss_limit_order(default_conf, mocker): api_mock = MagicMock() order_id = 'test_prod_buy_{}'.format(randint(0, 10 ** 6)) - order_type = 'stop_loss' + order_type = 'stop_loss_limit' api_mock.create_order = MagicMock(return_value={ 'id': order_id, From 4ffc74d5facac4ee9243bdf355421de34cdbfe6c Mon Sep 17 00:00:00 2001 From: misagh Date: Tue, 27 Nov 2018 19:05:59 +0100 Subject: [PATCH 496/699] if buy order is rejected or expired the bot should exit the buy loop --- freqtrade/freqtradebot.py | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index 7258413f1..c5edc71ae 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -474,9 +474,21 @@ class FreqtradeBot(object): amount = stake_amount / buy_limit - order_id = self.exchange.buy(pair=pair, ordertype=self.strategy.order_types['buy'], + order = self.exchange.buy(pair=pair, ordertype=self.strategy.order_types['buy'], amount=amount, rate=buy_limit, - time_in_force=self.strategy.order_time_in_force['buy'])['id'] + time_in_force=self.strategy.order_time_in_force['buy']) + order_id = order['id'] + order_info = order['info'] + + # check if order is expired (in case of FOC or IOC orders) + # or rejected by the exchange. + if order_info['status'] == 'EXPIRED' or order_info['status'] == 'REJECTED': + order_type = self.strategy.order_types['buy'] + status = order_info['status'] + logger.warning('Buy %s order for %s is %s by %s.', + order_type, pair_s, status, self.exchange.name) + return False + self.rpc.send_msg({ 'type': RPCMessageType.BUY_NOTIFICATION, From 50a384130fe7830cd5c3edc3ab2ecda5dd030f0a Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Wed, 28 Nov 2018 13:34:07 +0100 Subject: [PATCH 497/699] Update ccxt from 1.17.543 to 1.17.545 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 12c6f5cda..15c7d1858 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -ccxt==1.17.543 +ccxt==1.17.545 SQLAlchemy==1.2.14 python-telegram-bot==11.1.0 arrow==0.12.1 From da94e97c602861492cf03a38eb37c0d6e382e94e Mon Sep 17 00:00:00 2001 From: misagh Date: Wed, 28 Nov 2018 13:58:53 +0100 Subject: [PATCH 498/699] in case trade is not open, then handle_stoploss_on_exchange should not be called --- freqtrade/freqtradebot.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index aeb8ce50c..c72451df6 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -554,7 +554,7 @@ class FreqtradeBot(object): trade.update(order) - if self.strategy.order_types.get('stoploss_on_exchange'): + if self.strategy.order_types.get('stoploss_on_exchange') and trade.is_open: result = self.handle_stoploss_on_exchange(trade) if result: self.wallets.update() @@ -675,7 +675,7 @@ class FreqtradeBot(object): # If trade is open and the buy order is fulfilled but there is no stoploss, # then we add a stoploss on exchange - if trade.is_open and not trade.open_order_id and not trade.stoploss_order_id: + if not trade.open_order_id and not trade.stoploss_order_id: if self.edge: stoploss = self.edge.stoploss(pair=trade.pair) else: @@ -692,7 +692,7 @@ class FreqtradeBot(object): )['id'] trade.stoploss_order_id = str(stoploss_order_id) - # Or there is already a stoploss on exchange. + # Or the trade open and there is already a stoploss on exchange. # so we check if it is hit ... elif trade.stoploss_order_id: logger.debug('Handling stoploss on exchange %s ...', trade) From fb755880fad8cc2e0b0145544117b7243eac26dd Mon Sep 17 00:00:00 2001 From: misagh Date: Wed, 28 Nov 2018 14:16:50 +0100 Subject: [PATCH 499/699] logs added in case stop loss on exchange is hit --- freqtrade/persistence.py | 1 + freqtrade/tests/test_freqtradebot.py | 3 ++- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/freqtrade/persistence.py b/freqtrade/persistence.py index 26b0d9d93..64663a2fd 100644 --- a/freqtrade/persistence.py +++ b/freqtrade/persistence.py @@ -255,6 +255,7 @@ class Trade(_DECL_BASE): self.close(order['price']) elif order_type == 'stop_loss_limit': self.stoploss_order_id = None + logger.info('STOP_LOSS_LIMIT is hit for %s.', self) self.close(order['price']) else: raise ValueError(f'Unknown order type: {order_type}') diff --git a/freqtrade/tests/test_freqtradebot.py b/freqtrade/tests/test_freqtradebot.py index 995a3e8ff..81ade608a 100644 --- a/freqtrade/tests/test_freqtradebot.py +++ b/freqtrade/tests/test_freqtradebot.py @@ -899,7 +899,7 @@ def test_add_stoploss_on_exchange(mocker, default_conf, limit_buy_order) -> None assert trade.is_open is True -def test_handle_stoploss_on_exchange(mocker, default_conf, fee, +def test_handle_stoploss_on_exchange(mocker, default_conf, fee, caplog, markets, limit_buy_order, limit_sell_order) -> None: stoploss_limit = MagicMock(return_value={'id': 13434334}) patch_RPCManager(mocker) @@ -961,6 +961,7 @@ def test_handle_stoploss_on_exchange(mocker, default_conf, fee, }) mocker.patch('freqtrade.exchange.Exchange.get_order', stoploss_order_hit) assert freqtrade.handle_stoploss_on_exchange(trade) is True + assert log_has('STOP_LOSS_LIMIT is hit for {}.'.format(trade), caplog.record_tuples) assert trade.stoploss_order_id is None assert trade.is_open is False From e9305b6592ddcba35fef7d647f8d58ef79ccf40a Mon Sep 17 00:00:00 2001 From: misagh Date: Wed, 28 Nov 2018 15:36:32 +0100 Subject: [PATCH 500/699] position size fixed --- freqtrade/edge/__init__.py | 11 ++++++++--- freqtrade/freqtradebot.py | 4 +++- freqtrade/wallets.py | 22 ++++++++++++++++++++++ 3 files changed, 33 insertions(+), 4 deletions(-) diff --git a/freqtrade/edge/__init__.py b/freqtrade/edge/__init__.py index f5961d9b4..f7e8e5ea9 100644 --- a/freqtrade/edge/__init__.py +++ b/freqtrade/edge/__init__.py @@ -159,11 +159,16 @@ class Edge(): return True - def stake_amount(self, pair: str, capital: float) -> float: + def stake_amount(self, pair: str, free_capital: float, total_capital: float) -> float: stoploss = self._cached_pairs[pair].stoploss - available_capital = capital * self._capital_percentage + available_capital = total_capital * self._capital_percentage allowed_capital_at_risk = round(available_capital * self._allowed_risk, 15) - position_size = abs(round((allowed_capital_at_risk / stoploss), 15)) + max_position_size = abs(round((allowed_capital_at_risk / stoploss), 15)) + position_size = min(max_position_size, free_capital) + logger.info( + 'position size is %s for pair %s, stoploss %s and available capital of %s.', + position_size, pair, stoploss, available_capital + ) return position_size def stoploss(self, pair: str) -> float: diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index 937fa0d3f..90f100eac 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -334,7 +334,9 @@ class FreqtradeBot(object): """ if self.edge: stake_amount = self.edge.stake_amount( - pair, self.wallets.get_free(self.config['stake_currency']) + pair, + self.wallets.get_free(self.config['stake_currency']), + self.wallets.get_total(self.config['stake_currency']) ) return stake_amount else: diff --git a/freqtrade/wallets.py b/freqtrade/wallets.py index bf6f8b027..59d8fa3da 100644 --- a/freqtrade/wallets.py +++ b/freqtrade/wallets.py @@ -40,6 +40,28 @@ class Wallets(object): else: return 0 + def get_used(self, currency) -> float: + + if self.exchange._conf['dry_run']: + return 999.9 + + balance = self.wallets.get(currency) + if balance and balance.used: + return balance.used + else: + return 0 + + def get_total(self, currency) -> float: + + if self.exchange._conf['dry_run']: + return 999.9 + + balance = self.wallets.get(currency) + if balance and balance.total: + return balance.total + else: + return 0 + def update(self) -> None: balances = self.exchange.get_balances() From c913fef80c5c31d8007b5d437e9d371196440695 Mon Sep 17 00:00:00 2001 From: misagh Date: Wed, 28 Nov 2018 15:45:11 +0100 Subject: [PATCH 501/699] =?UTF-8?q?stop=20loss=20limit=20when=20hit,=20the?= =?UTF-8?q?=20close=20price=20is=20=E2=80=9Caverage=E2=80=9D?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- freqtrade/persistence.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/persistence.py b/freqtrade/persistence.py index 64663a2fd..592a88acb 100644 --- a/freqtrade/persistence.py +++ b/freqtrade/persistence.py @@ -256,7 +256,7 @@ class Trade(_DECL_BASE): elif order_type == 'stop_loss_limit': self.stoploss_order_id = None logger.info('STOP_LOSS_LIMIT is hit for %s.', self) - self.close(order['price']) + self.close(order['average']) else: raise ValueError(f'Unknown order type: {order_type}') cleanup() From 1a5465fb508ca20318e36913cfb0cd2f12a995b1 Mon Sep 17 00:00:00 2001 From: misagh Date: Wed, 28 Nov 2018 19:35:10 +0100 Subject: [PATCH 502/699] logs enriched in case of stop loss on exchange, test fixed --- freqtrade/exchange/__init__.py | 8 +++++--- freqtrade/tests/test_freqtradebot.py | 3 ++- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/freqtrade/exchange/__init__.py b/freqtrade/exchange/__init__.py index 2480dbe32..baa9d573d 100644 --- a/freqtrade/exchange/__init__.py +++ b/freqtrade/exchange/__init__.py @@ -378,13 +378,15 @@ class Exchange(object): except ccxt.InsufficientFunds as e: raise DependencyException( - f'Insufficient funds to place stoploss limit order on market {pair}.' - f'Tried to put a stoploss amount {amount} at rate {rate} (total {rate*amount}).' + f'Insufficient funds to place stoploss limit order on market {pair}. ' + f'Tried to put a stoploss amount {amount} with ' + f'stop {stop_price} and limit {rate} (total {rate*amount}).' f'Message: {e}') except ccxt.InvalidOrder as e: raise DependencyException( f'Could not place stoploss limit order on market {pair}.' - f'Tried to place stoploss amount {amount} at rate {rate} (total {rate*amount}).' + f'Tried to place stoploss amount {amount} with ' + f'stop {stop_price} and limit {rate} (total {rate*amount}).' f'Message: {e}') except (ccxt.NetworkError, ccxt.ExchangeError) as e: raise TemporaryError( diff --git a/freqtrade/tests/test_freqtradebot.py b/freqtrade/tests/test_freqtradebot.py index 81ade608a..147d47b72 100644 --- a/freqtrade/tests/test_freqtradebot.py +++ b/freqtrade/tests/test_freqtradebot.py @@ -957,7 +957,8 @@ def test_handle_stoploss_on_exchange(mocker, default_conf, fee, caplog, stoploss_order_hit = MagicMock(return_value={ 'status': 'closed', 'type': 'stop_loss_limit', - 'price': 2 + 'price': 3, + 'average': 2 }) mocker.patch('freqtrade.exchange.Exchange.get_order', stoploss_order_hit) assert freqtrade.handle_stoploss_on_exchange(trade) is True From e698590bb25a3327b68a30897ae96bdec052cc1d Mon Sep 17 00:00:00 2001 From: misagh Date: Wed, 28 Nov 2018 20:04:56 +0100 Subject: [PATCH 503/699] avoid generating logs on each iteration --- freqtrade/edge/__init__.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/freqtrade/edge/__init__.py b/freqtrade/edge/__init__.py index f7e8e5ea9..0b4227ed0 100644 --- a/freqtrade/edge/__init__.py +++ b/freqtrade/edge/__init__.py @@ -53,6 +53,7 @@ class Edge(): self.edge_config = self.config.get('edge', {}) self._cached_pairs: Dict[str, Any] = {} # Keeps a list of pairs + self._final: list = [] # checking max_open_trades. it should be -1 as with Edge # the number of trades is determined by position size @@ -178,7 +179,6 @@ class Edge(): """ Filters out and sorts "pairs" according to Edge calculated pairs """ - final = [] for pair, info in self._cached_pairs.items(): if info.expectancy > float(self.edge_config.get('minimum_expectancy', 0.2)) and \ @@ -186,12 +186,14 @@ class Edge(): pair in pairs: final.append(pair) - if final: - logger.info('Edge validated only %s', final) - else: - logger.info('Edge removed all pairs as no pair with minimum expectancy was found !') + if self._final != final: + self._final = final + if self._final: + logger.info('Edge validated only %s', self._final) + else: + logger.info('Edge removed all pairs as no pair with minimum expectancy was found !') - return final + return self._final def _fill_calculable_fields(self, result: DataFrame) -> DataFrame: """ From 38592c6fa60f0892d0d32e450b1036830bab2c3c Mon Sep 17 00:00:00 2001 From: Matthias Date: Thu, 29 Nov 2018 07:07:47 +0100 Subject: [PATCH 504/699] Add binance config sample, improve invalid pair message --- config_binance.json.example | 83 ++++++++++++++++++++++++++++++++++ freqtrade/exchange/__init__.py | 3 +- 2 files changed, 85 insertions(+), 1 deletion(-) create mode 100644 config_binance.json.example diff --git a/config_binance.json.example b/config_binance.json.example new file mode 100644 index 000000000..da000efa7 --- /dev/null +++ b/config_binance.json.example @@ -0,0 +1,83 @@ +{ + "max_open_trades": 3, + "stake_currency": "BTC", + "stake_amount": 0.05, + "fiat_display_currency": "USD", + "ticker_interval" : "5m", + "dry_run": true, + "trailing_stop": false, + "unfilledtimeout": { + "buy": 10, + "sell": 30 + }, + "bid_strategy": { + "ask_last_balance": 0.0, + "use_order_book": false, + "order_book_top": 1, + "check_depth_of_market": { + "enabled": false, + "bids_to_ask_delta": 1 + } + }, + "ask_strategy":{ + "use_order_book": false, + "order_book_min": 1, + "order_book_max": 9 + }, + "exchange": { + "name": "binance", + "key": "your_exchange_key", + "secret": "your_exchange_secret", + "ccxt_config": {"enableRateLimit": true}, + "ccxt_async_config": { + "enableRateLimit": false + }, + "pair_whitelist": [ + "AST/BTC", + "ETC/BTC", + "ETH/BTC", + "EOS/BTC", + "IOTA/BTC", + "LTC/BTC", + "MTH/BTC", + "NCASH/BTC", + "TNT/BTC", + "XMR/BTC", + "XLM/BTC", + "XRP/BTC" + ], + "pair_blacklist": [ + "DOGE/BTC" + ] + }, + "experimental": { + "use_sell_signal": false, + "sell_profit_only": false, + "ignore_roi_if_buy_signal": false + }, + "edge": { + "enabled": false, + "process_throttle_secs": 3600, + "calculate_since_number_of_days": 7, + "total_capital_in_stake_currency": 0.5, + "allowed_risk": 0.01, + "stoploss_range_min": -0.01, + "stoploss_range_max": -0.1, + "stoploss_range_step": -0.01, + "minimum_winrate": 0.60, + "minimum_expectancy": 0.20, + "min_trade_number": 10, + "max_trade_duration_minute": 1440, + "remove_pumps": false + }, + "telegram": { + "enabled": false, + "token": "your_telegram_token", + "chat_id": "your_telegram_chat_id" + }, + "initial_state": "running", + "forcebuy_enable": false, + "internals": { + "process_throttle_secs": 5 + } +} diff --git a/freqtrade/exchange/__init__.py b/freqtrade/exchange/__init__.py index 350c730a4..4573e461c 100644 --- a/freqtrade/exchange/__init__.py +++ b/freqtrade/exchange/__init__.py @@ -207,7 +207,8 @@ class Exchange(object): f'Pair {pair} not compatible with stake_currency: {stake_cur}') if self.markets and pair not in self.markets: raise OperationalException( - f'Pair {pair} is not available at {self.name}') + f'Pair {pair} is not available at {self.name}' + f'Please remove {pair} from your whitelist.') def validate_timeframes(self, timeframe: List[str]) -> None: """ From cb9104fd8a1fb4a9862ae042baae9e5b69f608d6 Mon Sep 17 00:00:00 2001 From: Matthias Date: Thu, 29 Nov 2018 07:36:37 +0100 Subject: [PATCH 505/699] Add BNB as blacklist to align to documentation --- config_binance.json.example | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config_binance.json.example b/config_binance.json.example index da000efa7..7773a8c39 100644 --- a/config_binance.json.example +++ b/config_binance.json.example @@ -47,7 +47,7 @@ "XRP/BTC" ], "pair_blacklist": [ - "DOGE/BTC" + "BNB/BTC" ] }, "experimental": { From a61daed8e95915f56ce36d5027335fac9b64f4bd Mon Sep 17 00:00:00 2001 From: misagh Date: Thu, 29 Nov 2018 12:24:04 +0100 Subject: [PATCH 506/699] logs enriched --- freqtrade/edge/__init__.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/freqtrade/edge/__init__.py b/freqtrade/edge/__init__.py index 0b4227ed0..f27309479 100644 --- a/freqtrade/edge/__init__.py +++ b/freqtrade/edge/__init__.py @@ -167,7 +167,10 @@ class Edge(): max_position_size = abs(round((allowed_capital_at_risk / stoploss), 15)) position_size = min(max_position_size, free_capital) logger.info( - 'position size is %s for pair %s, stoploss %s and available capital of %s.', + 'winrate: %s, expectancy: %s, position size: %s, pair: %s,' + ' stoploss: %s, available capital: %s.', + self._cached_pairs[pair].winrate, + self._cached_pairs[pair].expectancy, position_size, pair, stoploss, available_capital ) return position_size From 6bedcc5d79b3e71d01f5bcccba7f98cd900e45fe Mon Sep 17 00:00:00 2001 From: misagh Date: Thu, 29 Nov 2018 13:22:41 +0100 Subject: [PATCH 507/699] log enriched for time in force --- freqtrade/freqtradebot.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index c5edc71ae..9268341c5 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -484,9 +484,10 @@ class FreqtradeBot(object): # or rejected by the exchange. if order_info['status'] == 'EXPIRED' or order_info['status'] == 'REJECTED': order_type = self.strategy.order_types['buy'] + order_tif = self.strategy.order_time_in_force['buy'] status = order_info['status'] - logger.warning('Buy %s order for %s is %s by %s.', - order_type, pair_s, status, self.exchange.name) + logger.warning('Buy %s order with time in force %s for %s is %s by %s.', + order_tif, order_type, pair_s, status, self.exchange.name) return False From bc2f6d3b71722f9703a5423488759a44669fba79 Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Thu, 29 Nov 2018 13:34:07 +0100 Subject: [PATCH 508/699] Update ccxt from 1.17.545 to 1.17.556 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 15c7d1858..e2a1332ba 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -ccxt==1.17.545 +ccxt==1.17.556 SQLAlchemy==1.2.14 python-telegram-bot==11.1.0 arrow==0.12.1 From 3d37c5d76725e0b56aaf0b2591781e88e408cba8 Mon Sep 17 00:00:00 2001 From: misagh Date: Thu, 29 Nov 2018 18:31:08 +0100 Subject: [PATCH 509/699] edge non existing stoploss fixed. solves #1370 --- freqtrade/edge/__init__.py | 8 +++++++- freqtrade/tests/edge/test_edge.py | 11 +++++++++++ 2 files changed, 18 insertions(+), 1 deletion(-) diff --git a/freqtrade/edge/__init__.py b/freqtrade/edge/__init__.py index 009b80664..2b1849d19 100644 --- a/freqtrade/edge/__init__.py +++ b/freqtrade/edge/__init__.py @@ -157,7 +157,13 @@ class Edge(): return position_size def stoploss(self, pair: str) -> float: - return self._cached_pairs[pair].stoploss + if pair in self._cached_pairs: + return self._cached_pairs[pair].stoploss + else: + logger.warning('tried to access stoploss of a non-existing pair, ' + 'strategy stoploss is returned instead.') + return self.strategy.stoploss + def adjust(self, pairs) -> list: """ diff --git a/freqtrade/tests/edge/test_edge.py b/freqtrade/tests/edge/test_edge.py index fac055c17..d2b1f8030 100644 --- a/freqtrade/tests/edge/test_edge.py +++ b/freqtrade/tests/edge/test_edge.py @@ -152,6 +152,17 @@ def test_stoploss(mocker, default_conf): assert edge.stoploss('E/F') == -0.01 +def test_nonexisting_stoploss(mocker, default_conf): + freqtrade = get_patched_freqtradebot(mocker, default_conf) + edge = Edge(default_conf, freqtrade.exchange, freqtrade.strategy) + mocker.patch('freqtrade.edge.Edge._cached_pairs', mocker.PropertyMock( + return_value={ + 'E/F': PairInfo(-0.01, 0.66, 3.71, 0.50, 1.71, 10, 60), + } + )) + + assert edge.stoploss('N/O') == -0.1 + def _validate_ohlc(buy_ohlc_sell_matrice): for index, ohlc in enumerate(buy_ohlc_sell_matrice): # if not high < open < low or not high < close < low From 74ca34f2dee9982235478c19e7a86ce74fa89c37 Mon Sep 17 00:00:00 2001 From: misagh Date: Thu, 29 Nov 2018 18:45:37 +0100 Subject: [PATCH 510/699] flaking8 --- freqtrade/edge/__init__.py | 1 - freqtrade/tests/edge/test_edge.py | 1 + 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/edge/__init__.py b/freqtrade/edge/__init__.py index 2b1849d19..4cb0dbc31 100644 --- a/freqtrade/edge/__init__.py +++ b/freqtrade/edge/__init__.py @@ -164,7 +164,6 @@ class Edge(): 'strategy stoploss is returned instead.') return self.strategy.stoploss - def adjust(self, pairs) -> list: """ Filters out and sorts "pairs" according to Edge calculated pairs diff --git a/freqtrade/tests/edge/test_edge.py b/freqtrade/tests/edge/test_edge.py index d2b1f8030..50c4ade3d 100644 --- a/freqtrade/tests/edge/test_edge.py +++ b/freqtrade/tests/edge/test_edge.py @@ -163,6 +163,7 @@ def test_nonexisting_stoploss(mocker, default_conf): assert edge.stoploss('N/O') == -0.1 + def _validate_ohlc(buy_ohlc_sell_matrice): for index, ohlc in enumerate(buy_ohlc_sell_matrice): # if not high < open < low or not high < close < low From efcec736b55205f732493c381a3d2671b9ae1fa4 Mon Sep 17 00:00:00 2001 From: Matthias Date: Thu, 29 Nov 2018 20:02:12 +0100 Subject: [PATCH 511/699] refactor startup_messages to rpc_manger this cleans up freqtradebot slightly --- freqtrade/freqtradebot.py | 34 +------------------------ freqtrade/rpc/rpc_manager.py | 34 ++++++++++++++++++++++++- freqtrade/tests/rpc/test_rpc_manager.py | 20 +++++++++++++++ 3 files changed, 54 insertions(+), 34 deletions(-) diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index 7a974d385..0c6369ef4 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -107,7 +107,7 @@ class FreqtradeBot(object): }) logger.info('Changing state to: %s', state.name) if state == State.RUNNING: - self._startup_messages() + self.rpc.startup_messages(self.config) if state == State.STOPPED: time.sleep(1) @@ -121,38 +121,6 @@ class FreqtradeBot(object): min_secs=min_secs) return state - def _startup_messages(self) -> None: - if self.config.get('dry_run', False): - self.rpc.send_msg({ - 'type': RPCMessageType.WARNING_NOTIFICATION, - 'status': 'Dry run is enabled. All trades are simulated.' - }) - stake_currency = self.config['stake_currency'] - stake_amount = self.config['stake_amount'] - minimal_roi = self.config['minimal_roi'] - ticker_interval = self.config['ticker_interval'] - exchange_name = self.config['exchange']['name'] - strategy_name = self.config.get('strategy', '') - self.rpc.send_msg({ - 'type': RPCMessageType.CUSTOM_NOTIFICATION, - 'status': f'*Exchange:* `{exchange_name}`\n' - f'*Stake per trade:* `{stake_amount} {stake_currency}`\n' - f'*Minimum ROI:* `{minimal_roi}`\n' - f'*Ticker Interval:* `{ticker_interval}`\n' - f'*Strategy:* `{strategy_name}`' - }) - if self.config.get('dynamic_whitelist', False): - top_pairs = 'top volume ' + str(self.config.get('dynamic_whitelist', 20)) - specific_pairs = '' - else: - top_pairs = 'whitelisted' - specific_pairs = '\n' + ', '.join(self.config['exchange'].get('pair_whitelist', '')) - self.rpc.send_msg({ - 'type': RPCMessageType.STATUS_NOTIFICATION, - 'status': f'Searching for {top_pairs} {stake_currency} pairs to buy and sell...' - f'{specific_pairs}' - }) - def _throttle(self, func: Callable[..., Any], min_secs: float, *args, **kwargs) -> Any: """ Throttles the given callable that it diff --git a/freqtrade/rpc/rpc_manager.py b/freqtrade/rpc/rpc_manager.py index 022578378..74a4e3bdc 100644 --- a/freqtrade/rpc/rpc_manager.py +++ b/freqtrade/rpc/rpc_manager.py @@ -4,7 +4,7 @@ This module contains class to manage RPC communications (Telegram, Slack, ...) import logging from typing import List, Dict, Any -from freqtrade.rpc import RPC +from freqtrade.rpc import RPC, RPCMessageType logger = logging.getLogger(__name__) @@ -51,3 +51,35 @@ class RPCManager(object): for mod in self.registered_modules: logger.debug('Forwarding message to rpc.%s', mod.name) mod.send_msg(msg) + + def startup_messages(self, config) -> None: + if config.get('dry_run', False): + self.send_msg({ + 'type': RPCMessageType.WARNING_NOTIFICATION, + 'status': 'Dry run is enabled. All trades are simulated.' + }) + stake_currency = config['stake_currency'] + stake_amount = config['stake_amount'] + minimal_roi = config['minimal_roi'] + ticker_interval = config['ticker_interval'] + exchange_name = config['exchange']['name'] + strategy_name = config.get('strategy', '') + self.send_msg({ + 'type': RPCMessageType.CUSTOM_NOTIFICATION, + 'status': f'*Exchange:* `{exchange_name}`\n' + f'*Stake per trade:* `{stake_amount} {stake_currency}`\n' + f'*Minimum ROI:* `{minimal_roi}`\n' + f'*Ticker Interval:* `{ticker_interval}`\n' + f'*Strategy:* `{strategy_name}`' + }) + if config.get('dynamic_whitelist', False): + top_pairs = 'top volume ' + str(config.get('dynamic_whitelist', 20)) + specific_pairs = '' + else: + top_pairs = 'whitelisted' + specific_pairs = '\n' + ', '.join(config['exchange'].get('pair_whitelist', '')) + self.send_msg({ + 'type': RPCMessageType.STATUS_NOTIFICATION, + 'status': f'Searching for {top_pairs} {stake_currency} pairs to buy and sell...' + f'{specific_pairs}' + }) diff --git a/freqtrade/tests/rpc/test_rpc_manager.py b/freqtrade/tests/rpc/test_rpc_manager.py index 90c693830..cbb858522 100644 --- a/freqtrade/tests/rpc/test_rpc_manager.py +++ b/freqtrade/tests/rpc/test_rpc_manager.py @@ -113,3 +113,23 @@ def test_init_webhook_enabled(mocker, default_conf, caplog) -> None: assert log_has('Enabling rpc.webhook ...', caplog.record_tuples) assert len(rpc_manager.registered_modules) == 1 assert 'webhook' in [mod.name for mod in rpc_manager.registered_modules] + + +def test_startupmessages_telegram_enabled(mocker, default_conf, caplog) -> None: + telegram_mock = mocker.patch('freqtrade.rpc.telegram.Telegram.send_msg', MagicMock()) + mocker.patch('freqtrade.rpc.telegram.Telegram._init', MagicMock()) + + freqtradebot = get_patched_freqtradebot(mocker, default_conf) + rpc_manager = RPCManager(freqtradebot) + rpc_manager.startup_messages(default_conf) + + assert telegram_mock.call_count == 3 + assert "*Exchange:* `bittrex`" in telegram_mock.call_args_list[1][0][0]['status'] + + telegram_mock.reset_mock() + default_conf['dry_run'] = True + default_conf['dynamic_whitelist'] = 20 + + rpc_manager.startup_messages(default_conf) + assert telegram_mock.call_count == 3 + assert "Dry run is enabled." in telegram_mock.call_args_list[0][0][0]['status'] From 42c8888fa1967bf024ed8afb0e4414cb66e142ee Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Fri, 30 Nov 2018 13:34:08 +0100 Subject: [PATCH 512/699] Update ccxt from 1.17.556 to 1.17.563 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index e2a1332ba..f7db2ea09 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -ccxt==1.17.556 +ccxt==1.17.563 SQLAlchemy==1.2.14 python-telegram-bot==11.1.0 arrow==0.12.1 From f554647efd8e072ddea0bd86b6e9abb33e40daca Mon Sep 17 00:00:00 2001 From: misagh Date: Fri, 30 Nov 2018 14:14:31 +0100 Subject: [PATCH 513/699] =?UTF-8?q?=E2=80=9Cchecking=20sell=E2=80=9D=20mes?= =?UTF-8?q?sage=20removed=20to=20debug?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- freqtrade/freqtradebot.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index 7a974d385..a9a51aae0 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -648,7 +648,7 @@ class FreqtradeBot(object): return True break else: - logger.info('checking sell') + logger.debug('checking sell') if self.check_sell(trade, sell_rate, buy, sell): return True From 7767470af845279df6bea54436bf534ed7d41583 Mon Sep 17 00:00:00 2001 From: misagh Date: Fri, 30 Nov 2018 17:50:03 +0100 Subject: [PATCH 514/699] =?UTF-8?q?return=20stake=20amount=20of=20strategy?= =?UTF-8?q?=20if=20edge=20doesn=E2=80=99t=20have=20any?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- freqtrade/edge/__init__.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/freqtrade/edge/__init__.py b/freqtrade/edge/__init__.py index f27309479..74fabc409 100644 --- a/freqtrade/edge/__init__.py +++ b/freqtrade/edge/__init__.py @@ -161,6 +161,11 @@ class Edge(): return True def stake_amount(self, pair: str, free_capital: float, total_capital: float) -> float: + if pair not in self._cached_pairs: + logger.warning("cannot find %s in calculated pairs, " + "stake_amount of strategy is used instead.", pair) + return self.strategy.stake_amount + stoploss = self._cached_pairs[pair].stoploss available_capital = total_capital * self._capital_percentage allowed_capital_at_risk = round(available_capital * self._allowed_risk, 15) From 12471e012e1dd43155c84c063f33e6b7c4013dd6 Mon Sep 17 00:00:00 2001 From: misagh Date: Fri, 30 Nov 2018 17:59:51 +0100 Subject: [PATCH 515/699] added tests for position sizing --- freqtrade/edge/__init__.py | 2 +- freqtrade/tests/edge/test_edge.py | 40 ++++++++++++++++++++++++++++--- 2 files changed, 38 insertions(+), 4 deletions(-) diff --git a/freqtrade/edge/__init__.py b/freqtrade/edge/__init__.py index 4800691e8..b76821da0 100644 --- a/freqtrade/edge/__init__.py +++ b/freqtrade/edge/__init__.py @@ -164,7 +164,7 @@ class Edge(): if pair not in self._cached_pairs: logger.warning("cannot find %s in calculated pairs, " "stake_amount of strategy is used instead.", pair) - return self.strategy.stake_amount + return self.config['stake_amount'] stoploss = self._cached_pairs[pair].stoploss available_capital = total_capital * self._capital_percentage diff --git a/freqtrade/tests/edge/test_edge.py b/freqtrade/tests/edge/test_edge.py index d5457f342..d453f6f27 100644 --- a/freqtrade/tests/edge/test_edge.py +++ b/freqtrade/tests/edge/test_edge.py @@ -5,6 +5,7 @@ import pytest import logging from freqtrade.tests.conftest import get_patched_freqtradebot from freqtrade.edge import Edge, PairInfo +from freqtrade import constants from pandas import DataFrame, to_datetime from freqtrade.strategy.interface import SellType from freqtrade.tests.optimize import (BTrade, BTContainer, _build_backtest_dataframe, @@ -152,9 +153,9 @@ def test_stoploss(mocker, edge_conf): assert edge.stoploss('E/F') == -0.01 -def test_nonexisting_stoploss(mocker, default_conf): - freqtrade = get_patched_freqtradebot(mocker, default_conf) - edge = Edge(default_conf, freqtrade.exchange, freqtrade.strategy) +def test_nonexisting_stoploss(mocker, edge_conf): + freqtrade = get_patched_freqtradebot(mocker, edge_conf) + edge = Edge(edge_conf, freqtrade.exchange, freqtrade.strategy) mocker.patch('freqtrade.edge.Edge._cached_pairs', mocker.PropertyMock( return_value={ 'E/F': PairInfo(-0.01, 0.66, 3.71, 0.50, 1.71, 10, 60), @@ -164,6 +165,39 @@ def test_nonexisting_stoploss(mocker, default_conf): assert edge.stoploss('N/O') == -0.1 +def test_stake_amount(mocker, edge_conf): + freqtrade = get_patched_freqtradebot(mocker, edge_conf) + edge = Edge(edge_conf, freqtrade.exchange, freqtrade.strategy) + mocker.patch('freqtrade.edge.Edge._cached_pairs', mocker.PropertyMock( + return_value={ + 'E/F': PairInfo(-0.02, 0.66, 3.71, 0.50, 1.71, 10, 60), + } + )) + free = 100 + total = 100 + assert edge.stake_amount('E/F', free, total) == 25 + + free = 20 + total = 100 + assert edge.stake_amount('E/F', free, total) == 20 + + free = 0 + total = 100 + assert edge.stake_amount('E/F', free, total) == 0 + + +def test_nonexisting_stake_amount(mocker, edge_conf): + freqtrade = get_patched_freqtradebot(mocker, edge_conf) + edge = Edge(edge_conf, freqtrade.exchange, freqtrade.strategy) + mocker.patch('freqtrade.edge.Edge._cached_pairs', mocker.PropertyMock( + return_value={ + 'E/F': PairInfo(-0.01, 0.66, 3.71, 0.50, 1.71, 10, 60), + } + )) + + assert edge.stake_amount('N/O', 1, 2) == constants.UNLIMITED_STAKE_AMOUNT + + def _validate_ohlc(buy_ohlc_sell_matrice): for index, ohlc in enumerate(buy_ohlc_sell_matrice): # if not high < open < low or not high < close < low From 11101e66686e858a74b9e405a6bf870e90f582da Mon Sep 17 00:00:00 2001 From: misagh Date: Fri, 30 Nov 2018 18:07:45 +0100 Subject: [PATCH 516/699] config full aded --- config_binance.json.example | 2 +- config_full.json.example | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/config_binance.json.example b/config_binance.json.example index 7773a8c39..3d11f317a 100644 --- a/config_binance.json.example +++ b/config_binance.json.example @@ -59,7 +59,7 @@ "enabled": false, "process_throttle_secs": 3600, "calculate_since_number_of_days": 7, - "total_capital_in_stake_currency": 0.5, + "capital_available_percentage": 0.5, "allowed_risk": 0.01, "stoploss_range_min": -0.01, "stoploss_range_max": -0.1, diff --git a/config_full.json.example b/config_full.json.example index b0719bcc6..7e54eb5ca 100644 --- a/config_full.json.example +++ b/config_full.json.example @@ -67,7 +67,8 @@ "edge": { "enabled": false, "process_throttle_secs": 3600, - "calculate_since_number_of_days": 2, + "calculate_since_number_of_days": 7, + "capital_available_percentage": 0.5, "allowed_risk": 0.01, "stoploss_range_min": -0.01, "stoploss_range_max": -0.1, From aadc9f052ab63d9a168f3518a4891cc4718ea985 Mon Sep 17 00:00:00 2001 From: misagh Date: Fri, 30 Nov 2018 18:10:22 +0100 Subject: [PATCH 517/699] conf schema --- freqtrade/constants.py | 1 + 1 file changed, 1 insertion(+) diff --git a/freqtrade/constants.py b/freqtrade/constants.py index 055fee3b2..82f392990 100644 --- a/freqtrade/constants.py +++ b/freqtrade/constants.py @@ -191,6 +191,7 @@ CONF_SCHEMA = { "process_throttle_secs": {'type': 'integer', 'minimum': 600}, "calculate_since_number_of_days": {'type': 'integer'}, "allowed_risk": {'type': 'number'}, + "capital_available_percentage": {'type': 'number'}, "stoploss_range_min": {'type': 'number'}, "stoploss_range_max": {'type': 'number'}, "stoploss_range_step": {'type': 'number'}, From c61ede4182048611d74436bc0c610cc350ae0a76 Mon Sep 17 00:00:00 2001 From: misagh Date: Fri, 30 Nov 2018 18:20:29 +0100 Subject: [PATCH 518/699] documentation updated --- docs/edge.md | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/docs/edge.md b/docs/edge.md index e5575554b..499b91e7b 100644 --- a/docs/edge.md +++ b/docs/edge.md @@ -82,9 +82,7 @@ Edge dictates the stake amount for each trade to the bot according to the follow Allowed capital at risk is calculated as follows: -**allowed capital at risk** = **total capital** X **allowed risk per trade** - -**total capital** is your stake amount. +**allowed capital at risk** = **capital_available_percentage** X **allowed risk per trade** **Stoploss** is calculated as described above against historical data. @@ -93,7 +91,7 @@ Your position size then will be: **position size** = **allowed capital at risk** / **stoploss** Example: -Let's say your stake amount is 3 ETH, you would allow 1% of risk for each trade. thus your allowed capital at risk would be **3 x 0.01 = 0.03 ETH**. Let's assume Edge has calculated that for **XLM/ETH** market your stoploss should be at 2%. So your position size will be **0.03 / 0.02= 1.5ETH**.
+Let's say the stake currency is ETH and you have 10 ETH on the exchange, your **capital_available_percentage** is 50% and you would allow 1% of risk for each trade. thus your available capital for trading is **10 x 0.5 = 5 ETH** and allowed capital at risk would be **5 x 0.01 = 0.05 ETH**. Let's assume Edge has calculated that for **XLM/ETH** market your stoploss should be at 2%. So your position size will be **0.05 / 0.02= 2.5ETH**.
## Configurations Edge has following configurations: @@ -111,6 +109,11 @@ Number of days of data against which Edge calculates Win Rate, Risk Reward and E Note that it downloads historical data so increasing this number would lead to slowing down the bot
(default to 7) +#### capital_available_percentage +This is the percentage of the total capital on exchange in stake currency.
+As an example if you have 100 USDT available in your wallet on the exchange and this value is 0.5 (which is 50%), then the bot will use a maximum amount of 50 USDT for trading and considers it as available capital. +(default to 0.5) + #### allowed_risk Percentage of allowed risk per trade
(default to 0.01 [1%]) From 7e86ec31be1e58ed93808ba660bc670ddc947859 Mon Sep 17 00:00:00 2001 From: misagh Date: Fri, 30 Nov 2018 18:23:16 +0100 Subject: [PATCH 519/699] tests added for wallet additional functions --- freqtrade/tests/test_wallets.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/freqtrade/tests/test_wallets.py b/freqtrade/tests/test_wallets.py index 88366a869..8d9adc74c 100644 --- a/freqtrade/tests/test_wallets.py +++ b/freqtrade/tests/test_wallets.py @@ -58,6 +58,8 @@ def test_sync_wallet_at_boot(mocker, default_conf): assert freqtrade.wallets.wallets['GAS'].used == 0.1 assert freqtrade.wallets.wallets['GAS'].total == 0.260439 assert freqtrade.wallets.get_free('GAS') == 0.270739 + assert freqtrade.wallets.get_used('GAS') == 0.1 + assert freqtrade.wallets.get_total('GAS') == 0.260439 def test_sync_wallet_missing_data(mocker, default_conf): From 4a2d60370c652884407c0ce66ce90c2021a73d9a Mon Sep 17 00:00:00 2001 From: misagh Date: Fri, 30 Nov 2018 18:28:18 +0100 Subject: [PATCH 520/699] adding dots at the end of sentences --- docs/edge.md | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/docs/edge.md b/docs/edge.md index 499b91e7b..27afcd507 100644 --- a/docs/edge.md +++ b/docs/edge.md @@ -97,7 +97,7 @@ Let's say the stake currency is ETH and you have 10 ETH on the exchange, your ** Edge has following configurations: #### enabled -If true, then Edge will run periodically
+If true, then Edge will run periodically.
(default to false) #### process_throttle_secs @@ -106,24 +106,24 @@ How often should Edge run in seconds?
#### calculate_since_number_of_days Number of days of data against which Edge calculates Win Rate, Risk Reward and Expectancy -Note that it downloads historical data so increasing this number would lead to slowing down the bot
+Note that it downloads historical data so increasing this number would lead to slowing down the bot.
(default to 7) #### capital_available_percentage This is the percentage of the total capital on exchange in stake currency.
-As an example if you have 100 USDT available in your wallet on the exchange and this value is 0.5 (which is 50%), then the bot will use a maximum amount of 50 USDT for trading and considers it as available capital. +As an example if you have 100 USDT available in your wallet on the exchange and this value is 0.5 (which is 50%), then the bot will use a maximum amount of 50 USDT for trading and considers it as available capital.
(default to 0.5) #### allowed_risk -Percentage of allowed risk per trade
+Percentage of allowed risk per trade.
(default to 0.01 [1%]) #### stoploss_range_min -Minimum stoploss
+Minimum stoploss.
(default to -0.01) #### stoploss_range_max -Maximum stoploss
+Maximum stoploss.
(default to -0.10) #### stoploss_range_step From f04655c012c90ceffb01a562e80a8d8c72f6e10c Mon Sep 17 00:00:00 2001 From: Matthias Date: Fri, 30 Nov 2018 20:13:50 +0100 Subject: [PATCH 521/699] Test exceptions in sell-stoploss --- freqtrade/tests/exchange/test_exchange.py | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/freqtrade/tests/exchange/test_exchange.py b/freqtrade/tests/exchange/test_exchange.py index ec7c2acae..d1f391266 100644 --- a/freqtrade/tests/exchange/test_exchange.py +++ b/freqtrade/tests/exchange/test_exchange.py @@ -1182,6 +1182,27 @@ def test_stoploss_limit_order(default_conf, mocker): assert api_mock.create_order.call_args[0][4] == 200 assert api_mock.create_order.call_args[0][5] == {'stopPrice': 220} + # test exception handling + with pytest.raises(DependencyException): + api_mock.create_order = MagicMock(side_effect=ccxt.InsufficientFunds) + exchange = get_patched_exchange(mocker, default_conf, api_mock) + exchange.stoploss_limit(pair='ETH/BTC', amount=1, stop_price=220, rate=200) + + with pytest.raises(DependencyException): + api_mock.create_order = MagicMock(side_effect=ccxt.InvalidOrder) + exchange = get_patched_exchange(mocker, default_conf, api_mock) + exchange.stoploss_limit(pair='ETH/BTC', amount=1, stop_price=220, rate=200) + + with pytest.raises(TemporaryError): + api_mock.create_order = MagicMock(side_effect=ccxt.NetworkError) + exchange = get_patched_exchange(mocker, default_conf, api_mock) + exchange.stoploss_limit(pair='ETH/BTC', amount=1, stop_price=220, rate=200) + + with pytest.raises(OperationalException): + api_mock.create_order = MagicMock(side_effect=ccxt.BaseError) + exchange = get_patched_exchange(mocker, default_conf, api_mock) + exchange.stoploss_limit(pair='ETH/BTC', amount=1, stop_price=220, rate=200) + def test_stoploss_limit_order_dry_run(default_conf, mocker): api_mock = MagicMock() From d4f83a7516c916ea65e2013f632a75f0686c56c4 Mon Sep 17 00:00:00 2001 From: Matthias Date: Fri, 30 Nov 2018 20:15:56 +0100 Subject: [PATCH 522/699] Fix missing mock in test_add_stoploss_on_exchange --- freqtrade/tests/test_freqtradebot.py | 1 + 1 file changed, 1 insertion(+) diff --git a/freqtrade/tests/test_freqtradebot.py b/freqtrade/tests/test_freqtradebot.py index 147d47b72..a3638d08a 100644 --- a/freqtrade/tests/test_freqtradebot.py +++ b/freqtrade/tests/test_freqtradebot.py @@ -876,6 +876,7 @@ def test_execute_buy(mocker, default_conf, fee, markets, limit_buy_order) -> Non def test_add_stoploss_on_exchange(mocker, default_conf, limit_buy_order) -> None: patch_RPCManager(mocker) + patch_exchange(mocker) mocker.patch('freqtrade.freqtradebot.FreqtradeBot.handle_trade', MagicMock(return_value=True)) mocker.patch('freqtrade.exchange.Exchange.get_order', return_value=limit_buy_order) mocker.patch('freqtrade.exchange.Exchange.get_trades_for_order', return_value=[]) From 24f573f3b01e3e59eafc856b03dbf57d38d61a9a Mon Sep 17 00:00:00 2001 From: Misagh Date: Sat, 1 Dec 2018 10:01:11 +0100 Subject: [PATCH 523/699] log "Found no sell signal for whitelisted ..." changed (#1378) * sell log enriched and put modify on debug --- freqtrade/freqtradebot.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index 62b0a0d2c..d85a533b7 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -629,7 +629,7 @@ class FreqtradeBot(object): if self.check_sell(trade, sell_rate, buy, sell): return True - logger.info('Found no sell signals for whitelisted currencies. Trying again..') + logger.debug('Found no sell signal for %s.', trade) return False def handle_stoploss_on_exchange(self, trade: Trade) -> bool: From bf990ec59954a722c0803dee02071d8749dcc586 Mon Sep 17 00:00:00 2001 From: misagh Date: Sat, 1 Dec 2018 10:50:41 +0100 Subject: [PATCH 524/699] test fixed and flake --- freqtrade/freqtradebot.py | 7 +++++-- freqtrade/tests/test_freqtradebot.py | 7 ++++--- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index 53477f0ca..a73a2e98f 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -798,8 +798,11 @@ class FreqtradeBot(object): if sell_reason in (SellType.STOP_LOSS, SellType.TRAILING_STOP_LOSS): sell_type = 'stoploss' - if self.config.get('dry_run', False) and sell_type == 'stoploss' and self.strategy.order_types['stoploss_on_exchange']: - limit = trade.stop_loss + # if stoploss is on exchange and we are on dry_run mode, + # we consider the sell price stop price + if self.config.get('dry_run', False) and sell_type == 'stoploss' \ + and self.strategy.order_types['stoploss_on_exchange']: + limit = trade.stop_loss # First cancelling stoploss on exchange ... if self.strategy.order_types.get('stoploss_on_exchange') and trade.stoploss_order_id: diff --git a/freqtrade/tests/test_freqtradebot.py b/freqtrade/tests/test_freqtradebot.py index fb42dd518..912adacaf 100644 --- a/freqtrade/tests/test_freqtradebot.py +++ b/freqtrade/tests/test_freqtradebot.py @@ -1516,8 +1516,9 @@ def test_execute_sell_up(default_conf, ticker, fee, ticker_sell_up, markets, moc } == last_msg -def test_execute_sell_down_live(default_conf, ticker, fee, - ticker_sell_down, markets, mocker) -> None: +def test_execute_sell_down(default_conf, ticker, fee, + ticker_sell_down, markets, mocker) -> None: + rpc_mock = patch_RPCManager(mocker) mocker.patch.multiple( 'freqtrade.exchange.Exchange', @@ -1540,7 +1541,7 @@ def test_execute_sell_down_live(default_conf, ticker, fee, 'freqtrade.exchange.Exchange', get_ticker=ticker_sell_down ) - default_conf['dry_run'] = False + freqtrade.execute_sell(trade=trade, limit=ticker_sell_down()['bid'], sell_reason=SellType.STOP_LOSS) From 042e631f87720304b71237a03a75d9d74546465d Mon Sep 17 00:00:00 2001 From: misagh Date: Sat, 1 Dec 2018 10:52:36 +0100 Subject: [PATCH 525/699] rollback on futile change --- freqtrade/tests/test_freqtradebot.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/freqtrade/tests/test_freqtradebot.py b/freqtrade/tests/test_freqtradebot.py index 912adacaf..bd0651ffc 100644 --- a/freqtrade/tests/test_freqtradebot.py +++ b/freqtrade/tests/test_freqtradebot.py @@ -1516,8 +1516,7 @@ def test_execute_sell_up(default_conf, ticker, fee, ticker_sell_up, markets, moc } == last_msg -def test_execute_sell_down(default_conf, ticker, fee, - ticker_sell_down, markets, mocker) -> None: +def test_execute_sell_down(default_conf, ticker, fee, ticker_sell_down, markets, mocker) -> None: rpc_mock = patch_RPCManager(mocker) mocker.patch.multiple( From b1c81acfcb08ddbc3b8774143e99afa42a32fd6b Mon Sep 17 00:00:00 2001 From: misagh Date: Sat, 1 Dec 2018 10:53:21 +0100 Subject: [PATCH 526/699] another futile one --- freqtrade/tests/test_freqtradebot.py | 1 - 1 file changed, 1 deletion(-) diff --git a/freqtrade/tests/test_freqtradebot.py b/freqtrade/tests/test_freqtradebot.py index bd0651ffc..a9b3ffc5d 100644 --- a/freqtrade/tests/test_freqtradebot.py +++ b/freqtrade/tests/test_freqtradebot.py @@ -1517,7 +1517,6 @@ def test_execute_sell_up(default_conf, ticker, fee, ticker_sell_up, markets, moc def test_execute_sell_down(default_conf, ticker, fee, ticker_sell_down, markets, mocker) -> None: - rpc_mock = patch_RPCManager(mocker) mocker.patch.multiple( 'freqtrade.exchange.Exchange', From 9c987fdedd3b22794027af01d2ad01df22c71f54 Mon Sep 17 00:00:00 2001 From: misagh Date: Sat, 1 Dec 2018 10:56:33 +0100 Subject: [PATCH 527/699] variable name changed (_final_pairs) --- freqtrade/edge/__init__.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/freqtrade/edge/__init__.py b/freqtrade/edge/__init__.py index b76821da0..4c7ac7950 100644 --- a/freqtrade/edge/__init__.py +++ b/freqtrade/edge/__init__.py @@ -53,7 +53,7 @@ class Edge(): self.edge_config = self.config.get('edge', {}) self._cached_pairs: Dict[str, Any] = {} # Keeps a list of pairs - self._final: list = [] + self._final_pairs: list = [] # checking max_open_trades. it should be -1 as with Edge # the number of trades is determined by position size @@ -199,14 +199,14 @@ class Edge(): pair in pairs: final.append(pair) - if self._final != final: - self._final = final - if self._final: - logger.info('Edge validated only %s', self._final) + if self._final_pairs != final: + self._final_pairs = final + if self._final_pairs: + logger.info('Edge validated only %s', self._final_pairs) else: logger.info('Edge removed all pairs as no pair with minimum expectancy was found !') - return self._final + return self._final_pairs def _fill_calculable_fields(self, result: DataFrame) -> DataFrame: """ From 86d9457ea13f1f802646400ac7dbc991637d9daf Mon Sep 17 00:00:00 2001 From: misagh Date: Sat, 1 Dec 2018 10:58:05 +0100 Subject: [PATCH 528/699] removing unnecessary variable before returning the result --- freqtrade/freqtradebot.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index 66e32ec4b..e687b4a8d 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -333,12 +333,11 @@ class FreqtradeBot(object): :return: float: Stake Amount """ if self.edge: - stake_amount = self.edge.stake_amount( + return self.edge.stake_amount( pair, self.wallets.get_free(self.config['stake_currency']), self.wallets.get_total(self.config['stake_currency']) ) - return stake_amount else: stake_amount = self.config['stake_amount'] From c4f17f1c4588a61cbc4e15d4ea1a6d7d66a6fefa Mon Sep 17 00:00:00 2001 From: misagh Date: Sat, 1 Dec 2018 10:58:47 +0100 Subject: [PATCH 529/699] config json updated --- config.json.example | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config.json.example b/config.json.example index bbd9648da..323ff711e 100644 --- a/config.json.example +++ b/config.json.example @@ -57,7 +57,7 @@ "enabled": false, "process_throttle_secs": 3600, "calculate_since_number_of_days": 7, - "total_capital_in_stake_currency": 0.5, + "capital_available_percentage": 0.5, "allowed_risk": 0.01, "stoploss_range_min": -0.01, "stoploss_range_max": -0.1, From 9c0be99ff7b4e99bbe88d2b0cee3fb1ce1d3d0e8 Mon Sep 17 00:00:00 2001 From: misagh Date: Sat, 1 Dec 2018 11:00:33 +0100 Subject: [PATCH 530/699] rounding float at the end --- freqtrade/edge/__init__.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/freqtrade/edge/__init__.py b/freqtrade/edge/__init__.py index 4c7ac7950..a7bf2199a 100644 --- a/freqtrade/edge/__init__.py +++ b/freqtrade/edge/__init__.py @@ -168,8 +168,8 @@ class Edge(): stoploss = self._cached_pairs[pair].stoploss available_capital = total_capital * self._capital_percentage - allowed_capital_at_risk = round(available_capital * self._allowed_risk, 15) - max_position_size = abs(round((allowed_capital_at_risk / stoploss), 15)) + allowed_capital_at_risk = available_capital * self._allowed_risk + max_position_size = abs(allowed_capital_at_risk / stoploss) position_size = min(max_position_size, free_capital) logger.info( 'winrate: %s, expectancy: %s, position size: %s, pair: %s,' @@ -178,7 +178,7 @@ class Edge(): self._cached_pairs[pair].expectancy, position_size, pair, stoploss, available_capital ) - return position_size + return round(position_size, 15) def stoploss(self, pair: str) -> float: if pair in self._cached_pairs: From 88d277ea557ba70156ae30e2ebc71817a6bbd22d Mon Sep 17 00:00:00 2001 From: misagh Date: Sat, 1 Dec 2018 11:08:18 +0100 Subject: [PATCH 531/699] adding required config for edge --- freqtrade/constants.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/freqtrade/constants.py b/freqtrade/constants.py index 82f392990..cd22815e9 100644 --- a/freqtrade/constants.py +++ b/freqtrade/constants.py @@ -200,7 +200,8 @@ CONF_SCHEMA = { "min_trade_number": {'type': 'number'}, "max_trade_duration_minute": {'type': 'integer'}, "remove_pumps": {'type': 'boolean'} - } + }, + 'required': ['process_throttle_secs', 'allowed_risk', 'capital_available_percentage'] } }, 'anyOf': [ From 4431e3bdb61bb7af1fa7db61d4d9e9f634d06b46 Mon Sep 17 00:00:00 2001 From: misagh Date: Sat, 1 Dec 2018 11:40:13 +0100 Subject: [PATCH 532/699] position size explanation enriched --- docs/edge.md | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/docs/edge.md b/docs/edge.md index 27afcd507..7ecd4e75b 100644 --- a/docs/edge.md +++ b/docs/edge.md @@ -90,8 +90,14 @@ Your position size then will be: **position size** = **allowed capital at risk** / **stoploss** -Example: -Let's say the stake currency is ETH and you have 10 ETH on the exchange, your **capital_available_percentage** is 50% and you would allow 1% of risk for each trade. thus your available capital for trading is **10 x 0.5 = 5 ETH** and allowed capital at risk would be **5 x 0.01 = 0.05 ETH**. Let's assume Edge has calculated that for **XLM/ETH** market your stoploss should be at 2%. So your position size will be **0.05 / 0.02= 2.5ETH**.
+Example:
+Let's say the stake currency is ETH and you have 10 ETH on the exchange, your **capital_available_percentage** is 50% and you would allow 1% of risk for each trade. thus your available capital for trading is **10 x 0.5 = 5 ETH** and allowed capital at risk would be **5 x 0.01 = 0.05 ETH**.
+Let's assume Edge has calculated that for **XLM/ETH** market your stoploss should be at 2%. So your position size will be **0.05 / 0.02 = 2.5ETH**.
+Bot takes a position of 2.5ETH on XLM/ETH (call it trade 1). Up next, you receive another buy signal while trade 1 is still open. This time on BTC/ETH market. Edge calculated stoploss for this market at 4%. So your position size would be 0.05 / 0.04 = 1.25ETH (call it trade 2).
+Note that available capital for trading didn’t change for trade 2 even if you had already trade 1. The available capital doesn’t mean the free amount on your wallet.
+Now you have two trades open. The Bot receives yet another buy signal for another market: **ADA/ETH**. This time the stoploss is calculated at 1%. So your position size is **0.05 / 0.01 = 5ETH**. But there are already 4ETH blocked in two previous trades. So the position size for this third trade would be 1ETH.
+Available capital doesn’t change before a position is sold. Let’s assume that trade 1 receives a sell signal and it is sold with a profit of 1ETH. Your total capital on exchange would be 11 ETH and the available capital for trading becomes 5.5ETH.
+So the Bot receives another buy signal for trade 4 with a stoploss at 2% then your position size would be 0.055 / 0.02 = 2.75 ## Configurations Edge has following configurations: From ee62adf4f7c4d2d323dd4740c2e2fa765d7846b7 Mon Sep 17 00:00:00 2001 From: misagh Date: Sat, 1 Dec 2018 11:41:30 +0100 Subject: [PATCH 533/699] highlight --- docs/edge.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/edge.md b/docs/edge.md index 7ecd4e75b..5082b79d2 100644 --- a/docs/edge.md +++ b/docs/edge.md @@ -97,7 +97,7 @@ Bot takes a position of 2.5ETH on XLM/ETH (call it trade 1). Up next, you receiv Note that available capital for trading didn’t change for trade 2 even if you had already trade 1. The available capital doesn’t mean the free amount on your wallet.
Now you have two trades open. The Bot receives yet another buy signal for another market: **ADA/ETH**. This time the stoploss is calculated at 1%. So your position size is **0.05 / 0.01 = 5ETH**. But there are already 4ETH blocked in two previous trades. So the position size for this third trade would be 1ETH.
Available capital doesn’t change before a position is sold. Let’s assume that trade 1 receives a sell signal and it is sold with a profit of 1ETH. Your total capital on exchange would be 11 ETH and the available capital for trading becomes 5.5ETH.
-So the Bot receives another buy signal for trade 4 with a stoploss at 2% then your position size would be 0.055 / 0.02 = 2.75 +So the Bot receives another buy signal for trade 4 with a stoploss at 2% then your position size would be **0.055 / 0.02 = 2.75**. ## Configurations Edge has following configurations: From 1d41a917887b75188c2a31211346e1b778338318 Mon Sep 17 00:00:00 2001 From: misagh Date: Sat, 1 Dec 2018 11:48:41 +0100 Subject: [PATCH 534/699] =?UTF-8?q?stake=5Famount=20in=20case=20it=20doesn?= =?UTF-8?q?=E2=80=99t=20exist?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- freqtrade/edge/__init__.py | 22 +++++++++------------- 1 file changed, 9 insertions(+), 13 deletions(-) diff --git a/freqtrade/edge/__init__.py b/freqtrade/edge/__init__.py index a7bf2199a..e5f913e2d 100644 --- a/freqtrade/edge/__init__.py +++ b/freqtrade/edge/__init__.py @@ -161,23 +161,19 @@ class Edge(): return True def stake_amount(self, pair: str, free_capital: float, total_capital: float) -> float: - if pair not in self._cached_pairs: - logger.warning("cannot find %s in calculated pairs, " - "stake_amount of strategy is used instead.", pair) - return self.config['stake_amount'] - - stoploss = self._cached_pairs[pair].stoploss + stoploss = self.stoploss(pair) available_capital = total_capital * self._capital_percentage allowed_capital_at_risk = available_capital * self._allowed_risk max_position_size = abs(allowed_capital_at_risk / stoploss) position_size = min(max_position_size, free_capital) - logger.info( - 'winrate: %s, expectancy: %s, position size: %s, pair: %s,' - ' stoploss: %s, available capital: %s.', - self._cached_pairs[pair].winrate, - self._cached_pairs[pair].expectancy, - position_size, pair, stoploss, available_capital - ) + if pair in self._cached_pairs: + logger.info( + 'winrate: %s, expectancy: %s, position size: %s, pair: %s,' + ' stoploss: %s, available capital: %s.', + self._cached_pairs[pair].winrate, + self._cached_pairs[pair].expectancy, + position_size, pair, stoploss, available_capital + ) return round(position_size, 15) def stoploss(self, pair: str) -> float: From 33f1cc13b3a3db99287c805d4293c657dc4ee763 Mon Sep 17 00:00:00 2001 From: misagh Date: Sat, 1 Dec 2018 11:56:16 +0100 Subject: [PATCH 535/699] fixing tests --- freqtrade/tests/conftest.py | 1 - freqtrade/tests/edge/test_edge.py | 5 ++--- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/freqtrade/tests/conftest.py b/freqtrade/tests/conftest.py index 75c8032f1..29297ac61 100644 --- a/freqtrade/tests/conftest.py +++ b/freqtrade/tests/conftest.py @@ -63,7 +63,6 @@ def patch_edge(mocker) -> None: 'LTC/BTC': PairInfo(-0.21, 0.66, 3.71, 0.50, 1.71, 11, 20), } )) - mocker.patch('freqtrade.edge.Edge.stoploss', MagicMock(return_value=-0.20)) mocker.patch('freqtrade.edge.Edge.calculate', MagicMock(return_value=True)) diff --git a/freqtrade/tests/edge/test_edge.py b/freqtrade/tests/edge/test_edge.py index d453f6f27..4b63ceb1a 100644 --- a/freqtrade/tests/edge/test_edge.py +++ b/freqtrade/tests/edge/test_edge.py @@ -191,11 +191,10 @@ def test_nonexisting_stake_amount(mocker, edge_conf): edge = Edge(edge_conf, freqtrade.exchange, freqtrade.strategy) mocker.patch('freqtrade.edge.Edge._cached_pairs', mocker.PropertyMock( return_value={ - 'E/F': PairInfo(-0.01, 0.66, 3.71, 0.50, 1.71, 10, 60), + 'E/F': PairInfo(-0.11, 0.66, 3.71, 0.50, 1.71, 10, 60), } )) - - assert edge.stake_amount('N/O', 1, 2) == constants.UNLIMITED_STAKE_AMOUNT + assert edge.stake_amount('N/O', 1, 2) == 0.1 def _validate_ohlc(buy_ohlc_sell_matrice): From bd673178cec320cc6343e84f1586c2cc0f0360f2 Mon Sep 17 00:00:00 2001 From: misagh Date: Sat, 1 Dec 2018 11:56:53 +0100 Subject: [PATCH 536/699] constants removed --- freqtrade/tests/edge/test_edge.py | 1 - 1 file changed, 1 deletion(-) diff --git a/freqtrade/tests/edge/test_edge.py b/freqtrade/tests/edge/test_edge.py index 4b63ceb1a..74227cfe8 100644 --- a/freqtrade/tests/edge/test_edge.py +++ b/freqtrade/tests/edge/test_edge.py @@ -5,7 +5,6 @@ import pytest import logging from freqtrade.tests.conftest import get_patched_freqtradebot from freqtrade.edge import Edge, PairInfo -from freqtrade import constants from pandas import DataFrame, to_datetime from freqtrade.strategy.interface import SellType from freqtrade.tests.optimize import (BTrade, BTContainer, _build_backtest_dataframe, From 7ddbaa70ad8a9f5643dbec74eac7065353838348 Mon Sep 17 00:00:00 2001 From: misagh Date: Sat, 1 Dec 2018 12:06:48 +0100 Subject: [PATCH 537/699] USDT to ETH conversion. 1 USDT = 1 ETH --- docs/edge.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/edge.md b/docs/edge.md index 5082b79d2..829910484 100644 --- a/docs/edge.md +++ b/docs/edge.md @@ -117,7 +117,7 @@ Note that it downloads historical data so increasing this number would lead to s #### capital_available_percentage This is the percentage of the total capital on exchange in stake currency.
-As an example if you have 100 USDT available in your wallet on the exchange and this value is 0.5 (which is 50%), then the bot will use a maximum amount of 50 USDT for trading and considers it as available capital.
+As an example if you have 10 ETH available in your wallet on the exchange and this value is 0.5 (which is 50%), then the bot will use a maximum amount of 5 ETH for trading and considers it as available capital.
(default to 0.5) #### allowed_risk From 2d17346b0eec75df2597671c69e4e822eb8f16df Mon Sep 17 00:00:00 2001 From: misagh Date: Sat, 1 Dec 2018 13:01:51 +0100 Subject: [PATCH 538/699] explaining arbitrary stake amount in comment --- freqtrade/edge/__init__.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/freqtrade/edge/__init__.py b/freqtrade/edge/__init__.py index e5f913e2d..4dfcfc8ba 100644 --- a/freqtrade/edge/__init__.py +++ b/freqtrade/edge/__init__.py @@ -223,9 +223,13 @@ class Edge(): # 0.05% is 0.0005 # fee = 0.001 + # we set stake amount to an arbitraty amount. + # as it doesn't change the calculation. + # all returned values are relative. they are percentages. stake = 0.015 - fee = self.fee + + fee = self.fee open_fee = fee / 2 close_fee = fee / 2 From a5414b843703adf51eadbdba54f03397a91e4bf9 Mon Sep 17 00:00:00 2001 From: misagh Date: Sat, 1 Dec 2018 13:02:45 +0100 Subject: [PATCH 539/699] flake8 --- freqtrade/edge/__init__.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/freqtrade/edge/__init__.py b/freqtrade/edge/__init__.py index 4dfcfc8ba..a212c9849 100644 --- a/freqtrade/edge/__init__.py +++ b/freqtrade/edge/__init__.py @@ -223,12 +223,10 @@ class Edge(): # 0.05% is 0.0005 # fee = 0.001 - # we set stake amount to an arbitraty amount. + # we set stake amount to an arbitrary amount. # as it doesn't change the calculation. # all returned values are relative. they are percentages. stake = 0.015 - - fee = self.fee open_fee = fee / 2 close_fee = fee / 2 From b594bc7cccaf9c9964fdde218aa1a708a79736f1 Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Sat, 1 Dec 2018 13:34:06 +0100 Subject: [PATCH 540/699] Update ccxt from 1.17.563 to 1.17.566 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index f7db2ea09..e16ca8d37 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -ccxt==1.17.563 +ccxt==1.17.566 SQLAlchemy==1.2.14 python-telegram-bot==11.1.0 arrow==0.12.1 From 35d678c5058a0277ad224ac29696bab81f5d2247 Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Sun, 2 Dec 2018 13:34:06 +0100 Subject: [PATCH 541/699] Update ccxt from 1.17.566 to 1.17.572 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index e16ca8d37..cee3e19be 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -ccxt==1.17.566 +ccxt==1.17.572 SQLAlchemy==1.2.14 python-telegram-bot==11.1.0 arrow==0.12.1 From bb828c308f9e881d25614553bde6021ae84e19ab Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 2 Dec 2018 16:03:34 +0100 Subject: [PATCH 542/699] Remove unnecessary test-file --- freqtrade/tests/optimize/test_optimize.py | 3 +- freqtrade/tests/strategy/test_interface.py | 8 ++++-- freqtrade/tests/test_dataframe.py | 32 ---------------------- 3 files changed, 8 insertions(+), 35 deletions(-) delete mode 100644 freqtrade/tests/test_dataframe.py diff --git a/freqtrade/tests/optimize/test_optimize.py b/freqtrade/tests/optimize/test_optimize.py index b58c92d5c..d73f31ad5 100644 --- a/freqtrade/tests/optimize/test_optimize.py +++ b/freqtrade/tests/optimize/test_optimize.py @@ -57,7 +57,8 @@ def test_load_data_30min_ticker(ticker_history, mocker, caplog, default_conf) -> mocker.patch('freqtrade.exchange.Exchange.get_history', return_value=ticker_history) file = os.path.join(os.path.dirname(__file__), '..', 'testdata', 'UNITTEST_BTC-30m.json') _backup_file(file, copy_file=True) - optimize.load_data(None, pairs=['UNITTEST/BTC'], ticker_interval='30m') + ld = optimize.load_data(None, pairs=['UNITTEST/BTC'], ticker_interval='30m') + assert isinstance(ld, dict) assert os.path.isfile(file) is True assert not log_has('Download the pair: "UNITTEST/BTC", Interval: 30m', caplog.record_tuples) _clean_test_file(file) diff --git a/freqtrade/tests/strategy/test_interface.py b/freqtrade/tests/strategy/test_interface.py index fedd355af..79c485590 100644 --- a/freqtrade/tests/strategy/test_interface.py +++ b/freqtrade/tests/strategy/test_interface.py @@ -179,6 +179,10 @@ def test_analyze_ticker_skip_analyze(ticker_history, mocker, caplog) -> None: strategy.process_only_new_candles = True ret = strategy.analyze_ticker(ticker_history, {'pair': 'ETH/BTC'}) + assert 'high' in ret.columns + assert 'low' in ret.columns + assert 'close' in ret.columns + assert isinstance(ret, DataFrame) assert ind_mock.call_count == 1 assert buy_mock.call_count == 1 assert buy_mock.call_count == 1 @@ -193,8 +197,8 @@ def test_analyze_ticker_skip_analyze(ticker_history, mocker, caplog) -> None: assert buy_mock.call_count == 1 assert buy_mock.call_count == 1 # only skipped analyze adds buy and sell columns, otherwise it's all mocked - assert 'buy' in ret - assert 'sell' in ret + assert 'buy' in ret.columns + assert 'sell' in ret.columns assert ret['buy'].sum() == 0 assert ret['sell'].sum() == 0 assert not log_has('TA Analysis Launched', caplog.record_tuples) diff --git a/freqtrade/tests/test_dataframe.py b/freqtrade/tests/test_dataframe.py deleted file mode 100644 index 6afb83a3f..000000000 --- a/freqtrade/tests/test_dataframe.py +++ /dev/null @@ -1,32 +0,0 @@ -# pragma pylint: disable=missing-docstring, C0103 - -import pandas - -from freqtrade.optimize import load_data -from freqtrade.resolvers import StrategyResolver - -_pairs = ['ETH/BTC'] - - -def load_dataframe_pair(pairs, strategy): - ld = load_data(None, ticker_interval='5m', pairs=pairs) - assert isinstance(ld, dict) - assert isinstance(pairs[0], str) - dataframe = ld[pairs[0]] - - dataframe = strategy.analyze_ticker(dataframe, {'pair': pairs[0]}) - return dataframe - - -def test_dataframe_load(): - strategy = StrategyResolver({'strategy': 'DefaultStrategy'}).strategy - dataframe = load_dataframe_pair(_pairs, strategy) - assert isinstance(dataframe, pandas.core.frame.DataFrame) - - -def test_dataframe_columns_exists(): - strategy = StrategyResolver({'strategy': 'DefaultStrategy'}).strategy - dataframe = load_dataframe_pair(_pairs, strategy) - assert 'high' in dataframe.columns - assert 'low' in dataframe.columns - assert 'close' in dataframe.columns From e8fbe77ebc4cdcd2ab9e490fb69715b30993982f Mon Sep 17 00:00:00 2001 From: Matthias Date: Fri, 30 Nov 2018 06:34:56 +0100 Subject: [PATCH 543/699] Refactor static whitelist to module --- freqtrade/freqtradebot.py | 47 ++++----------------- freqtrade/pairlist/StaticList.py | 70 ++++++++++++++++++++++++++++++++ freqtrade/pairlist/__init__.py | 0 3 files changed, 79 insertions(+), 38 deletions(-) create mode 100644 freqtrade/pairlist/StaticList.py create mode 100644 freqtrade/pairlist/__init__.py diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index a73a2e98f..3e8874bb6 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -25,6 +25,7 @@ from freqtrade.resolvers import StrategyResolver from freqtrade.state import State from freqtrade.strategy.interface import SellType, IStrategy from freqtrade.exchange.exchange_helpers import order_book_to_dataframe +from freqtrade.pairlist.StaticList import StaticList logger = logging.getLogger(__name__) @@ -59,6 +60,7 @@ class FreqtradeBot(object): self.persistence = None self.exchange = Exchange(self.config) self.wallets = Wallets(self.exchange) + self.pairlists = StaticList(self, self.config) # Initializing Edge only if enabled self.edge = Edge(self.config, self.exchange, self.strategy) if \ @@ -148,11 +150,13 @@ class FreqtradeBot(object): try: nb_assets = self.config.get('dynamic_whitelist', None) # Refresh whitelist based on wallet maintenance - sanitized_list = self._refresh_whitelist( - self._gen_pair_whitelist( - self.config['stake_currency'] - ) if nb_assets else self.config['exchange']['pair_whitelist'] - ) + self.pairlists.refresh_whitelist() + sanitized_list = self.pairlists.whitelist + # sanitized_list = self._refresh_whitelist( + # self._gen_pair_whitelist( + # self.config['stake_currency'] + # ) if nb_assets else self.lists.get_whitelist() + # ) # Keep only the subsets of pairs wanted (up to nb_assets) self.active_pair_whitelist = sanitized_list[:nb_assets] if nb_assets else sanitized_list @@ -227,39 +231,6 @@ class FreqtradeBot(object): pairs = [s['symbol'] for s in sorted_tickers] return pairs - def _refresh_whitelist(self, whitelist: List[str]) -> List[str]: - """ - Check available markets and remove pair from whitelist if necessary - :param whitelist: the sorted list (based on BaseVolume) of pairs the user might want to - trade - :return: the list of pairs the user wants to trade without the one unavailable or - black_listed - """ - sanitized_whitelist = whitelist - markets = self.exchange.get_markets() - - markets = [m for m in markets if m['quote'] == self.config['stake_currency']] - known_pairs = set() - for market in markets: - pair = market['symbol'] - # pair is not int the generated dynamic market, or in the blacklist ... ignore it - if pair not in whitelist or pair in self.config['exchange'].get('pair_blacklist', []): - continue - # else the pair is valid - known_pairs.add(pair) - # Market is not active - if not market['active']: - sanitized_whitelist.remove(pair) - logger.info( - 'Ignoring %s from whitelist. Market is not active.', - pair - ) - - # We need to remove pairs that are unknown - final_list = [x for x in sanitized_whitelist if x in known_pairs] - - return final_list - def get_target_bid(self, pair: str, ticker: Dict[str, float]) -> float: """ Calculates bid target between current ask price and last price diff --git a/freqtrade/pairlist/StaticList.py b/freqtrade/pairlist/StaticList.py new file mode 100644 index 000000000..ea4a9ac98 --- /dev/null +++ b/freqtrade/pairlist/StaticList.py @@ -0,0 +1,70 @@ +""" +Static List provider + +Provides lists as configured in config.json + + """ +import logging +from typing import List, Optional + +logger = logging.getLogger(__name__) + + +class StaticList(object): + + def __init__(self, freqtrade, config: dict) -> None: + self._freqtrade = freqtrade + self._config = config + self._whitelist = self._config['exchange']['pair_whitelist'] + self._blacklist = self._config['exchange'].get('pair_blacklist', []) + self.refresh_whitelist() + + @property + def whitelist(self) -> List[str]: + """ Contains the current whitelist """ + return self._whitelist + + @property + def blacklist(self) -> List[str]: + return self._blacklist + + def refresh_whitelist(self) -> bool: + """ + Refreshes whitelist. + """ + return self.validate_whitelist(self._config['exchange']['pair_whitelist']) + + def validate_whitelist(self, whitelist: List[str]) -> bool: + """ + Check available markets and remove pair from whitelist if necessary + :param whitelist: the sorted list (based on BaseVolume) of pairs the user might want to + trade + :return: the list of pairs the user wants to trade without the one unavailable or + black_listed + """ + sanitized_whitelist = whitelist + markets = self._freqtrade.exchange.get_markets() + + # Filter to markets in stake currency + markets = [m for m in markets if m['quote'] == self._config['stake_currency']] + known_pairs = set() + + for market in markets: + pair = market['symbol'] + # pair is not int the generated dynamic market, or in the blacklist ... ignore it + if pair not in whitelist or pair in self.blacklist: + continue + # else the pair is valid + known_pairs.add(pair) + # Market is not active + if not market['active']: + sanitized_whitelist.remove(pair) + logger.info( + 'Ignoring %s from whitelist. Market is not active.', + pair + ) + + # We need to remove pairs that are unknown + self._whitelist = [x for x in sanitized_whitelist if x in known_pairs] + + return True diff --git a/freqtrade/pairlist/__init__.py b/freqtrade/pairlist/__init__.py new file mode 100644 index 000000000..e69de29bb From 1738633efc2696add32450cc30355dbf94721a1b Mon Sep 17 00:00:00 2001 From: Matthias Date: Fri, 30 Nov 2018 06:54:20 +0100 Subject: [PATCH 544/699] Fix refresh_whitelist tests --- freqtrade/pairlist/StaticList.py | 2 +- freqtrade/tests/test_acl_pair.py | 16 +++++++--------- 2 files changed, 8 insertions(+), 10 deletions(-) diff --git a/freqtrade/pairlist/StaticList.py b/freqtrade/pairlist/StaticList.py index ea4a9ac98..8b0decdf6 100644 --- a/freqtrade/pairlist/StaticList.py +++ b/freqtrade/pairlist/StaticList.py @@ -17,7 +17,7 @@ class StaticList(object): self._config = config self._whitelist = self._config['exchange']['pair_whitelist'] self._blacklist = self._config['exchange'].get('pair_blacklist', []) - self.refresh_whitelist() + # self.refresh_whitelist() @property def whitelist(self) -> List[str]: diff --git a/freqtrade/tests/test_acl_pair.py b/freqtrade/tests/test_acl_pair.py index 38df3cb38..58dd3dff2 100644 --- a/freqtrade/tests/test_acl_pair.py +++ b/freqtrade/tests/test_acl_pair.py @@ -6,9 +6,7 @@ from freqtrade.tests.conftest import get_patched_freqtradebot import pytest -# whitelist, blacklist, filtering, all of that will -# eventually become some rules to run on a generic ACL engine -# perhaps try to anticipate that by using some python package +# whitelist, blacklist, @pytest.fixture(scope="function") @@ -33,26 +31,26 @@ def test_refresh_market_pair_not_in_whitelist(mocker, markets, whitelist_conf): freqtradebot = get_patched_freqtradebot(mocker, whitelist_conf) mocker.patch('freqtrade.exchange.Exchange.get_markets', markets) - refreshedwhitelist = freqtradebot._refresh_whitelist( + freqtradebot.pairlists.validate_whitelist( whitelist_conf['exchange']['pair_whitelist'] + ['XXX/BTC'] ) # List ordered by BaseVolume whitelist = ['ETH/BTC', 'TKN/BTC'] # Ensure all except those in whitelist are removed - assert whitelist == refreshedwhitelist + assert whitelist == freqtradebot.pairlists.whitelist def test_refresh_whitelist(mocker, markets, whitelist_conf): freqtradebot = get_patched_freqtradebot(mocker, whitelist_conf) mocker.patch('freqtrade.exchange.Exchange.get_markets', markets) - refreshedwhitelist = freqtradebot._refresh_whitelist( + freqtradebot.pairlists.validate_whitelist( whitelist_conf['exchange']['pair_whitelist']) # List ordered by BaseVolume whitelist = ['ETH/BTC', 'TKN/BTC'] # Ensure all except those in whitelist are removed - assert whitelist == refreshedwhitelist + assert whitelist == freqtradebot.pairlists.whitelist def test_refresh_whitelist_dynamic(mocker, markets, tickers, whitelist_conf): @@ -67,11 +65,11 @@ def test_refresh_whitelist_dynamic(mocker, markets, tickers, whitelist_conf): # argument: use the whitelist dynamically by exchange-volume whitelist = ['ETH/BTC', 'TKN/BTC'] - refreshedwhitelist = freqtradebot._refresh_whitelist( + freqtradebot._refresh_whitelist( freqtradebot._gen_pair_whitelist(whitelist_conf['stake_currency']) ) - assert whitelist == refreshedwhitelist + assert whitelist == freqtradebot.pairlists.whitelist def test_refresh_whitelist_dynamic_empty(mocker, markets_empty, whitelist_conf): From 8fd713f3ae09d465d205501eda28c98d5e7addfc Mon Sep 17 00:00:00 2001 From: Matthias Date: Fri, 30 Nov 2018 07:02:08 +0100 Subject: [PATCH 545/699] validate_whitelist should return the list again --- freqtrade/pairlist/StaticList.py | 10 ++++------ freqtrade/tests/test_acl_pair.py | 11 +++++------ 2 files changed, 9 insertions(+), 12 deletions(-) diff --git a/freqtrade/pairlist/StaticList.py b/freqtrade/pairlist/StaticList.py index 8b0decdf6..225f89b2e 100644 --- a/freqtrade/pairlist/StaticList.py +++ b/freqtrade/pairlist/StaticList.py @@ -28,13 +28,13 @@ class StaticList(object): def blacklist(self) -> List[str]: return self._blacklist - def refresh_whitelist(self) -> bool: + def refresh_whitelist(self) -> None: """ Refreshes whitelist. """ - return self.validate_whitelist(self._config['exchange']['pair_whitelist']) + self._whitelist = self.validate_whitelist(self._config['exchange']['pair_whitelist']) - def validate_whitelist(self, whitelist: List[str]) -> bool: + def validate_whitelist(self, whitelist: List[str]) -> List[str]: """ Check available markets and remove pair from whitelist if necessary :param whitelist: the sorted list (based on BaseVolume) of pairs the user might want to @@ -65,6 +65,4 @@ class StaticList(object): ) # We need to remove pairs that are unknown - self._whitelist = [x for x in sanitized_whitelist if x in known_pairs] - - return True + return [x for x in sanitized_whitelist if x in known_pairs] diff --git a/freqtrade/tests/test_acl_pair.py b/freqtrade/tests/test_acl_pair.py index 58dd3dff2..6ec11e664 100644 --- a/freqtrade/tests/test_acl_pair.py +++ b/freqtrade/tests/test_acl_pair.py @@ -31,22 +31,21 @@ def test_refresh_market_pair_not_in_whitelist(mocker, markets, whitelist_conf): freqtradebot = get_patched_freqtradebot(mocker, whitelist_conf) mocker.patch('freqtrade.exchange.Exchange.get_markets', markets) - freqtradebot.pairlists.validate_whitelist( - whitelist_conf['exchange']['pair_whitelist'] + ['XXX/BTC'] - ) + freqtradebot.pairlists.refresh_whitelist() # List ordered by BaseVolume whitelist = ['ETH/BTC', 'TKN/BTC'] # Ensure all except those in whitelist are removed assert whitelist == freqtradebot.pairlists.whitelist + # Ensure config dict hasn't been changed + assert (whitelist_conf['exchange']['pair_whitelist'] == + freqtradebot.config['exchange']['pair_whitelist']) def test_refresh_whitelist(mocker, markets, whitelist_conf): freqtradebot = get_patched_freqtradebot(mocker, whitelist_conf) mocker.patch('freqtrade.exchange.Exchange.get_markets', markets) - freqtradebot.pairlists.validate_whitelist( - whitelist_conf['exchange']['pair_whitelist']) - + freqtradebot.pairlists.refresh_whitelist() # List ordered by BaseVolume whitelist = ['ETH/BTC', 'TKN/BTC'] # Ensure all except those in whitelist are removed From 58c7adae0aff2b6121243811f1aa02a3fd03d481 Mon Sep 17 00:00:00 2001 From: Matthias Date: Fri, 30 Nov 2018 07:06:02 +0100 Subject: [PATCH 546/699] Test for blacklist --- freqtrade/pairlist/StaticList.py | 2 +- freqtrade/tests/test_acl_pair.py | 5 +++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/freqtrade/pairlist/StaticList.py b/freqtrade/pairlist/StaticList.py index 225f89b2e..3c46fab11 100644 --- a/freqtrade/pairlist/StaticList.py +++ b/freqtrade/pairlist/StaticList.py @@ -30,7 +30,7 @@ class StaticList(object): def refresh_whitelist(self) -> None: """ - Refreshes whitelist. + Refreshes whitelist and assigns it to self._whitelist """ self._whitelist = self.validate_whitelist(self._config['exchange']['pair_whitelist']) diff --git a/freqtrade/tests/test_acl_pair.py b/freqtrade/tests/test_acl_pair.py index 6ec11e664..d9fd93d09 100644 --- a/freqtrade/tests/test_acl_pair.py +++ b/freqtrade/tests/test_acl_pair.py @@ -6,7 +6,7 @@ from freqtrade.tests.conftest import get_patched_freqtradebot import pytest -# whitelist, blacklist, +# whitelist, blacklist @pytest.fixture(scope="function") @@ -41,7 +41,7 @@ def test_refresh_market_pair_not_in_whitelist(mocker, markets, whitelist_conf): freqtradebot.config['exchange']['pair_whitelist']) -def test_refresh_whitelist(mocker, markets, whitelist_conf): +def test_refresh_pairlists(mocker, markets, whitelist_conf): freqtradebot = get_patched_freqtradebot(mocker, whitelist_conf) mocker.patch('freqtrade.exchange.Exchange.get_markets', markets) @@ -50,6 +50,7 @@ def test_refresh_whitelist(mocker, markets, whitelist_conf): whitelist = ['ETH/BTC', 'TKN/BTC'] # Ensure all except those in whitelist are removed assert whitelist == freqtradebot.pairlists.whitelist + assert whitelist_conf['exchange']['pair_blacklist'] == freqtradebot.pairlists.blacklist def test_refresh_whitelist_dynamic(mocker, markets, tickers, whitelist_conf): From d09dbfe2e6e6a233174c9863aff4bb3b6fc1076a Mon Sep 17 00:00:00 2001 From: Matthias Date: Fri, 30 Nov 2018 20:08:50 +0100 Subject: [PATCH 547/699] Add volumePairList - refactor tests to correct file --- freqtrade/freqtradebot.py | 48 ++------- .../{StaticList.py => StaticPairList.py} | 6 +- freqtrade/pairlist/VolumePairList.py | 98 +++++++++++++++++++ freqtrade/tests/test_acl_pair.py | 46 +++++++-- freqtrade/tests/test_freqtradebot.py | 31 ------ 5 files changed, 149 insertions(+), 80 deletions(-) rename freqtrade/pairlist/{StaticList.py => StaticPairList.py} (91%) create mode 100644 freqtrade/pairlist/VolumePairList.py diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index 3e8874bb6..d8f789e6a 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -12,8 +12,6 @@ from typing import Any, Callable, Dict, List, Optional import arrow from requests.exceptions import RequestException -from cachetools import TTLCache, cached - from freqtrade import (DependencyException, OperationalException, TemporaryError, __version__, constants, persistence) from freqtrade.exchange import Exchange @@ -25,7 +23,8 @@ from freqtrade.resolvers import StrategyResolver from freqtrade.state import State from freqtrade.strategy.interface import SellType, IStrategy from freqtrade.exchange.exchange_helpers import order_book_to_dataframe -from freqtrade.pairlist.StaticList import StaticList +from freqtrade.pairlist.StaticPairList import StaticPairList +from freqtrade.pairlist.VolumePairList import VolumePairList logger = logging.getLogger(__name__) @@ -60,7 +59,11 @@ class FreqtradeBot(object): self.persistence = None self.exchange = Exchange(self.config) self.wallets = Wallets(self.exchange) - self.pairlists = StaticList(self, self.config) + if self.config.get('dynamic_whitelist', None): + self.pairlists = VolumePairList(self, self.config) + + else: + self.pairlists = StaticPairList(self, self.config) # Initializing Edge only if enabled self.edge = Edge(self.config, self.exchange, self.strategy) if \ @@ -148,18 +151,9 @@ class FreqtradeBot(object): """ state_changed = False try: - nb_assets = self.config.get('dynamic_whitelist', None) - # Refresh whitelist based on wallet maintenance + # Refresh whitelist self.pairlists.refresh_whitelist() - sanitized_list = self.pairlists.whitelist - # sanitized_list = self._refresh_whitelist( - # self._gen_pair_whitelist( - # self.config['stake_currency'] - # ) if nb_assets else self.lists.get_whitelist() - # ) - - # Keep only the subsets of pairs wanted (up to nb_assets) - self.active_pair_whitelist = sanitized_list[:nb_assets] if nb_assets else sanitized_list + self.active_pair_whitelist = self.pairlists.whitelist # Calculating Edge positiong # Should be called before refresh_tickers @@ -207,30 +201,6 @@ class FreqtradeBot(object): self.state = State.STOPPED return state_changed - @cached(TTLCache(maxsize=1, ttl=1800)) - def _gen_pair_whitelist(self, base_currency: str, key: str = 'quoteVolume') -> List[str]: - """ - Updates the whitelist with with a dynamically generated list - :param base_currency: base currency as str - :param key: sort key (defaults to 'quoteVolume') - :return: List of pairs - """ - - if not self.exchange.exchange_has('fetchTickers'): - raise OperationalException( - 'Exchange does not support dynamic whitelist.' - 'Please edit your config and restart the bot' - ) - - tickers = self.exchange.get_tickers() - # check length so that we make sure that '/' is actually in the string - tickers = [v for k, v in tickers.items() - if len(k.split('/')) == 2 and k.split('/')[1] == base_currency] - - sorted_tickers = sorted(tickers, reverse=True, key=lambda t: t[key]) - pairs = [s['symbol'] for s in sorted_tickers] - return pairs - def get_target_bid(self, pair: str, ticker: Dict[str, float]) -> float: """ Calculates bid target between current ask price and last price diff --git a/freqtrade/pairlist/StaticList.py b/freqtrade/pairlist/StaticPairList.py similarity index 91% rename from freqtrade/pairlist/StaticList.py rename to freqtrade/pairlist/StaticPairList.py index 3c46fab11..8466d7b82 100644 --- a/freqtrade/pairlist/StaticList.py +++ b/freqtrade/pairlist/StaticPairList.py @@ -10,7 +10,7 @@ from typing import List, Optional logger = logging.getLogger(__name__) -class StaticList(object): +class StaticPairList(object): def __init__(self, freqtrade, config: dict) -> None: self._freqtrade = freqtrade @@ -32,9 +32,9 @@ class StaticList(object): """ Refreshes whitelist and assigns it to self._whitelist """ - self._whitelist = self.validate_whitelist(self._config['exchange']['pair_whitelist']) + self._whitelist = self._validate_whitelist(self._config['exchange']['pair_whitelist']) - def validate_whitelist(self, whitelist: List[str]) -> List[str]: + def _validate_whitelist(self, whitelist: List[str]) -> List[str]: """ Check available markets and remove pair from whitelist if necessary :param whitelist: the sorted list (based on BaseVolume) of pairs the user might want to diff --git a/freqtrade/pairlist/VolumePairList.py b/freqtrade/pairlist/VolumePairList.py new file mode 100644 index 000000000..c79f3c5c0 --- /dev/null +++ b/freqtrade/pairlist/VolumePairList.py @@ -0,0 +1,98 @@ +""" +Static List provider + +Provides lists as configured in config.json + + """ +import logging +from typing import List +from cachetools import TTLCache, cached + +from freqtrade.pairlist.StaticPairList import StaticPairList +from freqtrade import OperationalException +logger = logging.getLogger(__name__) + + +class VolumePairList(StaticPairList): + + def __init__(self, freqtrade, config: dict) -> None: + self._freqtrade = freqtrade + self._config = config + self._whitelist = self._config['exchange']['pair_whitelist'] + self._blacklist = self._config['exchange'].get('pair_blacklist', []) + self._number_pairs = self._config.get('dynamic_whitelist', None) + if not self._freqtrade.exchange.exchange_has('fetchTickers'): + raise OperationalException( + 'Exchange does not support dynamic whitelist.' + 'Please edit your config and restart the bot' + ) + # self.refresh_whitelist() + + @property + def whitelist(self) -> List[str]: + """ Contains the current whitelist """ + return self._whitelist + + @property + def blacklist(self) -> List[str]: + return self._blacklist + + def refresh_whitelist(self) -> None: + """ + Refreshes whitelist and assigns it to self._whitelist + """ + # Generate dynamic whitelist + pairs = self._gen_pair_whitelist(self._config['stake_currency']) + # Validate whitelist to only have active market pairs + self._whitelist = self._validate_whitelist(pairs)[:self._number_pairs] + + @cached(TTLCache(maxsize=1, ttl=1800)) + def _gen_pair_whitelist(self, base_currency: str, key: str = 'quoteVolume') -> List[str]: + """ + Updates the whitelist with with a dynamically generated list + :param base_currency: base currency as str + :param key: sort key (defaults to 'quoteVolume') + :return: List of pairs + """ + + tickers = self._freqtrade.exchange.get_tickers() + # check length so that we make sure that '/' is actually in the string + tickers = [v for k, v in tickers.items() + if len(k.split('/')) == 2 and k.split('/')[1] == base_currency] + + sorted_tickers = sorted(tickers, reverse=True, key=lambda t: t[key]) + pairs = [s['symbol'] for s in sorted_tickers] + return pairs + + def _validate_whitelist(self, whitelist: List[str]) -> List[str]: + """ + Check available markets and remove pair from whitelist if necessary + :param whitelist: the sorted list (based on BaseVolume) of pairs the user might want to + trade + :return: the list of pairs the user wants to trade without the one unavailable or + black_listed + """ + sanitized_whitelist = whitelist + markets = self._freqtrade.exchange.get_markets() + + # Filter to markets in stake currency + markets = [m for m in markets if m['quote'] == self._config['stake_currency']] + known_pairs = set() + + for market in markets: + pair = market['symbol'] + # pair is not int the generated dynamic market, or in the blacklist ... ignore it + if pair not in whitelist or pair in self.blacklist: + continue + # else the pair is valid + known_pairs.add(pair) + # Market is not active + if not market['active']: + sanitized_whitelist.remove(pair) + logger.info( + 'Ignoring %s from whitelist. Market is not active.', + pair + ) + + # We need to remove pairs that are unknown + return [x for x in sanitized_whitelist if x in known_pairs] diff --git a/freqtrade/tests/test_acl_pair.py b/freqtrade/tests/test_acl_pair.py index d9fd93d09..997c171ad 100644 --- a/freqtrade/tests/test_acl_pair.py +++ b/freqtrade/tests/test_acl_pair.py @@ -2,6 +2,7 @@ from unittest.mock import MagicMock +from freqtrade import OperationalException from freqtrade.tests.conftest import get_patched_freqtradebot import pytest @@ -35,7 +36,7 @@ def test_refresh_market_pair_not_in_whitelist(mocker, markets, whitelist_conf): # List ordered by BaseVolume whitelist = ['ETH/BTC', 'TKN/BTC'] # Ensure all except those in whitelist are removed - assert whitelist == freqtradebot.pairlists.whitelist + assert set(whitelist) == set(freqtradebot.pairlists.whitelist) # Ensure config dict hasn't been changed assert (whitelist_conf['exchange']['pair_whitelist'] == freqtradebot.config['exchange']['pair_whitelist']) @@ -49,11 +50,12 @@ def test_refresh_pairlists(mocker, markets, whitelist_conf): # List ordered by BaseVolume whitelist = ['ETH/BTC', 'TKN/BTC'] # Ensure all except those in whitelist are removed - assert whitelist == freqtradebot.pairlists.whitelist + assert set(whitelist)== set(freqtradebot.pairlists.whitelist) assert whitelist_conf['exchange']['pair_blacklist'] == freqtradebot.pairlists.blacklist def test_refresh_whitelist_dynamic(mocker, markets, tickers, whitelist_conf): + whitelist_conf['dynamic_whitelist'] = 5 freqtradebot = get_patched_freqtradebot(mocker, whitelist_conf) mocker.patch.multiple( 'freqtrade.exchange.Exchange', @@ -64,10 +66,7 @@ def test_refresh_whitelist_dynamic(mocker, markets, tickers, whitelist_conf): # argument: use the whitelist dynamically by exchange-volume whitelist = ['ETH/BTC', 'TKN/BTC'] - - freqtradebot._refresh_whitelist( - freqtradebot._gen_pair_whitelist(whitelist_conf['stake_currency']) - ) + freqtradebot.pairlists.refresh_whitelist() assert whitelist == freqtradebot.pairlists.whitelist @@ -79,7 +78,40 @@ def test_refresh_whitelist_dynamic_empty(mocker, markets_empty, whitelist_conf): # argument: use the whitelist dynamically by exchange-volume whitelist = [] whitelist_conf['exchange']['pair_whitelist'] = [] - freqtradebot._refresh_whitelist(whitelist) + freqtradebot.pairlists.refresh_whitelist() pairslist = whitelist_conf['exchange']['pair_whitelist'] assert set(whitelist) == set(pairslist) + + +def test_gen_pair_whitelist(mocker, default_conf, tickers) -> None: + freqtrade = get_patched_freqtradebot(mocker, default_conf) + mocker.patch('freqtrade.exchange.Exchange.get_tickers', tickers) + mocker.patch('freqtrade.exchange.Exchange.exchange_has', MagicMock(return_value=True)) + + # Test to retrieved BTC sorted on quoteVolume (default) + freqtrade.pairlists.refresh_whitelist() + + whitelist = freqtrade.pairlists.whitelist + assert whitelist == ['ETH/BTC', 'TKN/BTC', 'BLK/BTC', 'LTC/BTC'] + + # Test to retrieve BTC sorted on bidVolume + whitelist = freqtrade._gen_pair_whitelist(base_currency='BTC', key='bidVolume') + assert whitelist == ['LTC/BTC', 'TKN/BTC', 'ETH/BTC', 'BLK/BTC'] + + # Test with USDT sorted on quoteVolume (default) + whitelist = freqtrade._gen_pair_whitelist(base_currency='USDT') + assert whitelist == ['TKN/USDT', 'ETH/USDT', 'LTC/USDT', 'BLK/USDT'] + + # Test with ETH (our fixture does not have ETH, so result should be empty) + whitelist = freqtrade._gen_pair_whitelist(base_currency='ETH') + assert whitelist == [] + + +def test_gen_pair_whitelist_not_supported(mocker, default_conf, tickers) -> None: + freqtrade = get_patched_freqtradebot(mocker, default_conf) + mocker.patch('freqtrade.exchange.Exchange.get_tickers', tickers) + mocker.patch('freqtrade.exchange.Exchange.exchange_has', MagicMock(return_value=False)) + + with pytest.raises(OperationalException): + freqtrade._gen_pair_whitelist(base_currency='BTC') diff --git a/freqtrade/tests/test_freqtradebot.py b/freqtrade/tests/test_freqtradebot.py index a9b3ffc5d..71d451b94 100644 --- a/freqtrade/tests/test_freqtradebot.py +++ b/freqtrade/tests/test_freqtradebot.py @@ -136,37 +136,6 @@ def test_throttle_with_assets(mocker, default_conf) -> None: assert result == -1 -def test_gen_pair_whitelist(mocker, default_conf, tickers) -> None: - freqtrade = get_patched_freqtradebot(mocker, default_conf) - mocker.patch('freqtrade.exchange.Exchange.get_tickers', tickers) - mocker.patch('freqtrade.exchange.Exchange.exchange_has', MagicMock(return_value=True)) - - # Test to retrieved BTC sorted on quoteVolume (default) - whitelist = freqtrade._gen_pair_whitelist(base_currency='BTC') - assert whitelist == ['ETH/BTC', 'TKN/BTC', 'BLK/BTC', 'LTC/BTC'] - - # Test to retrieve BTC sorted on bidVolume - whitelist = freqtrade._gen_pair_whitelist(base_currency='BTC', key='bidVolume') - assert whitelist == ['LTC/BTC', 'TKN/BTC', 'ETH/BTC', 'BLK/BTC'] - - # Test with USDT sorted on quoteVolume (default) - whitelist = freqtrade._gen_pair_whitelist(base_currency='USDT') - assert whitelist == ['TKN/USDT', 'ETH/USDT', 'LTC/USDT', 'BLK/USDT'] - - # Test with ETH (our fixture does not have ETH, so result should be empty) - whitelist = freqtrade._gen_pair_whitelist(base_currency='ETH') - assert whitelist == [] - - -def test_gen_pair_whitelist_not_supported(mocker, default_conf, tickers) -> None: - freqtrade = get_patched_freqtradebot(mocker, default_conf) - mocker.patch('freqtrade.exchange.Exchange.get_tickers', tickers) - mocker.patch('freqtrade.exchange.Exchange.exchange_has', MagicMock(return_value=False)) - - with pytest.raises(OperationalException): - freqtrade._gen_pair_whitelist(base_currency='BTC') - - def test_get_trade_stake_amount(default_conf, ticker, limit_buy_order, fee, mocker) -> None: patch_RPCManager(mocker) patch_exchange(mocker) From 26187ef6c7863a3c25b9e02398690c8faf310d3d Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 2 Dec 2018 22:12:12 +0100 Subject: [PATCH 548/699] patch exchange_has --- freqtrade/tests/rpc/test_rpc.py | 1 + freqtrade/tests/rpc/test_rpc_telegram.py | 1 + freqtrade/tests/test_freqtradebot.py | 1 + 3 files changed, 3 insertions(+) diff --git a/freqtrade/tests/rpc/test_rpc.py b/freqtrade/tests/rpc/test_rpc.py index ff72ef634..b0b598654 100644 --- a/freqtrade/tests/rpc/test_rpc.py +++ b/freqtrade/tests/rpc/test_rpc.py @@ -663,6 +663,7 @@ def test_rpc_whitelist_dynamic(mocker, default_conf) -> None: patch_coinmarketcap(mocker) patch_exchange(mocker) default_conf['dynamic_whitelist'] = 4 + mocker.patch('freqtrade.exchange.Exchange.exchange_has', MagicMock(return_value=True)) mocker.patch('freqtrade.rpc.telegram.Telegram', MagicMock()) freqtradebot = FreqtradeBot(default_conf) diff --git a/freqtrade/tests/rpc/test_rpc_telegram.py b/freqtrade/tests/rpc/test_rpc_telegram.py index a8e8bf003..4f29c5a03 100644 --- a/freqtrade/tests/rpc/test_rpc_telegram.py +++ b/freqtrade/tests/rpc/test_rpc_telegram.py @@ -1033,6 +1033,7 @@ def test_whitelist_dynamic(default_conf, update, mocker) -> None: _init=MagicMock(), _send_msg=msg_mock ) + mocker.patch('freqtrade.exchange.Exchange.exchange_has', MagicMock(return_value=True)) default_conf['dynamic_whitelist'] = 4 freqtradebot = get_patched_freqtradebot(mocker, default_conf) diff --git a/freqtrade/tests/test_freqtradebot.py b/freqtrade/tests/test_freqtradebot.py index 71d451b94..ca604340b 100644 --- a/freqtrade/tests/test_freqtradebot.py +++ b/freqtrade/tests/test_freqtradebot.py @@ -2533,5 +2533,6 @@ def test_order_book_ask_strategy(default_conf, limit_buy_order, limit_sell_order def test_startup_messages(default_conf, mocker): default_conf['dynamic_whitelist'] = 20 + mocker.patch('freqtrade.exchange.Exchange.exchange_has', MagicMock(return_value=True)) freqtrade = get_patched_freqtradebot(mocker, default_conf) assert freqtrade.state is State.RUNNING From 3a086aac58f4f2fd3530910d1adc3154e9c3ea73 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 2 Dec 2018 22:49:30 +0100 Subject: [PATCH 549/699] Add commit and message to container --- build_helpers/publish_docker.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/build_helpers/publish_docker.sh b/build_helpers/publish_docker.sh index c2b40ba81..9d82fc2d5 100755 --- a/build_helpers/publish_docker.sh +++ b/build_helpers/publish_docker.sh @@ -4,6 +4,9 @@ TAG=$(echo "${TRAVIS_BRANCH}" | sed -e "s/\//_/") +# Add commit and commit_message to docker container +echo "${TRAVIS_COMMIT} ${TRAVIS_COMMIT_MESSAGE}" > freqtrade_commit + if [ "${TRAVIS_EVENT_TYPE}" = "cron" ]; then echo "event ${TRAVIS_EVENT_TYPE}: full rebuild - skipping cache" docker build -t freqtrade:${TAG} . From 99f7c3752afb0ba03643baa16de046f2b0a0782d Mon Sep 17 00:00:00 2001 From: Pan Long Date: Mon, 3 Dec 2018 07:21:01 +0800 Subject: [PATCH 550/699] Correct Edge links It was pointing to a fork instead of freqtrade/freqtrade --- docs/index.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/index.md b/docs/index.md index c64b9c188..43890b053 100644 --- a/docs/index.md +++ b/docs/index.md @@ -21,12 +21,12 @@ Pull-request. Do not hesitate to reach us on - [Bot commands](https://github.com/freqtrade/freqtrade/blob/develop/docs/bot-usage.md#bot-commands) - [Backtesting commands](https://github.com/freqtrade/freqtrade/blob/develop/docs/bot-usage.md#backtesting-commands) - [Hyperopt commands](https://github.com/freqtrade/freqtrade/blob/develop/docs/bot-usage.md#hyperopt-commands) - - [Edge commands](https://github.com/mishaker/freqtrade/blob/develop/docs/bot-usage.md#edge-commands) + - [Edge commands](https://github.com/freqtrade/freqtrade/blob/develop/docs/bot-usage.md#edge-commands) - [Bot Optimization](https://github.com/freqtrade/freqtrade/blob/develop/docs/bot-optimization.md) - [Change your strategy](https://github.com/freqtrade/freqtrade/blob/develop/docs/bot-optimization.md#change-your-strategy) - [Add more Indicator](https://github.com/freqtrade/freqtrade/blob/develop/docs/bot-optimization.md#add-more-indicator) - [Test your strategy with Backtesting](https://github.com/freqtrade/freqtrade/blob/develop/docs/backtesting.md) - - [Edge positioning](https://github.com/mishaker/freqtrade/blob/money_mgt/docs/edge.md) + - [Edge positioning](https://github.com/freqtrade/freqtrade/blob/money_mgt/docs/edge.md) - [Find optimal parameters with Hyperopt](https://github.com/freqtrade/freqtrade/blob/develop/docs/hyperopt.md) - [Control the bot with telegram](https://github.com/freqtrade/freqtrade/blob/develop/docs/telegram-usage.md) - [Receive notifications via webhook](https://github.com/freqtrade/freqtrade/blob/develop/docs/webhook-config.md) From 11da297c25ed32016bba2cb71eda44deb6263e70 Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Mon, 3 Dec 2018 13:34:08 +0100 Subject: [PATCH 551/699] Update ccxt from 1.17.572 to 1.17.574 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index cee3e19be..592c009d6 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -ccxt==1.17.572 +ccxt==1.17.574 SQLAlchemy==1.2.14 python-telegram-bot==11.1.0 arrow==0.12.1 From 3360e777a13d3ca910f097b381e0b83d516dde3b Mon Sep 17 00:00:00 2001 From: Matthias Date: Mon, 3 Dec 2018 19:29:35 +0100 Subject: [PATCH 552/699] Fix flake adn mypy --- freqtrade/freqtradebot.py | 4 ++-- freqtrade/pairlist/StaticPairList.py | 2 +- freqtrade/tests/test_acl_pair.py | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index d8f789e6a..ebe30cec6 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -60,10 +60,10 @@ class FreqtradeBot(object): self.exchange = Exchange(self.config) self.wallets = Wallets(self.exchange) if self.config.get('dynamic_whitelist', None): - self.pairlists = VolumePairList(self, self.config) + self.pairlists: StaticPairList = VolumePairList(self, self.config) else: - self.pairlists = StaticPairList(self, self.config) + self.pairlists: StaticPairList = StaticPairList(self, self.config) # Initializing Edge only if enabled self.edge = Edge(self.config, self.exchange, self.strategy) if \ diff --git a/freqtrade/pairlist/StaticPairList.py b/freqtrade/pairlist/StaticPairList.py index 8466d7b82..24ad30deb 100644 --- a/freqtrade/pairlist/StaticPairList.py +++ b/freqtrade/pairlist/StaticPairList.py @@ -5,7 +5,7 @@ Provides lists as configured in config.json """ import logging -from typing import List, Optional +from typing import List logger = logging.getLogger(__name__) diff --git a/freqtrade/tests/test_acl_pair.py b/freqtrade/tests/test_acl_pair.py index 997c171ad..8e41ad860 100644 --- a/freqtrade/tests/test_acl_pair.py +++ b/freqtrade/tests/test_acl_pair.py @@ -50,7 +50,7 @@ def test_refresh_pairlists(mocker, markets, whitelist_conf): # List ordered by BaseVolume whitelist = ['ETH/BTC', 'TKN/BTC'] # Ensure all except those in whitelist are removed - assert set(whitelist)== set(freqtradebot.pairlists.whitelist) + assert set(whitelist) == set(freqtradebot.pairlists.whitelist) assert whitelist_conf['exchange']['pair_blacklist'] == freqtradebot.pairlists.blacklist From b5192193fda4df5e3afa06399c22aa7a8f7889ba Mon Sep 17 00:00:00 2001 From: misagh Date: Mon, 3 Dec 2018 19:45:00 +0100 Subject: [PATCH 553/699] total amount passed to edge should consider open trades too --- freqtrade/freqtradebot.py | 3 ++- freqtrade/persistence.py | 11 +++++++++++ 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index ccb45be2c..1e4b2e678 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -305,7 +305,8 @@ class FreqtradeBot(object): return self.edge.stake_amount( pair, self.wallets.get_free(self.config['stake_currency']), - self.wallets.get_total(self.config['stake_currency']) + self.wallets.get_total(self.config['stake_currency']) + + Trade.calc_total_open_trades_in_stake_currency() ) else: stake_amount = self.config['stake_amount'] diff --git a/freqtrade/persistence.py b/freqtrade/persistence.py index 592a88acb..20ae8c43a 100644 --- a/freqtrade/persistence.py +++ b/freqtrade/persistence.py @@ -14,6 +14,7 @@ from sqlalchemy.exc import NoSuchModuleError from sqlalchemy.ext.declarative import declarative_base from sqlalchemy.orm.scoping import scoped_session from sqlalchemy.orm.session import sessionmaker +from sqlalchemy import func from sqlalchemy.pool import StaticPool from freqtrade import OperationalException @@ -349,3 +350,13 @@ class Trade(_DECL_BASE): ) profit_percent = (close_trade_price / open_trade_price) - 1 return float(f"{profit_percent:.8f}") + + def calc_total_open_trades_in_stake_currency() -> float: + """ + Calculates total invested amount in open trades + in stake currency + """ + total_open_stake_amount = Trade.session.query(func.sum(Trade.stake_amount))\ + .filter(Trade.is_open.is_(True))\ + .scalar() + return total_open_stake_amount or 0 From 43bafc391f77e1b937b6e3b18ea74ab1d9b54e1e Mon Sep 17 00:00:00 2001 From: misagh Date: Mon, 3 Dec 2018 19:46:22 +0100 Subject: [PATCH 554/699] static method added --- freqtrade/persistence.py | 1 + 1 file changed, 1 insertion(+) diff --git a/freqtrade/persistence.py b/freqtrade/persistence.py index 20ae8c43a..aeab63f22 100644 --- a/freqtrade/persistence.py +++ b/freqtrade/persistence.py @@ -351,6 +351,7 @@ class Trade(_DECL_BASE): profit_percent = (close_trade_price / open_trade_price) - 1 return float(f"{profit_percent:.8f}") + @staticmethod def calc_total_open_trades_in_stake_currency() -> float: """ Calculates total invested amount in open trades From 108d9a111714c5d7a26aaa72980a03912e93819a Mon Sep 17 00:00:00 2001 From: misagh Date: Mon, 3 Dec 2018 19:55:37 +0100 Subject: [PATCH 555/699] function name refactored --- freqtrade/freqtradebot.py | 2 +- freqtrade/persistence.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index 1e4b2e678..6baef76bc 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -306,7 +306,7 @@ class FreqtradeBot(object): pair, self.wallets.get_free(self.config['stake_currency']), self.wallets.get_total(self.config['stake_currency']) + - Trade.calc_total_open_trades_in_stake_currency() + Trade.total_open_trades_stakes() ) else: stake_amount = self.config['stake_amount'] diff --git a/freqtrade/persistence.py b/freqtrade/persistence.py index aeab63f22..71752d58e 100644 --- a/freqtrade/persistence.py +++ b/freqtrade/persistence.py @@ -352,7 +352,7 @@ class Trade(_DECL_BASE): return float(f"{profit_percent:.8f}") @staticmethod - def calc_total_open_trades_in_stake_currency() -> float: + def total_open_trades_stakes() -> float: """ Calculates total invested amount in open trades in stake currency From 1b3ecb8343b7de2424a5d5bced73400d3a4cefa4 Mon Sep 17 00:00:00 2001 From: Matthias Date: Mon, 3 Dec 2018 20:00:18 +0100 Subject: [PATCH 556/699] Deprecate --dynamic-whitelist --- freqtrade/arguments.py | 3 ++- freqtrade/configuration.py | 12 ++++++++---- freqtrade/constants.py | 8 ++++++++ freqtrade/freqtradebot.py | 3 +-- freqtrade/pairlist/VolumePairList.py | 3 ++- freqtrade/tests/test_acl_pair.py | 7 ++++--- freqtrade/tests/test_configuration.py | 8 +++++--- 7 files changed, 30 insertions(+), 14 deletions(-) diff --git a/freqtrade/arguments.py b/freqtrade/arguments.py index 84e1a0f77..a33848c5f 100644 --- a/freqtrade/arguments.py +++ b/freqtrade/arguments.py @@ -115,7 +115,8 @@ class Arguments(object): self.parser.add_argument( '--dynamic-whitelist', help='dynamically generate and update whitelist' - ' based on 24h BaseVolume (default: %(const)s)', + ' based on 24h BaseVolume (default: %(const)s)' + ' DEPRECATED.', dest='dynamic_whitelist', const=constants.DYNAMIC_WHITELIST, type=int, diff --git a/freqtrade/configuration.py b/freqtrade/configuration.py index feec0cb43..7ddcf1639 100644 --- a/freqtrade/configuration.py +++ b/freqtrade/configuration.py @@ -110,10 +110,14 @@ class Configuration(object): # Add dynamic_whitelist if found if 'dynamic_whitelist' in self.args and self.args.dynamic_whitelist: - config.update({'dynamic_whitelist': self.args.dynamic_whitelist}) - logger.info( - 'Parameter --dynamic-whitelist detected. ' - 'Using dynamically generated whitelist. ' + # Update to volumePairList (the previous default) + config['whitelist'] = {'method': 'VolumePairList', + 'config': {'number_assets': self.args.dynamic_whitelist} + } + logger.warning( + 'Parameter --dynamic-whitelist has been deprecated, ' + 'and will be completely replaced by the whitelist dict in the future. ' + 'For now: using dynamically generated whitelist based on VolumePairList. ' '(not applicable with Backtesting and Hyperopt)' ) diff --git a/freqtrade/constants.py b/freqtrade/constants.py index f8fb91240..c200137cb 100644 --- a/freqtrade/constants.py +++ b/freqtrade/constants.py @@ -124,6 +124,14 @@ CONF_SCHEMA = { 'ignore_roi_if_buy_signal_true': {'type': 'boolean'} } }, + 'whitelist': { + 'type': 'object', + 'properties': { + 'method': {'type': 'string'}, + 'config': {'type': 'object'} + }, + 'required': ['method'] + }, 'telegram': { 'type': 'object', 'properties': { diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index ebe30cec6..19a22b87d 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -59,9 +59,8 @@ class FreqtradeBot(object): self.persistence = None self.exchange = Exchange(self.config) self.wallets = Wallets(self.exchange) - if self.config.get('dynamic_whitelist', None): + if self.config.get('whitelist', {}).get('method') == 'VolumePairList': self.pairlists: StaticPairList = VolumePairList(self, self.config) - else: self.pairlists: StaticPairList = StaticPairList(self, self.config) diff --git a/freqtrade/pairlist/VolumePairList.py b/freqtrade/pairlist/VolumePairList.py index c79f3c5c0..41f9943a1 100644 --- a/freqtrade/pairlist/VolumePairList.py +++ b/freqtrade/pairlist/VolumePairList.py @@ -18,9 +18,10 @@ class VolumePairList(StaticPairList): def __init__(self, freqtrade, config: dict) -> None: self._freqtrade = freqtrade self._config = config + self._whitelistconf = self._config.get('whitelist', {}).get('config') self._whitelist = self._config['exchange']['pair_whitelist'] self._blacklist = self._config['exchange'].get('pair_blacklist', []) - self._number_pairs = self._config.get('dynamic_whitelist', None) + self._number_pairs = self._whitelistconf.get('number_assets') if not self._freqtrade.exchange.exchange_has('fetchTickers'): raise OperationalException( 'Exchange does not support dynamic whitelist.' diff --git a/freqtrade/tests/test_acl_pair.py b/freqtrade/tests/test_acl_pair.py index 8e41ad860..a12de9a81 100644 --- a/freqtrade/tests/test_acl_pair.py +++ b/freqtrade/tests/test_acl_pair.py @@ -4,7 +4,6 @@ from unittest.mock import MagicMock from freqtrade import OperationalException from freqtrade.tests.conftest import get_patched_freqtradebot - import pytest # whitelist, blacklist @@ -109,9 +108,11 @@ def test_gen_pair_whitelist(mocker, default_conf, tickers) -> None: def test_gen_pair_whitelist_not_supported(mocker, default_conf, tickers) -> None: - freqtrade = get_patched_freqtradebot(mocker, default_conf) + default_conf['whitelist'] = {'method': 'VolumePairList', + 'config': {'number_assets': 10} + } mocker.patch('freqtrade.exchange.Exchange.get_tickers', tickers) mocker.patch('freqtrade.exchange.Exchange.exchange_has', MagicMock(return_value=False)) with pytest.raises(OperationalException): - freqtrade._gen_pair_whitelist(base_currency='BTC') + get_patched_freqtradebot(mocker, default_conf) diff --git a/freqtrade/tests/test_configuration.py b/freqtrade/tests/test_configuration.py index 23fefd3cd..951d831b2 100644 --- a/freqtrade/tests/test_configuration.py +++ b/freqtrade/tests/test_configuration.py @@ -119,7 +119,8 @@ def test_load_config_with_params(default_conf, mocker) -> None: configuration = Configuration(args) validated_conf = configuration.load_config() - assert validated_conf.get('dynamic_whitelist') == 10 + assert validated_conf.get('whitelist', {}).get('method') == 'VolumePairList' + assert validated_conf.get('whitelist', {}).get('config').get('number_assets') == 10 assert validated_conf.get('strategy') == 'TestStrategy' assert validated_conf.get('strategy_path') == '/some/path' assert validated_conf.get('db_url') == 'sqlite:///someurl' @@ -194,8 +195,9 @@ def test_show_info(default_conf, mocker, caplog) -> None: configuration.get_config() assert log_has( - 'Parameter --dynamic-whitelist detected. ' - 'Using dynamically generated whitelist. ' + 'Parameter --dynamic-whitelist has been deprecated, ' + 'and will be completely replaced by the whitelist dict in the future. ' + 'For now: using dynamically generated whitelist based on VolumePairList. ' '(not applicable with Backtesting and Hyperopt)', caplog.record_tuples ) From ef1208b3664d7ebe1dd6ff568b9592a46b8bfc7c Mon Sep 17 00:00:00 2001 From: Matthias Date: Mon, 3 Dec 2018 20:31:25 +0100 Subject: [PATCH 557/699] Fix rpc messages --- freqtrade/freqtradebot.py | 2 +- freqtrade/pairlist/StaticPairList.py | 24 +++++++++++++++++++++++- freqtrade/pairlist/VolumePairList.py | 16 +++++++--------- freqtrade/rpc/rpc.py | 3 ++- freqtrade/rpc/rpc_manager.py | 12 +++--------- freqtrade/rpc/telegram.py | 6 ++---- freqtrade/tests/rpc/test_rpc.py | 7 +++++-- freqtrade/tests/rpc/test_rpc_manager.py | 4 +++- freqtrade/tests/rpc/test_rpc_telegram.py | 4 +++- freqtrade/tests/test_acl_pair.py | 6 ++++-- freqtrade/tests/test_arguments.py | 3 ++- freqtrade/tests/test_configuration.py | 4 +--- freqtrade/tests/test_freqtradebot.py | 4 +++- 13 files changed, 59 insertions(+), 36 deletions(-) diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index 19a22b87d..17b818b27 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -112,7 +112,7 @@ class FreqtradeBot(object): }) logger.info('Changing state to: %s', state.name) if state == State.RUNNING: - self.rpc.startup_messages(self.config) + self.rpc.startup_messages(self.config, self.pairlists) if state == State.STOPPED: time.sleep(1) diff --git a/freqtrade/pairlist/StaticPairList.py b/freqtrade/pairlist/StaticPairList.py index 24ad30deb..5b0e37357 100644 --- a/freqtrade/pairlist/StaticPairList.py +++ b/freqtrade/pairlist/StaticPairList.py @@ -19,15 +19,37 @@ class StaticPairList(object): self._blacklist = self._config['exchange'].get('pair_blacklist', []) # self.refresh_whitelist() + @property + def name(self) -> str: + """ + Gets name of the class + -> no need to overwrite in subclasses + """ + return self.__class__.__name__ + @property def whitelist(self) -> List[str]: - """ Contains the current whitelist """ + """ + Has the current whitelist + -> no need to overwrite in subclasses + """ return self._whitelist @property def blacklist(self) -> List[str]: + """ + Has the current blacklist + -> no need to overwrite in subclasses + """ return self._blacklist + def short_desc(self) -> str: + """ + Short whitelist method description - used for startup-messages + -> Please overwrite in subclasses + """ + return f"{self.name}: {self.whitelist}" + def refresh_whitelist(self) -> None: """ Refreshes whitelist and assigns it to self._whitelist diff --git a/freqtrade/pairlist/VolumePairList.py b/freqtrade/pairlist/VolumePairList.py index 41f9943a1..f45c57676 100644 --- a/freqtrade/pairlist/VolumePairList.py +++ b/freqtrade/pairlist/VolumePairList.py @@ -21,7 +21,7 @@ class VolumePairList(StaticPairList): self._whitelistconf = self._config.get('whitelist', {}).get('config') self._whitelist = self._config['exchange']['pair_whitelist'] self._blacklist = self._config['exchange'].get('pair_blacklist', []) - self._number_pairs = self._whitelistconf.get('number_assets') + self._number_pairs = self._whitelistconf['number_assets'] if not self._freqtrade.exchange.exchange_has('fetchTickers'): raise OperationalException( 'Exchange does not support dynamic whitelist.' @@ -29,14 +29,12 @@ class VolumePairList(StaticPairList): ) # self.refresh_whitelist() - @property - def whitelist(self) -> List[str]: - """ Contains the current whitelist """ - return self._whitelist - - @property - def blacklist(self) -> List[str]: - return self._blacklist + def short_desc(self) -> str: + """ + Short whitelist method description - used for startup-messages + -> Please overwrite in subclasses + """ + return f"{self.name} - top {self._whitelistconf['number_assets']} volume pairs." def refresh_whitelist(self) -> None: """ diff --git a/freqtrade/rpc/rpc.py b/freqtrade/rpc/rpc.py index 50cbd27ef..6ca32e4be 100644 --- a/freqtrade/rpc/rpc.py +++ b/freqtrade/rpc/rpc.py @@ -446,7 +446,8 @@ class RPC(object): def _rpc_whitelist(self) -> Dict: """ Returns the currently active whitelist""" - res = {'method': self._freqtrade.config.get('dynamic_whitelist', 0) or 'static', + res = {'method': self._freqtrade.pairlists.name, + 'length': len(self._freqtrade.pairlists.whitelist), 'whitelist': self._freqtrade.active_pair_whitelist } return res diff --git a/freqtrade/rpc/rpc_manager.py b/freqtrade/rpc/rpc_manager.py index 74a4e3bdc..de861677d 100644 --- a/freqtrade/rpc/rpc_manager.py +++ b/freqtrade/rpc/rpc_manager.py @@ -52,7 +52,7 @@ class RPCManager(object): logger.debug('Forwarding message to rpc.%s', mod.name) mod.send_msg(msg) - def startup_messages(self, config) -> None: + def startup_messages(self, config, pairlist) -> None: if config.get('dry_run', False): self.send_msg({ 'type': RPCMessageType.WARNING_NOTIFICATION, @@ -72,14 +72,8 @@ class RPCManager(object): f'*Ticker Interval:* `{ticker_interval}`\n' f'*Strategy:* `{strategy_name}`' }) - if config.get('dynamic_whitelist', False): - top_pairs = 'top volume ' + str(config.get('dynamic_whitelist', 20)) - specific_pairs = '' - else: - top_pairs = 'whitelisted' - specific_pairs = '\n' + ', '.join(config['exchange'].get('pair_whitelist', '')) self.send_msg({ 'type': RPCMessageType.STATUS_NOTIFICATION, - 'status': f'Searching for {top_pairs} {stake_currency} pairs to buy and sell...' - f'{specific_pairs}' + 'status': f'Searching for {stake_currency} pairs to buy and sell ' + f'based on {pairlist.short_desc()}' }) diff --git a/freqtrade/rpc/telegram.py b/freqtrade/rpc/telegram.py index 55c5abef2..9a3abaea4 100644 --- a/freqtrade/rpc/telegram.py +++ b/freqtrade/rpc/telegram.py @@ -447,10 +447,8 @@ class Telegram(RPC): """ try: whitelist = self._rpc_whitelist() - if whitelist['method'] == 'static': - message = f"Using static whitelist with `{len(whitelist['whitelist'])}` pairs \n" - else: - message = f"Dynamic whitelist with `{whitelist['method']}` pairs\n" + + message = f"Using whitelist `{whitelist['method']}` with {whitelist['length']} pairs\n" message += f"`{', '.join(whitelist['whitelist'])}`" logger.debug(message) diff --git a/freqtrade/tests/rpc/test_rpc.py b/freqtrade/tests/rpc/test_rpc.py index b0b598654..54bb45cd4 100644 --- a/freqtrade/tests/rpc/test_rpc.py +++ b/freqtrade/tests/rpc/test_rpc.py @@ -662,12 +662,15 @@ def test_rpc_whitelist(mocker, default_conf) -> None: def test_rpc_whitelist_dynamic(mocker, default_conf) -> None: patch_coinmarketcap(mocker) patch_exchange(mocker) - default_conf['dynamic_whitelist'] = 4 + default_conf['whitelist'] = {'method': 'VolumePairList', + 'config': {'number_assets': 4} + } mocker.patch('freqtrade.exchange.Exchange.exchange_has', MagicMock(return_value=True)) mocker.patch('freqtrade.rpc.telegram.Telegram', MagicMock()) freqtradebot = FreqtradeBot(default_conf) rpc = RPC(freqtradebot) ret = rpc._rpc_whitelist() - assert ret['method'] == 4 + assert ret['method'] == 'VolumePairList' + assert ret['length'] == 4 assert ret['whitelist'] == default_conf['exchange']['pair_whitelist'] diff --git a/freqtrade/tests/rpc/test_rpc_manager.py b/freqtrade/tests/rpc/test_rpc_manager.py index cbb858522..9359fc927 100644 --- a/freqtrade/tests/rpc/test_rpc_manager.py +++ b/freqtrade/tests/rpc/test_rpc_manager.py @@ -128,7 +128,9 @@ def test_startupmessages_telegram_enabled(mocker, default_conf, caplog) -> None: telegram_mock.reset_mock() default_conf['dry_run'] = True - default_conf['dynamic_whitelist'] = 20 + default_conf['whitelist'] = {'method': 'VolumePairList', + 'config': {'number_assets': 20} + } rpc_manager.startup_messages(default_conf) assert telegram_mock.call_count == 3 diff --git a/freqtrade/tests/rpc/test_rpc_telegram.py b/freqtrade/tests/rpc/test_rpc_telegram.py index 4f29c5a03..8a97abf58 100644 --- a/freqtrade/tests/rpc/test_rpc_telegram.py +++ b/freqtrade/tests/rpc/test_rpc_telegram.py @@ -1034,7 +1034,9 @@ def test_whitelist_dynamic(default_conf, update, mocker) -> None: _send_msg=msg_mock ) mocker.patch('freqtrade.exchange.Exchange.exchange_has', MagicMock(return_value=True)) - default_conf['dynamic_whitelist'] = 4 + default_conf['whitelist'] = {'method': 'VolumePairList', + 'config': {'number_assets': 4} + } freqtradebot = get_patched_freqtradebot(mocker, default_conf) telegram = Telegram(freqtradebot) diff --git a/freqtrade/tests/test_acl_pair.py b/freqtrade/tests/test_acl_pair.py index a12de9a81..5acd9abda 100644 --- a/freqtrade/tests/test_acl_pair.py +++ b/freqtrade/tests/test_acl_pair.py @@ -54,14 +54,16 @@ def test_refresh_pairlists(mocker, markets, whitelist_conf): def test_refresh_whitelist_dynamic(mocker, markets, tickers, whitelist_conf): - whitelist_conf['dynamic_whitelist'] = 5 - freqtradebot = get_patched_freqtradebot(mocker, whitelist_conf) + whitelist_conf['whitelist'] = {'method': 'VolumePairList', + 'config': {'number_assets': 5} + } mocker.patch.multiple( 'freqtrade.exchange.Exchange', get_markets=markets, get_tickers=tickers, exchange_has=MagicMock(return_value=True) ) + freqtradebot = get_patched_freqtradebot(mocker, whitelist_conf) # argument: use the whitelist dynamically by exchange-volume whitelist = ['ETH/BTC', 'TKN/BTC'] diff --git a/freqtrade/tests/test_arguments.py b/freqtrade/tests/test_arguments.py index e09aeb1df..d28ab30af 100644 --- a/freqtrade/tests/test_arguments.py +++ b/freqtrade/tests/test_arguments.py @@ -17,7 +17,8 @@ def test_parse_args_none() -> None: def test_parse_args_defaults() -> None: args = Arguments([], '').get_parsed_arg() assert args.config == 'config.json' - assert args.dynamic_whitelist is None + assert args.strategy_path is None + assert args.datadir is None assert args.loglevel == 0 diff --git a/freqtrade/tests/test_configuration.py b/freqtrade/tests/test_configuration.py index 951d831b2..5eda9d871 100644 --- a/freqtrade/tests/test_configuration.py +++ b/freqtrade/tests/test_configuration.py @@ -102,7 +102,7 @@ def test_load_config(default_conf, mocker) -> None: assert validated_conf.get('strategy') == 'DefaultStrategy' assert validated_conf.get('strategy_path') is None - assert 'dynamic_whitelist' not in validated_conf + assert 'edge' not in validated_conf def test_load_config_with_params(default_conf, mocker) -> None: @@ -133,7 +133,6 @@ def test_load_config_with_params(default_conf, mocker) -> None: )) arglist = [ - '--dynamic-whitelist', '10', '--strategy', 'TestStrategy', '--strategy-path', '/some/path' ] @@ -152,7 +151,6 @@ def test_load_config_with_params(default_conf, mocker) -> None: )) arglist = [ - '--dynamic-whitelist', '10', '--strategy', 'TestStrategy', '--strategy-path', '/some/path' ] diff --git a/freqtrade/tests/test_freqtradebot.py b/freqtrade/tests/test_freqtradebot.py index ca604340b..0cdc9b0e6 100644 --- a/freqtrade/tests/test_freqtradebot.py +++ b/freqtrade/tests/test_freqtradebot.py @@ -2532,7 +2532,9 @@ def test_order_book_ask_strategy(default_conf, limit_buy_order, limit_sell_order def test_startup_messages(default_conf, mocker): - default_conf['dynamic_whitelist'] = 20 + default_conf['whitelist'] = {'method': 'VolumePairList', + 'config': {'number_assets': 20} + } mocker.patch('freqtrade.exchange.Exchange.exchange_has', MagicMock(return_value=True)) freqtrade = get_patched_freqtradebot(mocker, default_conf) assert freqtrade.state is State.RUNNING From 18ad3388b4e23158bc540a7fb45dd102949c0071 Mon Sep 17 00:00:00 2001 From: Matthias Date: Mon, 3 Dec 2018 20:38:15 +0100 Subject: [PATCH 558/699] Some more tests adapted to pairlists --- freqtrade/tests/rpc/test_rpc.py | 2 +- freqtrade/tests/rpc/test_rpc_manager.py | 4 ++-- freqtrade/tests/rpc/test_rpc_telegram.py | 4 ++-- freqtrade/tests/test_freqtradebot.py | 2 +- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/freqtrade/tests/rpc/test_rpc.py b/freqtrade/tests/rpc/test_rpc.py index 54bb45cd4..87124a9a7 100644 --- a/freqtrade/tests/rpc/test_rpc.py +++ b/freqtrade/tests/rpc/test_rpc.py @@ -655,7 +655,7 @@ def test_rpc_whitelist(mocker, default_conf) -> None: freqtradebot = FreqtradeBot(default_conf) rpc = RPC(freqtradebot) ret = rpc._rpc_whitelist() - assert ret['method'] == 'static' + assert ret['method'] == 'StaticPairList' assert ret['whitelist'] == default_conf['exchange']['pair_whitelist'] diff --git a/freqtrade/tests/rpc/test_rpc_manager.py b/freqtrade/tests/rpc/test_rpc_manager.py index 9359fc927..15d9c20c6 100644 --- a/freqtrade/tests/rpc/test_rpc_manager.py +++ b/freqtrade/tests/rpc/test_rpc_manager.py @@ -121,7 +121,7 @@ def test_startupmessages_telegram_enabled(mocker, default_conf, caplog) -> None: freqtradebot = get_patched_freqtradebot(mocker, default_conf) rpc_manager = RPCManager(freqtradebot) - rpc_manager.startup_messages(default_conf) + rpc_manager.startup_messages(default_conf, freqtradebot.pairlists) assert telegram_mock.call_count == 3 assert "*Exchange:* `bittrex`" in telegram_mock.call_args_list[1][0][0]['status'] @@ -132,6 +132,6 @@ def test_startupmessages_telegram_enabled(mocker, default_conf, caplog) -> None: 'config': {'number_assets': 20} } - rpc_manager.startup_messages(default_conf) + rpc_manager.startup_messages(default_conf, freqtradebot.pairlists) assert telegram_mock.call_count == 3 assert "Dry run is enabled." in telegram_mock.call_args_list[0][0][0]['status'] diff --git a/freqtrade/tests/rpc/test_rpc_telegram.py b/freqtrade/tests/rpc/test_rpc_telegram.py index 8a97abf58..f63393935 100644 --- a/freqtrade/tests/rpc/test_rpc_telegram.py +++ b/freqtrade/tests/rpc/test_rpc_telegram.py @@ -1021,7 +1021,7 @@ def test_whitelist_static(default_conf, update, mocker) -> None: telegram._whitelist(bot=MagicMock(), update=update) assert msg_mock.call_count == 1 - assert ('Using static whitelist with `4` pairs \n`ETH/BTC, LTC/BTC, XRP/BTC, NEO/BTC`' + assert ('Using whitelist `StaticPairList` with 4 pairs\n`ETH/BTC, LTC/BTC, XRP/BTC, NEO/BTC`' in msg_mock.call_args_list[0][0][0]) @@ -1043,7 +1043,7 @@ def test_whitelist_dynamic(default_conf, update, mocker) -> None: telegram._whitelist(bot=MagicMock(), update=update) assert msg_mock.call_count == 1 - assert ('Dynamic whitelist with `4` pairs\n`ETH/BTC, LTC/BTC, XRP/BTC, NEO/BTC`' + assert ('Using whitelist `VolumePairList` with 4 pairs\n`ETH/BTC, LTC/BTC, XRP/BTC, NEO/BTC`' in msg_mock.call_args_list[0][0][0]) diff --git a/freqtrade/tests/test_freqtradebot.py b/freqtrade/tests/test_freqtradebot.py index 0cdc9b0e6..a10389261 100644 --- a/freqtrade/tests/test_freqtradebot.py +++ b/freqtrade/tests/test_freqtradebot.py @@ -217,7 +217,7 @@ def test_edge_called_in_process(mocker, edge_conf) -> None: patch_exchange(mocker) freqtrade = FreqtradeBot(edge_conf) - freqtrade._refresh_whitelist = _refresh_whitelist + freqtrade.pairlists._validate_whitelist = _refresh_whitelist patch_get_signal(freqtrade) freqtrade._process() assert freqtrade.active_pair_whitelist == ['NEO/BTC', 'LTC/BTC'] From 0929f59680f302b82b61a6894b9424c784892796 Mon Sep 17 00:00:00 2001 From: Matthias Date: Mon, 3 Dec 2018 20:48:51 +0100 Subject: [PATCH 559/699] Refactor pairlist-tests --- freqtrade/constants.py | 4 ++-- freqtrade/tests/pairlist/__init__.py | 0 .../{test_acl_pair.py => pairlist/test_pairlist.py} | 12 ++++++++++++ 3 files changed, 14 insertions(+), 2 deletions(-) create mode 100644 freqtrade/tests/pairlist/__init__.py rename freqtrade/tests/{test_acl_pair.py => pairlist/test_pairlist.py} (89%) diff --git a/freqtrade/constants.py b/freqtrade/constants.py index c200137cb..e0799d46f 100644 --- a/freqtrade/constants.py +++ b/freqtrade/constants.py @@ -15,7 +15,7 @@ DEFAULT_DB_DRYRUN_URL = 'sqlite://' UNLIMITED_STAKE_AMOUNT = 'unlimited' REQUIRED_ORDERTYPES = ['buy', 'sell', 'stoploss', 'stoploss_on_exchange'] ORDERTYPE_POSSIBILITIES = ['limit', 'market'] - +AVAILABLE_PAIRLISTS = ['StaticPairList', 'VolumePairList'] TICKER_INTERVAL_MINUTES = { '1m': 1, @@ -127,7 +127,7 @@ CONF_SCHEMA = { 'whitelist': { 'type': 'object', 'properties': { - 'method': {'type': 'string'}, + 'method': {'type': 'string', 'enum': AVAILABLE_PAIRLISTS}, 'config': {'type': 'object'} }, 'required': ['method'] diff --git a/freqtrade/tests/pairlist/__init__.py b/freqtrade/tests/pairlist/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/freqtrade/tests/test_acl_pair.py b/freqtrade/tests/pairlist/test_pairlist.py similarity index 89% rename from freqtrade/tests/test_acl_pair.py rename to freqtrade/tests/pairlist/test_pairlist.py index 5acd9abda..d0a5bb1f6 100644 --- a/freqtrade/tests/test_acl_pair.py +++ b/freqtrade/tests/pairlist/test_pairlist.py @@ -3,6 +3,7 @@ from unittest.mock import MagicMock from freqtrade import OperationalException +from freqtrade.constants import AVAILABLE_PAIRLISTS from freqtrade.tests.conftest import get_patched_freqtradebot import pytest @@ -22,6 +23,9 @@ def whitelist_conf(default_conf): default_conf['exchange']['pair_blacklist'] = [ 'BLK/BTC' ] + default_conf['whitelist'] = {'method': 'StaticPairList', + 'config': {'number_assets': 3} + } return default_conf @@ -118,3 +122,11 @@ def test_gen_pair_whitelist_not_supported(mocker, default_conf, tickers) -> None with pytest.raises(OperationalException): get_patched_freqtradebot(mocker, default_conf) + + +@pytest.mark.parametrize("pairlist", AVAILABLE_PAIRLISTS) +def test_pairlist_class(mocker, whitelist_conf, pairlist): + whitelist_conf['whitelist']['method'] = pairlist + freqtrade = get_patched_freqtradebot(mocker, whitelist_conf) + assert freqtrade.pairlists.name == pairlist + # TODO: add more tests From ab60571ac7117ba5a140ae9d7c41316c7529f60a Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 4 Dec 2018 06:13:39 +0100 Subject: [PATCH 560/699] Add sample config --- config_full.json.example | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/config_full.json.example b/config_full.json.example index 6134e9cad..da4de5cec 100644 --- a/config_full.json.example +++ b/config_full.json.example @@ -39,6 +39,12 @@ "stoploss": "market", "stoploss_on_exchange": "false" }, + "whitelist": { + "method": "VolumePairList", + "config": { + "number_assets": 20 + } + }, "exchange": { "name": "bittrex", "key": "your_exchange_key", From ba3218a87d7928ee697da0796182c78b323a7ba1 Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 4 Dec 2018 07:12:56 +0100 Subject: [PATCH 561/699] Support multiple sorting variants --- config_full.json.example | 3 ++- freqtrade/pairlist/VolumePairList.py | 12 +++++++++++- 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/config_full.json.example b/config_full.json.example index da4de5cec..551d39452 100644 --- a/config_full.json.example +++ b/config_full.json.example @@ -42,7 +42,8 @@ "whitelist": { "method": "VolumePairList", "config": { - "number_assets": 20 + "number_assets": 20, + "sort_key": "quoteVolume" } }, "exchange": { diff --git a/freqtrade/pairlist/VolumePairList.py b/freqtrade/pairlist/VolumePairList.py index f45c57676..aec35a727 100644 --- a/freqtrade/pairlist/VolumePairList.py +++ b/freqtrade/pairlist/VolumePairList.py @@ -12,6 +12,8 @@ from freqtrade.pairlist.StaticPairList import StaticPairList from freqtrade import OperationalException logger = logging.getLogger(__name__) +SORT_VALUES = ['askVolume', 'bidVolume', 'quoteVolume'] + class VolumePairList(StaticPairList): @@ -22,13 +24,21 @@ class VolumePairList(StaticPairList): self._whitelist = self._config['exchange']['pair_whitelist'] self._blacklist = self._config['exchange'].get('pair_blacklist', []) self._number_pairs = self._whitelistconf['number_assets'] + self._sort_key = self._whitelistconf.get('sort_key', 'quoteVolume') + if not self._freqtrade.exchange.exchange_has('fetchTickers'): raise OperationalException( 'Exchange does not support dynamic whitelist.' 'Please edit your config and restart the bot' ) + if not self.validate_keys(self._sort_key): + raise OperationalException( + f'key {self._sort_key} not in {SORT_VALUES}') # self.refresh_whitelist() + def validate_keys(self, key): + return key in SORT_VALUES + def short_desc(self) -> str: """ Short whitelist method description - used for startup-messages @@ -41,7 +51,7 @@ class VolumePairList(StaticPairList): Refreshes whitelist and assigns it to self._whitelist """ # Generate dynamic whitelist - pairs = self._gen_pair_whitelist(self._config['stake_currency']) + pairs = self._gen_pair_whitelist(self._config['stake_currency'], self._sort_key) # Validate whitelist to only have active market pairs self._whitelist = self._validate_whitelist(pairs)[:self._number_pairs] From bf678164c79cb6ee4445561cfd156ce3fb75f46f Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 4 Dec 2018 07:16:34 +0100 Subject: [PATCH 562/699] remove default param - fix tests --- freqtrade/pairlist/VolumePairList.py | 2 +- freqtrade/tests/pairlist/test_pairlist.py | 44 ++++++++++++++++------- 2 files changed, 33 insertions(+), 13 deletions(-) diff --git a/freqtrade/pairlist/VolumePairList.py b/freqtrade/pairlist/VolumePairList.py index aec35a727..30f60f252 100644 --- a/freqtrade/pairlist/VolumePairList.py +++ b/freqtrade/pairlist/VolumePairList.py @@ -56,7 +56,7 @@ class VolumePairList(StaticPairList): self._whitelist = self._validate_whitelist(pairs)[:self._number_pairs] @cached(TTLCache(maxsize=1, ttl=1800)) - def _gen_pair_whitelist(self, base_currency: str, key: str = 'quoteVolume') -> List[str]: + def _gen_pair_whitelist(self, base_currency: str, key: str) -> List[str]: """ Updates the whitelist with with a dynamically generated list :param base_currency: base currency as str diff --git a/freqtrade/tests/pairlist/test_pairlist.py b/freqtrade/tests/pairlist/test_pairlist.py index d0a5bb1f6..12ca559f6 100644 --- a/freqtrade/tests/pairlist/test_pairlist.py +++ b/freqtrade/tests/pairlist/test_pairlist.py @@ -76,7 +76,7 @@ def test_refresh_whitelist_dynamic(mocker, markets, tickers, whitelist_conf): assert whitelist == freqtradebot.pairlists.whitelist -def test_refresh_whitelist_dynamic_empty(mocker, markets_empty, whitelist_conf): +def test_VolumePairList_refresh_empty(mocker, markets_empty, whitelist_conf): freqtradebot = get_patched_freqtradebot(mocker, whitelist_conf) mocker.patch('freqtrade.exchange.Exchange.get_markets', markets_empty) @@ -89,27 +89,27 @@ def test_refresh_whitelist_dynamic_empty(mocker, markets_empty, whitelist_conf): assert set(whitelist) == set(pairslist) -def test_gen_pair_whitelist(mocker, default_conf, tickers) -> None: - freqtrade = get_patched_freqtradebot(mocker, default_conf) - mocker.patch('freqtrade.exchange.Exchange.get_tickers', tickers) +def test_VolumePairList_whitelist_gen(mocker, whitelist_conf, markets, tickers) -> None: + whitelist_conf['whitelist']['method'] = 'VolumePairList' mocker.patch('freqtrade.exchange.Exchange.exchange_has', MagicMock(return_value=True)) + freqtrade = get_patched_freqtradebot(mocker, whitelist_conf) + mocker.patch('freqtrade.exchange.Exchange.get_markets', markets) + mocker.patch('freqtrade.exchange.Exchange.get_tickers', tickers) # Test to retrieved BTC sorted on quoteVolume (default) - freqtrade.pairlists.refresh_whitelist() - - whitelist = freqtrade.pairlists.whitelist + whitelist = freqtrade.pairlists._gen_pair_whitelist(base_currency='BTC', key='quoteVolume') assert whitelist == ['ETH/BTC', 'TKN/BTC', 'BLK/BTC', 'LTC/BTC'] # Test to retrieve BTC sorted on bidVolume - whitelist = freqtrade._gen_pair_whitelist(base_currency='BTC', key='bidVolume') + whitelist = freqtrade.pairlists._gen_pair_whitelist(base_currency='BTC', key='bidVolume') assert whitelist == ['LTC/BTC', 'TKN/BTC', 'ETH/BTC', 'BLK/BTC'] # Test with USDT sorted on quoteVolume (default) - whitelist = freqtrade._gen_pair_whitelist(base_currency='USDT') + whitelist = freqtrade.pairlists._gen_pair_whitelist(base_currency='USDT', key='quoteVolume') assert whitelist == ['TKN/USDT', 'ETH/USDT', 'LTC/USDT', 'BLK/USDT'] # Test with ETH (our fixture does not have ETH, so result should be empty) - whitelist = freqtrade._gen_pair_whitelist(base_currency='ETH') + whitelist = freqtrade.pairlists._gen_pair_whitelist(base_currency='ETH', key='quoteVolume') assert whitelist == [] @@ -125,8 +125,28 @@ def test_gen_pair_whitelist_not_supported(mocker, default_conf, tickers) -> None @pytest.mark.parametrize("pairlist", AVAILABLE_PAIRLISTS) -def test_pairlist_class(mocker, whitelist_conf, pairlist): +def test_pairlist_class(mocker, whitelist_conf, markets, pairlist): whitelist_conf['whitelist']['method'] = pairlist + mocker.patch('freqtrade.exchange.Exchange.get_markets', markets) + mocker.patch('freqtrade.exchange.Exchange.exchange_has', MagicMock(return_value=True)) freqtrade = get_patched_freqtradebot(mocker, whitelist_conf) + assert freqtrade.pairlists.name == pairlist - # TODO: add more tests + assert pairlist in freqtrade.pairlists.short_desc() + assert isinstance(freqtrade.pairlists.whitelist, list) + assert isinstance(freqtrade.pairlists.blacklist, list) + + whitelist = ['ETH/BTC', 'TKN/BTC'] + new_whitelist = freqtrade.pairlists._validate_whitelist(whitelist) + + assert set(whitelist) == set(new_whitelist) + + whitelist = ['ETH/BTC', 'TKN/BTC', 'TRX/ETH'] + new_whitelist = freqtrade.pairlists._validate_whitelist(whitelist) + # TRX/ETH was removed + assert set(['ETH/BTC', 'TKN/BTC']) == set(new_whitelist) + + whitelist = ['ETH/BTC', 'TKN/BTC', 'BLK/BTC'] + new_whitelist = freqtrade.pairlists._validate_whitelist(whitelist) + # BLK/BTC is in blacklist ... + assert set(['ETH/BTC', 'TKN/BTC']) == set(new_whitelist) From 32b6cd9dff8f025e7400f51b4c14b81806f4f149 Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Tue, 4 Dec 2018 13:34:07 +0100 Subject: [PATCH 563/699] Update ccxt from 1.17.574 to 1.17.581 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 592c009d6..2befc4d5c 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -ccxt==1.17.574 +ccxt==1.17.581 SQLAlchemy==1.2.14 python-telegram-bot==11.1.0 arrow==0.12.1 From e7684b446b3c737a01da2d88dd0be08a4f4b78d8 Mon Sep 17 00:00:00 2001 From: misagh Date: Tue, 4 Dec 2018 17:05:35 +0100 Subject: [PATCH 564/699] capital in trade extracted to a separated argument --- freqtrade/edge/__init__.py | 10 +++++++--- freqtrade/freqtradebot.py | 2 +- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/freqtrade/edge/__init__.py b/freqtrade/edge/__init__.py index a212c9849..49acbd3e7 100644 --- a/freqtrade/edge/__init__.py +++ b/freqtrade/edge/__init__.py @@ -160,19 +160,23 @@ class Edge(): return True - def stake_amount(self, pair: str, free_capital: float, total_capital: float) -> float: + def stake_amount(self, pair: str, free_capital: float, + total_capital: float, capital_in_trade: float) -> float: stoploss = self.stoploss(pair) - available_capital = total_capital * self._capital_percentage + available_capital = (total_capital + capital_in_trade) * self._capital_percentage allowed_capital_at_risk = available_capital * self._allowed_risk max_position_size = abs(allowed_capital_at_risk / stoploss) position_size = min(max_position_size, free_capital) if pair in self._cached_pairs: logger.info( 'winrate: %s, expectancy: %s, position size: %s, pair: %s,' + ' capital in trade: %s, free capital: %s, total capital: %s,' ' stoploss: %s, available capital: %s.', self._cached_pairs[pair].winrate, self._cached_pairs[pair].expectancy, - position_size, pair, stoploss, available_capital + position_size, pair, + capital_in_trade, free_capital, total_capital, + stoploss, available_capital ) return round(position_size, 15) diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index 6baef76bc..d782f4342 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -305,7 +305,7 @@ class FreqtradeBot(object): return self.edge.stake_amount( pair, self.wallets.get_free(self.config['stake_currency']), - self.wallets.get_total(self.config['stake_currency']) + + self.wallets.get_total(self.config['stake_currency']), Trade.total_open_trades_stakes() ) else: From 24f9ea29c605351ee61a7d4c8dbdabe7b158ca53 Mon Sep 17 00:00:00 2001 From: misagh Date: Tue, 4 Dec 2018 17:13:46 +0100 Subject: [PATCH 565/699] tests fixed --- freqtrade/tests/edge/test_edge.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/freqtrade/tests/edge/test_edge.py b/freqtrade/tests/edge/test_edge.py index 74227cfe8..008413ff1 100644 --- a/freqtrade/tests/edge/test_edge.py +++ b/freqtrade/tests/edge/test_edge.py @@ -174,15 +174,18 @@ def test_stake_amount(mocker, edge_conf): )) free = 100 total = 100 - assert edge.stake_amount('E/F', free, total) == 25 + in_trade = 25 + assert edge.stake_amount('E/F', free, total, in_trade) == 31.25 free = 20 total = 100 - assert edge.stake_amount('E/F', free, total) == 20 + in_trade = 25 + assert edge.stake_amount('E/F', free, total, in_trade) == 20 free = 0 total = 100 - assert edge.stake_amount('E/F', free, total) == 0 + in_trade = 25 + assert edge.stake_amount('E/F', free, total, in_trade) == 0 def test_nonexisting_stake_amount(mocker, edge_conf): @@ -193,7 +196,8 @@ def test_nonexisting_stake_amount(mocker, edge_conf): 'E/F': PairInfo(-0.11, 0.66, 3.71, 0.50, 1.71, 10, 60), } )) - assert edge.stake_amount('N/O', 1, 2) == 0.1 + # should use strategy stoploss + assert edge.stake_amount('N/O', 1, 2, 1) == 0.15 def _validate_ohlc(buy_ohlc_sell_matrice): From 33e9ed5a5ee8290813d19e9bc7a3d54d6958f9c7 Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 4 Dec 2018 19:58:26 +0100 Subject: [PATCH 566/699] Print sellreason in sell-message --- docs/webhook-config.md | 1 + freqtrade/freqtradebot.py | 1 + freqtrade/rpc/telegram.py | 23 ++++++++++++----------- 3 files changed, 14 insertions(+), 11 deletions(-) diff --git a/docs/webhook-config.md b/docs/webhook-config.md index 71524187a..e5509d6c9 100644 --- a/docs/webhook-config.md +++ b/docs/webhook-config.md @@ -66,6 +66,7 @@ Possible parameters are: * profit_fiat * stake_currency * fiat_currency +* sell_reason ### Webhookstatus diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index a73a2e98f..53c012a17 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -834,6 +834,7 @@ class FreqtradeBot(object): 'current_rate': current_rate, 'profit_amount': profit_trade, 'profit_percent': profit_percent, + 'sell_reason': sell_reason.value } # For regular case, when the configuration exists diff --git a/freqtrade/rpc/telegram.py b/freqtrade/rpc/telegram.py index 55c5abef2..032593e9e 100644 --- a/freqtrade/rpc/telegram.py +++ b/freqtrade/rpc/telegram.py @@ -125,9 +125,9 @@ class Telegram(RPC): else: msg['stake_amount_fiat'] = 0 - message = "*{exchange}:* Buying [{pair}]({market_url})\n" \ - "with limit `{limit:.8f}\n" \ - "({stake_amount:.6f} {stake_currency}".format(**msg) + message = ("*{exchange}:* Buying [{pair}]({market_url})\n" + "with limit `{limit:.8f}\n" + "({stake_amount:.6f} {stake_currency}").format(**msg) if msg.get('fiat_currency', None): message += ",{stake_amount_fiat:.3f} {fiat_currency}".format(**msg) @@ -137,12 +137,13 @@ class Telegram(RPC): msg['amount'] = round(msg['amount'], 8) msg['profit_percent'] = round(msg['profit_percent'] * 100, 2) - message = "*{exchange}:* Selling [{pair}]({market_url})\n" \ - "*Limit:* `{limit:.8f}`\n" \ - "*Amount:* `{amount:.8f}`\n" \ - "*Open Rate:* `{open_rate:.8f}`\n" \ - "*Current Rate:* `{current_rate:.8f}`\n" \ - "*Profit:* `{profit_percent:.2f}%`".format(**msg) + message = ("*{exchange}:* Selling [{pair}]({market_url})\n" + "*Limit:* `{limit:.8f}`\n" + "*Amount:* `{amount:.8f}`\n" + "*Open Rate:* `{open_rate:.8f}`\n" + "*Current Rate:* `{current_rate:.8f}`\n" + "*Sell Reason:* `{sell_reason}`\n" + "*Profit:* `{profit_percent:.2f}%`").format(**msg) # Check if all sell properties are available. # This might not be the case if the message origin is triggered by /forcesell @@ -150,8 +151,8 @@ class Telegram(RPC): and self._fiat_converter): msg['profit_fiat'] = self._fiat_converter.convert_amount( msg['profit_amount'], msg['stake_currency'], msg['fiat_currency']) - message += '` ({gain}: {profit_amount:.8f} {stake_currency}`' \ - '` / {profit_fiat:.3f} {fiat_currency})`'.format(**msg) + message += ('` ({gain}: {profit_amount:.8f} {stake_currency}`' + '` / {profit_fiat:.3f} {fiat_currency})`').format(**msg) elif msg['type'] == RPCMessageType.STATUS_NOTIFICATION: message = '*Status:* `{status}`'.format(**msg) From 4143e2c032297c796c3041010225537c3d1a586b Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 4 Dec 2018 19:58:43 +0100 Subject: [PATCH 567/699] adapt tests to send sell-reason in sell-message --- freqtrade/tests/rpc/test_rpc_telegram.py | 42 +++++++++++++++--------- freqtrade/tests/rpc/test_rpc_webhook.py | 2 ++ freqtrade/tests/test_freqtradebot.py | 7 ++++ 3 files changed, 35 insertions(+), 16 deletions(-) diff --git a/freqtrade/tests/rpc/test_rpc_telegram.py b/freqtrade/tests/rpc/test_rpc_telegram.py index a8e8bf003..900f583eb 100644 --- a/freqtrade/tests/rpc/test_rpc_telegram.py +++ b/freqtrade/tests/rpc/test_rpc_telegram.py @@ -17,6 +17,7 @@ from freqtrade.freqtradebot import FreqtradeBot from freqtrade.persistence import Trade from freqtrade.rpc import RPCMessageType from freqtrade.rpc.telegram import Telegram, authorized_only +from freqtrade.strategy.interface import SellType from freqtrade.state import State from freqtrade.tests.conftest import (get_patched_freqtradebot, log_has, patch_exchange) @@ -729,6 +730,7 @@ def test_forcesell_handle(default_conf, update, ticker, fee, 'profit_percent': 0.0611052, 'stake_currency': 'BTC', 'fiat_currency': 'USD', + 'sell_reason': SellType.FORCE_SELL.value } == last_msg @@ -782,6 +784,7 @@ def test_forcesell_down_handle(default_conf, update, ticker, fee, 'profit_percent': -0.05478342, 'stake_currency': 'BTC', 'fiat_currency': 'USD', + 'sell_reason': SellType.FORCE_SELL.value } == last_msg @@ -827,6 +830,7 @@ def test_forcesell_all_handle(default_conf, update, ticker, fee, markets, mocker 'profit_percent': -0.00589291, 'stake_currency': 'BTC', 'fiat_currency': 'USD', + 'sell_reason': SellType.FORCE_SELL.value } == msg @@ -1127,16 +1131,18 @@ def test_send_msg_sell_notification(default_conf, mocker) -> None: 'profit_amount': -0.05746268, 'profit_percent': -0.57405275, 'stake_currency': 'ETH', - 'fiat_currency': 'USD' + 'fiat_currency': 'USD', + 'sell_reason': SellType.STOP_LOSS.value }) assert msg_mock.call_args[0][0] \ - == '*Binance:* Selling [KEY/ETH]' \ - '(https://www.binance.com/tradeDetail.html?symbol=KEY_ETH)\n' \ - '*Limit:* `0.00003201`\n' \ - '*Amount:* `1333.33333333`\n' \ - '*Open Rate:* `0.00007500`\n' \ - '*Current Rate:* `0.00003201`\n' \ - '*Profit:* `-57.41%`` (loss: -0.05746268 ETH`` / -24.812 USD)`' + == ('*Binance:* Selling [KEY/ETH]' + '(https://www.binance.com/tradeDetail.html?symbol=KEY_ETH)\n' + '*Limit:* `0.00003201`\n' + '*Amount:* `1333.33333333`\n' + '*Open Rate:* `0.00007500`\n' + '*Current Rate:* `0.00003201`\n' + '*Sell Reason:* `stop_loss`\n' + '*Profit:* `-57.41%`` (loss: -0.05746268 ETH`` / -24.812 USD)`') msg_mock.reset_mock() telegram.send_msg({ @@ -1152,15 +1158,17 @@ def test_send_msg_sell_notification(default_conf, mocker) -> None: 'profit_amount': -0.05746268, 'profit_percent': -0.57405275, 'stake_currency': 'ETH', + 'sell_reason': SellType.STOP_LOSS.value }) assert msg_mock.call_args[0][0] \ - == '*Binance:* Selling [KEY/ETH]' \ - '(https://www.binance.com/tradeDetail.html?symbol=KEY_ETH)\n' \ - '*Limit:* `0.00003201`\n' \ - '*Amount:* `1333.33333333`\n' \ - '*Open Rate:* `0.00007500`\n' \ - '*Current Rate:* `0.00003201`\n' \ - '*Profit:* `-57.41%`' + == ('*Binance:* Selling [KEY/ETH]' + '(https://www.binance.com/tradeDetail.html?symbol=KEY_ETH)\n' + '*Limit:* `0.00003201`\n' + '*Amount:* `1333.33333333`\n' + '*Open Rate:* `0.00007500`\n' + '*Current Rate:* `0.00003201`\n' + '*Sell Reason:* `stop_loss`\n' + '*Profit:* `-57.41%`') # Reset singleton function to avoid random breaks telegram._fiat_converter.convert_amount = old_convamount @@ -1278,7 +1286,8 @@ def test_send_msg_sell_notification_no_fiat(default_conf, mocker) -> None: 'profit_amount': -0.05746268, 'profit_percent': -0.57405275, 'stake_currency': 'ETH', - 'fiat_currency': 'USD' + 'fiat_currency': 'USD', + 'sell_reason': SellType.STOP_LOSS.value }) assert msg_mock.call_args[0][0] \ == '*Binance:* Selling [KEY/ETH]' \ @@ -1287,6 +1296,7 @@ def test_send_msg_sell_notification_no_fiat(default_conf, mocker) -> None: '*Amount:* `1333.33333333`\n' \ '*Open Rate:* `0.00007500`\n' \ '*Current Rate:* `0.00003201`\n' \ + '*Sell Reason:* `stop_loss`\n' \ '*Profit:* `-57.41%`' diff --git a/freqtrade/tests/rpc/test_rpc_webhook.py b/freqtrade/tests/rpc/test_rpc_webhook.py index bfe0416b0..002308815 100644 --- a/freqtrade/tests/rpc/test_rpc_webhook.py +++ b/freqtrade/tests/rpc/test_rpc_webhook.py @@ -7,6 +7,7 @@ from requests import RequestException from freqtrade.rpc import RPCMessageType from freqtrade.rpc.webhook import Webhook +from freqtrade.strategy.interface import SellType from freqtrade.tests.conftest import get_patched_freqtradebot, log_has @@ -80,6 +81,7 @@ def test_send_msg(default_conf, mocker): 'profit_amount': 0.001, 'profit_percent': 0.20, 'stake_currency': 'BTC', + 'sell_reason': SellType.STOP_LOSS.value } webhook.send_msg(msg=msg) assert msg_mock.call_count == 1 diff --git a/freqtrade/tests/test_freqtradebot.py b/freqtrade/tests/test_freqtradebot.py index a9b3ffc5d..be84829e2 100644 --- a/freqtrade/tests/test_freqtradebot.py +++ b/freqtrade/tests/test_freqtradebot.py @@ -1513,6 +1513,7 @@ def test_execute_sell_up(default_conf, ticker, fee, ticker_sell_up, markets, moc 'profit_percent': 0.0611052, 'stake_currency': 'BTC', 'fiat_currency': 'USD', + 'sell_reason': SellType.ROI.value } == last_msg @@ -1559,6 +1560,7 @@ def test_execute_sell_down(default_conf, ticker, fee, ticker_sell_down, markets, 'profit_percent': -0.05478342, 'stake_currency': 'BTC', 'fiat_currency': 'USD', + 'sell_reason': SellType.STOP_LOSS.value } == last_msg @@ -1613,6 +1615,8 @@ def test_execute_sell_down_stoploss_on_exchange_dry_run(default_conf, ticker, fe 'profit_percent': -0.01493766, 'stake_currency': 'BTC', 'fiat_currency': 'USD', + 'sell_reason': SellType.STOP_LOSS.value + } == last_msg @@ -1781,6 +1785,8 @@ def test_execute_sell_without_conf_sell_up(default_conf, ticker, fee, 'current_rate': 1.172e-05, 'profit_amount': 6.126e-05, 'profit_percent': 0.0611052, + 'sell_reason': SellType.ROI.value + } == last_msg @@ -1827,6 +1833,7 @@ def test_execute_sell_without_conf_sell_down(default_conf, ticker, fee, 'current_rate': 1.044e-05, 'profit_amount': -5.492e-05, 'profit_percent': -0.05478342, + 'sell_reason': SellType.STOP_LOSS.value } == last_msg From 6ab907bef1f11a0478b11a0423aa7e8041c61f39 Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 4 Dec 2018 20:23:03 +0100 Subject: [PATCH 568/699] Rename config whitelist to pairlist --- config_full.json.example | 2 +- freqtrade/configuration.py | 6 +++--- freqtrade/constants.py | 2 +- freqtrade/freqtradebot.py | 2 +- freqtrade/pairlist/VolumePairList.py | 2 +- freqtrade/tests/pairlist/test_pairlist.py | 22 +++++++++++----------- freqtrade/tests/rpc/test_rpc.py | 6 +++--- freqtrade/tests/rpc/test_rpc_telegram.py | 6 +++--- freqtrade/tests/test_configuration.py | 4 ++-- freqtrade/tests/test_freqtradebot.py | 6 +++--- 10 files changed, 29 insertions(+), 29 deletions(-) diff --git a/config_full.json.example b/config_full.json.example index 551d39452..e21ccbe4f 100644 --- a/config_full.json.example +++ b/config_full.json.example @@ -39,7 +39,7 @@ "stoploss": "market", "stoploss_on_exchange": "false" }, - "whitelist": { + "pairlist": { "method": "VolumePairList", "config": { "number_assets": 20, diff --git a/freqtrade/configuration.py b/freqtrade/configuration.py index 7ddcf1639..cb087a1ab 100644 --- a/freqtrade/configuration.py +++ b/freqtrade/configuration.py @@ -111,9 +111,9 @@ class Configuration(object): # Add dynamic_whitelist if found if 'dynamic_whitelist' in self.args and self.args.dynamic_whitelist: # Update to volumePairList (the previous default) - config['whitelist'] = {'method': 'VolumePairList', - 'config': {'number_assets': self.args.dynamic_whitelist} - } + config['pairlist'] = {'method': 'VolumePairList', + 'config': {'number_assets': self.args.dynamic_whitelist} + } logger.warning( 'Parameter --dynamic-whitelist has been deprecated, ' 'and will be completely replaced by the whitelist dict in the future. ' diff --git a/freqtrade/constants.py b/freqtrade/constants.py index e0799d46f..178aafe79 100644 --- a/freqtrade/constants.py +++ b/freqtrade/constants.py @@ -124,7 +124,7 @@ CONF_SCHEMA = { 'ignore_roi_if_buy_signal_true': {'type': 'boolean'} } }, - 'whitelist': { + 'pairlist': { 'type': 'object', 'properties': { 'method': {'type': 'string', 'enum': AVAILABLE_PAIRLISTS}, diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index 17b818b27..54374b209 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -59,7 +59,7 @@ class FreqtradeBot(object): self.persistence = None self.exchange = Exchange(self.config) self.wallets = Wallets(self.exchange) - if self.config.get('whitelist', {}).get('method') == 'VolumePairList': + if self.config.get('pairlist', {}).get('method') == 'VolumePairList': self.pairlists: StaticPairList = VolumePairList(self, self.config) else: self.pairlists: StaticPairList = StaticPairList(self, self.config) diff --git a/freqtrade/pairlist/VolumePairList.py b/freqtrade/pairlist/VolumePairList.py index 30f60f252..6565caae6 100644 --- a/freqtrade/pairlist/VolumePairList.py +++ b/freqtrade/pairlist/VolumePairList.py @@ -20,7 +20,7 @@ class VolumePairList(StaticPairList): def __init__(self, freqtrade, config: dict) -> None: self._freqtrade = freqtrade self._config = config - self._whitelistconf = self._config.get('whitelist', {}).get('config') + self._whitelistconf = self._config.get('pairlist', {}).get('config') self._whitelist = self._config['exchange']['pair_whitelist'] self._blacklist = self._config['exchange'].get('pair_blacklist', []) self._number_pairs = self._whitelistconf['number_assets'] diff --git a/freqtrade/tests/pairlist/test_pairlist.py b/freqtrade/tests/pairlist/test_pairlist.py index 12ca559f6..b94859efa 100644 --- a/freqtrade/tests/pairlist/test_pairlist.py +++ b/freqtrade/tests/pairlist/test_pairlist.py @@ -23,9 +23,9 @@ def whitelist_conf(default_conf): default_conf['exchange']['pair_blacklist'] = [ 'BLK/BTC' ] - default_conf['whitelist'] = {'method': 'StaticPairList', - 'config': {'number_assets': 3} - } + default_conf['pairlist'] = {'method': 'StaticPairList', + 'config': {'number_assets': 3} + } return default_conf @@ -58,9 +58,9 @@ def test_refresh_pairlists(mocker, markets, whitelist_conf): def test_refresh_whitelist_dynamic(mocker, markets, tickers, whitelist_conf): - whitelist_conf['whitelist'] = {'method': 'VolumePairList', - 'config': {'number_assets': 5} - } + whitelist_conf['pairlist'] = {'method': 'VolumePairList', + 'config': {'number_assets': 5} + } mocker.patch.multiple( 'freqtrade.exchange.Exchange', get_markets=markets, @@ -90,7 +90,7 @@ def test_VolumePairList_refresh_empty(mocker, markets_empty, whitelist_conf): def test_VolumePairList_whitelist_gen(mocker, whitelist_conf, markets, tickers) -> None: - whitelist_conf['whitelist']['method'] = 'VolumePairList' + whitelist_conf['pairlist']['method'] = 'VolumePairList' mocker.patch('freqtrade.exchange.Exchange.exchange_has', MagicMock(return_value=True)) freqtrade = get_patched_freqtradebot(mocker, whitelist_conf) mocker.patch('freqtrade.exchange.Exchange.get_markets', markets) @@ -114,9 +114,9 @@ def test_VolumePairList_whitelist_gen(mocker, whitelist_conf, markets, tickers) def test_gen_pair_whitelist_not_supported(mocker, default_conf, tickers) -> None: - default_conf['whitelist'] = {'method': 'VolumePairList', - 'config': {'number_assets': 10} - } + default_conf['pairlist'] = {'method': 'VolumePairList', + 'config': {'number_assets': 10} + } mocker.patch('freqtrade.exchange.Exchange.get_tickers', tickers) mocker.patch('freqtrade.exchange.Exchange.exchange_has', MagicMock(return_value=False)) @@ -126,7 +126,7 @@ def test_gen_pair_whitelist_not_supported(mocker, default_conf, tickers) -> None @pytest.mark.parametrize("pairlist", AVAILABLE_PAIRLISTS) def test_pairlist_class(mocker, whitelist_conf, markets, pairlist): - whitelist_conf['whitelist']['method'] = pairlist + whitelist_conf['pairlist']['method'] = pairlist mocker.patch('freqtrade.exchange.Exchange.get_markets', markets) mocker.patch('freqtrade.exchange.Exchange.exchange_has', MagicMock(return_value=True)) freqtrade = get_patched_freqtradebot(mocker, whitelist_conf) diff --git a/freqtrade/tests/rpc/test_rpc.py b/freqtrade/tests/rpc/test_rpc.py index 87124a9a7..2b271af31 100644 --- a/freqtrade/tests/rpc/test_rpc.py +++ b/freqtrade/tests/rpc/test_rpc.py @@ -662,9 +662,9 @@ def test_rpc_whitelist(mocker, default_conf) -> None: def test_rpc_whitelist_dynamic(mocker, default_conf) -> None: patch_coinmarketcap(mocker) patch_exchange(mocker) - default_conf['whitelist'] = {'method': 'VolumePairList', - 'config': {'number_assets': 4} - } + default_conf['pairlist'] = {'method': 'VolumePairList', + 'config': {'number_assets': 4} + } mocker.patch('freqtrade.exchange.Exchange.exchange_has', MagicMock(return_value=True)) mocker.patch('freqtrade.rpc.telegram.Telegram', MagicMock()) diff --git a/freqtrade/tests/rpc/test_rpc_telegram.py b/freqtrade/tests/rpc/test_rpc_telegram.py index f63393935..c1a19f2a4 100644 --- a/freqtrade/tests/rpc/test_rpc_telegram.py +++ b/freqtrade/tests/rpc/test_rpc_telegram.py @@ -1034,9 +1034,9 @@ def test_whitelist_dynamic(default_conf, update, mocker) -> None: _send_msg=msg_mock ) mocker.patch('freqtrade.exchange.Exchange.exchange_has', MagicMock(return_value=True)) - default_conf['whitelist'] = {'method': 'VolumePairList', - 'config': {'number_assets': 4} - } + default_conf['pairlist'] = {'method': 'VolumePairList', + 'config': {'number_assets': 4} + } freqtradebot = get_patched_freqtradebot(mocker, default_conf) telegram = Telegram(freqtradebot) diff --git a/freqtrade/tests/test_configuration.py b/freqtrade/tests/test_configuration.py index 5eda9d871..d013f1059 100644 --- a/freqtrade/tests/test_configuration.py +++ b/freqtrade/tests/test_configuration.py @@ -119,8 +119,8 @@ def test_load_config_with_params(default_conf, mocker) -> None: configuration = Configuration(args) validated_conf = configuration.load_config() - assert validated_conf.get('whitelist', {}).get('method') == 'VolumePairList' - assert validated_conf.get('whitelist', {}).get('config').get('number_assets') == 10 + assert validated_conf.get('pairlist', {}).get('method') == 'VolumePairList' + assert validated_conf.get('pairlist', {}).get('config').get('number_assets') == 10 assert validated_conf.get('strategy') == 'TestStrategy' assert validated_conf.get('strategy_path') == '/some/path' assert validated_conf.get('db_url') == 'sqlite:///someurl' diff --git a/freqtrade/tests/test_freqtradebot.py b/freqtrade/tests/test_freqtradebot.py index a10389261..6c61b893c 100644 --- a/freqtrade/tests/test_freqtradebot.py +++ b/freqtrade/tests/test_freqtradebot.py @@ -2532,9 +2532,9 @@ def test_order_book_ask_strategy(default_conf, limit_buy_order, limit_sell_order def test_startup_messages(default_conf, mocker): - default_conf['whitelist'] = {'method': 'VolumePairList', - 'config': {'number_assets': 20} - } + default_conf['pairlist'] = {'method': 'VolumePairList', + 'config': {'number_assets': 20} + } mocker.patch('freqtrade.exchange.Exchange.exchange_has', MagicMock(return_value=True)) freqtrade = get_patched_freqtradebot(mocker, default_conf) assert freqtrade.state is State.RUNNING From 1c3ce265f15060539898a5f85ab7690f75c69b2b Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 4 Dec 2018 20:24:52 +0100 Subject: [PATCH 569/699] documentation for pairlists --- docs/bot-usage.md | 4 +++- docs/configuration.md | 35 +++++++++++++++++++++++++++++++---- 2 files changed, 34 insertions(+), 5 deletions(-) diff --git a/docs/bot-usage.md b/docs/bot-usage.md index ec8873b12..114e7613e 100644 --- a/docs/bot-usage.md +++ b/docs/bot-usage.md @@ -36,7 +36,7 @@ optional arguments: --strategy-path PATH specify additional strategy lookup path --dynamic-whitelist [INT] dynamically generate and update whitelist based on 24h - BaseVolume (default: 20) + BaseVolume (default: 20) DEPRECATED --db-url PATH Override trades database URL, this is useful if dry_run is enabled or in custom deployments (default: sqlite:///tradesv3.sqlite) @@ -89,6 +89,8 @@ This is very simple. Copy paste your strategy file into the folder ### How to use --dynamic-whitelist? +> Dynamic-whitelist is deprecated. Please move your configurations to the configuration as outlined [here](docs/configuration.md#Dynamic-Pairlists) + Per default `--dynamic-whitelist` will retrieve the 20 currencies based on BaseVolume. This value can be changed when you run the script. diff --git a/docs/configuration.md b/docs/configuration.md index 5b8baa43b..6d8db6600 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -34,8 +34,8 @@ The table below will list all configuration parameters. | `bid_strategy.ask_last_balance` | 0.0 | Yes | Set the bidding price. More information below. | `bid_strategy.use_order_book` | false | No | Allows buying of pair using the rates in Order Book Bids. | `bid_strategy.order_book_top` | 0 | No | Bot will use the top N rate in Order Book Bids. Ie. a value of 2 will allow the bot to pick the 2nd bid rate in Order Book Bids. -| `bid_strategy.check_depth_of_market.enabled` | false | No | Does not buy if the % difference of buy orders and sell orders is met in Order Book. -| `bid_strategy.check_depth_of_market.bids_to_ask_delta` | 0 | No | The % difference of buy orders and sell orders found in Order Book. A value lesser than 1 means sell orders is greater, while value greater than 1 means buy orders is higher. +| `bid_strategy. check_depth_of_market.enabled` | false | No | Does not buy if the % difference of buy orders and sell orders is met in Order Book. +| `bid_strategy. check_depth_of_market.bids_to_ask_delta` | 0 | No | The % difference of buy orders and sell orders found in Order Book. A value lesser than 1 means sell orders is greater, while value greater than 1 means buy orders is higher. | `ask_strategy.use_order_book` | false | No | Allows selling of open traded pair using the rates in Order Book Asks. | `ask_strategy.order_book_min` | 0 | No | Bot will scan from the top min to max Order Book Asks searching for a profitable rate. | `ask_strategy.order_book_max` | 0 | No | Bot will scan from the top min to max Order Book Asks searching for a profitable rate. @@ -52,6 +52,8 @@ The table below will list all configuration parameters. | `experimental.use_sell_signal` | false | No | Use your sell strategy in addition of the `minimal_roi`. | `experimental.sell_profit_only` | false | No | waits until you have made a positive profit before taking a sell decision. | `experimental.ignore_roi_if_buy_signal` | false | No | Does not sell if the buy-signal is still active. Takes preference over `minimal_roi` and `use_sell_signal` +| `pairlist.method` | StaticPairList | No | Use Static whitelist. [More information below](#dynamic-pairlists). +| `pairlist.config` | None | No | Additional configuration for dynamic pairlists. [More information below](#dynamic-pairlists). | `telegram.enabled` | true | Yes | Enable or not the usage of Telegram. | `telegram.token` | token | No | Your Telegram bot token. Only required if `telegram.enabled` is `true`. | `telegram.chat_id` | chat_id | No | Your personal Telegram account id. Only required if `telegram.enabled` is `true`. @@ -147,7 +149,7 @@ This can be set in the configuration or in the strategy. Configuration overwrite If this is configured, all 4 values (`"buy"`, `"sell"`, `"stoploss"`, `"stoploss_on_exchange"`) need to be present, otherwise the bot warn about it and will fail to start. The below is the default which is used if this is not configured in either Strategy or configuration. -``` json +``` python "order_types": { "buy": "limit", "sell": "limit", @@ -211,13 +213,38 @@ creating trades. Once you will be happy with your bot performance, you can switch it to production mode. +### Dynamic Pairlists + +Dynamic pairlists allow configuration of the pair-selection. +Basically, the bot runs against all pairs (with that stake) on the exchange, and a number of assets (`number_assets`) is selected based on different criteria. + +By *default*, a Static Pairlist is used (configured as `"pair_whitelist"` under the `"exchange"` section of this configuration). + +#### Available Pairlist methods + +* `"StaticPairList"` + * uses configuration from `exchange.pair_whitelist` and `exchange.pair_blacklist` +* `"VolumePairList"` + * Formerly available as `--dynamic-whitelist []` + * Selects `number_assets` top pairs based on `sort_key`, which can be one of `askVolume`, `bidVolume` and `quoteVolume`, defaults to `quoteVolume`. + +```json +"pairlist": { + "method": "VolumePairList", + "config": { + "number_assets": 20, + "sort_key": "quoteVolume" + } + }, +``` + ## Switch to production mode In production mode, the bot will engage your money. Be careful a wrong strategy can lose all your money. Be aware of what you are doing when you run it in production mode. -### To switch your bot in production mode: +### To switch your bot in production mode 1. Edit your `config.json` file From e3876bcf0fadf0465728a262d9382c8c18c077ed Mon Sep 17 00:00:00 2001 From: misagh Date: Tue, 4 Dec 2018 20:36:44 +0100 Subject: [PATCH 570/699] removing AON as it is not supported in binance. will be added once TIF is added for other exchanges --- freqtrade/constants.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/constants.py b/freqtrade/constants.py index 31c3fe9a9..4c3b86e7e 100644 --- a/freqtrade/constants.py +++ b/freqtrade/constants.py @@ -16,7 +16,7 @@ UNLIMITED_STAKE_AMOUNT = 'unlimited' REQUIRED_ORDERTIF = ['buy', 'sell'] REQUIRED_ORDERTYPES = ['buy', 'sell', 'stoploss', 'stoploss_on_exchange'] ORDERTYPE_POSSIBILITIES = ['limit', 'market'] -ORDERTIF_POSSIBILITIES = ['gtc', 'aon', 'fok', 'ioc'] +ORDERTIF_POSSIBILITIES = ['gtc', 'fok', 'ioc'] TICKER_INTERVAL_MINUTES = { From 910601ba1d6e1af9201369859eda2502247149f0 Mon Sep 17 00:00:00 2001 From: misagh Date: Tue, 4 Dec 2018 20:50:35 +0100 Subject: [PATCH 571/699] =?UTF-8?q?in=20case=20exchange=20doesn=E2=80=99t?= =?UTF-8?q?=20return=20order=20info=20=E2=80=A6?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- freqtrade/freqtradebot.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index ff73a036f..7894fc6cf 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -447,11 +447,11 @@ class FreqtradeBot(object): amount=amount, rate=buy_limit, time_in_force=self.strategy.order_time_in_force['buy']) order_id = order['id'] - order_info = order['info'] + order_info = order.get('info', {}) # check if order is expired (in case of FOC or IOC orders) # or rejected by the exchange. - if order_info['status'] == 'EXPIRED' or order_info['status'] == 'REJECTED': + if order_info.get('status', '') == 'EXPIRED' or order_info.get('status', '') == 'REJECTED': order_type = self.strategy.order_types['buy'] order_tif = self.strategy.order_time_in_force['buy'] status = order_info['status'] From d12cc39a5ec8f192992fac6fb14c78c31159cad0 Mon Sep 17 00:00:00 2001 From: misagh Date: Tue, 4 Dec 2018 20:59:55 +0100 Subject: [PATCH 572/699] some visual happyness --- freqtrade/exchange/__init__.py | 3 ++- freqtrade/freqtradebot.py | 8 ++++---- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/freqtrade/exchange/__init__.py b/freqtrade/exchange/__init__.py index ce638e042..727766d0f 100644 --- a/freqtrade/exchange/__init__.py +++ b/freqtrade/exchange/__init__.py @@ -275,7 +275,8 @@ class Exchange(object): price = ceil(big_price) / pow(10, symbol_prec) return price - def buy(self, pair: str, ordertype: str, amount: float, rate: float, time_in_force='gtc') -> Dict: + def buy(self, pair: str, ordertype: str, amount: float, + rate: float, time_in_force='gtc') -> Dict: if self._conf['dry_run']: order_id = f'dry_run_buy_{randint(0, 10**6)}' self._dry_run_open_orders[order_id] = { diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index 7894fc6cf..52d4b88ec 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -444,14 +444,15 @@ class FreqtradeBot(object): amount = stake_amount / buy_limit order = self.exchange.buy(pair=pair, ordertype=self.strategy.order_types['buy'], - amount=amount, rate=buy_limit, - time_in_force=self.strategy.order_time_in_force['buy']) + amount=amount, rate=buy_limit, + time_in_force=self.strategy.order_time_in_force['buy']) order_id = order['id'] order_info = order.get('info', {}) # check if order is expired (in case of FOC or IOC orders) # or rejected by the exchange. - if order_info.get('status', '') == 'EXPIRED' or order_info.get('status', '') == 'REJECTED': + order_status = order_info.get('status', '') + if order_status == 'EXPIRED' or order_status == 'REJECTED': order_type = self.strategy.order_types['buy'] order_tif = self.strategy.order_time_in_force['buy'] status = order_info['status'] @@ -459,7 +460,6 @@ class FreqtradeBot(object): order_tif, order_type, pair_s, status, self.exchange.name) return False - self.rpc.send_msg({ 'type': RPCMessageType.BUY_NOTIFICATION, 'exchange': self.exchange.name.capitalize(), From 37ebe05c6dee78a347493cd1f50b3aedec239539 Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Wed, 5 Dec 2018 13:34:06 +0100 Subject: [PATCH 573/699] Update ccxt from 1.17.581 to 1.17.583 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 2befc4d5c..b59104dfe 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -ccxt==1.17.581 +ccxt==1.17.583 SQLAlchemy==1.2.14 python-telegram-bot==11.1.0 arrow==0.12.1 From 21906e4892d2a04546fdfdf6875d670eac83dbda Mon Sep 17 00:00:00 2001 From: Matthias Date: Wed, 5 Dec 2018 19:48:50 +0100 Subject: [PATCH 574/699] Remove duplicate code --- freqtrade/pairlist/VolumePairList.py | 36 ++-------------------------- 1 file changed, 2 insertions(+), 34 deletions(-) diff --git a/freqtrade/pairlist/VolumePairList.py b/freqtrade/pairlist/VolumePairList.py index 6565caae6..bd562da83 100644 --- a/freqtrade/pairlist/VolumePairList.py +++ b/freqtrade/pairlist/VolumePairList.py @@ -31,12 +31,12 @@ class VolumePairList(StaticPairList): 'Exchange does not support dynamic whitelist.' 'Please edit your config and restart the bot' ) - if not self.validate_keys(self._sort_key): + if not self._validate_keys(self._sort_key): raise OperationalException( f'key {self._sort_key} not in {SORT_VALUES}') # self.refresh_whitelist() - def validate_keys(self, key): + def _validate_keys(self, key): return key in SORT_VALUES def short_desc(self) -> str: @@ -73,35 +73,3 @@ class VolumePairList(StaticPairList): pairs = [s['symbol'] for s in sorted_tickers] return pairs - def _validate_whitelist(self, whitelist: List[str]) -> List[str]: - """ - Check available markets and remove pair from whitelist if necessary - :param whitelist: the sorted list (based on BaseVolume) of pairs the user might want to - trade - :return: the list of pairs the user wants to trade without the one unavailable or - black_listed - """ - sanitized_whitelist = whitelist - markets = self._freqtrade.exchange.get_markets() - - # Filter to markets in stake currency - markets = [m for m in markets if m['quote'] == self._config['stake_currency']] - known_pairs = set() - - for market in markets: - pair = market['symbol'] - # pair is not int the generated dynamic market, or in the blacklist ... ignore it - if pair not in whitelist or pair in self.blacklist: - continue - # else the pair is valid - known_pairs.add(pair) - # Market is not active - if not market['active']: - sanitized_whitelist.remove(pair) - logger.info( - 'Ignoring %s from whitelist. Market is not active.', - pair - ) - - # We need to remove pairs that are unknown - return [x for x in sanitized_whitelist if x in known_pairs] From 1a10e1286193a8fa7d76b7dfc7dd479b17ca6ace Mon Sep 17 00:00:00 2001 From: Matthias Date: Wed, 5 Dec 2018 19:48:59 +0100 Subject: [PATCH 575/699] Documentation and developer documentation --- CONTRIBUTING.md | 4 ++++ docs/developer.md | 60 +++++++++++++++++++++++++++++++++++++++++++++++ docs/index.md | 3 ++- 3 files changed, 66 insertions(+), 1 deletion(-) create mode 100644 docs/developer.md diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 58185b27c..c9a967834 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -72,6 +72,10 @@ pip3.6 install mypy mypy freqtrade ``` +## Getting started + +Best start by reading the [documentation](https://github.com/freqtrade/freqtrade/blob/develop/docs/index.md) to get a feel for what is possible with the bot, or head straight to the [Developer-documentation](https://github.com/freqtrade/freqtrade/blob/develop/docs/developer.md) (WIP) which should help you getting started. + ## (Core)-Committer Guide ### Process: Pull Requests diff --git a/docs/developer.md b/docs/developer.md new file mode 100644 index 000000000..35410de1f --- /dev/null +++ b/docs/developer.md @@ -0,0 +1,60 @@ +# Development Help + +This page is intended for developers of FreqTrade, people who want to contribute to the FreqTrade codebase or documentation, or people who want to understand the source code of the application they're running. + +All contributions, bug reports, bug fixes, documentation improvements, enhancements and ideas are welcome. We [track issues](https://github.com/freqtrade/freqtrade/issues) on [GitHub](https://github.com) and also have a dev channel in [slack](https://join.slack.com/t/highfrequencybot/shared_invite/enQtMjQ5NTM0OTYzMzY3LWMxYzE3M2MxNDdjMGM3ZTYwNzFjMGIwZGRjNTc3ZGU3MGE3NzdmZGMwNmU3NDM5ZTNmM2Y3NjRiNzk4NmM4OGE) where you can ask questions. + + +## Module + +### Dynamic Pairlist + +You have a great idea for a new pair selection algorithm you would like to try out? Great. +Hopefully you also want to contribute this back upstream. + +Whatever your motivations are - This should get you off the ground in trying to develop a new Pairlist provider. + +First of all, have a look at the [VolumePairList](https://github.com/freqtrade/freqtrade/blob/develop/freqtrade/pairlist/VolumePairList.py) provider, and best copy this file with a name of your new Pairlist Provider. + +This is a simple provider, which however serves as a good example on how to start developing. + +Next, modify the classname of the provider (ideally align this with the Filename). + +Now, let's step through the methods which require actions: + +#### configuration + +Configuration for PairListProvider is done in the bot configuration file in the element `"pairlist"`. +This Pairlist-object may contain a `"config"` dict with additional configurations for the configured pairlist. +By convention, `"number_assets"` is used to specify the maximum number of pairs to keep in the whitelist. Please follow this to ensure a consistent user experience. + +Additional elements can be configured as needed. `VolumePairList` uses `"sort_key"` to specify the sorting value - however feel free to specify whatever is necessary for your great algorithm to be successfull and dynamic. + +#### short_desc + +Returns a description used for Telegram messages. +This should coutain the name of the Provider, as well as a short description containing the number of assets. Please follow the format `"PairlistName - top/bottom X pairs"`. + +#### refresh_whitelist + +Override this method and run all calculations needed in this method. +This is called with each iteration of the bot - so consider implementing caching for compute/network heavy calculations. + +Assign the resulting whiteslist to `self._whitelist` and `self._blacklist` respectively. These will then be used to run the bot in this iteration. Pairs with open trades will be added to the whitelist to have the sell-methods run correctly. + +Please also run `self._validate_whitelist(pairs)` and to check and remove pairs with inactive markets. This function is available in the Parent class (`StaticPairList`) and should ideally not be overwritten. + +##### sample + +``` python + def refresh_whitelist(self) -> None: + # Generate dynamic whitelist + pairs = self._gen_pair_whitelist(self._config['stake_currency'], self._sort_key) + # Validate whitelist to only have active market pairs + self._whitelist = self._validate_whitelist(pairs)[:self._number_pairs] +``` + +#### _gen_pair_whitelist + +This is a simple method used by `VolumePairList` - however serves as a good example. +It implements caching (`@cached(TTLCache(maxsize=1, ttl=1800))`) as well as a configuration option to allow different (but similar) strategies to work with the same PairListProvider. diff --git a/docs/index.md b/docs/index.md index c64b9c188..2f579643e 100644 --- a/docs/index.md +++ b/docs/index.md @@ -35,4 +35,5 @@ Pull-request. Do not hesitate to reach us on - [Run tests & Check PEP8 compliance](https://github.com/freqtrade/freqtrade/blob/develop/CONTRIBUTING.md) - [FAQ](https://github.com/freqtrade/freqtrade/blob/develop/docs/faq.md) - [SQL cheatsheet](https://github.com/freqtrade/freqtrade/blob/develop/docs/sql_cheatsheet.md) -- [Sandbox Testing](https://github.com/freqtrade/freqtrade/blob/develop/docs/sandbox-testing.md)) +- [Sandbox Testing](https://github.com/freqtrade/freqtrade/blob/develop/docs/sandbox-testing.md) +- [Developer Docs](https://github.com/freqtrade/freqtrade/blob/develop/docs/developer.md) From 43031aa3bb61828b764bde099075c2213c855023 Mon Sep 17 00:00:00 2001 From: Matthias Date: Wed, 5 Dec 2018 20:44:41 +0100 Subject: [PATCH 576/699] Add missing path-error handler for hyperopt --- freqtrade/resolvers/hyperopt_resolver.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/freqtrade/resolvers/hyperopt_resolver.py b/freqtrade/resolvers/hyperopt_resolver.py index da7b65648..eb91c0e89 100644 --- a/freqtrade/resolvers/hyperopt_resolver.py +++ b/freqtrade/resolvers/hyperopt_resolver.py @@ -52,11 +52,14 @@ class HyperOptResolver(IResolver): abs_paths.insert(0, Path(extra_dir)) for _path in abs_paths: - hyperopt = self._search_object(directory=_path, object_type=IHyperOpt, - object_name=hyperopt_name) - if hyperopt: - logger.info('Using resolved hyperopt %s from \'%s\'', hyperopt_name, _path) - return hyperopt + try: + hyperopt = self._search_object(directory=_path, object_type=IHyperOpt, + object_name=hyperopt_name) + if hyperopt: + logger.info('Using resolved hyperopt %s from \'%s\'', hyperopt_name, _path) + return hyperopt + except FileNotFoundError: + logger.warning('Path "%s" does not exist', _path.relative_to(Path.cwd())) raise ImportError( "Impossible to load Hyperopt '{}'. This class does not exist" From 3e2fa580291066b3c4fd0948a4fdf0eea6935569 Mon Sep 17 00:00:00 2001 From: Matthias Date: Wed, 5 Dec 2018 20:44:56 +0100 Subject: [PATCH 577/699] load pairlists via resolver --- docs/developer.md | 14 +++- freqtrade/freqtradebot.py | 12 ++-- freqtrade/pairlist/IPairList.py | 91 ++++++++++++++++++++++++ freqtrade/pairlist/StaticPairList.py | 72 ++----------------- freqtrade/pairlist/VolumePairList.py | 16 ++--- freqtrade/resolvers/__init__.py | 1 + freqtrade/resolvers/pairlist_resolver.py | 59 +++++++++++++++ 7 files changed, 179 insertions(+), 86 deletions(-) create mode 100644 freqtrade/pairlist/IPairList.py create mode 100644 freqtrade/resolvers/pairlist_resolver.py diff --git a/docs/developer.md b/docs/developer.md index 35410de1f..c564acffb 100644 --- a/docs/developer.md +++ b/docs/developer.md @@ -20,6 +20,16 @@ This is a simple provider, which however serves as a good example on how to star Next, modify the classname of the provider (ideally align this with the Filename). +The base-class provides the an instance of the bot (`self._freqtrade`), as well as the configuration (`self._config`), and initiates both `_blacklist` and `_whitelist`. + +```python + self._freqtrade = freqtrade + self._config = config + self._whitelist = self._config['exchange']['pair_whitelist'] + self._blacklist = self._config['exchange'].get('pair_blacklist', []) +``` + + Now, let's step through the methods which require actions: #### configuration @@ -35,7 +45,7 @@ Additional elements can be configured as needed. `VolumePairList` uses `"sort_ke Returns a description used for Telegram messages. This should coutain the name of the Provider, as well as a short description containing the number of assets. Please follow the format `"PairlistName - top/bottom X pairs"`. -#### refresh_whitelist +#### refresh_pairlist Override this method and run all calculations needed in this method. This is called with each iteration of the bot - so consider implementing caching for compute/network heavy calculations. @@ -47,7 +57,7 @@ Please also run `self._validate_whitelist(pairs)` and to check and remove pairs ##### sample ``` python - def refresh_whitelist(self) -> None: + def refresh_pairlist(self) -> None: # Generate dynamic whitelist pairs = self._gen_pair_whitelist(self._config['stake_currency'], self._sort_key) # Validate whitelist to only have active market pairs diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index 54374b209..01de328f6 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -19,12 +19,10 @@ from freqtrade.wallets import Wallets from freqtrade.edge import Edge from freqtrade.persistence import Trade from freqtrade.rpc import RPCManager, RPCMessageType -from freqtrade.resolvers import StrategyResolver +from freqtrade.resolvers import StrategyResolver, PairListResolver from freqtrade.state import State from freqtrade.strategy.interface import SellType, IStrategy from freqtrade.exchange.exchange_helpers import order_book_to_dataframe -from freqtrade.pairlist.StaticPairList import StaticPairList -from freqtrade.pairlist.VolumePairList import VolumePairList logger = logging.getLogger(__name__) @@ -59,10 +57,8 @@ class FreqtradeBot(object): self.persistence = None self.exchange = Exchange(self.config) self.wallets = Wallets(self.exchange) - if self.config.get('pairlist', {}).get('method') == 'VolumePairList': - self.pairlists: StaticPairList = VolumePairList(self, self.config) - else: - self.pairlists: StaticPairList = StaticPairList(self, self.config) + pairlistname = self.config.get('pairlist', {}).get('method', 'StaticPairList') + self.pairlists = PairListResolver(pairlistname, self, self.config).pairlist # Initializing Edge only if enabled self.edge = Edge(self.config, self.exchange, self.strategy) if \ @@ -151,7 +147,7 @@ class FreqtradeBot(object): state_changed = False try: # Refresh whitelist - self.pairlists.refresh_whitelist() + self.pairlists.refresh_pairlist() self.active_pair_whitelist = self.pairlists.whitelist # Calculating Edge positiong diff --git a/freqtrade/pairlist/IPairList.py b/freqtrade/pairlist/IPairList.py new file mode 100644 index 000000000..6b5b0db4b --- /dev/null +++ b/freqtrade/pairlist/IPairList.py @@ -0,0 +1,91 @@ +""" +Static List provider + +Provides lists as configured in config.json + + """ +import logging +from abc import ABC, abstractmethod +from typing import List + +logger = logging.getLogger(__name__) + + +class IPairList(ABC): + + def __init__(self, freqtrade, config: dict) -> None: + self._freqtrade = freqtrade + self._config = config + self._whitelist = self._config['exchange']['pair_whitelist'] + self._blacklist = self._config['exchange'].get('pair_blacklist', []) + + @property + def name(self) -> str: + """ + Gets name of the class + -> no need to overwrite in subclasses + """ + return self.__class__.__name__ + + @property + def whitelist(self) -> List[str]: + """ + Has the current whitelist + -> no need to overwrite in subclasses + """ + return self._whitelist + + @property + def blacklist(self) -> List[str]: + """ + Has the current blacklist + -> no need to overwrite in subclasses + """ + return self._blacklist + + @abstractmethod + def short_desc(self) -> str: + """ + Short whitelist method description - used for startup-messages + -> Please overwrite in subclasses + """ + + @abstractmethod + def refresh_pairlist(self) -> None: + """ + Refreshes pairlists and assigns them to self._whitelist and self._blacklist respectively + -> Please overwrite in subclasses + """ + + def _validate_whitelist(self, whitelist: List[str]) -> List[str]: + """ + Check available markets and remove pair from whitelist if necessary + :param whitelist: the sorted list (based on BaseVolume) of pairs the user might want to + trade + :return: the list of pairs the user wants to trade without the one unavailable or + black_listed + """ + sanitized_whitelist = whitelist + markets = self._freqtrade.exchange.get_markets() + + # Filter to markets in stake currency + markets = [m for m in markets if m['quote'] == self._config['stake_currency']] + known_pairs = set() + + for market in markets: + pair = market['symbol'] + # pair is not int the generated dynamic market, or in the blacklist ... ignore it + if pair not in whitelist or pair in self.blacklist: + continue + # else the pair is valid + known_pairs.add(pair) + # Market is not active + if not market['active']: + sanitized_whitelist.remove(pair) + logger.info( + 'Ignoring %s from whitelist. Market is not active.', + pair + ) + + # We need to remove pairs that are unknown + return [x for x in sanitized_whitelist if x in known_pairs] diff --git a/freqtrade/pairlist/StaticPairList.py b/freqtrade/pairlist/StaticPairList.py index 5b0e37357..5896e814a 100644 --- a/freqtrade/pairlist/StaticPairList.py +++ b/freqtrade/pairlist/StaticPairList.py @@ -5,43 +5,16 @@ Provides lists as configured in config.json """ import logging -from typing import List + +from freqtrade.pairlist.IPairList import IPairList logger = logging.getLogger(__name__) -class StaticPairList(object): +class StaticPairList(IPairList): def __init__(self, freqtrade, config: dict) -> None: - self._freqtrade = freqtrade - self._config = config - self._whitelist = self._config['exchange']['pair_whitelist'] - self._blacklist = self._config['exchange'].get('pair_blacklist', []) - # self.refresh_whitelist() - - @property - def name(self) -> str: - """ - Gets name of the class - -> no need to overwrite in subclasses - """ - return self.__class__.__name__ - - @property - def whitelist(self) -> List[str]: - """ - Has the current whitelist - -> no need to overwrite in subclasses - """ - return self._whitelist - - @property - def blacklist(self) -> List[str]: - """ - Has the current blacklist - -> no need to overwrite in subclasses - """ - return self._blacklist + super().__init__(freqtrade, config) def short_desc(self) -> str: """ @@ -50,41 +23,8 @@ class StaticPairList(object): """ return f"{self.name}: {self.whitelist}" - def refresh_whitelist(self) -> None: + def refresh_pairlist(self) -> None: """ - Refreshes whitelist and assigns it to self._whitelist + Refreshes pairlists and assigns them to self._whitelist and self._blacklist respectively """ self._whitelist = self._validate_whitelist(self._config['exchange']['pair_whitelist']) - - def _validate_whitelist(self, whitelist: List[str]) -> List[str]: - """ - Check available markets and remove pair from whitelist if necessary - :param whitelist: the sorted list (based on BaseVolume) of pairs the user might want to - trade - :return: the list of pairs the user wants to trade without the one unavailable or - black_listed - """ - sanitized_whitelist = whitelist - markets = self._freqtrade.exchange.get_markets() - - # Filter to markets in stake currency - markets = [m for m in markets if m['quote'] == self._config['stake_currency']] - known_pairs = set() - - for market in markets: - pair = market['symbol'] - # pair is not int the generated dynamic market, or in the blacklist ... ignore it - if pair not in whitelist or pair in self.blacklist: - continue - # else the pair is valid - known_pairs.add(pair) - # Market is not active - if not market['active']: - sanitized_whitelist.remove(pair) - logger.info( - 'Ignoring %s from whitelist. Market is not active.', - pair - ) - - # We need to remove pairs that are unknown - return [x for x in sanitized_whitelist if x in known_pairs] diff --git a/freqtrade/pairlist/VolumePairList.py b/freqtrade/pairlist/VolumePairList.py index bd562da83..9d3d46169 100644 --- a/freqtrade/pairlist/VolumePairList.py +++ b/freqtrade/pairlist/VolumePairList.py @@ -8,21 +8,18 @@ import logging from typing import List from cachetools import TTLCache, cached -from freqtrade.pairlist.StaticPairList import StaticPairList +from freqtrade.pairlist.IPairList import IPairList from freqtrade import OperationalException logger = logging.getLogger(__name__) SORT_VALUES = ['askVolume', 'bidVolume', 'quoteVolume'] -class VolumePairList(StaticPairList): +class VolumePairList(IPairList): def __init__(self, freqtrade, config: dict) -> None: - self._freqtrade = freqtrade - self._config = config + super().__init__(freqtrade, config) self._whitelistconf = self._config.get('pairlist', {}).get('config') - self._whitelist = self._config['exchange']['pair_whitelist'] - self._blacklist = self._config['exchange'].get('pair_blacklist', []) self._number_pairs = self._whitelistconf['number_assets'] self._sort_key = self._whitelistconf.get('sort_key', 'quoteVolume') @@ -34,7 +31,6 @@ class VolumePairList(StaticPairList): if not self._validate_keys(self._sort_key): raise OperationalException( f'key {self._sort_key} not in {SORT_VALUES}') - # self.refresh_whitelist() def _validate_keys(self, key): return key in SORT_VALUES @@ -46,9 +42,10 @@ class VolumePairList(StaticPairList): """ return f"{self.name} - top {self._whitelistconf['number_assets']} volume pairs." - def refresh_whitelist(self) -> None: + def refresh_pairlist(self) -> None: """ - Refreshes whitelist and assigns it to self._whitelist + Refreshes pairlists and assigns them to self._whitelist and self._blacklist respectively + -> Please overwrite in subclasses """ # Generate dynamic whitelist pairs = self._gen_pair_whitelist(self._config['stake_currency'], self._sort_key) @@ -72,4 +69,3 @@ class VolumePairList(StaticPairList): sorted_tickers = sorted(tickers, reverse=True, key=lambda t: t[key]) pairs = [s['symbol'] for s in sorted_tickers] return pairs - diff --git a/freqtrade/resolvers/__init__.py b/freqtrade/resolvers/__init__.py index 84e3bcdcd..da2987b27 100644 --- a/freqtrade/resolvers/__init__.py +++ b/freqtrade/resolvers/__init__.py @@ -1,3 +1,4 @@ from freqtrade.resolvers.iresolver import IResolver # noqa: F401 from freqtrade.resolvers.hyperopt_resolver import HyperOptResolver # noqa: F401 +from freqtrade.resolvers.pairlist_resolver import PairListResolver # noqa: F401 from freqtrade.resolvers.strategy_resolver import StrategyResolver # noqa: F401 diff --git a/freqtrade/resolvers/pairlist_resolver.py b/freqtrade/resolvers/pairlist_resolver.py new file mode 100644 index 000000000..286cea5bf --- /dev/null +++ b/freqtrade/resolvers/pairlist_resolver.py @@ -0,0 +1,59 @@ +# pragma pylint: disable=attribute-defined-outside-init + +""" +This module load custom hyperopts +""" +import logging +from pathlib import Path + +from freqtrade.pairlist.IPairList import IPairList +from freqtrade.resolvers import IResolver + +logger = logging.getLogger(__name__) + + +class PairListResolver(IResolver): + """ + This class contains all the logic to load custom hyperopt class + """ + + __slots__ = ['pairlist'] + + def __init__(self, pairlist_name: str, freqtrade, config: dict) -> None: + """ + Load the custom class from config parameter + :param config: configuration dictionary or None + """ + self.pairlist = self._load_pairlist(pairlist_name, kwargs={'freqtrade': freqtrade, + 'config': config}) + + def _load_pairlist( + self, pairlist_name: str, kwargs: dict) -> IPairList: + """ + Search and loads the specified pairlist. + :param pairlist_name: name of the module to import + :param extra_dir: additional directory to search for the given pairlist + :return: PairList instance or None + """ + current_path = Path(__file__).parent.parent.joinpath('pairlist').resolve() + + abs_paths = [ + current_path.parent.parent.joinpath('user_data/pairlist'), + current_path, + ] + + for _path in abs_paths: + try: + pairlist = self._search_object(directory=_path, object_type=IPairList, + object_name=pairlist_name, + kwargs=kwargs) + if pairlist: + logger.info('Using resolved pairlist %s from \'%s\'', pairlist_name, _path) + return pairlist + except FileNotFoundError: + logger.warning('Path "%s" does not exist', _path.relative_to(Path.cwd())) + + raise ImportError( + "Impossible to load Pairlist '{}'. This class does not exist" + " or contains Python code errors".format(pairlist_name) + ) From 2f0d7a1aea8510266bbec118fd394991af622b15 Mon Sep 17 00:00:00 2001 From: Matthias Date: Wed, 5 Dec 2018 20:45:11 +0100 Subject: [PATCH 578/699] Add specific test --- freqtrade/tests/pairlist/test_pairlist.py | 20 +++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/freqtrade/tests/pairlist/test_pairlist.py b/freqtrade/tests/pairlist/test_pairlist.py index b94859efa..83c3817a7 100644 --- a/freqtrade/tests/pairlist/test_pairlist.py +++ b/freqtrade/tests/pairlist/test_pairlist.py @@ -4,6 +4,7 @@ from unittest.mock import MagicMock from freqtrade import OperationalException from freqtrade.constants import AVAILABLE_PAIRLISTS +from freqtrade.resolvers import PairListResolver from freqtrade.tests.conftest import get_patched_freqtradebot import pytest @@ -30,12 +31,21 @@ def whitelist_conf(default_conf): return default_conf +def test_load_pairlist_noexist(mocker, markets, default_conf): + freqtradebot = get_patched_freqtradebot(mocker, default_conf) + mocker.patch('freqtrade.exchange.Exchange.get_markets', markets) + with pytest.raises(ImportError, + match=r"Impossible to load Pairlist 'NonexistingPairList'." + r" This class does not exist or contains Python code errors"): + PairListResolver('NonexistingPairList', freqtradebot, default_conf).pairlist + + def test_refresh_market_pair_not_in_whitelist(mocker, markets, whitelist_conf): freqtradebot = get_patched_freqtradebot(mocker, whitelist_conf) mocker.patch('freqtrade.exchange.Exchange.get_markets', markets) - freqtradebot.pairlists.refresh_whitelist() + freqtradebot.pairlists.refresh_pairlist() # List ordered by BaseVolume whitelist = ['ETH/BTC', 'TKN/BTC'] # Ensure all except those in whitelist are removed @@ -49,7 +59,7 @@ def test_refresh_pairlists(mocker, markets, whitelist_conf): freqtradebot = get_patched_freqtradebot(mocker, whitelist_conf) mocker.patch('freqtrade.exchange.Exchange.get_markets', markets) - freqtradebot.pairlists.refresh_whitelist() + freqtradebot.pairlists.refresh_pairlist() # List ordered by BaseVolume whitelist = ['ETH/BTC', 'TKN/BTC'] # Ensure all except those in whitelist are removed @@ -57,7 +67,7 @@ def test_refresh_pairlists(mocker, markets, whitelist_conf): assert whitelist_conf['exchange']['pair_blacklist'] == freqtradebot.pairlists.blacklist -def test_refresh_whitelist_dynamic(mocker, markets, tickers, whitelist_conf): +def test_refresh_pairlist_dynamic(mocker, markets, tickers, whitelist_conf): whitelist_conf['pairlist'] = {'method': 'VolumePairList', 'config': {'number_assets': 5} } @@ -71,7 +81,7 @@ def test_refresh_whitelist_dynamic(mocker, markets, tickers, whitelist_conf): # argument: use the whitelist dynamically by exchange-volume whitelist = ['ETH/BTC', 'TKN/BTC'] - freqtradebot.pairlists.refresh_whitelist() + freqtradebot.pairlists.refresh_pairlist() assert whitelist == freqtradebot.pairlists.whitelist @@ -83,7 +93,7 @@ def test_VolumePairList_refresh_empty(mocker, markets_empty, whitelist_conf): # argument: use the whitelist dynamically by exchange-volume whitelist = [] whitelist_conf['exchange']['pair_whitelist'] = [] - freqtradebot.pairlists.refresh_whitelist() + freqtradebot.pairlists.refresh_pairlist() pairslist = whitelist_conf['exchange']['pair_whitelist'] assert set(whitelist) == set(pairslist) From 0c10719037137778f3b81e7d90fe0ca5aa1058ba Mon Sep 17 00:00:00 2001 From: Matthias Date: Thu, 6 Dec 2018 06:57:07 +0100 Subject: [PATCH 579/699] Specify JsonValidatorversion explicitly without doing that, it exclusiveMaximum raises an exception as jsonschema defaults to the latest version (Draft6) which changes behaviour of this property. fixes #1233 --- freqtrade/configuration.py | 2 +- freqtrade/tests/test_configuration.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/freqtrade/configuration.py b/freqtrade/configuration.py index feec0cb43..e9daf4a99 100644 --- a/freqtrade/configuration.py +++ b/freqtrade/configuration.py @@ -275,7 +275,7 @@ class Configuration(object): :return: Returns the config if valid, otherwise throw an exception """ try: - validate(conf, constants.CONF_SCHEMA) + validate(conf, constants.CONF_SCHEMA, Draft4Validator) return conf except ValidationError as exception: logger.critical( diff --git a/freqtrade/tests/test_configuration.py b/freqtrade/tests/test_configuration.py index 23fefd3cd..5a0ae2636 100644 --- a/freqtrade/tests/test_configuration.py +++ b/freqtrade/tests/test_configuration.py @@ -6,7 +6,7 @@ import logging from unittest.mock import MagicMock import pytest -from jsonschema import validate, ValidationError +from jsonschema import validate, ValidationError, Draft4Validator from freqtrade import constants from freqtrade import OperationalException @@ -486,4 +486,4 @@ def test_load_config_warn_forcebuy(default_conf, mocker, caplog) -> None: def test_validate_default_conf(default_conf) -> None: - validate(default_conf, constants.CONF_SCHEMA) + validate(default_conf, constants.CONF_SCHEMA, Draft4Validator) From bf1841d2a81b1231d91409d1053299963a87c057 Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Thu, 6 Dec 2018 13:34:06 +0100 Subject: [PATCH 580/699] Update ccxt from 1.17.583 to 1.18.2 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index b59104dfe..61a153d45 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -ccxt==1.17.583 +ccxt==1.18.2 SQLAlchemy==1.2.14 python-telegram-bot==11.1.0 arrow==0.12.1 From 0ea7dc92722e82f059f43d71538395bf5c06a03d Mon Sep 17 00:00:00 2001 From: misagh Date: Thu, 6 Dec 2018 13:51:06 +0100 Subject: [PATCH 581/699] test added for total open trade stake amount from schalchemy --- freqtrade/tests/test_freqtradebot.py | 33 ++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/freqtrade/tests/test_freqtradebot.py b/freqtrade/tests/test_freqtradebot.py index e8d80b91f..ed3271838 100644 --- a/freqtrade/tests/test_freqtradebot.py +++ b/freqtrade/tests/test_freqtradebot.py @@ -342,6 +342,39 @@ def test_edge_should_ignore_strategy_stoploss(limit_buy_order, fee, markets, assert freqtrade.handle_trade(trade) is False +def test_total_open_trades_stakes(mocker, default_conf, ticker, + limit_buy_order, fee, markets) -> None: + patch_RPCManager(mocker) + patch_exchange(mocker) + default_conf['stake_amount'] = 0.0000098751 + mocker.patch.multiple( + 'freqtrade.exchange.Exchange', + get_ticker=ticker, + buy=MagicMock(return_value={'id': limit_buy_order['id']}), + get_fee=fee, + get_markets=markets + ) + freqtrade = FreqtradeBot(default_conf) + patch_get_signal(freqtrade) + freqtrade.create_trade() + trade = Trade.query.first() + + assert trade is not None + assert trade.stake_amount == 0.0000098751 + assert trade.is_open + assert trade.open_date is not None + + freqtrade.create_trade() + trade = Trade.query.order_by(Trade.id.desc()).first() + + assert trade is not None + assert trade.stake_amount == 0.0000098751 + assert trade.is_open + assert trade.open_date is not None + + assert Trade.total_open_trades_stakes() == 1.97502e-05 + + def test_get_min_pair_stake_amount(mocker, default_conf) -> None: patch_RPCManager(mocker) patch_exchange(mocker) From a63f123b6ddc9ece400c5503b51f38e05fb6e948 Mon Sep 17 00:00:00 2001 From: Matthias Date: Thu, 6 Dec 2018 19:36:33 +0100 Subject: [PATCH 582/699] Check if number_assets is defined, as it's required by VolumePairList --- docs/developer.md | 2 +- freqtrade/pairlist/VolumePairList.py | 4 ++++ freqtrade/tests/pairlist/test_pairlist.py | 8 ++++++++ 3 files changed, 13 insertions(+), 1 deletion(-) diff --git a/docs/developer.md b/docs/developer.md index c564acffb..9137f16ca 100644 --- a/docs/developer.md +++ b/docs/developer.md @@ -43,7 +43,7 @@ Additional elements can be configured as needed. `VolumePairList` uses `"sort_ke #### short_desc Returns a description used for Telegram messages. -This should coutain the name of the Provider, as well as a short description containing the number of assets. Please follow the format `"PairlistName - top/bottom X pairs"`. +This should contain the name of the Provider, as well as a short description containing the number of assets. Please follow the format `"PairlistName - top/bottom X pairs"`. #### refresh_pairlist diff --git a/freqtrade/pairlist/VolumePairList.py b/freqtrade/pairlist/VolumePairList.py index 9d3d46169..262e4bf59 100644 --- a/freqtrade/pairlist/VolumePairList.py +++ b/freqtrade/pairlist/VolumePairList.py @@ -20,6 +20,10 @@ class VolumePairList(IPairList): def __init__(self, freqtrade, config: dict) -> None: super().__init__(freqtrade, config) self._whitelistconf = self._config.get('pairlist', {}).get('config') + if 'number_assets' not in self._whitelistconf: + raise OperationalException( + f'`number_assets` not specified. Please check your configuration ' + 'for "pairlist.config.number_assets"') self._number_pairs = self._whitelistconf['number_assets'] self._sort_key = self._whitelistconf.get('sort_key', 'quoteVolume') diff --git a/freqtrade/tests/pairlist/test_pairlist.py b/freqtrade/tests/pairlist/test_pairlist.py index 83c3817a7..9f90aac6e 100644 --- a/freqtrade/tests/pairlist/test_pairlist.py +++ b/freqtrade/tests/pairlist/test_pairlist.py @@ -85,6 +85,14 @@ def test_refresh_pairlist_dynamic(mocker, markets, tickers, whitelist_conf): assert whitelist == freqtradebot.pairlists.whitelist + whitelist_conf['pairlist'] = {'method': 'VolumePairList', + 'config': {} + } + with pytest.raises(OperationalException, + match=r'`number_assets` not specified. Please check your configuration ' + r'for "pairlist.config.number_assets"'): + PairListResolver('VolumePairList', freqtradebot, whitelist_conf).pairlist + def test_VolumePairList_refresh_empty(mocker, markets_empty, whitelist_conf): freqtradebot = get_patched_freqtradebot(mocker, whitelist_conf) From 8f19c83f6b8fb2f21d8f8ad8c925ae95d3ac907a Mon Sep 17 00:00:00 2001 From: Matthias Date: Thu, 6 Dec 2018 19:39:25 +0100 Subject: [PATCH 583/699] Refrase documentation --- docs/configuration.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/configuration.md b/docs/configuration.md index 6d8db6600..c4c0bed28 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -215,8 +215,8 @@ production mode. ### Dynamic Pairlists -Dynamic pairlists allow configuration of the pair-selection. -Basically, the bot runs against all pairs (with that stake) on the exchange, and a number of assets (`number_assets`) is selected based on different criteria. +Dynamic pairlists select pairs for you based on the logic configured. +The bot runs against all pairs (with that stake) on the exchange, and a number of assets (`number_assets`) is selected based on the selected criteria. By *default*, a Static Pairlist is used (configured as `"pair_whitelist"` under the `"exchange"` section of this configuration). From ac9f19aee5f3932c2b63a0109dc6113ba6ee9fd9 Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Fri, 7 Dec 2018 13:34:07 +0100 Subject: [PATCH 584/699] Update ccxt from 1.18.2 to 1.18.10 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 61a153d45..b3fe9f5a6 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -ccxt==1.18.2 +ccxt==1.18.10 SQLAlchemy==1.2.14 python-telegram-bot==11.1.0 arrow==0.12.1 From b3b6eda2ba6b0f367f4a10ddc4f7d07234fedc80 Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Sat, 8 Dec 2018 13:33:06 +0100 Subject: [PATCH 585/699] Update ccxt from 1.18.10 to 1.18.11 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index b3fe9f5a6..a434b2f2c 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -ccxt==1.18.10 +ccxt==1.18.11 SQLAlchemy==1.2.14 python-telegram-bot==11.1.0 arrow==0.12.1 From 3b951c3817dd949a6d8f0e63802f8bd7293f00f3 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 9 Dec 2018 09:03:17 +0100 Subject: [PATCH 586/699] Drop indexes on renamed table avoid naming conflicts on recreate (indexes are not renamed, and keeping them on backup tables does not really make sense). fixes #1396 --- freqtrade/persistence.py | 3 +++ freqtrade/tests/test_persistence.py | 2 ++ 2 files changed, 5 insertions(+) diff --git a/freqtrade/persistence.py b/freqtrade/persistence.py index 71752d58e..a393eb318 100644 --- a/freqtrade/persistence.py +++ b/freqtrade/persistence.py @@ -100,6 +100,9 @@ def check_migrate(engine) -> None: # Schema migration necessary engine.execute(f"alter table trades rename to {table_back_name}") + # drop indexes on backup table + for index in inspector.get_indexes(table_back_name): + engine.execute(f"drop index {index['name']}") # let SQLAlchemy create the schema as required _DECL_BASE.metadata.create_all(engine) diff --git a/freqtrade/tests/test_persistence.py b/freqtrade/tests/test_persistence.py index d0a209f40..a7b21bc1d 100644 --- a/freqtrade/tests/test_persistence.py +++ b/freqtrade/tests/test_persistence.py @@ -446,6 +446,8 @@ def test_migrate_new(mocker, default_conf, fee, caplog): # Create table using the old format engine.execute(create_table_old) + engine.execute("create index ix_trades_is_open on trades(is_open)") + engine.execute("create index ix_trades_pair on trades(pair)") engine.execute(insert_table_old) # fake previous backup From acb96eb501803aee9ad731c3e2534f40895ffdff Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Sun, 9 Dec 2018 13:33:06 +0100 Subject: [PATCH 587/699] Update ccxt from 1.18.11 to 1.18.13 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index a434b2f2c..72c416b60 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -ccxt==1.18.11 +ccxt==1.18.13 SQLAlchemy==1.2.14 python-telegram-bot==11.1.0 arrow==0.12.1 From b35199a772f4d786179fe811a358ed903f55fabc Mon Sep 17 00:00:00 2001 From: misagh Date: Sun, 9 Dec 2018 15:59:05 +0100 Subject: [PATCH 588/699] intermediary commit before extracting the logic --- freqtrade/freqtradebot.py | 30 ++++++++++++++++++++---------- 1 file changed, 20 insertions(+), 10 deletions(-) diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index 907734313..9a7975e17 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -367,6 +367,7 @@ class FreqtradeBot(object): pair_url = self.exchange.get_pair_detail_url(pair) stake_currency = self.config['stake_currency'] fiat_currency = self.config.get('fiat_display_currency', None) + time_in_force = self.strategy.order_time_in_force['buy'] if price: buy_limit = price @@ -386,20 +387,29 @@ class FreqtradeBot(object): order = self.exchange.buy(pair=pair, ordertype=self.strategy.order_types['buy'], amount=amount, rate=buy_limit, - time_in_force=self.strategy.order_time_in_force['buy']) + time_in_force=time_in_force) order_id = order['id'] - order_info = order.get('info', {}) + order_status = order.get('status', None) - # check if order is expired (in case of FOC or IOC orders) - # or rejected by the exchange. - order_status = order_info.get('status', '') - if order_status == 'EXPIRED' or order_status == 'REJECTED': + # in case of FOK or IOC orders we can check immediately + # if the order is fulfilled fully or partially + if order_status == 'expired' or order_status == 'rejected': order_type = self.strategy.order_types['buy'] order_tif = self.strategy.order_time_in_force['buy'] - status = order_info['status'] - logger.warning('Buy %s order with time in force %s for %s is %s by %s.', - order_tif, order_type, pair_s, status, self.exchange.name) - return False + + # return false is order is not filled + if float(order['filled']) == 0: + logger.warning('Buy %s order with time in force %s for %s is %s by %s.' + ' zero amount is fulfilled.', + order_tif, order_type, pair_s, order_status, self.exchange.name) + return False + else: # the order is partially fulfilled + logger.warning('Buy %s order with time in force %s for %s is %s by %s.' + ' %s amount fulfilled out of %s (%s remaining which is canceled).', + order_tif, order_type, pair_s, order_status, self.exchange.name, + order['filled'], order['amount'], order['remaining'] + ) + self.rpc.send_msg({ 'type': RPCMessageType.BUY_NOTIFICATION, From 2f5c8941ebbd6a1ca34d5bdd3939494e33af08f2 Mon Sep 17 00:00:00 2001 From: misagh Date: Sun, 9 Dec 2018 16:00:04 +0100 Subject: [PATCH 589/699] removing unnecessary default value --- freqtrade/exchange/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/exchange/__init__.py b/freqtrade/exchange/__init__.py index 727766d0f..85abf2939 100644 --- a/freqtrade/exchange/__init__.py +++ b/freqtrade/exchange/__init__.py @@ -276,7 +276,7 @@ class Exchange(object): return price def buy(self, pair: str, ordertype: str, amount: float, - rate: float, time_in_force='gtc') -> Dict: + rate: float, time_in_force) -> Dict: if self._conf['dry_run']: order_id = f'dry_run_buy_{randint(0, 10**6)}' self._dry_run_open_orders[order_id] = { From 20d794e2658b062e1995313621aa0e6f33274d12 Mon Sep 17 00:00:00 2001 From: misagh Date: Sun, 9 Dec 2018 16:04:28 +0100 Subject: [PATCH 590/699] mistake in previous commit --- freqtrade/optimize/edge_cli.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/optimize/edge_cli.py b/freqtrade/optimize/edge_cli.py index fbe81812d..a2189f6c1 100644 --- a/freqtrade/optimize/edge_cli.py +++ b/freqtrade/optimize/edge_cli.py @@ -55,7 +55,7 @@ class EdgeCli(object): 'average duration (min)'] for result in results.items(): - if result[1].nb_trades > 0 and result[1].winrate > 0.60: + if result[1].nb_trades > 0: tabular_data.append([ result[0], result[1].stoploss, From 663e33d2efc389f51870054fb9e3a2e17622518a Mon Sep 17 00:00:00 2001 From: misagh Date: Sun, 9 Dec 2018 16:06:00 +0100 Subject: [PATCH 591/699] if condition refactored --- freqtrade/exchange/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/exchange/__init__.py b/freqtrade/exchange/__init__.py index 85abf2939..e9d819beb 100644 --- a/freqtrade/exchange/__init__.py +++ b/freqtrade/exchange/__init__.py @@ -240,7 +240,7 @@ class Exchange(object): Checks if order time in force configured in strategy/config are supported """ if any(v != 'gtc' for k, v in order_time_in_force.items()): - if not self.name == 'Binance': + if self.name is not 'Binance': raise OperationalException( f'Time in force policies are not supporetd for {self.name} yet.') From 866b7aee8ec2ec964bf14eab7a8aedbe2f9d38c1 Mon Sep 17 00:00:00 2001 From: misagh Date: Sun, 9 Dec 2018 16:22:21 +0100 Subject: [PATCH 592/699] tests fixed --- freqtrade/freqtradebot.py | 4 ++-- freqtrade/tests/exchange/test_exchange.py | 27 +++++++++++++++++------ 2 files changed, 22 insertions(+), 9 deletions(-) diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index 9a7975e17..3a124568e 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -403,13 +403,13 @@ class FreqtradeBot(object): ' zero amount is fulfilled.', order_tif, order_type, pair_s, order_status, self.exchange.name) return False - else: # the order is partially fulfilled + else: # the order is partially fulfilled logger.warning('Buy %s order with time in force %s for %s is %s by %s.' ' %s amount fulfilled out of %s (%s remaining which is canceled).', order_tif, order_type, pair_s, order_status, self.exchange.name, order['filled'], order['amount'], order['remaining'] ) - + return False self.rpc.send_msg({ 'type': RPCMessageType.BUY_NOTIFICATION, diff --git a/freqtrade/tests/exchange/test_exchange.py b/freqtrade/tests/exchange/test_exchange.py index d1f391266..e021b2de8 100644 --- a/freqtrade/tests/exchange/test_exchange.py +++ b/freqtrade/tests/exchange/test_exchange.py @@ -426,7 +426,8 @@ def test_buy_dry_run(default_conf, mocker): default_conf['dry_run'] = True exchange = get_patched_exchange(mocker, default_conf) - order = exchange.buy(pair='ETH/BTC', ordertype='limit', amount=1, rate=200) + order = exchange.buy(pair='ETH/BTC', ordertype='limit', + amount=1, rate=200, time_in_force='gtc') assert 'id' in order assert 'dry_run_buy_' in order['id'] @@ -435,6 +436,7 @@ def test_buy_prod(default_conf, mocker): api_mock = MagicMock() order_id = 'test_prod_buy_{}'.format(randint(0, 10 ** 6)) order_type = 'market' + time_in_force = 'gtc' api_mock.create_order = MagicMock(return_value={ 'id': order_id, 'info': { @@ -446,7 +448,9 @@ def test_buy_prod(default_conf, mocker): mocker.patch('freqtrade.exchange.Exchange.symbol_price_prec', lambda s, x, y: y) exchange = get_patched_exchange(mocker, default_conf, api_mock) - order = exchange.buy(pair='ETH/BTC', ordertype=order_type, amount=1, rate=200) + order = exchange.buy(pair='ETH/BTC', ordertype=order_type, + amount=1, rate=200, time_in_force=time_in_force) + assert 'id' in order assert 'info' in order assert order['id'] == order_id @@ -458,7 +462,12 @@ def test_buy_prod(default_conf, mocker): api_mock.create_order.reset_mock() order_type = 'limit' - order = exchange.buy(pair='ETH/BTC', ordertype=order_type, amount=1, rate=200) + order = exchange.buy( + pair='ETH/BTC', + ordertype=order_type, + amount=1, + rate=200, + time_in_force=time_in_force) assert api_mock.create_order.call_args[0][0] == 'ETH/BTC' assert api_mock.create_order.call_args[0][1] == order_type assert api_mock.create_order.call_args[0][2] == 'buy' @@ -469,22 +478,26 @@ def test_buy_prod(default_conf, mocker): with pytest.raises(DependencyException): api_mock.create_order = MagicMock(side_effect=ccxt.InsufficientFunds) exchange = get_patched_exchange(mocker, default_conf, api_mock) - exchange.buy(pair='ETH/BTC', ordertype=order_type, amount=1, rate=200) + exchange.buy(pair='ETH/BTC', ordertype=order_type, + amount=1, rate=200, time_in_force=time_in_force) with pytest.raises(DependencyException): api_mock.create_order = MagicMock(side_effect=ccxt.InvalidOrder) exchange = get_patched_exchange(mocker, default_conf, api_mock) - exchange.buy(pair='ETH/BTC', ordertype=order_type, amount=1, rate=200) + exchange.buy(pair='ETH/BTC', ordertype=order_type, + amount=1, rate=200, time_in_force=time_in_force) with pytest.raises(TemporaryError): api_mock.create_order = MagicMock(side_effect=ccxt.NetworkError) exchange = get_patched_exchange(mocker, default_conf, api_mock) - exchange.buy(pair='ETH/BTC', ordertype=order_type, amount=1, rate=200) + exchange.buy(pair='ETH/BTC', ordertype=order_type, + amount=1, rate=200, time_in_force=time_in_force) with pytest.raises(OperationalException): api_mock.create_order = MagicMock(side_effect=ccxt.BaseError) exchange = get_patched_exchange(mocker, default_conf, api_mock) - exchange.buy(pair='ETH/BTC', ordertype=order_type, amount=1, rate=200) + exchange.buy(pair='ETH/BTC', ordertype=order_type, + amount=1, rate=200, time_in_force=time_in_force) def test_sell_dry_run(default_conf, mocker): From d904667c87f8080cdbe66b4bf3d33894d71ba238 Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Mon, 10 Dec 2018 13:33:07 +0100 Subject: [PATCH 593/699] Update ccxt from 1.18.13 to 1.18.17 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 72c416b60..7f6472f8a 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -ccxt==1.18.13 +ccxt==1.18.17 SQLAlchemy==1.2.14 python-telegram-bot==11.1.0 arrow==0.12.1 From 6018f2d252e6f22bd4464c814fe6b2fde48a5c7e Mon Sep 17 00:00:00 2001 From: misagh Date: Mon, 10 Dec 2018 18:52:24 +0100 Subject: [PATCH 594/699] order status handled in case of IOC and FOK --- freqtrade/freqtradebot.py | 38 ++++++++++++++++++++++++++------------ 1 file changed, 26 insertions(+), 12 deletions(-) diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index 3a124568e..7adc4fa27 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -370,12 +370,12 @@ class FreqtradeBot(object): time_in_force = self.strategy.order_time_in_force['buy'] if price: - buy_limit = price + buy_limit_requested = price else: # Calculate amount - buy_limit = self.get_target_bid(pair, self.exchange.get_ticker(pair)) + buy_limit_requested = self.get_target_bid(pair, self.exchange.get_ticker(pair)) - min_stake_amount = self._get_min_pair_stake_amount(pair_s, buy_limit) + min_stake_amount = self._get_min_pair_stake_amount(pair_s, buy_limit_requested) if min_stake_amount is not None and min_stake_amount > stake_amount: logger.warning( f'Can\'t open a new trade for {pair_s}: stake amount' @@ -383,16 +383,17 @@ class FreqtradeBot(object): ) return False - amount = stake_amount / buy_limit + amount = stake_amount / buy_limit_requested order = self.exchange.buy(pair=pair, ordertype=self.strategy.order_types['buy'], - amount=amount, rate=buy_limit, + amount=amount, rate=buy_limit_requested, time_in_force=time_in_force) order_id = order['id'] order_status = order.get('status', None) - # in case of FOK or IOC orders we can check immediately - # if the order is fulfilled fully or partially + # we assume the order is executed at the price requested + buy_limit_filled_price = buy_limit_requested + if order_status == 'expired' or order_status == 'rejected': order_type = self.strategy.order_types['buy'] order_tif = self.strategy.order_time_in_force['buy'] @@ -403,20 +404,33 @@ class FreqtradeBot(object): ' zero amount is fulfilled.', order_tif, order_type, pair_s, order_status, self.exchange.name) return False - else: # the order is partially fulfilled + else: + # the order is partially fulfilled + # in case of IOC orders we can check immediately + # if the order is fulfilled fully or partially logger.warning('Buy %s order with time in force %s for %s is %s by %s.' ' %s amount fulfilled out of %s (%s remaining which is canceled).', order_tif, order_type, pair_s, order_status, self.exchange.name, order['filled'], order['amount'], order['remaining'] ) - return False + stake_amount = order['price'] + amount = order['amount'] + buy_limit_filled_price = order['average'] + order_id = None + + # in case of FOK the order may be filled immediately and fully + elif order_status == 'filled': + stake_amount = order['price'] + amount = order['amount'] + buy_limit_filled_price = order['average'] + order_id = None self.rpc.send_msg({ 'type': RPCMessageType.BUY_NOTIFICATION, 'exchange': self.exchange.name.capitalize(), 'pair': pair_s, 'market_url': pair_url, - 'limit': buy_limit, + 'limit': buy_limit_requested, 'stake_amount': stake_amount, 'stake_currency': stake_currency, 'fiat_currency': fiat_currency @@ -430,8 +444,8 @@ class FreqtradeBot(object): amount=amount, fee_open=fee, fee_close=fee, - open_rate=buy_limit, - open_rate_requested=buy_limit, + open_rate=buy_limit_filled_price, + open_rate_requested=buy_limit_requested, open_date=datetime.utcnow(), exchange=self.exchange.id, open_order_id=order_id, From e6fd7da43f65ccde9977970a9d1e9f177d62bed4 Mon Sep 17 00:00:00 2001 From: misagh Date: Mon, 10 Dec 2018 19:09:20 +0100 Subject: [PATCH 595/699] adding test: create order should consider TIF --- freqtrade/tests/exchange/test_exchange.py | 30 +++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/freqtrade/tests/exchange/test_exchange.py b/freqtrade/tests/exchange/test_exchange.py index e021b2de8..02314b32c 100644 --- a/freqtrade/tests/exchange/test_exchange.py +++ b/freqtrade/tests/exchange/test_exchange.py @@ -500,6 +500,36 @@ def test_buy_prod(default_conf, mocker): amount=1, rate=200, time_in_force=time_in_force) +def test_buy_considers_time_in_force(default_conf, mocker): + api_mock = MagicMock() + order_id = 'test_prod_buy_{}'.format(randint(0, 10 ** 6)) + order_type = 'market' + time_in_force = 'ioc' + api_mock.create_order = MagicMock(return_value={ + 'id': order_id, + 'info': { + 'foo': 'bar' + } + }) + default_conf['dry_run'] = False + mocker.patch('freqtrade.exchange.Exchange.symbol_amount_prec', lambda s, x, y: y) + mocker.patch('freqtrade.exchange.Exchange.symbol_price_prec', lambda s, x, y: y) + exchange = get_patched_exchange(mocker, default_conf, api_mock) + + order = exchange.buy(pair='ETH/BTC', ordertype=order_type, + amount=1, rate=200, time_in_force=time_in_force) + + assert 'id' in order + assert 'info' in order + assert order['id'] == order_id + assert api_mock.create_order.call_args[0][0] == 'ETH/BTC' + assert api_mock.create_order.call_args[0][1] == order_type + assert api_mock.create_order.call_args[0][2] == 'buy' + assert api_mock.create_order.call_args[0][3] == 1 + assert api_mock.create_order.call_args[0][4] is None + assert api_mock.create_order.call_args[0][5] == {'timeInForce': 'ioc'} + + def test_sell_dry_run(default_conf, mocker): default_conf['dry_run'] = True exchange = get_patched_exchange(mocker, default_conf) From adcaa8439e37f192d6f0e1d91d4ebdf3cabc1972 Mon Sep 17 00:00:00 2001 From: misagh Date: Mon, 10 Dec 2018 19:17:56 +0100 Subject: [PATCH 596/699] test_strategy_override_order_tif added --- freqtrade/tests/strategy/test_strategy.py | 35 +++++++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/freqtrade/tests/strategy/test_strategy.py b/freqtrade/tests/strategy/test_strategy.py index 271fe4d32..08d95fc5d 100644 --- a/freqtrade/tests/strategy/test_strategy.py +++ b/freqtrade/tests/strategy/test_strategy.py @@ -221,6 +221,41 @@ def test_strategy_override_order_types(caplog): StrategyResolver(config) +def test_strategy_override_order_tif(caplog): + caplog.set_level(logging.INFO) + + order_time_in_force = { + 'buy': 'fok', + 'sell': 'gtc', + } + + config = { + 'strategy': 'DefaultStrategy', + 'order_time_in_force': order_time_in_force + } + resolver = StrategyResolver(config) + + assert resolver.strategy.order_time_in_force + for method in ['buy', 'sell']: + assert resolver.strategy.order_time_in_force[method] == order_time_in_force[method] + + assert ('freqtrade.resolvers.strategy_resolver', + logging.INFO, + "Override strategy 'order_time_in_force' with value in config file:" + " {'buy': 'fok', 'sell': 'gtc'}." + ) in caplog.record_tuples + + config = { + 'strategy': 'DefaultStrategy', + 'order_time_in_force': {'buy': 'fok'} + } + # Raise error for invalid configuration + with pytest.raises(ImportError, + match=r"Impossible to load Strategy 'DefaultStrategy'. " + r"Order-time-in-force mapping is incomplete."): + StrategyResolver(config) + + def test_deprecate_populate_indicators(result): default_location = path.join(path.dirname(path.realpath(__file__))) resolver = StrategyResolver({'strategy': 'TestStrategyLegacy', From 36de4518098944875d79d88fd4854104499786ea Mon Sep 17 00:00:00 2001 From: Matthias Date: Mon, 10 Dec 2018 19:54:43 +0100 Subject: [PATCH 597/699] Remove class-level variables --- freqtrade/exchange/__init__.py | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/freqtrade/exchange/__init__.py b/freqtrade/exchange/__init__.py index 65e912a1f..40ac9b58a 100644 --- a/freqtrade/exchange/__init__.py +++ b/freqtrade/exchange/__init__.py @@ -64,14 +64,8 @@ def retrier(f): class Exchange(object): - # Current selected exchange - _api: ccxt.Exchange = None - _api_async: ccxt_async.Exchange = None _conf: Dict = {} - # Holds all open sell orders for dry_run - _dry_run_open_orders: Dict[str, Any] = {} - def __init__(self, config: dict) -> None: """ Initializes this module with the given config, @@ -89,13 +83,17 @@ class Exchange(object): # Holds candles self.klines: Dict[str, Any] = {} + # Holds all open sell orders for dry_run + self._dry_run_open_orders: Dict[str, Any] = {} + if config['dry_run']: logger.info('Instance is running with dry_run enabled') exchange_config = config['exchange'] - self._api = self._init_ccxt(exchange_config, ccxt_kwargs=exchange_config.get('ccxt_config')) - self._api_async = self._init_ccxt(exchange_config, ccxt_async, - ccxt_kwargs=exchange_config.get('ccxt_async_config')) + self._api: ccxt.Exchange = self._init_ccxt( + exchange_config, ccxt_kwargs=exchange_config.get('ccxt_config')) + self._api_async: ccxt_async.Exchange = self._init_ccxt( + exchange_config, ccxt_async, ccxt_kwargs=exchange_config.get('ccxt_async_config')) logger.info('Using Exchange "%s"', self.name) From e2bff9d5cb86e11335663f52bcb06dff8e373243 Mon Sep 17 00:00:00 2001 From: Matthias Date: Mon, 10 Dec 2018 20:22:21 +0100 Subject: [PATCH 598/699] Remove assigning klines from download method --- freqtrade/exchange/__init__.py | 24 +++++++++++------------ freqtrade/tests/exchange/test_exchange.py | 13 +++++++----- 2 files changed, 20 insertions(+), 17 deletions(-) diff --git a/freqtrade/exchange/__init__.py b/freqtrade/exchange/__init__.py index 40ac9b58a..06c6d9382 100644 --- a/freqtrade/exchange/__init__.py +++ b/freqtrade/exchange/__init__.py @@ -489,9 +489,9 @@ class Exchange(object): # Combine tickers data: List = [] - for tick in tickers: - if tick[0] == pair: - data.extend(tick[1]) + for p, ticker in tickers: + if p == pair: + data.extend(ticker) # Sort data again after extending the result - above calls return in "async order" order data = sorted(data, key=lambda x: x[0]) logger.info("downloaded %s with length %s.", pair, len(data)) @@ -502,9 +502,16 @@ class Exchange(object): Refresh tickers asyncronously and return the result. """ logger.debug("Refreshing klines for %d pairs", len(pair_list)) - asyncio.get_event_loop().run_until_complete( + ticklist = asyncio.get_event_loop().run_until_complete( self.async_get_candles_history(pair_list, ticker_interval)) + for pair, ticks in ticklist: + # keeping last candle time as last refreshed time of the pair + if ticks: + self._pairs_last_refresh_time[pair] = ticks[-1][0] // 1000 + # keeping candles in cache + self.klines[pair] = ticks + async def async_get_candles_history(self, pairs: List[str], tick_interval: str) -> List[Tuple[str, List]]: """Download ohlcv history for pair-list asyncronously """ @@ -528,7 +535,7 @@ class Exchange(object): # so we fetch it from local cache if (not since_ms and self._pairs_last_refresh_time.get(pair, 0) + interval_in_sec >= - arrow.utcnow().timestamp): + arrow.utcnow().timestamp and pair in self.klines): data = self.klines[pair] logger.debug("Using cached klines data for %s ...", pair) else: @@ -542,13 +549,6 @@ class Exchange(object): if data and data[0][0] > data[-1][0]: data = sorted(data, key=lambda x: x[0]) - # keeping last candle time as last refreshed time of the pair - if data: - self._pairs_last_refresh_time[pair] = data[-1][0] // 1000 - - # keeping candles in cache - self.klines[pair] = data - logger.debug("done fetching %s ...", pair) return pair, data diff --git a/freqtrade/tests/exchange/test_exchange.py b/freqtrade/tests/exchange/test_exchange.py index d1f391266..b711dd3ab 100644 --- a/freqtrade/tests/exchange/test_exchange.py +++ b/freqtrade/tests/exchange/test_exchange.py @@ -737,7 +737,7 @@ def test_get_history(default_conf, mocker, caplog): def test_refresh_tickers(mocker, default_conf, caplog) -> None: tick = [ [ - 1511686200000, # unix timestamp ms + arrow.utcnow().timestamp * 1000, # unix timestamp ms 1, # open 2, # high 3, # low @@ -757,9 +757,16 @@ def test_refresh_tickers(mocker, default_conf, caplog) -> None: assert log_has(f'Refreshing klines for {len(pairs)} pairs', caplog.record_tuples) assert exchange.klines + assert exchange._api_async.fetch_ohlcv.call_count == 2 for pair in pairs: assert exchange.klines[pair] + # test caching + exchange.refresh_tickers(['IOTA/ETH', 'XRP/ETH'], '5m') + + assert exchange._api_async.fetch_ohlcv.call_count == 2 + assert log_has(f"Using cached klines data for {pairs[0]} ...", caplog.record_tuples) + @pytest.mark.asyncio async def test__async_get_candle_history(default_conf, mocker, caplog): @@ -788,10 +795,6 @@ async def test__async_get_candle_history(default_conf, mocker, caplog): assert res[1] == tick assert exchange._api_async.fetch_ohlcv.call_count == 1 assert not log_has(f"Using cached klines data for {pair} ...", caplog.record_tuples) - # test caching - res = await exchange._async_get_candle_history(pair, "5m") - assert exchange._api_async.fetch_ohlcv.call_count == 1 - assert log_has(f"Using cached klines data for {pair} ...", caplog.record_tuples) # exchange = Exchange(default_conf) await async_ccxt_exception(mocker, default_conf, MagicMock(), From 523dea4a045e432d8bc75d5899caf4fc606f4755 Mon Sep 17 00:00:00 2001 From: Matthias Date: Mon, 10 Dec 2018 20:22:41 +0100 Subject: [PATCH 599/699] remove hacky workaround not needed anymore --- freqtrade/edge/__init__.py | 14 -------------- 1 file changed, 14 deletions(-) diff --git a/freqtrade/edge/__init__.py b/freqtrade/edge/__init__.py index 49acbd3e7..9bd43e37f 100644 --- a/freqtrade/edge/__init__.py +++ b/freqtrade/edge/__init__.py @@ -144,20 +144,6 @@ class Edge(): self._cached_pairs = self._process_expectancy(trades_df) self._last_updated = arrow.utcnow().timestamp - # Not a nice hack but probably simplest solution: - # When backtest load data it loads the delta between disk and exchange - # The problem is that exchange consider that recent. - # it is but it is incomplete (c.f. _async_get_candle_history) - # So it causes get_signal to exit cause incomplete ticker_hist - # A patch to that would be update _pairs_last_refresh_time of exchange - # so it will download again all pairs - # Another solution is to add new data to klines instead of reassigning it: - # self.klines[pair].update(data) instead of self.klines[pair] = data in exchange package. - # But that means indexing timestamp and having a verification so that - # there is no empty range between two timestaps (recently added and last - # one) - self.exchange._pairs_last_refresh_time = {} - return True def stake_amount(self, pair: str, free_capital: float, From 8c1901ad1e6ab95f92226a8fa27c331c0baa766f Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 11 Dec 2018 07:11:43 +0100 Subject: [PATCH 600/699] Extract caching logic from lowestlevel fetch_ohlcv function --- freqtrade/exchange/__init__.py | 61 +++++++++++++++++----------------- 1 file changed, 30 insertions(+), 31 deletions(-) diff --git a/freqtrade/exchange/__init__.py b/freqtrade/exchange/__init__.py index 06c6d9382..c0fb6629f 100644 --- a/freqtrade/exchange/__init__.py +++ b/freqtrade/exchange/__init__.py @@ -126,12 +126,12 @@ class Exchange(object): raise OperationalException(f'Exchange {name} is not supported') ex_config = { - 'apiKey': exchange_config.get('key'), - 'secret': exchange_config.get('secret'), - 'password': exchange_config.get('password'), - 'uid': exchange_config.get('uid', ''), - 'enableRateLimit': exchange_config.get('ccxt_rate_limit', True) - } + 'apiKey': exchange_config.get('key'), + 'secret': exchange_config.get('secret'), + 'password': exchange_config.get('password'), + 'uid': exchange_config.get('uid', ''), + 'enableRateLimit': exchange_config.get('ccxt_rate_limit', True) + } if ccxt_kwargs: logger.info('Applying additional ccxt config: %s', ccxt_kwargs) ex_config.update(ccxt_kwargs) @@ -499,25 +499,36 @@ class Exchange(object): def refresh_tickers(self, pair_list: List[str], ticker_interval: str) -> None: """ - Refresh tickers asyncronously and return the result. + Refresh tickers asyncronously and set `klines` of this object with the result """ logger.debug("Refreshing klines for %d pairs", len(pair_list)) - ticklist = asyncio.get_event_loop().run_until_complete( + asyncio.get_event_loop().run_until_complete( self.async_get_candles_history(pair_list, ticker_interval)) - for pair, ticks in ticklist: - # keeping last candle time as last refreshed time of the pair - if ticks: - self._pairs_last_refresh_time[pair] = ticks[-1][0] // 1000 - # keeping candles in cache - self.klines[pair] = ticks - async def async_get_candles_history(self, pairs: List[str], tick_interval: str) -> List[Tuple[str, List]]: """Download ohlcv history for pair-list asyncronously """ - input_coroutines = [self._async_get_candle_history( - symbol, tick_interval) for symbol in pairs] + # Calculating ticker interval in second + interval_in_sec = constants.TICKER_INTERVAL_MINUTES[tick_interval] * 60 + input_coroutines = [] + + # Gather corotines to run + for pair in pairs: + if not (self._pairs_last_refresh_time.get(pair, 0) + interval_in_sec >= + arrow.utcnow().timestamp and pair in self.klines): + input_coroutines.append(self._async_get_candle_history(pair, tick_interval)) + else: + logger.debug("Using cached klines data for %s ...", pair) + tickers = await asyncio.gather(*input_coroutines, return_exceptions=True) + + # handle caching + for pair, ticks in tickers: + # keeping last candle time as last refreshed time of the pair + if ticks: + self._pairs_last_refresh_time[pair] = ticks[-1][0] // 1000 + # keeping parsed dataframe in cache + self.klines[pair] = ticks return tickers @retrier_async @@ -527,20 +538,8 @@ class Exchange(object): # fetch ohlcv asynchronously logger.debug("fetching %s since %s ...", pair, since_ms) - # Calculating ticker interval in second - interval_in_sec = constants.TICKER_INTERVAL_MINUTES[tick_interval] * 60 - - # If (last update time) + (interval in second) is greater or equal than now - # that means we don't have to hit the API as there is no new candle - # so we fetch it from local cache - if (not since_ms and - self._pairs_last_refresh_time.get(pair, 0) + interval_in_sec >= - arrow.utcnow().timestamp and pair in self.klines): - data = self.klines[pair] - logger.debug("Using cached klines data for %s ...", pair) - else: - data = await self._api_async.fetch_ohlcv(pair, timeframe=tick_interval, - since=since_ms) + data = await self._api_async.fetch_ohlcv(pair, timeframe=tick_interval, + since=since_ms) # Because some exchange sort Tickers ASC and other DESC. # Ex: Bittrex returns a list of tickers ASC (oldest first, newest last) From 97e7b0d9f6bcd3e7e35d8a5776545243bfafb0f8 Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Tue, 11 Dec 2018 13:33:06 +0100 Subject: [PATCH 601/699] Update ccxt from 1.18.17 to 1.18.18 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 7f6472f8a..eb9f2876a 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -ccxt==1.18.17 +ccxt==1.18.18 SQLAlchemy==1.2.14 python-telegram-bot==11.1.0 arrow==0.12.1 From 70ad8a06c39552edc52537c53931c28e60e490b5 Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Tue, 11 Dec 2018 13:33:08 +0100 Subject: [PATCH 602/699] Update requests from 2.20.1 to 2.21.0 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index eb9f2876a..3cdf22439 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,7 +3,7 @@ SQLAlchemy==1.2.14 python-telegram-bot==11.1.0 arrow==0.12.1 cachetools==3.0.0 -requests==2.20.1 +requests==2.21.0 urllib3==1.24.1 wrapt==1.10.11 pandas==0.23.4 From 0ab8ac1c1d52d632909274142b0826f0fd82a71d Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 11 Dec 2018 19:18:28 +0100 Subject: [PATCH 603/699] Add test to verify downloading history does not modify _pairs_last_refresh_time --- freqtrade/tests/optimize/test_optimize.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/freqtrade/tests/optimize/test_optimize.py b/freqtrade/tests/optimize/test_optimize.py index d73f31ad5..970041a4f 100644 --- a/freqtrade/tests/optimize/test_optimize.py +++ b/freqtrade/tests/optimize/test_optimize.py @@ -283,13 +283,15 @@ def test_download_pairs_exception(ticker_history, mocker, caplog, default_conf) def test_download_backtesting_testdata(ticker_history, mocker, default_conf) -> None: mocker.patch('freqtrade.exchange.Exchange.get_history', return_value=ticker_history) exchange = get_patched_exchange(mocker, default_conf) - + # Tst that pairs-cached is not touched. + assert not exchange._pairs_last_refresh_time # Download a 1 min ticker file file1 = os.path.join(os.path.dirname(__file__), '..', 'testdata', 'XEL_BTC-1m.json') _backup_file(file1) download_backtesting_testdata(None, exchange, pair="XEL/BTC", tick_interval='1m') assert os.path.isfile(file1) is True _clean_test_file(file1) + assert not exchange._pairs_last_refresh_time # Download a 5 min ticker file file2 = os.path.join(os.path.dirname(__file__), '..', 'testdata', 'STORJ_BTC-5m.json') @@ -298,6 +300,7 @@ def test_download_backtesting_testdata(ticker_history, mocker, default_conf) -> download_backtesting_testdata(None, exchange, pair="STORJ/BTC", tick_interval='5m') assert os.path.isfile(file2) is True _clean_test_file(file2) + assert not exchange._pairs_last_refresh_time def test_download_backtesting_testdata2(mocker, default_conf) -> None: From 0f2c5478054e74b4d5c836a98f88b2a975262440 Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 11 Dec 2018 20:26:53 +0100 Subject: [PATCH 604/699] Move fiat-convert to subfolder --- freqtrade/{ => rpc}/fiat_convert.py | 0 .../tests/{ => rpc}/test_fiat_convert.py | 19 +++++++++++-------- 2 files changed, 11 insertions(+), 8 deletions(-) rename freqtrade/{ => rpc}/fiat_convert.py (100%) rename freqtrade/tests/{ => rpc}/test_fiat_convert.py (91%) diff --git a/freqtrade/fiat_convert.py b/freqtrade/rpc/fiat_convert.py similarity index 100% rename from freqtrade/fiat_convert.py rename to freqtrade/rpc/fiat_convert.py diff --git a/freqtrade/tests/test_fiat_convert.py b/freqtrade/tests/rpc/test_fiat_convert.py similarity index 91% rename from freqtrade/tests/test_fiat_convert.py rename to freqtrade/tests/rpc/test_fiat_convert.py index 8fd3b66b4..7d857d2f1 100644 --- a/freqtrade/tests/test_fiat_convert.py +++ b/freqtrade/tests/rpc/test_fiat_convert.py @@ -7,7 +7,7 @@ from unittest.mock import MagicMock import pytest from requests.exceptions import RequestException -from freqtrade.fiat_convert import CryptoFiat, CryptoToFiatConverter +from freqtrade.rpc.fiat_convert import CryptoFiat, CryptoToFiatConverter from freqtrade.tests.conftest import log_has, patch_coinmarketcap @@ -81,16 +81,18 @@ def test_fiat_convert_find_price(mocker): assert fiat_convert.get_price(crypto_symbol='XRP', fiat_symbol='USD') == 0.0 - mocker.patch('freqtrade.fiat_convert.CryptoToFiatConverter._find_price', return_value=12345.0) + mocker.patch('freqtrade.rpc.fiat_convert.CryptoToFiatConverter._find_price', + return_value=12345.0) assert fiat_convert.get_price(crypto_symbol='BTC', fiat_symbol='USD') == 12345.0 assert fiat_convert.get_price(crypto_symbol='btc', fiat_symbol='usd') == 12345.0 - mocker.patch('freqtrade.fiat_convert.CryptoToFiatConverter._find_price', return_value=13000.2) + mocker.patch('freqtrade.rpc.fiat_convert.CryptoToFiatConverter._find_price', + return_value=13000.2) assert fiat_convert.get_price(crypto_symbol='BTC', fiat_symbol='EUR') == 13000.2 def test_fiat_convert_unsupported_crypto(mocker, caplog): - mocker.patch('freqtrade.fiat_convert.CryptoToFiatConverter._cryptomap', return_value=[]) + mocker.patch('freqtrade.rpc.fiat_convert.CryptoToFiatConverter._cryptomap', return_value=[]) patch_coinmarketcap(mocker) fiat_convert = CryptoToFiatConverter() assert fiat_convert._find_price(crypto_symbol='CRYPTO_123', fiat_symbol='EUR') == 0.0 @@ -100,7 +102,8 @@ def test_fiat_convert_unsupported_crypto(mocker, caplog): def test_fiat_convert_get_price(mocker): patch_coinmarketcap(mocker) - mocker.patch('freqtrade.fiat_convert.CryptoToFiatConverter._find_price', return_value=28000.0) + mocker.patch('freqtrade.rpc.fiat_convert.CryptoToFiatConverter._find_price', + return_value=28000.0) fiat_convert = CryptoToFiatConverter() @@ -157,7 +160,7 @@ def test_fiat_init_network_exception(mocker): # Because CryptoToFiatConverter is a Singleton we reset the listings listmock = MagicMock(side_effect=RequestException) mocker.patch.multiple( - 'freqtrade.fiat_convert.Market', + 'freqtrade.rpc.fiat_convert.Market', listings=listmock, ) # with pytest.raises(RequestEsxception): @@ -187,7 +190,7 @@ def test_fiat_invalid_response(mocker, caplog): # Because CryptoToFiatConverter is a Singleton we reset the listings listmock = MagicMock(return_value="{'novalidjson':DEADBEEFf}") mocker.patch.multiple( - 'freqtrade.fiat_convert.Market', + 'freqtrade.rpc.fiat_convert.Market', listings=listmock, ) # with pytest.raises(RequestEsxception): @@ -203,7 +206,7 @@ def test_fiat_invalid_response(mocker, caplog): def test_convert_amount(mocker): patch_coinmarketcap(mocker) - mocker.patch('freqtrade.fiat_convert.CryptoToFiatConverter.get_price', return_value=12345.0) + mocker.patch('freqtrade.rpc.fiat_convert.CryptoToFiatConverter.get_price', return_value=12345.0) fiat_convert = CryptoToFiatConverter() result = fiat_convert.convert_amount( From efc709501ae8329a9baf22d91f1cb4265f41e874 Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 11 Dec 2018 20:27:30 +0100 Subject: [PATCH 605/699] move fiat-convert to rpc - adjust imports --- freqtrade/rpc/rpc.py | 2 +- freqtrade/rpc/telegram.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/freqtrade/rpc/rpc.py b/freqtrade/rpc/rpc.py index 6ca32e4be..e83d9d41b 100644 --- a/freqtrade/rpc/rpc.py +++ b/freqtrade/rpc/rpc.py @@ -14,9 +14,9 @@ from numpy import mean, nan_to_num, NAN from pandas import DataFrame from freqtrade import TemporaryError, DependencyException -from freqtrade.fiat_convert import CryptoToFiatConverter from freqtrade.misc import shorten_date from freqtrade.persistence import Trade +from freqtrade.rpc.fiat_convert import CryptoToFiatConverter from freqtrade.state import State from freqtrade.strategy.interface import SellType diff --git a/freqtrade/rpc/telegram.py b/freqtrade/rpc/telegram.py index 56c58f2cc..be2498d78 100644 --- a/freqtrade/rpc/telegram.py +++ b/freqtrade/rpc/telegram.py @@ -12,8 +12,8 @@ from telegram.error import NetworkError, TelegramError from telegram.ext import CommandHandler, Updater from freqtrade.__init__ import __version__ -from freqtrade.fiat_convert import CryptoToFiatConverter from freqtrade.rpc import RPC, RPCException, RPCMessageType +from freqtrade.rpc.fiat_convert import CryptoToFiatConverter logger = logging.getLogger(__name__) From 81b4940eef6b9321684d95f40f10a0c0690d7dab Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 11 Dec 2018 20:27:54 +0100 Subject: [PATCH 606/699] Adjust tests to new fiat-convert location --- freqtrade/tests/conftest.py | 3 +-- freqtrade/tests/rpc/test_rpc.py | 11 ++++++----- freqtrade/tests/rpc/test_rpc_telegram.py | 9 ++++++--- 3 files changed, 13 insertions(+), 10 deletions(-) diff --git a/freqtrade/tests/conftest.py b/freqtrade/tests/conftest.py index df1a1cdc4..1b03f1b19 100644 --- a/freqtrade/tests/conftest.py +++ b/freqtrade/tests/conftest.py @@ -82,7 +82,6 @@ def get_patched_freqtradebot(mocker, config) -> FreqtradeBot: :param config: Config to pass to the bot :return: None """ - # mocker.patch('freqtrade.fiat_convert.Market', {'price_usd': 12345.0}) patch_coinmarketcap(mocker, {'price_usd': 12345.0}) mocker.patch('freqtrade.freqtradebot.RPCManager', MagicMock()) mocker.patch('freqtrade.freqtradebot.persistence.init', MagicMock()) @@ -107,7 +106,7 @@ def patch_coinmarketcap(mocker, value: Optional[Dict[str, float]] = None) -> Non 'website_slug': 'ethereum'} ]}) mocker.patch.multiple( - 'freqtrade.fiat_convert.Market', + 'freqtrade.rpc.fiat_convert.Market', ticker=tickermock, listings=listmock, diff --git a/freqtrade/tests/rpc/test_rpc.py b/freqtrade/tests/rpc/test_rpc.py index 2b271af31..bb685cad5 100644 --- a/freqtrade/tests/rpc/test_rpc.py +++ b/freqtrade/tests/rpc/test_rpc.py @@ -8,10 +8,10 @@ import pytest from numpy import isnan from freqtrade import TemporaryError, DependencyException -from freqtrade.fiat_convert import CryptoToFiatConverter from freqtrade.freqtradebot import FreqtradeBot from freqtrade.persistence import Trade from freqtrade.rpc import RPC, RPCException +from freqtrade.rpc.fiat_convert import CryptoToFiatConverter from freqtrade.state import State from freqtrade.tests.test_freqtradebot import patch_get_signal from freqtrade.tests.conftest import patch_coinmarketcap, patch_exchange @@ -171,7 +171,7 @@ def test_rpc_daily_profit(default_conf, update, ticker, fee, def test_rpc_trade_statistics(default_conf, ticker, ticker_sell_up, fee, limit_buy_order, limit_sell_order, markets, mocker) -> None: mocker.patch.multiple( - 'freqtrade.fiat_convert.Market', + 'freqtrade.rpc.fiat_convert.Market', ticker=MagicMock(return_value={'price_usd': 15000.0}), ) patch_coinmarketcap(mocker) @@ -260,10 +260,11 @@ def test_rpc_trade_statistics_closed(mocker, default_conf, ticker, fee, markets, ticker_sell_up, limit_buy_order, limit_sell_order): patch_exchange(mocker) mocker.patch.multiple( - 'freqtrade.fiat_convert.Market', + 'freqtrade.rpc.fiat_convert.Market', ticker=MagicMock(return_value={'price_usd': 15000.0}), ) - mocker.patch('freqtrade.fiat_convert.CryptoToFiatConverter._find_price', return_value=15000.0) + mocker.patch('freqtrade.rpc.fiat_convert.CryptoToFiatConverter._find_price', + return_value=15000.0) mocker.patch('freqtrade.rpc.telegram.Telegram', MagicMock()) mocker.patch.multiple( 'freqtrade.exchange.Exchange', @@ -328,7 +329,7 @@ def test_rpc_balance_handle(default_conf, mocker): # ETH will be skipped due to mocked Error below mocker.patch.multiple( - 'freqtrade.fiat_convert.Market', + 'freqtrade.rpc.fiat_convert.Market', ticker=MagicMock(return_value={'price_usd': 15000.0}), ) patch_coinmarketcap(mocker) diff --git a/freqtrade/tests/rpc/test_rpc_telegram.py b/freqtrade/tests/rpc/test_rpc_telegram.py index cd4445a1e..686a92469 100644 --- a/freqtrade/tests/rpc/test_rpc_telegram.py +++ b/freqtrade/tests/rpc/test_rpc_telegram.py @@ -737,7 +737,8 @@ def test_forcesell_handle(default_conf, update, ticker, fee, def test_forcesell_down_handle(default_conf, update, ticker, fee, ticker_sell_down, markets, mocker) -> None: patch_coinmarketcap(mocker, value={'price_usd': 15000.0}) - mocker.patch('freqtrade.fiat_convert.CryptoToFiatConverter._find_price', return_value=15000.0) + mocker.patch('freqtrade.rpc.fiat_convert.CryptoToFiatConverter._find_price', + return_value=15000.0) rpc_mock = mocker.patch('freqtrade.rpc.telegram.Telegram.send_msg', MagicMock()) mocker.patch('freqtrade.rpc.telegram.Telegram._init', MagicMock()) mocker.patch.multiple( @@ -791,7 +792,8 @@ def test_forcesell_down_handle(default_conf, update, ticker, fee, def test_forcesell_all_handle(default_conf, update, ticker, fee, markets, mocker) -> None: patch_coinmarketcap(mocker, value={'price_usd': 15000.0}) patch_exchange(mocker) - mocker.patch('freqtrade.fiat_convert.CryptoToFiatConverter._find_price', return_value=15000.0) + mocker.patch('freqtrade.rpc.fiat_convert.CryptoToFiatConverter._find_price', + return_value=15000.0) rpc_mock = mocker.patch('freqtrade.rpc.telegram.Telegram.send_msg', MagicMock()) mocker.patch('freqtrade.rpc.telegram.Telegram._init', MagicMock()) mocker.patch('freqtrade.exchange.Exchange.get_pair_detail_url', MagicMock()) @@ -836,7 +838,8 @@ def test_forcesell_all_handle(default_conf, update, ticker, fee, markets, mocker def test_forcesell_handle_invalid(default_conf, update, mocker) -> None: patch_coinmarketcap(mocker, value={'price_usd': 15000.0}) - mocker.patch('freqtrade.fiat_convert.CryptoToFiatConverter._find_price', return_value=15000.0) + mocker.patch('freqtrade.rpc.fiat_convert.CryptoToFiatConverter._find_price', + return_value=15000.0) msg_mock = MagicMock() mocker.patch.multiple( 'freqtrade.rpc.telegram.Telegram', From 8d8b53f4d1a6fad07dbf4d6931f2f5f855f4f802 Mon Sep 17 00:00:00 2001 From: misagh Date: Wed, 12 Dec 2018 13:05:55 +0100 Subject: [PATCH 607/699] added tests for IOC and FOK --- freqtrade/freqtradebot.py | 10 +++---- freqtrade/tests/test_freqtradebot.py | 44 ++++++++++++++++++++++++++++ 2 files changed, 49 insertions(+), 5 deletions(-) diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index 7adc4fa27..099c62591 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -413,16 +413,16 @@ class FreqtradeBot(object): order_tif, order_type, pair_s, order_status, self.exchange.name, order['filled'], order['amount'], order['remaining'] ) - stake_amount = order['price'] + stake_amount = order['cost'] amount = order['amount'] - buy_limit_filled_price = order['average'] + buy_limit_filled_price = order['price'] order_id = None # in case of FOK the order may be filled immediately and fully - elif order_status == 'filled': - stake_amount = order['price'] + elif order_status == 'closed': + stake_amount = order['cost'] amount = order['amount'] - buy_limit_filled_price = order['average'] + buy_limit_filled_price = order['price'] order_id = None self.rpc.send_msg({ diff --git a/freqtrade/tests/test_freqtradebot.py b/freqtrade/tests/test_freqtradebot.py index 70cf8edc5..0b6a14112 100644 --- a/freqtrade/tests/test_freqtradebot.py +++ b/freqtrade/tests/test_freqtradebot.py @@ -863,6 +863,13 @@ def test_execute_buy(mocker, default_conf, fee, markets, limit_buy_order) -> Non assert call_args['rate'] == bid assert call_args['amount'] == stake_amount / bid + # Should create an open trade with an open order id + # As the order is not fulfilled yet + trade = Trade.query.first() + assert trade + assert trade.is_open is True + assert trade.open_order_id == limit_buy_order['id'] + # Test calling with price fix_price = 0.06 assert freqtrade.execute_buy(pair, stake_amount, fix_price) @@ -875,6 +882,43 @@ def test_execute_buy(mocker, default_conf, fee, markets, limit_buy_order) -> Non assert call_args['rate'] == fix_price assert call_args['amount'] == stake_amount / fix_price + # In case of closed order + limit_buy_order['status'] = 'closed' + limit_buy_order['price'] = 10 + limit_buy_order['cost'] = 100 + mocker.patch('freqtrade.exchange.Exchange.buy', MagicMock(return_value=limit_buy_order)) + assert freqtrade.execute_buy(pair, stake_amount) + trade = Trade.query.all()[2] + assert trade + assert trade.open_order_id is None + assert trade.open_rate == 10 + assert trade.stake_amount == 100 + + # In case of rejected or expired order and partially filled + limit_buy_order['status'] = 'expired' + limit_buy_order['amount'] = 90.99181073 + limit_buy_order['filled'] = 80.99181073 + limit_buy_order['remaining'] = 10.00 + limit_buy_order['price'] = 0.5 + limit_buy_order['cost'] = 40.495905365 + mocker.patch('freqtrade.exchange.Exchange.buy', MagicMock(return_value=limit_buy_order)) + assert freqtrade.execute_buy(pair, stake_amount) + trade = Trade.query.all()[3] + assert trade + assert trade.open_order_id is None + assert trade.open_rate == 0.5 + assert trade.stake_amount == 40.495905365 + + # In case of the order is rejected and not filled at all + limit_buy_order['status'] = 'rejected' + limit_buy_order['amount'] = 90.99181073 + limit_buy_order['filled'] = 0.0 + limit_buy_order['remaining'] = 90.99181073 + limit_buy_order['price'] = 0.5 + limit_buy_order['cost'] = 0.0 + mocker.patch('freqtrade.exchange.Exchange.buy', MagicMock(return_value=limit_buy_order)) + assert not freqtrade.execute_buy(pair, stake_amount) + def test_add_stoploss_on_exchange(mocker, default_conf, limit_buy_order) -> None: patch_RPCManager(mocker) From fd953bab8c6e3697eb2c41a51b51d1f2f9c546d3 Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Wed, 12 Dec 2018 13:32:07 +0100 Subject: [PATCH 608/699] Update ccxt from 1.18.18 to 1.18.24 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 3cdf22439..72df60c1c 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -ccxt==1.18.18 +ccxt==1.18.24 SQLAlchemy==1.2.14 python-telegram-bot==11.1.0 arrow==0.12.1 From 79f5c4adfeff5f55a0216034bd75972b33ccfeba Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Wed, 12 Dec 2018 13:32:09 +0100 Subject: [PATCH 609/699] Update sqlalchemy from 1.2.14 to 1.2.15 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 72df60c1c..88e53287a 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,5 @@ ccxt==1.18.24 -SQLAlchemy==1.2.14 +SQLAlchemy==1.2.15 python-telegram-bot==11.1.0 arrow==0.12.1 cachetools==3.0.0 From aa1262bea6fe0bb55e3c20289ef0d9b9bb65a5c8 Mon Sep 17 00:00:00 2001 From: misagh Date: Wed, 12 Dec 2018 13:33:03 +0100 Subject: [PATCH 610/699] typo corrected --- freqtrade/freqtradebot.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index 099c62591..8e0440dc3 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -398,7 +398,7 @@ class FreqtradeBot(object): order_type = self.strategy.order_types['buy'] order_tif = self.strategy.order_time_in_force['buy'] - # return false is order is not filled + # return false if the order is not filled if float(order['filled']) == 0: logger.warning('Buy %s order with time in force %s for %s is %s by %s.' ' zero amount is fulfilled.', @@ -430,7 +430,7 @@ class FreqtradeBot(object): 'exchange': self.exchange.name.capitalize(), 'pair': pair_s, 'market_url': pair_url, - 'limit': buy_limit_requested, + 'limit': buy_limit_filled_price, 'stake_amount': stake_amount, 'stake_currency': stake_currency, 'fiat_currency': fiat_currency From 627ab9f583fad5fd9ef5ece90d1fd956d2ff2355 Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 11 Dec 2018 19:47:48 +0100 Subject: [PATCH 611/699] pass around dataframe instead of list --- freqtrade/exchange/__init__.py | 18 +++++++++++++----- freqtrade/exchange/exchange_helpers.py | 1 + freqtrade/freqtradebot.py | 5 ++--- freqtrade/strategy/interface.py | 13 ++++++------- 4 files changed, 22 insertions(+), 15 deletions(-) diff --git a/freqtrade/exchange/__init__.py b/freqtrade/exchange/__init__.py index c0fb6629f..02104062d 100644 --- a/freqtrade/exchange/__init__.py +++ b/freqtrade/exchange/__init__.py @@ -7,12 +7,14 @@ from typing import List, Dict, Tuple, Any, Optional from datetime import datetime from math import floor, ceil +import arrow import asyncio import ccxt import ccxt.async_support as ccxt_async -import arrow +from pandas import DataFrame from freqtrade import constants, OperationalException, DependencyException, TemporaryError +from freqtrade.exchange.exchange_helpers import parse_ticker_dataframe logger = logging.getLogger(__name__) @@ -81,7 +83,7 @@ class Exchange(object): self._pairs_last_refresh_time: Dict[str, int] = {} # Holds candles - self.klines: Dict[str, Any] = {} + self._klines: Dict[str, DataFrame] = {} # Holds all open sell orders for dry_run self._dry_run_open_orders: Dict[str, Any] = {} @@ -155,6 +157,12 @@ class Exchange(object): """exchange ccxt id""" return self._api.id + def klines(self, pair: str) -> DataFrame: + if pair in self._klines: + return self._klines.get(pair).copy() + else: + return None + def set_sandbox(self, api, exchange_config: dict, name: str): if exchange_config.get('sandbox'): if api.urls.get('test'): @@ -499,7 +507,7 @@ class Exchange(object): def refresh_tickers(self, pair_list: List[str], ticker_interval: str) -> None: """ - Refresh tickers asyncronously and set `klines` of this object with the result + Refresh tickers asyncronously and set `_klines` of this object with the result """ logger.debug("Refreshing klines for %d pairs", len(pair_list)) asyncio.get_event_loop().run_until_complete( @@ -515,7 +523,7 @@ class Exchange(object): # Gather corotines to run for pair in pairs: if not (self._pairs_last_refresh_time.get(pair, 0) + interval_in_sec >= - arrow.utcnow().timestamp and pair in self.klines): + arrow.utcnow().timestamp and pair in self._klines): input_coroutines.append(self._async_get_candle_history(pair, tick_interval)) else: logger.debug("Using cached klines data for %s ...", pair) @@ -528,7 +536,7 @@ class Exchange(object): if ticks: self._pairs_last_refresh_time[pair] = ticks[-1][0] // 1000 # keeping parsed dataframe in cache - self.klines[pair] = ticks + self._klines[pair] = parse_ticker_dataframe(ticks) return tickers @retrier_async diff --git a/freqtrade/exchange/exchange_helpers.py b/freqtrade/exchange/exchange_helpers.py index 84e68d4bb..729a4a987 100644 --- a/freqtrade/exchange/exchange_helpers.py +++ b/freqtrade/exchange/exchange_helpers.py @@ -14,6 +14,7 @@ def parse_ticker_dataframe(ticker: list) -> DataFrame: :param ticker: ticker list, as returned by exchange.async_get_candle_history :return: DataFrame """ + logger.debug("Parsing tickerlist to dataframe") cols = ['date', 'open', 'high', 'low', 'close', 'volume'] frame = DataFrame(ticker, columns=cols) diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index 1297a1c2c..e4f6311d5 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -317,7 +317,7 @@ class FreqtradeBot(object): # running get_signal on historical data fetched for _pair in whitelist: - (buy, sell) = self.strategy.get_signal(_pair, interval, self.exchange.klines.get(_pair)) + (buy, sell) = self.strategy.get_signal(_pair, interval, self.exchange.klines(_pair)) if buy and not sell: stake_amount = self._get_trade_stake_amount(_pair) if not stake_amount: @@ -540,9 +540,8 @@ class FreqtradeBot(object): (buy, sell) = (False, False) experimental = self.config.get('experimental', {}) if experimental.get('use_sell_signal') or experimental.get('ignore_roi_if_buy_signal'): - ticker = self.exchange.klines.get(trade.pair) (buy, sell) = self.strategy.get_signal(trade.pair, self.strategy.ticker_interval, - ticker) + self.exchange.klines(trade.pair)) config_ask_strategy = self.config.get('ask_strategy', {}) if config_ask_strategy.get('use_order_book', False): diff --git a/freqtrade/strategy/interface.py b/freqtrade/strategy/interface.py index 141dd996c..d8f379d53 100644 --- a/freqtrade/strategy/interface.py +++ b/freqtrade/strategy/interface.py @@ -6,7 +6,7 @@ import logging from abc import ABC, abstractmethod from datetime import datetime from enum import Enum -from typing import Dict, List, NamedTuple, Optional, Tuple +from typing import Dict, List, NamedTuple, Tuple import warnings import arrow @@ -122,15 +122,13 @@ class IStrategy(ABC): """ return self.__class__.__name__ - def analyze_ticker(self, ticker_history: List[Dict], metadata: dict) -> DataFrame: + def analyze_ticker(self, dataframe: DataFrame, metadata: dict) -> DataFrame: """ Parses the given ticker history and returns a populated DataFrame add several TA indicators and buy signal to it :return DataFrame with ticker data and indicator data """ - dataframe = parse_ticker_dataframe(ticker_history) - pair = str(metadata.get('pair')) # Test if seen this pair and last candle before. @@ -155,19 +153,20 @@ class IStrategy(ABC): return dataframe def get_signal(self, pair: str, interval: str, - ticker_hist: Optional[List[Dict]]) -> Tuple[bool, bool]: + dataframe: DataFrame) -> Tuple[bool, bool]: """ Calculates current signal based several technical analysis indicators :param pair: pair in format ANT/BTC :param interval: Interval to use (in min) + :param dataframe: Dataframe to analyze :return: (Buy, Sell) A bool-tuple indicating buy/sell signal """ - if not ticker_hist: + if isinstance(dataframe, DataFrame) and dataframe.empty: logger.warning('Empty ticker history for pair %s', pair) return False, False try: - dataframe = self.analyze_ticker(ticker_hist, {'pair': pair}) + dataframe = self.analyze_ticker(dataframe, {'pair': pair}) except ValueError as error: logger.warning( 'Unable to analyze ticker for pair %s: %s', From fe3990af3d7398c4367975b61e7516cd368077ab Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 11 Dec 2018 19:48:36 +0100 Subject: [PATCH 612/699] Adjust some tests to dataframe passing --- freqtrade/tests/conftest.py | 7 ++++- freqtrade/tests/exchange/test_exchange.py | 18 +++++++++--- .../tests/exchange/test_exchange_helpers.py | 6 +++- freqtrade/tests/strategy/test_interface.py | 29 ++++++++++--------- scripts/plot_dataframe.py | 2 +- 5 files changed, 42 insertions(+), 20 deletions(-) diff --git a/freqtrade/tests/conftest.py b/freqtrade/tests/conftest.py index 1b03f1b19..59c8835b5 100644 --- a/freqtrade/tests/conftest.py +++ b/freqtrade/tests/conftest.py @@ -481,7 +481,7 @@ def order_book_l2(): @pytest.fixture -def ticker_history(): +def ticker_history_list(): return [ [ 1511686200000, # unix timestamp ms @@ -510,6 +510,11 @@ def ticker_history(): ] +@pytest.fixture +def ticker_history(ticker_history_list): + return parse_ticker_dataframe(ticker_history_list) + + @pytest.fixture def tickers(): return MagicMock(return_value={ diff --git a/freqtrade/tests/exchange/test_exchange.py b/freqtrade/tests/exchange/test_exchange.py index b711dd3ab..0524565fe 100644 --- a/freqtrade/tests/exchange/test_exchange.py +++ b/freqtrade/tests/exchange/test_exchange.py @@ -9,6 +9,7 @@ from unittest.mock import Mock, MagicMock, PropertyMock import arrow import ccxt import pytest +from pandas import DataFrame from freqtrade import DependencyException, OperationalException, TemporaryError from freqtrade.exchange import API_RETRY_COUNT, Exchange @@ -737,12 +738,20 @@ def test_get_history(default_conf, mocker, caplog): def test_refresh_tickers(mocker, default_conf, caplog) -> None: tick = [ [ - arrow.utcnow().timestamp * 1000, # unix timestamp ms + (arrow.utcnow().timestamp - 1) * 1000, # unix timestamp ms 1, # open 2, # high 3, # low 4, # close 5, # volume (in quote currency) + ], + [ + arrow.utcnow().timestamp * 1000, # unix timestamp ms + 3, # open + 1, # high + 4, # low + 6, # close + 5, # volume (in quote currency) ] ] @@ -752,14 +761,15 @@ def test_refresh_tickers(mocker, default_conf, caplog) -> None: pairs = ['IOTA/ETH', 'XRP/ETH'] # empty dicts - assert not exchange.klines + assert not exchange._klines exchange.refresh_tickers(['IOTA/ETH', 'XRP/ETH'], '5m') assert log_has(f'Refreshing klines for {len(pairs)} pairs', caplog.record_tuples) - assert exchange.klines + assert exchange._klines assert exchange._api_async.fetch_ohlcv.call_count == 2 for pair in pairs: - assert exchange.klines[pair] + assert isinstance(exchange.klines(pair), DataFrame) + assert len(exchange.klines(pair)) > 0 # test caching exchange.refresh_tickers(['IOTA/ETH', 'XRP/ETH'], '5m') diff --git a/freqtrade/tests/exchange/test_exchange_helpers.py b/freqtrade/tests/exchange/test_exchange_helpers.py index 82525e805..57a24c69c 100644 --- a/freqtrade/tests/exchange/test_exchange_helpers.py +++ b/freqtrade/tests/exchange/test_exchange_helpers.py @@ -1,6 +1,8 @@ # pragma pylint: disable=missing-docstring, C0103 +import logging from freqtrade.exchange.exchange_helpers import parse_ticker_dataframe +from freqtrade.tests.conftest import log_has def test_dataframe_correct_length(result): @@ -13,9 +15,11 @@ def test_dataframe_correct_columns(result): ['date', 'open', 'high', 'low', 'close', 'volume'] -def test_parse_ticker_dataframe(ticker_history): +def test_parse_ticker_dataframe(ticker_history, caplog): columns = ['date', 'open', 'high', 'low', 'close', 'volume'] + caplog.set_level(logging.DEBUG) # Test file with BV data dataframe = parse_ticker_dataframe(ticker_history) assert dataframe.columns.tolist() == columns + assert log_has('Parsing tickerlist to dataframe', caplog.record_tuples) diff --git a/freqtrade/tests/strategy/test_interface.py b/freqtrade/tests/strategy/test_interface.py index 79c485590..ffef568de 100644 --- a/freqtrade/tests/strategy/test_interface.py +++ b/freqtrade/tests/strategy/test_interface.py @@ -16,62 +16,64 @@ from freqtrade.strategy.default_strategy import DefaultStrategy _STRATEGY = DefaultStrategy(config={}) -def test_returns_latest_buy_signal(mocker, default_conf): +def test_returns_latest_buy_signal(mocker, default_conf, ticker_history): mocker.patch.object( _STRATEGY, 'analyze_ticker', return_value=DataFrame([{'buy': 1, 'sell': 0, 'date': arrow.utcnow()}]) ) - assert _STRATEGY.get_signal('ETH/BTC', '5m', MagicMock()) == (True, False) + assert _STRATEGY.get_signal('ETH/BTC', '5m', ticker_history) == (True, False) mocker.patch.object( _STRATEGY, 'analyze_ticker', return_value=DataFrame([{'buy': 0, 'sell': 1, 'date': arrow.utcnow()}]) ) - assert _STRATEGY.get_signal('ETH/BTC', '5m', MagicMock()) == (False, True) + assert _STRATEGY.get_signal('ETH/BTC', '5m', ticker_history) == (False, True) -def test_returns_latest_sell_signal(mocker, default_conf): +def test_returns_latest_sell_signal(mocker, default_conf, ticker_history): mocker.patch.object( _STRATEGY, 'analyze_ticker', return_value=DataFrame([{'sell': 1, 'buy': 0, 'date': arrow.utcnow()}]) ) - assert _STRATEGY.get_signal('ETH/BTC', '5m', MagicMock()) == (False, True) + assert _STRATEGY.get_signal('ETH/BTC', '5m', ticker_history) == (False, True) mocker.patch.object( _STRATEGY, 'analyze_ticker', return_value=DataFrame([{'sell': 0, 'buy': 1, 'date': arrow.utcnow()}]) ) - assert _STRATEGY.get_signal('ETH/BTC', '5m', MagicMock()) == (True, False) + assert _STRATEGY.get_signal('ETH/BTC', '5m', ticker_history) == (True, False) def test_get_signal_empty(default_conf, mocker, caplog): assert (False, False) == _STRATEGY.get_signal('foo', default_conf['ticker_interval'], - None) + DataFrame()) assert log_has('Empty ticker history for pair foo', caplog.record_tuples) -def test_get_signal_exception_valueerror(default_conf, mocker, caplog): +def test_get_signal_exception_valueerror(default_conf, mocker, caplog, ticker_history): caplog.set_level(logging.INFO) mocker.patch.object( _STRATEGY, 'analyze_ticker', side_effect=ValueError('xyz') ) - assert (False, False) == _STRATEGY.get_signal('foo', default_conf['ticker_interval'], 1) + assert (False, False) == _STRATEGY.get_signal('foo', default_conf['ticker_interval'], + ticker_history) assert log_has('Unable to analyze ticker for pair foo: xyz', caplog.record_tuples) -def test_get_signal_empty_dataframe(default_conf, mocker, caplog): +def test_get_signal_empty_dataframe(default_conf, mocker, caplog, ticker_history): caplog.set_level(logging.INFO) mocker.patch.object( _STRATEGY, 'analyze_ticker', return_value=DataFrame([]) ) - assert (False, False) == _STRATEGY.get_signal('xyz', default_conf['ticker_interval'], 1) + assert (False, False) == _STRATEGY.get_signal('xyz', default_conf['ticker_interval'], + ticker_history) assert log_has('Empty dataframe for pair xyz', caplog.record_tuples) -def test_get_signal_old_dataframe(default_conf, mocker, caplog): +def test_get_signal_old_dataframe(default_conf, mocker, caplog, ticker_history): caplog.set_level(logging.INFO) # default_conf defines a 5m interval. we check interval * 2 + 5m # this is necessary as the last candle is removed (partial candles) by default @@ -81,7 +83,8 @@ def test_get_signal_old_dataframe(default_conf, mocker, caplog): _STRATEGY, 'analyze_ticker', return_value=DataFrame(ticks) ) - assert (False, False) == _STRATEGY.get_signal('xyz', default_conf['ticker_interval'], 1) + assert (False, False) == _STRATEGY.get_signal('xyz', default_conf['ticker_interval'], + ticker_history) assert log_has( 'Outdated history for pair xyz. Last tick is 16 minutes old', caplog.record_tuples diff --git a/scripts/plot_dataframe.py b/scripts/plot_dataframe.py index 8fd3a43bd..5a8a25309 100755 --- a/scripts/plot_dataframe.py +++ b/scripts/plot_dataframe.py @@ -139,7 +139,7 @@ def plot_analyzed_dataframe(args: Namespace) -> None: if args.live: logger.info('Downloading pair.') exchange.refresh_tickers([pair], tick_interval) - tickers[pair] = exchange.klines[pair] + tickers[pair] = exchange.klines(pair) else: tickers = optimize.load_data( datadir=_CONF.get("datadir"), From 7a533de1a815650ba264f97babad1daa511fb197 Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 11 Dec 2018 20:07:14 +0100 Subject: [PATCH 613/699] Use list ticker history for backtesting --- freqtrade/exchange/__init__.py | 2 +- freqtrade/optimize/backtesting.py | 2 +- freqtrade/tests/optimize/test_optimize.py | 12 ++++++------ freqtrade/tests/test_misc.py | 4 ++-- 4 files changed, 10 insertions(+), 10 deletions(-) diff --git a/freqtrade/exchange/__init__.py b/freqtrade/exchange/__init__.py index 02104062d..2bb529250 100644 --- a/freqtrade/exchange/__init__.py +++ b/freqtrade/exchange/__init__.py @@ -159,7 +159,7 @@ class Exchange(object): def klines(self, pair: str) -> DataFrame: if pair in self._klines: - return self._klines.get(pair).copy() + return self._klines[pair].copy() else: return None diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index f950ddb3c..61a2c3e9d 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -362,7 +362,7 @@ class Backtesting(object): if self.config.get('live'): logger.info('Downloading data for all pairs in whitelist ...') self.exchange.refresh_tickers(pairs, self.ticker_interval) - data = self.exchange.klines + data = self.exchange._klines else: logger.info('Using local backtesting data (using whitelist in given config) ...') diff --git a/freqtrade/tests/optimize/test_optimize.py b/freqtrade/tests/optimize/test_optimize.py index 970041a4f..dde76332b 100644 --- a/freqtrade/tests/optimize/test_optimize.py +++ b/freqtrade/tests/optimize/test_optimize.py @@ -85,11 +85,11 @@ def test_load_data_1min_ticker(ticker_history, mocker, caplog) -> None: _clean_test_file(file) -def test_load_data_with_new_pair_1min(ticker_history, mocker, caplog, default_conf) -> None: +def test_load_data_with_new_pair_1min(ticker_history_list, mocker, caplog, default_conf) -> None: """ Test load_data() with 1 min ticker """ - mocker.patch('freqtrade.exchange.Exchange.get_history', return_value=ticker_history) + mocker.patch('freqtrade.exchange.Exchange.get_history', return_value=ticker_history_list) exchange = get_patched_exchange(mocker, default_conf) file = os.path.join(os.path.dirname(__file__), '..', 'testdata', 'MEME_BTC-1m.json') @@ -119,8 +119,8 @@ def test_testdata_path() -> None: assert os.path.join('freqtrade', 'tests', 'testdata') in make_testdata_path(None) -def test_download_pairs(ticker_history, mocker, default_conf) -> None: - mocker.patch('freqtrade.exchange.Exchange.get_history', return_value=ticker_history) +def test_download_pairs(ticker_history_list, mocker, default_conf) -> None: + mocker.patch('freqtrade.exchange.Exchange.get_history', return_value=ticker_history_list) exchange = get_patched_exchange(mocker, default_conf) file1_1 = os.path.join(os.path.dirname(__file__), '..', 'testdata', 'MEME_BTC-1m.json') file1_5 = os.path.join(os.path.dirname(__file__), '..', 'testdata', 'MEME_BTC-5m.json') @@ -280,8 +280,8 @@ def test_download_pairs_exception(ticker_history, mocker, caplog, default_conf) assert log_has('Failed to download the pair: "MEME/BTC", Interval: 1m', caplog.record_tuples) -def test_download_backtesting_testdata(ticker_history, mocker, default_conf) -> None: - mocker.patch('freqtrade.exchange.Exchange.get_history', return_value=ticker_history) +def test_download_backtesting_testdata(ticker_history_list, mocker, default_conf) -> None: + mocker.patch('freqtrade.exchange.Exchange.get_history', return_value=ticker_history_list) exchange = get_patched_exchange(mocker, default_conf) # Tst that pairs-cached is not touched. assert not exchange._pairs_last_refresh_time diff --git a/freqtrade/tests/test_misc.py b/freqtrade/tests/test_misc.py index 26e0c5ee6..e405457a1 100644 --- a/freqtrade/tests/test_misc.py +++ b/freqtrade/tests/test_misc.py @@ -16,8 +16,8 @@ def test_shorten_date() -> None: assert shorten_date(str_data) == str_shorten_data -def test_datesarray_to_datetimearray(ticker_history): - dataframes = parse_ticker_dataframe(ticker_history) +def test_datesarray_to_datetimearray(ticker_history_list): + dataframes = parse_ticker_dataframe(ticker_history_list) dates = datesarray_to_datetimearray(dataframes['date']) assert isinstance(dates[0], datetime.datetime) From d6ba4f0e810feecfb0d6cc6696c37701beb2e77b Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 11 Dec 2018 20:30:43 +0100 Subject: [PATCH 614/699] Fix last 2 tests to use DF as data container --- freqtrade/tests/optimize/test_backtesting.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/freqtrade/tests/optimize/test_backtesting.py b/freqtrade/tests/optimize/test_backtesting.py index e832e3a9b..4f80d618f 100644 --- a/freqtrade/tests/optimize/test_backtesting.py +++ b/freqtrade/tests/optimize/test_backtesting.py @@ -840,7 +840,7 @@ def test_backtest_start_live(default_conf, mocker, caplog): 'Using stake_currency: BTC ...', 'Using stake_amount: 0.001 ...', 'Downloading data for all pairs in whitelist ...', - 'Measuring data from 2017-11-14T19:31:00+00:00 up to 2017-11-14T22:58:00+00:00 (0 days)..', + 'Measuring data from 2017-11-14T19:31:00+00:00 up to 2017-11-14T22:57:00+00:00 (0 days)..', 'Parameter --enable-position-stacking detected ...' ] @@ -899,7 +899,7 @@ def test_backtest_start_multi_strat(default_conf, mocker, caplog): 'Using stake_currency: BTC ...', 'Using stake_amount: 0.001 ...', 'Downloading data for all pairs in whitelist ...', - 'Measuring data from 2017-11-14T19:31:00+00:00 up to 2017-11-14T22:58:00+00:00 (0 days)..', + 'Measuring data from 2017-11-14T19:31:00+00:00 up to 2017-11-14T22:57:00+00:00 (0 days)..', 'Parameter --enable-position-stacking detected ...', 'Running backtesting for Strategy DefaultStrategy', 'Running backtesting for Strategy TestStrategy', From 5c3dcf3e2bf532facf344a086df26afdec47d1cc Mon Sep 17 00:00:00 2001 From: Matthias Date: Wed, 12 Dec 2018 19:35:51 +0100 Subject: [PATCH 615/699] Test for wrong inputs (empty / none-dataframes) in get_signal --- freqtrade/strategy/interface.py | 2 +- freqtrade/tests/strategy/test_interface.py | 5 +++++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/freqtrade/strategy/interface.py b/freqtrade/strategy/interface.py index d8f379d53..1d6ca5ac8 100644 --- a/freqtrade/strategy/interface.py +++ b/freqtrade/strategy/interface.py @@ -161,7 +161,7 @@ class IStrategy(ABC): :param dataframe: Dataframe to analyze :return: (Buy, Sell) A bool-tuple indicating buy/sell signal """ - if isinstance(dataframe, DataFrame) and dataframe.empty: + if not isinstance(dataframe, DataFrame) or dataframe.empty: logger.warning('Empty ticker history for pair %s', pair) return False, False diff --git a/freqtrade/tests/strategy/test_interface.py b/freqtrade/tests/strategy/test_interface.py index ffef568de..d42296462 100644 --- a/freqtrade/tests/strategy/test_interface.py +++ b/freqtrade/tests/strategy/test_interface.py @@ -49,6 +49,11 @@ def test_get_signal_empty(default_conf, mocker, caplog): assert (False, False) == _STRATEGY.get_signal('foo', default_conf['ticker_interval'], DataFrame()) assert log_has('Empty ticker history for pair foo', caplog.record_tuples) + caplog.clear() + + assert (False, False) == _STRATEGY.get_signal('bar', default_conf['ticker_interval'], + []) + assert log_has('Empty ticker history for pair bar', caplog.record_tuples) def test_get_signal_exception_valueerror(default_conf, mocker, caplog, ticker_history): From 7e3955b04c242600304a27dcf5263bd425b87419 Mon Sep 17 00:00:00 2001 From: Matthias Date: Wed, 12 Dec 2018 20:03:41 +0100 Subject: [PATCH 616/699] Fix edge-cli comments (refer to edge, not backtest --- freqtrade/optimize/edge_cli.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/freqtrade/optimize/edge_cli.py b/freqtrade/optimize/edge_cli.py index a2189f6c1..a98f0c877 100644 --- a/freqtrade/optimize/edge_cli.py +++ b/freqtrade/optimize/edge_cli.py @@ -1,7 +1,7 @@ # pragma pylint: disable=missing-docstring, W0212, too-many-arguments """ -This module contains the backtesting logic +This module contains the edge backtesting interface """ import logging from argparse import Namespace @@ -19,11 +19,11 @@ logger = logging.getLogger(__name__) class EdgeCli(object): """ - Backtesting class, this class contains all the logic to run a backtest + EdgeCli class, this class contains all the logic to run edge backtesting - To run a backtest: - backtesting = Backtesting(config) - backtesting.start() + To run a edge backtest: + edge = EdgeCli(config) + edge.start() """ def __init__(self, config: Dict[str, Any]) -> None: @@ -77,7 +77,7 @@ class EdgeCli(object): def setup_configuration(args: Namespace) -> Dict[str, Any]: """ - Prepare the configuration for the backtesting + Prepare the configuration for edge backtesting :param args: Cli args from Arguments() :return: Configuration """ From df5a280169d93b35dfee19355a8e14c116bd7e63 Mon Sep 17 00:00:00 2001 From: misagh Date: Wed, 12 Dec 2018 20:11:27 +0100 Subject: [PATCH 617/699] fix edge doc broken link --- docs/index.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/index.md b/docs/index.md index 9eb0d445c..8fa24e996 100644 --- a/docs/index.md +++ b/docs/index.md @@ -26,7 +26,7 @@ Pull-request. Do not hesitate to reach us on - [Change your strategy](https://github.com/freqtrade/freqtrade/blob/develop/docs/bot-optimization.md#change-your-strategy) - [Add more Indicator](https://github.com/freqtrade/freqtrade/blob/develop/docs/bot-optimization.md#add-more-indicator) - [Test your strategy with Backtesting](https://github.com/freqtrade/freqtrade/blob/develop/docs/backtesting.md) - - [Edge positioning](https://github.com/freqtrade/freqtrade/blob/money_mgt/docs/edge.md) + - [Edge positioning](https://github.com/freqtrade/freqtrade/blob/develop/docs/edge.md) - [Find optimal parameters with Hyperopt](https://github.com/freqtrade/freqtrade/blob/develop/docs/hyperopt.md) - [Control the bot with telegram](https://github.com/freqtrade/freqtrade/blob/develop/docs/telegram-usage.md) - [Receive notifications via webhook](https://github.com/freqtrade/freqtrade/blob/develop/docs/webhook-config.md) From 6b4bab272f42584bedeb9c5c21118d84dc1253fd Mon Sep 17 00:00:00 2001 From: Matthias Date: Wed, 12 Dec 2018 20:20:07 +0100 Subject: [PATCH 618/699] Add link to edge documentation in bot-usage --- docs/bot-usage.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/bot-usage.md b/docs/bot-usage.md index 114e7613e..5451c8459 100644 --- a/docs/bot-usage.md +++ b/docs/bot-usage.md @@ -240,6 +240,8 @@ optional arguments: --stoplosses=-0.01,-0.1,-0.001 ``` +To understand edge and how to read the results, please read the [edge documentation](edge.md). + ## A parameter missing in the configuration? All parameters for `main.py`, `backtesting`, `hyperopt` are referenced From 6c9c03b3d52a25f001c2e546a680966e353881e2 Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Thu, 13 Dec 2018 13:32:07 +0100 Subject: [PATCH 619/699] Update ccxt from 1.18.24 to 1.18.32 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 88e53287a..bff055a63 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -ccxt==1.18.24 +ccxt==1.18.32 SQLAlchemy==1.2.15 python-telegram-bot==11.1.0 arrow==0.12.1 From aca243086e869815380d863fc68f0ce3d18afea0 Mon Sep 17 00:00:00 2001 From: Matthias Date: Thu, 13 Dec 2018 19:43:17 +0100 Subject: [PATCH 620/699] Fix comment --- freqtrade/strategy/interface.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/strategy/interface.py b/freqtrade/strategy/interface.py index 1d6ca5ac8..094831511 100644 --- a/freqtrade/strategy/interface.py +++ b/freqtrade/strategy/interface.py @@ -132,7 +132,7 @@ class IStrategy(ABC): pair = str(metadata.get('pair')) # Test if seen this pair and last candle before. - # always run if process_only_new_candles is set to true + # always run if process_only_new_candles is set to false if (not self.process_only_new_candles or self._last_candle_seen_per_pair.get(pair, None) != dataframe.iloc[-1]['date']): # Defs that only make change on new candle data. From 030ecbfc17bb84c651200b68ace43d1660c2b250 Mon Sep 17 00:00:00 2001 From: Matthias Date: Wed, 12 Dec 2018 19:57:25 +0100 Subject: [PATCH 621/699] move exchange_helpers to data module --- freqtrade/data/__init__.py | 0 freqtrade/{exchange/exchange_helpers.py => data/convert.py} | 4 ++-- freqtrade/exchange/__init__.py | 2 +- .../test_exchange_helpers.py => data/test_convert.py} | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) create mode 100644 freqtrade/data/__init__.py rename freqtrade/{exchange/exchange_helpers.py => data/convert.py} (93%) rename freqtrade/tests/{exchange/test_exchange_helpers.py => data/test_convert.py} (91%) diff --git a/freqtrade/data/__init__.py b/freqtrade/data/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/freqtrade/exchange/exchange_helpers.py b/freqtrade/data/convert.py similarity index 93% rename from freqtrade/exchange/exchange_helpers.py rename to freqtrade/data/convert.py index 729a4a987..e09618a64 100644 --- a/freqtrade/exchange/exchange_helpers.py +++ b/freqtrade/data/convert.py @@ -1,5 +1,5 @@ """ -Functions to analyze ticker data with indicators and produce buy and sell signals +Functions to convert data from one format to another """ import logging import pandas as pd @@ -10,7 +10,7 @@ logger = logging.getLogger(__name__) def parse_ticker_dataframe(ticker: list) -> DataFrame: """ - Analyses the trend for the given ticker history + Converts a ticker-list (format ccxt.fetch_ohlcv) to a Dataframe :param ticker: ticker list, as returned by exchange.async_get_candle_history :return: DataFrame """ diff --git a/freqtrade/exchange/__init__.py b/freqtrade/exchange/__init__.py index 2bb529250..4a29770fe 100644 --- a/freqtrade/exchange/__init__.py +++ b/freqtrade/exchange/__init__.py @@ -14,7 +14,7 @@ import ccxt.async_support as ccxt_async from pandas import DataFrame from freqtrade import constants, OperationalException, DependencyException, TemporaryError -from freqtrade.exchange.exchange_helpers import parse_ticker_dataframe +from freqtrade.data.convert import parse_ticker_dataframe logger = logging.getLogger(__name__) diff --git a/freqtrade/tests/exchange/test_exchange_helpers.py b/freqtrade/tests/data/test_convert.py similarity index 91% rename from freqtrade/tests/exchange/test_exchange_helpers.py rename to freqtrade/tests/data/test_convert.py index 57a24c69c..8e8bed997 100644 --- a/freqtrade/tests/exchange/test_exchange_helpers.py +++ b/freqtrade/tests/data/test_convert.py @@ -1,7 +1,7 @@ # pragma pylint: disable=missing-docstring, C0103 import logging -from freqtrade.exchange.exchange_helpers import parse_ticker_dataframe +from freqtrade.data.convert import parse_ticker_dataframe from freqtrade.tests.conftest import log_has From 453f62cdfa1883a7b6bb160882789497da6678e3 Mon Sep 17 00:00:00 2001 From: Matthias Date: Wed, 12 Dec 2018 19:57:40 +0100 Subject: [PATCH 622/699] Adjust imports --- freqtrade/freqtradebot.py | 6 +++--- freqtrade/strategy/interface.py | 2 +- freqtrade/tests/conftest.py | 2 +- freqtrade/tests/strategy/test_default_strategy.py | 2 +- freqtrade/tests/test_misc.py | 2 +- 5 files changed, 7 insertions(+), 7 deletions(-) diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index e4f6311d5..14a48df3e 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -14,15 +14,15 @@ from requests.exceptions import RequestException from freqtrade import (DependencyException, OperationalException, TemporaryError, __version__, constants, persistence) -from freqtrade.exchange import Exchange -from freqtrade.wallets import Wallets +from freqtrade.data.convert import order_book_to_dataframe from freqtrade.edge import Edge +from freqtrade.exchange import Exchange from freqtrade.persistence import Trade from freqtrade.rpc import RPCManager, RPCMessageType from freqtrade.resolvers import StrategyResolver, PairListResolver from freqtrade.state import State from freqtrade.strategy.interface import SellType, IStrategy -from freqtrade.exchange.exchange_helpers import order_book_to_dataframe +from freqtrade.wallets import Wallets logger = logging.getLogger(__name__) diff --git a/freqtrade/strategy/interface.py b/freqtrade/strategy/interface.py index 094831511..6182cac46 100644 --- a/freqtrade/strategy/interface.py +++ b/freqtrade/strategy/interface.py @@ -13,7 +13,7 @@ import arrow from pandas import DataFrame from freqtrade import constants -from freqtrade.exchange.exchange_helpers import parse_ticker_dataframe +from freqtrade.data.convert import parse_ticker_dataframe from freqtrade.persistence import Trade logger = logging.getLogger(__name__) diff --git a/freqtrade/tests/conftest.py b/freqtrade/tests/conftest.py index 59c8835b5..85a9dabf4 100644 --- a/freqtrade/tests/conftest.py +++ b/freqtrade/tests/conftest.py @@ -11,7 +11,7 @@ import pytest from telegram import Chat, Message, Update from freqtrade import constants -from freqtrade.exchange.exchange_helpers import parse_ticker_dataframe +from freqtrade.data.convert import parse_ticker_dataframe from freqtrade.exchange import Exchange from freqtrade.edge import Edge, PairInfo from freqtrade.freqtradebot import FreqtradeBot diff --git a/freqtrade/tests/strategy/test_default_strategy.py b/freqtrade/tests/strategy/test_default_strategy.py index 6acfc439f..ac574fcbc 100644 --- a/freqtrade/tests/strategy/test_default_strategy.py +++ b/freqtrade/tests/strategy/test_default_strategy.py @@ -3,7 +3,7 @@ import json import pytest from pandas import DataFrame -from freqtrade.exchange.exchange_helpers import parse_ticker_dataframe +from freqtrade.data.convert import parse_ticker_dataframe from freqtrade.strategy.default_strategy import DefaultStrategy diff --git a/freqtrade/tests/test_misc.py b/freqtrade/tests/test_misc.py index e405457a1..a3408576f 100644 --- a/freqtrade/tests/test_misc.py +++ b/freqtrade/tests/test_misc.py @@ -3,7 +3,7 @@ import datetime from unittest.mock import MagicMock -from freqtrade.exchange.exchange_helpers import parse_ticker_dataframe +from freqtrade.data.convert import parse_ticker_dataframe from freqtrade.misc import (common_datearray, datesarray_to_datetimearray, file_dump_json, format_ms_time, shorten_date) from freqtrade.optimize.__init__ import load_tickerdata_file From 1f298028846ce36b6370b671fe0fc89b1192c0e4 Mon Sep 17 00:00:00 2001 From: Matthias Date: Wed, 12 Dec 2018 20:08:00 +0100 Subject: [PATCH 623/699] only export what's needed --- freqtrade/data/__init__.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/freqtrade/data/__init__.py b/freqtrade/data/__init__.py index e69de29bb..c7d0b5983 100644 --- a/freqtrade/data/__init__.py +++ b/freqtrade/data/__init__.py @@ -0,0 +1,3 @@ +__all__ = [ + 'convert' +] From b38195e9b355b5b4c820254a8853ab407db0c500 Mon Sep 17 00:00:00 2001 From: Matthias Date: Wed, 12 Dec 2018 20:16:03 +0100 Subject: [PATCH 624/699] Rename to converter --- freqtrade/data/{convert.py => converter.py} | 0 freqtrade/exchange/__init__.py | 2 +- freqtrade/freqtradebot.py | 2 +- freqtrade/strategy/interface.py | 2 +- freqtrade/tests/conftest.py | 2 +- freqtrade/tests/data/{test_convert.py => test_converter.py} | 2 +- freqtrade/tests/strategy/test_default_strategy.py | 2 +- freqtrade/tests/test_misc.py | 2 +- 8 files changed, 7 insertions(+), 7 deletions(-) rename freqtrade/data/{convert.py => converter.py} (100%) rename freqtrade/tests/data/{test_convert.py => test_converter.py} (93%) diff --git a/freqtrade/data/convert.py b/freqtrade/data/converter.py similarity index 100% rename from freqtrade/data/convert.py rename to freqtrade/data/converter.py diff --git a/freqtrade/exchange/__init__.py b/freqtrade/exchange/__init__.py index 4a29770fe..1d4b504f0 100644 --- a/freqtrade/exchange/__init__.py +++ b/freqtrade/exchange/__init__.py @@ -14,7 +14,7 @@ import ccxt.async_support as ccxt_async from pandas import DataFrame from freqtrade import constants, OperationalException, DependencyException, TemporaryError -from freqtrade.data.convert import parse_ticker_dataframe +from freqtrade.data.converter import parse_ticker_dataframe logger = logging.getLogger(__name__) diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index 14a48df3e..6222072ef 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -14,7 +14,7 @@ from requests.exceptions import RequestException from freqtrade import (DependencyException, OperationalException, TemporaryError, __version__, constants, persistence) -from freqtrade.data.convert import order_book_to_dataframe +from freqtrade.data.converter import order_book_to_dataframe from freqtrade.edge import Edge from freqtrade.exchange import Exchange from freqtrade.persistence import Trade diff --git a/freqtrade/strategy/interface.py b/freqtrade/strategy/interface.py index 6182cac46..c8284b44f 100644 --- a/freqtrade/strategy/interface.py +++ b/freqtrade/strategy/interface.py @@ -13,7 +13,7 @@ import arrow from pandas import DataFrame from freqtrade import constants -from freqtrade.data.convert import parse_ticker_dataframe +from freqtrade.data.converter import parse_ticker_dataframe from freqtrade.persistence import Trade logger = logging.getLogger(__name__) diff --git a/freqtrade/tests/conftest.py b/freqtrade/tests/conftest.py index 85a9dabf4..975a3a9f2 100644 --- a/freqtrade/tests/conftest.py +++ b/freqtrade/tests/conftest.py @@ -11,7 +11,7 @@ import pytest from telegram import Chat, Message, Update from freqtrade import constants -from freqtrade.data.convert import parse_ticker_dataframe +from freqtrade.data.converter import parse_ticker_dataframe from freqtrade.exchange import Exchange from freqtrade.edge import Edge, PairInfo from freqtrade.freqtradebot import FreqtradeBot diff --git a/freqtrade/tests/data/test_convert.py b/freqtrade/tests/data/test_converter.py similarity index 93% rename from freqtrade/tests/data/test_convert.py rename to freqtrade/tests/data/test_converter.py index 8e8bed997..54f32b341 100644 --- a/freqtrade/tests/data/test_convert.py +++ b/freqtrade/tests/data/test_converter.py @@ -1,7 +1,7 @@ # pragma pylint: disable=missing-docstring, C0103 import logging -from freqtrade.data.convert import parse_ticker_dataframe +from freqtrade.data.converter import parse_ticker_dataframe from freqtrade.tests.conftest import log_has diff --git a/freqtrade/tests/strategy/test_default_strategy.py b/freqtrade/tests/strategy/test_default_strategy.py index ac574fcbc..45ed54c4d 100644 --- a/freqtrade/tests/strategy/test_default_strategy.py +++ b/freqtrade/tests/strategy/test_default_strategy.py @@ -3,7 +3,7 @@ import json import pytest from pandas import DataFrame -from freqtrade.data.convert import parse_ticker_dataframe +from freqtrade.data.converter import parse_ticker_dataframe from freqtrade.strategy.default_strategy import DefaultStrategy diff --git a/freqtrade/tests/test_misc.py b/freqtrade/tests/test_misc.py index a3408576f..bec18f8b3 100644 --- a/freqtrade/tests/test_misc.py +++ b/freqtrade/tests/test_misc.py @@ -3,7 +3,7 @@ import datetime from unittest.mock import MagicMock -from freqtrade.data.convert import parse_ticker_dataframe +from freqtrade.data.converter import parse_ticker_dataframe from freqtrade.misc import (common_datearray, datesarray_to_datetimearray, file_dump_json, format_ms_time, shorten_date) from freqtrade.optimize.__init__ import load_tickerdata_file From 1a3fcd47717dc2a997884eded9971251f20b55bf Mon Sep 17 00:00:00 2001 From: Matthias Date: Thu, 13 Dec 2018 06:12:10 +0100 Subject: [PATCH 625/699] extract data-handling methods from optimize --- freqtrade/data/history.py | 257 +++++++++++++++++++++++++++++++++ freqtrade/optimize/__init__.py | 247 +------------------------------ 2 files changed, 259 insertions(+), 245 deletions(-) create mode 100644 freqtrade/data/history.py diff --git a/freqtrade/data/history.py b/freqtrade/data/history.py new file mode 100644 index 000000000..a101c62f8 --- /dev/null +++ b/freqtrade/data/history.py @@ -0,0 +1,257 @@ +# pragma pylint: disable=missing-docstring + +import gzip +try: + import ujson as json + _UJSON = True +except ImportError: + # see mypy/issues/1153 + import json # type: ignore + _UJSON = False +import logging +import os +from typing import Optional, List, Dict, Tuple, Any + +import arrow +from pandas import DataFrame + +from freqtrade import misc, constants, OperationalException +from freqtrade.exchange import Exchange +from freqtrade.arguments import TimeRange + +logger = logging.getLogger(__name__) + + +def json_load(data): + """Try to load data with ujson""" + if _UJSON: + return json.load(data, precise_float=True) + else: + return json.load(data) + + +def trim_tickerlist(tickerlist: List[Dict], timerange: TimeRange) -> List[Dict]: + """ + Trim tickerlist based on given timerange + """ + if not tickerlist: + return tickerlist + + start_index = 0 + stop_index = len(tickerlist) + + if timerange.starttype == 'line': + stop_index = timerange.startts + if timerange.starttype == 'index': + start_index = timerange.startts + elif timerange.starttype == 'date': + while (start_index < len(tickerlist) and + tickerlist[start_index][0] < timerange.startts * 1000): + start_index += 1 + + if timerange.stoptype == 'line': + start_index = len(tickerlist) + timerange.stopts + if timerange.stoptype == 'index': + stop_index = timerange.stopts + elif timerange.stoptype == 'date': + while (stop_index > 0 and + tickerlist[stop_index-1][0] > timerange.stopts * 1000): + stop_index -= 1 + + if start_index > stop_index: + raise ValueError(f'The timerange [{timerange.startts},{timerange.stopts}] is incorrect') + + return tickerlist[start_index:stop_index] + + +def load_tickerdata_file( + datadir: str, pair: str, + ticker_interval: str, + timerange: Optional[TimeRange] = None) -> Optional[List[Dict]]: + """ + Load a pair from file, + :return dict OR empty if unsuccesful + """ + path = make_testdata_path(datadir) + pair_s = pair.replace('/', '_') + file = os.path.join(path, f'{pair_s}-{ticker_interval}.json') + gzipfile = file + '.gz' + + # If the file does not exist we download it when None is returned. + # If file exists, read the file, load the json + if os.path.isfile(gzipfile): + logger.debug('Loading ticker data from file %s', gzipfile) + with gzip.open(gzipfile) as tickerdata: + pairdata = json.load(tickerdata) + elif os.path.isfile(file): + logger.debug('Loading ticker data from file %s', file) + with open(file) as tickerdata: + pairdata = json.load(tickerdata) + else: + return None + + if timerange: + pairdata = trim_tickerlist(pairdata, timerange) + return pairdata + + +def load_data(datadir: str, + ticker_interval: str, + pairs: List[str], + refresh_pairs: Optional[bool] = False, + exchange: Optional[Exchange] = None, + timerange: TimeRange = TimeRange(None, None, 0, 0)) -> Dict[str, List]: + """ + Loads ticker history data for the given parameters + :return: dict + """ + result = {} + + # If the user force the refresh of pairs + if refresh_pairs: + logger.info('Download data for all pairs and store them in %s', datadir) + if not exchange: + raise OperationalException("Exchange needs to be initialized when " + "calling load_data with refresh_pairs=True") + download_pairs(datadir, exchange, pairs, ticker_interval, timerange=timerange) + + for pair in pairs: + pairdata = load_tickerdata_file(datadir, pair, ticker_interval, timerange=timerange) + if pairdata: + if timerange.starttype == 'date' and pairdata[0][0] > timerange.startts * 1000: + logger.warning('Missing data at start for pair %s, data starts at %s', + pair, + arrow.get(pairdata[0][0] // 1000).strftime('%Y-%m-%d %H:%M:%S')) + if timerange.stoptype == 'date' and pairdata[-1][0] < timerange.stopts * 1000: + logger.warning('Missing data at end for pair %s, data ends at %s', + pair, + arrow.get(pairdata[-1][0] // 1000).strftime('%Y-%m-%d %H:%M:%S')) + result[pair] = pairdata + else: + logger.warning( + 'No data for pair: "%s", Interval: %s. ' + 'Use --refresh-pairs-cached to download the data', + pair, + ticker_interval + ) + + return result + + +def make_testdata_path(datadir: str) -> str: + """Return the path where testdata files are stored""" + return datadir or os.path.abspath( + os.path.join( + os.path.dirname(__file__), '..', 'tests', 'testdata' + ) + ) + + +def download_pairs(datadir, exchange: Exchange, pairs: List[str], + ticker_interval: str, + timerange: TimeRange = TimeRange(None, None, 0, 0)) -> bool: + """For each pairs passed in parameters, download the ticker intervals""" + for pair in pairs: + try: + download_backtesting_testdata(datadir, + exchange=exchange, + pair=pair, + tick_interval=ticker_interval, + timerange=timerange) + except BaseException: + logger.info( + 'Failed to download the pair: "%s", Interval: %s', + pair, + ticker_interval + ) + return False + return True + + +def load_cached_data_for_updating(filename: str, + tick_interval: str, + timerange: Optional[TimeRange]) -> Tuple[ + List[Any], + Optional[int]]: + """ + Load cached data and choose what part of the data should be updated + """ + + since_ms = None + + # user sets timerange, so find the start time + if timerange: + if timerange.starttype == 'date': + since_ms = timerange.startts * 1000 + elif timerange.stoptype == 'line': + num_minutes = timerange.stopts * constants.TICKER_INTERVAL_MINUTES[tick_interval] + since_ms = arrow.utcnow().shift(minutes=num_minutes).timestamp * 1000 + + # read the cached file + if os.path.isfile(filename): + with open(filename, "rt") as file: + data = json_load(file) + # remove the last item, because we are not sure if it is correct + # it could be fetched when the candle was incompleted + if data: + data.pop() + else: + data = [] + + if data: + if since_ms and since_ms < data[0][0]: + # the data is requested for earlier period than the cache has + # so fully redownload all the data + data = [] + else: + # a part of the data was already downloaded, so + # download unexist data only + since_ms = data[-1][0] + 1 + + return (data, since_ms) + + +def download_backtesting_testdata(datadir: str, + exchange: Exchange, + pair: str, + tick_interval: str = '5m', + timerange: Optional[TimeRange] = None) -> None: + """ + Download the latest ticker intervals from the exchange for the pair passed in parameters + The data is downloaded starting from the last correct ticker interval data that + exists in a cache. If timerange starts earlier than the data in the cache, + the full data will be redownloaded + + Based on @Rybolov work: https://github.com/rybolov/freqtrade-data + :param pair: pair to download + :param tick_interval: ticker interval + :param timerange: range of time to download + :return: None + + """ + path = make_testdata_path(datadir) + filepair = pair.replace("/", "_") + filename = os.path.join(path, f'{filepair}-{tick_interval}.json') + + logger.info( + 'Download the pair: "%s", Interval: %s', + pair, + tick_interval + ) + + data, since_ms = load_cached_data_for_updating(filename, tick_interval, timerange) + + logger.debug("Current Start: %s", misc.format_ms_time(data[1][0]) if data else 'None') + logger.debug("Current End: %s", misc.format_ms_time(data[-1][0]) if data else 'None') + + # Default since_ms to 30 days if nothing is given + new_data = exchange.get_history(pair=pair, tick_interval=tick_interval, + since_ms=since_ms if since_ms + else + int(arrow.utcnow().shift(days=-30).float_timestamp) * 1000) + data.extend(new_data) + + logger.debug("New Start: %s", misc.format_ms_time(data[0][0])) + logger.debug("New End: %s", misc.format_ms_time(data[-1][0])) + + misc.file_dump_json(filename, data) diff --git a/freqtrade/optimize/__init__.py b/freqtrade/optimize/__init__.py index b1407de18..34d1aaf1c 100644 --- a/freqtrade/optimize/__init__.py +++ b/freqtrade/optimize/__init__.py @@ -1,69 +1,20 @@ # pragma pylint: disable=missing-docstring -import gzip -try: - import ujson as json - _UJSON = True -except ImportError: - # see mypy/issues/1153 - import json # type: ignore - _UJSON = False import logging -import os from datetime import datetime -from typing import Optional, List, Dict, Tuple, Any +from typing import List, Dict, Tuple import operator import arrow from pandas import DataFrame -from freqtrade import misc, constants, OperationalException -from freqtrade.exchange import Exchange + from freqtrade.arguments import TimeRange from freqtrade.optimize.default_hyperopt import DefaultHyperOpts # noqa: F401 logger = logging.getLogger(__name__) -def json_load(data): - """Try to load data with ujson""" - if _UJSON: - return json.load(data, precise_float=True) - else: - return json.load(data) - - -def trim_tickerlist(tickerlist: List[Dict], timerange: TimeRange) -> List[Dict]: - if not tickerlist: - return tickerlist - - start_index = 0 - stop_index = len(tickerlist) - - if timerange.starttype == 'line': - stop_index = timerange.startts - if timerange.starttype == 'index': - start_index = timerange.startts - elif timerange.starttype == 'date': - while (start_index < len(tickerlist) and - tickerlist[start_index][0] < timerange.startts * 1000): - start_index += 1 - - if timerange.stoptype == 'line': - start_index = len(tickerlist) + timerange.stopts - if timerange.stoptype == 'index': - stop_index = timerange.stopts - elif timerange.stoptype == 'date': - while (stop_index > 0 and - tickerlist[stop_index-1][0] > timerange.stopts * 1000): - stop_index -= 1 - - if start_index > stop_index: - raise ValueError(f'The timerange [{timerange.startts},{timerange.stopts}] is incorrect') - - return tickerlist[start_index:stop_index] - - def get_timeframe(data: Dict[str, DataFrame]) -> Tuple[arrow.Arrow, arrow.Arrow]: """ Get the maximum timeframe for the given backtest data @@ -98,197 +49,3 @@ def validate_backtest_data(data: Dict[str, DataFrame], min_date: datetime, logger.warning("%s has missing frames: expected %s, got %s, that's %s missing values", pair, expected_frames, dflen, expected_frames - dflen) return found_missing - - -def load_tickerdata_file( - datadir: str, pair: str, - ticker_interval: str, - timerange: Optional[TimeRange] = None) -> Optional[List[Dict]]: - """ - Load a pair from file, - :return dict OR empty if unsuccesful - """ - path = make_testdata_path(datadir) - pair_s = pair.replace('/', '_') - file = os.path.join(path, f'{pair_s}-{ticker_interval}.json') - gzipfile = file + '.gz' - - # If the file does not exist we download it when None is returned. - # If file exists, read the file, load the json - if os.path.isfile(gzipfile): - logger.debug('Loading ticker data from file %s', gzipfile) - with gzip.open(gzipfile) as tickerdata: - pairdata = json.load(tickerdata) - elif os.path.isfile(file): - logger.debug('Loading ticker data from file %s', file) - with open(file) as tickerdata: - pairdata = json.load(tickerdata) - else: - return None - - if timerange: - pairdata = trim_tickerlist(pairdata, timerange) - return pairdata - - -def load_data(datadir: str, - ticker_interval: str, - pairs: List[str], - refresh_pairs: Optional[bool] = False, - exchange: Optional[Exchange] = None, - timerange: TimeRange = TimeRange(None, None, 0, 0)) -> Dict[str, List]: - """ - Loads ticker history data for the given parameters - :return: dict - """ - result = {} - - # If the user force the refresh of pairs - if refresh_pairs: - logger.info('Download data for all pairs and store them in %s', datadir) - if not exchange: - raise OperationalException("Exchange needs to be initialized when " - "calling load_data with refresh_pairs=True") - download_pairs(datadir, exchange, pairs, ticker_interval, timerange=timerange) - - for pair in pairs: - pairdata = load_tickerdata_file(datadir, pair, ticker_interval, timerange=timerange) - if pairdata: - if timerange.starttype == 'date' and pairdata[0][0] > timerange.startts * 1000: - logger.warning('Missing data at start for pair %s, data starts at %s', - pair, - arrow.get(pairdata[0][0] // 1000).strftime('%Y-%m-%d %H:%M:%S')) - if timerange.stoptype == 'date' and pairdata[-1][0] < timerange.stopts * 1000: - logger.warning('Missing data at end for pair %s, data ends at %s', - pair, - arrow.get(pairdata[-1][0] // 1000).strftime('%Y-%m-%d %H:%M:%S')) - result[pair] = pairdata - else: - logger.warning( - 'No data for pair: "%s", Interval: %s. ' - 'Use --refresh-pairs-cached to download the data', - pair, - ticker_interval - ) - - return result - - -def make_testdata_path(datadir: str) -> str: - """Return the path where testdata files are stored""" - return datadir or os.path.abspath( - os.path.join( - os.path.dirname(__file__), '..', 'tests', 'testdata' - ) - ) - - -def download_pairs(datadir, exchange: Exchange, pairs: List[str], - ticker_interval: str, - timerange: TimeRange = TimeRange(None, None, 0, 0)) -> bool: - """For each pairs passed in parameters, download the ticker intervals""" - for pair in pairs: - try: - download_backtesting_testdata(datadir, - exchange=exchange, - pair=pair, - tick_interval=ticker_interval, - timerange=timerange) - except BaseException: - logger.info( - 'Failed to download the pair: "%s", Interval: %s', - pair, - ticker_interval - ) - return False - return True - - -def load_cached_data_for_updating(filename: str, - tick_interval: str, - timerange: Optional[TimeRange]) -> Tuple[ - List[Any], - Optional[int]]: - """ - Load cached data and choose what part of the data should be updated - """ - - since_ms = None - - # user sets timerange, so find the start time - if timerange: - if timerange.starttype == 'date': - since_ms = timerange.startts * 1000 - elif timerange.stoptype == 'line': - num_minutes = timerange.stopts * constants.TICKER_INTERVAL_MINUTES[tick_interval] - since_ms = arrow.utcnow().shift(minutes=num_minutes).timestamp * 1000 - - # read the cached file - if os.path.isfile(filename): - with open(filename, "rt") as file: - data = json_load(file) - # remove the last item, because we are not sure if it is correct - # it could be fetched when the candle was incompleted - if data: - data.pop() - else: - data = [] - - if data: - if since_ms and since_ms < data[0][0]: - # the data is requested for earlier period than the cache has - # so fully redownload all the data - data = [] - else: - # a part of the data was already downloaded, so - # download unexist data only - since_ms = data[-1][0] + 1 - - return (data, since_ms) - - -def download_backtesting_testdata(datadir: str, - exchange: Exchange, - pair: str, - tick_interval: str = '5m', - timerange: Optional[TimeRange] = None) -> None: - - """ - Download the latest ticker intervals from the exchange for the pair passed in parameters - The data is downloaded starting from the last correct ticker interval data that - exists in a cache. If timerange starts earlier than the data in the cache, - the full data will be redownloaded - - Based on @Rybolov work: https://github.com/rybolov/freqtrade-data - :param pair: pair to download - :param tick_interval: ticker interval - :param timerange: range of time to download - :return: None - - """ - path = make_testdata_path(datadir) - filepair = pair.replace("/", "_") - filename = os.path.join(path, f'{filepair}-{tick_interval}.json') - - logger.info( - 'Download the pair: "%s", Interval: %s', - pair, - tick_interval - ) - - data, since_ms = load_cached_data_for_updating(filename, tick_interval, timerange) - - logger.debug("Current Start: %s", misc.format_ms_time(data[1][0]) if data else 'None') - logger.debug("Current End: %s", misc.format_ms_time(data[-1][0]) if data else 'None') - - # Default since_ms to 30 days if nothing is given - new_data = exchange.get_history(pair=pair, tick_interval=tick_interval, - since_ms=since_ms if since_ms - else - int(arrow.utcnow().shift(days=-30).float_timestamp) * 1000) - data.extend(new_data) - - logger.debug("New Start: %s", misc.format_ms_time(data[0][0])) - logger.debug("New End: %s", misc.format_ms_time(data[-1][0])) - - misc.file_dump_json(filename, data) From 0250a96febd35ad57cca268910b313b3c095528e Mon Sep 17 00:00:00 2001 From: Matthias Date: Thu, 13 Dec 2018 06:26:04 +0100 Subject: [PATCH 626/699] Sort imports --- freqtrade/tests/edge/test_edge.py | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/freqtrade/tests/edge/test_edge.py b/freqtrade/tests/edge/test_edge.py index 008413ff1..a855e275b 100644 --- a/freqtrade/tests/edge/test_edge.py +++ b/freqtrade/tests/edge/test_edge.py @@ -1,19 +1,21 @@ # pragma pylint: disable=missing-docstring, C0103, C0330 # pragma pylint: disable=protected-access, too-many-lines, invalid-name, too-many-arguments -import pytest import logging -from freqtrade.tests.conftest import get_patched_freqtradebot -from freqtrade.edge import Edge, PairInfo -from pandas import DataFrame, to_datetime -from freqtrade.strategy.interface import SellType -from freqtrade.tests.optimize import (BTrade, BTContainer, _build_backtest_dataframe, - _get_frame_time_from_offset) +import math +from unittest.mock import MagicMock + import arrow import numpy as np -import math +import pytest +from pandas import DataFrame, to_datetime -from unittest.mock import MagicMock +from freqtrade.edge import Edge, PairInfo +from freqtrade.strategy.interface import SellType +from freqtrade.tests.conftest import get_patched_freqtradebot +from freqtrade.tests.optimize import (BTContainer, BTrade, + _build_backtest_dataframe, + _get_frame_time_from_offset) # Cases to be tested: # 1) Open trade should be removed from the end From 432cc0028327c6e28a6e22e0369a425ed9dec14b Mon Sep 17 00:00:00 2001 From: Matthias Date: Thu, 13 Dec 2018 06:34:10 +0100 Subject: [PATCH 627/699] Adjust imports to data.history --- freqtrade/edge/__init__.py | 7 ++++--- freqtrade/optimize/backtesting.py | 3 ++- freqtrade/optimize/hyperopt.py | 3 ++- 3 files changed, 8 insertions(+), 5 deletions(-) diff --git a/freqtrade/edge/__init__.py b/freqtrade/edge/__init__.py index 9bd43e37f..589ac6fe0 100644 --- a/freqtrade/edge/__init__.py +++ b/freqtrade/edge/__init__.py @@ -8,10 +8,11 @@ import numpy as np import utils_find_1st as utf1st from pandas import DataFrame -import freqtrade.optimize as optimize from freqtrade import constants, OperationalException from freqtrade.arguments import Arguments from freqtrade.arguments import TimeRange +from freqtrade.data import history +from freqtrade.optimize import get_timeframe from freqtrade.strategy.interface import SellType @@ -47,7 +48,7 @@ class Edge(): self.strategy = strategy self.ticker_interval = self.strategy.ticker_interval self.tickerdata_to_dataframe = self.strategy.tickerdata_to_dataframe - self.get_timeframe = optimize.get_timeframe + self.get_timeframe = get_timeframe self.advise_sell = self.strategy.advise_sell self.advise_buy = self.strategy.advise_buy @@ -97,7 +98,7 @@ class Edge(): logger.info('Using stake_currency: %s ...', self.config['stake_currency']) logger.info('Using local backtesting data (using whitelist in given config) ...') - data = optimize.load_data( + data = history.load_data( self.config['datadir'], pairs=pairs, ticker_interval=self.ticker_interval, diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index 61a2c3e9d..0d025e696 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -18,6 +18,7 @@ from freqtrade import DependencyException, constants from freqtrade.arguments import Arguments from freqtrade.configuration import Configuration from freqtrade.exchange import Exchange +from freqtrade.data import history from freqtrade.misc import file_dump_json from freqtrade.persistence import Trade from freqtrade.resolvers import StrategyResolver @@ -368,7 +369,7 @@ class Backtesting(object): timerange = Arguments.parse_timerange(None if self.config.get( 'timerange') is None else str(self.config.get('timerange'))) - data = optimize.load_data( + data = history.load_data( self.config['datadir'], pairs=pairs, ticker_interval=self.ticker_interval, diff --git a/freqtrade/optimize/hyperopt.py b/freqtrade/optimize/hyperopt.py index c879fabe5..2ac55dd9b 100644 --- a/freqtrade/optimize/hyperopt.py +++ b/freqtrade/optimize/hyperopt.py @@ -20,7 +20,8 @@ from skopt.space import Dimension from freqtrade.arguments import Arguments from freqtrade.configuration import Configuration -from freqtrade.optimize import load_data, get_timeframe +from freqtrade.data.history import load_data +from freqtrade.optimize import get_timeframe from freqtrade.optimize.backtesting import Backtesting from freqtrade.resolvers import HyperOptResolver From 4ca6aad99ad023ccab67df0a0d126e0d69fab7f6 Mon Sep 17 00:00:00 2001 From: Matthias Date: Thu, 13 Dec 2018 06:34:37 +0100 Subject: [PATCH 628/699] Adjust imports in scripts --- scripts/download_backtest_data.py | 2 +- scripts/plot_dataframe.py | 4 ++-- scripts/plot_profit.py | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/scripts/download_backtest_data.py b/scripts/download_backtest_data.py index b83eb2c4b..ebfef05d5 100755 --- a/scripts/download_backtest_data.py +++ b/scripts/download_backtest_data.py @@ -9,7 +9,7 @@ import arrow from freqtrade import arguments from freqtrade.arguments import TimeRange from freqtrade.exchange import Exchange -from freqtrade.optimize import download_backtesting_testdata +from freqtrade.data.history import download_backtesting_testdata from freqtrade.configuration import set_loggers import logging diff --git a/scripts/plot_dataframe.py b/scripts/plot_dataframe.py index 5a8a25309..a22735c6a 100755 --- a/scripts/plot_dataframe.py +++ b/scripts/plot_dataframe.py @@ -38,9 +38,9 @@ import pytz from plotly import tools from plotly.offline import plot -import freqtrade.optimize as optimize from freqtrade import persistence from freqtrade.arguments import Arguments, TimeRange +from freqtrade.data import history from freqtrade.exchange import Exchange from freqtrade.optimize.backtesting import setup_configuration from freqtrade.persistence import Trade @@ -141,7 +141,7 @@ def plot_analyzed_dataframe(args: Namespace) -> None: exchange.refresh_tickers([pair], tick_interval) tickers[pair] = exchange.klines(pair) else: - tickers = optimize.load_data( + tickers = history.load_data( datadir=_CONF.get("datadir"), pairs=[pair], ticker_interval=tick_interval, diff --git a/scripts/plot_profit.py b/scripts/plot_profit.py index 53f14ca3c..5e84943a5 100755 --- a/scripts/plot_profit.py +++ b/scripts/plot_profit.py @@ -27,8 +27,8 @@ import plotly.graph_objs as go from freqtrade.arguments import Arguments from freqtrade.configuration import Configuration from freqtrade import constants +from freqtrade.data as history from freqtrade.resolvers import StrategyResolver -import freqtrade.optimize as optimize import freqtrade.misc as misc @@ -120,7 +120,7 @@ def plot_profit(args: Namespace) -> None: pairs = list(set(pairs) & set(filter_pairs)) logger.info('Filter, keep pairs %s' % pairs) - tickers = optimize.load_data( + tickers = history.load_data( datadir=config.get('datadir'), pairs=pairs, ticker_interval=tick_interval, From 92c800d925f8cf15e8e444c1ee04e26b12235ac5 Mon Sep 17 00:00:00 2001 From: Matthias Date: Thu, 13 Dec 2018 06:35:28 +0100 Subject: [PATCH 629/699] Adjust tests to data.history --- freqtrade/tests/edge/test_edge.py | 2 +- freqtrade/tests/optimize/test_backtesting.py | 29 ++++++++++---------- freqtrade/tests/optimize/test_hyperopt.py | 2 +- freqtrade/tests/strategy/test_interface.py | 2 +- freqtrade/tests/test_misc.py | 2 +- 5 files changed, 19 insertions(+), 18 deletions(-) diff --git a/freqtrade/tests/edge/test_edge.py b/freqtrade/tests/edge/test_edge.py index a855e275b..1f4673fe7 100644 --- a/freqtrade/tests/edge/test_edge.py +++ b/freqtrade/tests/edge/test_edge.py @@ -288,7 +288,7 @@ def test_edge_process_downloaded_data(mocker, edge_conf): edge_conf['datadir'] = None freqtrade = get_patched_freqtradebot(mocker, edge_conf) mocker.patch('freqtrade.exchange.Exchange.get_fee', MagicMock(return_value=0.001)) - mocker.patch('freqtrade.optimize.load_data', mocked_load_data) + mocker.patch('freqtrade.data.history.load_data', mocked_load_data) edge = Edge(edge_conf, freqtrade.exchange, freqtrade.strategy) assert edge.calculate() diff --git a/freqtrade/tests/optimize/test_backtesting.py b/freqtrade/tests/optimize/test_backtesting.py index 4f80d618f..fd1c957eb 100644 --- a/freqtrade/tests/optimize/test_backtesting.py +++ b/freqtrade/tests/optimize/test_backtesting.py @@ -11,14 +11,15 @@ import pandas as pd import pytest from arrow import Arrow -from freqtrade import DependencyException, constants, optimize +from freqtrade import DependencyException, constants from freqtrade.arguments import Arguments, TimeRange +from freqtrade.data import history from freqtrade.optimize import get_timeframe from freqtrade.optimize.backtesting import (Backtesting, setup_configuration, start) -from freqtrade.tests.conftest import log_has, patch_exchange -from freqtrade.strategy.interface import SellType from freqtrade.strategy.default_strategy import DefaultStrategy +from freqtrade.strategy.interface import SellType +from freqtrade.tests.conftest import log_has, patch_exchange def get_args(args) -> List[str]: @@ -34,8 +35,8 @@ def trim_dictlist(dict_list, num): def load_data_test(what): timerange = TimeRange(None, 'line', 0, -101) - data = optimize.load_data(None, ticker_interval='1m', - pairs=['UNITTEST/BTC'], timerange=timerange) + data = history.load_data(None, ticker_interval='1m', + pairs=['UNITTEST/BTC'], timerange=timerange) pair = data['UNITTEST/BTC'] datalen = len(pair) # Depending on the what parameter we now adjust the @@ -110,21 +111,21 @@ def simple_backtest(config, contour, num_results, mocker) -> None: def mocked_load_data(datadir, pairs=[], ticker_interval='0m', refresh_pairs=False, timerange=None, exchange=None): - tickerdata = optimize.load_tickerdata_file(datadir, 'UNITTEST/BTC', '1m', timerange=timerange) + tickerdata = history.load_tickerdata_file(datadir, 'UNITTEST/BTC', '1m', timerange=timerange) pairdata = {'UNITTEST/BTC': tickerdata} return pairdata # use for mock ccxt.fetch_ohlvc' def _load_pair_as_ticks(pair, tickfreq): - ticks = optimize.load_data(None, ticker_interval=tickfreq, pairs=[pair]) + ticks = history.load_data(None, ticker_interval=tickfreq, pairs=[pair]) ticks = trim_dictlist(ticks, -201) return ticks[pair] # FIX: fixturize this? def _make_backtest_conf(mocker, conf=None, pair='UNITTEST/BTC', record=None): - data = optimize.load_data(None, ticker_interval='1m', pairs=[pair]) + data = history.load_data(None, ticker_interval='1m', pairs=[pair]) data = trim_dictlist(data, -201) patch_exchange(mocker) backtesting = Backtesting(conf) @@ -332,7 +333,7 @@ def test_backtesting_init(mocker, default_conf) -> None: def test_tickerdata_to_dataframe(default_conf, mocker) -> None: patch_exchange(mocker) timerange = TimeRange(None, 'line', 0, -100) - tick = optimize.load_tickerdata_file(None, 'UNITTEST/BTC', '1m', timerange=timerange) + tick = history.load_tickerdata_file(None, 'UNITTEST/BTC', '1m', timerange=timerange) tickerlist = {'UNITTEST/BTC': tick} backtesting = Backtesting(default_conf) @@ -447,7 +448,7 @@ def test_backtesting_start(default_conf, mocker, caplog) -> None: def get_timeframe(input1): return Arrow(2017, 11, 14, 21, 17), Arrow(2017, 11, 14, 22, 59) - mocker.patch('freqtrade.optimize.load_data', mocked_load_data) + mocker.patch('freqtrade.data.history.load_data', mocked_load_data) mocker.patch('freqtrade.optimize.get_timeframe', get_timeframe) mocker.patch('freqtrade.exchange.Exchange.refresh_tickers', MagicMock()) patch_exchange(mocker) @@ -482,7 +483,7 @@ def test_backtesting_start_no_data(default_conf, mocker, caplog) -> None: def get_timeframe(input1): return Arrow(2017, 11, 14, 21, 17), Arrow(2017, 11, 14, 22, 59) - mocker.patch('freqtrade.optimize.load_data', MagicMock(return_value={})) + mocker.patch('freqtrade.data.history.load_data', MagicMock(return_value={})) mocker.patch('freqtrade.optimize.get_timeframe', get_timeframe) mocker.patch('freqtrade.exchange.Exchange.refresh_tickers', MagicMock()) patch_exchange(mocker) @@ -511,7 +512,7 @@ def test_backtest(default_conf, fee, mocker) -> None: patch_exchange(mocker) backtesting = Backtesting(default_conf) pair = 'UNITTEST/BTC' - data = optimize.load_data(None, ticker_interval='5m', pairs=['UNITTEST/BTC']) + data = history.load_data(None, ticker_interval='5m', pairs=['UNITTEST/BTC']) data = trim_dictlist(data, -200) data_processed = backtesting.strategy.tickerdata_to_dataframe(data) min_date, max_date = get_timeframe(data_processed) @@ -564,7 +565,7 @@ def test_backtest_1min_ticker_interval(default_conf, fee, mocker) -> None: backtesting = Backtesting(default_conf) # Run a backtesting for an exiting 5min ticker_interval - data = optimize.load_data(None, ticker_interval='1m', pairs=['UNITTEST/BTC']) + data = history.load_data(None, ticker_interval='1m', pairs=['UNITTEST/BTC']) data = trim_dictlist(data, -200) processed = backtesting.strategy.tickerdata_to_dataframe(data) min_date, max_date = get_timeframe(processed) @@ -688,7 +689,7 @@ def test_backtest_multi_pair(default_conf, fee, mocker): mocker.patch('freqtrade.exchange.Exchange.get_fee', fee) patch_exchange(mocker) pairs = ['ADA/BTC', 'DASH/BTC', 'ETH/BTC', 'LTC/BTC', 'NXT/BTC'] - data = optimize.load_data(None, ticker_interval='5m', pairs=pairs) + data = history.load_data(None, ticker_interval='5m', pairs=pairs) data = trim_dictlist(data, -500) # We need to enable sell-signal - otherwise it sells on ROI!! default_conf['experimental'] = {"use_sell_signal": True} diff --git a/freqtrade/tests/optimize/test_hyperopt.py b/freqtrade/tests/optimize/test_hyperopt.py index 9ee51434c..c41c2d3c0 100644 --- a/freqtrade/tests/optimize/test_hyperopt.py +++ b/freqtrade/tests/optimize/test_hyperopt.py @@ -6,7 +6,7 @@ from unittest.mock import MagicMock import pandas as pd import pytest -from freqtrade.optimize import load_tickerdata_file +from freqtrade.data.history import load_tickerdata_file from freqtrade.optimize.hyperopt import Hyperopt, start from freqtrade.resolvers import StrategyResolver from freqtrade.tests.conftest import log_has, patch_exchange diff --git a/freqtrade/tests/strategy/test_interface.py b/freqtrade/tests/strategy/test_interface.py index d42296462..e3e5f6224 100644 --- a/freqtrade/tests/strategy/test_interface.py +++ b/freqtrade/tests/strategy/test_interface.py @@ -7,7 +7,7 @@ import arrow from pandas import DataFrame from freqtrade.arguments import TimeRange -from freqtrade.optimize.__init__ import load_tickerdata_file +from freqtrade.data.history import load_tickerdata_file from freqtrade.persistence import Trade from freqtrade.tests.conftest import get_patched_exchange, log_has from freqtrade.strategy.default_strategy import DefaultStrategy diff --git a/freqtrade/tests/test_misc.py b/freqtrade/tests/test_misc.py index bec18f8b3..d8a2fe697 100644 --- a/freqtrade/tests/test_misc.py +++ b/freqtrade/tests/test_misc.py @@ -6,7 +6,7 @@ from unittest.mock import MagicMock from freqtrade.data.converter import parse_ticker_dataframe from freqtrade.misc import (common_datearray, datesarray_to_datetimearray, file_dump_json, format_ms_time, shorten_date) -from freqtrade.optimize.__init__ import load_tickerdata_file +from freqtrade.data.history import load_tickerdata_file from freqtrade.strategy.default_strategy import DefaultStrategy From 17a820e5c0fecdc0f483fb5b2fab38c4b3f829c2 Mon Sep 17 00:00:00 2001 From: Matthias Date: Thu, 13 Dec 2018 06:35:42 +0100 Subject: [PATCH 630/699] Move tests from test_optimize to test_history --- freqtrade/tests/data/test_history.py | 472 +++++++++++++++++++++ freqtrade/tests/optimize/test_optimize.py | 476 +--------------------- 2 files changed, 477 insertions(+), 471 deletions(-) create mode 100644 freqtrade/tests/data/test_history.py diff --git a/freqtrade/tests/data/test_history.py b/freqtrade/tests/data/test_history.py new file mode 100644 index 000000000..585e7e2a7 --- /dev/null +++ b/freqtrade/tests/data/test_history.py @@ -0,0 +1,472 @@ +# pragma pylint: disable=missing-docstring, protected-access, C0103 + +import json +import os +import uuid +from shutil import copyfile + +import arrow + +from freqtrade.arguments import TimeRange +from freqtrade.data import history +from freqtrade.data.history import (download_backtesting_testdata, + download_pairs, + load_cached_data_for_updating, + load_tickerdata_file, + make_testdata_path, + trim_tickerlist) +from freqtrade.misc import file_dump_json +from freqtrade.tests.conftest import get_patched_exchange, log_has + +# Change this if modifying UNITTEST/BTC testdatafile +_BTC_UNITTEST_LENGTH = 13681 + + +def _backup_file(file: str, copy_file: bool = False) -> None: + """ + Backup existing file to avoid deleting the user file + :param file: complete path to the file + :param touch_file: create an empty file in replacement + :return: None + """ + file_swp = file + '.swp' + if os.path.isfile(file): + os.rename(file, file_swp) + + if copy_file: + copyfile(file_swp, file) + + +def _clean_test_file(file: str) -> None: + """ + Backup existing file to avoid deleting the user file + :param file: complete path to the file + :return: None + """ + file_swp = file + '.swp' + # 1. Delete file from the test + if os.path.isfile(file): + os.remove(file) + + # 2. Rollback to the initial file + if os.path.isfile(file_swp): + os.rename(file_swp, file) + + +def test_load_data_30min_ticker(ticker_history, mocker, caplog, default_conf) -> None: + mocker.patch('freqtrade.exchange.Exchange.get_history', return_value=ticker_history) + file = os.path.join(os.path.dirname(__file__), '..', 'testdata', 'UNITTEST_BTC-30m.json') + _backup_file(file, copy_file=True) + ld = history.load_data(None, pairs=['UNITTEST/BTC'], ticker_interval='30m') + assert isinstance(ld, dict) + assert os.path.isfile(file) is True + assert not log_has('Download the pair: "UNITTEST/BTC", Interval: 30m', caplog.record_tuples) + _clean_test_file(file) + + +def test_load_data_5min_ticker(ticker_history, mocker, caplog, default_conf) -> None: + mocker.patch('freqtrade.exchange.Exchange.get_history', return_value=ticker_history) + + file = os.path.join(os.path.dirname(__file__), '..', 'testdata', 'UNITTEST_BTC-5m.json') + _backup_file(file, copy_file=True) + history.load_data(None, pairs=['UNITTEST/BTC'], ticker_interval='5m') + assert os.path.isfile(file) is True + assert not log_has('Download the pair: "UNITTEST/BTC", Interval: 5m', caplog.record_tuples) + _clean_test_file(file) + + +def test_load_data_1min_ticker(ticker_history, mocker, caplog) -> None: + mocker.patch('freqtrade.exchange.Exchange.get_history', return_value=ticker_history) + file = os.path.join(os.path.dirname(__file__), '..', 'testdata', 'UNITTEST_BTC-1m.json') + _backup_file(file, copy_file=True) + history.load_data(None, ticker_interval='1m', pairs=['UNITTEST/BTC']) + assert os.path.isfile(file) is True + assert not log_has('Download the pair: "UNITTEST/BTC", Interval: 1m', caplog.record_tuples) + _clean_test_file(file) + + +def test_load_data_with_new_pair_1min(ticker_history_list, mocker, caplog, default_conf) -> None: + """ + Test load_data() with 1 min ticker + """ + mocker.patch('freqtrade.exchange.Exchange.get_history', return_value=ticker_history_list) + exchange = get_patched_exchange(mocker, default_conf) + file = os.path.join(os.path.dirname(__file__), '..', 'testdata', 'MEME_BTC-1m.json') + + _backup_file(file) + # do not download a new pair if refresh_pairs isn't set + history.load_data(None, + ticker_interval='1m', + refresh_pairs=False, + pairs=['MEME/BTC']) + assert os.path.isfile(file) is False + assert log_has('No data for pair: "MEME/BTC", Interval: 1m. ' + 'Use --refresh-pairs-cached to download the data', + caplog.record_tuples) + + # download a new pair if refresh_pairs is set + history.load_data(None, + ticker_interval='1m', + refresh_pairs=True, + exchange=exchange, + pairs=['MEME/BTC']) + assert os.path.isfile(file) is True + assert log_has('Download the pair: "MEME/BTC", Interval: 1m', caplog.record_tuples) + _clean_test_file(file) + + +def test_testdata_path() -> None: + assert os.path.join('freqtrade', 'tests', 'testdata') in make_testdata_path(None) + + +def test_download_pairs(ticker_history_list, mocker, default_conf) -> None: + mocker.patch('freqtrade.exchange.Exchange.get_history', return_value=ticker_history_list) + exchange = get_patched_exchange(mocker, default_conf) + file1_1 = os.path.join(os.path.dirname(__file__), '..', 'testdata', 'MEME_BTC-1m.json') + file1_5 = os.path.join(os.path.dirname(__file__), '..', 'testdata', 'MEME_BTC-5m.json') + file2_1 = os.path.join(os.path.dirname(__file__), '..', 'testdata', 'CFI_BTC-1m.json') + file2_5 = os.path.join(os.path.dirname(__file__), '..', 'testdata', 'CFI_BTC-5m.json') + + _backup_file(file1_1) + _backup_file(file1_5) + _backup_file(file2_1) + _backup_file(file2_5) + + assert os.path.isfile(file1_1) is False + assert os.path.isfile(file2_1) is False + + assert download_pairs(None, exchange, + pairs=['MEME/BTC', 'CFI/BTC'], ticker_interval='1m') is True + + assert os.path.isfile(file1_1) is True + assert os.path.isfile(file2_1) is True + + # clean files freshly downloaded + _clean_test_file(file1_1) + _clean_test_file(file2_1) + + assert os.path.isfile(file1_5) is False + assert os.path.isfile(file2_5) is False + + assert download_pairs(None, exchange, + pairs=['MEME/BTC', 'CFI/BTC'], ticker_interval='5m') is True + + assert os.path.isfile(file1_5) is True + assert os.path.isfile(file2_5) is True + + # clean files freshly downloaded + _clean_test_file(file1_5) + _clean_test_file(file2_5) + + +def test_load_cached_data_for_updating(mocker) -> None: + datadir = os.path.join(os.path.dirname(__file__), '..', 'testdata') + + test_data = None + test_filename = os.path.join(datadir, 'UNITTEST_BTC-1m.json') + with open(test_filename, "rt") as file: + test_data = json.load(file) + + # change now time to test 'line' cases + # now = last cached item + 1 hour + now_ts = test_data[-1][0] / 1000 + 60 * 60 + mocker.patch('arrow.utcnow', return_value=arrow.get(now_ts)) + + # timeframe starts earlier than the cached data + # should fully update data + timerange = TimeRange('date', None, test_data[0][0] / 1000 - 1, 0) + data, start_ts = load_cached_data_for_updating(test_filename, + '1m', + timerange) + assert data == [] + assert start_ts == test_data[0][0] - 1000 + + # same with 'line' timeframe + num_lines = (test_data[-1][0] - test_data[1][0]) / 1000 / 60 + 120 + data, start_ts = load_cached_data_for_updating(test_filename, + '1m', + TimeRange(None, 'line', 0, -num_lines)) + assert data == [] + assert start_ts < test_data[0][0] - 1 + + # timeframe starts in the center of the cached data + # should return the chached data w/o the last item + timerange = TimeRange('date', None, test_data[0][0] / 1000 + 1, 0) + data, start_ts = load_cached_data_for_updating(test_filename, + '1m', + timerange) + assert data == test_data[:-1] + assert test_data[-2][0] < start_ts < test_data[-1][0] + + # same with 'line' timeframe + num_lines = (test_data[-1][0] - test_data[1][0]) / 1000 / 60 + 30 + timerange = TimeRange(None, 'line', 0, -num_lines) + data, start_ts = load_cached_data_for_updating(test_filename, + '1m', + timerange) + assert data == test_data[:-1] + assert test_data[-2][0] < start_ts < test_data[-1][0] + + # timeframe starts after the chached data + # should return the chached data w/o the last item + timerange = TimeRange('date', None, test_data[-1][0] / 1000 + 1, 0) + data, start_ts = load_cached_data_for_updating(test_filename, + '1m', + timerange) + assert data == test_data[:-1] + assert test_data[-2][0] < start_ts < test_data[-1][0] + + # same with 'line' timeframe + num_lines = 30 + timerange = TimeRange(None, 'line', 0, -num_lines) + data, start_ts = load_cached_data_for_updating(test_filename, + '1m', + timerange) + assert data == test_data[:-1] + assert test_data[-2][0] < start_ts < test_data[-1][0] + + # no timeframe is set + # should return the chached data w/o the last item + num_lines = 30 + timerange = TimeRange(None, 'line', 0, -num_lines) + data, start_ts = load_cached_data_for_updating(test_filename, + '1m', + timerange) + assert data == test_data[:-1] + assert test_data[-2][0] < start_ts < test_data[-1][0] + + # no datafile exist + # should return timestamp start time + timerange = TimeRange('date', None, now_ts - 10000, 0) + data, start_ts = load_cached_data_for_updating(test_filename + 'unexist', + '1m', + timerange) + assert data == [] + assert start_ts == (now_ts - 10000) * 1000 + + # same with 'line' timeframe + num_lines = 30 + timerange = TimeRange(None, 'line', 0, -num_lines) + data, start_ts = load_cached_data_for_updating(test_filename + 'unexist', + '1m', + timerange) + assert data == [] + assert start_ts == (now_ts - num_lines * 60) * 1000 + + # no datafile exist, no timeframe is set + # should return an empty array and None + data, start_ts = load_cached_data_for_updating(test_filename + 'unexist', + '1m', + None) + assert data == [] + assert start_ts is None + + +def test_download_pairs_exception(ticker_history, mocker, caplog, default_conf) -> None: + mocker.patch('freqtrade.exchange.Exchange.get_history', return_value=ticker_history) + mocker.patch('freqtrade.data.history.download_backtesting_testdata', + side_effect=BaseException('File Error')) + exchange = get_patched_exchange(mocker, default_conf) + + file1_1 = os.path.join(os.path.dirname(__file__), '..', 'testdata', 'MEME_BTC-1m.json') + file1_5 = os.path.join(os.path.dirname(__file__), '..', 'testdata', 'MEME_BTC-5m.json') + _backup_file(file1_1) + _backup_file(file1_5) + + download_pairs(None, exchange, pairs=['MEME/BTC'], ticker_interval='1m') + # clean files freshly downloaded + _clean_test_file(file1_1) + _clean_test_file(file1_5) + assert log_has('Failed to download the pair: "MEME/BTC", Interval: 1m', caplog.record_tuples) + + +def test_download_backtesting_testdata(ticker_history_list, mocker, default_conf) -> None: + mocker.patch('freqtrade.exchange.Exchange.get_history', return_value=ticker_history_list) + exchange = get_patched_exchange(mocker, default_conf) + # Tst that pairs-cached is not touched. + assert not exchange._pairs_last_refresh_time + # Download a 1 min ticker file + file1 = os.path.join(os.path.dirname(__file__), '..', 'testdata', 'XEL_BTC-1m.json') + _backup_file(file1) + download_backtesting_testdata(None, exchange, pair="XEL/BTC", tick_interval='1m') + assert os.path.isfile(file1) is True + _clean_test_file(file1) + assert not exchange._pairs_last_refresh_time + + # Download a 5 min ticker file + file2 = os.path.join(os.path.dirname(__file__), '..', 'testdata', 'STORJ_BTC-5m.json') + _backup_file(file2) + + download_backtesting_testdata(None, exchange, pair="STORJ/BTC", tick_interval='5m') + assert os.path.isfile(file2) is True + _clean_test_file(file2) + assert not exchange._pairs_last_refresh_time + + +def test_download_backtesting_testdata2(mocker, default_conf) -> None: + tick = [ + [1509836520000, 0.00162008, 0.00162008, 0.00162008, 0.00162008, 108.14853839], + [1509836580000, 0.00161, 0.00161, 0.00161, 0.00161, 82.390199] + ] + json_dump_mock = mocker.patch('freqtrade.misc.file_dump_json', return_value=None) + mocker.patch('freqtrade.exchange.Exchange.get_history', return_value=tick) + exchange = get_patched_exchange(mocker, default_conf) + download_backtesting_testdata(None, exchange, pair="UNITTEST/BTC", tick_interval='1m') + download_backtesting_testdata(None, exchange, pair="UNITTEST/BTC", tick_interval='3m') + assert json_dump_mock.call_count == 2 + + +def test_load_tickerdata_file() -> None: + # 7 does not exist in either format. + assert not load_tickerdata_file(None, 'UNITTEST/BTC', '7m') + # 1 exists only as a .json + tickerdata = load_tickerdata_file(None, 'UNITTEST/BTC', '1m') + assert _BTC_UNITTEST_LENGTH == len(tickerdata) + # 8 .json is empty and will fail if it's loaded. .json.gz is a copy of 1.json + tickerdata = load_tickerdata_file(None, 'UNITTEST/BTC', '8m') + assert _BTC_UNITTEST_LENGTH == len(tickerdata) + + +def test_load_partial_missing(caplog) -> None: + # Make sure we start fresh - test missing data at start + start = arrow.get('2018-01-01T00:00:00') + end = arrow.get('2018-01-11T00:00:00') + tickerdata = history.load_data(None, '5m', ['UNITTEST/BTC'], + refresh_pairs=False, + timerange=TimeRange('date', 'date', + start.timestamp, end.timestamp)) + # timedifference in 5 minutes + td = ((end - start).total_seconds() // 60 // 5) + 1 + assert td != len(tickerdata['UNITTEST/BTC']) + start_real = arrow.get(tickerdata['UNITTEST/BTC'][0][0] / 1000) + assert log_has(f'Missing data at start for pair ' + f'UNITTEST/BTC, data starts at {start_real.strftime("%Y-%m-%d %H:%M:%S")}', + caplog.record_tuples) + # Make sure we start fresh - test missing data at end + caplog.clear() + start = arrow.get('2018-01-10T00:00:00') + end = arrow.get('2018-02-20T00:00:00') + tickerdata = history.load_data(None, '5m', ['UNITTEST/BTC'], + refresh_pairs=False, + timerange=TimeRange('date', 'date', + start.timestamp, end.timestamp)) + # timedifference in 5 minutes + td = ((end - start).total_seconds() // 60 // 5) + 1 + assert td != len(tickerdata['UNITTEST/BTC']) + end_real = arrow.get(tickerdata['UNITTEST/BTC'][-1][0] / 1000) + assert log_has(f'Missing data at end for pair ' + f'UNITTEST/BTC, data ends at {end_real.strftime("%Y-%m-%d %H:%M:%S")}', + caplog.record_tuples) + + +def test_init(default_conf, mocker) -> None: + exchange = get_patched_exchange(mocker, default_conf) + assert {} == history.load_data( + '', + exchange=exchange, + pairs=[], + refresh_pairs=True, + ticker_interval=default_conf['ticker_interval'] + ) + + +def test_trim_tickerlist() -> None: + file = os.path.join(os.path.dirname(__file__), '..', 'testdata', 'UNITTEST_BTC-1m.json') + with open(file) as data_file: + ticker_list = json.load(data_file) + ticker_list_len = len(ticker_list) + + # Test the pattern ^(-\d+)$ + # This pattern uses the latest N elements + timerange = TimeRange(None, 'line', 0, -5) + ticker = trim_tickerlist(ticker_list, timerange) + ticker_len = len(ticker) + + assert ticker_len == 5 + assert ticker_list[0] is not ticker[0] # The first element should be different + assert ticker_list[-1] is ticker[-1] # The last element must be the same + + # Test the pattern ^(\d+)-$ + # This pattern keep X element from the end + timerange = TimeRange('line', None, 5, 0) + ticker = trim_tickerlist(ticker_list, timerange) + ticker_len = len(ticker) + + assert ticker_len == 5 + assert ticker_list[0] is ticker[0] # The first element must be the same + assert ticker_list[-1] is not ticker[-1] # The last element should be different + + # Test the pattern ^(\d+)-(\d+)$ + # This pattern extract a window + timerange = TimeRange('index', 'index', 5, 10) + ticker = trim_tickerlist(ticker_list, timerange) + ticker_len = len(ticker) + + assert ticker_len == 5 + assert ticker_list[0] is not ticker[0] # The first element should be different + assert ticker_list[5] is ticker[0] # The list starts at the index 5 + assert ticker_list[9] is ticker[-1] # The list ends at the index 9 (5 elements) + + # Test the pattern ^(\d{8})-(\d{8})$ + # This pattern extract a window between the dates + timerange = TimeRange('date', 'date', ticker_list[5][0] / 1000, ticker_list[10][0] / 1000 - 1) + ticker = trim_tickerlist(ticker_list, timerange) + ticker_len = len(ticker) + + assert ticker_len == 5 + assert ticker_list[0] is not ticker[0] # The first element should be different + assert ticker_list[5] is ticker[0] # The list starts at the index 5 + assert ticker_list[9] is ticker[-1] # The list ends at the index 9 (5 elements) + + # Test the pattern ^-(\d{8})$ + # This pattern extracts elements from the start to the date + timerange = TimeRange(None, 'date', 0, ticker_list[10][0] / 1000 - 1) + ticker = trim_tickerlist(ticker_list, timerange) + ticker_len = len(ticker) + + assert ticker_len == 10 + assert ticker_list[0] is ticker[0] # The start of the list is included + assert ticker_list[9] is ticker[-1] # The element 10 is not included + + # Test the pattern ^(\d{8})-$ + # This pattern extracts elements from the date to now + timerange = TimeRange('date', None, ticker_list[10][0] / 1000 - 1, None) + ticker = trim_tickerlist(ticker_list, timerange) + ticker_len = len(ticker) + + assert ticker_len == ticker_list_len - 10 + assert ticker_list[10] is ticker[0] # The first element is element #10 + assert ticker_list[-1] is ticker[-1] # The last element is the same + + # Test a wrong pattern + # This pattern must return the list unchanged + timerange = TimeRange(None, None, None, 5) + ticker = trim_tickerlist(ticker_list, timerange) + ticker_len = len(ticker) + + assert ticker_list_len == ticker_len + + +def test_file_dump_json() -> None: + file = os.path.join(os.path.dirname(__file__), '..', 'testdata', + 'test_{id}.json'.format(id=str(uuid.uuid4()))) + data = {'bar': 'foo'} + + # check the file we will create does not exist + assert os.path.isfile(file) is False + + # Create the Json file + file_dump_json(file, data) + + # Check the file was create + assert os.path.isfile(file) is True + + # Open the Json file created and test the data is in it + with open(file) as data_file: + json_from_file = json.load(data_file) + + assert 'bar' in json_from_file + assert json_from_file['bar'] == 'foo' + + # Remove the file + _clean_test_file(file) diff --git a/freqtrade/tests/optimize/test_optimize.py b/freqtrade/tests/optimize/test_optimize.py index dde76332b..2019c9fde 100644 --- a/freqtrade/tests/optimize/test_optimize.py +++ b/freqtrade/tests/optimize/test_optimize.py @@ -1,475 +1,9 @@ # pragma pylint: disable=missing-docstring, protected-access, C0103 - -import json -import os -import uuid -from shutil import copyfile - -import arrow - from freqtrade import optimize, constants from freqtrade.arguments import TimeRange -from freqtrade.misc import file_dump_json -from freqtrade.optimize.__init__ import (download_backtesting_testdata, - download_pairs, - load_cached_data_for_updating, - load_tickerdata_file, - make_testdata_path, trim_tickerlist) +from freqtrade.data import history from freqtrade.strategy.default_strategy import DefaultStrategy -from freqtrade.tests.conftest import get_patched_exchange, log_has, patch_exchange - -# Change this if modifying UNITTEST/BTC testdatafile -_BTC_UNITTEST_LENGTH = 13681 - - -def _backup_file(file: str, copy_file: bool = False) -> None: - """ - Backup existing file to avoid deleting the user file - :param file: complete path to the file - :param touch_file: create an empty file in replacement - :return: None - """ - file_swp = file + '.swp' - if os.path.isfile(file): - os.rename(file, file_swp) - - if copy_file: - copyfile(file_swp, file) - - -def _clean_test_file(file: str) -> None: - """ - Backup existing file to avoid deleting the user file - :param file: complete path to the file - :return: None - """ - file_swp = file + '.swp' - # 1. Delete file from the test - if os.path.isfile(file): - os.remove(file) - - # 2. Rollback to the initial file - if os.path.isfile(file_swp): - os.rename(file_swp, file) - - -def test_load_data_30min_ticker(ticker_history, mocker, caplog, default_conf) -> None: - mocker.patch('freqtrade.exchange.Exchange.get_history', return_value=ticker_history) - file = os.path.join(os.path.dirname(__file__), '..', 'testdata', 'UNITTEST_BTC-30m.json') - _backup_file(file, copy_file=True) - ld = optimize.load_data(None, pairs=['UNITTEST/BTC'], ticker_interval='30m') - assert isinstance(ld, dict) - assert os.path.isfile(file) is True - assert not log_has('Download the pair: "UNITTEST/BTC", Interval: 30m', caplog.record_tuples) - _clean_test_file(file) - - -def test_load_data_5min_ticker(ticker_history, mocker, caplog, default_conf) -> None: - mocker.patch('freqtrade.exchange.Exchange.get_history', return_value=ticker_history) - - file = os.path.join(os.path.dirname(__file__), '..', 'testdata', 'UNITTEST_BTC-5m.json') - _backup_file(file, copy_file=True) - optimize.load_data(None, pairs=['UNITTEST/BTC'], ticker_interval='5m') - assert os.path.isfile(file) is True - assert not log_has('Download the pair: "UNITTEST/BTC", Interval: 5m', caplog.record_tuples) - _clean_test_file(file) - - -def test_load_data_1min_ticker(ticker_history, mocker, caplog) -> None: - mocker.patch('freqtrade.exchange.Exchange.get_history', return_value=ticker_history) - file = os.path.join(os.path.dirname(__file__), '..', 'testdata', 'UNITTEST_BTC-1m.json') - _backup_file(file, copy_file=True) - optimize.load_data(None, ticker_interval='1m', pairs=['UNITTEST/BTC']) - assert os.path.isfile(file) is True - assert not log_has('Download the pair: "UNITTEST/BTC", Interval: 1m', caplog.record_tuples) - _clean_test_file(file) - - -def test_load_data_with_new_pair_1min(ticker_history_list, mocker, caplog, default_conf) -> None: - """ - Test load_data() with 1 min ticker - """ - mocker.patch('freqtrade.exchange.Exchange.get_history', return_value=ticker_history_list) - exchange = get_patched_exchange(mocker, default_conf) - file = os.path.join(os.path.dirname(__file__), '..', 'testdata', 'MEME_BTC-1m.json') - - _backup_file(file) - # do not download a new pair if refresh_pairs isn't set - optimize.load_data(None, - ticker_interval='1m', - refresh_pairs=False, - pairs=['MEME/BTC']) - assert os.path.isfile(file) is False - assert log_has('No data for pair: "MEME/BTC", Interval: 1m. ' - 'Use --refresh-pairs-cached to download the data', - caplog.record_tuples) - - # download a new pair if refresh_pairs is set - optimize.load_data(None, - ticker_interval='1m', - refresh_pairs=True, - exchange=exchange, - pairs=['MEME/BTC']) - assert os.path.isfile(file) is True - assert log_has('Download the pair: "MEME/BTC", Interval: 1m', caplog.record_tuples) - _clean_test_file(file) - - -def test_testdata_path() -> None: - assert os.path.join('freqtrade', 'tests', 'testdata') in make_testdata_path(None) - - -def test_download_pairs(ticker_history_list, mocker, default_conf) -> None: - mocker.patch('freqtrade.exchange.Exchange.get_history', return_value=ticker_history_list) - exchange = get_patched_exchange(mocker, default_conf) - file1_1 = os.path.join(os.path.dirname(__file__), '..', 'testdata', 'MEME_BTC-1m.json') - file1_5 = os.path.join(os.path.dirname(__file__), '..', 'testdata', 'MEME_BTC-5m.json') - file2_1 = os.path.join(os.path.dirname(__file__), '..', 'testdata', 'CFI_BTC-1m.json') - file2_5 = os.path.join(os.path.dirname(__file__), '..', 'testdata', 'CFI_BTC-5m.json') - - _backup_file(file1_1) - _backup_file(file1_5) - _backup_file(file2_1) - _backup_file(file2_5) - - assert os.path.isfile(file1_1) is False - assert os.path.isfile(file2_1) is False - - assert download_pairs(None, exchange, - pairs=['MEME/BTC', 'CFI/BTC'], ticker_interval='1m') is True - - assert os.path.isfile(file1_1) is True - assert os.path.isfile(file2_1) is True - - # clean files freshly downloaded - _clean_test_file(file1_1) - _clean_test_file(file2_1) - - assert os.path.isfile(file1_5) is False - assert os.path.isfile(file2_5) is False - - assert download_pairs(None, exchange, - pairs=['MEME/BTC', 'CFI/BTC'], ticker_interval='5m') is True - - assert os.path.isfile(file1_5) is True - assert os.path.isfile(file2_5) is True - - # clean files freshly downloaded - _clean_test_file(file1_5) - _clean_test_file(file2_5) - - -def test_load_cached_data_for_updating(mocker) -> None: - datadir = os.path.join(os.path.dirname(__file__), '..', 'testdata') - - test_data = None - test_filename = os.path.join(datadir, 'UNITTEST_BTC-1m.json') - with open(test_filename, "rt") as file: - test_data = json.load(file) - - # change now time to test 'line' cases - # now = last cached item + 1 hour - now_ts = test_data[-1][0] / 1000 + 60 * 60 - mocker.patch('arrow.utcnow', return_value=arrow.get(now_ts)) - - # timeframe starts earlier than the cached data - # should fully update data - timerange = TimeRange('date', None, test_data[0][0] / 1000 - 1, 0) - data, start_ts = load_cached_data_for_updating(test_filename, - '1m', - timerange) - assert data == [] - assert start_ts == test_data[0][0] - 1000 - - # same with 'line' timeframe - num_lines = (test_data[-1][0] - test_data[1][0]) / 1000 / 60 + 120 - data, start_ts = load_cached_data_for_updating(test_filename, - '1m', - TimeRange(None, 'line', 0, -num_lines)) - assert data == [] - assert start_ts < test_data[0][0] - 1 - - # timeframe starts in the center of the cached data - # should return the chached data w/o the last item - timerange = TimeRange('date', None, test_data[0][0] / 1000 + 1, 0) - data, start_ts = load_cached_data_for_updating(test_filename, - '1m', - timerange) - assert data == test_data[:-1] - assert test_data[-2][0] < start_ts < test_data[-1][0] - - # same with 'line' timeframe - num_lines = (test_data[-1][0] - test_data[1][0]) / 1000 / 60 + 30 - timerange = TimeRange(None, 'line', 0, -num_lines) - data, start_ts = load_cached_data_for_updating(test_filename, - '1m', - timerange) - assert data == test_data[:-1] - assert test_data[-2][0] < start_ts < test_data[-1][0] - - # timeframe starts after the chached data - # should return the chached data w/o the last item - timerange = TimeRange('date', None, test_data[-1][0] / 1000 + 1, 0) - data, start_ts = load_cached_data_for_updating(test_filename, - '1m', - timerange) - assert data == test_data[:-1] - assert test_data[-2][0] < start_ts < test_data[-1][0] - - # same with 'line' timeframe - num_lines = 30 - timerange = TimeRange(None, 'line', 0, -num_lines) - data, start_ts = load_cached_data_for_updating(test_filename, - '1m', - timerange) - assert data == test_data[:-1] - assert test_data[-2][0] < start_ts < test_data[-1][0] - - # no timeframe is set - # should return the chached data w/o the last item - num_lines = 30 - timerange = TimeRange(None, 'line', 0, -num_lines) - data, start_ts = load_cached_data_for_updating(test_filename, - '1m', - timerange) - assert data == test_data[:-1] - assert test_data[-2][0] < start_ts < test_data[-1][0] - - # no datafile exist - # should return timestamp start time - timerange = TimeRange('date', None, now_ts - 10000, 0) - data, start_ts = load_cached_data_for_updating(test_filename + 'unexist', - '1m', - timerange) - assert data == [] - assert start_ts == (now_ts - 10000) * 1000 - - # same with 'line' timeframe - num_lines = 30 - timerange = TimeRange(None, 'line', 0, -num_lines) - data, start_ts = load_cached_data_for_updating(test_filename + 'unexist', - '1m', - timerange) - assert data == [] - assert start_ts == (now_ts - num_lines * 60) * 1000 - - # no datafile exist, no timeframe is set - # should return an empty array and None - data, start_ts = load_cached_data_for_updating(test_filename + 'unexist', - '1m', - None) - assert data == [] - assert start_ts is None - - -def test_download_pairs_exception(ticker_history, mocker, caplog, default_conf) -> None: - mocker.patch('freqtrade.exchange.Exchange.get_history', return_value=ticker_history) - mocker.patch('freqtrade.optimize.__init__.download_backtesting_testdata', - side_effect=BaseException('File Error')) - exchange = get_patched_exchange(mocker, default_conf) - - file1_1 = os.path.join(os.path.dirname(__file__), '..', 'testdata', 'MEME_BTC-1m.json') - file1_5 = os.path.join(os.path.dirname(__file__), '..', 'testdata', 'MEME_BTC-5m.json') - _backup_file(file1_1) - _backup_file(file1_5) - - download_pairs(None, exchange, pairs=['MEME/BTC'], ticker_interval='1m') - # clean files freshly downloaded - _clean_test_file(file1_1) - _clean_test_file(file1_5) - assert log_has('Failed to download the pair: "MEME/BTC", Interval: 1m', caplog.record_tuples) - - -def test_download_backtesting_testdata(ticker_history_list, mocker, default_conf) -> None: - mocker.patch('freqtrade.exchange.Exchange.get_history', return_value=ticker_history_list) - exchange = get_patched_exchange(mocker, default_conf) - # Tst that pairs-cached is not touched. - assert not exchange._pairs_last_refresh_time - # Download a 1 min ticker file - file1 = os.path.join(os.path.dirname(__file__), '..', 'testdata', 'XEL_BTC-1m.json') - _backup_file(file1) - download_backtesting_testdata(None, exchange, pair="XEL/BTC", tick_interval='1m') - assert os.path.isfile(file1) is True - _clean_test_file(file1) - assert not exchange._pairs_last_refresh_time - - # Download a 5 min ticker file - file2 = os.path.join(os.path.dirname(__file__), '..', 'testdata', 'STORJ_BTC-5m.json') - _backup_file(file2) - - download_backtesting_testdata(None, exchange, pair="STORJ/BTC", tick_interval='5m') - assert os.path.isfile(file2) is True - _clean_test_file(file2) - assert not exchange._pairs_last_refresh_time - - -def test_download_backtesting_testdata2(mocker, default_conf) -> None: - tick = [ - [1509836520000, 0.00162008, 0.00162008, 0.00162008, 0.00162008, 108.14853839], - [1509836580000, 0.00161, 0.00161, 0.00161, 0.00161, 82.390199] - ] - json_dump_mock = mocker.patch('freqtrade.misc.file_dump_json', return_value=None) - mocker.patch('freqtrade.exchange.Exchange.get_history', return_value=tick) - exchange = get_patched_exchange(mocker, default_conf) - download_backtesting_testdata(None, exchange, pair="UNITTEST/BTC", tick_interval='1m') - download_backtesting_testdata(None, exchange, pair="UNITTEST/BTC", tick_interval='3m') - assert json_dump_mock.call_count == 2 - - -def test_load_tickerdata_file() -> None: - # 7 does not exist in either format. - assert not load_tickerdata_file(None, 'UNITTEST/BTC', '7m') - # 1 exists only as a .json - tickerdata = load_tickerdata_file(None, 'UNITTEST/BTC', '1m') - assert _BTC_UNITTEST_LENGTH == len(tickerdata) - # 8 .json is empty and will fail if it's loaded. .json.gz is a copy of 1.json - tickerdata = load_tickerdata_file(None, 'UNITTEST/BTC', '8m') - assert _BTC_UNITTEST_LENGTH == len(tickerdata) - - -def test_load_partial_missing(caplog) -> None: - # Make sure we start fresh - test missing data at start - start = arrow.get('2018-01-01T00:00:00') - end = arrow.get('2018-01-11T00:00:00') - tickerdata = optimize.load_data(None, '5m', ['UNITTEST/BTC'], - refresh_pairs=False, - timerange=TimeRange('date', 'date', - start.timestamp, end.timestamp)) - # timedifference in 5 minutes - td = ((end - start).total_seconds() // 60 // 5) + 1 - assert td != len(tickerdata['UNITTEST/BTC']) - start_real = arrow.get(tickerdata['UNITTEST/BTC'][0][0] / 1000) - assert log_has(f'Missing data at start for pair ' - f'UNITTEST/BTC, data starts at {start_real.strftime("%Y-%m-%d %H:%M:%S")}', - caplog.record_tuples) - # Make sure we start fresh - test missing data at end - caplog.clear() - start = arrow.get('2018-01-10T00:00:00') - end = arrow.get('2018-02-20T00:00:00') - tickerdata = optimize.load_data(None, '5m', ['UNITTEST/BTC'], - refresh_pairs=False, - timerange=TimeRange('date', 'date', - start.timestamp, end.timestamp)) - # timedifference in 5 minutes - td = ((end - start).total_seconds() // 60 // 5) + 1 - assert td != len(tickerdata['UNITTEST/BTC']) - end_real = arrow.get(tickerdata['UNITTEST/BTC'][-1][0] / 1000) - assert log_has(f'Missing data at end for pair ' - f'UNITTEST/BTC, data ends at {end_real.strftime("%Y-%m-%d %H:%M:%S")}', - caplog.record_tuples) - - -def test_init(default_conf, mocker) -> None: - exchange = get_patched_exchange(mocker, default_conf) - assert {} == optimize.load_data( - '', - exchange=exchange, - pairs=[], - refresh_pairs=True, - ticker_interval=default_conf['ticker_interval'] - ) - - -def test_trim_tickerlist() -> None: - file = os.path.join(os.path.dirname(__file__), '..', 'testdata', 'UNITTEST_BTC-1m.json') - with open(file) as data_file: - ticker_list = json.load(data_file) - ticker_list_len = len(ticker_list) - - # Test the pattern ^(-\d+)$ - # This pattern uses the latest N elements - timerange = TimeRange(None, 'line', 0, -5) - ticker = trim_tickerlist(ticker_list, timerange) - ticker_len = len(ticker) - - assert ticker_len == 5 - assert ticker_list[0] is not ticker[0] # The first element should be different - assert ticker_list[-1] is ticker[-1] # The last element must be the same - - # Test the pattern ^(\d+)-$ - # This pattern keep X element from the end - timerange = TimeRange('line', None, 5, 0) - ticker = trim_tickerlist(ticker_list, timerange) - ticker_len = len(ticker) - - assert ticker_len == 5 - assert ticker_list[0] is ticker[0] # The first element must be the same - assert ticker_list[-1] is not ticker[-1] # The last element should be different - - # Test the pattern ^(\d+)-(\d+)$ - # This pattern extract a window - timerange = TimeRange('index', 'index', 5, 10) - ticker = trim_tickerlist(ticker_list, timerange) - ticker_len = len(ticker) - - assert ticker_len == 5 - assert ticker_list[0] is not ticker[0] # The first element should be different - assert ticker_list[5] is ticker[0] # The list starts at the index 5 - assert ticker_list[9] is ticker[-1] # The list ends at the index 9 (5 elements) - - # Test the pattern ^(\d{8})-(\d{8})$ - # This pattern extract a window between the dates - timerange = TimeRange('date', 'date', ticker_list[5][0] / 1000, ticker_list[10][0] / 1000 - 1) - ticker = trim_tickerlist(ticker_list, timerange) - ticker_len = len(ticker) - - assert ticker_len == 5 - assert ticker_list[0] is not ticker[0] # The first element should be different - assert ticker_list[5] is ticker[0] # The list starts at the index 5 - assert ticker_list[9] is ticker[-1] # The list ends at the index 9 (5 elements) - - # Test the pattern ^-(\d{8})$ - # This pattern extracts elements from the start to the date - timerange = TimeRange(None, 'date', 0, ticker_list[10][0] / 1000 - 1) - ticker = trim_tickerlist(ticker_list, timerange) - ticker_len = len(ticker) - - assert ticker_len == 10 - assert ticker_list[0] is ticker[0] # The start of the list is included - assert ticker_list[9] is ticker[-1] # The element 10 is not included - - # Test the pattern ^(\d{8})-$ - # This pattern extracts elements from the date to now - timerange = TimeRange('date', None, ticker_list[10][0] / 1000 - 1, None) - ticker = trim_tickerlist(ticker_list, timerange) - ticker_len = len(ticker) - - assert ticker_len == ticker_list_len - 10 - assert ticker_list[10] is ticker[0] # The first element is element #10 - assert ticker_list[-1] is ticker[-1] # The last element is the same - - # Test a wrong pattern - # This pattern must return the list unchanged - timerange = TimeRange(None, None, None, 5) - ticker = trim_tickerlist(ticker_list, timerange) - ticker_len = len(ticker) - - assert ticker_list_len == ticker_len - - -def test_file_dump_json() -> None: - file = os.path.join(os.path.dirname(__file__), '..', 'testdata', - 'test_{id}.json'.format(id=str(uuid.uuid4()))) - data = {'bar': 'foo'} - - # check the file we will create does not exist - assert os.path.isfile(file) is False - - # Create the Json file - file_dump_json(file, data) - - # Check the file was create - assert os.path.isfile(file) is True - - # Open the Json file created and test the data is in it - with open(file) as data_file: - json_from_file = json.load(data_file) - - assert 'bar' in json_from_file - assert json_from_file['bar'] == 'foo' - - # Remove the file - _clean_test_file(file) +from freqtrade.tests.conftest import log_has, patch_exchange def test_get_timeframe(default_conf, mocker) -> None: @@ -477,7 +11,7 @@ def test_get_timeframe(default_conf, mocker) -> None: strategy = DefaultStrategy(default_conf) data = strategy.tickerdata_to_dataframe( - optimize.load_data( + history.load_data( None, ticker_interval='1m', pairs=['UNITTEST/BTC'] @@ -493,7 +27,7 @@ def test_validate_backtest_data_warn(default_conf, mocker, caplog) -> None: strategy = DefaultStrategy(default_conf) data = strategy.tickerdata_to_dataframe( - optimize.load_data( + history.load_data( None, ticker_interval='1m', pairs=['UNITTEST/BTC'] @@ -515,7 +49,7 @@ def test_validate_backtest_data(default_conf, mocker, caplog) -> None: timerange = TimeRange('index', 'index', 200, 250) data = strategy.tickerdata_to_dataframe( - optimize.load_data( + history.load_data( None, ticker_interval='5m', pairs=['UNITTEST/BTC'], From 407139b0e0fc686f82768b3aa04d7987142bec73 Mon Sep 17 00:00:00 2001 From: Matthias Date: Thu, 13 Dec 2018 06:37:07 +0100 Subject: [PATCH 631/699] remove unused imports --- freqtrade/data/history.py | 1 - freqtrade/optimize/__init__.py | 4 +--- 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/freqtrade/data/history.py b/freqtrade/data/history.py index a101c62f8..cc151ea52 100644 --- a/freqtrade/data/history.py +++ b/freqtrade/data/history.py @@ -13,7 +13,6 @@ import os from typing import Optional, List, Dict, Tuple, Any import arrow -from pandas import DataFrame from freqtrade import misc, constants, OperationalException from freqtrade.exchange import Exchange diff --git a/freqtrade/optimize/__init__.py b/freqtrade/optimize/__init__.py index 34d1aaf1c..19b8dd90a 100644 --- a/freqtrade/optimize/__init__.py +++ b/freqtrade/optimize/__init__.py @@ -2,14 +2,12 @@ import logging from datetime import datetime -from typing import List, Dict, Tuple +from typing import Dict, Tuple import operator import arrow from pandas import DataFrame - -from freqtrade.arguments import TimeRange from freqtrade.optimize.default_hyperopt import DefaultHyperOpts # noqa: F401 logger = logging.getLogger(__name__) From 43039aa6ab08cdcd08ca37df34f7e05f469baf77 Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Fri, 14 Dec 2018 13:32:07 +0100 Subject: [PATCH 632/699] Update ccxt from 1.18.32 to 1.18.36 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index bff055a63..595b4082a 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -ccxt==1.18.32 +ccxt==1.18.36 SQLAlchemy==1.2.15 python-telegram-bot==11.1.0 arrow==0.12.1 From c42d5002a178415882fdd29e78702018326464b8 Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Fri, 14 Dec 2018 13:32:09 +0100 Subject: [PATCH 633/699] Update pytest from 4.0.1 to 4.0.2 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 595b4082a..3f06e6695 100644 --- a/requirements.txt +++ b/requirements.txt @@ -13,7 +13,7 @@ scipy==1.1.0 jsonschema==2.6.0 numpy==1.15.4 TA-Lib==0.4.17 -pytest==4.0.1 +pytest==4.0.2 pytest-mock==1.10.0 pytest-asyncio==0.9.0 pytest-cov==2.6.0 From 7e463b209c99162f2213cf75d16bf0fe1400b405 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 15 Dec 2018 13:28:00 +0100 Subject: [PATCH 634/699] Add link to contributing for "wanna help" --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 571709e3b..30ca839a9 100644 --- a/README.md +++ b/README.md @@ -62,7 +62,7 @@ hesitate to read the source code and understand the mechanism of this bot. - [Requirements](#requirements) - [Min hardware required](#min-hardware-required) - [Software requirements](#software-requirements) -- [Wanna help?] +- [Wanna help?](https://github.com/freqtrade/freqtrade/blob/develop/CONTRIBUTING.md) ## Quick start From 4ad507f8dda9f72b81bc865ca283bda6239927e9 Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Sat, 15 Dec 2018 13:32:06 +0100 Subject: [PATCH 635/699] Update ccxt from 1.18.36 to 1.18.37 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 3f06e6695..6ad6cd2d4 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -ccxt==1.18.36 +ccxt==1.18.37 SQLAlchemy==1.2.15 python-telegram-bot==11.1.0 arrow==0.12.1 From a34c2cf64b9ddb7e18c2f6886cbe56a28bd28785 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 15 Dec 2018 13:40:02 +0100 Subject: [PATCH 636/699] Add missing test-module __init__.py --- freqtrade/tests/data/__init__.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 freqtrade/tests/data/__init__.py diff --git a/freqtrade/tests/data/__init__.py b/freqtrade/tests/data/__init__.py new file mode 100644 index 000000000..e69de29bb From f261911285343a870bda9ad281af4e971e15b95d Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 15 Dec 2018 13:54:35 +0100 Subject: [PATCH 637/699] replace os.path with pathlib.Path --- freqtrade/data/history.py | 30 +++++++++++++----------------- scripts/plot_dataframe.py | 2 +- scripts/plot_profit.py | 8 ++++---- 3 files changed, 18 insertions(+), 22 deletions(-) diff --git a/freqtrade/data/history.py b/freqtrade/data/history.py index cc151ea52..21636f8c0 100644 --- a/freqtrade/data/history.py +++ b/freqtrade/data/history.py @@ -9,7 +9,7 @@ except ImportError: import json # type: ignore _UJSON = False import logging -import os +from pathlib import Path from typing import Optional, List, Dict, Tuple, Any import arrow @@ -64,7 +64,7 @@ def trim_tickerlist(tickerlist: List[Dict], timerange: TimeRange) -> List[Dict]: def load_tickerdata_file( - datadir: str, pair: str, + datadir: Path, pair: str, ticker_interval: str, timerange: Optional[TimeRange] = None) -> Optional[List[Dict]]: """ @@ -73,16 +73,16 @@ def load_tickerdata_file( """ path = make_testdata_path(datadir) pair_s = pair.replace('/', '_') - file = os.path.join(path, f'{pair_s}-{ticker_interval}.json') - gzipfile = file + '.gz' + file = path.joinpath(f'{pair_s}-{ticker_interval}.json') + gzipfile = file.with_suffix('.gz') # If the file does not exist we download it when None is returned. # If file exists, read the file, load the json - if os.path.isfile(gzipfile): + if gzipfile.is_file(): logger.debug('Loading ticker data from file %s', gzipfile) with gzip.open(gzipfile) as tickerdata: pairdata = json.load(tickerdata) - elif os.path.isfile(file): + elif file.is_file(): logger.debug('Loading ticker data from file %s', file) with open(file) as tickerdata: pairdata = json.load(tickerdata) @@ -94,7 +94,7 @@ def load_tickerdata_file( return pairdata -def load_data(datadir: str, +def load_data(datadir: Path, ticker_interval: str, pairs: List[str], refresh_pairs: Optional[bool] = False, @@ -137,13 +137,9 @@ def load_data(datadir: str, return result -def make_testdata_path(datadir: str) -> str: +def make_testdata_path(datadir: Optional[Path]) -> Path: """Return the path where testdata files are stored""" - return datadir or os.path.abspath( - os.path.join( - os.path.dirname(__file__), '..', 'tests', 'testdata' - ) - ) + return datadir or (Path(__file__).parent.parent / "tests" / "testdata").resolve() def download_pairs(datadir, exchange: Exchange, pairs: List[str], @@ -167,7 +163,7 @@ def download_pairs(datadir, exchange: Exchange, pairs: List[str], return True -def load_cached_data_for_updating(filename: str, +def load_cached_data_for_updating(filename: Path, tick_interval: str, timerange: Optional[TimeRange]) -> Tuple[ List[Any], @@ -187,7 +183,7 @@ def load_cached_data_for_updating(filename: str, since_ms = arrow.utcnow().shift(minutes=num_minutes).timestamp * 1000 # read the cached file - if os.path.isfile(filename): + if filename.is_file(): with open(filename, "rt") as file: data = json_load(file) # remove the last item, because we are not sure if it is correct @@ -210,7 +206,7 @@ def load_cached_data_for_updating(filename: str, return (data, since_ms) -def download_backtesting_testdata(datadir: str, +def download_backtesting_testdata(datadir: Path, exchange: Exchange, pair: str, tick_interval: str = '5m', @@ -230,7 +226,7 @@ def download_backtesting_testdata(datadir: str, """ path = make_testdata_path(datadir) filepair = pair.replace("/", "_") - filename = os.path.join(path, f'{filepair}-{tick_interval}.json') + filename = path.joinpath(f'{filepair}-{tick_interval}.json') logger.info( 'Download the pair: "%s", Interval: %s', diff --git a/scripts/plot_dataframe.py b/scripts/plot_dataframe.py index a22735c6a..ae9cd7f1d 100755 --- a/scripts/plot_dataframe.py +++ b/scripts/plot_dataframe.py @@ -142,7 +142,7 @@ def plot_analyzed_dataframe(args: Namespace) -> None: tickers[pair] = exchange.klines(pair) else: tickers = history.load_data( - datadir=_CONF.get("datadir"), + datadir=Path(_CONF.get("datadir")), pairs=[pair], ticker_interval=tick_interval, refresh_pairs=_CONF.get('refresh_pairs', False), diff --git a/scripts/plot_profit.py b/scripts/plot_profit.py index 5e84943a5..a1561bc89 100755 --- a/scripts/plot_profit.py +++ b/scripts/plot_profit.py @@ -13,10 +13,10 @@ Optional Cli parameters --export-filename: Specify where the backtest export is located. """ import logging -import os import sys import json from argparse import Namespace +from pathlib import Path from typing import List, Optional import numpy as np @@ -27,7 +27,7 @@ import plotly.graph_objs as go from freqtrade.arguments import Arguments from freqtrade.configuration import Configuration from freqtrade import constants -from freqtrade.data as history +from freqtrade.data import history from freqtrade.resolvers import StrategyResolver import freqtrade.misc as misc @@ -121,7 +121,7 @@ def plot_profit(args: Namespace) -> None: logger.info('Filter, keep pairs %s' % pairs) tickers = history.load_data( - datadir=config.get('datadir'), + datadir=Path(config.get('datadir')), pairs=pairs, ticker_interval=tick_interval, refresh_pairs=False, @@ -187,7 +187,7 @@ def plot_profit(args: Namespace) -> None: ) fig.append_trace(pair_profit, 3, 1) - plot(fig, filename=os.path.join('user_data', 'freqtrade-profit-plot.html')) + plot(fig, filename=str(Path('user_data').joinpath('freqtrade-profit-plot.html'))) def define_index(min_date: int, max_date: int, interval: str) -> int: From 21aba1620c3bebc8089681ae2b2648edb1329995 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 15 Dec 2018 13:55:16 +0100 Subject: [PATCH 638/699] Replace calls to load_data --- freqtrade/data/history.py | 4 ++-- freqtrade/edge/__init__.py | 5 +++-- freqtrade/optimize/backtesting.py | 2 +- freqtrade/optimize/hyperopt.py | 7 ++++--- scripts/download_backtest_data.py | 2 +- 5 files changed, 11 insertions(+), 9 deletions(-) diff --git a/freqtrade/data/history.py b/freqtrade/data/history.py index 21636f8c0..00dc8470b 100644 --- a/freqtrade/data/history.py +++ b/freqtrade/data/history.py @@ -74,7 +74,7 @@ def load_tickerdata_file( path = make_testdata_path(datadir) pair_s = pair.replace('/', '_') file = path.joinpath(f'{pair_s}-{ticker_interval}.json') - gzipfile = file.with_suffix('.gz') + gzipfile = file.with_suffix(file.suffix + '.gz') # If the file does not exist we download it when None is returned. # If file exists, read the file, load the json @@ -94,7 +94,7 @@ def load_tickerdata_file( return pairdata -def load_data(datadir: Path, +def load_data(datadir: Optional[Path], ticker_interval: str, pairs: List[str], refresh_pairs: Optional[bool] = False, diff --git a/freqtrade/edge/__init__.py b/freqtrade/edge/__init__.py index 589ac6fe0..29f34ce27 100644 --- a/freqtrade/edge/__init__.py +++ b/freqtrade/edge/__init__.py @@ -1,9 +1,10 @@ # pragma pylint: disable=W0603 """ Edge positioning package """ import logging +from pathlib import Path from typing import Any, Dict, NamedTuple -import arrow +import arrow import numpy as np import utils_find_1st as utf1st from pandas import DataFrame @@ -99,7 +100,7 @@ class Edge(): logger.info('Using local backtesting data (using whitelist in given config) ...') data = history.load_data( - self.config['datadir'], + Path(self.config.get('datadir')) if self.config.get('datadir') else None, pairs=pairs, ticker_interval=self.ticker_interval, refresh_pairs=self._refresh_pairs, diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index 0d025e696..1640adef4 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -370,7 +370,7 @@ class Backtesting(object): timerange = Arguments.parse_timerange(None if self.config.get( 'timerange') is None else str(self.config.get('timerange'))) data = history.load_data( - self.config['datadir'], + Path(self.config.get('datadir')) if self.config.get('datadir') else None, pairs=pairs, ticker_interval=self.ticker_interval, refresh_pairs=self.config.get('refresh_pairs', False), diff --git a/freqtrade/optimize/hyperopt.py b/freqtrade/optimize/hyperopt.py index 2ac55dd9b..a405217e9 100644 --- a/freqtrade/optimize/hyperopt.py +++ b/freqtrade/optimize/hyperopt.py @@ -5,11 +5,12 @@ This module contains the hyperopt logic """ import logging -import multiprocessing +from argparse import Namespace import os import sys -from argparse import Namespace +from pathlib import Path from math import exp +import multiprocessing from operator import itemgetter from typing import Any, Dict, List @@ -240,7 +241,7 @@ class Hyperopt(Backtesting): timerange = Arguments.parse_timerange(None if self.config.get( 'timerange') is None else str(self.config.get('timerange'))) data = load_data( - datadir=str(self.config.get('datadir')), + datadir=Path(self.config.get('datadir')) if self.config.get('datadir') else None, pairs=self.config['exchange']['pair_whitelist'], ticker_interval=self.ticker_interval, timerange=timerange diff --git a/scripts/download_backtest_data.py b/scripts/download_backtest_data.py index ebfef05d5..6235fe667 100755 --- a/scripts/download_backtest_data.py +++ b/scripts/download_backtest_data.py @@ -82,7 +82,7 @@ for pair in PAIRS: dl_file.unlink() print(f'downloading pair {pair}, interval {tick_interval}') - download_backtesting_testdata(str(dl_path), exchange=exchange, + download_backtesting_testdata(dl_path, exchange=exchange, pair=pair, tick_interval=tick_interval, timerange=timerange) From 6c02cc59939c8355106e7ce64de5805d855fd10b Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 15 Dec 2018 14:10:45 +0100 Subject: [PATCH 639/699] Adjust test to pathlib --- freqtrade/data/history.py | 2 +- freqtrade/edge/__init__.py | 2 +- freqtrade/optimize/backtesting.py | 2 +- freqtrade/optimize/hyperopt.py | 2 +- freqtrade/tests/data/test_history.py | 13 +++++++------ 5 files changed, 11 insertions(+), 10 deletions(-) diff --git a/freqtrade/data/history.py b/freqtrade/data/history.py index 00dc8470b..793aee0d1 100644 --- a/freqtrade/data/history.py +++ b/freqtrade/data/history.py @@ -64,7 +64,7 @@ def trim_tickerlist(tickerlist: List[Dict], timerange: TimeRange) -> List[Dict]: def load_tickerdata_file( - datadir: Path, pair: str, + datadir: Optional[Path], pair: str, ticker_interval: str, timerange: Optional[TimeRange] = None) -> Optional[List[Dict]]: """ diff --git a/freqtrade/edge/__init__.py b/freqtrade/edge/__init__.py index 29f34ce27..72e234773 100644 --- a/freqtrade/edge/__init__.py +++ b/freqtrade/edge/__init__.py @@ -100,7 +100,7 @@ class Edge(): logger.info('Using local backtesting data (using whitelist in given config) ...') data = history.load_data( - Path(self.config.get('datadir')) if self.config.get('datadir') else None, + Path(self.config['datadir']) if self.config.get('datadir') else None, pairs=pairs, ticker_interval=self.ticker_interval, refresh_pairs=self._refresh_pairs, diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index 1640adef4..4447df966 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -370,7 +370,7 @@ class Backtesting(object): timerange = Arguments.parse_timerange(None if self.config.get( 'timerange') is None else str(self.config.get('timerange'))) data = history.load_data( - Path(self.config.get('datadir')) if self.config.get('datadir') else None, + Path(self.config['datadir']) if self.config.get('datadir') else None, pairs=pairs, ticker_interval=self.ticker_interval, refresh_pairs=self.config.get('refresh_pairs', False), diff --git a/freqtrade/optimize/hyperopt.py b/freqtrade/optimize/hyperopt.py index a405217e9..2d08fec81 100644 --- a/freqtrade/optimize/hyperopt.py +++ b/freqtrade/optimize/hyperopt.py @@ -241,7 +241,7 @@ class Hyperopt(Backtesting): timerange = Arguments.parse_timerange(None if self.config.get( 'timerange') is None else str(self.config.get('timerange'))) data = load_data( - datadir=Path(self.config.get('datadir')) if self.config.get('datadir') else None, + datadir=Path(self.config['datadir']) if self.config.get('datadir') else None, pairs=self.config['exchange']['pair_whitelist'], ticker_interval=self.ticker_interval, timerange=timerange diff --git a/freqtrade/tests/data/test_history.py b/freqtrade/tests/data/test_history.py index 585e7e2a7..4f6d5d247 100644 --- a/freqtrade/tests/data/test_history.py +++ b/freqtrade/tests/data/test_history.py @@ -2,6 +2,7 @@ import json import os +from pathlib import Path import uuid from shutil import copyfile @@ -116,7 +117,7 @@ def test_load_data_with_new_pair_1min(ticker_history_list, mocker, caplog, defau def test_testdata_path() -> None: - assert os.path.join('freqtrade', 'tests', 'testdata') in make_testdata_path(None) + assert str(Path('freqtrade') / 'tests' / 'testdata') in str(make_testdata_path(None)) def test_download_pairs(ticker_history_list, mocker, default_conf) -> None: @@ -160,10 +161,10 @@ def test_download_pairs(ticker_history_list, mocker, default_conf) -> None: def test_load_cached_data_for_updating(mocker) -> None: - datadir = os.path.join(os.path.dirname(__file__), '..', 'testdata') + datadir = Path(__file__).parent.parent.joinpath('testdata') test_data = None - test_filename = os.path.join(datadir, 'UNITTEST_BTC-1m.json') + test_filename = datadir.joinpath('UNITTEST_BTC-1m.json') with open(test_filename, "rt") as file: test_data = json.load(file) @@ -238,7 +239,7 @@ def test_load_cached_data_for_updating(mocker) -> None: # no datafile exist # should return timestamp start time timerange = TimeRange('date', None, now_ts - 10000, 0) - data, start_ts = load_cached_data_for_updating(test_filename + 'unexist', + data, start_ts = load_cached_data_for_updating(test_filename.with_name('unexist'), '1m', timerange) assert data == [] @@ -247,7 +248,7 @@ def test_load_cached_data_for_updating(mocker) -> None: # same with 'line' timeframe num_lines = 30 timerange = TimeRange(None, 'line', 0, -num_lines) - data, start_ts = load_cached_data_for_updating(test_filename + 'unexist', + data, start_ts = load_cached_data_for_updating(test_filename.with_name('unexist'), '1m', timerange) assert data == [] @@ -255,7 +256,7 @@ def test_load_cached_data_for_updating(mocker) -> None: # no datafile exist, no timeframe is set # should return an empty array and None - data, start_ts = load_cached_data_for_updating(test_filename + 'unexist', + data, start_ts = load_cached_data_for_updating(test_filename.with_name('unexist'), '1m', None) assert data == [] From b4f1a80dc13a5a12fab1f32f4c4457fb307be1ae Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 15 Dec 2018 14:21:14 +0100 Subject: [PATCH 640/699] Add edge oneliner to index --- README.md | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 30ca839a9..858611d3b 100644 --- a/README.md +++ b/README.md @@ -34,13 +34,15 @@ hesitate to read the source code and understand the mechanism of this bot. - [x] **Dry-run**: Run the bot without playing money. - [x] **Backtesting**: Run a simulation of your buy/sell strategy. - [x] **Strategy Optimization by machine learning**: Use machine learning to optimize your buy/sell strategy parameters with real exchange data. -- [x] **Whitelist crypto-currencies**: Select which crypto-currency you want to trade. +- [x] **Risk Reward calculation based trading**: Builtin win rate and risk reward ratio using [edge](https://github.com/freqtrade/freqtrade/blob/develop/docs/edge.md). +- [x] **Whitelist crypto-currencies**: Select which crypto-currency you want to trade or use dynamic whitelists. - [x] **Blacklist crypto-currencies**: Select which crypto-currency you want to avoid. - [x] **Manageable via Telegram**: Manage the bot with Telegram - [x] **Display profit/loss in fiat**: Display your profit/loss in 33 fiat. - [x] **Daily summary of profit/loss**: Provide a daily summary of your profit/loss. - [x] **Performance status report**: Provide a performance status of your current trades. + ## Table of Contents - [Quick start](#quick-start) @@ -51,6 +53,7 @@ hesitate to read the source code and understand the mechanism of this bot. - [Backtesting](https://github.com/freqtrade/freqtrade/blob/develop/docs/backtesting.md) - [Hyperopt](https://github.com/freqtrade/freqtrade/blob/develop/docs/hyperopt.md) - [Sandbox Testing](https://github.com/freqtrade/freqtrade/blob/develop/docs/sandbox-testing.md) + - [Edge](https://github.com/freqtrade/freqtrade/blob/develop/docs/edge.md) - [Basic Usage](#basic-usage) - [Bot commands](#bot-commands) - [Telegram RPC commands](#telegram-rpc-commands) @@ -63,6 +66,7 @@ hesitate to read the source code and understand the mechanism of this bot. - [Min hardware required](#min-hardware-required) - [Software requirements](#software-requirements) - [Wanna help?](https://github.com/freqtrade/freqtrade/blob/develop/CONTRIBUTING.md) + - [Dev - getting started](https://github.com/freqtrade/freqtrade/blob/develop/docs/developer.md) (WIP) ## Quick start From c1a32bc6c8f390c07fb6dfd6d4f1387089b47595 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 15 Dec 2018 14:22:49 +0100 Subject: [PATCH 641/699] use json_load to load data - otherwise unforseen problems could appear due to the default beeing ujson --- freqtrade/data/history.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/freqtrade/data/history.py b/freqtrade/data/history.py index 793aee0d1..c397f7adb 100644 --- a/freqtrade/data/history.py +++ b/freqtrade/data/history.py @@ -81,11 +81,11 @@ def load_tickerdata_file( if gzipfile.is_file(): logger.debug('Loading ticker data from file %s', gzipfile) with gzip.open(gzipfile) as tickerdata: - pairdata = json.load(tickerdata) + pairdata = json_load(tickerdata) elif file.is_file(): logger.debug('Loading ticker data from file %s', file) with open(file) as tickerdata: - pairdata = json.load(tickerdata) + pairdata = json_load(tickerdata) else: return None From 1c5031b468865f800c295b32fb2a6443748b5680 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 15 Dec 2018 14:28:37 +0100 Subject: [PATCH 642/699] load_data to return dataframe --- freqtrade/data/history.py | 8 +++++--- freqtrade/strategy/interface.py | 3 +-- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/freqtrade/data/history.py b/freqtrade/data/history.py index c397f7adb..e561c7252 100644 --- a/freqtrade/data/history.py +++ b/freqtrade/data/history.py @@ -13,8 +13,10 @@ from pathlib import Path from typing import Optional, List, Dict, Tuple, Any import arrow +from pandas import DataFrame from freqtrade import misc, constants, OperationalException +from freqtrade.data.converter import parse_ticker_dataframe from freqtrade.exchange import Exchange from freqtrade.arguments import TimeRange @@ -69,7 +71,7 @@ def load_tickerdata_file( timerange: Optional[TimeRange] = None) -> Optional[List[Dict]]: """ Load a pair from file, - :return dict OR empty if unsuccesful + :return dict(:) or None if unsuccesful """ path = make_testdata_path(datadir) pair_s = pair.replace('/', '_') @@ -99,7 +101,7 @@ def load_data(datadir: Optional[Path], pairs: List[str], refresh_pairs: Optional[bool] = False, exchange: Optional[Exchange] = None, - timerange: TimeRange = TimeRange(None, None, 0, 0)) -> Dict[str, List]: + timerange: TimeRange = TimeRange(None, None, 0, 0)) -> Dict[str, DataFrame]: """ Loads ticker history data for the given parameters :return: dict @@ -125,7 +127,7 @@ def load_data(datadir: Optional[Path], logger.warning('Missing data at end for pair %s, data ends at %s', pair, arrow.get(pairdata[-1][0] // 1000).strftime('%Y-%m-%d %H:%M:%S')) - result[pair] = pairdata + result[pair] = parse_ticker_dataframe(pairdata) else: logger.warning( 'No data for pair: "%s", Interval: %s. ' diff --git a/freqtrade/strategy/interface.py b/freqtrade/strategy/interface.py index c8284b44f..700b65f04 100644 --- a/freqtrade/strategy/interface.py +++ b/freqtrade/strategy/interface.py @@ -13,7 +13,6 @@ import arrow from pandas import DataFrame from freqtrade import constants -from freqtrade.data.converter import parse_ticker_dataframe from freqtrade.persistence import Trade logger = logging.getLogger(__name__) @@ -326,7 +325,7 @@ class IStrategy(ABC): """ Creates a dataframe and populates indicators for given ticker data """ - return {pair: self.advise_indicators(parse_ticker_dataframe(pair_data), {'pair': pair}) + return {pair: self.advise_indicators(pair_data, {'pair': pair}) for pair, pair_data in tickerdata.items()} def advise_indicators(self, dataframe: DataFrame, metadata: dict) -> DataFrame: From 34ea214f7c783a63c444e378a7104659bcb4fee6 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 15 Dec 2018 14:42:21 +0100 Subject: [PATCH 643/699] Fix some tests to use dataframe --- freqtrade/data/converter.py | 1 + freqtrade/tests/data/test_history.py | 5 +++-- freqtrade/tests/edge/test_edge.py | 4 +++- freqtrade/tests/test_misc.py | 2 +- 4 files changed, 8 insertions(+), 4 deletions(-) diff --git a/freqtrade/data/converter.py b/freqtrade/data/converter.py index e09618a64..18b8fd84c 100644 --- a/freqtrade/data/converter.py +++ b/freqtrade/data/converter.py @@ -32,6 +32,7 @@ def parse_ticker_dataframe(ticker: list) -> DataFrame: 'volume': 'max', }) frame.drop(frame.tail(1).index, inplace=True) # eliminate partial candle + logger.debug('Droppling last candle') return frame diff --git a/freqtrade/tests/data/test_history.py b/freqtrade/tests/data/test_history.py index 4f6d5d247..5d279a529 100644 --- a/freqtrade/tests/data/test_history.py +++ b/freqtrade/tests/data/test_history.py @@ -339,7 +339,7 @@ def test_load_partial_missing(caplog) -> None: # timedifference in 5 minutes td = ((end - start).total_seconds() // 60 // 5) + 1 assert td != len(tickerdata['UNITTEST/BTC']) - start_real = arrow.get(tickerdata['UNITTEST/BTC'][0][0] / 1000) + start_real = tickerdata['UNITTEST/BTC'].iloc[0, 0] assert log_has(f'Missing data at start for pair ' f'UNITTEST/BTC, data starts at {start_real.strftime("%Y-%m-%d %H:%M:%S")}', caplog.record_tuples) @@ -354,7 +354,8 @@ def test_load_partial_missing(caplog) -> None: # timedifference in 5 minutes td = ((end - start).total_seconds() // 60 // 5) + 1 assert td != len(tickerdata['UNITTEST/BTC']) - end_real = arrow.get(tickerdata['UNITTEST/BTC'][-1][0] / 1000) + # Shift endtime with +5 - as last candle is dropped (partial candle) + end_real = arrow.get(tickerdata['UNITTEST/BTC'].iloc[-1, 0]).shift(minutes=5) assert log_has(f'Missing data at end for pair ' f'UNITTEST/BTC, data ends at {end_real.strftime("%Y-%m-%d %H:%M:%S")}', caplog.record_tuples) diff --git a/freqtrade/tests/edge/test_edge.py b/freqtrade/tests/edge/test_edge.py index 1f4673fe7..4fbe9c494 100644 --- a/freqtrade/tests/edge/test_edge.py +++ b/freqtrade/tests/edge/test_edge.py @@ -10,6 +10,7 @@ import numpy as np import pytest from pandas import DataFrame, to_datetime +from freqtrade.data.converter import parse_ticker_dataframe from freqtrade.edge import Edge, PairInfo from freqtrade.strategy.interface import SellType from freqtrade.tests.conftest import get_patched_freqtradebot @@ -280,7 +281,8 @@ def mocked_load_data(datadir, pairs=[], ticker_interval='0m', refresh_pairs=Fals 123.45 ] for x in range(0, 500)] - pairdata = {'NEO/BTC': ETHBTC, 'LTC/BTC': LTCBTC} + pairdata = {'NEO/BTC': parse_ticker_dataframe(ETHBTC), + 'LTC/BTC': parse_ticker_dataframe(LTCBTC)} return pairdata diff --git a/freqtrade/tests/test_misc.py b/freqtrade/tests/test_misc.py index d8a2fe697..017bf372f 100644 --- a/freqtrade/tests/test_misc.py +++ b/freqtrade/tests/test_misc.py @@ -34,7 +34,7 @@ def test_datesarray_to_datetimearray(ticker_history_list): def test_common_datearray(default_conf) -> None: strategy = DefaultStrategy(default_conf) tick = load_tickerdata_file(None, 'UNITTEST/BTC', '1m') - tickerlist = {'UNITTEST/BTC': tick} + tickerlist = {'UNITTEST/BTC': parse_ticker_dataframe(tick)} dataframes = strategy.tickerdata_to_dataframe(tickerlist) dates = common_datearray(dataframes) From d0c9791ca661f6782eaaff0eabb11eeb11bb8034 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 15 Dec 2018 15:34:25 +0100 Subject: [PATCH 644/699] Fix tests to support load_data with dataframe --- freqtrade/tests/optimize/test_backtesting.py | 62 +++++++++----------- freqtrade/tests/optimize/test_hyperopt.py | 5 +- freqtrade/tests/strategy/test_interface.py | 3 +- 3 files changed, 33 insertions(+), 37 deletions(-) diff --git a/freqtrade/tests/optimize/test_backtesting.py b/freqtrade/tests/optimize/test_backtesting.py index fd1c957eb..f30d6b224 100644 --- a/freqtrade/tests/optimize/test_backtesting.py +++ b/freqtrade/tests/optimize/test_backtesting.py @@ -14,6 +14,7 @@ from arrow import Arrow from freqtrade import DependencyException, constants from freqtrade.arguments import Arguments, TimeRange from freqtrade.data import history +from freqtrade.data.converter import parse_ticker_dataframe from freqtrade.optimize import get_timeframe from freqtrade.optimize.backtesting import (Backtesting, setup_configuration, start) @@ -35,22 +36,13 @@ def trim_dictlist(dict_list, num): def load_data_test(what): timerange = TimeRange(None, 'line', 0, -101) - data = history.load_data(None, ticker_interval='1m', - pairs=['UNITTEST/BTC'], timerange=timerange) - pair = data['UNITTEST/BTC'] + pair = history.load_tickerdata_file(None, ticker_interval='1m', + pair='UNITTEST/BTC', timerange=timerange) datalen = len(pair) - # Depending on the what parameter we now adjust the - # loaded data looks: - # pair :: [[ 1509836520000, unix timestamp in ms - # 0.00162008, open - # 0.00162008, high - # 0.00162008, low - # 0.00162008, close - # 108.14853839 base volume - # ]] + base = 0.001 if what == 'raise': - return {'UNITTEST/BTC': [ + data = [ [ pair[x][0], # Keep old dates x * base, # But replace O,H,L,C @@ -59,9 +51,9 @@ def load_data_test(what): x * base, pair[x][5], # Keep old volume ] for x in range(0, datalen) - ]} + ] if what == 'lower': - return {'UNITTEST/BTC': [ + data = [ [ pair[x][0], # Keep old dates 1 - x * base, # But replace O,H,L,C @@ -70,10 +62,10 @@ def load_data_test(what): 1 - x * base, pair[x][5] # Keep old volume ] for x in range(0, datalen) - ]} + ] if what == 'sine': hz = 0.1 # frequency - return {'UNITTEST/BTC': [ + data = [ [ pair[x][0], # Keep old dates math.sin(x * hz) / 1000 + base, # But replace O,H,L,C @@ -82,8 +74,8 @@ def load_data_test(what): math.sin(x * hz) / 1000 + base, pair[x][5] # Keep old volume ] for x in range(0, datalen) - ]} - return data + ] + return {'UNITTEST/BTC': parse_ticker_dataframe(data)} def simple_backtest(config, contour, num_results, mocker) -> None: @@ -112,15 +104,15 @@ def simple_backtest(config, contour, num_results, mocker) -> None: def mocked_load_data(datadir, pairs=[], ticker_interval='0m', refresh_pairs=False, timerange=None, exchange=None): tickerdata = history.load_tickerdata_file(datadir, 'UNITTEST/BTC', '1m', timerange=timerange) - pairdata = {'UNITTEST/BTC': tickerdata} + pairdata = {'UNITTEST/BTC': parse_ticker_dataframe(tickerdata)} return pairdata # use for mock ccxt.fetch_ohlvc' def _load_pair_as_ticks(pair, tickfreq): - ticks = history.load_data(None, ticker_interval=tickfreq, pairs=[pair]) - ticks = trim_dictlist(ticks, -201) - return ticks[pair] + ticks = history.load_tickerdata_file(None, ticker_interval=tickfreq, pair=pair) + ticks = ticks[-201:] + return ticks # FIX: fixturize this? @@ -334,7 +326,7 @@ def test_tickerdata_to_dataframe(default_conf, mocker) -> None: patch_exchange(mocker) timerange = TimeRange(None, 'line', 0, -100) tick = history.load_tickerdata_file(None, 'UNITTEST/BTC', '1m', timerange=timerange) - tickerlist = {'UNITTEST/BTC': tick} + tickerlist = {'UNITTEST/BTC': parse_ticker_dataframe(tick)} backtesting = Backtesting(default_conf) data = backtesting.strategy.tickerdata_to_dataframe(tickerlist) @@ -512,8 +504,9 @@ def test_backtest(default_conf, fee, mocker) -> None: patch_exchange(mocker) backtesting = Backtesting(default_conf) pair = 'UNITTEST/BTC' - data = history.load_data(None, ticker_interval='5m', pairs=['UNITTEST/BTC']) - data = trim_dictlist(data, -200) + timerange = TimeRange(None, 'line', 0, -201) + data = history.load_data(None, ticker_interval='5m', pairs=['UNITTEST/BTC'], + timerange=timerange) data_processed = backtesting.strategy.tickerdata_to_dataframe(data) min_date, max_date = get_timeframe(data_processed) results = backtesting.backtest( @@ -537,8 +530,8 @@ def test_backtest(default_conf, fee, mocker) -> None: Arrow(2018, 1, 30, 3, 30, 0).datetime], 'close_time': [Arrow(2018, 1, 29, 22, 35, 0).datetime, Arrow(2018, 1, 30, 4, 15, 0).datetime], - 'open_index': [77, 183], - 'close_index': [124, 192], + 'open_index': [78, 184], + 'close_index': [125, 193], 'trade_duration': [235, 45], 'open_at_end': [False, False], 'open_rate': [0.104445, 0.10302485], @@ -564,9 +557,10 @@ def test_backtest_1min_ticker_interval(default_conf, fee, mocker) -> None: patch_exchange(mocker) backtesting = Backtesting(default_conf) - # Run a backtesting for an exiting 5min ticker_interval - data = history.load_data(None, ticker_interval='1m', pairs=['UNITTEST/BTC']) - data = trim_dictlist(data, -200) + # Run a backtesting for an exiting 1min ticker_interval + timerange = TimeRange(None, 'line', 0, -200) + data = history.load_data(None, ticker_interval='1m', pairs=['UNITTEST/BTC'], + timerange=timerange) processed = backtesting.strategy.tickerdata_to_dataframe(data) min_date, max_date = get_timeframe(processed) results = backtesting.backtest( @@ -652,7 +646,7 @@ def test_backtest_alternate_buy_sell(default_conf, fee, mocker): # 200 candles in backtest data # won't buy on first (shifted by 1) # 100 buys signals - assert len(results) == 99 + assert len(results) == 100 # One trade was force-closed at the end assert len(results.loc[results.open_at_end]) == 0 @@ -841,7 +835,7 @@ def test_backtest_start_live(default_conf, mocker, caplog): 'Using stake_currency: BTC ...', 'Using stake_amount: 0.001 ...', 'Downloading data for all pairs in whitelist ...', - 'Measuring data from 2017-11-14T19:31:00+00:00 up to 2017-11-14T22:57:00+00:00 (0 days)..', + 'Measuring data from 2017-11-14T19:31:00+00:00 up to 2017-11-14T22:58:00+00:00 (0 days)..', 'Parameter --enable-position-stacking detected ...' ] @@ -900,7 +894,7 @@ def test_backtest_start_multi_strat(default_conf, mocker, caplog): 'Using stake_currency: BTC ...', 'Using stake_amount: 0.001 ...', 'Downloading data for all pairs in whitelist ...', - 'Measuring data from 2017-11-14T19:31:00+00:00 up to 2017-11-14T22:57:00+00:00 (0 days)..', + 'Measuring data from 2017-11-14T19:31:00+00:00 up to 2017-11-14T22:58:00+00:00 (0 days)..', 'Parameter --enable-position-stacking detected ...', 'Running backtesting for Strategy DefaultStrategy', 'Running backtesting for Strategy TestStrategy', diff --git a/freqtrade/tests/optimize/test_hyperopt.py b/freqtrade/tests/optimize/test_hyperopt.py index c41c2d3c0..5a40441bb 100644 --- a/freqtrade/tests/optimize/test_hyperopt.py +++ b/freqtrade/tests/optimize/test_hyperopt.py @@ -6,6 +6,7 @@ from unittest.mock import MagicMock import pandas as pd import pytest +from freqtrade.data.converter import parse_ticker_dataframe from freqtrade.data.history import load_tickerdata_file from freqtrade.optimize.hyperopt import Hyperopt, start from freqtrade.resolvers import StrategyResolver @@ -242,7 +243,7 @@ def test_has_space(hyperopt): def test_populate_indicators(hyperopt) -> None: tick = load_tickerdata_file(None, 'UNITTEST/BTC', '1m') - tickerlist = {'UNITTEST/BTC': tick} + tickerlist = {'UNITTEST/BTC': parse_ticker_dataframe(tick)} dataframes = hyperopt.strategy.tickerdata_to_dataframe(tickerlist) dataframe = hyperopt.custom_hyperopt.populate_indicators(dataframes['UNITTEST/BTC'], {'pair': 'UNITTEST/BTC'}) @@ -255,7 +256,7 @@ def test_populate_indicators(hyperopt) -> None: def test_buy_strategy_generator(hyperopt) -> None: tick = load_tickerdata_file(None, 'UNITTEST/BTC', '1m') - tickerlist = {'UNITTEST/BTC': tick} + tickerlist = {'UNITTEST/BTC': parse_ticker_dataframe(tick)} dataframes = hyperopt.strategy.tickerdata_to_dataframe(tickerlist) dataframe = hyperopt.custom_hyperopt.populate_indicators(dataframes['UNITTEST/BTC'], {'pair': 'UNITTEST/BTC'}) diff --git a/freqtrade/tests/strategy/test_interface.py b/freqtrade/tests/strategy/test_interface.py index e3e5f6224..22ba9a2b6 100644 --- a/freqtrade/tests/strategy/test_interface.py +++ b/freqtrade/tests/strategy/test_interface.py @@ -7,6 +7,7 @@ import arrow from pandas import DataFrame from freqtrade.arguments import TimeRange +from freqtrade.data.converter import parse_ticker_dataframe from freqtrade.data.history import load_tickerdata_file from freqtrade.persistence import Trade from freqtrade.tests.conftest import get_patched_exchange, log_has @@ -110,7 +111,7 @@ def test_tickerdata_to_dataframe(default_conf) -> None: timerange = TimeRange(None, 'line', 0, -100) tick = load_tickerdata_file(None, 'UNITTEST/BTC', '1m', timerange=timerange) - tickerlist = {'UNITTEST/BTC': tick} + tickerlist = {'UNITTEST/BTC': parse_ticker_dataframe(tick)} data = strategy.tickerdata_to_dataframe(tickerlist) assert len(data['UNITTEST/BTC']) == 99 # partial candle was removed From d421e4e8afb7b345363f5732d4171b4d355ba59a Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 15 Dec 2018 19:15:38 +0100 Subject: [PATCH 645/699] update edge description --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 858611d3b..0d0724d3a 100644 --- a/README.md +++ b/README.md @@ -34,7 +34,7 @@ hesitate to read the source code and understand the mechanism of this bot. - [x] **Dry-run**: Run the bot without playing money. - [x] **Backtesting**: Run a simulation of your buy/sell strategy. - [x] **Strategy Optimization by machine learning**: Use machine learning to optimize your buy/sell strategy parameters with real exchange data. -- [x] **Risk Reward calculation based trading**: Builtin win rate and risk reward ratio using [edge](https://github.com/freqtrade/freqtrade/blob/develop/docs/edge.md). +- [x] **Edge position sizing** Calculate your win rate, risk reward ratio, the best stoploss and adjust your position size before taking a position for each specific market. [Learn more](https://github.com/freqtrade/freqtrade/blob/develop/docs/edge.md) - [x] **Whitelist crypto-currencies**: Select which crypto-currency you want to trade or use dynamic whitelists. - [x] **Blacklist crypto-currencies**: Select which crypto-currency you want to avoid. - [x] **Manageable via Telegram**: Manage the bot with Telegram From acd07d40a0d6c0390892fbbf921eeda072aa02ac Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 15 Dec 2018 19:52:52 +0100 Subject: [PATCH 646/699] Cleanup some comments and code formatting --- freqtrade/data/history.py | 44 +++++++++++++-------------------------- 1 file changed, 14 insertions(+), 30 deletions(-) diff --git a/freqtrade/data/history.py b/freqtrade/data/history.py index e561c7252..241b5a058 100644 --- a/freqtrade/data/history.py +++ b/freqtrade/data/history.py @@ -70,7 +70,7 @@ def load_tickerdata_file( ticker_interval: str, timerange: Optional[TimeRange] = None) -> Optional[List[Dict]]: """ - Load a pair from file, + Load a pair from file, either .json.gz or .json :return dict(:) or None if unsuccesful """ path = make_testdata_path(datadir) @@ -78,8 +78,7 @@ def load_tickerdata_file( file = path.joinpath(f'{pair_s}-{ticker_interval}.json') gzipfile = file.with_suffix(file.suffix + '.gz') - # If the file does not exist we download it when None is returned. - # If file exists, read the file, load the json + # Try gzip file first, otherwise regular json file. if gzipfile.is_file(): logger.debug('Loading ticker data from file %s', gzipfile) with gzip.open(gzipfile) as tickerdata: @@ -129,12 +128,9 @@ def load_data(datadir: Optional[Path], arrow.get(pairdata[-1][0] // 1000).strftime('%Y-%m-%d %H:%M:%S')) result[pair] = parse_ticker_dataframe(pairdata) else: - logger.warning( - 'No data for pair: "%s", Interval: %s. ' - 'Use --refresh-pairs-cached to download the data', - pair, - ticker_interval - ) + logger.warning('No data for pair: "%s", Interval: %s. ' + 'Use --refresh-pairs-cached to download the data', + pair, ticker_interval) return result @@ -156,20 +152,15 @@ def download_pairs(datadir, exchange: Exchange, pairs: List[str], tick_interval=ticker_interval, timerange=timerange) except BaseException: - logger.info( - 'Failed to download the pair: "%s", Interval: %s', - pair, - ticker_interval - ) + logger.info('Failed to download the pair: "%s", Interval: %s', + pair, ticker_interval) return False return True -def load_cached_data_for_updating(filename: Path, - tick_interval: str, - timerange: Optional[TimeRange]) -> Tuple[ - List[Any], - Optional[int]]: +def load_cached_data_for_updating(filename: Path, tick_interval: str, + timerange: Optional[TimeRange]) -> Tuple[List[Any], + Optional[int]]: """ Load cached data and choose what part of the data should be updated """ @@ -188,8 +179,7 @@ def load_cached_data_for_updating(filename: Path, if filename.is_file(): with open(filename, "rt") as file: data = json_load(file) - # remove the last item, because we are not sure if it is correct - # it could be fetched when the candle was incompleted + # remove the last item, could be incomplete candle if data: data.pop() else: @@ -197,12 +187,10 @@ def load_cached_data_for_updating(filename: Path, if data: if since_ms and since_ms < data[0][0]: - # the data is requested for earlier period than the cache has - # so fully redownload all the data + # Earlier data than existing data requested, redownload all data = [] else: - # a part of the data was already downloaded, so - # download unexist data only + # a part of the data was already downloaded, so download unexist data only since_ms = data[-1][0] + 1 return (data, since_ms) @@ -230,11 +218,7 @@ def download_backtesting_testdata(datadir: Path, filepair = pair.replace("/", "_") filename = path.joinpath(f'{filepair}-{tick_interval}.json') - logger.info( - 'Download the pair: "%s", Interval: %s', - pair, - tick_interval - ) + logger.info('Download the pair: "%s", Interval: %s', pair, tick_interval) data, since_ms = load_cached_data_for_updating(filename, tick_interval, timerange) From 429f846ad1aa0ac1b29476d79c8033254d1bb918 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 15 Dec 2018 20:14:13 +0100 Subject: [PATCH 647/699] Switch load_data to kwargs --- freqtrade/edge/__init__.py | 2 +- freqtrade/optimize/backtesting.py | 2 +- freqtrade/tests/data/test_history.py | 16 ++++++++-------- freqtrade/tests/optimize/test_backtesting.py | 8 ++++---- freqtrade/tests/optimize/test_optimize.py | 6 +++--- 5 files changed, 17 insertions(+), 17 deletions(-) diff --git a/freqtrade/edge/__init__.py b/freqtrade/edge/__init__.py index 72e234773..8ccfc90de 100644 --- a/freqtrade/edge/__init__.py +++ b/freqtrade/edge/__init__.py @@ -100,7 +100,7 @@ class Edge(): logger.info('Using local backtesting data (using whitelist in given config) ...') data = history.load_data( - Path(self.config['datadir']) if self.config.get('datadir') else None, + datadir=Path(self.config['datadir']) if self.config.get('datadir') else None, pairs=pairs, ticker_interval=self.ticker_interval, refresh_pairs=self._refresh_pairs, diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index 4447df966..cc05c5de8 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -370,7 +370,7 @@ class Backtesting(object): timerange = Arguments.parse_timerange(None if self.config.get( 'timerange') is None else str(self.config.get('timerange'))) data = history.load_data( - Path(self.config['datadir']) if self.config.get('datadir') else None, + datadir=Path(self.config['datadir']) if self.config.get('datadir') else None, pairs=pairs, ticker_interval=self.ticker_interval, refresh_pairs=self.config.get('refresh_pairs', False), diff --git a/freqtrade/tests/data/test_history.py b/freqtrade/tests/data/test_history.py index 5d279a529..a40021fba 100644 --- a/freqtrade/tests/data/test_history.py +++ b/freqtrade/tests/data/test_history.py @@ -58,7 +58,7 @@ def test_load_data_30min_ticker(ticker_history, mocker, caplog, default_conf) -> mocker.patch('freqtrade.exchange.Exchange.get_history', return_value=ticker_history) file = os.path.join(os.path.dirname(__file__), '..', 'testdata', 'UNITTEST_BTC-30m.json') _backup_file(file, copy_file=True) - ld = history.load_data(None, pairs=['UNITTEST/BTC'], ticker_interval='30m') + ld = history.load_data(datadir=None, pairs=['UNITTEST/BTC'], ticker_interval='30m') assert isinstance(ld, dict) assert os.path.isfile(file) is True assert not log_has('Download the pair: "UNITTEST/BTC", Interval: 30m', caplog.record_tuples) @@ -70,7 +70,7 @@ def test_load_data_5min_ticker(ticker_history, mocker, caplog, default_conf) -> file = os.path.join(os.path.dirname(__file__), '..', 'testdata', 'UNITTEST_BTC-5m.json') _backup_file(file, copy_file=True) - history.load_data(None, pairs=['UNITTEST/BTC'], ticker_interval='5m') + history.load_data(datadir=None, pairs=['UNITTEST/BTC'], ticker_interval='5m') assert os.path.isfile(file) is True assert not log_has('Download the pair: "UNITTEST/BTC", Interval: 5m', caplog.record_tuples) _clean_test_file(file) @@ -80,7 +80,7 @@ def test_load_data_1min_ticker(ticker_history, mocker, caplog) -> None: mocker.patch('freqtrade.exchange.Exchange.get_history', return_value=ticker_history) file = os.path.join(os.path.dirname(__file__), '..', 'testdata', 'UNITTEST_BTC-1m.json') _backup_file(file, copy_file=True) - history.load_data(None, ticker_interval='1m', pairs=['UNITTEST/BTC']) + history.load_data(datadir=None, ticker_interval='1m', pairs=['UNITTEST/BTC']) assert os.path.isfile(file) is True assert not log_has('Download the pair: "UNITTEST/BTC", Interval: 1m', caplog.record_tuples) _clean_test_file(file) @@ -96,7 +96,7 @@ def test_load_data_with_new_pair_1min(ticker_history_list, mocker, caplog, defau _backup_file(file) # do not download a new pair if refresh_pairs isn't set - history.load_data(None, + history.load_data(datadir=None, ticker_interval='1m', refresh_pairs=False, pairs=['MEME/BTC']) @@ -106,7 +106,7 @@ def test_load_data_with_new_pair_1min(ticker_history_list, mocker, caplog, defau caplog.record_tuples) # download a new pair if refresh_pairs is set - history.load_data(None, + history.load_data(datadir=None, ticker_interval='1m', refresh_pairs=True, exchange=exchange, @@ -347,8 +347,8 @@ def test_load_partial_missing(caplog) -> None: caplog.clear() start = arrow.get('2018-01-10T00:00:00') end = arrow.get('2018-02-20T00:00:00') - tickerdata = history.load_data(None, '5m', ['UNITTEST/BTC'], - refresh_pairs=False, + tickerdata = history.load_data(datadir=None, ticker_interval='5m', + pairs=['UNITTEST/BTC'], refresh_pairs=False, timerange=TimeRange('date', 'date', start.timestamp, end.timestamp)) # timedifference in 5 minutes @@ -364,7 +364,7 @@ def test_load_partial_missing(caplog) -> None: def test_init(default_conf, mocker) -> None: exchange = get_patched_exchange(mocker, default_conf) assert {} == history.load_data( - '', + datadir='', exchange=exchange, pairs=[], refresh_pairs=True, diff --git a/freqtrade/tests/optimize/test_backtesting.py b/freqtrade/tests/optimize/test_backtesting.py index f30d6b224..8f7c80523 100644 --- a/freqtrade/tests/optimize/test_backtesting.py +++ b/freqtrade/tests/optimize/test_backtesting.py @@ -117,7 +117,7 @@ def _load_pair_as_ticks(pair, tickfreq): # FIX: fixturize this? def _make_backtest_conf(mocker, conf=None, pair='UNITTEST/BTC', record=None): - data = history.load_data(None, ticker_interval='1m', pairs=[pair]) + data = history.load_data(datadir=None, ticker_interval='1m', pairs=[pair]) data = trim_dictlist(data, -201) patch_exchange(mocker) backtesting = Backtesting(conf) @@ -505,7 +505,7 @@ def test_backtest(default_conf, fee, mocker) -> None: backtesting = Backtesting(default_conf) pair = 'UNITTEST/BTC' timerange = TimeRange(None, 'line', 0, -201) - data = history.load_data(None, ticker_interval='5m', pairs=['UNITTEST/BTC'], + data = history.load_data(datadir=None, ticker_interval='5m', pairs=['UNITTEST/BTC'], timerange=timerange) data_processed = backtesting.strategy.tickerdata_to_dataframe(data) min_date, max_date = get_timeframe(data_processed) @@ -559,7 +559,7 @@ def test_backtest_1min_ticker_interval(default_conf, fee, mocker) -> None: # Run a backtesting for an exiting 1min ticker_interval timerange = TimeRange(None, 'line', 0, -200) - data = history.load_data(None, ticker_interval='1m', pairs=['UNITTEST/BTC'], + data = history.load_data(datadir=None, ticker_interval='1m', pairs=['UNITTEST/BTC'], timerange=timerange) processed = backtesting.strategy.tickerdata_to_dataframe(data) min_date, max_date = get_timeframe(processed) @@ -683,7 +683,7 @@ def test_backtest_multi_pair(default_conf, fee, mocker): mocker.patch('freqtrade.exchange.Exchange.get_fee', fee) patch_exchange(mocker) pairs = ['ADA/BTC', 'DASH/BTC', 'ETH/BTC', 'LTC/BTC', 'NXT/BTC'] - data = history.load_data(None, ticker_interval='5m', pairs=pairs) + data = history.load_data(datadir=None, ticker_interval='5m', pairs=pairs) data = trim_dictlist(data, -500) # We need to enable sell-signal - otherwise it sells on ROI!! default_conf['experimental'] = {"use_sell_signal": True} diff --git a/freqtrade/tests/optimize/test_optimize.py b/freqtrade/tests/optimize/test_optimize.py index 2019c9fde..02d0e9c36 100644 --- a/freqtrade/tests/optimize/test_optimize.py +++ b/freqtrade/tests/optimize/test_optimize.py @@ -12,7 +12,7 @@ def test_get_timeframe(default_conf, mocker) -> None: data = strategy.tickerdata_to_dataframe( history.load_data( - None, + datadir=None, ticker_interval='1m', pairs=['UNITTEST/BTC'] ) @@ -28,7 +28,7 @@ def test_validate_backtest_data_warn(default_conf, mocker, caplog) -> None: data = strategy.tickerdata_to_dataframe( history.load_data( - None, + datadir=None, ticker_interval='1m', pairs=['UNITTEST/BTC'] ) @@ -50,7 +50,7 @@ def test_validate_backtest_data(default_conf, mocker, caplog) -> None: timerange = TimeRange('index', 'index', 200, 250) data = strategy.tickerdata_to_dataframe( history.load_data( - None, + datadir=None, ticker_interval='5m', pairs=['UNITTEST/BTC'], timerange=timerange From 8a3c2a0c63be8f7bf648fe47cfac5921f23c4d2f Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 15 Dec 2018 20:31:25 +0100 Subject: [PATCH 648/699] allow only loading 1 pair if necessary * simplify tests nad remove unnecessary mocking --- freqtrade/data/history.py | 45 ++++++++++++++++++---------- freqtrade/tests/data/test_history.py | 28 +++++++---------- 2 files changed, 40 insertions(+), 33 deletions(-) diff --git a/freqtrade/data/history.py b/freqtrade/data/history.py index 241b5a058..11b759aa5 100644 --- a/freqtrade/data/history.py +++ b/freqtrade/data/history.py @@ -95,6 +95,31 @@ def load_tickerdata_file( return pairdata +def load_pair_history(pair: str, + ticker_interval: str, + datadir: Optional[Path], + timerange: TimeRange = TimeRange(None, None, 0, 0)) -> DataFrame: + """ + Loads cached ticker history for the given pair. + """ + + pairdata = load_tickerdata_file(datadir, pair, ticker_interval, timerange=timerange) + if pairdata: + if timerange.starttype == 'date' and pairdata[0][0] > timerange.startts * 1000: + logger.warning('Missing data at start for pair %s, data starts at %s', + pair, arrow.get(pairdata[0][0] // 1000).strftime('%Y-%m-%d %H:%M:%S')) + if timerange.stoptype == 'date' and pairdata[-1][0] < timerange.stopts * 1000: + logger.warning('Missing data at end for pair %s, data ends at %s', + pair, + arrow.get(pairdata[-1][0] // 1000).strftime('%Y-%m-%d %H:%M:%S')) + return parse_ticker_dataframe(pairdata) + else: + logger.warning('No data for pair: "%s", Interval: %s. ' + 'Use --refresh-pairs-cached to download the data', + pair, ticker_interval) + return None + + def load_data(datadir: Optional[Path], ticker_interval: str, pairs: List[str], @@ -116,22 +141,10 @@ def load_data(datadir: Optional[Path], download_pairs(datadir, exchange, pairs, ticker_interval, timerange=timerange) for pair in pairs: - pairdata = load_tickerdata_file(datadir, pair, ticker_interval, timerange=timerange) - if pairdata: - if timerange.starttype == 'date' and pairdata[0][0] > timerange.startts * 1000: - logger.warning('Missing data at start for pair %s, data starts at %s', - pair, - arrow.get(pairdata[0][0] // 1000).strftime('%Y-%m-%d %H:%M:%S')) - if timerange.stoptype == 'date' and pairdata[-1][0] < timerange.stopts * 1000: - logger.warning('Missing data at end for pair %s, data ends at %s', - pair, - arrow.get(pairdata[-1][0] // 1000).strftime('%Y-%m-%d %H:%M:%S')) - result[pair] = parse_ticker_dataframe(pairdata) - else: - logger.warning('No data for pair: "%s", Interval: %s. ' - 'Use --refresh-pairs-cached to download the data', - pair, ticker_interval) - + hist = load_pair_history(pair=pair, ticker_interval=ticker_interval, + datadir=datadir, timerange=timerange) + if hist is not None: + result[pair] = hist return result diff --git a/freqtrade/tests/data/test_history.py b/freqtrade/tests/data/test_history.py index a40021fba..8e7185b33 100644 --- a/freqtrade/tests/data/test_history.py +++ b/freqtrade/tests/data/test_history.py @@ -7,6 +7,7 @@ import uuid from shutil import copyfile import arrow +from pandas import DataFrame from freqtrade.arguments import TimeRange from freqtrade.data import history @@ -54,26 +55,19 @@ def _clean_test_file(file: str) -> None: os.rename(file_swp, file) -def test_load_data_30min_ticker(ticker_history, mocker, caplog, default_conf) -> None: - mocker.patch('freqtrade.exchange.Exchange.get_history', return_value=ticker_history) - file = os.path.join(os.path.dirname(__file__), '..', 'testdata', 'UNITTEST_BTC-30m.json') - _backup_file(file, copy_file=True) - ld = history.load_data(datadir=None, pairs=['UNITTEST/BTC'], ticker_interval='30m') - assert isinstance(ld, dict) - assert os.path.isfile(file) is True +def test_load_data_30min_ticker(mocker, caplog, default_conf) -> None: + ld = history.load_pair_history(pair='UNITTEST/BTC', ticker_interval='30m', datadir=None) + assert isinstance(ld, DataFrame) assert not log_has('Download the pair: "UNITTEST/BTC", Interval: 30m', caplog.record_tuples) - _clean_test_file(file) -def test_load_data_5min_ticker(ticker_history, mocker, caplog, default_conf) -> None: - mocker.patch('freqtrade.exchange.Exchange.get_history', return_value=ticker_history) - - file = os.path.join(os.path.dirname(__file__), '..', 'testdata', 'UNITTEST_BTC-5m.json') - _backup_file(file, copy_file=True) - history.load_data(datadir=None, pairs=['UNITTEST/BTC'], ticker_interval='5m') - assert os.path.isfile(file) is True - assert not log_has('Download the pair: "UNITTEST/BTC", Interval: 5m', caplog.record_tuples) - _clean_test_file(file) +def test_load_data_7min_ticker(mocker, caplog, default_conf) -> None: + ld = history.load_pair_history(pair='UNITTEST/BTC', ticker_interval='7m', datadir=None) + assert not isinstance(ld, DataFrame) + assert ld is None + assert log_has( + 'No data for pair: "UNITTEST/BTC", Interval: 7m. ' + 'Use --refresh-pairs-cached to download the data', caplog.record_tuples) def test_load_data_1min_ticker(ticker_history, mocker, caplog) -> None: From f5b2430cdaa2c7ad80d033d295b8ecb36b2ab619 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 16 Dec 2018 09:58:46 +0100 Subject: [PATCH 649/699] Fix docstrings and typo --- freqtrade/data/__init__.py | 4 ++++ freqtrade/data/converter.py | 2 +- freqtrade/data/history.py | 7 ++++++- 3 files changed, 11 insertions(+), 2 deletions(-) diff --git a/freqtrade/data/__init__.py b/freqtrade/data/__init__.py index c7d0b5983..f0d6a7bad 100644 --- a/freqtrade/data/__init__.py +++ b/freqtrade/data/__init__.py @@ -1,3 +1,7 @@ +""" +Module to handle data operations for freqtrade bot +""" +# limit what's imported when using `from freqtrad.data import *`` __all__ = [ 'convert' ] diff --git a/freqtrade/data/converter.py b/freqtrade/data/converter.py index 18b8fd84c..ee915ec19 100644 --- a/freqtrade/data/converter.py +++ b/freqtrade/data/converter.py @@ -32,7 +32,7 @@ def parse_ticker_dataframe(ticker: list) -> DataFrame: 'volume': 'max', }) frame.drop(frame.tail(1).index, inplace=True) # eliminate partial candle - logger.debug('Droppling last candle') + logger.debug('Dropping last candle') return frame diff --git a/freqtrade/data/history.py b/freqtrade/data/history.py index 11b759aa5..c4655b5f6 100644 --- a/freqtrade/data/history.py +++ b/freqtrade/data/history.py @@ -1,4 +1,9 @@ -# pragma pylint: disable=missing-docstring +""" +Handle historic data (ohlcv). +includes: +* load data for a pair (or a list of pairs) from disk +* download data from exchange and store to disk +""" import gzip try: From ebb80b690612b168e97cdf6caaba3518e65379be Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 16 Dec 2018 09:58:54 +0100 Subject: [PATCH 650/699] remove ujson / json fallback hack as it's now in requirements --- freqtrade/data/history.py | 20 ++++++++------------ 1 file changed, 8 insertions(+), 12 deletions(-) diff --git a/freqtrade/data/history.py b/freqtrade/data/history.py index c4655b5f6..a5a3f52da 100644 --- a/freqtrade/data/history.py +++ b/freqtrade/data/history.py @@ -6,19 +6,14 @@ includes: """ import gzip -try: - import ujson as json - _UJSON = True -except ImportError: - # see mypy/issues/1153 - import json # type: ignore - _UJSON = False + import logging from pathlib import Path from typing import Optional, List, Dict, Tuple, Any import arrow from pandas import DataFrame +import ujson from freqtrade import misc, constants, OperationalException from freqtrade.data.converter import parse_ticker_dataframe @@ -29,11 +24,12 @@ logger = logging.getLogger(__name__) def json_load(data): - """Try to load data with ujson""" - if _UJSON: - return json.load(data, precise_float=True) - else: - return json.load(data) + """ + load data with ujson + Use this to have a consistent experience, + otherwise "precise_float" needs to be passed to all load operations + """ + return ujson.load(data, precise_float=True) def trim_tickerlist(tickerlist: List[Dict], timerange: TimeRange) -> List[Dict]: From 043cefd60af23841a08e73c4d6cee5c875c104b9 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 16 Dec 2018 10:17:11 +0100 Subject: [PATCH 651/699] allow reloading single pair --- freqtrade/data/history.py | 36 ++++++++++++++++++---------- freqtrade/tests/data/test_history.py | 28 ++++++++++++++-------- scripts/download_backtest_data.py | 2 +- 3 files changed, 42 insertions(+), 24 deletions(-) diff --git a/freqtrade/data/history.py b/freqtrade/data/history.py index a5a3f52da..e8c5e7816 100644 --- a/freqtrade/data/history.py +++ b/freqtrade/data/history.py @@ -99,12 +99,28 @@ def load_tickerdata_file( def load_pair_history(pair: str, ticker_interval: str, datadir: Optional[Path], - timerange: TimeRange = TimeRange(None, None, 0, 0)) -> DataFrame: + timerange: TimeRange = TimeRange(None, None, 0, 0), + refresh_pairs: bool = False, + exchange: Optional[Exchange] = None, + ) -> DataFrame: """ Loads cached ticker history for the given pair. """ pairdata = load_tickerdata_file(datadir, pair, ticker_interval, timerange=timerange) + # If the user force the refresh of pairs + if refresh_pairs: + if not exchange: + raise OperationalException("Exchange needs to be initialized when " + "calling load_data with refresh_pairs=True") + + logger.info('Download data for all pairs and store them in %s', datadir) + download_backtesting_testdata(datadir=datadir, + exchange=exchange, + pair=pair, + tick_interval=ticker_interval, + timerange=timerange) + if pairdata: if timerange.starttype == 'date' and pairdata[0][0] > timerange.startts * 1000: logger.warning('Missing data at start for pair %s, data starts at %s', @@ -124,26 +140,20 @@ def load_pair_history(pair: str, def load_data(datadir: Optional[Path], ticker_interval: str, pairs: List[str], - refresh_pairs: Optional[bool] = False, + refresh_pairs: bool = False, exchange: Optional[Exchange] = None, timerange: TimeRange = TimeRange(None, None, 0, 0)) -> Dict[str, DataFrame]: """ - Loads ticker history data for the given parameters + Loads ticker history data for a list of pairs the given parameters :return: dict """ result = {} - # If the user force the refresh of pairs - if refresh_pairs: - logger.info('Download data for all pairs and store them in %s', datadir) - if not exchange: - raise OperationalException("Exchange needs to be initialized when " - "calling load_data with refresh_pairs=True") - download_pairs(datadir, exchange, pairs, ticker_interval, timerange=timerange) - for pair in pairs: hist = load_pair_history(pair=pair, ticker_interval=ticker_interval, - datadir=datadir, timerange=timerange) + datadir=datadir, timerange=timerange, + refresh_pairs=refresh_pairs, + exchange=exchange) if hist is not None: result[pair] = hist return result @@ -160,7 +170,7 @@ def download_pairs(datadir, exchange: Exchange, pairs: List[str], """For each pairs passed in parameters, download the ticker intervals""" for pair in pairs: try: - download_backtesting_testdata(datadir, + download_backtesting_testdata(datadir=datadir, exchange=exchange, pair=pair, tick_interval=ticker_interval, diff --git a/freqtrade/tests/data/test_history.py b/freqtrade/tests/data/test_history.py index 8e7185b33..908f17df7 100644 --- a/freqtrade/tests/data/test_history.py +++ b/freqtrade/tests/data/test_history.py @@ -8,7 +8,9 @@ from shutil import copyfile import arrow from pandas import DataFrame +import pytest +from freqtrade import OperationalException from freqtrade.arguments import TimeRange from freqtrade.data import history from freqtrade.data.history import (download_backtesting_testdata, @@ -82,7 +84,7 @@ def test_load_data_1min_ticker(ticker_history, mocker, caplog) -> None: def test_load_data_with_new_pair_1min(ticker_history_list, mocker, caplog, default_conf) -> None: """ - Test load_data() with 1 min ticker + Test load_pair_history() with 1 min ticker """ mocker.patch('freqtrade.exchange.Exchange.get_history', return_value=ticker_history_list) exchange = get_patched_exchange(mocker, default_conf) @@ -90,23 +92,29 @@ def test_load_data_with_new_pair_1min(ticker_history_list, mocker, caplog, defau _backup_file(file) # do not download a new pair if refresh_pairs isn't set - history.load_data(datadir=None, - ticker_interval='1m', - refresh_pairs=False, - pairs=['MEME/BTC']) + history.load_pair_history(datadir=None, + ticker_interval='1m', + refresh_pairs=False, + pair='MEME/BTC') assert os.path.isfile(file) is False assert log_has('No data for pair: "MEME/BTC", Interval: 1m. ' 'Use --refresh-pairs-cached to download the data', caplog.record_tuples) # download a new pair if refresh_pairs is set - history.load_data(datadir=None, - ticker_interval='1m', - refresh_pairs=True, - exchange=exchange, - pairs=['MEME/BTC']) + history.load_pair_history(datadir=None, + ticker_interval='1m', + refresh_pairs=True, + exchange=exchange, + pair='MEME/BTC') assert os.path.isfile(file) is True assert log_has('Download the pair: "MEME/BTC", Interval: 1m', caplog.record_tuples) + with pytest.raises(OperationalException, match=r'Exchange needs to be initialized when.*'): + history.load_pair_history(datadir=None, + ticker_interval='1m', + refresh_pairs=True, + exchange=None, + pair='MEME/BTC') _clean_test_file(file) diff --git a/scripts/download_backtest_data.py b/scripts/download_backtest_data.py index 6235fe667..c32ac1018 100755 --- a/scripts/download_backtest_data.py +++ b/scripts/download_backtest_data.py @@ -82,7 +82,7 @@ for pair in PAIRS: dl_file.unlink() print(f'downloading pair {pair}, interval {tick_interval}') - download_backtesting_testdata(dl_path, exchange=exchange, + download_backtesting_testdata(datadir=dl_path, exchange=exchange, pair=pair, tick_interval=tick_interval, timerange=timerange) From 8826a1df5f9bcf17afa449bebdc602c1bb9cccf1 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 16 Dec 2018 10:19:49 +0100 Subject: [PATCH 652/699] Add missing tests for trim_tickerlist --- freqtrade/tests/data/test_history.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/freqtrade/tests/data/test_history.py b/freqtrade/tests/data/test_history.py index 908f17df7..53e03c186 100644 --- a/freqtrade/tests/data/test_history.py +++ b/freqtrade/tests/data/test_history.py @@ -450,6 +450,19 @@ def test_trim_tickerlist() -> None: assert ticker_list_len == ticker_len + # Test invalid timerange (start after stop) + timerange = TimeRange('index', 'index', 10, 5) + with pytest.raises(ValueError, match=r'The timerange .* is incorrect'): + trim_tickerlist(ticker_list, timerange) + + assert ticker_list_len == ticker_len + + # passing empty list + timerange = TimeRange(None, None, None, 5) + ticker = trim_tickerlist([], timerange) + assert 0 == len(ticker) + assert not ticker + def test_file_dump_json() -> None: file = os.path.join(os.path.dirname(__file__), '..', 'testdata', From 8bd4d03e135bfea5ebb81fbbc469d0ee6624389a Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 16 Dec 2018 10:29:53 +0100 Subject: [PATCH 653/699] remove download_pairs --- freqtrade/data/history.py | 60 ++++++++++++++++----------------------- 1 file changed, 24 insertions(+), 36 deletions(-) diff --git a/freqtrade/data/history.py b/freqtrade/data/history.py index e8c5e7816..ca645df8c 100644 --- a/freqtrade/data/history.py +++ b/freqtrade/data/history.py @@ -164,24 +164,6 @@ def make_testdata_path(datadir: Optional[Path]) -> Path: return datadir or (Path(__file__).parent.parent / "tests" / "testdata").resolve() -def download_pairs(datadir, exchange: Exchange, pairs: List[str], - ticker_interval: str, - timerange: TimeRange = TimeRange(None, None, 0, 0)) -> bool: - """For each pairs passed in parameters, download the ticker intervals""" - for pair in pairs: - try: - download_backtesting_testdata(datadir=datadir, - exchange=exchange, - pair=pair, - tick_interval=ticker_interval, - timerange=timerange) - except BaseException: - logger.info('Failed to download the pair: "%s", Interval: %s', - pair, ticker_interval) - return False - return True - - def load_cached_data_for_updating(filename: Path, tick_interval: str, timerange: Optional[TimeRange]) -> Tuple[List[Any], Optional[int]]: @@ -220,11 +202,11 @@ def load_cached_data_for_updating(filename: Path, tick_interval: str, return (data, since_ms) -def download_backtesting_testdata(datadir: Path, +def download_backtesting_testdata(datadir: Optional[Path], exchange: Exchange, pair: str, tick_interval: str = '5m', - timerange: Optional[TimeRange] = None) -> None: + timerange: Optional[TimeRange] = None) -> bool: """ Download the latest ticker intervals from the exchange for the pair passed in parameters The data is downloaded starting from the last correct ticker interval data that @@ -238,25 +220,31 @@ def download_backtesting_testdata(datadir: Path, :return: None """ - path = make_testdata_path(datadir) - filepair = pair.replace("/", "_") - filename = path.joinpath(f'{filepair}-{tick_interval}.json') + try: + path = make_testdata_path(datadir) + filepair = pair.replace("/", "_") + filename = path.joinpath(f'{filepair}-{tick_interval}.json') - logger.info('Download the pair: "%s", Interval: %s', pair, tick_interval) + logger.info('Download the pair: "%s", Interval: %s', pair, tick_interval) - data, since_ms = load_cached_data_for_updating(filename, tick_interval, timerange) + data, since_ms = load_cached_data_for_updating(filename, tick_interval, timerange) - logger.debug("Current Start: %s", misc.format_ms_time(data[1][0]) if data else 'None') - logger.debug("Current End: %s", misc.format_ms_time(data[-1][0]) if data else 'None') + logger.debug("Current Start: %s", misc.format_ms_time(data[1][0]) if data else 'None') + logger.debug("Current End: %s", misc.format_ms_time(data[-1][0]) if data else 'None') - # Default since_ms to 30 days if nothing is given - new_data = exchange.get_history(pair=pair, tick_interval=tick_interval, - since_ms=since_ms if since_ms - else - int(arrow.utcnow().shift(days=-30).float_timestamp) * 1000) - data.extend(new_data) + # Default since_ms to 30 days if nothing is given + new_data = exchange.get_history(pair=pair, tick_interval=tick_interval, + since_ms=since_ms if since_ms + else + int(arrow.utcnow().shift(days=-30).float_timestamp) * 1000) + data.extend(new_data) - logger.debug("New Start: %s", misc.format_ms_time(data[0][0])) - logger.debug("New End: %s", misc.format_ms_time(data[-1][0])) + logger.debug("New Start: %s", misc.format_ms_time(data[0][0])) + logger.debug("New End: %s", misc.format_ms_time(data[-1][0])) - misc.file_dump_json(filename, data) + misc.file_dump_json(filename, data) + return True + except BaseException: + logger.info('Failed to download the pair: "%s", Interval: %s', + pair, tick_interval) + return False From 50938d410a235aa2b17495e29c1fb7e0805082e6 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 16 Dec 2018 10:30:13 +0100 Subject: [PATCH 654/699] Remove tests for download_pairs --- freqtrade/tests/data/test_history.py | 134 ++++++++++++--------------- 1 file changed, 60 insertions(+), 74 deletions(-) diff --git a/freqtrade/tests/data/test_history.py b/freqtrade/tests/data/test_history.py index 53e03c186..007153a19 100644 --- a/freqtrade/tests/data/test_history.py +++ b/freqtrade/tests/data/test_history.py @@ -14,7 +14,6 @@ from freqtrade import OperationalException from freqtrade.arguments import TimeRange from freqtrade.data import history from freqtrade.data.history import (download_backtesting_testdata, - download_pairs, load_cached_data_for_updating, load_tickerdata_file, make_testdata_path, @@ -122,46 +121,6 @@ def test_testdata_path() -> None: assert str(Path('freqtrade') / 'tests' / 'testdata') in str(make_testdata_path(None)) -def test_download_pairs(ticker_history_list, mocker, default_conf) -> None: - mocker.patch('freqtrade.exchange.Exchange.get_history', return_value=ticker_history_list) - exchange = get_patched_exchange(mocker, default_conf) - file1_1 = os.path.join(os.path.dirname(__file__), '..', 'testdata', 'MEME_BTC-1m.json') - file1_5 = os.path.join(os.path.dirname(__file__), '..', 'testdata', 'MEME_BTC-5m.json') - file2_1 = os.path.join(os.path.dirname(__file__), '..', 'testdata', 'CFI_BTC-1m.json') - file2_5 = os.path.join(os.path.dirname(__file__), '..', 'testdata', 'CFI_BTC-5m.json') - - _backup_file(file1_1) - _backup_file(file1_5) - _backup_file(file2_1) - _backup_file(file2_5) - - assert os.path.isfile(file1_1) is False - assert os.path.isfile(file2_1) is False - - assert download_pairs(None, exchange, - pairs=['MEME/BTC', 'CFI/BTC'], ticker_interval='1m') is True - - assert os.path.isfile(file1_1) is True - assert os.path.isfile(file2_1) is True - - # clean files freshly downloaded - _clean_test_file(file1_1) - _clean_test_file(file2_1) - - assert os.path.isfile(file1_5) is False - assert os.path.isfile(file2_5) is False - - assert download_pairs(None, exchange, - pairs=['MEME/BTC', 'CFI/BTC'], ticker_interval='5m') is True - - assert os.path.isfile(file1_5) is True - assert os.path.isfile(file2_5) is True - - # clean files freshly downloaded - _clean_test_file(file1_5) - _clean_test_file(file2_5) - - def test_load_cached_data_for_updating(mocker) -> None: datadir = Path(__file__).parent.parent.joinpath('testdata') @@ -265,45 +224,52 @@ def test_load_cached_data_for_updating(mocker) -> None: assert start_ts is None -def test_download_pairs_exception(ticker_history, mocker, caplog, default_conf) -> None: - mocker.patch('freqtrade.exchange.Exchange.get_history', return_value=ticker_history) - mocker.patch('freqtrade.data.history.download_backtesting_testdata', - side_effect=BaseException('File Error')) - exchange = get_patched_exchange(mocker, default_conf) - - file1_1 = os.path.join(os.path.dirname(__file__), '..', 'testdata', 'MEME_BTC-1m.json') - file1_5 = os.path.join(os.path.dirname(__file__), '..', 'testdata', 'MEME_BTC-5m.json') - _backup_file(file1_1) - _backup_file(file1_5) - - download_pairs(None, exchange, pairs=['MEME/BTC'], ticker_interval='1m') - # clean files freshly downloaded - _clean_test_file(file1_1) - _clean_test_file(file1_5) - assert log_has('Failed to download the pair: "MEME/BTC", Interval: 1m', caplog.record_tuples) - - def test_download_backtesting_testdata(ticker_history_list, mocker, default_conf) -> None: mocker.patch('freqtrade.exchange.Exchange.get_history', return_value=ticker_history_list) exchange = get_patched_exchange(mocker, default_conf) - # Tst that pairs-cached is not touched. - assert not exchange._pairs_last_refresh_time - # Download a 1 min ticker file - file1 = os.path.join(os.path.dirname(__file__), '..', 'testdata', 'XEL_BTC-1m.json') - _backup_file(file1) - download_backtesting_testdata(None, exchange, pair="XEL/BTC", tick_interval='1m') - assert os.path.isfile(file1) is True - _clean_test_file(file1) - assert not exchange._pairs_last_refresh_time + file1_1 = os.path.join(os.path.dirname(__file__), '..', 'testdata', 'MEME_BTC-1m.json') + file1_5 = os.path.join(os.path.dirname(__file__), '..', 'testdata', 'MEME_BTC-5m.json') + file2_1 = os.path.join(os.path.dirname(__file__), '..', 'testdata', 'CFI_BTC-1m.json') + file2_5 = os.path.join(os.path.dirname(__file__), '..', 'testdata', 'CFI_BTC-5m.json') - # Download a 5 min ticker file - file2 = os.path.join(os.path.dirname(__file__), '..', 'testdata', 'STORJ_BTC-5m.json') - _backup_file(file2) + _backup_file(file1_1) + _backup_file(file1_5) + _backup_file(file2_1) + _backup_file(file2_5) - download_backtesting_testdata(None, exchange, pair="STORJ/BTC", tick_interval='5m') - assert os.path.isfile(file2) is True - _clean_test_file(file2) + assert os.path.isfile(file1_1) is False + assert os.path.isfile(file2_1) is False + + assert download_backtesting_testdata(datadir=None, exchange=exchange, + pair='MEME/BTC', + tick_interval='1m') + assert download_backtesting_testdata(datadir=None, exchange=exchange, + pair='CFI/BTC', + tick_interval='1m') assert not exchange._pairs_last_refresh_time + assert os.path.isfile(file1_1) is True + assert os.path.isfile(file2_1) is True + + # clean files freshly downloaded + _clean_test_file(file1_1) + _clean_test_file(file2_1) + + assert os.path.isfile(file1_5) is False + assert os.path.isfile(file2_5) is False + + assert download_backtesting_testdata(datadir=None, exchange=exchange, + pair='MEME/BTC', + tick_interval='5m') + assert download_backtesting_testdata(datadir=None, exchange=exchange, + pair='CFI/BTC', + tick_interval='5m') + assert not exchange._pairs_last_refresh_time + assert os.path.isfile(file1_5) is True + assert os.path.isfile(file2_5) is True + + # clean files freshly downloaded + _clean_test_file(file1_5) + _clean_test_file(file2_5) def test_download_backtesting_testdata2(mocker, default_conf) -> None: @@ -319,6 +285,26 @@ def test_download_backtesting_testdata2(mocker, default_conf) -> None: assert json_dump_mock.call_count == 2 +def test_download_backtesting_data_exception(ticker_history, mocker, caplog, default_conf) -> None: + mocker.patch('freqtrade.exchange.Exchange.get_history', + side_effect=BaseException('File Error')) + + exchange = get_patched_exchange(mocker, default_conf) + + file1_1 = os.path.join(os.path.dirname(__file__), '..', 'testdata', 'MEME_BTC-1m.json') + file1_5 = os.path.join(os.path.dirname(__file__), '..', 'testdata', 'MEME_BTC-5m.json') + _backup_file(file1_1) + _backup_file(file1_5) + + assert not download_backtesting_testdata(datadir=None, exchange=exchange, + pair='MEME/BTC', + tick_interval='1m') + # clean files freshly downloaded + _clean_test_file(file1_1) + _clean_test_file(file1_5) + assert log_has('Failed to download the pair: "MEME/BTC", Interval: 1m', caplog.record_tuples) + + def test_load_tickerdata_file() -> None: # 7 does not exist in either format. assert not load_tickerdata_file(None, 'UNITTEST/BTC', '7m') From eb7034c7a7589f6ddc80a729e078ffe5e77e83bb Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 16 Dec 2018 10:33:08 +0100 Subject: [PATCH 655/699] Rename download_backtest_testdata to download_pair_history --- freqtrade/data/history.py | 20 +++++++------- freqtrade/tests/data/test_history.py | 40 ++++++++++++++-------------- scripts/download_backtest_data.py | 10 +++---- 3 files changed, 35 insertions(+), 35 deletions(-) diff --git a/freqtrade/data/history.py b/freqtrade/data/history.py index ca645df8c..79f45223d 100644 --- a/freqtrade/data/history.py +++ b/freqtrade/data/history.py @@ -115,11 +115,11 @@ def load_pair_history(pair: str, "calling load_data with refresh_pairs=True") logger.info('Download data for all pairs and store them in %s', datadir) - download_backtesting_testdata(datadir=datadir, - exchange=exchange, - pair=pair, - tick_interval=ticker_interval, - timerange=timerange) + download_pair_history(datadir=datadir, + exchange=exchange, + pair=pair, + tick_interval=ticker_interval, + timerange=timerange) if pairdata: if timerange.starttype == 'date' and pairdata[0][0] > timerange.startts * 1000: @@ -202,11 +202,11 @@ def load_cached_data_for_updating(filename: Path, tick_interval: str, return (data, since_ms) -def download_backtesting_testdata(datadir: Optional[Path], - exchange: Exchange, - pair: str, - tick_interval: str = '5m', - timerange: Optional[TimeRange] = None) -> bool: +def download_pair_history(datadir: Optional[Path], + exchange: Exchange, + pair: str, + tick_interval: str = '5m', + timerange: Optional[TimeRange] = None) -> bool: """ Download the latest ticker intervals from the exchange for the pair passed in parameters The data is downloaded starting from the last correct ticker interval data that diff --git a/freqtrade/tests/data/test_history.py b/freqtrade/tests/data/test_history.py index 007153a19..8923a60b6 100644 --- a/freqtrade/tests/data/test_history.py +++ b/freqtrade/tests/data/test_history.py @@ -13,7 +13,7 @@ import pytest from freqtrade import OperationalException from freqtrade.arguments import TimeRange from freqtrade.data import history -from freqtrade.data.history import (download_backtesting_testdata, +from freqtrade.data.history import (download_pair_history, load_cached_data_for_updating, load_tickerdata_file, make_testdata_path, @@ -224,7 +224,7 @@ def test_load_cached_data_for_updating(mocker) -> None: assert start_ts is None -def test_download_backtesting_testdata(ticker_history_list, mocker, default_conf) -> None: +def test_download_pair_history(ticker_history_list, mocker, default_conf) -> None: mocker.patch('freqtrade.exchange.Exchange.get_history', return_value=ticker_history_list) exchange = get_patched_exchange(mocker, default_conf) file1_1 = os.path.join(os.path.dirname(__file__), '..', 'testdata', 'MEME_BTC-1m.json') @@ -240,12 +240,12 @@ def test_download_backtesting_testdata(ticker_history_list, mocker, default_conf assert os.path.isfile(file1_1) is False assert os.path.isfile(file2_1) is False - assert download_backtesting_testdata(datadir=None, exchange=exchange, - pair='MEME/BTC', - tick_interval='1m') - assert download_backtesting_testdata(datadir=None, exchange=exchange, - pair='CFI/BTC', - tick_interval='1m') + assert download_pair_history(datadir=None, exchange=exchange, + pair='MEME/BTC', + tick_interval='1m') + assert download_pair_history(datadir=None, exchange=exchange, + pair='CFI/BTC', + tick_interval='1m') assert not exchange._pairs_last_refresh_time assert os.path.isfile(file1_1) is True assert os.path.isfile(file2_1) is True @@ -257,12 +257,12 @@ def test_download_backtesting_testdata(ticker_history_list, mocker, default_conf assert os.path.isfile(file1_5) is False assert os.path.isfile(file2_5) is False - assert download_backtesting_testdata(datadir=None, exchange=exchange, - pair='MEME/BTC', - tick_interval='5m') - assert download_backtesting_testdata(datadir=None, exchange=exchange, - pair='CFI/BTC', - tick_interval='5m') + assert download_pair_history(datadir=None, exchange=exchange, + pair='MEME/BTC', + tick_interval='5m') + assert download_pair_history(datadir=None, exchange=exchange, + pair='CFI/BTC', + tick_interval='5m') assert not exchange._pairs_last_refresh_time assert os.path.isfile(file1_5) is True assert os.path.isfile(file2_5) is True @@ -272,7 +272,7 @@ def test_download_backtesting_testdata(ticker_history_list, mocker, default_conf _clean_test_file(file2_5) -def test_download_backtesting_testdata2(mocker, default_conf) -> None: +def test_download_pair_history2(mocker, default_conf) -> None: tick = [ [1509836520000, 0.00162008, 0.00162008, 0.00162008, 0.00162008, 108.14853839], [1509836580000, 0.00161, 0.00161, 0.00161, 0.00161, 82.390199] @@ -280,8 +280,8 @@ def test_download_backtesting_testdata2(mocker, default_conf) -> None: json_dump_mock = mocker.patch('freqtrade.misc.file_dump_json', return_value=None) mocker.patch('freqtrade.exchange.Exchange.get_history', return_value=tick) exchange = get_patched_exchange(mocker, default_conf) - download_backtesting_testdata(None, exchange, pair="UNITTEST/BTC", tick_interval='1m') - download_backtesting_testdata(None, exchange, pair="UNITTEST/BTC", tick_interval='3m') + download_pair_history(None, exchange, pair="UNITTEST/BTC", tick_interval='1m') + download_pair_history(None, exchange, pair="UNITTEST/BTC", tick_interval='3m') assert json_dump_mock.call_count == 2 @@ -296,9 +296,9 @@ def test_download_backtesting_data_exception(ticker_history, mocker, caplog, def _backup_file(file1_1) _backup_file(file1_5) - assert not download_backtesting_testdata(datadir=None, exchange=exchange, - pair='MEME/BTC', - tick_interval='1m') + assert not download_pair_history(datadir=None, exchange=exchange, + pair='MEME/BTC', + tick_interval='1m') # clean files freshly downloaded _clean_test_file(file1_1) _clean_test_file(file1_5) diff --git a/scripts/download_backtest_data.py b/scripts/download_backtest_data.py index c32ac1018..2b4ca03d1 100755 --- a/scripts/download_backtest_data.py +++ b/scripts/download_backtest_data.py @@ -9,7 +9,7 @@ import arrow from freqtrade import arguments from freqtrade.arguments import TimeRange from freqtrade.exchange import Exchange -from freqtrade.data.history import download_backtesting_testdata +from freqtrade.data.history import download_pair_history from freqtrade.configuration import set_loggers import logging @@ -82,10 +82,10 @@ for pair in PAIRS: dl_file.unlink() print(f'downloading pair {pair}, interval {tick_interval}') - download_backtesting_testdata(datadir=dl_path, exchange=exchange, - pair=pair, - tick_interval=tick_interval, - timerange=timerange) + download_pair_history(datadir=dl_path, exchange=exchange, + pair=pair, + tick_interval=tick_interval, + timerange=timerange) if pairs_not_available: From 806ab3729f0b9a6f42757ff2dd604d6158bc8ebf Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 16 Dec 2018 14:14:17 +0100 Subject: [PATCH 656/699] Add / fix some comments --- freqtrade/data/__init__.py | 3 ++- freqtrade/data/history.py | 15 ++++++++------- 2 files changed, 10 insertions(+), 8 deletions(-) diff --git a/freqtrade/data/__init__.py b/freqtrade/data/__init__.py index f0d6a7bad..12e80d7a2 100644 --- a/freqtrade/data/__init__.py +++ b/freqtrade/data/__init__.py @@ -1,6 +1,7 @@ """ -Module to handle data operations for freqtrade bot +Module to handle data operations for freqtrade """ + # limit what's imported when using `from freqtrad.data import *`` __all__ = [ 'convert' diff --git a/freqtrade/data/history.py b/freqtrade/data/history.py index 79f45223d..f4ff46a1a 100644 --- a/freqtrade/data/history.py +++ b/freqtrade/data/history.py @@ -69,10 +69,10 @@ def trim_tickerlist(tickerlist: List[Dict], timerange: TimeRange) -> List[Dict]: def load_tickerdata_file( datadir: Optional[Path], pair: str, ticker_interval: str, - timerange: Optional[TimeRange] = None) -> Optional[List[Dict]]: + timerange: Optional[TimeRange] = None) -> Optional[list]: """ Load a pair from file, either .json.gz or .json - :return dict(:) or None if unsuccesful + :return tickerlist or None if unsuccesful """ path = make_testdata_path(datadir) pair_s = pair.replace('/', '_') @@ -105,6 +105,7 @@ def load_pair_history(pair: str, ) -> DataFrame: """ Loads cached ticker history for the given pair. + :return: DataFrame with ohlcv data """ pairdata = load_tickerdata_file(datadir, pair, ticker_interval, timerange=timerange) @@ -145,7 +146,7 @@ def load_data(datadir: Optional[Path], timerange: TimeRange = TimeRange(None, None, 0, 0)) -> Dict[str, DataFrame]: """ Loads ticker history data for a list of pairs the given parameters - :return: dict + :return: dict(:) """ result = {} @@ -185,9 +186,9 @@ def load_cached_data_for_updating(filename: Path, tick_interval: str, if filename.is_file(): with open(filename, "rt") as file: data = json_load(file) - # remove the last item, could be incomplete candle - if data: - data.pop() + # remove the last item, could be incomplete candle + if data: + data.pop() else: data = [] @@ -217,7 +218,7 @@ def download_pair_history(datadir: Optional[Path], :param pair: pair to download :param tick_interval: ticker interval :param timerange: range of time to download - :return: None + :return: bool with success state """ try: From 9d8a3b4ec5ca30ecfe6ff23d05ca0f7aacd386ab Mon Sep 17 00:00:00 2001 From: misagh Date: Sun, 16 Dec 2018 22:02:29 +0100 Subject: [PATCH 657/699] docs added --- docs/configuration.md | 22 +++++++++++++++++++++- 1 file changed, 21 insertions(+), 1 deletion(-) diff --git a/docs/configuration.md b/docs/configuration.md index c4c0bed28..4db6c2e70 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -40,6 +40,7 @@ The table below will list all configuration parameters. | `ask_strategy.order_book_min` | 0 | No | Bot will scan from the top min to max Order Book Asks searching for a profitable rate. | `ask_strategy.order_book_max` | 0 | No | Bot will scan from the top min to max Order Book Asks searching for a profitable rate. | `order_types` | None | No | Configure order-types depending on the action (`"buy"`, `"sell"`, `"stoploss"`, `"stoploss_on_exchange"`). +| `order_time_in_force` | None | No | Configure time in force for buy and sell orders (`"buy"`, `"sell"`). [More info blow](#user-content-understand-order-time_in_force). | `exchange.name` | bittrex | Yes | Name of the exchange class to use. [List below](#user-content-what-values-for-exchangename). | `exchange.key` | key | No | API key to use for the exchange. Only required when you are in production mode. | `exchange.secret` | secret | No | API secret to use for the exchange. Only required when you are in production mode. @@ -161,6 +162,25 @@ The below is the default which is used if this is not configured in either Strat **NOTE**: Not all exchanges support "market" orders. The following message will be shown if your exchange does not support market orders: `"Exchange does not support market orders."` +### Understand order_time_in_force +Order time in force defines the policy by which the order is executed on the exchange. Three commonly used time in force are:
+`GTC (Goog Till Canceled):` +This is most of the time the default time in force. It means the order will remain on exchange till it is canceled by user. It can be fully or partially fulfilled. If partially fulfilled, the remaining will stay on the exchange till cancelled.
+`FOK (Full Or Kill):` +It means if the order is not executed immediately AND fully then it is canceled by the exchange.
+`IOC (Immediate Or Canceled):` +It is the same as FOK (above) except it can be partially fulfilled. The remaining part is automatically cancelled by the exchange. +
+`order_time_in_force` contains a dict buy and sell time in force policy. This can be set in the configuration or in the strategy. Configuration overwrites strategy configurations.
+possible values are: `gtc` (defaul), `fok` or `ioc`.
+``` python + "order_time_in_force": { + "buy": "gtc", + "sell": "gtc" + }, +``` +**NOTE**: This is an ongoing work. For now it is supported only for binance and only for buy orders. Please don't change the default value unless you know what you are doing.
+ ### What values for exchange.name? Freqtrade is based on [CCXT library](https://github.com/ccxt/ccxt) that supports 115 cryptocurrency @@ -215,7 +235,7 @@ production mode. ### Dynamic Pairlists -Dynamic pairlists select pairs for you based on the logic configured. +Dynamic pairlists select pairs for you based on the logic configured. The bot runs against all pairs (with that stake) on the exchange, and a number of assets (`number_assets`) is selected based on the selected criteria. By *default*, a Static Pairlist is used (configured as `"pair_whitelist"` under the `"exchange"` section of this configuration). From a967b8918a70c803e474e4f4f036d9ecf96582ed Mon Sep 17 00:00:00 2001 From: misagh Date: Sun, 16 Dec 2018 22:05:15 +0100 Subject: [PATCH 658/699] broken link corrected --- docs/configuration.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/configuration.md b/docs/configuration.md index 4db6c2e70..3ef39127a 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -40,7 +40,7 @@ The table below will list all configuration parameters. | `ask_strategy.order_book_min` | 0 | No | Bot will scan from the top min to max Order Book Asks searching for a profitable rate. | `ask_strategy.order_book_max` | 0 | No | Bot will scan from the top min to max Order Book Asks searching for a profitable rate. | `order_types` | None | No | Configure order-types depending on the action (`"buy"`, `"sell"`, `"stoploss"`, `"stoploss_on_exchange"`). -| `order_time_in_force` | None | No | Configure time in force for buy and sell orders (`"buy"`, `"sell"`). [More info blow](#user-content-understand-order-time_in_force). +| `order_time_in_force` | None | No | Configure time in force for buy and sell orders (`"buy"`, `"sell"`). [More informatin below](#understand-order_time_in_force). | `exchange.name` | bittrex | Yes | Name of the exchange class to use. [List below](#user-content-what-values-for-exchangename). | `exchange.key` | key | No | API key to use for the exchange. Only required when you are in production mode. | `exchange.secret` | secret | No | API secret to use for the exchange. Only required when you are in production mode. From 2e7028442d08b91059c3e7b4b86a87ff500f6d34 Mon Sep 17 00:00:00 2001 From: misagh Date: Sun, 16 Dec 2018 22:07:03 +0100 Subject: [PATCH 659/699] reformatting --- docs/configuration.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/configuration.md b/docs/configuration.md index 3ef39127a..66572b446 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -164,11 +164,11 @@ The following message will be shown if your exchange does not support market ord ### Understand order_time_in_force Order time in force defines the policy by which the order is executed on the exchange. Three commonly used time in force are:
-`GTC (Goog Till Canceled):` +**GTC (Goog Till Canceled):** This is most of the time the default time in force. It means the order will remain on exchange till it is canceled by user. It can be fully or partially fulfilled. If partially fulfilled, the remaining will stay on the exchange till cancelled.
-`FOK (Full Or Kill):` +**FOK (Full Or Kill):** It means if the order is not executed immediately AND fully then it is canceled by the exchange.
-`IOC (Immediate Or Canceled):` +**IOC (Immediate Or Canceled):** It is the same as FOK (above) except it can be partially fulfilled. The remaining part is automatically cancelled by the exchange.
`order_time_in_force` contains a dict buy and sell time in force policy. This can be set in the configuration or in the strategy. Configuration overwrites strategy configurations.
From f756f1ad28a1ee9f856b8cd20659249ef7063d42 Mon Sep 17 00:00:00 2001 From: misagh Date: Sun, 16 Dec 2018 22:08:50 +0100 Subject: [PATCH 660/699] unnecessary explanation removed. --- docs/configuration.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/configuration.md b/docs/configuration.md index 66572b446..9b2860988 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -40,7 +40,7 @@ The table below will list all configuration parameters. | `ask_strategy.order_book_min` | 0 | No | Bot will scan from the top min to max Order Book Asks searching for a profitable rate. | `ask_strategy.order_book_max` | 0 | No | Bot will scan from the top min to max Order Book Asks searching for a profitable rate. | `order_types` | None | No | Configure order-types depending on the action (`"buy"`, `"sell"`, `"stoploss"`, `"stoploss_on_exchange"`). -| `order_time_in_force` | None | No | Configure time in force for buy and sell orders (`"buy"`, `"sell"`). [More informatin below](#understand-order_time_in_force). +| `order_time_in_force` | None | No | Configure time in force for buy and sell orders. [More informatin below](#understand-order_time_in_force). | `exchange.name` | bittrex | Yes | Name of the exchange class to use. [List below](#user-content-what-values-for-exchangename). | `exchange.key` | key | No | API key to use for the exchange. Only required when you are in production mode. | `exchange.secret` | secret | No | API secret to use for the exchange. Only required when you are in production mode. From 213155e6d394e64fe488e57eb9706c0879b1d3c0 Mon Sep 17 00:00:00 2001 From: misagh Date: Sun, 16 Dec 2018 22:09:46 +0100 Subject: [PATCH 661/699] typo --- docs/configuration.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/configuration.md b/docs/configuration.md index 9b2860988..bd36688a8 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -172,7 +172,7 @@ It means if the order is not executed immediately AND fully then it is canceled It is the same as FOK (above) except it can be partially fulfilled. The remaining part is automatically cancelled by the exchange.
`order_time_in_force` contains a dict buy and sell time in force policy. This can be set in the configuration or in the strategy. Configuration overwrites strategy configurations.
-possible values are: `gtc` (defaul), `fok` or `ioc`.
+possible values are: `gtc` (default), `fok` or `ioc`.
``` python "order_time_in_force": { "buy": "gtc", From c784b829e5a8ae6e4c9215bf7ce0a1da7e41976c Mon Sep 17 00:00:00 2001 From: misagh Date: Sun, 16 Dec 2018 22:11:51 +0100 Subject: [PATCH 662/699] typo --- docs/configuration.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/configuration.md b/docs/configuration.md index bd36688a8..521f330ec 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -40,7 +40,7 @@ The table below will list all configuration parameters. | `ask_strategy.order_book_min` | 0 | No | Bot will scan from the top min to max Order Book Asks searching for a profitable rate. | `ask_strategy.order_book_max` | 0 | No | Bot will scan from the top min to max Order Book Asks searching for a profitable rate. | `order_types` | None | No | Configure order-types depending on the action (`"buy"`, `"sell"`, `"stoploss"`, `"stoploss_on_exchange"`). -| `order_time_in_force` | None | No | Configure time in force for buy and sell orders. [More informatin below](#understand-order_time_in_force). +| `order_time_in_force` | None | No | Configure time in force for buy and sell orders. [More information below](#understand-order_time_in_force). | `exchange.name` | bittrex | Yes | Name of the exchange class to use. [List below](#user-content-what-values-for-exchangename). | `exchange.key` | key | No | API key to use for the exchange. Only required when you are in production mode. | `exchange.secret` | secret | No | API secret to use for the exchange. Only required when you are in production mode. From 7357d6b0895f1a1659a5f92ca1f17e9133b380e3 Mon Sep 17 00:00:00 2001 From: misagh Date: Sun, 16 Dec 2018 22:13:50 +0100 Subject: [PATCH 663/699] adding order type explanation link to doc. --- docs/configuration.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/configuration.md b/docs/configuration.md index c4c0bed28..a4a673228 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -39,7 +39,7 @@ The table below will list all configuration parameters. | `ask_strategy.use_order_book` | false | No | Allows selling of open traded pair using the rates in Order Book Asks. | `ask_strategy.order_book_min` | 0 | No | Bot will scan from the top min to max Order Book Asks searching for a profitable rate. | `ask_strategy.order_book_max` | 0 | No | Bot will scan from the top min to max Order Book Asks searching for a profitable rate. -| `order_types` | None | No | Configure order-types depending on the action (`"buy"`, `"sell"`, `"stoploss"`, `"stoploss_on_exchange"`). +| `order_types` | None | No | Configure order-types depending on the action (`"buy"`, `"sell"`, `"stoploss"`, `"stoploss_on_exchange"`). [More information below](#understand-order_types). | `exchange.name` | bittrex | Yes | Name of the exchange class to use. [List below](#user-content-what-values-for-exchangename). | `exchange.key` | key | No | API key to use for the exchange. Only required when you are in production mode. | `exchange.secret` | secret | No | API secret to use for the exchange. Only required when you are in production mode. @@ -215,7 +215,7 @@ production mode. ### Dynamic Pairlists -Dynamic pairlists select pairs for you based on the logic configured. +Dynamic pairlists select pairs for you based on the logic configured. The bot runs against all pairs (with that stake) on the exchange, and a number of assets (`number_assets`) is selected based on the selected criteria. By *default*, a Static Pairlist is used (configured as `"pair_whitelist"` under the `"exchange"` section of this configuration). From 5493d1a7e080ee0080f4336647d6e5179af0407d Mon Sep 17 00:00:00 2001 From: Matthias Date: Mon, 17 Dec 2018 06:32:59 +0100 Subject: [PATCH 664/699] Fix wonrly named module --- freqtrade/data/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/data/__init__.py b/freqtrade/data/__init__.py index 12e80d7a2..0a31d095c 100644 --- a/freqtrade/data/__init__.py +++ b/freqtrade/data/__init__.py @@ -4,5 +4,5 @@ Module to handle data operations for freqtrade # limit what's imported when using `from freqtrad.data import *`` __all__ = [ - 'convert' + 'converter' ] From 1372095c66888ea083cf3ddbb6480788cbe61b1e Mon Sep 17 00:00:00 2001 From: Pan Long Date: Sun, 2 Dec 2018 21:08:00 +0800 Subject: [PATCH 665/699] Seperate requirements to run the bot and to develop. - Add a requirements-dev.txt file which includes additional deps for development. - Add a Dockerfile.develop which installs all deps for development and also enables dev commands. - Change related documentations on how to run/dev the bot. --- .travis.yml | 2 +- Dockerfile.develop | 9 +++++++++ docs/installation.md | 7 +++++++ requirements-dev.txt | 8 ++++++++ requirements.txt | 7 ------- setup.sh | 10 ++++++++++ 6 files changed, 35 insertions(+), 8 deletions(-) create mode 100644 Dockerfile.develop create mode 100644 requirements-dev.txt diff --git a/.travis.yml b/.travis.yml index f1192e80c..4398a1386 100644 --- a/.travis.yml +++ b/.travis.yml @@ -20,7 +20,7 @@ install: - ./build_helpers/install_ta-lib.sh - export LD_LIBRARY_PATH=/usr/local/lib:$LD_LIBRARY_PATH - pip install --upgrade flake8 coveralls pytest-random-order pytest-asyncio mypy -- pip install -r requirements.txt +- pip install -r requirements-dev.txt - pip install -e . jobs: include: diff --git a/Dockerfile.develop b/Dockerfile.develop new file mode 100644 index 000000000..ef3e207a7 --- /dev/null +++ b/Dockerfile.develop @@ -0,0 +1,9 @@ +FROM freqtrade:latest + +# Install dependencies +COPY requirements-dev.txt /freqtrade/ +RUN pip install numpy --no-cache-dir \ + && pip install -r requirements-dev.txt --no-cache-dir + +# Empty the ENTRYPOINT to allow all commands +ENTRYPOINT [] diff --git a/docs/installation.md b/docs/installation.md index 18406e555..80ab2e35f 100644 --- a/docs/installation.md +++ b/docs/installation.md @@ -134,6 +134,13 @@ cd freqtrade docker build -t freqtrade . ``` +If you are developing using Docker, use `Dockerfile.develop` instead, which will also set up develop dependencies: + +```bash +cd freqtrade +docker build -f ./Dockerfile.develop -t freqtrade . +``` + For security reasons, your configuration file will not be included in the image, you will need to bind mount it. It is also advised to bind mount an SQLite database file (see the "5. Run a restartable docker image" section) to keep it between updates. ### 3. Verify the Docker image diff --git a/requirements-dev.txt b/requirements-dev.txt new file mode 100644 index 000000000..4a7416ca1 --- /dev/null +++ b/requirements-dev.txt @@ -0,0 +1,8 @@ +# Include all requirements to run the bot. +-r requirements.txt + +flake8==3.6.0 +pytest==4.0.2 +pytest-mock==1.10.0 +pytest-asyncio==0.9.0 +pytest-cov==2.6.0 diff --git a/requirements.txt b/requirements.txt index 6ad6cd2d4..99bb722d4 100644 --- a/requirements.txt +++ b/requirements.txt @@ -13,19 +13,12 @@ scipy==1.1.0 jsonschema==2.6.0 numpy==1.15.4 TA-Lib==0.4.17 -pytest==4.0.2 -pytest-mock==1.10.0 -pytest-asyncio==0.9.0 -pytest-cov==2.6.0 tabulate==0.8.2 coinmarketcap==5.0.3 # Required for hyperopt scikit-optimize==0.5.2 -# Required for plotting data -#plotly==3.1.1 - # find first, C search in arrays py_find_1st==1.1.3 diff --git a/setup.sh b/setup.sh index 1bb2ba397..b8e99e679 100755 --- a/setup.sh +++ b/setup.sh @@ -28,6 +28,16 @@ function updateenv () { pip3 install --quiet --upgrade pip pip3 install --quiet -r requirements.txt --upgrade pip3 install --quiet -r requirements.txt + + read -p "Do you want to install dependencies for dev [Y/N]? " + if [[ $REPLY =~ ^[Yy]$ ]] + then + pip3 install --quiet -r requirements-dev.txt --upgrade + pip3 install --quiet -r requirements-dev.txt + else + echo "Dev dependencies ignored." + fi + pip3 install --quiet -e . echo "pip3 install completed" echo From ac9189ebc06442bb7c90c42165dcf714311164ab Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Mon, 17 Dec 2018 13:32:07 +0100 Subject: [PATCH 666/699] Update ccxt from 1.18.37 to 1.18.39 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 6ad6cd2d4..eb2ed5736 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -ccxt==1.18.37 +ccxt==1.18.39 SQLAlchemy==1.2.15 python-telegram-bot==11.1.0 arrow==0.12.1 From 1483593e65cb85b69f4d0530cf80023d8bc27cce Mon Sep 17 00:00:00 2001 From: Pan Long Date: Mon, 17 Dec 2018 07:54:28 -0800 Subject: [PATCH 667/699] Fix instructions on building a dev Docker image. --- docs/installation.md | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/docs/installation.md b/docs/installation.md index 80ab2e35f..3d0ee9468 100644 --- a/docs/installation.md +++ b/docs/installation.md @@ -134,11 +134,10 @@ cd freqtrade docker build -t freqtrade . ``` -If you are developing using Docker, use `Dockerfile.develop` instead, which will also set up develop dependencies: +If you are developing using Docker, use `Dockerfile.develop` to build a dev Docker image, which will also set up develop dependencies: ```bash -cd freqtrade -docker build -f ./Dockerfile.develop -t freqtrade . +docker build -f ./Dockerfile.develop -t freqtrade-dev . ``` For security reasons, your configuration file will not be included in the image, you will need to bind mount it. It is also advised to bind mount an SQLite database file (see the "5. Run a restartable docker image" section) to keep it between updates. From 215ded2e0a6544c802b51abc84fc2e5c39c3b255 Mon Sep 17 00:00:00 2001 From: misagh Date: Mon, 17 Dec 2018 21:30:58 +0100 Subject: [PATCH 668/699] returning last candle close price for a pair --- freqtrade/exchange/__init__.py | 6 ++++++ freqtrade/tests/exchange/test_exchange.py | 3 +++ 2 files changed, 9 insertions(+) diff --git a/freqtrade/exchange/__init__.py b/freqtrade/exchange/__init__.py index e883175e7..4a827fcb5 100644 --- a/freqtrade/exchange/__init__.py +++ b/freqtrade/exchange/__init__.py @@ -164,6 +164,12 @@ class Exchange(object): else: return None + def last_kline_close(self, pair: str): + if pair in self._klines: + return self._klines[pair].iloc[-1]['close'] + else: + return None + def set_sandbox(self, api, exchange_config: dict, name: str): if exchange_config.get('sandbox'): if api.urls.get('test'): diff --git a/freqtrade/tests/exchange/test_exchange.py b/freqtrade/tests/exchange/test_exchange.py index 647440223..6299d8dc8 100644 --- a/freqtrade/tests/exchange/test_exchange.py +++ b/freqtrade/tests/exchange/test_exchange.py @@ -814,6 +814,9 @@ def test_refresh_tickers(mocker, default_conf, caplog) -> None: assert isinstance(exchange.klines(pair), DataFrame) assert len(exchange.klines(pair)) > 0 + # test last kline close price + assert exchange.last_kline_close('XRP/ETH') == 4 + # test caching exchange.refresh_tickers(['IOTA/ETH', 'XRP/ETH'], '5m') From b1e9fa754a0d7117895ab70288d1d59630ea86d2 Mon Sep 17 00:00:00 2001 From: Pan Long Date: Mon, 17 Dec 2018 13:53:22 -0800 Subject: [PATCH 669/699] Base dev Docker image on freqtradeorg/freqtrade:develop. --- Dockerfile.develop | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile.develop b/Dockerfile.develop index ef3e207a7..8f6871c55 100644 --- a/Dockerfile.develop +++ b/Dockerfile.develop @@ -1,4 +1,4 @@ -FROM freqtrade:latest +FROM freqtradeorg/freqtrade:develop # Install dependencies COPY requirements-dev.txt /freqtrade/ From fd4cfefda5a7d1d350352dc22c16d515a6c02981 Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Tue, 18 Dec 2018 13:32:07 +0100 Subject: [PATCH 670/699] Update ccxt from 1.18.39 to 1.18.40 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 2fdf0d234..7c24b878a 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -ccxt==1.18.39 +ccxt==1.18.40 SQLAlchemy==1.2.15 python-telegram-bot==11.1.0 arrow==0.12.1 From 2e06d52240bb6111c059451dfb3d11b819c86fca Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Tue, 18 Dec 2018 13:32:09 +0100 Subject: [PATCH 671/699] Update scipy from 1.1.0 to 1.2.0 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 7c24b878a..c3e34bf7c 100644 --- a/requirements.txt +++ b/requirements.txt @@ -9,7 +9,7 @@ wrapt==1.10.11 pandas==0.23.4 scikit-learn==0.20.1 joblib==0.13.0 -scipy==1.1.0 +scipy==1.2.0 jsonschema==2.6.0 numpy==1.15.4 TA-Lib==0.4.17 From ad4952731a84fb1d11b5bb1f8780b2ae588daa54 Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Wed, 19 Dec 2018 13:32:09 +0100 Subject: [PATCH 672/699] Update ccxt from 1.18.40 to 1.18.46 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index c3e34bf7c..ab60e0489 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -ccxt==1.18.40 +ccxt==1.18.46 SQLAlchemy==1.2.15 python-telegram-bot==11.1.0 arrow==0.12.1 From fc4384c96f143ca84763c0d5a7580a6cde623597 Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Thu, 20 Dec 2018 13:32:07 +0100 Subject: [PATCH 673/699] Update ccxt from 1.18.46 to 1.18.50 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index ab60e0489..db208cc84 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -ccxt==1.18.46 +ccxt==1.18.50 SQLAlchemy==1.2.15 python-telegram-bot==11.1.0 arrow==0.12.1 From 358b5d7e5dd01aba2ca1055ac0946b7a81919ba4 Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Thu, 20 Dec 2018 13:32:09 +0100 Subject: [PATCH 674/699] Update scikit-learn from 0.20.1 to 0.20.2 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index db208cc84..0bd34f848 100644 --- a/requirements.txt +++ b/requirements.txt @@ -7,7 +7,7 @@ requests==2.21.0 urllib3==1.24.1 wrapt==1.10.11 pandas==0.23.4 -scikit-learn==0.20.1 +scikit-learn==0.20.2 joblib==0.13.0 scipy==1.2.0 jsonschema==2.6.0 From a45ec1ed1c7f38ec85d10ec919712e51bea9952c Mon Sep 17 00:00:00 2001 From: misagh Date: Fri, 21 Dec 2018 10:20:01 +0100 Subject: [PATCH 675/699] adding copy as a parameter to klines --- freqtrade/exchange/__init__.py | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/freqtrade/exchange/__init__.py b/freqtrade/exchange/__init__.py index 4a827fcb5..9ea963c65 100644 --- a/freqtrade/exchange/__init__.py +++ b/freqtrade/exchange/__init__.py @@ -158,17 +158,12 @@ class Exchange(object): """exchange ccxt id""" return self._api.id - def klines(self, pair: str) -> DataFrame: + def klines(self, pair: str, copy=True) -> DataFrame: if pair in self._klines: return self._klines[pair].copy() else: return None - def last_kline_close(self, pair: str): - if pair in self._klines: - return self._klines[pair].iloc[-1]['close'] - else: - return None def set_sandbox(self, api, exchange_config: dict, name: str): if exchange_config.get('sandbox'): From a13b30b2de1eb4860a9a47fecdb036837c8616c1 Mon Sep 17 00:00:00 2001 From: misagh Date: Fri, 21 Dec 2018 10:21:31 +0100 Subject: [PATCH 676/699] removing test --- freqtrade/exchange/__init__.py | 1 - freqtrade/tests/exchange/test_exchange.py | 3 --- 2 files changed, 4 deletions(-) diff --git a/freqtrade/exchange/__init__.py b/freqtrade/exchange/__init__.py index 9ea963c65..181e60e8c 100644 --- a/freqtrade/exchange/__init__.py +++ b/freqtrade/exchange/__init__.py @@ -164,7 +164,6 @@ class Exchange(object): else: return None - def set_sandbox(self, api, exchange_config: dict, name: str): if exchange_config.get('sandbox'): if api.urls.get('test'): diff --git a/freqtrade/tests/exchange/test_exchange.py b/freqtrade/tests/exchange/test_exchange.py index 6299d8dc8..647440223 100644 --- a/freqtrade/tests/exchange/test_exchange.py +++ b/freqtrade/tests/exchange/test_exchange.py @@ -814,9 +814,6 @@ def test_refresh_tickers(mocker, default_conf, caplog) -> None: assert isinstance(exchange.klines(pair), DataFrame) assert len(exchange.klines(pair)) > 0 - # test last kline close price - assert exchange.last_kline_close('XRP/ETH') == 4 - # test caching exchange.refresh_tickers(['IOTA/ETH', 'XRP/ETH'], '5m') From 34e3af6ad41225fc532b265240aae18f9d84654c Mon Sep 17 00:00:00 2001 From: misagh Date: Fri, 21 Dec 2018 10:35:17 +0100 Subject: [PATCH 677/699] do not copy DF if copy is false --- freqtrade/exchange/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/exchange/__init__.py b/freqtrade/exchange/__init__.py index 181e60e8c..98522b98a 100644 --- a/freqtrade/exchange/__init__.py +++ b/freqtrade/exchange/__init__.py @@ -160,7 +160,7 @@ class Exchange(object): def klines(self, pair: str, copy=True) -> DataFrame: if pair in self._klines: - return self._klines[pair].copy() + return self._klines[pair].copy() if copy else self._klines[pair] else: return None From 41ef02a2920afd9bc382d697f1d72c2e68b317bf Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Fri, 21 Dec 2018 13:32:06 +0100 Subject: [PATCH 678/699] Update ccxt from 1.18.50 to 1.18.58 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 0bd34f848..835442ba9 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -ccxt==1.18.50 +ccxt==1.18.58 SQLAlchemy==1.2.15 python-telegram-bot==11.1.0 arrow==0.12.1 From 7c69dbae30fb3a0e4cfbc339fead2b69bbb44a79 Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Sat, 22 Dec 2018 13:33:07 +0100 Subject: [PATCH 679/699] Update ccxt from 1.18.58 to 1.18.60 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 835442ba9..5260015c6 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -ccxt==1.18.58 +ccxt==1.18.60 SQLAlchemy==1.2.15 python-telegram-bot==11.1.0 arrow==0.12.1 From 7243da3afe573dcb26275f62aeb7482b22ccf957 Mon Sep 17 00:00:00 2001 From: misagh Date: Sat, 22 Dec 2018 19:03:42 +0100 Subject: [PATCH 680/699] tests added for klines copy=True --- freqtrade/tests/exchange/test_exchange.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/freqtrade/tests/exchange/test_exchange.py b/freqtrade/tests/exchange/test_exchange.py index 647440223..29154bc39 100644 --- a/freqtrade/tests/exchange/test_exchange.py +++ b/freqtrade/tests/exchange/test_exchange.py @@ -814,6 +814,13 @@ def test_refresh_tickers(mocker, default_conf, caplog) -> None: assert isinstance(exchange.klines(pair), DataFrame) assert len(exchange.klines(pair)) > 0 + # klines function should return a different object on each call + # if copy is "True" + assert exchange.klines(pair) is not exchange.klines(pair) + assert exchange.klines(pair) is not exchange.klines(pair, copy=True) + assert exchange.klines(pair, copy=True) is not exchange.klines(pair, copy=True) + assert exchange.klines(pair, copy=False) is exchange.klines(pair, copy=False) + # test caching exchange.refresh_tickers(['IOTA/ETH', 'XRP/ETH'], '5m') From 741e3368642758a5dfb647d154e71ea89cf302b6 Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Sun, 23 Dec 2018 13:33:07 +0100 Subject: [PATCH 681/699] Update ccxt from 1.18.60 to 1.18.64 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 5260015c6..17750eda6 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -ccxt==1.18.60 +ccxt==1.18.64 SQLAlchemy==1.2.15 python-telegram-bot==11.1.0 arrow==0.12.1 From a5137e4fa4ac35b6e6492deabfa3b23bde86c8aa Mon Sep 17 00:00:00 2001 From: misagh Date: Mon, 24 Dec 2018 11:39:11 +0100 Subject: [PATCH 682/699] comparing float instead of int --- freqtrade/freqtradebot.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index 9b58f8ec6..4febe9dd0 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -694,7 +694,7 @@ class FreqtradeBot(object): ordertime = arrow.get(order['datetime']).datetime # Check if trade is still actually open - if int(order['remaining']) == 0: + if float(order['remaining']) == 0.0: self.wallets.update() continue From ae514585851ca879f294267a2e790a435480f355 Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Mon, 24 Dec 2018 13:33:08 +0100 Subject: [PATCH 683/699] Update ccxt from 1.18.64 to 1.18.67 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 17750eda6..efd8cd014 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -ccxt==1.18.64 +ccxt==1.18.67 SQLAlchemy==1.2.15 python-telegram-bot==11.1.0 arrow==0.12.1 From 8fbeb700d64d2ae2b0d65b0467b513a8e97874d5 Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 25 Dec 2018 13:00:48 +0100 Subject: [PATCH 684/699] move key/secret in download_backtest_data to correct location --- scripts/download_backtest_data.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/scripts/download_backtest_data.py b/scripts/download_backtest_data.py index 2b4ca03d1..6b41e5d99 100755 --- a/scripts/download_backtest_data.py +++ b/scripts/download_backtest_data.py @@ -54,12 +54,12 @@ print(f'About to download pairs: {PAIRS} to {dl_path}') # Init exchange -exchange = Exchange({'key': '', - 'secret': '', - 'stake_currency': '', +exchange = Exchange({'stake_currency': '', 'dry_run': True, 'exchange': { 'name': args.exchange, + 'key': '', + 'secret': '', 'pair_whitelist': [], 'ccxt_async_config': { "enableRateLimit": False From 34b93eb95292c689d1bb9319a2684e174fcbdfc8 Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 25 Dec 2018 13:14:28 +0100 Subject: [PATCH 685/699] Load config-file in download_backtest_data - --- freqtrade/arguments.py | 11 ++++++++- scripts/download_backtest_data.py | 40 +++++++++++++++++++------------ 2 files changed, 35 insertions(+), 16 deletions(-) diff --git a/freqtrade/arguments.py b/freqtrade/arguments.py index a33848c5f..19cccfe8b 100644 --- a/freqtrade/arguments.py +++ b/freqtrade/arguments.py @@ -378,6 +378,15 @@ class Arguments(object): metavar='PATH', ) + self.parser.add_argument( + '-c', '--config', + help='specify configuration file, used for additional exchange parameters', + dest='config', + default=None, + type=str, + metavar='PATH', + ) + self.parser.add_argument( '--days', help='Download data for number of days', @@ -389,7 +398,7 @@ class Arguments(object): self.parser.add_argument( '--exchange', - help='Exchange name (default: %(default)s)', + help='Exchange name (default: %(default)s). Only valid if no config is provided', dest='exchange', type=str, default='bittrex' diff --git a/scripts/download_backtest_data.py b/scripts/download_backtest_data.py index 6b41e5d99..c4a1b59c5 100755 --- a/scripts/download_backtest_data.py +++ b/scripts/download_backtest_data.py @@ -10,7 +10,7 @@ from freqtrade import arguments from freqtrade.arguments import TimeRange from freqtrade.exchange import Exchange from freqtrade.data.history import download_pair_history -from freqtrade.configuration import set_loggers +from freqtrade.configuration import Configuration, set_loggers import logging logging.basicConfig( @@ -27,7 +27,29 @@ args = arguments.parse_args() timeframes = args.timeframes -dl_path = Path(DEFAULT_DL_PATH).joinpath(args.exchange) +if args.config: + configuration = Configuration(args) + config = configuration._load_config_file(args.config) + + # Ensure we do not use Exchange credentials + config['exchange']['key'] = '' + config['exchange']['secret'] = '' +else: + config = {'stake_currency': '', + 'dry_run': True, + 'exchange': { + 'name': args.exchange, + 'key': '', + 'secret': '', + 'pair_whitelist': [], + 'ccxt_async_config': { + "enableRateLimit": False + } + } + } + + +dl_path = Path(DEFAULT_DL_PATH).joinpath(config['exchange']['name']) if args.export: dl_path = Path(args.export) @@ -52,20 +74,8 @@ if args.days: print(f'About to download pairs: {PAIRS} to {dl_path}') - # Init exchange -exchange = Exchange({'stake_currency': '', - 'dry_run': True, - 'exchange': { - 'name': args.exchange, - 'key': '', - 'secret': '', - 'pair_whitelist': [], - 'ccxt_async_config': { - "enableRateLimit": False - } - } - }) +exchange = Exchange(config) pairs_not_available = [] for pair in PAIRS: From 0e6dbfab5ee37ee63ae35fe5fd7770b3a161e24b Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Tue, 25 Dec 2018 13:33:06 +0100 Subject: [PATCH 686/699] Update ccxt from 1.18.67 to 1.18.69 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index efd8cd014..c77b3e37c 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -ccxt==1.18.67 +ccxt==1.18.69 SQLAlchemy==1.2.15 python-telegram-bot==11.1.0 arrow==0.12.1 From b28b2369da5206d40184fd2a03bc3fe81fd9c9df Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Wed, 26 Dec 2018 13:33:06 +0100 Subject: [PATCH 687/699] Update ccxt from 1.18.69 to 1.18.71 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index c77b3e37c..aa1147f9a 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -ccxt==1.18.69 +ccxt==1.18.71 SQLAlchemy==1.2.15 python-telegram-bot==11.1.0 arrow==0.12.1 From d95128986222f931a632f84dfd50df64e30d1e90 Mon Sep 17 00:00:00 2001 From: Matthias Date: Wed, 26 Dec 2018 13:06:25 +0100 Subject: [PATCH 688/699] Refactor strategy documentation --- docs/bot-optimization.md | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/docs/bot-optimization.md b/docs/bot-optimization.md index 0214ce5c5..b0e39e530 100644 --- a/docs/bot-optimization.md +++ b/docs/bot-optimization.md @@ -38,7 +38,7 @@ A strategy file contains all the information needed to build a good strategy: - Buy strategy rules - Sell strategy rules - Minimal ROI recommended -- Stoploss recommended +- Stoploss strongly recommended The bot also include a sample strategy called `TestStrategy` you can update: `user_data/strategies/test_strategy.py`. You can test it with the parameter: `--strategy TestStrategy` @@ -47,18 +47,7 @@ You can test it with the parameter: `--strategy TestStrategy` python3 ./freqtrade/main.py --strategy AwesomeStrategy ``` -### Specify custom strategy location - -If you want to use a strategy from a different folder you can pass `--strategy-path` - -```bash -python3 ./freqtrade/main.py --strategy AwesomeStrategy --strategy-path /some/folder -``` - -**For the following section we will use the [user_data/strategies/test_strategy.py](https://github.com/freqtrade/freqtrade/blob/develop/user_data/strategies/test_strategy.py) -file as reference.** - -### Buy strategy +### Buy strategy rules Edit the method `populate_buy_trend()` into your strategy file to update your buy strategy. @@ -83,7 +72,7 @@ def populate_buy_trend(self, dataframe: DataFrame, metadata: dict) -> DataFrame: return dataframe ``` -### Sell strategy +### Sell strategy rules Edit the method `populate_sell_trend()` into your strategy file to update your sell strategy. Please note that the sell-signal is only used if `use_sell_signal` is set to true in the configuration. @@ -172,6 +161,17 @@ Then uncomment indicators you need. The default buy strategy is located in the file [freqtrade/default_strategy.py](https://github.com/freqtrade/freqtrade/blob/develop/freqtrade/strategy/default_strategy.py). +### Specify custom strategy location + +If you want to use a strategy from a different folder you can pass `--strategy-path` + +```bash +python3 ./freqtrade/main.py --strategy AwesomeStrategy --strategy-path /some/folder +``` + +**For the following section we will use the [user_data/strategies/test_strategy.py](https://github.com/freqtrade/freqtrade/blob/develop/user_data/strategies/test_strategy.py) +file as reference.** + ### Further strategy ideas To get additional Ideas for strategies, head over to our [strategy repository](https://github.com/freqtrade/freqtrade-strategies). Feel free to use them as they are - but results will depend on the current market situation, pairs used etc. - therefore please backtest the strategy for your exchange/desired pairs first, evaluate carefully, use at your own risk. From f2beaf101c62594638820a6bca35b2279ddb63e7 Mon Sep 17 00:00:00 2001 From: Matthias Date: Wed, 26 Dec 2018 13:25:39 +0100 Subject: [PATCH 689/699] Add strategy documentation (fixes #818) --- docs/bot-optimization.md | 177 +++++++++++++++++++++++++++------------ 1 file changed, 125 insertions(+), 52 deletions(-) diff --git a/docs/bot-optimization.md b/docs/bot-optimization.md index b0e39e530..443191ed0 100644 --- a/docs/bot-optimization.md +++ b/docs/bot-optimization.md @@ -33,8 +33,14 @@ use your own file to not have to lose your parameters every time the default strategy file will be updated on Github. Put your custom strategy file into the folder `user_data/strategies`. +Best copy the test-strategy and modify this copy to avoid having bot-updates override your changes. +`cp user_data/strategies/test_strategy.py user_data/strategies/awesome-strategy.py` + +### Anatomy of a strategy + A strategy file contains all the information needed to build a good strategy: +- Indicators - Buy strategy rules - Sell strategy rules - Minimal ROI recommended @@ -47,62 +53,14 @@ You can test it with the parameter: `--strategy TestStrategy` python3 ./freqtrade/main.py --strategy AwesomeStrategy ``` -### Buy strategy rules +### Customize Indicators -Edit the method `populate_buy_trend()` into your strategy file to update your buy strategy. - -Sample from `user_data/strategies/test_strategy.py`: - -```python -def populate_buy_trend(self, dataframe: DataFrame, metadata: dict) -> DataFrame: - """ - Based on TA indicators, populates the buy signal for the given dataframe - :param dataframe: DataFrame populated with indicators - :param metadata: Additional information, like the currently traded pair - :return: DataFrame with buy column - """ - dataframe.loc[ - ( - (dataframe['adx'] > 30) & - (dataframe['tema'] <= dataframe['bb_middleband']) & - (dataframe['tema'] > dataframe['tema'].shift(1)) - ), - 'buy'] = 1 - - return dataframe -``` - -### Sell strategy rules - -Edit the method `populate_sell_trend()` into your strategy file to update your sell strategy. -Please note that the sell-signal is only used if `use_sell_signal` is set to true in the configuration. - -Sample from `user_data/strategies/test_strategy.py`: - -```python -def populate_sell_trend(self, dataframe: DataFrame, metadata: dict) -> DataFrame: - """ - Based on TA indicators, populates the sell signal for the given dataframe - :param dataframe: DataFrame populated with indicators - :param metadata: Additional information, like the currently traded pair - :return: DataFrame with buy column - """ - dataframe.loc[ - ( - (dataframe['adx'] > 70) & - (dataframe['tema'] > dataframe['bb_middleband']) & - (dataframe['tema'] < dataframe['tema'].shift(1)) - ), - 'sell'] = 1 - return dataframe -``` - -## Add more Indicators - -As you have seen, buy and sell strategies need indicators. You can add more indicators by extending the list contained in the method `populate_indicators()` from your strategy file. +Buy and sell strategies need indicators. You can add more indicators by extending the list contained in the method `populate_indicators()` from your strategy file. You should only add the indicators used in either `populate_buy_trend()`, `populate_sell_trend()`, or to populate another indicator, otherwise performance may suffer. +It's important to always return the dataframe without removing/modifying the columns `"open", "high", "low", "close", "volume"`, otherwise these fields would contain something unexpected. + Sample: ```python @@ -146,6 +104,121 @@ def populate_indicators(self, dataframe: DataFrame, metadata: dict) -> DataFrame return dataframe ``` +### Buy signal rules + +Edit the method `populate_buy_trend()` in your strategy file to update your buy strategy. + +It's important to always return the dataframe without removing/modifying the columns `"open", "high", "low", "close", "volume"`, otherwise these fields would contain something unexpected. + +This will method will also define a new column, `"buy"`, which needs to contain 1 for buys, and 0 for "no action". + +Sample from `user_data/strategies/test_strategy.py`: + +```python +def populate_buy_trend(self, dataframe: DataFrame, metadata: dict) -> DataFrame: + """ + Based on TA indicators, populates the buy signal for the given dataframe + :param dataframe: DataFrame populated with indicators + :param metadata: Additional information, like the currently traded pair + :return: DataFrame with buy column + """ + dataframe.loc[ + ( + (dataframe['adx'] > 30) & + (dataframe['tema'] <= dataframe['bb_middleband']) & + (dataframe['tema'] > dataframe['tema'].shift(1)) + ), + 'buy'] = 1 + + return dataframe +``` + +### Sell signal rules + +Edit the method `populate_sell_trend()` into your strategy file to update your sell strategy. +Please note that the sell-signal is only used if `use_sell_signal` is set to true in the configuration. + +It's important to always return the dataframe without removing/modifying the columns `"open", "high", "low", "close", "volume"`, otherwise these fields would contain something unexpected. + +This will method will also define a new column, `"sell"`, which needs to contain 1 for sells, and 0 for "no action". + +Sample from `user_data/strategies/test_strategy.py`: + +```python +def populate_sell_trend(self, dataframe: DataFrame, metadata: dict) -> DataFrame: + """ + Based on TA indicators, populates the sell signal for the given dataframe + :param dataframe: DataFrame populated with indicators + :param metadata: Additional information, like the currently traded pair + :return: DataFrame with buy column + """ + dataframe.loc[ + ( + (dataframe['adx'] > 70) & + (dataframe['tema'] > dataframe['bb_middleband']) & + (dataframe['tema'] < dataframe['tema'].shift(1)) + ), + 'sell'] = 1 + return dataframe +``` + +### Minimal ROI + +This dict defines the minimal Return On Investment (ROI) a trade should reach before selling, independent from the sell signal. + +It is of the following format, with the dict key (left side of the colon) being the minutes passed since the trade opened, and the value (right side of the colon) being the percentage. + +```python +minimal_roi = { + "40": 0.0, + "30": 0.01, + "20": 0.02, + "0": 0.04 +} +``` + +The above configuration would therefore mean: + +- Sell whenever 4% profit was reached +- Sell after 20 minutes when 2% profit was reached +- Sell after 20 minutes when 2% profit was reached +- Sell after 30 minutes when 1% profit was reached +- Sell after 40 minutes when the trade is non-loosing (no profit) + +The calculation does include fees. + +To disable ROI completely, set it to an insanely high number: + +```python +minimal_roi = { + "0": 100 +} +``` + +While technically not completely disabled, this would sell once the trade reaches 10000% Profit. + +### Stoploss + +Setting a stoploss is highly recommended to protect your capital from strong moves against you. + +Sample: + +``` python +stoploss = -0.10 +``` + +This would signify a stoploss of -10%. +If your exchange supports it, it's recommendet to also set `"stoploss_on_exchange"` in the order dict, so your stoploss is on the exchange and cannot be missed for network-problems (or other problems). + +For more information on order_types please look [here](https://github.com/freqtrade/freqtrade/blob/develop/docs/configuration.md#understand-order_types). + +### Ticker interval + +This is the set of candles the bot should download and use for the analysis. +Common values are `"1m"`, `"5m"`, `"15m"`, `"1h"`, however all values supported by your exchange should work. + +Please note that the same buy/sell signals may work with one interval, but not the other. + ### Metadata dict The metadata-dict (available for `populate_buy_trend`, `populate_sell_trend`, `populate_indicators`) contains additional information. From 5e3e7b69288e4e6f12ea442083010a481078a933 Mon Sep 17 00:00:00 2001 From: Matthias Date: Wed, 26 Dec 2018 13:32:35 +0100 Subject: [PATCH 690/699] correct TOC for bot-optimization.md --- docs/bot-optimization.md | 23 +++++++++++++++++------ 1 file changed, 17 insertions(+), 6 deletions(-) diff --git a/docs/bot-optimization.md b/docs/bot-optimization.md index 443191ed0..72b234dd5 100644 --- a/docs/bot-optimization.md +++ b/docs/bot-optimization.md @@ -7,7 +7,18 @@ indicators. - [Install a custom strategy file](#install-a-custom-strategy-file) - [Customize your strategy](#change-your-strategy) -- [Add more Indicator](#add-more-indicator) + - [Anatomy of a strategy](#anatomy-of-a-strategy) + - [Customize indicators](#customize-indicators) + - [Buy signal rules](#buy-signal-rules]) + - [Sell signal rules](#sell-signal-rules) + - [Minimal ROI](#minimal-roi) + - [Stoploss](#stoploss) + - [Ticker interval](#ticker-interval) + - [Metadata dict](#metadata-dict) + - [Where is the default strategy](#where-is-the-default-strategy) + - [Specify custom strategy location](#specify-custom-strategy-location) + - [Further strategy ideas](#further-strategy-ideas) + - [Where is the default strategy](#where-is-the-default-strategy) Since the version `0.16.0` the bot allows using custom strategy file. @@ -104,6 +115,11 @@ def populate_indicators(self, dataframe: DataFrame, metadata: dict) -> DataFrame return dataframe ``` +#### Want more indicator examples + +Look into the [user_data/strategies/test_strategy.py](https://github.com/freqtrade/freqtrade/blob/develop/user_data/strategies/test_strategy.py). +Then uncomment indicators you need. + ### Buy signal rules Edit the method `populate_buy_trend()` in your strategy file to update your buy strategy. @@ -224,11 +240,6 @@ Please note that the same buy/sell signals may work with one interval, but not t The metadata-dict (available for `populate_buy_trend`, `populate_sell_trend`, `populate_indicators`) contains additional information. Currently this is `pair`, which can be accessed using `metadata['pair']` - and will return a pair in the format `XRP/BTC`. -### Want more indicator examples - -Look into the [user_data/strategies/test_strategy.py](https://github.com/freqtrade/freqtrade/blob/develop/user_data/strategies/test_strategy.py). -Then uncomment indicators you need. - ### Where is the default strategy? The default buy strategy is located in the file From 32f43d32946172a7d3169129f7e5ae6f184d6e77 Mon Sep 17 00:00:00 2001 From: Matthias Date: Wed, 26 Dec 2018 13:52:56 +0100 Subject: [PATCH 691/699] Adjust pyup.yml to also "find" requirements-dev --- .pyup.yml | 32 ++++++++++++++++++++++++++++++-- 1 file changed, 30 insertions(+), 2 deletions(-) diff --git a/.pyup.yml b/.pyup.yml index a0833af39..74c1456ce 100644 --- a/.pyup.yml +++ b/.pyup.yml @@ -1,4 +1,32 @@ -# autogenerated pyup.io config file +# autogenerated pyup.io config file # see https://pyup.io/docs/configuration/ for all available options -schedule: every day +# configure updates globally +# default: all +# allowed: all, insecure, False +update: all + +# configure dependency pinning globally +# default: True +# allowed: True, False +pin: True + +schedule: "every day" + + +search: False +# Specify requirement files by hand, default is empty +# default: empty +# allowed: list +requirements: + - requirements.txt + - requirements-dev.txt + + +# configure the branch prefix the bot is using +# default: pyup- +branch_prefix: pyup/ + +# allow to close stale PRs +# default: True +close_prs: True From b2bc5d93964e98e8c2e1dbf5edfd1df47b8a0067 Mon Sep 17 00:00:00 2001 From: Matthias Date: Wed, 26 Dec 2018 14:02:17 +0100 Subject: [PATCH 692/699] Remove convert_backtestdata - this is not usefull anymore --- scripts/convert_backtestdata.py | 200 -------------------------------- 1 file changed, 200 deletions(-) delete mode 100755 scripts/convert_backtestdata.py diff --git a/scripts/convert_backtestdata.py b/scripts/convert_backtestdata.py deleted file mode 100755 index 96e0cbce8..000000000 --- a/scripts/convert_backtestdata.py +++ /dev/null @@ -1,200 +0,0 @@ -#!/usr/bin/env python3 -""" -Script to display when the bot will buy a specific pair - -Mandatory Cli parameters: --p / --pair: pair to examine - -Optional Cli parameters --d / --datadir: path to pair backtest data ---timerange: specify what timerange of data to use. --l / --live: Live, to download the latest ticker for the pair -""" -import logging -import sys -from argparse import Namespace -from os import path -import glob -import json -import re -from typing import List, Dict -import gzip - -from freqtrade.arguments import Arguments -from freqtrade import misc, constants -from pandas import DataFrame - -import dateutil.parser - -logger = logging.getLogger('freqtrade') - - -def load_old_file(filename) -> (List[Dict], bool): - if not path.isfile(filename): - logger.warning("filename %s does not exist", filename) - return (None, False) - logger.debug('Loading ticker data from file %s', filename) - - pairdata = None - - if filename.endswith('.gz'): - logger.debug('Loading ticker data from file %s', filename) - is_zip = True - with gzip.open(filename) as tickerdata: - pairdata = json.load(tickerdata) - else: - is_zip = False - with open(filename) as tickerdata: - pairdata = json.load(tickerdata) - return (pairdata, is_zip) - - -def parse_old_backtest_data(ticker) -> DataFrame: - """ - Reads old backtest data - Format: "O": 8.794e-05, - "H": 8.948e-05, - "L": 8.794e-05, - "C": 8.88e-05, - "V": 991.09056638, - "T": "2017-11-26T08:50:00", - "BV": 0.0877869 - """ - - columns = {'C': 'close', 'V': 'volume', 'O': 'open', - 'H': 'high', 'L': 'low', 'T': 'date'} - - frame = DataFrame(ticker) \ - .rename(columns=columns) - if 'BV' in frame: - frame.drop('BV', 1, inplace=True) - if 'date' not in frame: - logger.warning("Date not in frame - probably not a Ticker file") - return None - frame.sort_values('date', inplace=True) - return frame - - -def convert_dataframe(frame: DataFrame): - """Convert dataframe to new format""" - # reorder columns: - cols = ['date', 'open', 'high', 'low', 'close', 'volume'] - frame = frame[cols] - - # Make sure parsing/printing data is assumed to be UTC - frame['date'] = frame['date'].apply( - lambda d: int(dateutil.parser.parse(d+'+00:00').timestamp()) * 1000) - frame['date'] = frame['date'].astype('int64') - # Convert columns one by one to preserve type. - by_column = [frame[x].values.tolist() for x in frame.columns] - return list(list(x) for x in zip(*by_column)) - - -def convert_file(filename: str, filename_new: str) -> None: - """Converts a file from old format to ccxt format""" - (pairdata, is_zip) = load_old_file(filename) - if pairdata and type(pairdata) is list: - if type(pairdata[0]) is list: - logger.error("pairdata for %s already in new format", filename) - return - - frame = parse_old_backtest_data(pairdata) - # Convert frame to new format - if frame is not None: - frame1 = convert_dataframe(frame) - misc.file_dump_json(filename_new, frame1, is_zip) - - -def convert_main(args: Namespace) -> None: - """ - converts a folder given in --datadir from old to new format to support ccxt - """ - - workdir = path.join(args.datadir, "") - logger.info("Workdir: %s", workdir) - - for filename in glob.glob(workdir + "*.json"): - # swap currency names - ret = re.search(r'[A-Z_]{7,}', path.basename(filename)) - if args.norename: - filename_new = filename - else: - if not ret: - logger.warning("file %s could not be converted, could not extract currencies", - filename) - continue - pair = ret.group(0) - currencies = pair.split("_") - if len(currencies) != 2: - logger.warning("file %s could not be converted, could not extract currencies", - filename) - continue - - ret_integer = re.search(r'\d+(?=\.json)', path.basename(filename)) - ret_string = re.search(r'(\d+[mhdw])(?=\.json)', path.basename(filename)) - - if ret_integer: - minutes = int(ret_integer.group(0)) - # default to adding 'm' to end of minutes for new interval name - interval = str(minutes) + 'm' - # but check if there is a mapping between int and string also - for str_interval, minutes_interval in constants.TICKER_INTERVAL_MINUTES.items(): - if minutes_interval == minutes: - interval = str_interval - break - # change order on pairs if old ticker interval found - - filename_new = path.join(path.dirname(filename), - f"{currencies[1]}_{currencies[0]}-{interval}.json") - - elif ret_string: - interval = ret_string.group(0) - filename_new = path.join(path.dirname(filename), - f"{currencies[0]}_{currencies[1]}-{interval}.json") - - else: - logger.warning("file %s could not be converted, interval not found", filename) - continue - - logger.debug("Converting and renaming %s to %s", filename, filename_new) - convert_file(filename, filename_new) - - -def convert_parse_args(args: List[str]) -> Namespace: - """ - Parse args passed to the script - :param args: Cli arguments - :return: args: Array with all arguments - """ - arguments = Arguments(args, 'Convert datafiles') - arguments.parser.add_argument( - '-d', '--datadir', - help='path to backtest data (default: %(default)s', - dest='datadir', - default=path.join('freqtrade', 'tests', 'testdata'), - type=str, - metavar='PATH', - ) - arguments.parser.add_argument( - '-n', '--norename', - help='don''t rename files from BTC_ to _BTC - ' - 'Note that not renaming will overwrite source files', - dest='norename', - default=False, - action='store_true' - ) - - return arguments.parse_args() - - -def main(sysargv: List[str]) -> None: - """ - This function will initiate the bot and start the trading loop. - :return: None - """ - logger.info('Starting Dataframe conversation') - convert_main(convert_parse_args(sysargv)) - - -if __name__ == '__main__': - main(sys.argv[1:]) From 1fc0dcb9d817aafe4ea147133438595b8c148d2c Mon Sep 17 00:00:00 2001 From: Matthias Date: Thu, 27 Dec 2018 06:58:59 +0100 Subject: [PATCH 693/699] Fix typo in link --- docs/bot-optimization.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/bot-optimization.md b/docs/bot-optimization.md index 72b234dd5..844dd8581 100644 --- a/docs/bot-optimization.md +++ b/docs/bot-optimization.md @@ -9,7 +9,7 @@ indicators. - [Customize your strategy](#change-your-strategy) - [Anatomy of a strategy](#anatomy-of-a-strategy) - [Customize indicators](#customize-indicators) - - [Buy signal rules](#buy-signal-rules]) + - [Buy signal rules](#buy-signal-rules) - [Sell signal rules](#sell-signal-rules) - [Minimal ROI](#minimal-roi) - [Stoploss](#stoploss) From 37cde77e1899d7b7d493a3a186846de00032ccff Mon Sep 17 00:00:00 2001 From: Matthias Date: Thu, 27 Dec 2018 07:01:57 +0100 Subject: [PATCH 694/699] Fix typo --- docs/bot-optimization.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/bot-optimization.md b/docs/bot-optimization.md index 844dd8581..a3ab18d30 100644 --- a/docs/bot-optimization.md +++ b/docs/bot-optimization.md @@ -224,7 +224,7 @@ stoploss = -0.10 ``` This would signify a stoploss of -10%. -If your exchange supports it, it's recommendet to also set `"stoploss_on_exchange"` in the order dict, so your stoploss is on the exchange and cannot be missed for network-problems (or other problems). +If your exchange supports it, it's recommended to also set `"stoploss_on_exchange"` in the order dict, so your stoploss is on the exchange and cannot be missed for network-problems (or other problems). For more information on order_types please look [here](https://github.com/freqtrade/freqtrade/blob/develop/docs/configuration.md#understand-order_types). From 5b30815d7bdfd3d181fc9b00e2c8a9b2f3fd20e2 Mon Sep 17 00:00:00 2001 From: Matthias Date: Thu, 27 Dec 2018 07:03:28 +0100 Subject: [PATCH 695/699] Move "following section" part --- docs/bot-optimization.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/bot-optimization.md b/docs/bot-optimization.md index a3ab18d30..0fc41a2e3 100644 --- a/docs/bot-optimization.md +++ b/docs/bot-optimization.md @@ -64,6 +64,9 @@ You can test it with the parameter: `--strategy TestStrategy` python3 ./freqtrade/main.py --strategy AwesomeStrategy ``` +**For the following section we will use the [user_data/strategies/test_strategy.py](https://github.com/freqtrade/freqtrade/blob/develop/user_data/strategies/test_strategy.py) +file as reference.** + ### Customize Indicators Buy and sell strategies need indicators. You can add more indicators by extending the list contained in the method `populate_indicators()` from your strategy file. @@ -253,9 +256,6 @@ If you want to use a strategy from a different folder you can pass `--strategy-p python3 ./freqtrade/main.py --strategy AwesomeStrategy --strategy-path /some/folder ``` -**For the following section we will use the [user_data/strategies/test_strategy.py](https://github.com/freqtrade/freqtrade/blob/develop/user_data/strategies/test_strategy.py) -file as reference.** - ### Further strategy ideas To get additional Ideas for strategies, head over to our [strategy repository](https://github.com/freqtrade/freqtrade-strategies). Feel free to use them as they are - but results will depend on the current market situation, pairs used etc. - therefore please backtest the strategy for your exchange/desired pairs first, evaluate carefully, use at your own risk. From 8b38f44da6b82b4b6e5ea98eb75680ebee939b45 Mon Sep 17 00:00:00 2001 From: Oon Arfiandwi Date: Thu, 27 Dec 2018 14:56:35 +0800 Subject: [PATCH 696/699] Update installation reference for Raspbian Add `libffi-dev` as an additional package to install before process installation with `pip`. Add recommendation to use (mini)conda and remove package `scipy`, `pandas`, and `numpy` from `requrements.txt`. --- docs/installation.md | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/docs/installation.md b/docs/installation.md index 3d0ee9468..71ae5a6b5 100644 --- a/docs/installation.md +++ b/docs/installation.md @@ -260,15 +260,20 @@ sudo apt-get install python3.6 python3.6-venv python3.6-dev build-essential auto Before installing FreqTrade on a Raspberry Pi running the official Raspbian Image, make sure you have at least Python 3.6 installed. The default image only provides Python 3.5. Probably the easiest way to get a recent version of python is [miniconda](https://repo.continuum.io/miniconda/). -The following assumes that miniconda3 is installed and available in your environment, and is installed. -It's recommended to use (mini)conda for this as installation/compilation of `scipy` and `pandas` takes a long time. +The following assumes that miniconda3 is installed and available in your environment. Last miniconda3 installation file use python 3.4, we will update to python 3.6 on this installation. +It's recommended to use (mini)conda for this as installation/compilation of `numpy`, `scipy` and `pandas` takes a long time. +If you have installed it from (mini)conda, you can remove `numpy`, `scipy`, and `pandas` from `requirements.txt` before you install it with `pip`. + +Additional package to install on your Raspbian, `libffi-dev` required by cryptography (from python-telegram-bot). ``` bash conda config --add channels rpi conda install python=3.6 conda create -n freqtrade python=3.6 -conda install scipy pandas +conda activate freqtrade +conda install scipy pandas numpy +sudo apt install libffi-dev python3 -m pip install -r requirements.txt python3 -m pip install -e . ``` From 20cdabbe9c6e8260d75d406a6fb8d638a293e5f4 Mon Sep 17 00:00:00 2001 From: Matthias Date: Thu, 27 Dec 2018 09:31:21 +0100 Subject: [PATCH 697/699] Add test for market order --- freqtrade/tests/conftest.py | 30 +++++++++++++++++++++++++++++ freqtrade/tests/test_persistence.py | 25 ++++++++++++++++++++++++ 2 files changed, 55 insertions(+) diff --git a/freqtrade/tests/conftest.py b/freqtrade/tests/conftest.py index 975a3a9f2..0a2633c7c 100644 --- a/freqtrade/tests/conftest.py +++ b/freqtrade/tests/conftest.py @@ -387,6 +387,36 @@ def limit_buy_order(): } +@pytest.fixture(scope='function') +def market_buy_order(): + return { + 'id': 'mocked_market_buy', + 'type': 'market', + 'side': 'buy', + 'pair': 'mocked', + 'datetime': arrow.utcnow().isoformat(), + 'price': 0.00004099, + 'amount': 91.99181073, + 'remaining': 0.0, + 'status': 'closed' + } + + +@pytest.fixture +def market_sell_order(): + return { + 'id': 'mocked_limit_sell', + 'type': 'market', + 'side': 'sell', + 'pair': 'mocked', + 'datetime': arrow.utcnow().isoformat(), + 'price': 0.00004173, + 'amount': 91.99181073, + 'remaining': 0.0, + 'status': 'closed' + } + + @pytest.fixture def limit_buy_order_old(): return { diff --git a/freqtrade/tests/test_persistence.py b/freqtrade/tests/test_persistence.py index a7b21bc1d..b0307e31b 100644 --- a/freqtrade/tests/test_persistence.py +++ b/freqtrade/tests/test_persistence.py @@ -117,6 +117,31 @@ def test_update_with_bittrex(limit_buy_order, limit_sell_order, fee): assert trade.close_date is not None +@pytest.mark.usefixtures("init_persistence") +def test_update_market_order(market_buy_order, market_sell_order, fee): + trade = Trade( + pair='ETH/BTC', + stake_amount=0.001, + fee_open=fee.return_value, + fee_close=fee.return_value, + exchange='bittrex', + ) + + trade.open_order_id = 'something' + trade.update(market_buy_order) + assert trade.open_order_id is None + assert trade.open_rate == 0.00004099 + assert trade.close_profit is None + assert trade.close_date is None + + trade.open_order_id = 'something' + trade.update(market_sell_order) + assert trade.open_order_id is None + assert trade.close_rate == 0.00004173 + assert trade.close_profit == 0.01297561 + assert trade.close_date is not None + + @pytest.mark.usefixtures("init_persistence") def test_calc_open_close_trade_price(limit_buy_order, limit_sell_order, fee): trade = Trade( From 9af2fca7180495e78c8c78dc432fbc8a98daf37d Mon Sep 17 00:00:00 2001 From: Matthias Date: Thu, 27 Dec 2018 11:19:26 +0100 Subject: [PATCH 698/699] Add handling for market orders fixes #1427 and #1428 --- freqtrade/persistence.py | 9 +++++---- freqtrade/tests/test_persistence.py | 20 ++++++++++++++++++-- 2 files changed, 23 insertions(+), 6 deletions(-) diff --git a/freqtrade/persistence.py b/freqtrade/persistence.py index a393eb318..a14d22b98 100644 --- a/freqtrade/persistence.py +++ b/freqtrade/persistence.py @@ -247,16 +247,17 @@ class Trade(_DECL_BASE): if order['status'] == 'open' or order['price'] is None: return - logger.info('Updating trade (id=%d) ...', self.id) + logger.info('Updating trade (id=%s) ...', self.id) - if order_type == 'limit' and order['side'] == 'buy': + if order_type in ('market', 'limit') and order['side'] == 'buy': # Update open rate and actual amount self.open_rate = Decimal(order['price']) self.amount = Decimal(order['amount']) - logger.info('LIMIT_BUY has been fulfilled for %s.', self) + logger.info('%s_BUY has been fulfilled for %s.', order_type.upper(), self) self.open_order_id = None - elif order_type == 'limit' and order['side'] == 'sell': + elif order_type in ('market', 'limit') and order['side'] == 'sell': self.close(order['price']) + logger.info('%s_SELL has been fulfilled for %s.', order_type.upper(), self) elif order_type == 'stop_loss_limit': self.stoploss_order_id = None logger.info('STOP_LOSS_LIMIT is hit for %s.', self) diff --git a/freqtrade/tests/test_persistence.py b/freqtrade/tests/test_persistence.py index b0307e31b..e64a08262 100644 --- a/freqtrade/tests/test_persistence.py +++ b/freqtrade/tests/test_persistence.py @@ -62,7 +62,7 @@ def test_init_dryrun_db(default_conf, mocker): @pytest.mark.usefixtures("init_persistence") -def test_update_with_bittrex(limit_buy_order, limit_sell_order, fee): +def test_update_with_bittrex(limit_buy_order, limit_sell_order, fee, caplog): """ On this test we will buy and sell a crypto currency. @@ -91,6 +91,7 @@ def test_update_with_bittrex(limit_buy_order, limit_sell_order, fee): """ trade = Trade( + id=2, pair='ETH/BTC', stake_amount=0.001, fee_open=fee.return_value, @@ -108,18 +109,26 @@ def test_update_with_bittrex(limit_buy_order, limit_sell_order, fee): assert trade.open_rate == 0.00001099 assert trade.close_profit is None assert trade.close_date is None + assert log_has("LIMIT_BUY has been fulfilled for Trade(id=2, " + "pair=ETH/BTC, amount=90.99181073, open_rate=0.00001099, open_since=closed).", + caplog.record_tuples) + caplog.clear() trade.open_order_id = 'something' trade.update(limit_sell_order) assert trade.open_order_id is None assert trade.close_rate == 0.00001173 assert trade.close_profit == 0.06201058 assert trade.close_date is not None + assert log_has("LIMIT_SELL has been fulfilled for Trade(id=2, " + "pair=ETH/BTC, amount=90.99181073, open_rate=0.00001099, open_since=closed).", + caplog.record_tuples) @pytest.mark.usefixtures("init_persistence") -def test_update_market_order(market_buy_order, market_sell_order, fee): +def test_update_market_order(market_buy_order, market_sell_order, fee, caplog): trade = Trade( + id=1, pair='ETH/BTC', stake_amount=0.001, fee_open=fee.return_value, @@ -133,13 +142,20 @@ def test_update_market_order(market_buy_order, market_sell_order, fee): assert trade.open_rate == 0.00004099 assert trade.close_profit is None assert trade.close_date is None + assert log_has("MARKET_BUY has been fulfilled for Trade(id=1, " + "pair=ETH/BTC, amount=91.99181073, open_rate=0.00004099, open_since=closed).", + caplog.record_tuples) + caplog.clear() trade.open_order_id = 'something' trade.update(market_sell_order) assert trade.open_order_id is None assert trade.close_rate == 0.00004173 assert trade.close_profit == 0.01297561 assert trade.close_date is not None + assert log_has("MARKET_SELL has been fulfilled for Trade(id=1, " + "pair=ETH/BTC, amount=91.99181073, open_rate=0.00004099, open_since=closed).", + caplog.record_tuples) @pytest.mark.usefixtures("init_persistence") From 23d0cea01fde5dbfdb648fd50dde28854992bab3 Mon Sep 17 00:00:00 2001 From: Matthias Date: Thu, 27 Dec 2018 11:45:21 +0100 Subject: [PATCH 699/699] Version bump to 0.18.0 --- freqtrade/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/__init__.py b/freqtrade/__init__.py index 1d73e815e..2319135c6 100644 --- a/freqtrade/__init__.py +++ b/freqtrade/__init__.py @@ -1,5 +1,5 @@ """ FreqTrade bot """ -__version__ = '0.17.4' +__version__ = '0.18.0' class DependencyException(BaseException):