From c048e7229a3c98d1d99d1f0d27bae7d666cd95b9 Mon Sep 17 00:00:00 2001
From: Stefano Ariestasia <stefano.ariestasia@gmail.com>
Date: Sat, 22 Jul 2023 08:36:51 +0900
Subject: [PATCH 01/26] modify expectancy and expectancy ratio

---
 freqtrade/data/metrics.py                     | 53 ++++++++++++++-----
 .../optimize/optimize_reports/bt_output.py    |  2 +-
 .../optimize_reports/optimize_reports.py      |  7 +--
 freqtrade/rpc/rpc.py                          |  3 +-
 tests/data/test_btanalysis.py                 | 20 +++----
 5 files changed, 57 insertions(+), 28 deletions(-)

diff --git a/freqtrade/data/metrics.py b/freqtrade/data/metrics.py
index 138903b57..fe0d03522 100644
--- a/freqtrade/data/metrics.py
+++ b/freqtrade/data/metrics.py
@@ -203,21 +203,48 @@ def calculate_expectancy(trades: pd.DataFrame) -> float:
     if len(trades) == 0:
         return 0
 
-    expectancy = 1
+    expectancy = 0
 
-    profit_sum = trades.loc[trades['profit_abs'] > 0, 'profit_abs'].sum()
-    loss_sum = abs(trades.loc[trades['profit_abs'] < 0, 'profit_abs'].sum())
-    nb_win_trades = len(trades.loc[trades['profit_abs'] > 0])
-    nb_loss_trades = len(trades.loc[trades['profit_abs'] < 0])
+    winning_trades = trades.loc[trades['profit_abs'] > 0]
+    losing_trades = trades.loc[trades['profit_abs'] < 0]
+    profit_sum = winning_trades['profit_abs'].sum()
+    loss_sum = abs(losing_trades['profit_abs'].sum())
+    nb_win_trades = len(winning_trades)
+    nb_loss_trades = len(losing_trades)
 
-    if (nb_win_trades > 0) and (nb_loss_trades > 0):
-        average_win = profit_sum / nb_win_trades
-        average_loss = loss_sum / nb_loss_trades
-        risk_reward_ratio = average_win / average_loss
-        winrate = nb_win_trades / len(trades)
-        expectancy = ((1 + risk_reward_ratio) * winrate) - 1
-    elif nb_win_trades == 0:
-        expectancy = 0
+    average_win = profit_sum / nb_win_trades
+    average_loss = loss_sum / nb_loss_trades
+    winrate = nb_win_trades / len(trades)
+    loserate = nb_loss_trades / len(trades)
+    expectancy = (winrate * average_win) - (loserate * average_loss)
+
+    return expectancy
+
+
+def calculate_expectancy_ratio(trades: pd.DataFrame) -> float:
+    """
+    Calculate expectancy ratio
+    :param trades: DataFrame containing trades (requires columns close_date and profit_abs)
+    :return: expectancy ratio
+    """
+
+    expectancy_ratio = float('inf')
+
+    if len(trades) > 0:
+        winning_trades = trades.loc[trades['profit_abs'] > 0]
+        losing_trades = trades.loc[trades['profit_abs'] < 0]
+        profit_sum = winning_trades['profit_abs'].sum()
+        loss_sum = abs(losing_trades['profit_abs'].sum())
+        nb_win_trades = len(winning_trades)
+        nb_loss_trades = len(losing_trades)
+
+        average_win = (profit_sum / nb_win_trades) if nb_win_trades > 0 else 0
+        average_loss = (loss_sum / nb_loss_trades) if nb_loss_trades > 0 else 0
+
+        if (average_loss > 0):
+            risk_reward_ratio = average_win / average_loss
+            winrate = nb_win_trades / len(trades)
+            expectancy = ((1 + risk_reward_ratio) * winrate) - 1
 
     return expectancy
 
diff --git a/freqtrade/optimize/optimize_reports/bt_output.py b/freqtrade/optimize/optimize_reports/bt_output.py
index 1fd1f7a34..15d1030f2 100644
--- a/freqtrade/optimize/optimize_reports/bt_output.py
+++ b/freqtrade/optimize/optimize_reports/bt_output.py
@@ -233,7 +233,7 @@ def text_table_add_metrics(strat_results: Dict) -> str:
             ('Calmar', f"{strat_results['calmar']:.2f}" if 'calmar' in strat_results else 'N/A'),
             ('Profit factor', f'{strat_results["profit_factor"]:.2f}' if 'profit_factor'
                               in strat_results else 'N/A'),
-            ('Expectancy', f"{strat_results['expectancy']:.2f}" if 'expectancy'
+            ('Expectancy Ratio', f"{strat_results['expectancy_ratio']:.2f}" if 'expectancy_ratio'
                            in strat_results else 'N/A'),
             ('Trades per day', strat_results['trades_per_day']),
             ('Avg. daily profit %',
diff --git a/freqtrade/optimize/optimize_reports/optimize_reports.py b/freqtrade/optimize/optimize_reports/optimize_reports.py
index bb1106d38..854469975 100644
--- a/freqtrade/optimize/optimize_reports/optimize_reports.py
+++ b/freqtrade/optimize/optimize_reports/optimize_reports.py
@@ -7,8 +7,9 @@ from pandas import DataFrame, concat, to_datetime
 
 from freqtrade.constants import BACKTEST_BREAKDOWNS, DATETIME_PRINT_FORMAT, IntOrInf
 from freqtrade.data.metrics import (calculate_cagr, calculate_calmar, calculate_csum,
-                                    calculate_expectancy, calculate_market_change,
-                                    calculate_max_drawdown, calculate_sharpe, calculate_sortino)
+                                    calculate_expectancy, calculate_expectancy_ratio,
+                                    calculate_market_change, calculate_max_drawdown,
+                                    calculate_sharpe, calculate_sortino)
 from freqtrade.misc import decimals_per_coin, round_coin_value
 
 
@@ -414,7 +415,7 @@ def generate_strategy_stats(pairlist: List[str],
         'profit_total_long_abs': results.loc[~results['is_short'], 'profit_abs'].sum(),
         'profit_total_short_abs': results.loc[results['is_short'], 'profit_abs'].sum(),
         'cagr': calculate_cagr(backtest_days, start_balance, content['final_balance']),
-        'expectancy': calculate_expectancy(results),
+        'expectancy_ratio': calculate_expectancy_ratio(results),
         'sortino': calculate_sortino(results, min_date, max_date, start_balance),
         'sharpe': calculate_sharpe(results, min_date, max_date, start_balance),
         'calmar': calculate_calmar(results, min_date, max_date, start_balance),
diff --git a/freqtrade/rpc/rpc.py b/freqtrade/rpc/rpc.py
index b1e8520a5..440de5460 100644
--- a/freqtrade/rpc/rpc.py
+++ b/freqtrade/rpc/rpc.py
@@ -18,7 +18,8 @@ from freqtrade import __version__
 from freqtrade.configuration.timerange import TimeRange
 from freqtrade.constants import CANCEL_REASON, DATETIME_PRINT_FORMAT, Config
 from freqtrade.data.history import load_data
-from freqtrade.data.metrics import calculate_max_drawdown
+from freqtrade.data.metrics import (calculate_expectancy, calculate_expectancy_ratio,
+                                    calculate_max_drawdown)
 from freqtrade.enums import (CandleType, ExitCheckTuple, ExitType, MarketDirection, SignalDirection,
                              State, TradingMode)
 from freqtrade.exceptions import ExchangeError, PricingError
diff --git a/tests/data/test_btanalysis.py b/tests/data/test_btanalysis.py
index 5e377f851..76a33b386 100644
--- a/tests/data/test_btanalysis.py
+++ b/tests/data/test_btanalysis.py
@@ -13,10 +13,10 @@ from freqtrade.data.btanalysis import (BT_DATA_COLUMNS, analyze_trade_parallelis
                                        load_backtest_metadata, load_trades, load_trades_from_db)
 from freqtrade.data.history import load_data, load_pair_history
 from freqtrade.data.metrics import (calculate_cagr, calculate_calmar, calculate_csum,
-                                    calculate_expectancy, calculate_market_change,
-                                    calculate_max_drawdown, calculate_sharpe, calculate_sortino,
-                                    calculate_underwater, combine_dataframes_with_mean,
-                                    create_cum_profit)
+                                    calculate_expectancy, calculate_expectancy_ratio,
+                                    calculate_market_change, calculate_max_drawdown,
+                                    calculate_sharpe, calculate_sortino, calculate_underwater,
+                                    combine_dataframes_with_mean, create_cum_profit)
 from freqtrade.exceptions import OperationalException
 from freqtrade.util import dt_utc
 from tests.conftest import CURRENT_TEST_STRATEGY, create_mock_trades
@@ -339,16 +339,16 @@ def test_calculate_csum(testdatadir):
         csum_min, csum_max = calculate_csum(DataFrame())
 
 
-def test_calculate_expectancy(testdatadir):
+def test_calculate_expectancy_ratio(testdatadir):
     filename = testdatadir / "backtest_results/backtest-result.json"
     bt_data = load_backtest_data(filename)
 
-    expectancy = calculate_expectancy(DataFrame())
-    assert expectancy == 0.0
+    expectancy_ratio = calculate_expectancy_ratio(DataFrame())
+    assert expectancy_ratio == 0.0
 
-    expectancy = calculate_expectancy(bt_data)
-    assert isinstance(expectancy, float)
-    assert pytest.approx(expectancy) == 0.07151374226574791
+    expectancy_ratio = calculate_expectancy_ratio(bt_data)
+    assert isinstance(expectancy_ratio, float)
+    assert pytest.approx(expectancy_ratio) == 0.07151374226574791
 
 
 def test_calculate_sortino(testdatadir):

From 4812bcc28b04f0202b36f7340c0870fa6c7259ed Mon Sep 17 00:00:00 2001
From: Stefano Ariestasia <stefano.ariestasia@gmail.com>
Date: Sat, 22 Jul 2023 09:13:24 +0900
Subject: [PATCH 02/26] flake8 fiz

---
 freqtrade/data/metrics.py                             |  4 ++--
 freqtrade/optimize/optimize_reports/bt_output.py      |  2 +-
 .../optimize/optimize_reports/optimize_reports.py     |  5 ++---
 freqtrade/rpc/rpc.py                                  | 11 +++++++----
 4 files changed, 12 insertions(+), 10 deletions(-)

diff --git a/freqtrade/data/metrics.py b/freqtrade/data/metrics.py
index fe0d03522..294f66a66 100644
--- a/freqtrade/data/metrics.py
+++ b/freqtrade/data/metrics.py
@@ -244,9 +244,9 @@ def calculate_expectancy_ratio(trades: pd.DataFrame) -> float:
         if (average_loss > 0):
             risk_reward_ratio = average_win / average_loss
             winrate = nb_win_trades / len(trades)
-            expectancy = ((1 + risk_reward_ratio) * winrate) - 1
+            expectancy_ratio = ((1 + risk_reward_ratio) * winrate) - 1
 
-    return expectancy
+    return expectancy_ratio
 
 
 def calculate_sortino(trades: pd.DataFrame, min_date: datetime, max_date: datetime,
diff --git a/freqtrade/optimize/optimize_reports/bt_output.py b/freqtrade/optimize/optimize_reports/bt_output.py
index 15d1030f2..e833c5ce4 100644
--- a/freqtrade/optimize/optimize_reports/bt_output.py
+++ b/freqtrade/optimize/optimize_reports/bt_output.py
@@ -234,7 +234,7 @@ def text_table_add_metrics(strat_results: Dict) -> str:
             ('Profit factor', f'{strat_results["profit_factor"]:.2f}' if 'profit_factor'
                               in strat_results else 'N/A'),
             ('Expectancy Ratio', f"{strat_results['expectancy_ratio']:.2f}" if 'expectancy_ratio'
-                           in strat_results else 'N/A'),
+                                 in strat_results else 'N/A'),
             ('Trades per day', strat_results['trades_per_day']),
             ('Avg. daily profit %',
              f"{(strat_results['profit_total'] / strat_results['backtest_days']):.2%}"),
diff --git a/freqtrade/optimize/optimize_reports/optimize_reports.py b/freqtrade/optimize/optimize_reports/optimize_reports.py
index 854469975..e555077a8 100644
--- a/freqtrade/optimize/optimize_reports/optimize_reports.py
+++ b/freqtrade/optimize/optimize_reports/optimize_reports.py
@@ -7,9 +7,8 @@ from pandas import DataFrame, concat, to_datetime
 
 from freqtrade.constants import BACKTEST_BREAKDOWNS, DATETIME_PRINT_FORMAT, IntOrInf
 from freqtrade.data.metrics import (calculate_cagr, calculate_calmar, calculate_csum,
-                                    calculate_expectancy, calculate_expectancy_ratio,
-                                    calculate_market_change, calculate_max_drawdown,
-                                    calculate_sharpe, calculate_sortino)
+                                    calculate_expectancy_ratio, calculate_market_change,
+                                    calculate_max_drawdown, calculate_sharpe, calculate_sortino)
 from freqtrade.misc import decimals_per_coin, round_coin_value
 
 
diff --git a/freqtrade/rpc/rpc.py b/freqtrade/rpc/rpc.py
index 440de5460..e9adec793 100644
--- a/freqtrade/rpc/rpc.py
+++ b/freqtrade/rpc/rpc.py
@@ -530,10 +530,13 @@ class RPC:
         winrate = (winning_trades / closed_trade_count) if closed_trade_count > 0 else 0
         loserate = (1 - winrate)
 
-        expectancy, expectancy_ratio = self.__calc_expectancy(mean_winning_profit,
-                                                              mean_losing_profit,
-                                                              winrate,
-                                                              loserate)
+        # expectancy, expectancy_ratio = self.__calc_expectancy(mean_winning_profit,
+        #                                                       mean_losing_profit,
+        #                                                       winrate,
+        #                                                       loserate)
+
+        expectancy = calculate_expectancy(trades)
+        expectancy_ratio = calculate_expectancy_ratio(trades)
 
         trades_df = DataFrame([{'close_date': trade.close_date.strftime(DATETIME_PRINT_FORMAT),
                                 'profit_abs': trade.close_profit_abs}

From dcc3ef1309f4be6123bdd73016e9c97c361407d8 Mon Sep 17 00:00:00 2001
From: Stefano Ariestasia <stefano.ariestasia@gmail.com>
Date: Sat, 22 Jul 2023 09:18:22 +0900
Subject: [PATCH 03/26] flake8 fix

---
 freqtrade/rpc/rpc.py          | 7 ++++---
 tests/data/test_btanalysis.py | 8 ++++----
 2 files changed, 8 insertions(+), 7 deletions(-)

diff --git a/freqtrade/rpc/rpc.py b/freqtrade/rpc/rpc.py
index e9adec793..7d7dc4e77 100644
--- a/freqtrade/rpc/rpc.py
+++ b/freqtrade/rpc/rpc.py
@@ -535,12 +535,13 @@ class RPC:
         #                                                       winrate,
         #                                                       loserate)
 
-        expectancy = calculate_expectancy(trades)
-        expectancy_ratio = calculate_expectancy_ratio(trades)
-
         trades_df = DataFrame([{'close_date': trade.close_date.strftime(DATETIME_PRINT_FORMAT),
                                 'profit_abs': trade.close_profit_abs}
                                for trade in trades if not trade.is_open and trade.close_date])
+
+        expectancy = calculate_expectancy(trades_df)
+        expectancy_ratio = calculate_expectancy_ratio(trades_df)
+        
         max_drawdown_abs = 0.0
         max_drawdown = 0.0
         if len(trades_df) > 0:
diff --git a/tests/data/test_btanalysis.py b/tests/data/test_btanalysis.py
index 76a33b386..83b0182d5 100644
--- a/tests/data/test_btanalysis.py
+++ b/tests/data/test_btanalysis.py
@@ -13,10 +13,10 @@ from freqtrade.data.btanalysis import (BT_DATA_COLUMNS, analyze_trade_parallelis
                                        load_backtest_metadata, load_trades, load_trades_from_db)
 from freqtrade.data.history import load_data, load_pair_history
 from freqtrade.data.metrics import (calculate_cagr, calculate_calmar, calculate_csum,
-                                    calculate_expectancy, calculate_expectancy_ratio,
-                                    calculate_market_change, calculate_max_drawdown,
-                                    calculate_sharpe, calculate_sortino, calculate_underwater,
-                                    combine_dataframes_with_mean, create_cum_profit)
+                                    calculate_expectancy_ratio, calculate_market_change,
+                                    calculate_max_drawdown, calculate_sharpe, calculate_sortino,
+                                    calculate_underwater, combine_dataframes_with_mean,
+                                    create_cum_profit)
 from freqtrade.exceptions import OperationalException
 from freqtrade.util import dt_utc
 from tests.conftest import CURRENT_TEST_STRATEGY, create_mock_trades

From 11f24aff97e55ecf3b89b19870e3f40c661d46c2 Mon Sep 17 00:00:00 2001
From: Stefano Ariestasia <stefano.ariestasia@gmail.com>
Date: Sat, 22 Jul 2023 09:19:36 +0900
Subject: [PATCH 04/26] flake8

---
 freqtrade/rpc/rpc.py | 6 +-----
 1 file changed, 1 insertion(+), 5 deletions(-)

diff --git a/freqtrade/rpc/rpc.py b/freqtrade/rpc/rpc.py
index 7d7dc4e77..aaf91e456 100644
--- a/freqtrade/rpc/rpc.py
+++ b/freqtrade/rpc/rpc.py
@@ -524,11 +524,7 @@ class RPC:
 
         profit_factor = winning_profit / abs(losing_profit) if losing_profit else float('inf')
 
-        mean_winning_profit = (winning_profit / winning_trades) if winning_trades > 0 else 0
-        mean_losing_profit = (abs(losing_profit) / losing_trades) if losing_trades > 0 else 0
-
         winrate = (winning_trades / closed_trade_count) if closed_trade_count > 0 else 0
-        loserate = (1 - winrate)
 
         # expectancy, expectancy_ratio = self.__calc_expectancy(mean_winning_profit,
         #                                                       mean_losing_profit,
@@ -541,7 +537,7 @@ class RPC:
 
         expectancy = calculate_expectancy(trades_df)
         expectancy_ratio = calculate_expectancy_ratio(trades_df)
-        
+
         max_drawdown_abs = 0.0
         max_drawdown = 0.0
         if len(trades_df) > 0:

From 8621dc96e7fae04ecd0b6d01c474cc0106a23164 Mon Sep 17 00:00:00 2001
From: Stefano Ariestasia <stefano.ariestasia@gmail.com>
Date: Sat, 22 Jul 2023 09:44:24 +0900
Subject: [PATCH 05/26] fix tests

---
 freqtrade/data/metrics.py | 9 ++-------
 1 file changed, 2 insertions(+), 7 deletions(-)

diff --git a/freqtrade/data/metrics.py b/freqtrade/data/metrics.py
index 294f66a66..cea88b88f 100644
--- a/freqtrade/data/metrics.py
+++ b/freqtrade/data/metrics.py
@@ -200,11 +200,6 @@ def calculate_expectancy(trades: pd.DataFrame) -> float:
     :param trades: DataFrame containing trades (requires columns close_date and profit_abs)
     :return: expectancy
     """
-    if len(trades) == 0:
-        return 0
-
-    expectancy = 0
-
     winning_trades = trades.loc[trades['profit_abs'] > 0]
     losing_trades = trades.loc[trades['profit_abs'] < 0]
     profit_sum = winning_trades['profit_abs'].sum()
@@ -212,8 +207,8 @@ def calculate_expectancy(trades: pd.DataFrame) -> float:
     nb_win_trades = len(winning_trades)
     nb_loss_trades = len(losing_trades)
 
-    average_win = profit_sum / nb_win_trades
-    average_loss = loss_sum / nb_loss_trades
+    average_win = (profit_sum / nb_win_trades) if nb_win_trades > 0 else 0
+    average_loss = (loss_sum / nb_loss_trades) if nb_loss_trades > 0 else 0
     winrate = nb_win_trades / len(trades)
     loserate = nb_loss_trades / len(trades)
     expectancy = (winrate * average_win) - (loserate * average_loss)

From cfd8b068e7d8d82da0244d09c0fea86175685faf Mon Sep 17 00:00:00 2001
From: Stefano Ariestasia <stefano.ariestasia@gmail.com>
Date: Sat, 22 Jul 2023 11:25:53 +0900
Subject: [PATCH 06/26] add test for expectancy

---
 freqtrade/data/metrics.py     |  4 ++--
 tests/data/test_btanalysis.py | 10 ++++++++++
 2 files changed, 12 insertions(+), 2 deletions(-)

diff --git a/freqtrade/data/metrics.py b/freqtrade/data/metrics.py
index cea88b88f..ea28b17ce 100644
--- a/freqtrade/data/metrics.py
+++ b/freqtrade/data/metrics.py
@@ -209,8 +209,8 @@ def calculate_expectancy(trades: pd.DataFrame) -> float:
 
     average_win = (profit_sum / nb_win_trades) if nb_win_trades > 0 else 0
     average_loss = (loss_sum / nb_loss_trades) if nb_loss_trades > 0 else 0
-    winrate = nb_win_trades / len(trades)
-    loserate = nb_loss_trades / len(trades)
+    winrate = (nb_win_trades / len(trades)) if len(trades) > 0 else 0
+    loserate = (nb_loss_trades / len(trades)) if len(trades) > 0 else 0
     expectancy = (winrate * average_win) - (loserate * average_loss)
 
     return expectancy
diff --git a/tests/data/test_btanalysis.py b/tests/data/test_btanalysis.py
index 83b0182d5..cbf3bcf10 100644
--- a/tests/data/test_btanalysis.py
+++ b/tests/data/test_btanalysis.py
@@ -338,6 +338,16 @@ def test_calculate_csum(testdatadir):
     with pytest.raises(ValueError, match='Trade dataframe empty.'):
         csum_min, csum_max = calculate_csum(DataFrame())
 
+def test_calculate_expectancy(testdatadir):
+    filename = testdatadir / "backtest_results/backtest-result.json"
+    bt_data = load_backtest_data(filename)
+
+    expectancy = calculate_expectancy(DataFrame())
+    assert expectancy == 0.0
+
+    expectancy = calculate_expectancy(bt_data)
+    assert isinstance(expectancy, float)
+    assert pytest.approx(expectancy) == 0.07151374226574791
 
 def test_calculate_expectancy_ratio(testdatadir):
     filename = testdatadir / "backtest_results/backtest-result.json"

From b0639ab319675a56a612d3e03fe6fa9bb20f2c9d Mon Sep 17 00:00:00 2001
From: Stefano Ariestasia <stefano.ariestasia@gmail.com>
Date: Sat, 22 Jul 2023 11:29:08 +0900
Subject: [PATCH 07/26] flake8

---
 tests/data/test_btanalysis.py | 10 ++++++----
 1 file changed, 6 insertions(+), 4 deletions(-)

diff --git a/tests/data/test_btanalysis.py b/tests/data/test_btanalysis.py
index cbf3bcf10..cb78e6a9f 100644
--- a/tests/data/test_btanalysis.py
+++ b/tests/data/test_btanalysis.py
@@ -13,10 +13,10 @@ from freqtrade.data.btanalysis import (BT_DATA_COLUMNS, analyze_trade_parallelis
                                        load_backtest_metadata, load_trades, load_trades_from_db)
 from freqtrade.data.history import load_data, load_pair_history
 from freqtrade.data.metrics import (calculate_cagr, calculate_calmar, calculate_csum,
-                                    calculate_expectancy_ratio, calculate_market_change,
-                                    calculate_max_drawdown, calculate_sharpe, calculate_sortino,
-                                    calculate_underwater, combine_dataframes_with_mean,
-                                    create_cum_profit)
+                                    calculate_expectancy, calculate_expectancy_ratio,
+                                    calculate_market_change, calculate_max_drawdown,
+                                    calculate_sharpe, calculate_sortino, calculate_underwater,
+                                    combine_dataframes_with_mean, create_cum_profit)
 from freqtrade.exceptions import OperationalException
 from freqtrade.util import dt_utc
 from tests.conftest import CURRENT_TEST_STRATEGY, create_mock_trades
@@ -338,6 +338,7 @@ def test_calculate_csum(testdatadir):
     with pytest.raises(ValueError, match='Trade dataframe empty.'):
         csum_min, csum_max = calculate_csum(DataFrame())
 
+
 def test_calculate_expectancy(testdatadir):
     filename = testdatadir / "backtest_results/backtest-result.json"
     bt_data = load_backtest_data(filename)
@@ -349,6 +350,7 @@ def test_calculate_expectancy(testdatadir):
     assert isinstance(expectancy, float)
     assert pytest.approx(expectancy) == 0.07151374226574791
 
+
 def test_calculate_expectancy_ratio(testdatadir):
     filename = testdatadir / "backtest_results/backtest-result.json"
     bt_data = load_backtest_data(filename)

From ee3b69ea63d854e8a685027f33f983924ec8d8ce Mon Sep 17 00:00:00 2001
From: Stefano Ariestasia <stefano.ariestasia@gmail.com>
Date: Sat, 22 Jul 2023 11:37:22 +0900
Subject: [PATCH 08/26] fix test

---
 freqtrade/data/metrics.py | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/freqtrade/data/metrics.py b/freqtrade/data/metrics.py
index ea28b17ce..271446339 100644
--- a/freqtrade/data/metrics.py
+++ b/freqtrade/data/metrics.py
@@ -200,6 +200,10 @@ def calculate_expectancy(trades: pd.DataFrame) -> float:
     :param trades: DataFrame containing trades (requires columns close_date and profit_abs)
     :return: expectancy
     """
+
+    if len(trades) == 0:
+        return 0
+
     winning_trades = trades.loc[trades['profit_abs'] > 0]
     losing_trades = trades.loc[trades['profit_abs'] < 0]
     profit_sum = winning_trades['profit_abs'].sum()

From 3dd33cde00a9fdf5330595fa104b02771e0dbffd Mon Sep 17 00:00:00 2001
From: Stefano Ariestasia <stefano.ariestasia@gmail.com>
Date: Sat, 22 Jul 2023 11:39:10 +0900
Subject: [PATCH 09/26] fix test

---
 tests/data/test_btanalysis.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/data/test_btanalysis.py b/tests/data/test_btanalysis.py
index cb78e6a9f..4cd11a684 100644
--- a/tests/data/test_btanalysis.py
+++ b/tests/data/test_btanalysis.py
@@ -348,7 +348,7 @@ def test_calculate_expectancy(testdatadir):
 
     expectancy = calculate_expectancy(bt_data)
     assert isinstance(expectancy, float)
-    assert pytest.approx(expectancy) == 0.07151374226574791
+    assert pytest.approx(expectancy) == 5.820687070932315e-06
 
 
 def test_calculate_expectancy_ratio(testdatadir):

From 3552fa431b01b35d7b8a7e2b18c793c61d43cf54 Mon Sep 17 00:00:00 2001
From: Stefano Ariestasia <stefano.ariestasia@gmail.com>
Date: Sat, 22 Jul 2023 11:40:52 +0900
Subject: [PATCH 10/26] fix test

---
 tests/data/test_btanalysis.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/data/test_btanalysis.py b/tests/data/test_btanalysis.py
index 4cd11a684..8594ad4ec 100644
--- a/tests/data/test_btanalysis.py
+++ b/tests/data/test_btanalysis.py
@@ -356,7 +356,7 @@ def test_calculate_expectancy_ratio(testdatadir):
     bt_data = load_backtest_data(filename)
 
     expectancy_ratio = calculate_expectancy_ratio(DataFrame())
-    assert expectancy_ratio == 0.0
+    assert expectancy_ratio == float('inf')
 
     expectancy_ratio = calculate_expectancy_ratio(bt_data)
     assert isinstance(expectancy_ratio, float)

From c6ee8fcf5444e64eaa19d60968df46014538a165 Mon Sep 17 00:00:00 2001
From: Stefano Ariestasia <stefano.ariestasia@gmail.com>
Date: Sat, 22 Jul 2023 12:20:35 +0900
Subject: [PATCH 11/26] remove unused check

---
 freqtrade/data/metrics.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/freqtrade/data/metrics.py b/freqtrade/data/metrics.py
index 271446339..67660bfdf 100644
--- a/freqtrade/data/metrics.py
+++ b/freqtrade/data/metrics.py
@@ -213,8 +213,8 @@ def calculate_expectancy(trades: pd.DataFrame) -> float:
 
     average_win = (profit_sum / nb_win_trades) if nb_win_trades > 0 else 0
     average_loss = (loss_sum / nb_loss_trades) if nb_loss_trades > 0 else 0
-    winrate = (nb_win_trades / len(trades)) if len(trades) > 0 else 0
-    loserate = (nb_loss_trades / len(trades)) if len(trades) > 0 else 0
+    winrate = (nb_win_trades / len(trades))
+    loserate = (nb_loss_trades / len(trades))
     expectancy = (winrate * average_win) - (loserate * average_loss)
 
     return expectancy

From 40d7d05e4e2ed6b271d2ace0be8ee677a71b634f Mon Sep 17 00:00:00 2001
From: Stefano Ariestasia <stefano.ariestasia@gmail.com>
Date: Sat, 22 Jul 2023 17:29:43 +0900
Subject: [PATCH 12/26] merge 2 expectancy functions

---
 docs/backtesting.md                           |  4 +--
 freqtrade/data/metrics.py                     | 33 +++++++++++--------
 .../optimize/optimize_reports/bt_output.py    |  4 +--
 .../optimize_reports/optimize_reports.py      |  6 ++--
 freqtrade/rpc/rpc.py                          | 27 ++-------------
 tests/data/test_btanalysis.py                 | 25 +++++---------
 6 files changed, 37 insertions(+), 62 deletions(-)

diff --git a/docs/backtesting.md b/docs/backtesting.md
index 166c2b28b..abaf00a53 100644
--- a/docs/backtesting.md
+++ b/docs/backtesting.md
@@ -305,7 +305,7 @@ A backtesting result will look like that:
 | Sharpe                      | 2.97                |
 | Calmar                      | 6.29                |
 | Profit factor               | 1.11                |
-| Expectancy                  | -0.15               |
+| Expectancy (Ratio)          | -0.15 (-0.05)       |
 | Avg. stake amount           | 0.001      BTC      |
 | Total trade volume          | 0.429      BTC      |
 |                             |                     |
@@ -409,7 +409,7 @@ It contains some useful key metrics about performance of your strategy on backte
 | Sharpe                      | 2.97                |
 | Calmar                      | 6.29                |
 | Profit factor               | 1.11                |
-| Expectancy                  | -0.15               |
+| Expectancy (Ratio)          | -0.15 (-0.05)       |
 | Avg. stake amount           | 0.001      BTC      |
 | Total trade volume          | 0.429      BTC      |
 |                             |                     |
diff --git a/freqtrade/data/metrics.py b/freqtrade/data/metrics.py
index 67660bfdf..631ed71dd 100644
--- a/freqtrade/data/metrics.py
+++ b/freqtrade/data/metrics.py
@@ -201,23 +201,28 @@ def calculate_expectancy(trades: pd.DataFrame) -> float:
     :return: expectancy
     """
 
-    if len(trades) == 0:
-        return 0
+    expectancy = 0
+    expectancy_ratio = float('inf')
 
-    winning_trades = trades.loc[trades['profit_abs'] > 0]
-    losing_trades = trades.loc[trades['profit_abs'] < 0]
-    profit_sum = winning_trades['profit_abs'].sum()
-    loss_sum = abs(losing_trades['profit_abs'].sum())
-    nb_win_trades = len(winning_trades)
-    nb_loss_trades = len(losing_trades)
+    if len(trades) > 0:
+        winning_trades = trades.loc[trades['profit_abs'] > 0]
+        losing_trades = trades.loc[trades['profit_abs'] < 0]
+        profit_sum = winning_trades['profit_abs'].sum()
+        loss_sum = abs(losing_trades['profit_abs'].sum())
+        nb_win_trades = len(winning_trades)
+        nb_loss_trades = len(losing_trades)
 
-    average_win = (profit_sum / nb_win_trades) if nb_win_trades > 0 else 0
-    average_loss = (loss_sum / nb_loss_trades) if nb_loss_trades > 0 else 0
-    winrate = (nb_win_trades / len(trades))
-    loserate = (nb_loss_trades / len(trades))
-    expectancy = (winrate * average_win) - (loserate * average_loss)
+        average_win = (profit_sum / nb_win_trades) if nb_win_trades > 0 else 0
+        average_loss = (loss_sum / nb_loss_trades) if nb_loss_trades > 0 else 0
+        winrate = (nb_win_trades / len(trades))
+        loserate = (nb_loss_trades / len(trades))
 
-    return expectancy
+        expectancy = (winrate * average_win) - (loserate * average_loss)
+        if (average_loss > 0):
+            risk_reward_ratio = average_win / average_loss
+            expectancy_ratio = ((1 + risk_reward_ratio) * winrate) - 1
+
+    return expectancy, expectancy_ratio
 
 
 def calculate_expectancy_ratio(trades: pd.DataFrame) -> float:
diff --git a/freqtrade/optimize/optimize_reports/bt_output.py b/freqtrade/optimize/optimize_reports/bt_output.py
index e833c5ce4..85e91dc2a 100644
--- a/freqtrade/optimize/optimize_reports/bt_output.py
+++ b/freqtrade/optimize/optimize_reports/bt_output.py
@@ -233,8 +233,8 @@ def text_table_add_metrics(strat_results: Dict) -> str:
             ('Calmar', f"{strat_results['calmar']:.2f}" if 'calmar' in strat_results else 'N/A'),
             ('Profit factor', f'{strat_results["profit_factor"]:.2f}' if 'profit_factor'
                               in strat_results else 'N/A'),
-            ('Expectancy Ratio', f"{strat_results['expectancy_ratio']:.2f}" if 'expectancy_ratio'
-                                 in strat_results else 'N/A'),
+            ('Expectancy (Ratio)', f"{strat_results['expectancy']:.2f} "
+                                   f"({strat_results['expectancy_ratio']:.2f})"),
             ('Trades per day', strat_results['trades_per_day']),
             ('Avg. daily profit %',
              f"{(strat_results['profit_total'] / strat_results['backtest_days']):.2%}"),
diff --git a/freqtrade/optimize/optimize_reports/optimize_reports.py b/freqtrade/optimize/optimize_reports/optimize_reports.py
index e555077a8..729f61cc8 100644
--- a/freqtrade/optimize/optimize_reports/optimize_reports.py
+++ b/freqtrade/optimize/optimize_reports/optimize_reports.py
@@ -7,7 +7,7 @@ from pandas import DataFrame, concat, to_datetime
 
 from freqtrade.constants import BACKTEST_BREAKDOWNS, DATETIME_PRINT_FORMAT, IntOrInf
 from freqtrade.data.metrics import (calculate_cagr, calculate_calmar, calculate_csum,
-                                    calculate_expectancy_ratio, calculate_market_change,
+                                    calculate_expectancy, calculate_market_change,
                                     calculate_max_drawdown, calculate_sharpe, calculate_sortino)
 from freqtrade.misc import decimals_per_coin, round_coin_value
 
@@ -389,6 +389,7 @@ def generate_strategy_stats(pairlist: List[str],
     losing_profit = results.loc[results['profit_abs'] < 0, 'profit_abs'].sum()
     profit_factor = winning_profit / abs(losing_profit) if losing_profit else 0.0
 
+    expectancy, expectancy_ratio = calculate_expectancy(results)
     backtest_days = (max_date - min_date).days or 1
     strat_stats = {
         'trades': results.to_dict(orient='records'),
@@ -414,7 +415,8 @@ def generate_strategy_stats(pairlist: List[str],
         'profit_total_long_abs': results.loc[~results['is_short'], 'profit_abs'].sum(),
         'profit_total_short_abs': results.loc[results['is_short'], 'profit_abs'].sum(),
         'cagr': calculate_cagr(backtest_days, start_balance, content['final_balance']),
-        'expectancy_ratio': calculate_expectancy_ratio(results),
+        'expectancy': expectancy,
+        'expectancy_ratio': expectancy_ratio,
         'sortino': calculate_sortino(results, min_date, max_date, start_balance),
         'sharpe': calculate_sharpe(results, min_date, max_date, start_balance),
         'calmar': calculate_calmar(results, min_date, max_date, start_balance),
diff --git a/freqtrade/rpc/rpc.py b/freqtrade/rpc/rpc.py
index aaf91e456..a1b6317b6 100644
--- a/freqtrade/rpc/rpc.py
+++ b/freqtrade/rpc/rpc.py
@@ -18,8 +18,7 @@ from freqtrade import __version__
 from freqtrade.configuration.timerange import TimeRange
 from freqtrade.constants import CANCEL_REASON, DATETIME_PRINT_FORMAT, Config
 from freqtrade.data.history import load_data
-from freqtrade.data.metrics import (calculate_expectancy, calculate_expectancy_ratio,
-                                    calculate_max_drawdown)
+from freqtrade.data.metrics import calculate_expectancy, calculate_max_drawdown
 from freqtrade.enums import (CandleType, ExitCheckTuple, ExitType, MarketDirection, SignalDirection,
                              State, TradingMode)
 from freqtrade.exceptions import ExchangeError, PricingError
@@ -526,17 +525,11 @@ class RPC:
 
         winrate = (winning_trades / closed_trade_count) if closed_trade_count > 0 else 0
 
-        # expectancy, expectancy_ratio = self.__calc_expectancy(mean_winning_profit,
-        #                                                       mean_losing_profit,
-        #                                                       winrate,
-        #                                                       loserate)
-
         trades_df = DataFrame([{'close_date': trade.close_date.strftime(DATETIME_PRINT_FORMAT),
                                 'profit_abs': trade.close_profit_abs}
                                for trade in trades if not trade.is_open and trade.close_date])
 
-        expectancy = calculate_expectancy(trades_df)
-        expectancy_ratio = calculate_expectancy_ratio(trades_df)
+        expectancy, expectancy_ratio = calculate_expectancy(trades_df)
 
         max_drawdown_abs = 0.0
         max_drawdown = 0.0
@@ -626,22 +619,6 @@ class RPC:
 
         return est_stake, est_bot_stake
 
-    def __calc_expectancy(
-            self, mean_winning_profit: float, mean_losing_profit: float,
-            winrate: float, loserate: float) -> Tuple[float, float]:
-
-        expectancy = (
-            (winrate * mean_winning_profit) -
-            (loserate * mean_losing_profit)
-        )
-
-        expectancy_ratio = float('inf')
-        if mean_losing_profit > 0:
-            expectancy_ratio = (
-                ((1 + (mean_winning_profit / mean_losing_profit)) * winrate) - 1
-            )
-
-        return expectancy, expectancy_ratio
 
     def _rpc_balance(self, stake_currency: str, fiat_display_currency: str) -> Dict:
         """ Returns current account balance per crypto """
diff --git a/tests/data/test_btanalysis.py b/tests/data/test_btanalysis.py
index 8594ad4ec..6a100904c 100644
--- a/tests/data/test_btanalysis.py
+++ b/tests/data/test_btanalysis.py
@@ -13,10 +13,10 @@ from freqtrade.data.btanalysis import (BT_DATA_COLUMNS, analyze_trade_parallelis
                                        load_backtest_metadata, load_trades, load_trades_from_db)
 from freqtrade.data.history import load_data, load_pair_history
 from freqtrade.data.metrics import (calculate_cagr, calculate_calmar, calculate_csum,
-                                    calculate_expectancy, calculate_expectancy_ratio,
-                                    calculate_market_change, calculate_max_drawdown,
-                                    calculate_sharpe, calculate_sortino, calculate_underwater,
-                                    combine_dataframes_with_mean, create_cum_profit)
+                                    calculate_expectancy, calculate_market_change,
+                                    calculate_max_drawdown, calculate_sharpe, calculate_sortino,
+                                    calculate_underwater, combine_dataframes_with_mean,
+                                    create_cum_profit)
 from freqtrade.exceptions import OperationalException
 from freqtrade.util import dt_utc
 from tests.conftest import CURRENT_TEST_STRATEGY, create_mock_trades
@@ -343,23 +343,14 @@ def test_calculate_expectancy(testdatadir):
     filename = testdatadir / "backtest_results/backtest-result.json"
     bt_data = load_backtest_data(filename)
 
-    expectancy = calculate_expectancy(DataFrame())
+    expectancy, expectancy_ratio = calculate_expectancy(DataFrame())
     assert expectancy == 0.0
-
-    expectancy = calculate_expectancy(bt_data)
-    assert isinstance(expectancy, float)
-    assert pytest.approx(expectancy) == 5.820687070932315e-06
-
-
-def test_calculate_expectancy_ratio(testdatadir):
-    filename = testdatadir / "backtest_results/backtest-result.json"
-    bt_data = load_backtest_data(filename)
-
-    expectancy_ratio = calculate_expectancy_ratio(DataFrame())
     assert expectancy_ratio == float('inf')
 
-    expectancy_ratio = calculate_expectancy_ratio(bt_data)
+    expectancy, expectancy_ratio = calculate_expectancy(bt_data)
+    assert isinstance(expectancy, float)
     assert isinstance(expectancy_ratio, float)
+    assert pytest.approx(expectancy) == 5.820687070932315e-06
     assert pytest.approx(expectancy_ratio) == 0.07151374226574791
 
 

From e5f01ab2e8b105bcecdba89d2e0efff12f20c63b Mon Sep 17 00:00:00 2001
From: Stefano Ariestasia <stefano.ariestasia@gmail.com>
Date: Sat, 22 Jul 2023 17:45:58 +0900
Subject: [PATCH 13/26] pre-commit fix

---
 freqtrade/data/metrics.py | 30 +-----------------------------
 freqtrade/rpc/rpc.py      |  1 -
 2 files changed, 1 insertion(+), 30 deletions(-)

diff --git a/freqtrade/data/metrics.py b/freqtrade/data/metrics.py
index 631ed71dd..2cc95fa79 100644
--- a/freqtrade/data/metrics.py
+++ b/freqtrade/data/metrics.py
@@ -194,7 +194,7 @@ def calculate_cagr(days_passed: int, starting_balance: float, final_balance: flo
     return (final_balance / starting_balance) ** (1 / (days_passed / 365)) - 1
 
 
-def calculate_expectancy(trades: pd.DataFrame) -> float:
+def calculate_expectancy(trades: pd.DataFrame) -> Tuple[float, float]:
     """
     Calculate expectancy
     :param trades: DataFrame containing trades (requires columns close_date and profit_abs)
@@ -225,34 +225,6 @@ def calculate_expectancy(trades: pd.DataFrame) -> float:
     return expectancy, expectancy_ratio
 
 
-def calculate_expectancy_ratio(trades: pd.DataFrame) -> float:
-    """
-    Calculate expectancy ratio
-    :param trades: DataFrame containing trades (requires columns close_date and profit_abs)
-    :return: expectancy ratio
-    """
-
-    expectancy_ratio = float('inf')
-
-    if len(trades) > 0:
-        winning_trades = trades.loc[trades['profit_abs'] > 0]
-        losing_trades = trades.loc[trades['profit_abs'] < 0]
-        profit_sum = winning_trades['profit_abs'].sum()
-        loss_sum = abs(losing_trades['profit_abs'].sum())
-        nb_win_trades = len(winning_trades)
-        nb_loss_trades = len(losing_trades)
-
-        average_win = (profit_sum / nb_win_trades) if nb_win_trades > 0 else 0
-        average_loss = (loss_sum / nb_loss_trades) if nb_loss_trades > 0 else 0
-
-        if (average_loss > 0):
-            risk_reward_ratio = average_win / average_loss
-            winrate = nb_win_trades / len(trades)
-            expectancy_ratio = ((1 + risk_reward_ratio) * winrate) - 1
-
-    return expectancy_ratio
-
-
 def calculate_sortino(trades: pd.DataFrame, min_date: datetime, max_date: datetime,
                       starting_balance: float) -> float:
     """
diff --git a/freqtrade/rpc/rpc.py b/freqtrade/rpc/rpc.py
index a1b6317b6..32d6dfef3 100644
--- a/freqtrade/rpc/rpc.py
+++ b/freqtrade/rpc/rpc.py
@@ -619,7 +619,6 @@ class RPC:
 
         return est_stake, est_bot_stake
 
-
     def _rpc_balance(self, stake_currency: str, fiat_display_currency: str) -> Dict:
         """ Returns current account balance per crypto """
         currencies: List[Dict] = []

From 27a36bfb40dc651e526d028040baaf2265d06869 Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Sat, 22 Jul 2023 15:30:58 +0200
Subject: [PATCH 14/26] Bump lightgbm from 3.3.5 to 4.0.0 (#8923)

* Bump lightgbm from 3.3.5 to 4.0.0

Bumps [lightgbm](https://github.com/microsoft/LightGBM) from 3.3.5 to 4.0.0.
- [Release notes](https://github.com/microsoft/LightGBM/releases)
- [Commits](https://github.com/microsoft/LightGBM/compare/v3.3.5...v4.0.0)

---
updated-dependencies:
- dependency-name: lightgbm
  dependency-type: direct:production
  update-type: version-update:semver-major
...

Signed-off-by: dependabot[bot] <support@github.com>

* fix: ensure freqai lightgbm variants conform to v4.0.0

* remove random file

---------

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
Co-authored-by: robcaulk <rob.caulk@gmail.com>
---
 freqtrade/freqai/prediction_models/LightGBMClassifier.py     | 5 ++---
 freqtrade/freqai/prediction_models/LightGBMRegressor.py      | 2 +-
 .../freqai/prediction_models/LightGBMRegressorMultiTarget.py | 4 ++--
 requirements-freqai.txt                                      | 2 +-
 4 files changed, 6 insertions(+), 7 deletions(-)

diff --git a/freqtrade/freqai/prediction_models/LightGBMClassifier.py b/freqtrade/freqai/prediction_models/LightGBMClassifier.py
index 45f3a31d0..4c481adff 100644
--- a/freqtrade/freqai/prediction_models/LightGBMClassifier.py
+++ b/freqtrade/freqai/prediction_models/LightGBMClassifier.py
@@ -32,8 +32,8 @@ class LightGBMClassifier(BaseClassifierModel):
             eval_set = None
             test_weights = None
         else:
-            eval_set = (data_dictionary["test_features"].to_numpy(),
-                        data_dictionary["test_labels"].to_numpy()[:, 0])
+            eval_set = [(data_dictionary["test_features"].to_numpy(),
+                        data_dictionary["test_labels"].to_numpy()[:, 0])]
             test_weights = data_dictionary["test_weights"]
         X = data_dictionary["train_features"].to_numpy()
         y = data_dictionary["train_labels"].to_numpy()[:, 0]
@@ -42,7 +42,6 @@ class LightGBMClassifier(BaseClassifierModel):
         init_model = self.get_init_model(dk.pair)
 
         model = LGBMClassifier(**self.model_training_parameters)
-
         model.fit(X=X, y=y, eval_set=eval_set, sample_weight=train_weights,
                   eval_sample_weight=[test_weights], init_model=init_model)
 
diff --git a/freqtrade/freqai/prediction_models/LightGBMRegressor.py b/freqtrade/freqai/prediction_models/LightGBMRegressor.py
index 3d1c30ed3..15849f446 100644
--- a/freqtrade/freqai/prediction_models/LightGBMRegressor.py
+++ b/freqtrade/freqai/prediction_models/LightGBMRegressor.py
@@ -32,7 +32,7 @@ class LightGBMRegressor(BaseRegressionModel):
             eval_set = None
             eval_weights = None
         else:
-            eval_set = (data_dictionary["test_features"], data_dictionary["test_labels"])
+            eval_set = [(data_dictionary["test_features"], data_dictionary["test_labels"])]
             eval_weights = data_dictionary["test_weights"]
         X = data_dictionary["train_features"]
         y = data_dictionary["train_labels"]
diff --git a/freqtrade/freqai/prediction_models/LightGBMRegressorMultiTarget.py b/freqtrade/freqai/prediction_models/LightGBMRegressorMultiTarget.py
index 663a611f0..5827dcefe 100644
--- a/freqtrade/freqai/prediction_models/LightGBMRegressorMultiTarget.py
+++ b/freqtrade/freqai/prediction_models/LightGBMRegressorMultiTarget.py
@@ -42,10 +42,10 @@ class LightGBMRegressorMultiTarget(BaseRegressionModel):
             eval_weights = [data_dictionary["test_weights"]]
             eval_sets = [(None, None)] * data_dictionary['test_labels'].shape[1]  # type: ignore
             for i in range(data_dictionary['test_labels'].shape[1]):
-                eval_sets[i] = (  # type: ignore
+                eval_sets[i] = [(  # type: ignore
                     data_dictionary["test_features"],
                     data_dictionary["test_labels"].iloc[:, i]
-                )
+                )]
 
         init_model = self.get_init_model(dk.pair)
         if init_model:
diff --git a/requirements-freqai.txt b/requirements-freqai.txt
index ceb5488a6..325b92544 100644
--- a/requirements-freqai.txt
+++ b/requirements-freqai.txt
@@ -6,7 +6,7 @@
 scikit-learn==1.1.3
 joblib==1.3.1
 catboost==1.2; 'arm' not in platform_machine
-lightgbm==3.3.5
+lightgbm==4.0.0
 xgboost==1.7.6
 tensorboard==2.13.0
 datasieve==0.1.7

From 955a63725a4e0210afc9faf17d16af23bc13aded Mon Sep 17 00:00:00 2001
From: Matthias <xmatthias@outlook.com>
Date: Sat, 22 Jul 2023 19:43:20 +0200
Subject: [PATCH 15/26] Improve resiliance when showing older backtest results

---
 freqtrade/data/metrics.py                        | 2 +-
 freqtrade/optimize/optimize_reports/bt_output.py | 5 +++--
 tests/data/test_btanalysis.py                    | 9 +++++++++
 3 files changed, 13 insertions(+), 3 deletions(-)

diff --git a/freqtrade/data/metrics.py b/freqtrade/data/metrics.py
index 2cc95fa79..d4746c168 100644
--- a/freqtrade/data/metrics.py
+++ b/freqtrade/data/metrics.py
@@ -198,7 +198,7 @@ def calculate_expectancy(trades: pd.DataFrame) -> Tuple[float, float]:
     """
     Calculate expectancy
     :param trades: DataFrame containing trades (requires columns close_date and profit_abs)
-    :return: expectancy
+    :return: expectancy, expectancy_ratio
     """
 
     expectancy = 0
diff --git a/freqtrade/optimize/optimize_reports/bt_output.py b/freqtrade/optimize/optimize_reports/bt_output.py
index 85e91dc2a..07f23a5fa 100644
--- a/freqtrade/optimize/optimize_reports/bt_output.py
+++ b/freqtrade/optimize/optimize_reports/bt_output.py
@@ -233,8 +233,9 @@ def text_table_add_metrics(strat_results: Dict) -> str:
             ('Calmar', f"{strat_results['calmar']:.2f}" if 'calmar' in strat_results else 'N/A'),
             ('Profit factor', f'{strat_results["profit_factor"]:.2f}' if 'profit_factor'
                               in strat_results else 'N/A'),
-            ('Expectancy (Ratio)', f"{strat_results['expectancy']:.2f} "
-                                   f"({strat_results['expectancy_ratio']:.2f})"),
+            ('Expectancy (Ratio)', (
+                f"{strat_results['expectancy']:.2f} ({strat_results['expectancy_ratio']:.2f})" if
+                'expectancy_ratio' in strat_results else 'N/A')),
             ('Trades per day', strat_results['trades_per_day']),
             ('Avg. daily profit %',
              f"{(strat_results['profit_total'] / strat_results['backtest_days']):.2%}"),
diff --git a/tests/data/test_btanalysis.py b/tests/data/test_btanalysis.py
index 6a100904c..f11e5d772 100644
--- a/tests/data/test_btanalysis.py
+++ b/tests/data/test_btanalysis.py
@@ -353,6 +353,15 @@ def test_calculate_expectancy(testdatadir):
     assert pytest.approx(expectancy) == 5.820687070932315e-06
     assert pytest.approx(expectancy_ratio) == 0.07151374226574791
 
+    data = {
+        'profit_abs': [100, 200, 50, -150, 300, -100, 80, -30]
+    }
+    df = DataFrame(data)
+    expectancy, expectancy_ratio = calculate_expectancy(df)
+
+    assert pytest.approx(expectancy) == 56.25
+    assert pytest.approx(expectancy_ratio) == 0.60267857
+
 
 def test_calculate_sortino(testdatadir):
     filename = testdatadir / "backtest_results/backtest-result.json"

From 787e94924dd7459d2cbaa92e2a389a18fec3b767 Mon Sep 17 00:00:00 2001
From: Matthias <xmatthias@outlook.com>
Date: Sun, 23 Jul 2023 07:20:59 +0200
Subject: [PATCH 16/26] Update default expectancy ratio to 100

---
 freqtrade/data/metrics.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/freqtrade/data/metrics.py b/freqtrade/data/metrics.py
index d4746c168..c22dcccef 100644
--- a/freqtrade/data/metrics.py
+++ b/freqtrade/data/metrics.py
@@ -202,7 +202,7 @@ def calculate_expectancy(trades: pd.DataFrame) -> Tuple[float, float]:
     """
 
     expectancy = 0
-    expectancy_ratio = float('inf')
+    expectancy_ratio = 100
 
     if len(trades) > 0:
         winning_trades = trades.loc[trades['profit_abs'] > 0]

From 0eddc6b7add6d3b5b02d9559f0650e70b6980884 Mon Sep 17 00:00:00 2001
From: Stefano Ariestasia <stefano.ariestasia@gmail.com>
Date: Sun, 23 Jul 2023 14:27:45 +0900
Subject: [PATCH 17/26] update expectancy test

---
 tests/data/test_btanalysis.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/data/test_btanalysis.py b/tests/data/test_btanalysis.py
index f11e5d772..c1b007e77 100644
--- a/tests/data/test_btanalysis.py
+++ b/tests/data/test_btanalysis.py
@@ -345,7 +345,7 @@ def test_calculate_expectancy(testdatadir):
 
     expectancy, expectancy_ratio = calculate_expectancy(DataFrame())
     assert expectancy == 0.0
-    assert expectancy_ratio == float('inf')
+    assert expectancy_ratio == 100
 
     expectancy, expectancy_ratio = calculate_expectancy(bt_data)
     assert isinstance(expectancy, float)

From 4c23771d398ace1424b21dae5567e29a5b1f1736 Mon Sep 17 00:00:00 2001
From: Stefano Ariestasia <stefano.ariestasia@gmail.com>
Date: Sun, 23 Jul 2023 14:42:48 +0900
Subject: [PATCH 18/26] fix expectancy test

---
 tests/rpc/test_rpc_apiserver.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/rpc/test_rpc_apiserver.py b/tests/rpc/test_rpc_apiserver.py
index e0b87b9cc..fd9508060 100644
--- a/tests/rpc/test_rpc_apiserver.py
+++ b/tests/rpc/test_rpc_apiserver.py
@@ -846,7 +846,7 @@ def test_api_edge_disabled(botclient, mocker, ticker, fee, markets):
          'profit_closed_percent_sum': 1.5, 'profit_closed_ratio': 7.391275897987988e-07,
          'profit_closed_percent': 0.0, 'winning_trades': 2, 'losing_trades': 0,
          'profit_factor': None, 'winrate': 1.0, 'expectancy': 0.0003695635,
-         'expectancy_ratio': None, 'trading_volume': 91.074,
+         'expectancy_ratio': 100, 'trading_volume': 91.074,
          }
     ),
     (

From 8f04225282b16bbe906dcfc3f975272edefad00c Mon Sep 17 00:00:00 2001
From: Stefano Ariestasia <stefano.ariestasia@gmail.com>
Date: Sun, 23 Jul 2023 15:00:08 +0900
Subject: [PATCH 19/26] another test fix

---
 tests/rpc/test_rpc.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/rpc/test_rpc.py b/tests/rpc/test_rpc.py
index 65db6770a..3bc725f3a 100644
--- a/tests/rpc/test_rpc.py
+++ b/tests/rpc/test_rpc.py
@@ -403,7 +403,7 @@ def test_rpc_trade_statistics(default_conf_usdt, ticker, fee, mocker) -> None:
     assert res['latest_trade_date'] == ''
     assert res['latest_trade_timestamp'] == 0
     assert res['expectancy'] == 0
-    assert res['expectancy_ratio'] == float('inf')
+    assert res['expectancy_ratio'] == 100
 
     # Create some test data
     create_mock_trades_usdt(fee)

From 889a732e065286aa45b4aed57fbc408e53154e36 Mon Sep 17 00:00:00 2001
From: Matthias <xmatthias@outlook.com>
Date: Sun, 23 Jul 2023 17:57:04 +0200
Subject: [PATCH 20/26] Enhance bybit documentation

---
 docs/exchanges.md | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/docs/exchanges.md b/docs/exchanges.md
index 997d012e1..800a1c605 100644
--- a/docs/exchanges.md
+++ b/docs/exchanges.md
@@ -259,10 +259,15 @@ The configuration parameter `exchange.unknown_fee_rate` can be used to specify t
 
 Futures trading on bybit is currently supported for USDT markets, and will use isolated futures mode.
 Users with unified accounts (there's no way back) can create a Sub-account which will start as "non-unified", and can therefore use isolated futures.
-On startup, freqtrade will set the position mode to "One-way Mode" for the whole (sub)account. This avoids making this call over and over again (slowing down bot operations), but means that changes to this setting may result in exceptions and errors.
+On startup, freqtrade will set the position mode to "One-way Mode" for the whole (sub)account. This avoids making this call over and over again (slowing down bot operations), but means that changes to this setting may result in exceptions and errors
 
 As bybit doesn't provide funding rate history, the dry-run calculation is used for live trades as well.
 
+API Keys for live futures trading (Subaccount on non-unified) must have the following permissions:
+* Read-write
+* Contract - Orders
+* Contract - Positions
+
 !!! Tip "Stoploss on Exchange"
     Bybit (futures only) supports `stoploss_on_exchange` and uses `stop-loss-limit` orders. It provides great advantages, so we recommend to benefit from it by enabling stoploss on exchange.
     On futures, Bybit supports both `stop-limit` as well as `stop-market` orders. You can use either `"limit"` or `"market"` in the `order_types.stoploss` configuration setting to decide which type to use.

From 4549fb349c466b30c6dfba1ffd208774412398f7 Mon Sep 17 00:00:00 2001
From: Matthias <xmatthias@outlook.com>
Date: Sun, 23 Jul 2023 17:57:48 +0200
Subject: [PATCH 21/26] Add security notice about IP whitelisting on bybit docs

---
 docs/exchanges.md | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/docs/exchanges.md b/docs/exchanges.md
index 800a1c605..fb3049ba5 100644
--- a/docs/exchanges.md
+++ b/docs/exchanges.md
@@ -268,6 +268,8 @@ API Keys for live futures trading (Subaccount on non-unified) must have the foll
 * Contract - Orders
 * Contract - Positions
 
+We do strongly recommend to limit all API keys to the IP you're going to use it from.
+
 !!! Tip "Stoploss on Exchange"
     Bybit (futures only) supports `stoploss_on_exchange` and uses `stop-loss-limit` orders. It provides great advantages, so we recommend to benefit from it by enabling stoploss on exchange.
     On futures, Bybit supports both `stop-limit` as well as `stop-market` orders. You can use either `"limit"` or `"market"` in the `order_types.stoploss` configuration setting to decide which type to use.

From 8dfe43f370d3b4d323a4271864c11f680c23612d Mon Sep 17 00:00:00 2001
From: Matthias <xmatthias@outlook.com>
Date: Sun, 23 Jul 2023 18:28:43 +0200
Subject: [PATCH 22/26] Add timeout for webhooks

---
 freqtrade/rpc/webhook.py | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)

diff --git a/freqtrade/rpc/webhook.py b/freqtrade/rpc/webhook.py
index 80690ec0c..b9bdbd435 100644
--- a/freqtrade/rpc/webhook.py
+++ b/freqtrade/rpc/webhook.py
@@ -34,6 +34,7 @@ class Webhook(RPCHandler):
         self._format = self._config['webhook'].get('format', 'form')
         self._retries = self._config['webhook'].get('retries', 0)
         self._retry_delay = self._config['webhook'].get('retry_delay', 0.1)
+        self._timeout = self._config['webhook'].get('timeout', 10)
 
     def cleanup(self) -> None:
         """
@@ -107,12 +108,13 @@ class Webhook(RPCHandler):
 
             try:
                 if self._format == 'form':
-                    response = post(self._url, data=payload)
+                    response = post(self._url, data=payload, timeout=self._timeout)
                 elif self._format == 'json':
-                    response = post(self._url, json=payload)
+                    response = post(self._url, json=payload, timeout=self._timeout)
                 elif self._format == 'raw':
                     response = post(self._url, data=payload['data'],
-                                    headers={'Content-Type': 'text/plain'})
+                                    headers={'Content-Type': 'text/plain'},
+                                    timeout=self._timeout)
                 else:
                     raise NotImplementedError(f'Unknown format: {self._format}')
 

From ad82ad44073e01450c1d7d92199d862ebfc3c990 Mon Sep 17 00:00:00 2001
From: Matthias <xmatthias@outlook.com>
Date: Sun, 23 Jul 2023 18:28:49 +0200
Subject: [PATCH 23/26] webhook docs improvements

---
 docs/webhook-config.md | 10 +++++++++-
 1 file changed, 9 insertions(+), 1 deletion(-)

diff --git a/docs/webhook-config.md b/docs/webhook-config.md
index 00c369919..e18a05e9b 100644
--- a/docs/webhook-config.md
+++ b/docs/webhook-config.md
@@ -80,12 +80,18 @@ When using the Form-Encoded or JSON-Encoded configuration you can configure any
 
 The result would be a POST request with e.g. `Status: running` body and `Content-Type: text/plain` header.
 
-Optional parameters are available to enable automatic retries for webhook messages. The `webhook.retries` parameter can be set for the maximum number of retries the webhook request should attempt if it is unsuccessful (i.e. HTTP response status is not 200). By default this is set to `0` which is disabled. An additional `webhook.retry_delay` parameter can be set to specify the time in seconds between retry attempts. By default this is set to `0.1` (i.e. 100ms). Note that increasing the number of retries or retry delay may slow down the trader if there are connectivity issues with the webhook. Example configuration for retries:
+## Additional configurations
+
+The `webhook.retries` parameter can be set for the maximum number of retries the webhook request should attempt if it is unsuccessful (i.e. HTTP response status is not 200). By default this is set to `0` which is disabled. An additional `webhook.retry_delay` parameter can be set to specify the time in seconds between retry attempts. By default this is set to `0.1` (i.e. 100ms). Note that increasing the number of retries or retry delay may slow down the trader if there are connectivity issues with the webhook.
+You can also specify `webhook.timeout` - which defines how long the bot will wait until it assumes the other host as unresponsive (defaults to 10s).
+
+Example configuration for retries:
 
 ```json
   "webhook": {
         "enabled": true,
         "url": "https://<YOURHOOKURL>",
+        "timeout": 10,
         "retries": 3,
         "retry_delay": 0.2,
         "status": {
@@ -109,6 +115,8 @@ Custom messages can be sent to Webhook endpoints via the `self.dp.send_msg()` fu
 
 Different payloads can be configured for different events. Not all fields are necessary, but you should configure at least one of the dicts, otherwise the webhook will never be called.
 
+## Webhook Message types
+
 ### Entry
 
 The fields in `webhook.entry` are filled when the bot executes a long/short. Parameters are filled using string.format.

From d7916366bd32f448a328f9a9d805953df1cf7a84 Mon Sep 17 00:00:00 2001
From: Matthias <xmatthias@outlook.com>
Date: Sun, 23 Jul 2023 19:21:55 +0200
Subject: [PATCH 24/26] Adjust webhook tests to include timeout

---
 tests/rpc/test_rpc_webhook.py | 10 +++++++---
 1 file changed, 7 insertions(+), 3 deletions(-)

diff --git a/tests/rpc/test_rpc_webhook.py b/tests/rpc/test_rpc_webhook.py
index d0a0f5b1e..36b96ace5 100644
--- a/tests/rpc/test_rpc_webhook.py
+++ b/tests/rpc/test_rpc_webhook.py
@@ -381,7 +381,7 @@ def test__send_msg(default_conf, mocker, caplog):
     webhook._send_msg(msg)
 
     assert post.call_count == 1
-    assert post.call_args[1] == {'data': msg}
+    assert post.call_args[1] == {'data': msg, 'timeout': 10}
     assert post.call_args[0] == (default_conf['webhook']['url'], )
 
     post = MagicMock(side_effect=RequestException)
@@ -399,7 +399,7 @@ def test__send_msg_with_json_format(default_conf, mocker, caplog):
     mocker.patch("freqtrade.rpc.webhook.post", post)
     webhook._send_msg(msg)
 
-    assert post.call_args[1] == {'json': msg}
+    assert post.call_args[1] == {'json': msg, 'timeout': 10}
 
 
 def test__send_msg_with_raw_format(default_conf, mocker, caplog):
@@ -411,7 +411,11 @@ def test__send_msg_with_raw_format(default_conf, mocker, caplog):
     mocker.patch("freqtrade.rpc.webhook.post", post)
     webhook._send_msg(msg)
 
-    assert post.call_args[1] == {'data': msg['data'], 'headers': {'Content-Type': 'text/plain'}}
+    assert post.call_args[1] == {
+        'data': msg['data'],
+        'headers': {'Content-Type': 'text/plain'},
+        'timeout': 10
+    }
 
 
 def test_send_msg_discord(default_conf, mocker):

From fb7fb7f59c895ea094e77957d1e6be610fd39e0d Mon Sep 17 00:00:00 2001
From: Matthias <xmatthias@outlook.com>
Date: Sun, 23 Jul 2023 19:38:30 +0200
Subject: [PATCH 25/26] Add helper function detecting (prebuilt) docker
 environment

---
 freqtrade/configuration/detect_environment.py   | 8 ++++++++
 freqtrade/configuration/directory_operations.py | 4 ++--
 2 files changed, 10 insertions(+), 2 deletions(-)
 create mode 100644 freqtrade/configuration/detect_environment.py

diff --git a/freqtrade/configuration/detect_environment.py b/freqtrade/configuration/detect_environment.py
new file mode 100644
index 000000000..99d585e87
--- /dev/null
+++ b/freqtrade/configuration/detect_environment.py
@@ -0,0 +1,8 @@
+import os
+
+
+def running_in_docker() -> bool:
+    """
+    Check if we are running in a docker container
+    """
+    return os.environ.get('FT_APP_ENV') == 'docker'
diff --git a/freqtrade/configuration/directory_operations.py b/freqtrade/configuration/directory_operations.py
index e1313749b..267a74928 100644
--- a/freqtrade/configuration/directory_operations.py
+++ b/freqtrade/configuration/directory_operations.py
@@ -3,6 +3,7 @@ import shutil
 from pathlib import Path
 from typing import Optional
 
+from freqtrade.configuration.detect_environment import running_in_docker
 from freqtrade.constants import (USER_DATA_FILES, USERPATH_FREQAIMODELS, USERPATH_HYPEROPTS,
                                  USERPATH_NOTEBOOKS, USERPATH_STRATEGIES, Config)
 from freqtrade.exceptions import OperationalException
@@ -30,8 +31,7 @@ def chown_user_directory(directory: Path) -> None:
     Use Sudo to change permissions of the home-directory if necessary
     Only applies when running in docker!
     """
-    import os
-    if os.environ.get('FT_APP_ENV') == 'docker':
+    if running_in_docker():
         try:
             import subprocess
             subprocess.check_output(

From a00fcd68f87c0b5fcbcaf1abcff4ee4442a6b02e Mon Sep 17 00:00:00 2001
From: Matthias <xmatthias@outlook.com>
Date: Sun, 23 Jul 2023 19:44:43 +0200
Subject: [PATCH 26/26] Default to 0.0.0.0 if on API listen address for configs
 generated through docker

---
 freqtrade/commands/build_config_commands.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/freqtrade/commands/build_config_commands.py b/freqtrade/commands/build_config_commands.py
index 63bb5c211..311622458 100644
--- a/freqtrade/commands/build_config_commands.py
+++ b/freqtrade/commands/build_config_commands.py
@@ -5,6 +5,7 @@ from typing import Any, Dict, List
 
 from questionary import Separator, prompt
 
+from freqtrade.configuration.detect_environment import running_in_docker
 from freqtrade.configuration.directory_operations import chown_user_directory
 from freqtrade.constants import UNLIMITED_STAKE_AMOUNT
 from freqtrade.exceptions import OperationalException
@@ -179,7 +180,7 @@ def ask_user_config() -> Dict[str, Any]:
             "name": "api_server_listen_addr",
             "message": ("Insert Api server Listen Address (0.0.0.0 for docker, "
                         "otherwise best left untouched)"),
-            "default": "127.0.0.1",
+            "default": "127.0.0.1" if not running_in_docker() else "0.0.0.0",
             "when": lambda x: x['api_server']
         },
         {