From ffc4d8726346f4bf03f8c0bbae6bc5535392b013 Mon Sep 17 00:00:00 2001 From: robcaulk Date: Wed, 10 May 2023 09:48:36 +0000 Subject: [PATCH 01/25] add tensorboard integration to XGBoost and PyTorch et al --- docs/freqai-running.md | 5 +- freqtrade/freqai/__init__.py | 14 ++++ freqtrade/freqai/freqai_interface.py | 3 + .../prediction_models/PyTorchMLPClassifier.py | 1 + .../prediction_models/PyTorchMLPRegressor.py | 1 + .../PyTorchTransformerRegressor.py | 4 +- .../prediction_models/XGBoostRegressor.py | 4 + freqtrade/freqai/tensorboard.py | 77 +++++++++++++++++++ freqtrade/freqai/torch/PyTorchModelTrainer.py | 19 ++--- 9 files changed, 112 insertions(+), 16 deletions(-) create mode 100644 freqtrade/freqai/tensorboard.py diff --git a/docs/freqai-running.md b/docs/freqai-running.md index f3ccc546f..5ea67651f 100644 --- a/docs/freqai-running.md +++ b/docs/freqai-running.md @@ -158,7 +158,10 @@ This specific hyperopt would help you understand the appropriate `DI_values` for ## Using Tensorboard -CatBoost models benefit from tracking training metrics via Tensorboard. You can take advantage of the FreqAI integration to track training and evaluation performance across all coins and across all retrainings. Tensorboard is activated via the following command: +!!! note "Availability" + FreqAI includes tensorboard for a variety of models, including XGBoost, all PyTorch models, Reinforcement Learning, and Catboost. If you would like to see Tensorboard integrated into another model type, please open an issue on the [Freqtrade GitHub](https://github.com/freqtrade/freqtrade/issues) + +The easiest way to use tensorboard is to open a separate shell and run: ```bash cd freqtrade diff --git a/freqtrade/freqai/__init__.py b/freqtrade/freqai/__init__.py index e69de29bb..5fb6e5be0 100644 --- a/freqtrade/freqai/__init__.py +++ b/freqtrade/freqai/__init__.py @@ -0,0 +1,14 @@ +# ensure users can still use a non-torch freqai version +try: + from freqtrade.freqai.tensorboard import TensorBoardCallback, TensorboardLogger + TBLogger = TensorboardLogger + TBCallback = TensorBoardCallback +except ModuleNotFoundError: + from freqtrade.freqai.tensorboard import BaseTensorBoardCallback, BaseTensorboardLogger + TBLogger = BaseTensorboardLogger # type: ignore + TBCallback = BaseTensorBoardCallback # type: ignore + +__all__ = ( + "TBLogger", + "TBCallback" +) diff --git a/freqtrade/freqai/freqai_interface.py b/freqtrade/freqai/freqai_interface.py index 1669d1483..cf8097870 100644 --- a/freqtrade/freqai/freqai_interface.py +++ b/freqtrade/freqai/freqai_interface.py @@ -19,6 +19,7 @@ from freqtrade.data.dataprovider import DataProvider from freqtrade.enums import RunMode from freqtrade.exceptions import OperationalException from freqtrade.exchange import timeframe_to_seconds +from freqtrade.freqai import TBLogger from freqtrade.freqai.data_drawer import FreqaiDataDrawer from freqtrade.freqai.data_kitchen import FreqaiDataKitchen from freqtrade.freqai.utils import plot_feature_importance, record_params @@ -630,7 +631,9 @@ class IFreqaiModel(ABC): dk.find_features(unfiltered_dataframe) dk.find_labels(unfiltered_dataframe) + self.tb_logger = TBLogger(dk.data_path) model = self.train(unfiltered_dataframe, pair, dk) + self.tb_logger.close() self.dd.pair_dict[pair]["trained_timestamp"] = new_trained_timerange.stopts dk.set_new_model_names(pair, new_trained_timerange.stopts) diff --git a/freqtrade/freqai/prediction_models/PyTorchMLPClassifier.py b/freqtrade/freqai/prediction_models/PyTorchMLPClassifier.py index ea7981405..0ebe8d129 100644 --- a/freqtrade/freqai/prediction_models/PyTorchMLPClassifier.py +++ b/freqtrade/freqai/prediction_models/PyTorchMLPClassifier.py @@ -83,6 +83,7 @@ class PyTorchMLPClassifier(BasePyTorchClassifier): device=self.device, init_model=init_model, data_convertor=self.data_convertor, + tb_logger=self.tb_logger, **self.trainer_kwargs, ) trainer.fit(data_dictionary, self.splits) diff --git a/freqtrade/freqai/prediction_models/PyTorchMLPRegressor.py b/freqtrade/freqai/prediction_models/PyTorchMLPRegressor.py index 64f0f4b03..7d87f5226 100644 --- a/freqtrade/freqai/prediction_models/PyTorchMLPRegressor.py +++ b/freqtrade/freqai/prediction_models/PyTorchMLPRegressor.py @@ -77,6 +77,7 @@ class PyTorchMLPRegressor(BasePyTorchRegressor): device=self.device, init_model=init_model, data_convertor=self.data_convertor, + tb_logger=self.tb_logger, **self.trainer_kwargs, ) trainer.fit(data_dictionary, self.splits) diff --git a/freqtrade/freqai/prediction_models/PyTorchTransformerRegressor.py b/freqtrade/freqai/prediction_models/PyTorchTransformerRegressor.py index e760f6e68..d135b690b 100644 --- a/freqtrade/freqai/prediction_models/PyTorchTransformerRegressor.py +++ b/freqtrade/freqai/prediction_models/PyTorchTransformerRegressor.py @@ -32,8 +32,7 @@ class PyTorchTransformerRegressor(BasePyTorchRegressor): "trainer_kwargs": { "max_iters": 5000, "batch_size": 64, - "max_n_eval_batches": null, - "window_size": 10 + "max_n_eval_batches": null }, "model_kwargs": { "hidden_dim": 512, @@ -84,6 +83,7 @@ class PyTorchTransformerRegressor(BasePyTorchRegressor): init_model=init_model, data_convertor=self.data_convertor, window_size=self.window_size, + tb_logger=self.tb_logger, **self.trainer_kwargs, ) trainer.fit(data_dictionary, self.splits) diff --git a/freqtrade/freqai/prediction_models/XGBoostRegressor.py b/freqtrade/freqai/prediction_models/XGBoostRegressor.py index 93dfb319e..b4cdead65 100644 --- a/freqtrade/freqai/prediction_models/XGBoostRegressor.py +++ b/freqtrade/freqai/prediction_models/XGBoostRegressor.py @@ -3,6 +3,7 @@ from typing import Any, Dict from xgboost import XGBRegressor +from freqtrade.freqai import TBCallback from freqtrade.freqai.base_models.BaseRegressionModel import BaseRegressionModel from freqtrade.freqai.data_kitchen import FreqaiDataKitchen @@ -44,7 +45,10 @@ class XGBoostRegressor(BaseRegressionModel): model = XGBRegressor(**self.model_training_parameters) + model.set_params(callbacks=[TBCallback(dk.data_path)]) model.fit(X=X, y=y, sample_weight=sample_weight, eval_set=eval_set, sample_weight_eval_set=eval_weights, xgb_model=xgb_model) + # set the callbacks to empty so that we can serialize to disk later + model.set_params(callbacks=[]) return model diff --git a/freqtrade/freqai/tensorboard.py b/freqtrade/freqai/tensorboard.py new file mode 100644 index 000000000..cb536008e --- /dev/null +++ b/freqtrade/freqai/tensorboard.py @@ -0,0 +1,77 @@ +import logging +from pathlib import Path +from typing import Any + +import xgboost as xgb + + +logger = logging.getLogger(__name__) + + +class BaseTensorboardLogger: + def __init__(self, logdir: str = "tensorboard", id: str = "unique-id"): + logger.warning("Tensorboard is not installed, no logs will be written." + "Use ensure torch is installed, or use the torch/RL docker images") + + def log_scaler(self, tag: str, scalar_value: Any, step: int): + return + + def close(self): + return + + +class BaseTensorBoardCallback(xgb.callback.TrainingCallback): + + def __init__(self, logdir: str = "tensorboard", id: str = "uniqu-id", test_size=1): + logger.warning("Tensorboard is not installed, no logs will be written." + "Use ensure torch is installed, or use the torch/RL docker images") + + def after_iteration( + self, model, epoch: int, evals_log: xgb.callback.TrainingCallback.EvalsLog + ) -> bool: + return False + + def after_training(self, model): + return model + + +class TensorboardLogger(BaseTensorboardLogger): + def __init__(self, logdir: Path = Path("tensorboard")): + from torch.utils.tensorboard import SummaryWriter + self.writer: SummaryWriter = SummaryWriter(f"{str(logdir)}/tensorboard") + + def log_scalar(self, tag: str, scalar_value: Any, step: int): + self.writer.add_scalar(tag, scalar_value, step) + + def close(self): + self.writer.flush() + self.writer.close() + + +class TensorBoardCallback(BaseTensorBoardCallback): + + def __init__(self, logdir: Path = Path("tensorboard")): + from torch.utils.tensorboard import SummaryWriter + self.writer: SummaryWriter = SummaryWriter(f"{str(logdir)}/tensorboard") + + def after_iteration( + self, model, epoch: int, evals_log: xgb.callback.TrainingCallback.EvalsLog + ) -> bool: + if not evals_log: + return False + + for data, metric in evals_log.items(): + for metric_name, log in metric.items(): + score = log[-1][0] if isinstance(log[-1], tuple) else log[-1] + if data == "train": + self.writer.add_scalar("train_loss", score**2, epoch) + else: + self.writer.add_scalar("valid_loss", score**2, epoch) + + return False + + def after_training(self, model): + self.writer.flush() + self.writer.close() + + return model diff --git a/freqtrade/freqai/torch/PyTorchModelTrainer.py b/freqtrade/freqai/torch/PyTorchModelTrainer.py index a3b0d9b9c..6d0441f07 100644 --- a/freqtrade/freqai/torch/PyTorchModelTrainer.py +++ b/freqtrade/freqai/torch/PyTorchModelTrainer.py @@ -29,6 +29,7 @@ class PyTorchModelTrainer(PyTorchTrainerInterface): data_convertor: PyTorchDataConvertor, model_meta_data: Dict[str, Any] = {}, window_size: int = 1, + tb_logger: Any = None, **kwargs ): """ @@ -56,6 +57,7 @@ class PyTorchModelTrainer(PyTorchTrainerInterface): self.max_n_eval_batches: Optional[int] = kwargs.get("max_n_eval_batches", None) self.data_convertor = data_convertor self.window_size: int = window_size + self.tb_logger = tb_logger if init_model: self.load_from_checkpoint(init_model) @@ -81,8 +83,6 @@ class PyTorchModelTrainer(PyTorchTrainerInterface): ) self.model.train() for epoch in range(1, epochs + 1): - # training - losses = [] for i, batch_data in enumerate(data_loaders_dictionary["train"]): xb, yb = batch_data @@ -94,20 +94,15 @@ class PyTorchModelTrainer(PyTorchTrainerInterface): self.optimizer.zero_grad(set_to_none=True) loss.backward() self.optimizer.step() - losses.append(loss.item()) - train_loss = sum(losses) / len(losses) - log_message = f"epoch {epoch}/{epochs}: train loss {train_loss:.4f}" + self.tb_logger.log_scalar("train_loss", loss.item(), i) # evaluation if "test" in splits: - test_loss = self.estimate_loss( + self.estimate_loss( data_loaders_dictionary, self.max_n_eval_batches, "test" ) - log_message += f" ; test loss {test_loss:.4f}" - - logger.info(log_message) @torch.no_grad() def estimate_loss( @@ -115,10 +110,9 @@ class PyTorchModelTrainer(PyTorchTrainerInterface): data_loader_dictionary: Dict[str, DataLoader], max_n_eval_batches: Optional[int], split: str, - ) -> float: + ) -> None: self.model.eval() n_batches = 0 - losses = [] for i, batch_data in enumerate(data_loader_dictionary[split]): if max_n_eval_batches and i > max_n_eval_batches: n_batches += 1 @@ -129,10 +123,9 @@ class PyTorchModelTrainer(PyTorchTrainerInterface): yb_pred = self.model(xb) loss = self.criterion(yb_pred, yb) - losses.append(loss.item()) + self.tb_logger.log_scalar(f"{split}_loss", loss.item(), i) self.model.train() - return sum(losses) / len(losses) def create_data_loaders_dictionary( self, From b01aaa4d035e195ab71af0001e257d53d68aa0e4 Mon Sep 17 00:00:00 2001 From: robcaulk Date: Wed, 10 May 2023 10:11:33 +0000 Subject: [PATCH 02/25] ensure backtesting also produces tb_logs, make sure tests are working --- freqtrade/freqai/freqai_interface.py | 2 ++ tests/freqai/test_freqai_datakitchen.py | 1 + tests/freqai/test_freqai_interface.py | 2 -- 3 files changed, 3 insertions(+), 2 deletions(-) diff --git a/freqtrade/freqai/freqai_interface.py b/freqtrade/freqai/freqai_interface.py index b453500b8..3344f3df4 100644 --- a/freqtrade/freqai/freqai_interface.py +++ b/freqtrade/freqai/freqai_interface.py @@ -345,7 +345,9 @@ class IFreqaiModel(ABC): dk.find_labels(dataframe_train) try: + self.tb_logger = TBLogger(dk.data_path) self.model = self.train(dataframe_train, pair, dk) + self.tb_logger.close() except Exception as msg: logger.warning( f"Training {pair} raised exception {msg.__class__.__name__}. " diff --git a/tests/freqai/test_freqai_datakitchen.py b/tests/freqai/test_freqai_datakitchen.py index 3f0fc697d..c9d3a973c 100644 --- a/tests/freqai/test_freqai_datakitchen.py +++ b/tests/freqai/test_freqai_datakitchen.py @@ -194,3 +194,4 @@ def test_get_full_model_path(mocker, freqai_conf, model): model_path = freqai.dk.get_full_models_path(freqai_conf) assert model_path.is_dir() is True + shutil.rmtree(Path(freqai.dk.full_path)) diff --git a/tests/freqai/test_freqai_interface.py b/tests/freqai/test_freqai_interface.py index e27c8d2c0..5291185f0 100644 --- a/tests/freqai/test_freqai_interface.py +++ b/tests/freqai/test_freqai_interface.py @@ -552,8 +552,6 @@ def test_get_state_info(mocker, freqai_conf, dp_exists, caplog, tickers): if is_mac(): pytest.skip("Reinforcement learning module not available on intel based Mac OS") - if is_py11(): - pytest.skip("Reinforcement learning currently not available on python 3.11.") freqai_conf.update({"freqaimodel": "ReinforcementLearner"}) freqai_conf.update({"timerange": "20180110-20180130"}) From 6df5cb88787ad5e909c1d13de48e4ee8d066fd46 Mon Sep 17 00:00:00 2001 From: robcaulk Date: Wed, 10 May 2023 10:18:52 +0000 Subject: [PATCH 03/25] add install requirement to tensorboard doc --- docs/freqai-running.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/docs/freqai-running.md b/docs/freqai-running.md index 5ea67651f..ec6d28c57 100644 --- a/docs/freqai-running.md +++ b/docs/freqai-running.md @@ -161,6 +161,10 @@ This specific hyperopt would help you understand the appropriate `DI_values` for !!! note "Availability" FreqAI includes tensorboard for a variety of models, including XGBoost, all PyTorch models, Reinforcement Learning, and Catboost. If you would like to see Tensorboard integrated into another model type, please open an issue on the [Freqtrade GitHub](https://github.com/freqtrade/freqtrade/issues) +!!! danger "Requirements" + Tensorboard logging requires the FreqAI torch installation/docker image. + + The easiest way to use tensorboard is to open a separate shell and run: ```bash From 692fa390c61a358ad12828e4f70710b1058c1a86 Mon Sep 17 00:00:00 2001 From: robcaulk Date: Fri, 12 May 2023 07:56:44 +0000 Subject: [PATCH 04/25] fix the import logic, fix tests, put all tensorboard in a single folder --- .../RL/BaseReinforcementLearningModel.py | 2 +- freqtrade/freqai/__init__.py | 9 ++- .../ReinforcementLearner_multiproc.py | 2 +- freqtrade/freqai/tensorboard.py | 77 ------------------- .../TensorboardCallback.py | 0 .../freqai/tensorboard/base_tensorboard.py | 35 +++++++++ freqtrade/freqai/tensorboard/tensorboard.py | 52 +++++++++++++ tests/freqai/test_freqai_datakitchen.py | 2 +- tests/freqai/test_freqai_interface.py | 3 + 9 files changed, 98 insertions(+), 84 deletions(-) delete mode 100644 freqtrade/freqai/tensorboard.py rename freqtrade/freqai/{RL => tensorboard}/TensorboardCallback.py (100%) create mode 100644 freqtrade/freqai/tensorboard/base_tensorboard.py create mode 100644 freqtrade/freqai/tensorboard/tensorboard.py diff --git a/freqtrade/freqai/RL/BaseReinforcementLearningModel.py b/freqtrade/freqai/RL/BaseReinforcementLearningModel.py index e2c0f5fda..b024f58af 100644 --- a/freqtrade/freqai/RL/BaseReinforcementLearningModel.py +++ b/freqtrade/freqai/RL/BaseReinforcementLearningModel.py @@ -23,7 +23,7 @@ from freqtrade.freqai.data_kitchen import FreqaiDataKitchen from freqtrade.freqai.freqai_interface import IFreqaiModel from freqtrade.freqai.RL.Base5ActionRLEnv import Actions, Base5ActionRLEnv from freqtrade.freqai.RL.BaseEnvironment import BaseActions, BaseEnvironment, Positions -from freqtrade.freqai.RL.TensorboardCallback import TensorboardCallback +from freqtrade.freqai.tensorboard.TensorboardCallback import TensorboardCallback from freqtrade.persistence import Trade diff --git a/freqtrade/freqai/__init__.py b/freqtrade/freqai/__init__.py index 5fb6e5be0..14353de98 100644 --- a/freqtrade/freqai/__init__.py +++ b/freqtrade/freqai/__init__.py @@ -1,12 +1,13 @@ # ensure users can still use a non-torch freqai version try: - from freqtrade.freqai.tensorboard import TensorBoardCallback, TensorboardLogger + from freqtrade.freqai.tensorboard.tensorboard import TensorBoardCallback, TensorboardLogger TBLogger = TensorboardLogger TBCallback = TensorBoardCallback except ModuleNotFoundError: - from freqtrade.freqai.tensorboard import BaseTensorBoardCallback, BaseTensorboardLogger - TBLogger = BaseTensorboardLogger # type: ignore - TBCallback = BaseTensorBoardCallback # type: ignore + from freqtrade.freqai.tensorboard.base_tensorboard import (BaseTensorBoardCallback, + BaseTensorboardLogger) + TBLogger = BaseTensorboardLogger + TBCallback = BaseTensorBoardCallback __all__ = ( "TBLogger", diff --git a/freqtrade/freqai/prediction_models/ReinforcementLearner_multiproc.py b/freqtrade/freqai/prediction_models/ReinforcementLearner_multiproc.py index 73f617027..9f0b2d436 100644 --- a/freqtrade/freqai/prediction_models/ReinforcementLearner_multiproc.py +++ b/freqtrade/freqai/prediction_models/ReinforcementLearner_multiproc.py @@ -8,7 +8,7 @@ from stable_baselines3.common.vec_env import SubprocVecEnv, VecMonitor from freqtrade.freqai.data_kitchen import FreqaiDataKitchen from freqtrade.freqai.prediction_models.ReinforcementLearner import ReinforcementLearner from freqtrade.freqai.RL.BaseReinforcementLearningModel import make_env -from freqtrade.freqai.RL.TensorboardCallback import TensorboardCallback +from freqtrade.freqai.tensorboard.TensorboardCallback import TensorboardCallback logger = logging.getLogger(__name__) diff --git a/freqtrade/freqai/tensorboard.py b/freqtrade/freqai/tensorboard.py deleted file mode 100644 index cb536008e..000000000 --- a/freqtrade/freqai/tensorboard.py +++ /dev/null @@ -1,77 +0,0 @@ -import logging -from pathlib import Path -from typing import Any - -import xgboost as xgb - - -logger = logging.getLogger(__name__) - - -class BaseTensorboardLogger: - def __init__(self, logdir: str = "tensorboard", id: str = "unique-id"): - logger.warning("Tensorboard is not installed, no logs will be written." - "Use ensure torch is installed, or use the torch/RL docker images") - - def log_scaler(self, tag: str, scalar_value: Any, step: int): - return - - def close(self): - return - - -class BaseTensorBoardCallback(xgb.callback.TrainingCallback): - - def __init__(self, logdir: str = "tensorboard", id: str = "uniqu-id", test_size=1): - logger.warning("Tensorboard is not installed, no logs will be written." - "Use ensure torch is installed, or use the torch/RL docker images") - - def after_iteration( - self, model, epoch: int, evals_log: xgb.callback.TrainingCallback.EvalsLog - ) -> bool: - return False - - def after_training(self, model): - return model - - -class TensorboardLogger(BaseTensorboardLogger): - def __init__(self, logdir: Path = Path("tensorboard")): - from torch.utils.tensorboard import SummaryWriter - self.writer: SummaryWriter = SummaryWriter(f"{str(logdir)}/tensorboard") - - def log_scalar(self, tag: str, scalar_value: Any, step: int): - self.writer.add_scalar(tag, scalar_value, step) - - def close(self): - self.writer.flush() - self.writer.close() - - -class TensorBoardCallback(BaseTensorBoardCallback): - - def __init__(self, logdir: Path = Path("tensorboard")): - from torch.utils.tensorboard import SummaryWriter - self.writer: SummaryWriter = SummaryWriter(f"{str(logdir)}/tensorboard") - - def after_iteration( - self, model, epoch: int, evals_log: xgb.callback.TrainingCallback.EvalsLog - ) -> bool: - if not evals_log: - return False - - for data, metric in evals_log.items(): - for metric_name, log in metric.items(): - score = log[-1][0] if isinstance(log[-1], tuple) else log[-1] - if data == "train": - self.writer.add_scalar("train_loss", score**2, epoch) - else: - self.writer.add_scalar("valid_loss", score**2, epoch) - - return False - - def after_training(self, model): - self.writer.flush() - self.writer.close() - - return model diff --git a/freqtrade/freqai/RL/TensorboardCallback.py b/freqtrade/freqai/tensorboard/TensorboardCallback.py similarity index 100% rename from freqtrade/freqai/RL/TensorboardCallback.py rename to freqtrade/freqai/tensorboard/TensorboardCallback.py diff --git a/freqtrade/freqai/tensorboard/base_tensorboard.py b/freqtrade/freqai/tensorboard/base_tensorboard.py new file mode 100644 index 000000000..186658532 --- /dev/null +++ b/freqtrade/freqai/tensorboard/base_tensorboard.py @@ -0,0 +1,35 @@ +import logging +from pathlib import Path +from typing import Any + +import xgboost as xgb + + +logger = logging.getLogger(__name__) + + +class BaseTensorboardLogger: + def __init__(self, logdir: Path): + logger.warning("Tensorboard is not installed, no logs will be written." + "Ensure torch is installed, or use the torch/RL docker images") + + def log_scaler(self, tag: str, scalar_value: Any, step: int): + return + + def close(self): + return + + +class BaseTensorBoardCallback(xgb.callback.TrainingCallback): + + def __init__(self, logdir: Path): + logger.warning("Tensorboard is not installed, no logs will be written." + "Ensure torch is installed, or use the torch/RL docker images") + + def after_iteration( + self, model, epoch: int, evals_log: xgb.callback.TrainingCallback.EvalsLog + ) -> bool: + return False + + def after_training(self, model): + return model diff --git a/freqtrade/freqai/tensorboard/tensorboard.py b/freqtrade/freqai/tensorboard/tensorboard.py new file mode 100644 index 000000000..f9070be6e --- /dev/null +++ b/freqtrade/freqai/tensorboard/tensorboard.py @@ -0,0 +1,52 @@ +import logging +from pathlib import Path +from typing import Any + +from torch.utils.tensorboard import SummaryWriter +from xgboost import callback + +from freqtrade.freqai.tensorboard.base_tensorboard import (BaseTensorBoardCallback, + BaseTensorboardLogger) + + +logger = logging.getLogger(__name__) + + +class TensorboardLogger(BaseTensorboardLogger): + def __init__(self, logdir: Path): + self.writer: SummaryWriter = SummaryWriter(f"{str(logdir)}/tensorboard") + + def log_scalar(self, tag: str, scalar_value: Any, step: int): + self.writer.add_scalar(tag, scalar_value, step) + + def close(self): + self.writer.flush() + self.writer.close() + + +class TensorBoardCallback(BaseTensorBoardCallback): + + def __init__(self, logdir: Path): + self.writer: SummaryWriter = SummaryWriter(f"{str(logdir)}/tensorboard") + + def after_iteration( + self, model, epoch: int, evals_log: callback.TrainingCallback.EvalsLog + ) -> bool: + if not evals_log: + return False + + for data, metric in evals_log.items(): + for metric_name, log in metric.items(): + score = log[-1][0] if isinstance(log[-1], tuple) else log[-1] + if data == "train": + self.writer.add_scalar("train_loss", score**2, epoch) + else: + self.writer.add_scalar("valid_loss", score**2, epoch) + + return False + + def after_training(self, model): + self.writer.flush() + self.writer.close() + + return model diff --git a/tests/freqai/test_freqai_datakitchen.py b/tests/freqai/test_freqai_datakitchen.py index c9d3a973c..cbc4acd18 100644 --- a/tests/freqai/test_freqai_datakitchen.py +++ b/tests/freqai/test_freqai_datakitchen.py @@ -188,7 +188,7 @@ def test_get_full_model_path(mocker, freqai_conf, model): data_load_timerange = TimeRange.parse_timerange("20180110-20180130") new_timerange = TimeRange.parse_timerange("20180120-20180130") - + freqai.dk.set_paths('ADA/BTC', None) freqai.extract_data_and_train_model( new_timerange, "ADA/BTC", strategy, freqai.dk, data_load_timerange) diff --git a/tests/freqai/test_freqai_interface.py b/tests/freqai/test_freqai_interface.py index 5291185f0..a2e4f182a 100644 --- a/tests/freqai/test_freqai_interface.py +++ b/tests/freqai/test_freqai_interface.py @@ -282,6 +282,7 @@ def test_start_backtesting(mocker, freqai_conf, model, num_files, strat, caplog) df[f'%-constant_{i}'] = i metadata = {"pair": "LTC/BTC"} + freqai.dk.set_paths('LTC/BTC', None) freqai.start_backtesting(df, metadata, freqai.dk, strategy) model_folders = [x for x in freqai.dd.full_path.iterdir() if x.is_dir()] @@ -439,6 +440,7 @@ def test_principal_component_analysis(mocker, freqai_conf): data_load_timerange = TimeRange.parse_timerange("20180110-20180130") new_timerange = TimeRange.parse_timerange("20180120-20180130") + freqai.dk.set_paths('ADA/BTC', None) freqai.extract_data_and_train_model( new_timerange, "ADA/BTC", strategy, freqai.dk, data_load_timerange) @@ -472,6 +474,7 @@ def test_plot_feature_importance(mocker, freqai_conf): data_load_timerange = TimeRange.parse_timerange("20180110-20180130") new_timerange = TimeRange.parse_timerange("20180120-20180130") + freqai.dk.set_paths('ADA/BTC', None) freqai.extract_data_and_train_model( new_timerange, "ADA/BTC", strategy, freqai.dk, data_load_timerange) From 8261c988b990a3ad70abc22db5456990e27ccba9 Mon Sep 17 00:00:00 2001 From: robcaulk Date: Fri, 12 May 2023 09:11:14 +0000 Subject: [PATCH 05/25] try to fix mac CI --- tests/freqai/test_freqai_datakitchen.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/freqai/test_freqai_datakitchen.py b/tests/freqai/test_freqai_datakitchen.py index cbc4acd18..1c6038124 100644 --- a/tests/freqai/test_freqai_datakitchen.py +++ b/tests/freqai/test_freqai_datakitchen.py @@ -194,4 +194,3 @@ def test_get_full_model_path(mocker, freqai_conf, model): model_path = freqai.dk.get_full_models_path(freqai_conf) assert model_path.is_dir() is True - shutil.rmtree(Path(freqai.dk.full_path)) From ca7ad8a49be779c6148c1fb7e611700bf1f17f95 Mon Sep 17 00:00:00 2001 From: robcaulk Date: Fri, 12 May 2023 12:50:11 +0000 Subject: [PATCH 06/25] good old macos --- tests/freqai/test_freqai_datakitchen.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/freqai/test_freqai_datakitchen.py b/tests/freqai/test_freqai_datakitchen.py index 1c6038124..b21b599b6 100644 --- a/tests/freqai/test_freqai_datakitchen.py +++ b/tests/freqai/test_freqai_datakitchen.py @@ -188,7 +188,7 @@ def test_get_full_model_path(mocker, freqai_conf, model): data_load_timerange = TimeRange.parse_timerange("20180110-20180130") new_timerange = TimeRange.parse_timerange("20180120-20180130") - freqai.dk.set_paths('ADA/BTC', None) + # freqai.dk.set_paths('ADA/BTC', None) freqai.extract_data_and_train_model( new_timerange, "ADA/BTC", strategy, freqai.dk, data_load_timerange) From 6e5a9fe4c96a9a472e8cc914dcdba397ff3d0caf Mon Sep 17 00:00:00 2001 From: robcaulk Date: Fri, 12 May 2023 13:55:41 +0000 Subject: [PATCH 07/25] mac strikes again --- tests/freqai/test_freqai_datakitchen.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/tests/freqai/test_freqai_datakitchen.py b/tests/freqai/test_freqai_datakitchen.py index b21b599b6..13dc6b4b0 100644 --- a/tests/freqai/test_freqai_datakitchen.py +++ b/tests/freqai/test_freqai_datakitchen.py @@ -12,6 +12,7 @@ from freqtrade.freqai.data_kitchen import FreqaiDataKitchen from tests.conftest import get_patched_exchange, log_has_re from tests.freqai.conftest import (get_patched_data_kitchen, get_patched_freqai_strategy, make_data_dictionary, make_unfiltered_dataframe) +from tests.freqai.test_freqai_interface import is_mac @pytest.mark.parametrize( @@ -173,6 +174,9 @@ def test_get_full_model_path(mocker, freqai_conf, model): freqai_conf.update({"timerange": "20180110-20180130"}) freqai_conf.update({"strategy": "freqai_test_strat"}) + if is_mac(): + pytest.skip("Mac is confused during this test for unknown reasons") + strategy = get_patched_freqai_strategy(mocker, freqai_conf) exchange = get_patched_exchange(mocker, freqai_conf) strategy.dp = DataProvider(freqai_conf, exchange) @@ -188,7 +192,7 @@ def test_get_full_model_path(mocker, freqai_conf, model): data_load_timerange = TimeRange.parse_timerange("20180110-20180130") new_timerange = TimeRange.parse_timerange("20180120-20180130") - # freqai.dk.set_paths('ADA/BTC', None) + freqai.dk.set_paths('ADA/BTC', None) freqai.extract_data_and_train_model( new_timerange, "ADA/BTC", strategy, freqai.dk, data_load_timerange) From 43213cc6ff41e45b48be1c26dfd1e1e845bbbc68 Mon Sep 17 00:00:00 2001 From: Matthias Date: Fri, 12 May 2023 18:07:28 +0200 Subject: [PATCH 08/25] Revert testing Reinforcement lerning on Mac --- tests/freqai/test_freqai_interface.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/freqai/test_freqai_interface.py b/tests/freqai/test_freqai_interface.py index a2e4f182a..d92e0a63d 100644 --- a/tests/freqai/test_freqai_interface.py +++ b/tests/freqai/test_freqai_interface.py @@ -555,6 +555,8 @@ def test_get_state_info(mocker, freqai_conf, dp_exists, caplog, tickers): if is_mac(): pytest.skip("Reinforcement learning module not available on intel based Mac OS") + if is_py11(): + pytest.skip("Reinforcement learning currently not available on python 3.11.") freqai_conf.update({"freqaimodel": "ReinforcementLearner"}) freqai_conf.update({"timerange": "20180110-20180130"}) From 49b9b463b4354999eff7ffc1e9ab2bf0d48e82f1 Mon Sep 17 00:00:00 2001 From: Matthias Date: Fri, 12 May 2023 18:26:01 +0200 Subject: [PATCH 09/25] Move tensorboard callback exports to freqai.tensorboard. --- freqtrade/freqai/freqai_interface.py | 2 +- freqtrade/freqai/prediction_models/XGBoostRegressor.py | 2 +- freqtrade/freqai/{ => tensorboard}/__init__.py | 0 3 files changed, 2 insertions(+), 2 deletions(-) rename freqtrade/freqai/{ => tensorboard}/__init__.py (100%) diff --git a/freqtrade/freqai/freqai_interface.py b/freqtrade/freqai/freqai_interface.py index 3344f3df4..8e34d1c28 100644 --- a/freqtrade/freqai/freqai_interface.py +++ b/freqtrade/freqai/freqai_interface.py @@ -19,9 +19,9 @@ from freqtrade.data.dataprovider import DataProvider from freqtrade.enums import RunMode from freqtrade.exceptions import OperationalException from freqtrade.exchange import timeframe_to_seconds -from freqtrade.freqai import TBLogger from freqtrade.freqai.data_drawer import FreqaiDataDrawer from freqtrade.freqai.data_kitchen import FreqaiDataKitchen +from freqtrade.freqai.tensorboard import TBLogger from freqtrade.freqai.utils import plot_feature_importance, record_params from freqtrade.strategy.interface import IStrategy diff --git a/freqtrade/freqai/prediction_models/XGBoostRegressor.py b/freqtrade/freqai/prediction_models/XGBoostRegressor.py index b4cdead65..f1a2474da 100644 --- a/freqtrade/freqai/prediction_models/XGBoostRegressor.py +++ b/freqtrade/freqai/prediction_models/XGBoostRegressor.py @@ -3,9 +3,9 @@ from typing import Any, Dict from xgboost import XGBRegressor -from freqtrade.freqai import TBCallback from freqtrade.freqai.base_models.BaseRegressionModel import BaseRegressionModel from freqtrade.freqai.data_kitchen import FreqaiDataKitchen +from freqtrade.freqai.tensorboard import TBCallback logger = logging.getLogger(__name__) diff --git a/freqtrade/freqai/__init__.py b/freqtrade/freqai/tensorboard/__init__.py similarity index 100% rename from freqtrade/freqai/__init__.py rename to freqtrade/freqai/tensorboard/__init__.py From 6d7172ac445286afa5c8398b444089bb50d548fe Mon Sep 17 00:00:00 2001 From: Matthias Date: Fri, 12 May 2023 18:26:34 +0200 Subject: [PATCH 10/25] Re-add init file --- freqtrade/freqai/__init__.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 freqtrade/freqai/__init__.py diff --git a/freqtrade/freqai/__init__.py b/freqtrade/freqai/__init__.py new file mode 100644 index 000000000..e69de29bb From 871f1aabb72c8cbac9bb4f0ab5aef391c352c28f Mon Sep 17 00:00:00 2001 From: Matthias Date: Fri, 12 May 2023 18:33:46 +0200 Subject: [PATCH 11/25] Use tensorboard fallback for mac tests --- tests/freqai/test_freqai_interface.py | 26 +++++++++++++++++++++++++- 1 file changed, 25 insertions(+), 1 deletion(-) diff --git a/tests/freqai/test_freqai_interface.py b/tests/freqai/test_freqai_interface.py index d92e0a63d..7e74fe5ed 100644 --- a/tests/freqai/test_freqai_interface.py +++ b/tests/freqai/test_freqai_interface.py @@ -42,6 +42,30 @@ def can_run_model(model: str) -> None: pytest.skip("Reinforcement learning / PyTorch module not available on intel based Mac OS.") +@pytest.fixture(scope="function") +def import_fails_mac() -> None: + # Source of this test-method: + # https://stackoverflow.com/questions/2481511/mocking-importerror-in-python + if is_mac(): + import builtins + realimport = builtins.__import__ + + def mockedimport(name, *args, **kwargs): + if name in ["freqtrade.freqai.tensorboard.tensorboard"]: + raise ImportError(f"No module named '{name}'") + return realimport(name, *args, **kwargs) + + builtins.__import__ = mockedimport + + # Run test - then cleanup + yield + + # restore previous importfunction + builtins.__import__ = realimport + else: + yield + + @pytest.mark.parametrize('model, pca, dbscan, float32, can_short, shuffle, buffer', [ ('LightGBMRegressor', True, False, True, True, False, 0), ('XGBoostRegressor', False, True, False, True, False, 10), @@ -55,7 +79,7 @@ def can_run_model(model: str) -> None: ('ReinforcementLearner_test_3ac', False, False, False, True, False, 0), ('ReinforcementLearner_test_4ac', False, False, False, True, False, 0), ]) -def test_extract_data_and_train_model_Standard(mocker, freqai_conf, model, pca, +def test_extract_data_and_train_model_Standard(mocker, freqai_conf, model, pca, import_fails_mac, dbscan, float32, can_short, shuffle, buffer): can_run_model(model) From 400cbd1836e64bb51287587c4c1077cb7fba0250 Mon Sep 17 00:00:00 2001 From: Matthias Date: Fri, 12 May 2023 19:47:53 +0200 Subject: [PATCH 12/25] Fix types --- freqtrade/freqai/tensorboard/__init__.py | 4 ++-- tests/freqai/test_freqai_interface.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/freqtrade/freqai/tensorboard/__init__.py b/freqtrade/freqai/tensorboard/__init__.py index 14353de98..59862bc0d 100644 --- a/freqtrade/freqai/tensorboard/__init__.py +++ b/freqtrade/freqai/tensorboard/__init__.py @@ -6,8 +6,8 @@ try: except ModuleNotFoundError: from freqtrade.freqai.tensorboard.base_tensorboard import (BaseTensorBoardCallback, BaseTensorboardLogger) - TBLogger = BaseTensorboardLogger - TBCallback = BaseTensorBoardCallback + TBLogger = BaseTensorboardLogger # type: ignore + TBCallback = BaseTensorBoardCallback # type: ignore __all__ = ( "TBLogger", diff --git a/tests/freqai/test_freqai_interface.py b/tests/freqai/test_freqai_interface.py index 7e74fe5ed..ad615a5af 100644 --- a/tests/freqai/test_freqai_interface.py +++ b/tests/freqai/test_freqai_interface.py @@ -43,7 +43,7 @@ def can_run_model(model: str) -> None: @pytest.fixture(scope="function") -def import_fails_mac() -> None: +def import_fails_mac(): # Source of this test-method: # https://stackoverflow.com/questions/2481511/mocking-importerror-in-python if is_mac(): From 23e8932a443213b234287a42d3976de5bec115d6 Mon Sep 17 00:00:00 2001 From: Matthias Date: Fri, 12 May 2023 20:01:16 +0200 Subject: [PATCH 13/25] Mock tensorboard callbacks --- tests/freqai/test_freqai_interface.py | 31 ++++++--------------------- 1 file changed, 6 insertions(+), 25 deletions(-) diff --git a/tests/freqai/test_freqai_interface.py b/tests/freqai/test_freqai_interface.py index ad615a5af..b49e801ed 100644 --- a/tests/freqai/test_freqai_interface.py +++ b/tests/freqai/test_freqai_interface.py @@ -42,30 +42,6 @@ def can_run_model(model: str) -> None: pytest.skip("Reinforcement learning / PyTorch module not available on intel based Mac OS.") -@pytest.fixture(scope="function") -def import_fails_mac(): - # Source of this test-method: - # https://stackoverflow.com/questions/2481511/mocking-importerror-in-python - if is_mac(): - import builtins - realimport = builtins.__import__ - - def mockedimport(name, *args, **kwargs): - if name in ["freqtrade.freqai.tensorboard.tensorboard"]: - raise ImportError(f"No module named '{name}'") - return realimport(name, *args, **kwargs) - - builtins.__import__ = mockedimport - - # Run test - then cleanup - yield - - # restore previous importfunction - builtins.__import__ = realimport - else: - yield - - @pytest.mark.parametrize('model, pca, dbscan, float32, can_short, shuffle, buffer', [ ('LightGBMRegressor', True, False, True, True, False, 0), ('XGBoostRegressor', False, True, False, True, False, 10), @@ -79,10 +55,15 @@ def import_fails_mac(): ('ReinforcementLearner_test_3ac', False, False, False, True, False, 0), ('ReinforcementLearner_test_4ac', False, False, False, True, False, 0), ]) -def test_extract_data_and_train_model_Standard(mocker, freqai_conf, model, pca, import_fails_mac, +def test_extract_data_and_train_model_Standard(mocker, freqai_conf, model, pca, dbscan, float32, can_short, shuffle, buffer): can_run_model(model) + if is_mac(): + # MacOS CI is not friendly to tensorboard + mocker.patch('freqtrade.freqai.tensorboard.tensorboard.SummaryWriter') + mocker.patch('freqtrade.freqai.tensorboard.tensorboard.TensorBoardCallback.after_iteration') + model_save_ext = 'joblib' freqai_conf.update({"freqaimodel": model}) freqai_conf.update({"timerange": "20180110-20180130"}) From ab0f9d78ee5db8a111ece01624cda609bf58bdb7 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 13 May 2023 08:02:19 +0200 Subject: [PATCH 14/25] Mock tensorboard callbacks for all freqAI tests --- tests/freqai/conftest.py | 14 ++++++++++++++ tests/freqai/test_freqai_interface.py | 11 +---------- 2 files changed, 15 insertions(+), 10 deletions(-) diff --git a/tests/freqai/conftest.py b/tests/freqai/conftest.py index ab4a62a9e..eeb0182b6 100644 --- a/tests/freqai/conftest.py +++ b/tests/freqai/conftest.py @@ -1,3 +1,4 @@ +import platform from copy import deepcopy from pathlib import Path from typing import Any, Dict @@ -14,6 +15,19 @@ from freqtrade.resolvers.freqaimodel_resolver import FreqaiModelResolver from tests.conftest import get_patched_exchange +def is_mac() -> bool: + machine = platform.system() + return "Darwin" in machine + + +@pytest.fixture(scope="function", autouse=True) +def patch_tensorboard_maconly(mocker): + if is_mac(): + # MacOS CI is not friendly to tensorboard + mocker.patch('freqtrade.freqai.tensorboard.tensorboard.SummaryWriter') + mocker.patch('freqtrade.freqai.tensorboard.tensorboard.TensorBoardCallback.after_iteration') + + @pytest.fixture(scope="function") def freqai_conf(default_conf, tmpdir): freqaiconf = deepcopy(default_conf) diff --git a/tests/freqai/test_freqai_interface.py b/tests/freqai/test_freqai_interface.py index b49e801ed..b5314556f 100644 --- a/tests/freqai/test_freqai_interface.py +++ b/tests/freqai/test_freqai_interface.py @@ -15,7 +15,7 @@ from freqtrade.optimize.backtesting import Backtesting from freqtrade.persistence import Trade from freqtrade.plugins.pairlistmanager import PairListManager from tests.conftest import EXMS, create_mock_trades, get_patched_exchange, log_has_re -from tests.freqai.conftest import (get_patched_freqai_strategy, make_rl_config, +from tests.freqai.conftest import (get_patched_freqai_strategy, is_mac, make_rl_config, mock_pytorch_mlp_model_training_parameters) @@ -28,11 +28,6 @@ def is_arm() -> bool: return "arm" in machine or "aarch64" in machine -def is_mac() -> bool: - machine = platform.system() - return "Darwin" in machine - - def can_run_model(model: str) -> None: if (is_arm() or is_py11()) and "Catboost" in model: pytest.skip("CatBoost is not supported on ARM.") @@ -59,10 +54,6 @@ def test_extract_data_and_train_model_Standard(mocker, freqai_conf, model, pca, dbscan, float32, can_short, shuffle, buffer): can_run_model(model) - if is_mac(): - # MacOS CI is not friendly to tensorboard - mocker.patch('freqtrade.freqai.tensorboard.tensorboard.SummaryWriter') - mocker.patch('freqtrade.freqai.tensorboard.tensorboard.TensorBoardCallback.after_iteration') model_save_ext = 'joblib' freqai_conf.update({"freqaimodel": model}) From ab7a474ab6e986062538bfbcfe44387c1177966b Mon Sep 17 00:00:00 2001 From: robcaulk Date: Sun, 14 May 2023 12:03:15 +0000 Subject: [PATCH 15/25] try limiting tb_logger to pytorch only (XGBoost still gets its callback) --- freqtrade/freqai/freqai_interface.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/freqtrade/freqai/freqai_interface.py b/freqtrade/freqai/freqai_interface.py index 8e34d1c28..ae16d3b99 100644 --- a/freqtrade/freqai/freqai_interface.py +++ b/freqtrade/freqai/freqai_interface.py @@ -635,9 +635,11 @@ class IFreqaiModel(ABC): dk.find_features(unfiltered_dataframe) dk.find_labels(unfiltered_dataframe) - self.tb_logger = TBLogger(dk.data_path) + if self.dd.model_type == "pytorch": + self.tb_logger = TBLogger(dk.data_path) model = self.train(unfiltered_dataframe, pair, dk) - self.tb_logger.close() + if self.dd.model_type == "pytorch": + self.tb_logger.close() self.dd.pair_dict[pair]["trained_timestamp"] = trained_timestamp dk.set_new_model_names(pair, trained_timestamp) From 55a1a3afd63b5d7957c27f2ff847f9f3bb9c13f1 Mon Sep 17 00:00:00 2001 From: robcaulk Date: Sun, 14 May 2023 14:08:00 +0000 Subject: [PATCH 16/25] add config option for activating and deactivating tensorboard logger, ensure the various flavors are never activated simultaneously --- docs/freqai-parameter-table.md | 1 + docs/freqai-running.md | 6 +- .../RL/BaseReinforcementLearningModel.py | 12 +++- freqtrade/freqai/freqai_interface.py | 14 ++-- .../prediction_models/ReinforcementLearner.py | 7 +- .../prediction_models/XGBoostRegressor.py | 2 +- .../freqai/tensorboard/base_tensorboard.py | 4 +- freqtrade/freqai/tensorboard/tensorboard.py | 28 +++++--- freqtrade/freqai/utils.py | 64 ++++--------------- tests/freqai/test_freqai_datakitchen.py | 1 + tests/freqai/test_freqai_interface.py | 14 ++++ 11 files changed, 79 insertions(+), 74 deletions(-) diff --git a/docs/freqai-parameter-table.md b/docs/freqai-parameter-table.md index ef1a23401..cc92c2457 100644 --- a/docs/freqai-parameter-table.md +++ b/docs/freqai-parameter-table.md @@ -21,6 +21,7 @@ Mandatory parameters are marked as **Required** and have to be set in one of the | `continual_learning` | Use the final state of the most recently trained model as starting point for the new model, allowing for incremental learning (more information can be found [here](freqai-running.md#continual-learning)). Beware that this is currently a naive approach to incremental learning, and it has a high probability of overfitting/getting stuck in local minima while the market moves away from your model. We have the connections here primarily for experimental purposes and so that it is ready for more mature approaches to continual learning in chaotic systems like the crypto market.
**Datatype:** Boolean.
Default: `False`. | `write_metrics_to_disk` | Collect train timings, inference timings and cpu usage in json file.
**Datatype:** Boolean.
Default: `False` | `data_kitchen_thread_count` |
Designate the number of threads you want to use for data processing (outlier methods, normalization, etc.). This has no impact on the number of threads used for training. If user does not set it (default), FreqAI will use max number of threads - 2 (leaving 1 physical core available for Freqtrade bot and FreqUI)
**Datatype:** Positive integer. +| `activate_tensorboard` |
Indicate whether or not to activate tensorboard for the tensorboard enabled modules (currently Reinforcment Learning, XGBoost, Catboost, and PyTorch). Tensorboard needs Torch installed, which means you will need the torch/RL docker image or you need to answer "yes" to the install question about whether or not you wish to install Torch.
**Datatype:** Boolean.
Default: `True`. ### Feature parameters diff --git a/docs/freqai-running.md b/docs/freqai-running.md index cf1856041..55f302d40 100644 --- a/docs/freqai-running.md +++ b/docs/freqai-running.md @@ -168,7 +168,7 @@ This specific hyperopt would help you understand the appropriate `DI_values` for Tensorboard logging requires the FreqAI torch installation/docker image. -The easiest way to use tensorboard is to open a separate shell and run: +The easiest way to use tensorboard is to ensure `freqai.activate_tensorboard` is set to `True` (default setting) in your configuration file, run FreqAI, then open a separate shell and run: ```bash cd freqtrade @@ -178,3 +178,7 @@ tensorboard --logdir user_data/models/unique-id where `unique-id` is the `identifier` set in the `freqai` configuration file. This command must be run in a separate shell if you wish to view the output in your browser at 127.0.0.1:6060 (6060 is the default port used by Tensorboard). ![tensorboard](assets/tensorboard.jpg) + + +!!! note "Deactivate for improved performance" + Tensorboard logging can slow down training and should be deactivated for production use. diff --git a/freqtrade/freqai/RL/BaseReinforcementLearningModel.py b/freqtrade/freqai/RL/BaseReinforcementLearningModel.py index 8ee3c7c56..4d540ee36 100644 --- a/freqtrade/freqai/RL/BaseReinforcementLearningModel.py +++ b/freqtrade/freqai/RL/BaseReinforcementLearningModel.py @@ -68,8 +68,11 @@ class BaseReinforcementLearningModel(IFreqaiModel): self.unset_outlier_removal() self.net_arch = self.rl_config.get('net_arch', [128, 128]) self.dd.model_type = import_str - self.tensorboard_callback: TensorboardCallback = \ - TensorboardCallback(verbose=1, actions=BaseActions) + if self.activate_tensorboard: + self.tensorboard_callback: TensorboardCallback = \ + TensorboardCallback(verbose=1, actions=BaseActions) + else: + self.tenorboard_callback = None def unset_outlier_removal(self): """ @@ -156,7 +159,10 @@ class BaseReinforcementLearningModel(IFreqaiModel): best_model_save_path=str(dk.data_path)) actions = self.train_env.get_actions() - self.tensorboard_callback = TensorboardCallback(verbose=1, actions=actions) + if self.activate_tensorboard: + self.tensorboard_callback = TensorboardCallback(verbose=1, actions=actions) + else: + self.tensorboard_callback = None # type: ignore def pack_env_dict(self, pair: str) -> Dict[str, Any]: """ diff --git a/freqtrade/freqai/freqai_interface.py b/freqtrade/freqai/freqai_interface.py index ae16d3b99..9cfda05ee 100644 --- a/freqtrade/freqai/freqai_interface.py +++ b/freqtrade/freqai/freqai_interface.py @@ -21,8 +21,7 @@ from freqtrade.exceptions import OperationalException from freqtrade.exchange import timeframe_to_seconds from freqtrade.freqai.data_drawer import FreqaiDataDrawer from freqtrade.freqai.data_kitchen import FreqaiDataKitchen -from freqtrade.freqai.tensorboard import TBLogger -from freqtrade.freqai.utils import plot_feature_importance, record_params +from freqtrade.freqai.utils import get_tb_logger, plot_feature_importance, record_params from freqtrade.strategy.interface import IStrategy @@ -111,6 +110,7 @@ class IFreqaiModel(ABC): if self.ft_params.get('principal_component_analysis', False) and self.continual_learning: self.ft_params.update({'principal_component_analysis': False}) logger.warning('User tried to use PCA with continual learning. Deactivating PCA.') + self.activate_tensorboard: bool = self.freqai_info.get('activate_tensorboard', True) record_params(config, self.full_path) @@ -345,7 +345,8 @@ class IFreqaiModel(ABC): dk.find_labels(dataframe_train) try: - self.tb_logger = TBLogger(dk.data_path) + self.tb_logger = get_tb_logger(self.dd.model_type, dk.data_path, + self.activate_tensorboard) self.model = self.train(dataframe_train, pair, dk) self.tb_logger.close() except Exception as msg: @@ -635,11 +636,10 @@ class IFreqaiModel(ABC): dk.find_features(unfiltered_dataframe) dk.find_labels(unfiltered_dataframe) - if self.dd.model_type == "pytorch": - self.tb_logger = TBLogger(dk.data_path) + self.tb_logger = get_tb_logger(self.dd.model_type, dk.data_path, + self.activate_tensorboard) model = self.train(unfiltered_dataframe, pair, dk) - if self.dd.model_type == "pytorch": - self.tb_logger.close() + self.tb_logger.close() self.dd.pair_dict[pair]["trained_timestamp"] = trained_timestamp dk.set_new_model_names(pair, trained_timestamp) diff --git a/freqtrade/freqai/prediction_models/ReinforcementLearner.py b/freqtrade/freqai/prediction_models/ReinforcementLearner.py index 8c9d9bdef..0d6c52445 100644 --- a/freqtrade/freqai/prediction_models/ReinforcementLearner.py +++ b/freqtrade/freqai/prediction_models/ReinforcementLearner.py @@ -70,9 +70,14 @@ class ReinforcementLearner(BaseReinforcementLearningModel): model = self.dd.model_dictionary[dk.pair] model.set_env(self.train_env) + callbacks = [self.eval_callback] + + if self.activate_tensorboard: + callbacks.append(self.tensorboard_callback) + model.learn( total_timesteps=int(total_timesteps), - callback=[self.eval_callback, self.tensorboard_callback], + callback=callbacks, progress_bar=self.rl_config.get('progress_bar', False) ) diff --git a/freqtrade/freqai/prediction_models/XGBoostRegressor.py b/freqtrade/freqai/prediction_models/XGBoostRegressor.py index f1a2474da..f8b4d353d 100644 --- a/freqtrade/freqai/prediction_models/XGBoostRegressor.py +++ b/freqtrade/freqai/prediction_models/XGBoostRegressor.py @@ -45,7 +45,7 @@ class XGBoostRegressor(BaseRegressionModel): model = XGBRegressor(**self.model_training_parameters) - model.set_params(callbacks=[TBCallback(dk.data_path)]) + model.set_params(callbacks=[TBCallback(dk.data_path)], activate=self.activate_tensorboard) model.fit(X=X, y=y, sample_weight=sample_weight, eval_set=eval_set, sample_weight_eval_set=eval_weights, xgb_model=xgb_model) # set the callbacks to empty so that we can serialize to disk later diff --git a/freqtrade/freqai/tensorboard/base_tensorboard.py b/freqtrade/freqai/tensorboard/base_tensorboard.py index 186658532..8d396832b 100644 --- a/freqtrade/freqai/tensorboard/base_tensorboard.py +++ b/freqtrade/freqai/tensorboard/base_tensorboard.py @@ -9,7 +9,7 @@ logger = logging.getLogger(__name__) class BaseTensorboardLogger: - def __init__(self, logdir: Path): + def __init__(self, logdir: Path, activate: bool = True): logger.warning("Tensorboard is not installed, no logs will be written." "Ensure torch is installed, or use the torch/RL docker images") @@ -22,7 +22,7 @@ class BaseTensorboardLogger: class BaseTensorBoardCallback(xgb.callback.TrainingCallback): - def __init__(self, logdir: Path): + def __init__(self, logdir: Path, activate: bool = True): logger.warning("Tensorboard is not installed, no logs will be written." "Ensure torch is installed, or use the torch/RL docker images") diff --git a/freqtrade/freqai/tensorboard/tensorboard.py b/freqtrade/freqai/tensorboard/tensorboard.py index f9070be6e..46bf8dc61 100644 --- a/freqtrade/freqai/tensorboard/tensorboard.py +++ b/freqtrade/freqai/tensorboard/tensorboard.py @@ -13,25 +13,33 @@ logger = logging.getLogger(__name__) class TensorboardLogger(BaseTensorboardLogger): - def __init__(self, logdir: Path): - self.writer: SummaryWriter = SummaryWriter(f"{str(logdir)}/tensorboard") + def __init__(self, logdir: Path, activate: bool = True): + self.activate = activate + if self.activate: + self.writer: SummaryWriter = SummaryWriter(f"{str(logdir)}/tensorboard") def log_scalar(self, tag: str, scalar_value: Any, step: int): - self.writer.add_scalar(tag, scalar_value, step) + if self.activate: + self.writer.add_scalar(tag, scalar_value, step) def close(self): - self.writer.flush() - self.writer.close() + if self.activate: + self.writer.flush() + self.writer.close() class TensorBoardCallback(BaseTensorBoardCallback): - def __init__(self, logdir: Path): - self.writer: SummaryWriter = SummaryWriter(f"{str(logdir)}/tensorboard") + def __init__(self, logdir: Path, activate: bool = True): + self.activate = activate + if self.activate: + self.writer: SummaryWriter = SummaryWriter(f"{str(logdir)}/tensorboard") def after_iteration( self, model, epoch: int, evals_log: callback.TrainingCallback.EvalsLog ) -> bool: + if not self.activate: + return False if not evals_log: return False @@ -39,13 +47,15 @@ class TensorBoardCallback(BaseTensorBoardCallback): for metric_name, log in metric.items(): score = log[-1][0] if isinstance(log[-1], tuple) else log[-1] if data == "train": - self.writer.add_scalar("train_loss", score**2, epoch) + self.writer.add_scalar("train_loss", score, epoch) else: - self.writer.add_scalar("valid_loss", score**2, epoch) + self.writer.add_scalar("valid_loss", score, epoch) return False def after_training(self, model): + if not self.activate: + return model self.writer.flush() self.writer.close() diff --git a/freqtrade/freqai/utils.py b/freqtrade/freqai/utils.py index 2ba49ac40..cc803ae1e 100644 --- a/freqtrade/freqai/utils.py +++ b/freqtrade/freqai/utils.py @@ -1,7 +1,7 @@ import logging from datetime import datetime, timezone from pathlib import Path -from typing import Any, Dict +from typing import Any, Dict, Union import numpy as np import pandas as pd @@ -16,6 +16,8 @@ from freqtrade.exchange import timeframe_to_seconds from freqtrade.exchange.exchange import market_is_active from freqtrade.freqai.data_drawer import FreqaiDataDrawer from freqtrade.freqai.data_kitchen import FreqaiDataKitchen +from freqtrade.freqai.tensorboard import TBLogger +from freqtrade.freqai.tensorboard.base_tensorboard import BaseTensorboardLogger from freqtrade.plugins.pairlist.pairlist_helpers import dynamic_expand_pairlist @@ -92,55 +94,6 @@ def get_required_data_timerange(config: Config) -> TimeRange: return data_load_timerange -# Keep below for when we wish to download heterogeneously lengthed data for FreqAI. -# def download_all_data_for_training(dp: DataProvider, config: Config) -> None: -# """ -# Called only once upon start of bot to download the necessary data for -# populating indicators and training a FreqAI model. -# :param timerange: TimeRange = The full data timerange for populating the indicators -# and training the model. -# :param dp: DataProvider instance attached to the strategy -# """ - -# if dp._exchange is not None: -# markets = [p for p, m in dp._exchange.markets.items() if market_is_active(m) -# or config.get('include_inactive')] -# else: -# # This should not occur: -# raise OperationalException('No exchange object found.') - -# all_pairs = dynamic_expand_pairlist(config, markets) - -# if not dp._exchange: -# # Not realistic - this is only called in live mode. -# raise OperationalException("Dataprovider did not have an exchange attached.") - -# time = datetime.now(tz=timezone.utc).timestamp() - -# for tf in config["freqai"]["feature_parameters"].get("include_timeframes"): -# timerange = TimeRange() -# timerange.startts = int(time) -# timerange.stopts = int(time) -# startup_candles = dp.get_required_startup(str(tf)) -# tf_seconds = timeframe_to_seconds(str(tf)) -# timerange.subtract_start(tf_seconds * startup_candles) -# new_pairs_days = int((timerange.stopts - timerange.startts) / 86400) -# # FIXME: now that we are looping on `refresh_backtest_ohlcv_data`, the function -# # redownloads the funding rate for each pair. -# refresh_backtest_ohlcv_data( -# dp._exchange, -# pairs=all_pairs, -# timeframes=[tf], -# datadir=config["datadir"], -# timerange=timerange, -# new_pairs_days=new_pairs_days, -# erase=False, -# data_format=config.get("dataformat_ohlcv", "json"), -# trading_mode=config.get("trading_mode", "spot"), -# prepend=config.get("prepend_data", False), -# ) - - def plot_feature_importance(model: Any, pair: str, dk: FreqaiDataKitchen, count_max: int = 25) -> None: """ @@ -233,3 +186,14 @@ def get_timerange_backtest_live_models(config: Config) -> str: dd = FreqaiDataDrawer(models_path, config) timerange = dd.get_timerange_from_live_historic_predictions() return timerange.timerange_str + + +def get_tb_logger(model_type: str, path: Path, activate: bool) -> Union[TBLogger, + BaseTensorboardLogger]: + tb_logger: Union[TBLogger, BaseTensorboardLogger] + if model_type == "pytorch": + tb_logger = TBLogger(path, activate) + else: + tb_logger = BaseTensorboardLogger(path, activate) + + return tb_logger diff --git a/tests/freqai/test_freqai_datakitchen.py b/tests/freqai/test_freqai_datakitchen.py index 13dc6b4b0..6d6e10b94 100644 --- a/tests/freqai/test_freqai_datakitchen.py +++ b/tests/freqai/test_freqai_datakitchen.py @@ -183,6 +183,7 @@ def test_get_full_model_path(mocker, freqai_conf, model): strategy.freqai_info = freqai_conf.get("freqai", {}) freqai = strategy.freqai freqai.live = True + freqai.activate_tensorboard = False freqai.dk = FreqaiDataKitchen(freqai_conf) freqai.dk.live = True timerange = TimeRange.parse_timerange("20180110-20180130") diff --git a/tests/freqai/test_freqai_interface.py b/tests/freqai/test_freqai_interface.py index 407fb32ab..46e25462b 100644 --- a/tests/freqai/test_freqai_interface.py +++ b/tests/freqai/test_freqai_interface.py @@ -55,6 +55,10 @@ def test_extract_data_and_train_model_Standard(mocker, freqai_conf, model, pca, can_run_model(model) + test_tb = True + if is_mac(): + test_tb = False + model_save_ext = 'joblib' freqai_conf.update({"freqaimodel": model}) freqai_conf.update({"timerange": "20180110-20180130"}) @@ -90,6 +94,7 @@ def test_extract_data_and_train_model_Standard(mocker, freqai_conf, model, pca, strategy.freqai_info = freqai_conf.get("freqai", {}) freqai = strategy.freqai freqai.live = True + freqai.activate_tensorboard = test_tb freqai.can_short = can_short freqai.dk = FreqaiDataKitchen(freqai_conf) freqai.dk.live = True @@ -139,6 +144,7 @@ def test_extract_data_and_train_model_MultiTargets(mocker, freqai_conf, model, s strategy.freqai_info = freqai_conf.get("freqai", {}) freqai = strategy.freqai freqai.live = True + freqai.activate_tensorboard = False freqai.dk = FreqaiDataKitchen(freqai_conf) freqai.dk.live = True timerange = TimeRange.parse_timerange("20180110-20180130") @@ -183,6 +189,7 @@ def test_extract_data_and_train_model_Classifiers(mocker, freqai_conf, model): strategy.freqai_info = freqai_conf.get("freqai", {}) freqai = strategy.freqai freqai.live = True + freqai.activate_tensorboard = False freqai.dk = FreqaiDataKitchen(freqai_conf) freqai.dk.live = True timerange = TimeRange.parse_timerange("20180110-20180130") @@ -235,6 +242,9 @@ def test_extract_data_and_train_model_Classifiers(mocker, freqai_conf, model): ) def test_start_backtesting(mocker, freqai_conf, model, num_files, strat, caplog): can_run_model(model) + test_tb = True + if is_mac(): + test_tb = False freqai_conf.get("freqai", {}).update({"save_backtest_models": True}) freqai_conf['runmode'] = RunMode.BACKTEST @@ -267,6 +277,7 @@ def test_start_backtesting(mocker, freqai_conf, model, num_files, strat, caplog) strategy.freqai_info = freqai_conf.get("freqai", {}) freqai = strategy.freqai freqai.live = False + freqai.activate_tensorboard = test_tb freqai.dk = FreqaiDataKitchen(freqai_conf) timerange = TimeRange.parse_timerange("20180110-20180130") freqai.dd.load_all_pair_histories(timerange, freqai.dk) @@ -308,6 +319,7 @@ def test_start_backtesting_subdaily_backtest_period(mocker, freqai_conf): strategy.freqai_info = freqai_conf.get("freqai", {}) freqai = strategy.freqai freqai.live = False + freqai.activate_tensorboard = False freqai.dk = FreqaiDataKitchen(freqai_conf) timerange = TimeRange.parse_timerange("20180110-20180130") freqai.dd.load_all_pair_histories(timerange, freqai.dk) @@ -427,6 +439,7 @@ def test_principal_component_analysis(mocker, freqai_conf): strategy.freqai_info = freqai_conf.get("freqai", {}) freqai = strategy.freqai freqai.live = True + freqai.activate_tensorboard = False freqai.dk = FreqaiDataKitchen(freqai_conf) freqai.dk.live = True timerange = TimeRange.parse_timerange("20180110-20180130") @@ -460,6 +473,7 @@ def test_plot_feature_importance(mocker, freqai_conf): strategy.freqai_info = freqai_conf.get("freqai", {}) freqai = strategy.freqai freqai.live = True + freqai.activate_tensorboard = False freqai.dk = FreqaiDataKitchen(freqai_conf) freqai.dk.live = True timerange = TimeRange.parse_timerange("20180110-20180130") From 340d2191ffffdda4c267ad9a73ccde9fb25ef86f Mon Sep 17 00:00:00 2001 From: robcaulk Date: Sun, 14 May 2023 14:39:23 +0000 Subject: [PATCH 17/25] deactivate tensorboard by default --- .../RL/BaseReinforcementLearningModel.py | 12 +++--------- freqtrade/freqai/freqai_interface.py | 2 +- .../prediction_models/ReinforcementLearner.py | 15 +++++++-------- tests/freqai/test_freqai_interface.py | 18 +++++++++--------- 4 files changed, 20 insertions(+), 27 deletions(-) diff --git a/freqtrade/freqai/RL/BaseReinforcementLearningModel.py b/freqtrade/freqai/RL/BaseReinforcementLearningModel.py index 4d540ee36..8ee3c7c56 100644 --- a/freqtrade/freqai/RL/BaseReinforcementLearningModel.py +++ b/freqtrade/freqai/RL/BaseReinforcementLearningModel.py @@ -68,11 +68,8 @@ class BaseReinforcementLearningModel(IFreqaiModel): self.unset_outlier_removal() self.net_arch = self.rl_config.get('net_arch', [128, 128]) self.dd.model_type = import_str - if self.activate_tensorboard: - self.tensorboard_callback: TensorboardCallback = \ - TensorboardCallback(verbose=1, actions=BaseActions) - else: - self.tenorboard_callback = None + self.tensorboard_callback: TensorboardCallback = \ + TensorboardCallback(verbose=1, actions=BaseActions) def unset_outlier_removal(self): """ @@ -159,10 +156,7 @@ class BaseReinforcementLearningModel(IFreqaiModel): best_model_save_path=str(dk.data_path)) actions = self.train_env.get_actions() - if self.activate_tensorboard: - self.tensorboard_callback = TensorboardCallback(verbose=1, actions=actions) - else: - self.tensorboard_callback = None # type: ignore + self.tensorboard_callback = TensorboardCallback(verbose=1, actions=actions) def pack_env_dict(self, pair: str) -> Dict[str, Any]: """ diff --git a/freqtrade/freqai/freqai_interface.py b/freqtrade/freqai/freqai_interface.py index 9cfda05ee..ae3876d2c 100644 --- a/freqtrade/freqai/freqai_interface.py +++ b/freqtrade/freqai/freqai_interface.py @@ -110,7 +110,7 @@ class IFreqaiModel(ABC): if self.ft_params.get('principal_component_analysis', False) and self.continual_learning: self.ft_params.update({'principal_component_analysis': False}) logger.warning('User tried to use PCA with continual learning. Deactivating PCA.') - self.activate_tensorboard: bool = self.freqai_info.get('activate_tensorboard', True) + self.activate_tensorboard: bool = self.freqai_info.get('activate_tensorboard', False) record_params(config, self.full_path) diff --git a/freqtrade/freqai/prediction_models/ReinforcementLearner.py b/freqtrade/freqai/prediction_models/ReinforcementLearner.py index 0d6c52445..a11decc92 100644 --- a/freqtrade/freqai/prediction_models/ReinforcementLearner.py +++ b/freqtrade/freqai/prediction_models/ReinforcementLearner.py @@ -58,10 +58,14 @@ class ReinforcementLearner(BaseReinforcementLearningModel): policy_kwargs = dict(activation_fn=th.nn.ReLU, net_arch=self.net_arch) + if self.activate_tensorboard: + tb_path = Path(dk.full_path / "tensorboard" / dk.pair.split('/')[0]) + else: + tb_path = None + if dk.pair not in self.dd.model_dictionary or not self.continual_learning: model = self.MODELCLASS(self.policy_type, self.train_env, policy_kwargs=policy_kwargs, - tensorboard_log=Path( - dk.full_path / "tensorboard" / dk.pair.split('/')[0]), + tensorboard_log=tb_path, **self.freqai_info.get('model_training_parameters', {}) ) else: @@ -70,14 +74,9 @@ class ReinforcementLearner(BaseReinforcementLearningModel): model = self.dd.model_dictionary[dk.pair] model.set_env(self.train_env) - callbacks = [self.eval_callback] - - if self.activate_tensorboard: - callbacks.append(self.tensorboard_callback) - model.learn( total_timesteps=int(total_timesteps), - callback=callbacks, + callback=[self.eval_callback, self.tensorboard_callback], progress_bar=self.rl_config.get('progress_bar', False) ) diff --git a/tests/freqai/test_freqai_interface.py b/tests/freqai/test_freqai_interface.py index 46e25462b..575e5c7e6 100644 --- a/tests/freqai/test_freqai_interface.py +++ b/tests/freqai/test_freqai_interface.py @@ -55,9 +55,9 @@ def test_extract_data_and_train_model_Standard(mocker, freqai_conf, model, pca, can_run_model(model) - test_tb = True - if is_mac(): - test_tb = False + # test_tb = True + # if is_mac(): + # test_tb = False model_save_ext = 'joblib' freqai_conf.update({"freqaimodel": model}) @@ -94,7 +94,7 @@ def test_extract_data_and_train_model_Standard(mocker, freqai_conf, model, pca, strategy.freqai_info = freqai_conf.get("freqai", {}) freqai = strategy.freqai freqai.live = True - freqai.activate_tensorboard = test_tb + # freqai.activate_tensorboard = test_tb freqai.can_short = can_short freqai.dk = FreqaiDataKitchen(freqai_conf) freqai.dk.live = True @@ -233,7 +233,7 @@ def test_extract_data_and_train_model_Classifiers(mocker, freqai_conf, model): ("CatboostRegressor", 2, "freqai_test_strat"), ("PyTorchMLPRegressor", 2, "freqai_test_strat"), ("PyTorchTransformerRegressor", 2, "freqai_test_strat"), - ("ReinforcementLearner", 3, "freqai_rl_test_strat"), + ("ReinforcementLearner", 2, "freqai_rl_test_strat"), ("XGBoostClassifier", 2, "freqai_test_classifier"), ("LightGBMClassifier", 2, "freqai_test_classifier"), ("CatboostClassifier", 2, "freqai_test_classifier"), @@ -242,9 +242,9 @@ def test_extract_data_and_train_model_Classifiers(mocker, freqai_conf, model): ) def test_start_backtesting(mocker, freqai_conf, model, num_files, strat, caplog): can_run_model(model) - test_tb = True - if is_mac(): - test_tb = False + # test_tb = True + # if is_mac(): + # test_tb = False freqai_conf.get("freqai", {}).update({"save_backtest_models": True}) freqai_conf['runmode'] = RunMode.BACKTEST @@ -277,7 +277,7 @@ def test_start_backtesting(mocker, freqai_conf, model, num_files, strat, caplog) strategy.freqai_info = freqai_conf.get("freqai", {}) freqai = strategy.freqai freqai.live = False - freqai.activate_tensorboard = test_tb + # freqai.activate_tensorboard = test_tb freqai.dk = FreqaiDataKitchen(freqai_conf) timerange = TimeRange.parse_timerange("20180110-20180130") freqai.dd.load_all_pair_histories(timerange, freqai.dk) From 73e9047cd59d7278a4241b497c70bae5031de3a2 Mon Sep 17 00:00:00 2001 From: robcaulk Date: Sun, 14 May 2023 14:53:12 +0000 Subject: [PATCH 18/25] try to deactivate any test that has a callback --- tests/freqai/conftest.py | 8 -------- tests/freqai/test_freqai_interface.py | 3 +++ 2 files changed, 3 insertions(+), 8 deletions(-) diff --git a/tests/freqai/conftest.py b/tests/freqai/conftest.py index eeb0182b6..13bbc4fca 100644 --- a/tests/freqai/conftest.py +++ b/tests/freqai/conftest.py @@ -20,14 +20,6 @@ def is_mac() -> bool: return "Darwin" in machine -@pytest.fixture(scope="function", autouse=True) -def patch_tensorboard_maconly(mocker): - if is_mac(): - # MacOS CI is not friendly to tensorboard - mocker.patch('freqtrade.freqai.tensorboard.tensorboard.SummaryWriter') - mocker.patch('freqtrade.freqai.tensorboard.tensorboard.TensorBoardCallback.after_iteration') - - @pytest.fixture(scope="function") def freqai_conf(default_conf, tmpdir): freqaiconf = deepcopy(default_conf) diff --git a/tests/freqai/test_freqai_interface.py b/tests/freqai/test_freqai_interface.py index 575e5c7e6..5eff3f3f6 100644 --- a/tests/freqai/test_freqai_interface.py +++ b/tests/freqai/test_freqai_interface.py @@ -36,6 +36,9 @@ def can_run_model(model: str) -> None: if is_pytorch_model and is_mac() and not is_arm(): pytest.skip("Reinforcement learning / PyTorch module not available on intel based Mac OS.") + if "XGBoost" in model and is_mac(): + pytest.skip("Mac doesn't tb callbacks?") + @pytest.mark.parametrize('model, pca, dbscan, float32, can_short, shuffle, buffer', [ ('LightGBMRegressor', True, False, True, True, False, 0), From 1ed084557a353f4fc24d5181cccda55ad92d455a Mon Sep 17 00:00:00 2001 From: robcaulk Date: Sun, 14 May 2023 15:44:41 +0000 Subject: [PATCH 19/25] try even more deactivation --- freqtrade/freqai/utils.py | 2 +- tests/freqai/test_freqai_interface.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/freqtrade/freqai/utils.py b/freqtrade/freqai/utils.py index cc803ae1e..e40b143d7 100644 --- a/freqtrade/freqai/utils.py +++ b/freqtrade/freqai/utils.py @@ -191,7 +191,7 @@ def get_timerange_backtest_live_models(config: Config) -> str: def get_tb_logger(model_type: str, path: Path, activate: bool) -> Union[TBLogger, BaseTensorboardLogger]: tb_logger: Union[TBLogger, BaseTensorboardLogger] - if model_type == "pytorch": + if model_type == "pytorch" and activate: tb_logger = TBLogger(path, activate) else: tb_logger = BaseTensorboardLogger(path, activate) diff --git a/tests/freqai/test_freqai_interface.py b/tests/freqai/test_freqai_interface.py index 5eff3f3f6..70b2a3b08 100644 --- a/tests/freqai/test_freqai_interface.py +++ b/tests/freqai/test_freqai_interface.py @@ -37,7 +37,7 @@ def can_run_model(model: str) -> None: pytest.skip("Reinforcement learning / PyTorch module not available on intel based Mac OS.") if "XGBoost" in model and is_mac(): - pytest.skip("Mac doesn't tb callbacks?") + pytest.skip("Mac github CI does not have enough memory/threads to run this test.") @pytest.mark.parametrize('model, pca, dbscan, float32, can_short, shuffle, buffer', [ From 505f36a95f23bc279f2416ff277c163c80301ae9 Mon Sep 17 00:00:00 2001 From: robcaulk Date: Sun, 14 May 2023 15:49:24 +0000 Subject: [PATCH 20/25] try even more deactivation --- freqtrade/freqai/utils.py | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/freqtrade/freqai/utils.py b/freqtrade/freqai/utils.py index e40b143d7..8afc87870 100644 --- a/freqtrade/freqai/utils.py +++ b/freqtrade/freqai/utils.py @@ -1,7 +1,7 @@ import logging from datetime import datetime, timezone from pathlib import Path -from typing import Any, Dict, Union +from typing import Any, Dict import numpy as np import pandas as pd @@ -16,8 +16,6 @@ from freqtrade.exchange import timeframe_to_seconds from freqtrade.exchange.exchange import market_is_active from freqtrade.freqai.data_drawer import FreqaiDataDrawer from freqtrade.freqai.data_kitchen import FreqaiDataKitchen -from freqtrade.freqai.tensorboard import TBLogger -from freqtrade.freqai.tensorboard.base_tensorboard import BaseTensorboardLogger from freqtrade.plugins.pairlist.pairlist_helpers import dynamic_expand_pairlist @@ -188,12 +186,13 @@ def get_timerange_backtest_live_models(config: Config) -> str: return timerange.timerange_str -def get_tb_logger(model_type: str, path: Path, activate: bool) -> Union[TBLogger, - BaseTensorboardLogger]: - tb_logger: Union[TBLogger, BaseTensorboardLogger] +def get_tb_logger(model_type: str, path: Path, activate: bool) -> Any: + if model_type == "pytorch" and activate: + from freqtrade.freqai.tensorboard import TBLogger tb_logger = TBLogger(path, activate) else: - tb_logger = BaseTensorboardLogger(path, activate) + from freqtrade.freqai.tensorboard.base_tensorboard import BaseTensorboardLogger + tb_logger = BaseTensorboardLogger(path, activate) # type: ignore return tb_logger From 9242dfa355392f3ff323e0ac098c3674a3212fb2 Mon Sep 17 00:00:00 2001 From: robcaulk Date: Sun, 14 May 2023 16:05:49 +0000 Subject: [PATCH 21/25] try reactivating tb for some tests --- freqtrade/freqai/freqai_interface.py | 2 +- freqtrade/freqai/tensorboard/base_tensorboard.py | 2 +- tests/freqai/test_freqai_interface.py | 16 ++++++++-------- 3 files changed, 10 insertions(+), 10 deletions(-) diff --git a/freqtrade/freqai/freqai_interface.py b/freqtrade/freqai/freqai_interface.py index ae3876d2c..9cfda05ee 100644 --- a/freqtrade/freqai/freqai_interface.py +++ b/freqtrade/freqai/freqai_interface.py @@ -110,7 +110,7 @@ class IFreqaiModel(ABC): if self.ft_params.get('principal_component_analysis', False) and self.continual_learning: self.ft_params.update({'principal_component_analysis': False}) logger.warning('User tried to use PCA with continual learning. Deactivating PCA.') - self.activate_tensorboard: bool = self.freqai_info.get('activate_tensorboard', False) + self.activate_tensorboard: bool = self.freqai_info.get('activate_tensorboard', True) record_params(config, self.full_path) diff --git a/freqtrade/freqai/tensorboard/base_tensorboard.py b/freqtrade/freqai/tensorboard/base_tensorboard.py index 8d396832b..f94ce6165 100644 --- a/freqtrade/freqai/tensorboard/base_tensorboard.py +++ b/freqtrade/freqai/tensorboard/base_tensorboard.py @@ -13,7 +13,7 @@ class BaseTensorboardLogger: logger.warning("Tensorboard is not installed, no logs will be written." "Ensure torch is installed, or use the torch/RL docker images") - def log_scaler(self, tag: str, scalar_value: Any, step: int): + def log_scalar(self, tag: str, scalar_value: Any, step: int): return def close(self): diff --git a/tests/freqai/test_freqai_interface.py b/tests/freqai/test_freqai_interface.py index 70b2a3b08..5b0f73e92 100644 --- a/tests/freqai/test_freqai_interface.py +++ b/tests/freqai/test_freqai_interface.py @@ -58,9 +58,9 @@ def test_extract_data_and_train_model_Standard(mocker, freqai_conf, model, pca, can_run_model(model) - # test_tb = True - # if is_mac(): - # test_tb = False + test_tb = True + if is_mac(): + test_tb = False model_save_ext = 'joblib' freqai_conf.update({"freqaimodel": model}) @@ -97,7 +97,7 @@ def test_extract_data_and_train_model_Standard(mocker, freqai_conf, model, pca, strategy.freqai_info = freqai_conf.get("freqai", {}) freqai = strategy.freqai freqai.live = True - # freqai.activate_tensorboard = test_tb + freqai.activate_tensorboard = test_tb freqai.can_short = can_short freqai.dk = FreqaiDataKitchen(freqai_conf) freqai.dk.live = True @@ -245,9 +245,9 @@ def test_extract_data_and_train_model_Classifiers(mocker, freqai_conf, model): ) def test_start_backtesting(mocker, freqai_conf, model, num_files, strat, caplog): can_run_model(model) - # test_tb = True - # if is_mac(): - # test_tb = False + test_tb = True + if is_mac(): + test_tb = False freqai_conf.get("freqai", {}).update({"save_backtest_models": True}) freqai_conf['runmode'] = RunMode.BACKTEST @@ -280,7 +280,7 @@ def test_start_backtesting(mocker, freqai_conf, model, num_files, strat, caplog) strategy.freqai_info = freqai_conf.get("freqai", {}) freqai = strategy.freqai freqai.live = False - # freqai.activate_tensorboard = test_tb + freqai.activate_tensorboard = test_tb freqai.dk = FreqaiDataKitchen(freqai_conf) timerange = TimeRange.parse_timerange("20180110-20180130") freqai.dd.load_all_pair_histories(timerange, freqai.dk) From a225ab71e4cb5bab17e40b2b37e56f7a9d878528 Mon Sep 17 00:00:00 2001 From: robcaulk Date: Sun, 14 May 2023 16:18:33 +0000 Subject: [PATCH 22/25] revert file count --- tests/freqai/test_freqai_interface.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/freqai/test_freqai_interface.py b/tests/freqai/test_freqai_interface.py index 5b0f73e92..b69d72ef6 100644 --- a/tests/freqai/test_freqai_interface.py +++ b/tests/freqai/test_freqai_interface.py @@ -236,7 +236,7 @@ def test_extract_data_and_train_model_Classifiers(mocker, freqai_conf, model): ("CatboostRegressor", 2, "freqai_test_strat"), ("PyTorchMLPRegressor", 2, "freqai_test_strat"), ("PyTorchTransformerRegressor", 2, "freqai_test_strat"), - ("ReinforcementLearner", 2, "freqai_rl_test_strat"), + ("ReinforcementLearner", 3, "freqai_rl_test_strat"), ("XGBoostClassifier", 2, "freqai_test_classifier"), ("LightGBMClassifier", 2, "freqai_test_classifier"), ("CatboostClassifier", 2, "freqai_test_classifier"), From c7f7dd1d4ba61007c0dce4509fdaf697fea8a911 Mon Sep 17 00:00:00 2001 From: Matthias Date: Mon, 15 May 2023 18:27:12 +0200 Subject: [PATCH 23/25] Avoid unnecessary type ignore --- freqtrade/freqai/utils.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/freqtrade/freqai/utils.py b/freqtrade/freqai/utils.py index 8afc87870..1003ec87a 100644 --- a/freqtrade/freqai/utils.py +++ b/freqtrade/freqai/utils.py @@ -190,9 +190,7 @@ def get_tb_logger(model_type: str, path: Path, activate: bool) -> Any: if model_type == "pytorch" and activate: from freqtrade.freqai.tensorboard import TBLogger - tb_logger = TBLogger(path, activate) + return TBLogger(path, activate) else: - from freqtrade.freqai.tensorboard.base_tensorboard import BaseTensorboardLogger - tb_logger = BaseTensorboardLogger(path, activate) # type: ignore - - return tb_logger + from freqtrade.freqai.tensorboard import BaseTensorboardLogger + return BaseTensorboardLogger(path, activate) From 7d15c379cb8eb1e1cb8a58e46ee7a2ad24a4b3eb Mon Sep 17 00:00:00 2001 From: Matthias Date: Mon, 15 May 2023 19:26:51 +0200 Subject: [PATCH 24/25] Fix faulty removed import --- freqtrade/freqai/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/freqai/utils.py b/freqtrade/freqai/utils.py index 1003ec87a..b670a2aad 100644 --- a/freqtrade/freqai/utils.py +++ b/freqtrade/freqai/utils.py @@ -192,5 +192,5 @@ def get_tb_logger(model_type: str, path: Path, activate: bool) -> Any: from freqtrade.freqai.tensorboard import TBLogger return TBLogger(path, activate) else: - from freqtrade.freqai.tensorboard import BaseTensorboardLogger + from freqtrade.freqai.tensorboard.base_tensorboard import BaseTensorboardLogger return BaseTensorboardLogger(path, activate) From adeab13bdf4df765ff21007a81f8d0e9dcb3f70b Mon Sep 17 00:00:00 2001 From: robcaulk Date: Wed, 17 May 2023 07:21:48 +0000 Subject: [PATCH 25/25] cleanup tests, cross fingers that mac will pass --- freqtrade/freqai/tensorboard/base_tensorboard.py | 6 +++--- tests/freqai/conftest.py | 1 + tests/freqai/test_freqai_datakitchen.py | 1 - tests/freqai/test_freqai_interface.py | 8 -------- 4 files changed, 4 insertions(+), 12 deletions(-) diff --git a/freqtrade/freqai/tensorboard/base_tensorboard.py b/freqtrade/freqai/tensorboard/base_tensorboard.py index f94ce6165..c2d47137e 100644 --- a/freqtrade/freqai/tensorboard/base_tensorboard.py +++ b/freqtrade/freqai/tensorboard/base_tensorboard.py @@ -2,7 +2,7 @@ import logging from pathlib import Path from typing import Any -import xgboost as xgb +from xgboost.callback import TrainingCallback logger = logging.getLogger(__name__) @@ -20,14 +20,14 @@ class BaseTensorboardLogger: return -class BaseTensorBoardCallback(xgb.callback.TrainingCallback): +class BaseTensorBoardCallback(TrainingCallback): def __init__(self, logdir: Path, activate: bool = True): logger.warning("Tensorboard is not installed, no logs will be written." "Ensure torch is installed, or use the torch/RL docker images") def after_iteration( - self, model, epoch: int, evals_log: xgb.callback.TrainingCallback.EvalsLog + self, model, epoch: int, evals_log: TrainingCallback.EvalsLog ) -> bool: return False diff --git a/tests/freqai/conftest.py b/tests/freqai/conftest.py index 13bbc4fca..4c4891ceb 100644 --- a/tests/freqai/conftest.py +++ b/tests/freqai/conftest.py @@ -42,6 +42,7 @@ def freqai_conf(default_conf, tmpdir): "identifier": "uniqe-id100", "live_trained_timestamp": 0, "data_kitchen_thread_count": 2, + "activate_tensorboard": False, "feature_parameters": { "include_timeframes": ["5m"], "include_corr_pairlist": ["ADA/BTC"], diff --git a/tests/freqai/test_freqai_datakitchen.py b/tests/freqai/test_freqai_datakitchen.py index 6d6e10b94..13dc6b4b0 100644 --- a/tests/freqai/test_freqai_datakitchen.py +++ b/tests/freqai/test_freqai_datakitchen.py @@ -183,7 +183,6 @@ def test_get_full_model_path(mocker, freqai_conf, model): strategy.freqai_info = freqai_conf.get("freqai", {}) freqai = strategy.freqai freqai.live = True - freqai.activate_tensorboard = False freqai.dk = FreqaiDataKitchen(freqai_conf) freqai.dk.live = True timerange = TimeRange.parse_timerange("20180110-20180130") diff --git a/tests/freqai/test_freqai_interface.py b/tests/freqai/test_freqai_interface.py index b69d72ef6..61a7b7346 100644 --- a/tests/freqai/test_freqai_interface.py +++ b/tests/freqai/test_freqai_interface.py @@ -36,9 +36,6 @@ def can_run_model(model: str) -> None: if is_pytorch_model and is_mac() and not is_arm(): pytest.skip("Reinforcement learning / PyTorch module not available on intel based Mac OS.") - if "XGBoost" in model and is_mac(): - pytest.skip("Mac github CI does not have enough memory/threads to run this test.") - @pytest.mark.parametrize('model, pca, dbscan, float32, can_short, shuffle, buffer', [ ('LightGBMRegressor', True, False, True, True, False, 0), @@ -147,7 +144,6 @@ def test_extract_data_and_train_model_MultiTargets(mocker, freqai_conf, model, s strategy.freqai_info = freqai_conf.get("freqai", {}) freqai = strategy.freqai freqai.live = True - freqai.activate_tensorboard = False freqai.dk = FreqaiDataKitchen(freqai_conf) freqai.dk.live = True timerange = TimeRange.parse_timerange("20180110-20180130") @@ -192,7 +188,6 @@ def test_extract_data_and_train_model_Classifiers(mocker, freqai_conf, model): strategy.freqai_info = freqai_conf.get("freqai", {}) freqai = strategy.freqai freqai.live = True - freqai.activate_tensorboard = False freqai.dk = FreqaiDataKitchen(freqai_conf) freqai.dk.live = True timerange = TimeRange.parse_timerange("20180110-20180130") @@ -322,7 +317,6 @@ def test_start_backtesting_subdaily_backtest_period(mocker, freqai_conf): strategy.freqai_info = freqai_conf.get("freqai", {}) freqai = strategy.freqai freqai.live = False - freqai.activate_tensorboard = False freqai.dk = FreqaiDataKitchen(freqai_conf) timerange = TimeRange.parse_timerange("20180110-20180130") freqai.dd.load_all_pair_histories(timerange, freqai.dk) @@ -442,7 +436,6 @@ def test_principal_component_analysis(mocker, freqai_conf): strategy.freqai_info = freqai_conf.get("freqai", {}) freqai = strategy.freqai freqai.live = True - freqai.activate_tensorboard = False freqai.dk = FreqaiDataKitchen(freqai_conf) freqai.dk.live = True timerange = TimeRange.parse_timerange("20180110-20180130") @@ -476,7 +469,6 @@ def test_plot_feature_importance(mocker, freqai_conf): strategy.freqai_info = freqai_conf.get("freqai", {}) freqai = strategy.freqai freqai.live = True - freqai.activate_tensorboard = False freqai.dk = FreqaiDataKitchen(freqai_conf) freqai.dk.live = True timerange = TimeRange.parse_timerange("20180110-20180130")