From 46a1a2de101cb9f35d83cdfc6794e868a9312409 Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Thu, 11 Jan 2018 20:53:26 +0100 Subject: [PATCH 1/8] Update ta-lib from 0.4.14 to 0.4.15 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 8a3c8b9cf..e860baedd 100644 --- a/requirements.txt +++ b/requirements.txt @@ -11,7 +11,7 @@ scikit-learn==0.19.1 scipy==1.0.0 jsonschema==2.6.0 numpy==1.14.0 -TA-Lib==0.4.14 +TA-Lib==0.4.15 pytest==3.3.2 pytest-mock==1.6.3 pytest-cov==2.5.1 From 39c6e5263a422588adda63cdb358b947b5cbf988 Mon Sep 17 00:00:00 2001 From: Gerald Lonlas Date: Thu, 11 Jan 2018 21:09:04 -0800 Subject: [PATCH 2/8] Fix plot_dataframe.py --- scripts/plot_dataframe.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/plot_dataframe.py b/scripts/plot_dataframe.py index ce636a4b5..f07033637 100755 --- a/scripts/plot_dataframe.py +++ b/scripts/plot_dataframe.py @@ -10,7 +10,7 @@ from freqtrade.misc import common_args_parser def plot_parse_args(args ): - parser = common_args_parser(args, 'Graph utility') + parser = common_args_parser(description='Graph utility') parser.add_argument( '-p', '--pair', help = 'What currency pair', From a26cb4bc6b3b438df1f606c1ba9c3a7fa59b6ac6 Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Fri, 12 Jan 2018 11:08:23 +0100 Subject: [PATCH 3/8] Update pymarketcap from 3.3.145 to 3.3.147 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 8a3c8b9cf..52f4e7f7e 100644 --- a/requirements.txt +++ b/requirements.txt @@ -19,7 +19,7 @@ hyperopt==0.1 # do not upgrade networkx before this is fixed https://github.com/hyperopt/hyperopt/issues/325 networkx==1.11 tabulate==0.8.2 -pymarketcap==3.3.145 +pymarketcap==3.3.147 # Required for plotting data #matplotlib==2.1.0 From 3087ca0823a688f329a9ccbe265656db0048bccb Mon Sep 17 00:00:00 2001 From: Gerald Lonlas Date: Fri, 12 Jan 2018 22:56:39 -0800 Subject: [PATCH 4/8] Update freqtrade version --- freqtrade/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/__init__.py b/freqtrade/__init__.py index a190ca117..d1671e4c6 100644 --- a/freqtrade/__init__.py +++ b/freqtrade/__init__.py @@ -1,5 +1,5 @@ """ FreqTrade bot """ -__version__ = '0.14.3' +__version__ = '0.15.1' class DependencyException(BaseException): From f7a44d1cecd2c01188660c4e6244e06c21d3a872 Mon Sep 17 00:00:00 2001 From: Jean-Baptiste LE STANG Date: Sat, 13 Jan 2018 09:50:02 +0100 Subject: [PATCH 5/8] Fixing the ticker analysis with null value --- freqtrade/exchange/bittrex.py | 6 ++---- freqtrade/tests/exchange/test_exchange_bittrex.py | 13 ++++++++++++- 2 files changed, 14 insertions(+), 5 deletions(-) diff --git a/freqtrade/exchange/bittrex.py b/freqtrade/exchange/bittrex.py index a5fd68016..e6cacbd4e 100644 --- a/freqtrade/exchange/bittrex.py +++ b/freqtrade/exchange/bittrex.py @@ -123,10 +123,8 @@ class Bittrex(Exchange): message=data['message'], pair=pair)) - if not data.get('result') \ - or not data['result'].get('Bid') \ - or not data['result'].get('Ask') \ - or not data['result'].get('Last'): + if not data.get('result') or\ + not all(key in data.get('result', {}) for key in ['Bid', 'Ask', 'Last']): raise ContentDecodingError('{message} params=({pair})'.format( message='Got invalid response from bittrex', pair=pair)) diff --git a/freqtrade/tests/exchange/test_exchange_bittrex.py b/freqtrade/tests/exchange/test_exchange_bittrex.py index e01e7fa02..4302f9f5f 100644 --- a/freqtrade/tests/exchange/test_exchange_bittrex.py +++ b/freqtrade/tests/exchange/test_exchange_bittrex.py @@ -212,7 +212,18 @@ def test_exchange_bittrex_get_ticker_bad(): wb = make_wrap_bittrex() fb = FakeBittrex() fb.result = {'success': True, - 'result': {'Bid': 1}} # incomplete result + 'result': {'Bid': 1, 'Ask': 0}} # incomplete result + + with pytest.raises(ContentDecodingError, match=r'.*Got invalid response from bittrex params.*'): + wb.get_ticker('BTC_ETH') + fb.result = {'success': False, + 'message': 'gone bad' + } + with pytest.raises(btx.OperationalException, match=r'.*gone bad.*'): + wb.get_ticker('BTC_ETH') + + fb.result = {'success': True, + 'result': {}} # incomplete result with pytest.raises(ContentDecodingError, match=r'.*Got invalid response from bittrex params.*'): wb.get_ticker('BTC_ETH') fb.result = {'success': False, From e5b27baa590cf72fe2a8131837c064bf829874c2 Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Sat, 13 Jan 2018 13:38:23 +0100 Subject: [PATCH 6/8] Update pymarketcap from 3.3.147 to 3.3.148 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 80cc78e17..2bc86ad61 100644 --- a/requirements.txt +++ b/requirements.txt @@ -19,7 +19,7 @@ hyperopt==0.1 # do not upgrade networkx before this is fixed https://github.com/hyperopt/hyperopt/issues/325 networkx==1.11 tabulate==0.8.2 -pymarketcap==3.3.147 +pymarketcap==3.3.148 # Required for plotting data #matplotlib==2.1.0 From 3277e491f18835fd7697bc4a4467277b5ba27703 Mon Sep 17 00:00:00 2001 From: kryofly Date: Sat, 13 Jan 2018 17:39:36 +0100 Subject: [PATCH 7/8] support download for multiple testdata sets --- docs/backtesting.md | 15 +++++++ freqtrade/misc.py | 5 +++ .../tests/testdata/download_backtest_data.py | 41 +++++++++++-------- freqtrade/tests/testdata/pairs.json | 23 +++++++++++ 4 files changed, 68 insertions(+), 16 deletions(-) create mode 100644 freqtrade/tests/testdata/pairs.json diff --git a/docs/backtesting.md b/docs/backtesting.md index c426e2b5c..8574d9dc2 100644 --- a/docs/backtesting.md +++ b/docs/backtesting.md @@ -51,6 +51,21 @@ python3 ./freqtrade/main.py backtesting --realistic-simulation --live python3 ./freqtrade/main.py backtesting --datadir freqtrade/tests/testdata-20180101 ``` +To update your testdata directory, or download into another testdata directory: +```bash +mkdir freqtrade/tests/testdata-20180113 +cp freqtrade/tests/testdata/pairs.json freqtrade/tests/testdata-20180113 +cd freqtrade/tests/testdata-20180113 + +Possibly edit pairs.json file to include/exclude pairs + +python download_backtest_data.py -p pairs.json +``` + +The script will read your pairs.json file, and download ticker data +into the current working directory. + + For help about backtesting usage, please refer to [Backtesting commands](#backtesting-commands). diff --git a/freqtrade/misc.py b/freqtrade/misc.py index 3d70f6b25..979174f8d 100644 --- a/freqtrade/misc.py +++ b/freqtrade/misc.py @@ -15,6 +15,11 @@ from freqtrade import __version__ logger = logging.getLogger(__name__) +def file_dump_json(filename, data): + with open(filename, 'w') as fp: + json.dump(data, fp) + + class State(enum.Enum): RUNNING = 0 STOPPED = 1 diff --git a/freqtrade/tests/testdata/download_backtest_data.py b/freqtrade/tests/testdata/download_backtest_data.py index 37cd4c95f..0cb545b3a 100755 --- a/freqtrade/tests/testdata/download_backtest_data.py +++ b/freqtrade/tests/testdata/download_backtest_data.py @@ -1,29 +1,38 @@ #!/usr/bin/env python3 """This script generate json data from bittrex""" +import sys import json -from os import path from freqtrade import exchange from freqtrade.exchange import Bittrex +from freqtrade import misc -PAIRS = [ - 'BTC_BCC', 'BTC_ETH', 'BTC_MER', 'BTC_POWR', 'BTC_ETC', - 'BTC_OK', 'BTC_NEO', 'BTC_EMC2', 'BTC_DASH', 'BTC_LSK', - 'BTC_LTC', 'BTC_XZC', 'BTC_OMG', 'BTC_STRAT', 'BTC_XRP', - 'BTC_QTUM', 'BTC_WAVES', 'BTC_VTC', 'BTC_XLM', 'BTC_MCO' -] -TICKER_INTERVAL = 5 # ticker interval in minutes (currently implemented: 1 and 5) -OUTPUT_DIR = path.dirname(path.realpath(__file__)) +parser = misc.common_args_parser('download utility') +parser.add_argument( + '-p', '--pair', + help='JSON file containing pairs to download', + dest='pair', + default=None +) +args = parser.parse_args(sys.argv[1:]) + +TICKER_INTERVALS = [1, 5] # ticker interval in minutes (currently implemented: 1 and 5) +PAIRS = [] + +if args.pair: + with open(args.pair) as file: + PAIRS = json.load(file) +PAIRS = list(set(PAIRS)) + +print('About to download pairs:', PAIRS) # Init Bittrex exchange exchange._API = Bittrex({'key': '', 'secret': ''}) for pair in PAIRS: - data = exchange.get_ticker_history(pair, TICKER_INTERVAL) - filename = path.join(OUTPUT_DIR, '{}-{}.json'.format( - pair, - TICKER_INTERVAL, - )) - with open(filename, 'w') as fp: - json.dump(data, fp) + for tick_interval in TICKER_INTERVALS: + print('downloading pair %s, interval %s' % (pair, tick_interval)) + data = exchange.get_ticker_history(pair, tick_interval) + filename = '{}-{}.json'.format(pair, tick_interval) + misc.file_dump_json(filename, data) diff --git a/freqtrade/tests/testdata/pairs.json b/freqtrade/tests/testdata/pairs.json new file mode 100644 index 000000000..c3e339b4c --- /dev/null +++ b/freqtrade/tests/testdata/pairs.json @@ -0,0 +1,23 @@ +[ + "BTC_ADA", + "BTC_BAT", + "BTC_DASH", + "BTC_ETC", + "BTC_ETH", + "BTC_GBYTE", + "BTC_LSK", + "BTC_LTC", + "BTC_NEO", + "BTC_NXT", + "BTC_POWR", + "BTC_STORJ", + "BTC_QTUM", + "BTC_WAVES", + "BTC_VTC", + "BTC_XLM", + "BTC_XMR", + "BTC_XVG", + "BTC_XRP", + "BTC_ZEC" +] + From 344843d802241f26877c0820ea5e3b7e6a869f7e Mon Sep 17 00:00:00 2001 From: Gerald Lonlas Date: Sat, 13 Jan 2018 22:54:22 -0800 Subject: [PATCH 8/8] Update doc: 'cp' becomes 'cp -n', and add more FAQ questions --- docs/faq.md | 64 +++++++++++++++++++++++++++++++++++++----- docs/installation.md | 4 +-- docs/sql_cheatsheet.md | 12 ++++++++ 3 files changed, 71 insertions(+), 9 deletions(-) diff --git a/docs/faq.md b/docs/faq.md index 58929e15c..b3f15613a 100644 --- a/docs/faq.md +++ b/docs/faq.md @@ -2,20 +2,70 @@ #### I have waited 5 minutes, why hasn't the bot made any trades yet?! -Depending on the buy strategy, the amount of whitelisted coins, the situation of the market etc, it can take up to hours to find good entry position for a trade. Be patient! +Depending on the buy strategy, the amount of whitelisted coins, the +situation of the market etc, it can take up to hours to find good entry +position for a trade. Be patient! #### I have made 12 trades already, why is my total profit negative?! -I understand your disappointment but unfortunately 12 trades is just not enough to say anything. If you run backtesting, you can see that our current algorithm does leave you on the plus side, but that is after thousands of trades and even there, you will be left with losses on specific coins that you have traded tens if not hundreds of times. We of course constantly aim to improve the bot but it will _always_ be a gamble, which should leave you with modest wins on monthly basis but you can't say much from few trades. +I understand your disappointment but unfortunately 12 trades is just +not enough to say anything. If you run backtesting, you can see that our +current algorithm does leave you on the plus side, but that is after +thousands of trades and even there, you will be left with losses on +specific coins that you have traded tens if not hundreds of times. We +of course constantly aim to improve the bot but it will _always_ be a +gamble, which should leave you with modest wins on monthly basis but +you can't say much from few trades. -#### I’d like to change the stake amount. Can I just stop the bot with /stop and then change the config.json and run it again? +#### I’d like to change the stake amount. Can I just stop the bot with +/stop and then change the config.json and run it again? -Not quite. Trades are persisted to a database but the configuration is currently only read when the bot is killed and restarted. `/stop` more like pauses. You can stop your bot, adjust settings and start it again. +Not quite. Trades are persisted to a database but the configuration is +currently only read when the bot is killed and restarted. `/stop` more +like pauses. You can stop your bot, adjust settings and start it again. #### I want to improve the bot with a new strategy -That's great. We have a nice backtesting and hyperoptimizing setup. See the tutorial [[here|Testing-new-strategies-with-Hyperopt]]. +That's great. We have a nice backtesting and hyperoptimizing setup. See +the tutorial [here|Testing-new-strategies-with-Hyperopt](https://github.com/gcarq/freqtrade/blob/develop/docs/bot-usage.md#hyperopt-commands). -#### Is there a setting to only SELL the coins being held and not perform anymore BUYS? +#### Is there a setting to only SELL the coins being held and not +perform anymore BUYS? + +You can use the `/forcesell all` command from Telegram. + +### How many epoch do I need to get a good Hyperopt result? +Per default Hyperopts without `-e` or `--epochs` parameter will only +run 100 epochs, means 100 evals of your triggers, guards, .... Too few +to find a great result (unless if you are very lucky), so you probably +have to run it for 10.000 or more. But it will take an eternity to +compute. + +We recommend you to run it at least 10.000 epochs: +```bash +python3 ./freqtrade/main.py hyperopt -e 10000 +``` + +or if you want intermediate result to see +```bash +for i in {1..100}; do python3 ./freqtrade/main.py hyperopt -e 100; done +``` + +#### Why it is so long to run hyperopt? +Finding a great Hyperopt results takes time. + +If you wonder why it takes a while to find great hyperopt results + +This answer was written during the under the release 0.15.1, when we had +: +- 8 triggers +- 9 guards: let's say we evaluate even 10 values from each +- 1 stoploss calculation: let's say we want 10 values from that too to +be evaluated + +The following calculation is still very rough and not very precise +but it will give the idea. With only these triggers and guards there is +already 8*10^9*10 evaluations. A roughly total of 80 billion evals. +Did you run 100 000 evals? Congrats, you've done roughly 1 / 100 000 th +of the search space. -You can use the `/forcesell all` command from Telegram. \ No newline at end of file diff --git a/docs/installation.md b/docs/installation.md index 30431345b..59e0663ae 100644 --- a/docs/installation.md +++ b/docs/installation.md @@ -40,7 +40,7 @@ cd freqtrade ``` 4. Copy `config.sample` to `config.json` ```bash -cp config.json.example config.json +cp -n config.json.example config.json ``` To edit the config please refer to the [Bot Configuration](https://github.com/gcarq/freqtrade/blob/develop/docs/configuration.md) page 5. Create your DB file (Optional, the bot will create it if it is missing) @@ -194,7 +194,7 @@ The following steps are made for Linux/mac environment ## 4. Prepare the bot ```bash cd freqtrade -cp config.json.example config.json +cp -n config.json.example config.json ``` To edit the config please refer to [Bot Configuration](https://github.com/gcarq/freqtrade/blob/develop/docs/configuration.md) diff --git a/docs/sql_cheatsheet.md b/docs/sql_cheatsheet.md index 1edbcde5b..065f264f1 100644 --- a/docs/sql_cheatsheet.md +++ b/docs/sql_cheatsheet.md @@ -67,6 +67,18 @@ SET is_open=0, close_date='2017-12-20 03:08:45.103418', close_rate=0.19638016, c WHERE id=31; ``` +## Insert manually a new trade + +```sql +INSERT +INTO trades (exchange, pair, is_open, fee, open_rate, stake_amount, amount, open_date) +VALUES ('BITTREX', 'BTC_', 1, 0.0025, , , , '') +``` + +**Example:** +```sql +INSERT INTO trades (exchange, pair, is_open, fee, open_rate, stake_amount, amount, open_date) VALUES ('BITTREX', 'BTC_ETC', 1, 0.0025, 0.00258580, 0.002, 0.7715262081, '2017-11-28 12:44:24.000000') +``` ## Fix wrong fees in the table If your DB was created before