diff --git a/docs/sql_cheatsheet.md b/docs/sql_cheatsheet.md index 065f264f1..141eaeafe 100644 --- a/docs/sql_cheatsheet.md +++ b/docs/sql_cheatsheet.md @@ -32,9 +32,12 @@ CREATE TABLE trades ( exchange VARCHAR NOT NULL, pair VARCHAR NOT NULL, is_open BOOLEAN NOT NULL, - fee FLOAT NOT NULL, + fee_open FLOAT NOT NULL, + fee_close FLOAT NOT NULL, open_rate FLOAT, + open_rate_requested FLOAT, close_rate FLOAT, + close_rate_requested FLOAT, close_profit FLOAT, stake_amount FLOAT NOT NULL, amount FLOAT, @@ -71,13 +74,13 @@ WHERE id=31; ```sql INSERT -INTO trades (exchange, pair, is_open, fee, open_rate, stake_amount, amount, open_date) -VALUES ('BITTREX', 'BTC_', 1, 0.0025, , , , '') +INTO trades (exchange, pair, is_open, fee_open, fee_close, open_rate, stake_amount, amount, open_date) +VALUES ('BITTREX', 'BTC_', 1, 0.0025, 0.0025, , , , '') ``` **Example:** ```sql -INSERT INTO trades (exchange, pair, is_open, fee, open_rate, stake_amount, amount, open_date) VALUES ('BITTREX', 'BTC_ETC', 1, 0.0025, 0.00258580, 0.002, 0.7715262081, '2017-11-28 12:44:24.000000') +INSERT INTO trades (exchange, pair, is_open, fee_open, fee_close, open_rate, stake_amount, amount, open_date) VALUES ('BITTREX', 'BTC_ETC', 1, 0.0025, 0.0025, 0.00258580, 0.002, 0.7715262081, '2017-11-28 12:44:24.000000') ``` ## Fix wrong fees in the table @@ -87,4 +90,4 @@ If your DB was created before ```sql UPDATE trades SET fee=0.0025 WHERE fee=0.005; -``` \ No newline at end of file +``` diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index 0a332b952..56ace2ae2 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -330,6 +330,7 @@ class FreqtradeBot(object): fee_open=fee, fee_close=fee, open_rate=buy_limit, + open_rate_requested=buy_limit, open_date=datetime.utcnow(), exchange=exchange.get_id(), open_order_id=order_id @@ -538,6 +539,7 @@ class FreqtradeBot(object): # Execute sell and update trade record order_id = exchange.sell(str(trade.pair), limit, trade.amount)['id'] trade.open_order_id = order_id + trade.close_rate_requested = limit fmt_exp_profit = round(trade.calc_profit_percent(rate=limit) * 100, 2) profit_trade = trade.calc_profit(rate=limit) diff --git a/freqtrade/persistence.py b/freqtrade/persistence.py index ed81ad2ec..2d497662e 100644 --- a/freqtrade/persistence.py +++ b/freqtrade/persistence.py @@ -15,6 +15,8 @@ from sqlalchemy.ext.declarative import declarative_base from sqlalchemy.orm.scoping import scoped_session from sqlalchemy.orm.session import sessionmaker from sqlalchemy.pool import StaticPool +from sqlalchemy import inspect + logger = logging.getLogger(__name__) @@ -50,12 +52,61 @@ def init(config: dict, engine: Optional[Engine] = None) -> None: Trade.session = session() Trade.query = session.query_property() _DECL_BASE.metadata.create_all(engine) + check_migrate(engine) # Clean dry_run DB if _CONF.get('dry_run', False) and _CONF.get('dry_run_db', False): clean_dry_run_db() +def has_column(columns, searchname: str) -> bool: + return len(list(filter(lambda x: x["name"] == searchname, columns))) == 1 + + +def check_migrate(engine) -> None: + """ + Checks if migration is necessary and migrates if necessary + """ + inspector = inspect(engine) + + cols = inspector.get_columns('trades') + + if not has_column(cols, 'fee_open'): + # Schema migration necessary + engine.execute("alter table trades rename to trades_bak") + # let SQLAlchemy create the schema as required + _DECL_BASE.metadata.create_all(engine) + + # Copy data back - following the correct schema + engine.execute("""insert into trades + (id, exchange, pair, is_open, fee_open, fee_close, open_rate, + open_rate_requested, close_rate, close_rate_requested, close_profit, + stake_amount, amount, open_date, close_date, open_order_id) + select id, lower(exchange), + case + when instr(pair, '_') != 0 then + substr(pair, instr(pair, '_') + 1) || '/' || + substr(pair, 1, instr(pair, '_') - 1) + else pair + end + pair, + is_open, fee fee_open, fee fee_close, + open_rate, null open_rate_requested, close_rate, + null close_rate_requested, close_profit, + stake_amount, amount, open_date, close_date, open_order_id + from trades_bak + """) + + # Reread columns - the above recreated the table! + inspector = inspect(engine) + cols = inspector.get_columns('trades') + + if not has_column(cols, 'open_rate_requested'): + engine.execute("alter table trades add open_rate_requested float") + if not has_column(cols, 'close_rate_requested'): + engine.execute("alter table trades add close_rate_requested float") + + def cleanup() -> None: """ Flushes all pending operations to disk. @@ -88,7 +139,9 @@ class Trade(_DECL_BASE): fee_open = Column(Float, nullable=False, default=0.0) fee_close = Column(Float, nullable=False, default=0.0) open_rate = Column(Float) + open_rate_requested = Column(Float) close_rate = Column(Float) + close_rate_requested = Column(Float) close_profit = Column(Float) stake_amount = Column(Float, nullable=False) amount = Column(Float) diff --git a/freqtrade/tests/test_persistence.py b/freqtrade/tests/test_persistence.py index db8a5e9bd..3e0f50fbb 100644 --- a/freqtrade/tests/test_persistence.py +++ b/freqtrade/tests/test_persistence.py @@ -375,3 +375,105 @@ def test_clean_dry_run_db(default_conf, fee): # We have now only the prod assert len(Trade.query.filter(Trade.open_order_id.isnot(None)).all()) == 1 + + +def test_migrate_old(default_conf, fee): + """ + Test Database migration(starting with old pairformat) + """ + amount = 103.223 + create_table_old = """CREATE TABLE IF NOT EXISTS "trades" ( + id INTEGER NOT NULL, + exchange VARCHAR NOT NULL, + pair VARCHAR NOT NULL, + is_open BOOLEAN NOT NULL, + fee FLOAT NOT NULL, + open_rate FLOAT, + close_rate FLOAT, + close_profit FLOAT, + stake_amount FLOAT NOT NULL, + amount FLOAT, + open_date DATETIME NOT NULL, + close_date DATETIME, + open_order_id VARCHAR, + PRIMARY KEY (id), + CHECK (is_open IN (0, 1)) + );""" + insert_table_old = """INSERT INTO trades (exchange, pair, is_open, fee, + open_rate, stake_amount, amount, open_date) + VALUES ('BITTREX', 'BTC_ETC', 1, {fee}, + 0.00258580, {stake}, {amount}, + '2017-11-28 12:44:24.000000') + """.format(fee=fee.return_value, + stake=default_conf.get("stake_amount"), + amount=amount + ) + engine = create_engine('sqlite://') + # Create table using the old format + engine.execute(create_table_old) + engine.execute(insert_table_old) + # Run init to test migration + init(default_conf, engine) + + assert len(Trade.query.filter(Trade.id == 1).all()) == 1 + trade = Trade.query.filter(Trade.id == 1).first() + assert trade.fee_open == fee.return_value + assert trade.fee_close == fee.return_value + assert trade.open_rate_requested is None + assert trade.close_rate_requested is None + assert trade.is_open == 1 + assert trade.amount == amount + assert trade.stake_amount == default_conf.get("stake_amount") + assert trade.pair == "ETC/BTC" + assert trade.exchange == "bittrex" + + +def test_migrate_new(default_conf, fee): + """ + Test Database migration (starting with new pairformat) + """ + amount = 103.223 + create_table_old = """CREATE TABLE IF NOT EXISTS "trades" ( + id INTEGER NOT NULL, + exchange VARCHAR NOT NULL, + pair VARCHAR NOT NULL, + is_open BOOLEAN NOT NULL, + fee FLOAT NOT NULL, + open_rate FLOAT, + close_rate FLOAT, + close_profit FLOAT, + stake_amount FLOAT NOT NULL, + amount FLOAT, + open_date DATETIME NOT NULL, + close_date DATETIME, + open_order_id VARCHAR, + PRIMARY KEY (id), + CHECK (is_open IN (0, 1)) + );""" + insert_table_old = """INSERT INTO trades (exchange, pair, is_open, fee, + open_rate, stake_amount, amount, open_date) + VALUES ('binance', 'ETC/BTC', 1, {fee}, + 0.00258580, {stake}, {amount}, + '2019-11-28 12:44:24.000000') + """.format(fee=fee.return_value, + stake=default_conf.get("stake_amount"), + amount=amount + ) + engine = create_engine('sqlite://') + # Create table using the old format + engine.execute(create_table_old) + engine.execute(insert_table_old) + # Run init to test migration + init(default_conf, engine) + + assert len(Trade.query.filter(Trade.id == 1).all()) == 1 + trade = Trade.query.filter(Trade.id == 1).first() + assert trade.fee_open == fee.return_value + assert trade.fee_close == fee.return_value + assert trade.open_rate_requested is None + assert trade.close_rate_requested is None + assert trade.is_open == 1 + assert trade.amount == amount + assert trade.stake_amount == default_conf.get("stake_amount") + assert trade.pair == "ETC/BTC" + assert trade.exchange == "binance"