Add memory pool based fee estimates

- fee estimates can use ETA or mempool
 - require protocol version 1.2
 - remove fee_unit preference

gui/kivy/uix/dialogs/fee_dialog.py

@@ -32,7 +32,15 @@ Builder.load_string('''
                 text: _('Dynamic Fees')
             CheckBox:
                 id: dynfees
-                on_active: root.on_checkbox(self.active)
+                on_active: root.on_dynfees(self.active)
+        BoxLayout:
+            orientation: 'horizontal'
+            size_hint: 1, 0.5
+            Label:
+                text: _('Use mempool')
+            CheckBox:
+                id: mempool
+                on_active: root.on_mempool(self.active)
         Widget:
             size_hint: 1, 1
         BoxLayout:
@@ -60,7 +68,9 @@ class FeeDialog(Factory.Popup):
         self.config = config
         self.fee_rate = self.config.fee_per_kb()
         self.callback = callback
+        self.mempool = self.config.get('mempool_fees', False)
         self.dynfees = self.config.get('dynamic_fees', True)
+        self.ids.mempool.active = self.mempool
         self.ids.dynfees.active = self.dynfees
         self.update_slider()
         self.update_text()
@@ -71,34 +81,30 @@ class FeeDialog(Factory.Popup):
 
     def update_slider(self):
         slider = self.ids.slider
-        if self.dynfees:
-            slider.range = (0, 4)
-            slider.step = 1
-            slider.value = self.config.get('fee_level', 2)
-        else:
-            slider.range = (0, 9)
-            slider.step = 1
-            slider.value = self.config.static_fee_index(self.fee_rate)
+        maxp, pos, fee_rate = self.config.get_fee_slider(self.dynfees, self.mempool)
+        slider.range = (0, maxp)
+        slider.step = 1
+        slider.value = pos
 
-    def get_fee_text(self, value):
-        if self.ids.dynfees.active:
-            tooltip = fee_levels[value]
-            if self.config.has_fee_estimates():
-                dynfee = self.config.dynfee(value)
-                tooltip += '\n' + (self.app.format_amount_and_units(dynfee)) + '/kB'
+    def get_fee_text(self, pos):
+        dyn = self.dynfees
+        mempool = self.mempool
+        if dyn:
+            fee_rate = self.config.depth_to_fee(pos) if mempool else self.config.eta_to_fee(pos)
         else:
-            fee_rate = self.config.static_fee(value)
-            tooltip = self.app.format_amount_and_units(fee_rate) + '/kB'
-            if self.config.has_fee_estimates():
-                i = self.config.reverse_dynfee(fee_rate)
-                tooltip += '\n' + (_('low fee') if i < 0 else 'Within %d blocks'%i)
-        return tooltip
+            fee_rate = self.config.static_fee(pos)
+        target, tooltip = self.config.get_fee_text(pos, dyn, mempool, fee_rate)
+        return target
 
     def on_ok(self):
         value = int(self.ids.slider.value)
         self.config.set_key('dynamic_fees', self.dynfees, False)
+        self.config.set_key('mempool_fees', self.mempool, False)
         if self.dynfees:
-            self.config.set_key('fee_level', value, True)
+            if self.mempool:
+                self.config.set_key('depth_level', value, True)
+            else:
+                self.config.set_key('fee_level', value, True)
         else:
             self.config.set_key('fee_per_kb', self.config.static_fee(value), True)
         self.callback()
@@ -106,7 +112,12 @@ class FeeDialog(Factory.Popup):
     def on_slider(self, value):
         self.update_text()
 
-    def on_checkbox(self, b):
+    def on_dynfees(self, b):
         self.dynfees = b
         self.update_slider()
         self.update_text()
+
+    def on_mempool(self, b):
+        self.mempool = b
+        self.update_slider()
+        self.update_text()

gui/kivy/uix/dialogs/settings.py

@@ -204,10 +204,7 @@ class SettingsDialog(Factory.Popup):
         d.open()
 
     def fee_status(self):
-        if self.config.get('dynamic_fees', True):
-            return fee_levels[self.config.get('fee_level', 2)]
-        else:
-            return self.app.format_amount_and_units(self.config.fee_per_kb()) + '/kB'
+        return self.config.get_fee_status()
 
     def fee_dialog(self, label, dt):
         if self._fee_dialog is None:

gui/qt/amountedit.py

@@ -106,12 +106,7 @@ class BTCAmountEdit(AmountEdit):
 
 class FeerateEdit(BTCAmountEdit):
     def _base_unit(self):
-        p = self.decimal_point()
-        if p == 2:
-            return 'mBTC/kB'
-        if p == 0:
-            return 'sat/byte'
-        raise Exception('Unknown base unit')
+        return 'sat/byte'
 
     def get_amount(self):
         sat_per_byte_amount = BTCAmountEdit.get_amount(self)

gui/qt/fee_slider.py

@@ -1,6 +1,4 @@
-
 from electrum.i18n import _
-
 from PyQt5.QtGui import *
 from PyQt5.QtCore import *
 from PyQt5.QtWidgets import QSlider, QToolTip
@@ -22,37 +20,27 @@ class FeeSlider(QSlider):
 
     def moved(self, pos):
         with self.lock:
-            fee_rate = self.config.dynfee(pos) if self.dyn else self.config.static_fee(pos)
+            if self.dyn:
+                fee_rate = self.config.depth_to_fee(pos) if self.config.get('mempool_fees') else self.config.eta_to_fee(pos)
+            else:
+                fee_rate = self.config.static_fee(pos)
             tooltip = self.get_tooltip(pos, fee_rate)
             QToolTip.showText(QCursor.pos(), tooltip, self)
             self.setToolTip(tooltip)
             self.callback(self.dyn, pos, fee_rate)
 
     def get_tooltip(self, pos, fee_rate):
-        from electrum.util import fee_levels
-        rate_str = self.window.format_fee_rate(fee_rate) if fee_rate else _('unknown')
-        if self.dyn:
-            tooltip = fee_levels[pos] + '\n' + rate_str
-        else:
-            tooltip = 'Fixed rate: ' + rate_str
-            if self.config.has_fee_estimates():
-                i = self.config.reverse_dynfee(fee_rate)
-                tooltip += '\n' + (_('Low fee') if i < 0 else 'Within %d blocks'%i)
-        return tooltip
+        mempool = self.config.get('mempool_fees')
+        text, tooltip = self.config.get_fee_text(pos, self.dyn, mempool, fee_rate)
+        return text + '\n' + tooltip
 
     def update(self):
         with self.lock:
             self.dyn = self.config.is_dynfee()
-            if self.dyn:
-                pos = self.config.get('fee_level', 2)
-                fee_rate = self.config.dynfee(pos)
-                self.setRange(0, 4)
-                self.setValue(pos)
-            else:
-                fee_rate = self.config.fee_per_kb()
-                pos = self.config.static_fee_index(fee_rate)
-                self.setRange(0, 9)
-                self.setValue(pos)
+            mempool = self.config.get('mempool_fees')
+            maxp, pos, fee_rate = self.config.get_fee_slider(self.dyn, mempool)
+            self.setRange(0, maxp)
+            self.setValue(pos)
             tooltip = self.get_tooltip(pos, fee_rate)
             self.setToolTip(tooltip)
 

gui/qt/main_window.py

@@ -131,7 +131,6 @@ class ElectrumWindow(QMainWindow, MessageBoxMixin, PrintError):
         self.need_update = threading.Event()
 
         self.decimal_point = config.get('decimal_point', 5)
-        self.fee_unit = config.get('fee_unit', 0)
         self.num_zeros     = int(config.get('num_zeros',0))
 
         self.completions = QStringListModel()
@@ -293,7 +292,6 @@ class ElectrumWindow(QMainWindow, MessageBoxMixin, PrintError):
             self.need_update.set()
             self.gui_object.network_updated_signal_obj.network_updated_signal \
                 .emit(event, args)
-
         elif event == 'new_transaction':
             self.tx_notifications.append(args[0])
             self.notify_transactions_signal.emit()
@@ -315,6 +313,12 @@ class ElectrumWindow(QMainWindow, MessageBoxMixin, PrintError):
             if self.config.is_dynfee():
                 self.fee_slider.update()
                 self.do_update_fee()
+        elif event == 'fee_histogram':
+            if self.config.is_dynfee():
+                self.fee_slider.update()
+                self.do_update_fee()
+            # todo: update only unconfirmed tx
+            self.history_list.update()
         else:
             self.print_error("unexpected network_qt signal:", event, args)
 
@@ -636,10 +640,7 @@ class ElectrumWindow(QMainWindow, MessageBoxMixin, PrintError):
         return text
 
     def format_fee_rate(self, fee_rate):
-        if self.fee_unit == 0:
-            return format_satoshis(fee_rate/1000, False, self.num_zeros, 0, False)  + ' sat/byte'
-        else:
-            return self.format_amount(fee_rate) + ' ' + self.base_unit() + '/kB'
+        return format_satoshis(fee_rate/1000, False, self.num_zeros, 0, False)  + ' sat/byte'
 
     def get_decimal_point(self):
         return self.decimal_point
@@ -1076,7 +1077,10 @@ class ElectrumWindow(QMainWindow, MessageBoxMixin, PrintError):
 
         def fee_cb(dyn, pos, fee_rate):
             if dyn:
-                self.config.set_key('fee_level', pos, False)
+                if self.config.get('mempool_fees'):
+                    self.config.set_key('depth_level', pos, False)
+                else:
+                    self.config.set_key('fee_level', pos, False)
             else:
                 self.config.set_key('fee_per_kb', fee_rate, False)
 
@@ -1116,7 +1120,7 @@ class ElectrumWindow(QMainWindow, MessageBoxMixin, PrintError):
         self.size_e.setFixedWidth(140)
         self.size_e.setStyleSheet(ColorScheme.DEFAULT.as_stylesheet())
 
-        self.feerate_e = FeerateEdit(lambda: 2 if self.fee_unit else 0)
+        self.feerate_e = FeerateEdit(lambda: 0)
         self.feerate_e.setAmount(self.config.fee_per_byte())
         self.feerate_e.textEdited.connect(partial(on_fee_or_feerate, self.feerate_e, False))
         self.feerate_e.editingFinished.connect(partial(on_fee_or_feerate, self.feerate_e, True))
@@ -1256,9 +1260,6 @@ class ElectrumWindow(QMainWindow, MessageBoxMixin, PrintError):
         '''Recalculate the fee.  If the fee was manually input, retain it, but
         still build the TX to see if there are enough funds.
         '''
-        if not self.config.get('offline') and self.config.is_dynfee() and not self.config.has_fee_estimates():
-            self.statusBar().showMessage(_('Waiting for fee estimates...'))
-            return False
         freeze_fee = self.is_send_fee_frozen()
         freeze_feerate = self.is_send_feerate_frozen()
         amount = '!' if self.is_max else self.amount_e.get_amount()
@@ -2670,6 +2671,21 @@ class ElectrumWindow(QMainWindow, MessageBoxMixin, PrintError):
         nz.valueChanged.connect(on_nz)
         gui_widgets.append((nz_label, nz))
 
+        msg = '\n'.join([
+            _('Time based: fee rate is based on average confirmation time estimates'),
+            _('Mempool based: fee rate is targetting a depth in the memory pool')
+            ]
+        )
+        fee_type_label = HelpLabel(_('Fee estimation') + ':', msg)
+        fee_type_combo = QComboBox()
+        fee_type_combo.addItems([_('Time based'), _('Mempool based')])
+        fee_type_combo.setCurrentIndex(1 if self.config.get('mempool_fees') else 0)
+        def on_fee_type(x):
+            self.config.set_key('mempool_fees', x==1)
+            self.fee_slider.update()
+        fee_type_combo.currentIndexChanged.connect(on_fee_type)
+        fee_widgets.append((fee_type_label, fee_type_combo))
+
         def on_dynfee(x):
             self.config.set_key('dynamic_fees', x == Qt.Checked)
             self.fee_slider.update()
@@ -2699,18 +2715,6 @@ class ElectrumWindow(QMainWindow, MessageBoxMixin, PrintError):
         use_rbf_cb.stateChanged.connect(on_use_rbf)
         fee_widgets.append((use_rbf_cb, None))
 
-        self.fee_unit = self.config.get('fee_unit', 0)
-        fee_unit_label = HelpLabel(_('Fee Unit') + ':', '')
-        fee_unit_combo = QComboBox()
-        fee_unit_combo.addItems([_('sat/byte'), _('mBTC/kB')])
-        fee_unit_combo.setCurrentIndex(self.fee_unit)
-        def on_fee_unit(x):
-            self.fee_unit = x
-            self.config.set_key('fee_unit', x)
-            self.fee_slider.update()
-        fee_unit_combo.currentIndexChanged.connect(on_fee_unit)
-        fee_widgets.append((fee_unit_label, fee_unit_combo))
-
         msg = _('OpenAlias record, used to receive coins and to sign payment requests.') + '\n\n'\
               + _('The following alias providers are available:') + '\n'\
               + '\n'.join(['https://cryptoname.co/', 'http://xmr.link']) + '\n\n'\

gui/qt/transaction_dialog.py

@@ -221,8 +221,8 @@ class TxDialog(QDialog, MessageBoxMixin):
             self.date_label.setText(_("Date: {}").format(time_str))
             self.date_label.show()
         elif exp_n:
-            text = '%d blocks'%(exp_n) if exp_n > 0 else _('unknown (low fee)')
-            self.date_label.setText(_('Expected confirmation time') + ': ' + text)
+            text = '%.2f MB'%(exp_n/1000000)
+            self.date_label.setText(_('Position in mempool') + ': ' + text + _('from tip'))
             self.date_label.show()
         else:
             self.date_label.hide()

lib/bitcoin.py

@@ -102,7 +102,7 @@ NetworkConstants.set_mainnet()
 
 FEE_STEP = 10000
 MAX_FEE_RATE = 300000
-FEE_TARGETS = [25, 10, 5, 2]
+
 
 COINBASE_MATURITY = 100
 COIN = 100000000

lib/network.py

@@ -321,8 +321,10 @@ class Network(util.DaemonThread):
             self.queue_request('blockchain.scripthash.subscribe', [h])
 
     def request_fee_estimates(self):
+        from .simple_config import FEE_ETA_TARGETS
         self.config.requested_fee_estimates()
-        for i in bitcoin.FEE_TARGETS:
+        self.queue_request('mempool.get_fee_histogram', [])
+        for i in FEE_ETA_TARGETS:
             self.queue_request('blockchain.estimatefee', [i])
 
     def get_status_value(self, key):
@@ -332,6 +334,8 @@ class Network(util.DaemonThread):
             value = self.banner
         elif key == 'fee':
             value = self.config.fee_estimates
+        elif key == 'fee_histogram':
+            value = self.config.mempool_fees
         elif key == 'updated':
             value = (self.get_local_height(), self.get_server_height())
         elif key == 'servers':
@@ -543,6 +547,11 @@ class Network(util.DaemonThread):
         elif method == 'server.donation_address':
             if error is None:
                 self.donation_address = result
+        elif method == 'mempool.get_fee_histogram':
+            if error is None:
+                self.print_error(result)
+                self.config.mempool_fees = result
+                self.notify('fee_histogram')
         elif method == 'blockchain.estimatefee':
             if error is None and result > 0:
                 i = params[0]

lib/simple_config.py

@@ -6,9 +6,12 @@ import stat
 
 from copy import deepcopy
 from .util import (user_dir, print_error, PrintError,
-                   NoDynamicFeeEstimates)
+                   NoDynamicFeeEstimates, format_satoshis)
 
-from .bitcoin import MAX_FEE_RATE, FEE_TARGETS
+from .bitcoin import MAX_FEE_RATE
+
+FEE_ETA_TARGETS = [25, 10, 5, 2]
+FEE_DEPTH_TARGETS = [10000000, 5000000, 2000000, 1000000, 500000, 200000, 100000]
 
 config = None
 
@@ -48,6 +51,7 @@ class SimpleConfig(PrintError):
         # a thread-safe way.
         self.lock = threading.RLock()
 
+        self.mempool_fees = {}
         self.fee_estimates = {}
         self.fee_estimates_last_updated = {}
         self.last_time_fee_estimates_requested = 0  # zero ensures immediate fees
@@ -263,9 +267,9 @@ class SimpleConfig(PrintError):
             f = MAX_FEE_RATE
         return f
 
-    def dynfee(self, i):
+    def eta_to_fee(self, i):
         if i < 4:
-            j = FEE_TARGETS[i]
+            j = FEE_ETA_TARGETS[i]
             fee = self.fee_estimates.get(j)
         else:
             assert i == 4
@@ -276,15 +280,99 @@ class SimpleConfig(PrintError):
             fee = min(5*MAX_FEE_RATE, fee)
         return fee
 
-    def reverse_dynfee(self, fee_per_kb):
+    def fee_to_depth(self, target_fee):
+        depth = 0
+        for fee, s in self.mempool_fees:
+            depth += s
+            if fee < target_fee:
+                break
+        else:
+            return 0
+        return depth
+
+    def depth_to_fee(self, i):
+        target = self.depth_target(i)
+        depth = 0
+        for fee, s in self.mempool_fees:
+            depth += s
+            if depth > target:
+                break
+        else:
+            return 0
+        return fee * 1000
+
+    def depth_target(self, i):
+        return FEE_DEPTH_TARGETS[i]
+
+    def eta_target(self, i):
+        return FEE_ETA_TARGETS[i]
+
+    def fee_to_eta(self, fee_per_kb):
         import operator
-        l = list(self.fee_estimates.items()) + [(1, self.dynfee(4))]
+        l = list(self.fee_estimates.items()) + [(1, self.eta_to_fee(4))]
         dist = map(lambda x: (x[0], abs(x[1] - fee_per_kb)), l)
         min_target, min_value = min(dist, key=operator.itemgetter(1))
         if fee_per_kb < self.fee_estimates.get(25)/2:
             min_target = -1
         return min_target
 
+    def depth_tooltip(self, depth):
+        return "%.1f MB from tip"%(depth/1000000)
+
+    def eta_tooltip(self, x):
+        return 'Low fee' if x < 0 else 'Within %d blocks'%x
+
+    def get_fee_status(self):
+        dyn = self.is_dynfee()
+        mempool = self.get('mempool_fees')
+        pos = self.get('fee_level', 2) if mempool else self.get('depth_level', 2)
+        fee_rate = self.fee_per_kb()
+        target, tooltip = self.get_fee_text(pos, dyn, mempool, fee_rate)
+        return target
+
+    def get_fee_text(self, pos, dyn, mempool, fee_rate):
+        rate_str = (format_satoshis(fee_rate/1000, False, 0, 0, False)  + ' sat/byte') if fee_rate is not None else 'unknown'
+        if dyn:
+            if mempool:
+                depth = self.depth_target(pos)
+                text = self.depth_tooltip(depth)
+            else:
+                eta = self.eta_target(pos)
+                text = self.eta_tooltip(eta)
+            tooltip = rate_str
+        else:
+            text = rate_str
+            if mempool:
+                if self.has_fee_mempool():
+                    depth = self.fee_to_depth(fee_rate)
+                    tooltip = self.depth_tooltip(depth)
+                else:
+                    tooltip = ''
+            else:
+                if self.has_fee_etas():
+                    eta = self.fee_to_eta(fee_rate)
+                    tooltip = self.eta_tooltip(eta)
+                else:
+                    tooltip = ''
+        return text, tooltip
+
+    def get_fee_slider(self, dyn, mempool):
+        if dyn:
+            if mempool:
+                maxp = len(FEE_DEPTH_TARGETS) - 1
+                pos = min(maxp, self.get('depth_level', 2))
+                fee_rate = self.depth_to_fee(pos)
+            else:
+                maxp = len(FEE_ETA_TARGETS) - 1
+                pos = min(maxp, self.get('fee_level', 2))
+                fee_rate = self.eta_to_fee(pos)
+        else:
+            fee_rate = self.fee_per_kb()
+            pos = self.static_fee_index(fee_rate)
+            maxp= 9
+        return maxp, pos, fee_rate
+
+
     def static_fee(self, i):
         return self.fee_rates[i]
 
@@ -292,19 +380,27 @@ class SimpleConfig(PrintError):
         dist = list(map(lambda x: abs(x - value), self.fee_rates))
         return min(range(len(dist)), key=dist.__getitem__)
 
-    def has_fee_estimates(self):
-        return len(self.fee_estimates)==4
+    def has_fee_etas(self):
+        return len(self.fee_estimates) == 4
+
+    def has_fee_mempool(self):
+        return bool(self.mempool_fees)
 
     def is_dynfee(self):
         return self.get('dynamic_fees', True)
 
+    def use_mempool_fees(self):
+        return self.get('mempool_fees', False)
+
     def fee_per_kb(self):
         """Returns sat/kvB fee to pay for a txn.
         Note: might return None.
         """
-        dyn = self.is_dynfee()
-        if dyn:
-            fee_rate = self.dynfee(self.get('fee_level', 2))
+        if self.is_dynfee():
+            if self.use_mempool_fees():
+                fee_rate = self.depth_to_fee(self.get('depth_level', 2))
+            else:
+                fee_rate = self.eta_to_fee(self.get('fee_level', 2))
         else:
             fee_rate = self.get('fee_per_kb', self.max_fee_rate()/2)
         return fee_rate

lib/version.py

@@ -1,5 +1,5 @@
 ELECTRUM_VERSION = '3.0.6'   # version of the client package
-PROTOCOL_VERSION = '1.1'     # protocol version requested
+PROTOCOL_VERSION = '1.2'     # protocol version requested
 
 # The hash of the mnemonic seed must begin with this
 SEED_PREFIX      = '01'      # Standard wallet

lib/wallet.py

@@ -538,10 +538,10 @@ class Abstract_Wallet(PrintError):
                     status = _('Unconfirmed')
                     if fee is None:
                         fee = self.tx_fees.get(tx_hash)
-                    if fee and self.network.config.has_fee_estimates():
+                    if fee and self.network.config.has_fee_etas():
                         size = tx.estimated_size()
                         fee_per_kb = fee * 1000 / size
-                        exp_n = self.network.config.reverse_dynfee(fee_per_kb)
+                        exp_n = self.network.config.fee_to_eta(fee_per_kb)
                     can_bump = is_mine and not tx.is_final()
                 else:
                     status = _('Local')
@@ -860,18 +860,17 @@ class Abstract_Wallet(PrintError):
 
     def get_tx_status(self, tx_hash, height, conf, timestamp):
         from .util import format_time
+        exp_n = False
         if conf == 0:
             tx = self.transactions.get(tx_hash)
             if not tx:
                 return 3, 'unknown'
             is_final = tx and tx.is_final()
             fee = self.tx_fees.get(tx_hash)
-            if fee and self.network and self.network.config.has_fee_estimates():
-                size = len(tx.raw)/2
-                low_fee = int(self.network.config.dynfee(0)*size/1000)
-                is_lowfee = fee < low_fee * 0.5
-            else:
-                is_lowfee = False
+            if fee and self.network and self.network.config.has_fee_mempool():
+                size = tx.estimated_size()
+                fee_per_kb = fee * 1000 / size
+                exp_n = self.network.config.fee_to_depth(fee_per_kb//1000)
             if height == TX_HEIGHT_LOCAL:
                 status = 5
             elif height == TX_HEIGHT_UNCONF_PARENT:
@@ -888,6 +887,8 @@ class Abstract_Wallet(PrintError):
             status = 5 + min(conf, 6)
         time_str = format_time(timestamp) if timestamp else _("unknown")
         status_str = TX_STATUS[status] if status < 6 else time_str
+        if exp_n:
+            status_str += ' [%d sat/b, %.2f MB]'%(fee_per_kb//1000, exp_n/1000000)
         return status, status_str
 
     def relayfee(self):