Merge pull request #8713 from spesmilo/userspace_plugins

Allow users to downoad external plugins

contrib/make_plugin

@@ -0,0 +1,69 @@
+#!/usr/bin/python3
+import os
+import sys
+import hashlib
+import json
+import zipfile
+import zipimport
+
+# todo: use version number
+
+if len(sys.argv) != 2:
+    print(f"usage: {os.path.basename(__file__)} <plugin_directory>", file=sys.stderr)
+    sys.exit(1)
+
+source_dir = sys.argv[1]  # where the plugin source code is
+
+plugin_name = os.path.basename(source_dir)
+dest_dir = os.path.dirname(source_dir)
+zip_path = os.path.join(dest_dir, plugin_name + '.zip')
+
+# remove old zipfile
+if os.path.exists(zip_path):
+    os.unlink(zip_path)
+# create zipfile
+print('creating', zip_path)
+with zipfile.ZipFile(zip_path, 'w') as zip_object:
+    for folder_name, sub_folders, file_names in os.walk(source_dir):
+        for filename in file_names:
+            file_path = os.path.join(folder_name, filename)
+            dest_path = os.path.join(plugin_name, os.path.relpath(folder_name, source_dir), os.path.basename(file_path))
+            zip_object.write(file_path, dest_path)
+            print('added', dest_path)
+
+# read hash
+with open(zip_path, 'rb') as f:
+    s = f.read()
+_hash = bytes(hashlib.sha256(s).digest()).hex()
+
+# read metadata
+zip_file = zipimport.zipimporter(zip_path)
+module = zip_file.load_module(plugin_name)
+plugin_metadata = {
+    'hash': _hash,
+    'description': module.description,
+    'display_name': module.fullname,
+    'available_for': module.available_for,
+    'download_url': module.download_url,
+    'author': module.author,
+    'licence': module.licence,
+    'version': module.version,
+}
+print(json.dumps(plugin_metadata, indent=4))
+
+# update plugins.json file
+json_path = os.path.join(os.path.dirname(__file__), '..', 'electrum', 'plugins.json')
+with open(json_path, 'r') as f:
+    s = f.read()
+try:
+    metadata = json.loads(s)
+except:
+    metadata = {}
+metadata[plugin_name] = plugin_metadata
+with open(json_path, 'w') as f:
+    f.write(json.dumps(metadata, indent=4))
+
+# rename zip file
+if module.version:
+    zip_path_with_version = os.path.join(dest_dir, plugin_name + '-' + module.version + '.zip')
+    os.rename(zip_path, zip_path_with_version)

electrum/gui/qt/__init__.py

@@ -150,7 +150,7 @@ class ElectrumGui(BaseElectrumGui, Logger):
         self.reload_app_stylesheet()
 
         # always load 2fa
-        self.plugins.load_plugin('trustedcoin')
+        self.plugins.load_internal_plugin('trustedcoin')
 
         run_hook('init_qt', self)
 

electrum/gui/qt/plugins_dialog.py

@@ -56,22 +56,35 @@ class PluginsDialog(WindowModalDialog):
             self.grid.addWidget(widget, i, 1)
 
     def do_toggle(self, cb, name, i):
+        if self.plugins.requires_download(name):
+            cb.setChecked(False)
+            self.download_plugin_dialog(cb, name, i)
+            return
         p = self.plugins.toggle(name)
         cb.setChecked(bool(p))
         self.enable_settings_widget(p, name, i)
         # note: all enabled plugins will receive this hook:
         run_hook('init_qt', self.window.gui_object)
 
+    def download_plugin_dialog(self, cb, name, i):
+        import asyncio
+        if not self.window.question("Download plugin '%s'?"%name):
+            return
+        coro = self.plugins.download_external_plugin(name)
+        def on_success(x):
+            self.do_toggle(cb, name, i)
+        self.window.run_coroutine_dialog(coro, "Downloading '%s' "%name, on_result=on_success, on_cancelled=None)
+
     def show_list(self):
-        descriptions = self.plugins.descriptions.values()
+        descriptions = sorted(self.plugins.descriptions.items())
-        for i, descr in enumerate(descriptions):
+        i = 0
-            full_name = descr['__name__']
+        for name, descr in descriptions:
-            prefix, _separator, name = full_name.rpartition('.')
+            i += 1
             p = self.plugins.get(name)
             if descr.get('registers_keystore'):
                 continue
             try:
-                cb = QCheckBox(descr['fullname'])
+                cb = QCheckBox(descr['display_name'])
                 plugin_is_loaded = p is not None
                 cb_enabled = (not plugin_is_loaded and self.plugins.is_available(name, self.wallet)
                               or plugin_is_loaded and p.can_user_disable())

electrum/gui/qt/util.py

@@ -1237,6 +1237,10 @@ def icon_path(icon_basename: str):
 def read_QIcon(icon_basename: str) -> QIcon:
     return QIcon(icon_path(icon_basename))
 
+def read_QIcon_from_bytes(b: bytes) -> QIcon:
+    qp = QPixmap()
+    qp.loadFromData(b)
+    return QIcon(qp)
 
 class IconLabel(QWidget):
     HorizontalSpacing = 2

electrum/plugin.py

@@ -27,12 +27,18 @@ import pkgutil
 import importlib.util
 import time
 import threading
+import traceback
 import sys
+import json
 from typing import (NamedTuple, Any, Union, TYPE_CHECKING, Optional, Tuple,
                     Dict, Iterable, List, Sequence, Callable, TypeVar, Mapping, Set)
 import concurrent
+import zipimport
 from concurrent import futures
 from functools import wraps, partial
+from enum import IntEnum
+from packaging.version import parse as parse_version
+from electrum.version import ELECTRUM_VERSION
 
 from .i18n import _
 from .util import (profiler, DaemonThread, UserCancelled, ThreadJob, UserFacingException)
@@ -53,11 +59,11 @@ hook_names = set()
 hooks = {}
 
 
+
 class Plugins(DaemonThread):
 
     LOGGING_SHORTCUT = 'p'
     pkgpath = os.path.dirname(plugins.__file__)
-    _all_found_plugins = None  # type: Optional[Dict[str, dict]]
 
     @profiler
     def __init__(self, config: SimpleConfig, gui_name):
@@ -66,48 +72,49 @@ class Plugins(DaemonThread):
         self.config = config
         self.hw_wallets = {}
         self.plugins = {}  # type: Dict[str, BasePlugin]
+        self.internal_plugin_metadata = {}
+        self.external_plugin_metadata = {}
         self.gui_name = gui_name
-        self.descriptions = {}
         self.device_manager = DeviceMgr(config)
+        self.user_pkgpath = os.path.join(self.config.electrum_path_root(), 'plugins')
+        if not os.path.exists(self.user_pkgpath):
+            os.mkdir(self.user_pkgpath)
+        self.find_internal_plugins()
+        self.find_external_plugins()
         self.load_plugins()
         self.add_jobs(self.device_manager.thread_jobs())
         self.start()
 
-    @classmethod
+    @property
-    def find_all_plugins(cls) -> Mapping[str, dict]:
+    def descriptions(self):
-        """Return a map of all found plugins: name -> description.
+        return dict(list(self.internal_plugin_metadata.items()) + list(self.external_plugin_metadata.items()))
-        Note that plugins not available for the current GUI are also included.
-        """
-        if cls._all_found_plugins is None:
-            cls._all_found_plugins = dict()
-            iter_modules = list(pkgutil.iter_modules([cls.pkgpath]))
-            for loader, name, ispkg in iter_modules:
-                # FIXME pyinstaller binaries are packaging each built-in plugin twice:
-                #       once as data and once as code. To honor the "no duplicates" rule below,
-                #       we exclude the ones packaged as *code*, here:
-                if loader.__class__.__qualname__ == "FrozenImporter":
-                    continue
-                full_name = f'electrum.plugins.{name}'
-                spec = importlib.util.find_spec(full_name)
-                if spec is None:  # pkgutil found it but importlib can't ?!
-                    raise Exception(f"Error pre-loading {full_name}: no spec")
-                try:
-                    module = importlib.util.module_from_spec(spec)
-                    # sys.modules needs to be modified for relative imports to work
-                    # see https://stackoverflow.com/a/50395128
-                    sys.modules[spec.name] = module
-                    spec.loader.exec_module(module)
-                except Exception as e:
-                    raise Exception(f"Error pre-loading {full_name}: {repr(e)}") from e
-                d = module.__dict__
-                if name in cls._all_found_plugins:
-                    _logger.info(f"Found the following plugin modules: {iter_modules=}")
-                    raise Exception(f"duplicate plugins? for {name=}")
-                cls._all_found_plugins[name] = d
-        return cls._all_found_plugins
 
-    def load_plugins(self):
+    def find_internal_plugins(self) -> Mapping[str, dict]:
-        for name, d in self.find_all_plugins().items():
+        """Populates self.internal_plugin_metadata
+        """
+        iter_modules = list(pkgutil.iter_modules([self.pkgpath]))
+        for loader, name, ispkg in iter_modules:
+            # FIXME pyinstaller binaries are packaging each built-in plugin twice:
+            #       once as data and once as code. To honor the "no duplicates" rule below,
+            #       we exclude the ones packaged as *code*, here:
+            if loader.__class__.__qualname__ == "FrozenImporter":
+                continue
+            full_name = f'electrum.plugins.{name}'
+            spec = importlib.util.find_spec(full_name)
+            if spec is None:  # pkgutil found it but importlib can't ?!
+                raise Exception(f"Error pre-loading {full_name}: no spec")
+            try:
+                module = importlib.util.module_from_spec(spec)
+                # sys.modules needs to be modified for relative imports to work
+                # see https://stackoverflow.com/a/50395128
+                sys.modules[spec.name] = module
+                spec.loader.exec_module(module)
+            except Exception as e:
+                raise Exception(f"Error pre-loading {full_name}: {repr(e)}") from e
+            d = module.__dict__
+            if 'fullname' not in d:
+                continue
+            d['display_name'] = d['fullname']
             gui_good = self.gui_name in d.get('available_for', [])
             if not gui_good:
                 continue
@@ -117,13 +124,133 @@ class Plugins(DaemonThread):
             details = d.get('registers_keystore')
             if details:
                 self.register_keystore(name, gui_good, details)
-            self.descriptions[name] = d
+            if d.get('requires_wallet_type'):
-            if not d.get('requires_wallet_type') and self.config.get('use_' + name):
+                # trustedcoin will not be added to list
+                continue
+            if name in self.internal_plugin_metadata:
+                _logger.info(f"Found the following plugin modules: {iter_modules=}")
+                raise Exception(f"duplicate plugins? for {name=}")
+            self.internal_plugin_metadata[name] = d
+
+    def load_plugins(self):
+        self.load_internal_plugins()
+        self.load_external_plugins()
+
+    def load_internal_plugins(self):
+        for name, d in self.internal_plugin_metadata.items():
+            if not d.get('requires_wallet_type') and self.config.get('enable_plugin_' + name):
                 try:
-                    self.load_plugin(name)
+                    self.load_internal_plugin(name)
                 except BaseException as e:
                     self.logger.exception(f"cannot initialize plugin {name}: {e}")
 
+    def requires_download(self, name):
+        metadata = self.external_plugin_metadata.get(name)
+        if not metadata:
+            return False
+        if os.path.exists(self.external_plugin_path(name)):
+            return False
+        return True
+
+    def check_plugin_hash(self, name: str)-> bool:
+        from .crypto import sha256
+        metadata = self.external_plugin_metadata.get(name)
+        filename = self.external_plugin_path(name)
+        if not os.path.exists(filename):
+            return False
+        with open(filename, 'rb') as f:
+            s = f.read()
+        if sha256(s).hex() != metadata['hash']:
+            return False
+        return True
+
+    async def download_external_plugin(self, name):
+        import aiohttp
+        metadata = self.external_plugin_metadata.get(name)
+        if metadata is None:
+            raise Exception("unknown external plugin %s" % name)
+        url = metadata['download_url']
+        filename = self.external_plugin_path(name)
+        async with aiohttp.ClientSession() as session:
+            async with session.get(url) as resp:
+                if resp.status == 200:
+                    with open(filename, 'wb') as fd:
+                        async for chunk in resp.content.iter_chunked(10):
+                            fd.write(chunk)
+        if not self.check_plugin_hash(name):
+            os.unlink(filename)
+            raise Exception("wrong plugin hash %s" % name)
+
+    def load_external_plugin(self, name):
+        if name in self.plugins:
+            return self.plugins[name]
+        # If we do not have the metadata, it was not detected by `load_external_plugins`
+        # on startup, or added by manual user installation after that point.
+        metadata = self.external_plugin_metadata.get(name)
+        if metadata is None:
+            self.logger.exception("attempted to load unknown external plugin %s" % name)
+            return
+        filename = self.external_plugin_path(name)
+        if not os.path.exists(filename):
+            return
+        if not self.check_plugin_hash(name):
+            self.logger.exception("wrong hash for plugin '%s'" % name)
+            os.unlink(filename)
+            return
+        try:
+            zipfile = zipimport.zipimporter(filename)
+        except zipimport.ZipImportError:
+            self.logger.exception("unable to load zip plugin '%s'" % filename)
+            return
+        try:
+            module = zipfile.load_module(name)
+        except zipimport.ZipImportError as e:
+            self.logger.exception(f"unable to load zip plugin '{filename}' package '{name}'")
+            return
+        sys.modules['electrum_external_plugins.'+ name] = module
+        full_name = f'electrum_external_plugins.{name}.{self.gui_name}'
+        spec = importlib.util.find_spec(full_name)
+        if spec is None:
+            raise RuntimeError("%s implementation for %s plugin not found"
+                               % (self.gui_name, name))
+        module = importlib.util.module_from_spec(spec)
+        self._register_module(spec, module)
+        if sys.version_info >= (3, 10):
+            spec.loader.exec_module(module)
+        else:
+            module = spec.loader.load_module(full_name)
+        plugin = module.Plugin(self, self.config, name)
+        self.add_jobs(plugin.thread_jobs())
+        self.plugins[name] = plugin
+        self.logger.info(f"loaded external plugin {name}")
+        return plugin
+
+    @staticmethod
+    def _register_module(spec, module):
+        # sys.modules needs to be modified for relative imports to work
+        # see https://stackoverflow.com/a/50395128
+        sys.modules[spec.name] = module
+
+    def get_external_plugin_dir(self):
+        return self.user_pkgpath
+
+    def external_plugin_path(self, name):
+        return os.path.join(self.get_external_plugin_dir(), name + '.zip')
+
+    def find_external_plugins(self):
+        """ read json file """
+        from .constants import read_json
+        self.external_plugin_metadata = read_json('plugins.json', {})
+
+    def load_external_plugins(self):
+        for name, d in self.external_plugin_metadata.items():
+            if not d.get('requires_wallet_type') and self.config.get('use_' + name):
+                try:
+                    self.load_external_plugin(name)
+                except BaseException as e:
+                    traceback.print_exc(file=sys.stdout) # shouldn't this be... suppressed unless -v?
+                    self.logger.exception(f"cannot initialize plugin {name} {e!r}")
+
     def get(self, name):
         return self.plugins.get(name)
 
@@ -134,9 +261,17 @@ class Plugins(DaemonThread):
         """Imports the code of the given plugin.
         note: can be called from any thread.
         """
+        if name in self.internal_plugin_metadata:
+            return self.load_internal_plugin(name)
+        elif name in self.external_plugin_metadata:
+            return self.load_external_plugin(name)
+        else:
+            raise Exception()
+
+    def load_internal_plugin(self, name) -> 'BasePlugin':
         if name in self.plugins:
             return self.plugins[name]
-        full_name = f'electrum.plugins.{name}.{self.gui_name}'
+        full_name = f'electrum.plugins.{name}' + f'.{self.gui_name}'
         spec = importlib.util.find_spec(full_name)
         if spec is None:
             raise RuntimeError("%s implementation for %s plugin not found"
@@ -156,14 +291,14 @@ class Plugins(DaemonThread):
         self.remove_jobs(plugin.thread_jobs())
 
     def enable(self, name: str) -> 'BasePlugin':
-        self.config.set_key('use_' + name, True, save=True)
+        self.config.set_key('enable_plugin_' + name, True, save=True)
         p = self.get(name)
         if p:
             return p
         return self.load_plugin(name)
 
     def disable(self, name: str) -> None:
-        self.config.set_key('use_' + name, False, save=True)
+        self.config.set_key('enable_plugin_' + name, False, save=True)
         p = self.get(name)
         if not p:
             return
@@ -173,12 +308,7 @@ class Plugins(DaemonThread):
 
     @classmethod
     def is_plugin_enabler_config_key(cls, key: str) -> bool:
-        if not key.startswith('use_'):
+        return key.startswith('enable_plugin_')
-            return False
-        # note: the 'use_' prefix is not sufficient to check, there are
-        #       non-plugin-related config keys that also have it... hence:
-        name = key[4:]
-        return name in cls.find_all_plugins()
 
     def toggle(self, name: str) -> Optional['BasePlugin']:
         p = self.get(name)
@@ -204,7 +334,7 @@ class Plugins(DaemonThread):
             if gui_good:
                 try:
                     p = self.get_plugin(name)
-                    if p.is_enabled():
+                    if p.is_available():
                         out.append(HardwarePluginToScan(name=name,
                                                         description=details[2],
                                                         plugin=p,
@@ -276,7 +406,7 @@ class BasePlugin(Logger):
         self.parent = parent  # type: Plugins  # The plugins object
         self.name = name
         self.config = config
-        self.wallet = None
+        self.wallet = None # fixme: this field should not exist
         Logger.__init__(self)
         # add self to hooks
         for k in dir(self):
@@ -313,7 +443,7 @@ class BasePlugin(Logger):
         return []
 
     def is_enabled(self):
-        return self.is_available() and self.config.get('use_'+self.name) is True
+        return self.is_available() and self.config.get('enable_plugin_'+self.name) is True
 
     def is_available(self):
         return True
@@ -327,6 +457,14 @@ class BasePlugin(Logger):
     def settings_dialog(self, window):
         raise NotImplementedError()
 
+    def read_file(self, filename: str) -> bytes:
+        """ note: only for external plugins """
+        import zipfile
+        plugin_filename = self.parent.external_plugin_path(self.name)
+        with zipfile.ZipFile(plugin_filename) as myzip:
+            with myzip.open(os.path.join(self.name, filename)) as myfile:
+                s = myfile.read()
+                return s
 
 class DeviceUnpairableError(UserFacingException): pass
 class HardwarePluginLibraryUnavailable(Exception): pass

electrum/plugins.json

@@ -0,0 +1,14 @@
+{
+    "virtualkeyboard": {
+        "hash": "4ab551ec5226e7bb26e21991b33c75387e8de66bd9c5bc402e940e6f8a17050b",
+        "description": "Add an optional virtual keyboard to the password dialog.\nWarning: do not use this if it makes you pick a weaker password.",
+        "display_name": "Virtual Keyboard",
+        "available_for": [
+            "qt"
+        ],
+        "download_url": "https://raw.githubusercontent.com/spesmilo/electrum-plugins/master/virtualkeyboard-0.0.1.zip",
+        "author": "The Electrum developers",
+        "licence": "MIT",
+        "version": "0.0.1"
+    }
+}

electrum/plugins/swapserver/__init__.py

@@ -6,7 +6,7 @@ Submarine swap server for an Electrum daemon.
 
 Example setup:
 
-  electrum -o setconfig use_swapserver True
+  electrum -o setconfig enable_plugin_swapserver True
   electrum -o setconfig swapserver_port 5455
   electrum daemon -v
 

electrum/plugins/virtualkeyboard/__init__.py

@@ -1,5 +0,0 @@
-from electrum.i18n import _
-
-fullname = 'Virtual Keyboard'
-description = '%s\n%s' % (_("Add an optional virtual keyboard to the password dialog."), _("Warning: do not use this if it makes you pick a weaker password."))
-available_for = ['qt']

electrum/plugins/virtualkeyboard/qt.py

@@ -1,67 +0,0 @@
-import random
-
-from PyQt5.QtWidgets import (QVBoxLayout, QGridLayout, QPushButton)
-from PyQt5.QtGui import QFontMetrics
-
-from electrum.plugin import BasePlugin, hook
-from electrum.i18n import _
-
-
-class Plugin(BasePlugin):
-    vkb = None
-    vkb_index = 0
-
-    @hook
-    def password_dialog(self, pw, grid, pos):
-        vkb_button = QPushButton("+")
-        font_height = QFontMetrics(vkb_button.font()).height()
-        vkb_button.setFixedWidth(round(1.7 * font_height))
-        vkb_button.clicked.connect(lambda: self.toggle_vkb(grid, pw))
-        grid.addWidget(vkb_button, pos, 2)
-        self.kb_pos = 2
-        self.vkb = None
-
-    def toggle_vkb(self, grid, pw):
-        if self.vkb:
-            grid.removeItem(self.vkb)
-        self.vkb = self.virtual_keyboard(self.vkb_index, pw)
-        grid.addLayout(self.vkb, self.kb_pos, 0, 1, 3)
-        self.vkb_index += 1
-
-    def virtual_keyboard(self, i, pw):
-        i = i % 3
-        if i == 0:
-            chars = 'abcdefghijklmnopqrstuvwxyz '
-        elif i == 1:
-            chars = 'ABCDEFGHIJKLMNOPQRTSUVWXYZ '
-        elif i == 2:
-            chars = '1234567890!?.,;:/%&()[]{}+-'
-
-        n = len(chars)
-        s = []
-        for i in range(n):
-            while True:
-                k = random.randint(0, n - 1)
-                if k not in s:
-                    s.append(k)
-                    break
-
-        def add_target(t):
-            return lambda: pw.setText(str(pw.text()) + t)
-
-        font_height = QFontMetrics(QPushButton().font()).height()
-        btn_size = max(25, round(1.7 * font_height))
-
-        vbox = QVBoxLayout()
-        grid = QGridLayout()
-        grid.setSpacing(2)
-        for i in range(n):
-            l_button = QPushButton(chars[s[i]])
-            l_button.setFixedWidth(btn_size)
-            l_button.setFixedHeight(btn_size)
-            l_button.clicked.connect(add_target(chars[s[i]]))
-            grid.addWidget(l_button, i // 6, i % 6)
-
-        vbox.addLayout(grid)
-
-        return vbox

electrum/simple_config.py

@@ -242,14 +242,15 @@ class SimpleConfig(Logger):
         self.amt_precision_post_satoshi = self.BTC_AMOUNTS_PREC_POST_SAT
         self.amt_add_thousands_sep = self.BTC_AMOUNTS_ADD_THOUSANDS_SEP
 
-    def electrum_path(self):
+    def electrum_path_root(self):
         # Read electrum_path from command line
         # Otherwise use the user's default data directory.
-        path = self.get('electrum_path')
+        path = self.get('electrum_path') or self.user_dir()
-        if path is None:
-            path = self.user_dir()
-
         make_dir(path, allow_symlink=False)
+        return path
+
+    def electrum_path(self):
+        path = self.electrum_path_root()
         if self.get('testnet'):
             path = os.path.join(path, 'testnet')
             make_dir(path, allow_symlink=False)

tests/regtest.py

@@ -82,7 +82,7 @@ class TestLightningSwapserver(TestLightning):
         },
         'bob': {
             'lightning_listen': 'localhost:9735',
-            'use_swapserver': 'true',
+            'enable_plugin_swapserver': 'true',
         }
     }