__init__.py 10.3 KB
Newer Older
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
1 2
from future import standard_library
standard_library.install_aliases()
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
3
from future.utils import with_metaclass
4

J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
5
import os.path
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
6
import os
7
import pickle
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
8
import logging
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
9
import copy
10 11 12 13 14 15 16

import fnmatch
import inspect
import sys
import subprocess
import importlib
import yaml
17
import threading
18

J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
19
from .. import models, utils
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
20
from .. import api
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
21 22 23

logger = logging.getLogger(__name__)

24

J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
25
class PluginMeta(models.BaseMeta):
26
    _classes = {}
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
27 28 29 30 31 32 33

    def __new__(mcs, name, bases, attrs, **kwargs):
        plugin_type = []
        if hasattr(bases[0], 'plugin_type'):
            plugin_type += bases[0].plugin_type
        plugin_type.append(name)
        attrs['plugin_type'] = plugin_type
34 35 36 37 38 39 40 41 42 43
        cls = super(PluginMeta, mcs).__new__(mcs, name, bases, attrs)
        if name in mcs._classes:
            raise Exception(('The type of plugin {} already exists. '
                             'Please, choose a different name').format(name))
        mcs._classes[name] = cls
        return cls

    @classmethod
    def for_type(cls, ptype):
        return cls._classes[ptype]
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
44 45 46 47 48


class Plugin(with_metaclass(PluginMeta, models.Plugin)):

    def __init__(self, info=None, data_folder=None, **kwargs):
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
49 50 51 52
        """
        Provides a canonical name for plugins and serves as base for other
        kinds of plugins.
        """
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
53
        logger.debug("Initialising {}".format(info))
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
54
        super(Plugin, self).__init__(**kwargs)
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
55 56 57
        if info:
            self.update(info)
        if not self.validate():
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
58 59
            raise models.Error(message=("You need to provide configuration"
                                        "information for the plugin."))
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
60
        self.id = 'plugins/{}_{}'.format(self['name'], self['version'])
61
        self.is_activated = False
62
        self._lock = threading.Lock()
63
        self.data_folder = data_folder or os.getcwd()
64

J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
65 66 67
    def validate(self):
        return all(x in self for x in ('name', 'description', 'version'))

J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
68 69 70
    def get_folder(self):
        return os.path.dirname(inspect.getfile(self.__class__))

71 72 73 74 75 76
    def activate(self):
        pass

    def deactivate(self):
        pass

J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
77
    def test(self):
78 79 80
        if not hasattr(self, 'test_cases'):
            raise AttributeError(('Plugin {} [{}] does not have any defined '
                                  'test cases').format(self.id, inspect.getfile(self.__class__)))
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
81
        for case in self.test_cases:
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
82
            entry = models.Entry(case['entry'])
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
83 84
            given_parameters = case.get('params', {})
            params = api.parse_params(given_parameters, self.extra_params)
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
85 86 87 88 89 90 91 92 93 94
            fails = case.get('fails', False)
            try:
                res = list(self.analyse_entry(entry, params))
            except models.Error:
                if fails:
                    continue
                raise
            if fails:
                raise Exception('This test should have raised an exception.')

J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
95 96 97
            exp = case['expected']
            if not isinstance(exp, list):
                exp = [exp]
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
98
            utils.check_template(res, exp)
99 100
            for r in res:
                r.validate()
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
101

102 103 104
    def open(self, fpath, *args, **kwargs):
        if not os.path.isabs(fpath):
            fpath = os.path.join(self.data_folder, fpath)
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
105 106 107 108
        return open(fpath, *args, **kwargs)

    def serve(self, **kwargs):
        utils.serve(plugin=self, **kwargs)
109

110

J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
111 112 113 114
SenpyPlugin = Plugin


class AnalysisPlugin(Plugin):
115

116
    def analyse(self, *args, **kwargs):
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
117
        raise NotImplementedError(
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
118 119 120 121 122 123 124 125 126
            'Your method should implement either analyse or analyse_entry')

    def analyse_entry(self, entry, parameters):
        """ An implemented plugin should override this method.
        This base method is here to adapt old style plugins which only
        implement the *analyse* function.
        Note that this method may yield an annotated entry or a list of
        entries (e.g. in a tokenizer)
        """
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
127
        text = entry['nif:isString']
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
128 129 130 131 132
        params = copy.copy(parameters)
        params['input'] = text
        results = self.analyse(**params)
        for i in results.entries:
            yield i
133

J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
134 135 136 137 138 139
    def analyse_entries(self, entries, parameters):
        for entry in entries:
            logger.debug('Analysing entry with plugin {}: {}'.format(self, entry))
            for result in self.analyse_entry(entry, parameters):
                yield result

140

J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
141
class ConversionPlugin(Plugin):
142
    pass
143

144

J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
145
class SentimentPlugin(AnalysisPlugin, models.SentimentPlugin):
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
146 147
    minPolarityValue = 0
    maxPolarityValue = 1
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
148

149

J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
150
class EmotionPlugin(AnalysisPlugin, models.EmotionPlugin):
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
151 152
    minEmotionValue = 0
    maxEmotionValue = 1
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
153 154


J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
155
class EmotionConversionPlugin(ConversionPlugin):
156
    pass
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
157 158 159 160 161


class ShelfMixin(object):
    @property
    def sh(self):
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
162
        if not hasattr(self, '_sh') or self._sh is None:
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
163
            self._sh = {}
164
            if os.path.isfile(self.shelf_file):
165
                try:
166
                    with self.open(self.shelf_file, 'rb') as p:
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
167
                        self._sh = pickle.load(p)
168
                except (IndexError, EOFError, pickle.UnpicklingError):
169 170 171
                    logger.warning('{} has a corrupted shelf file!'.format(self.id))
                    if not self.get('force_shelf', False):
                        raise
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
172 173 174 175
        return self._sh

    @sh.deleter
    def sh(self):
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
176 177
        if os.path.isfile(self.shelf_file):
            os.remove(self.shelf_file)
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
178
            del self._sh
179
        self.save()
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
180

J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
181 182 183 184
    @sh.setter
    def sh(self, value):
        self._sh = value

J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
185 186
    @property
    def shelf_file(self):
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
187
        if 'shelf_file' not in self or not self['shelf_file']:
188
            self.shelf_file = os.path.join(self.data_folder, self.name + '.p')
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
189
        return self['shelf_file']
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
190

191
    def save(self):
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
192
        logger.debug('saving pickle')
193
        if hasattr(self, '_sh') and self._sh is not None:
194
            with self.open(self.shelf_file, 'wb') as f:
195
                pickle.dump(self._sh, f)
196 197 198 199 200 201 202 203


def pfilter(plugins, **kwargs):
    """ Filter plugins by different criteria """
    if isinstance(plugins, models.Plugins):
        plugins = plugins.plugins
    elif isinstance(plugins, dict):
        plugins = plugins.values()
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
204
    ptype = kwargs.pop('plugin_type', AnalysisPlugin)
205 206 207
    logger.debug('#' * 100)
    logger.debug('ptype {}'.format(ptype))
    if ptype:
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
208 209
        if isinstance(ptype, PluginMeta):
            ptype = ptype.__name__
210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231
        try:
            ptype = ptype[0].upper() + ptype[1:]
            pclass = globals()[ptype]
            logger.debug('Class: {}'.format(pclass))
            candidates = filter(lambda x: isinstance(x, pclass),
                                plugins)
        except KeyError:
            raise models.Error('{} is not a valid type'.format(ptype))
    else:
        candidates = plugins

    logger.debug(candidates)

    def matches(plug):
        res = all(getattr(plug, k, None) == v for (k, v) in kwargs.items())
        logger.debug(
            "matching {} with {}: {}".format(plug.name, kwargs, res))
        return res

    if kwargs:
        candidates = filter(matches, candidates)
    return {p.name: p for p in candidates}
232 233 234


def validate_info(info):
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
235
    return all(x in info for x in ('name',))
236 237


238 239 240
def load_module(name, root=None):
    if root:
        sys.path.append(root)
241
    tmp = importlib.import_module(name)
242 243
    if root:
        sys.path.remove(root)
244 245 246
    return tmp


247
def _log_subprocess_output(process):
248 249 250 251 252 253 254
    for line in iter(process.stdout.readline, b''):
        logger.info('%r', line)
    for line in iter(process.stderr.readline, b''):
        logger.error('%r', line)


def install_deps(*plugins):
255
    installed = False
256 257 258
    for info in plugins:
        requirements = info.get('requirements', [])
        if requirements:
259
            pip_args = [sys.executable, '-m', 'pip', 'install', '--use-wheel']
260 261 262 263 264 265
            for req in requirements:
                pip_args.append(req)
            logger.info('Installing requirements: ' + str(requirements))
            process = subprocess.Popen(pip_args,
                                       stdout=subprocess.PIPE,
                                       stderr=subprocess.PIPE)
266
            _log_subprocess_output(process)
267
            exitcode = process.wait()
268
            installed = True
269 270
            if exitcode != 0:
                raise models.Error("Dependencies not properly installed")
271
    return installed
272 273


J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
274 275 276 277 278 279 280 281 282 283 284
def get_plugin_class(module):
    candidate = None
    for _, obj in inspect.getmembers(module):
        if inspect.isclass(obj) and inspect.getmodule(obj) == module:
            logger.debug(("Found plugin class:"
                          " {}@{}").format(obj, inspect.getmodule(obj)))
            candidate = obj
            break
    return candidate


285
def load_plugin_from_info(info, root=None, validator=validate_info, install=True, *args, **kwargs):
286 287
    if not root and '_path' in info:
        root = os.path.dirname(info['_path'])
288
    if not validator(info):
289
        raise ValueError('Plugin info is not valid: {}'.format(info))
290 291
    module = info["module"]

292 293 294 295 296 297 298
    try:
        tmp = load_module(module, root)
    except ImportError:
        if not install:
            raise
        install_deps(info)
        tmp = load_module(module, root)
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
299 300 301
    cls = None
    if '@type' not in info:
        cls = get_plugin_class(tmp)
302 303
    else:
        cls = PluginMeta.from_type(info['@type'])
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
304 305 306
    if not cls:
        raise Exception("No valid plugin for: {}".format(module))
    return cls(info=info, *args, **kwargs)
307 308


309
def parse_plugin_info(fpath):
310 311 312
    logger.debug("Loading plugin: {}".format(fpath))
    with open(fpath, 'r') as f:
        info = yaml.load(f)
313 314 315 316 317
    info['_path'] = fpath
    name = info['name']
    return name, info


318
def load_plugin(fpath, *args, **kwargs):
319
    name, info = parse_plugin_info(fpath)
320
    logger.debug("Info: {}".format(info))
321
    plugin = load_plugin_from_info(info, *args, **kwargs)
322
    return name, plugin
323 324


325
def load_plugins(folders, loader=load_plugin, *args, **kwargs):
326 327 328
    plugins = {}
    for search_folder in folders:
        for root, dirnames, filenames in os.walk(search_folder):
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
329 330
            # Do not look for plugins in hidden or special folders
            dirnames[:] = [d for d in dirnames if d[0] not in ['.', '_']]
331
            for filename in fnmatch.filter(filenames, '*.senpy'):
332
                fpath = os.path.join(root, filename)
333
                name, plugin = loader(fpath, *args, **kwargs)
334 335 336
                if plugin and name:
                    plugins[name] = plugin
    return plugins