__init__.py 9.9 KB
Newer Older
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
1 2
from future import standard_library
standard_library.install_aliases()
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
3
from future.utils import with_metaclass
4

J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
5
import os.path
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
6
import os
7
import pickle
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
8
import logging
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
9
import copy
10 11 12 13 14 15 16

import fnmatch
import inspect
import sys
import subprocess
import importlib
import yaml
17
import threading
18

J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
19
from .. import models, utils
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
20
from .. import api
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
21 22 23

logger = logging.getLogger(__name__)

24

J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
25 26 27 28 29 30 31 32 33 34 35 36 37 38
class PluginMeta(models.BaseMeta):

    def __new__(mcs, name, bases, attrs, **kwargs):
        plugin_type = []
        if hasattr(bases[0], 'plugin_type'):
            plugin_type += bases[0].plugin_type
        plugin_type.append(name)
        attrs['plugin_type'] = plugin_type
        return super(PluginMeta, mcs).__new__(mcs, name, bases, attrs)


class Plugin(with_metaclass(PluginMeta, models.Plugin)):

    def __init__(self, info=None, data_folder=None, **kwargs):
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
39 40 41 42
        """
        Provides a canonical name for plugins and serves as base for other
        kinds of plugins.
        """
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
43
        logger.debug("Initialising {}".format(info))
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
44
        super(Plugin, self).__init__(**kwargs)
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
45 46 47
        if info:
            self.update(info)
        if not self.validate():
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
48 49
            raise models.Error(message=("You need to provide configuration"
                                        "information for the plugin."))
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
50
        self.id = 'plugins/{}_{}'.format(self['name'], self['version'])
51
        self.is_activated = False
52
        self._lock = threading.Lock()
53
        self.data_folder = data_folder or os.getcwd()
54

J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
55 56 57
    def validate(self):
        return all(x in self for x in ('name', 'description', 'version'))

J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
58 59 60
    def get_folder(self):
        return os.path.dirname(inspect.getfile(self.__class__))

61 62 63 64 65 66
    def activate(self):
        pass

    def deactivate(self):
        pass

J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
67
    def test(self):
68 69 70
        if not hasattr(self, 'test_cases'):
            raise AttributeError(('Plugin {} [{}] does not have any defined '
                                  'test cases').format(self.id, inspect.getfile(self.__class__)))
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
71
        for case in self.test_cases:
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
72
            entry = models.Entry(case['entry'])
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
73 74
            given_parameters = case.get('params', {})
            params = api.parse_params(given_parameters, self.extra_params)
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
75 76 77 78 79 80 81 82 83 84
            fails = case.get('fails', False)
            try:
                res = list(self.analyse_entry(entry, params))
            except models.Error:
                if fails:
                    continue
                raise
            if fails:
                raise Exception('This test should have raised an exception.')

J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
85 86 87
            exp = case['expected']
            if not isinstance(exp, list):
                exp = [exp]
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
88
            utils.check_template(res, exp)
89 90
            for r in res:
                r.validate()
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
91

92 93 94
    def open(self, fpath, *args, **kwargs):
        if not os.path.isabs(fpath):
            fpath = os.path.join(self.data_folder, fpath)
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
95 96 97 98
        return open(fpath, *args, **kwargs)

    def serve(self, **kwargs):
        utils.serve(plugin=self, **kwargs)
99

100

J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
101 102 103 104
SenpyPlugin = Plugin


class AnalysisPlugin(Plugin):
105

106
    def analyse(self, *args, **kwargs):
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
107
        raise NotImplementedError(
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
108 109 110 111 112 113 114 115 116
            'Your method should implement either analyse or analyse_entry')

    def analyse_entry(self, entry, parameters):
        """ An implemented plugin should override this method.
        This base method is here to adapt old style plugins which only
        implement the *analyse* function.
        Note that this method may yield an annotated entry or a list of
        entries (e.g. in a tokenizer)
        """
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
117
        text = entry['nif:isString']
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
118 119 120 121 122
        params = copy.copy(parameters)
        params['input'] = text
        results = self.analyse(**params)
        for i in results.entries:
            yield i
123

J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
124 125 126 127 128 129
    def analyse_entries(self, entries, parameters):
        for entry in entries:
            logger.debug('Analysing entry with plugin {}: {}'.format(self, entry))
            for result in self.analyse_entry(entry, parameters):
                yield result

130

J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
131
class ConversionPlugin(Plugin):
132
    pass
133

134

J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
135
class SentimentPlugin(AnalysisPlugin, models.SentimentPlugin):
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
136 137
    minPolarityValue = 0
    maxPolarityValue = 1
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
138

139

J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
140
class EmotionPlugin(AnalysisPlugin, models.EmotionPlugin):
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
141 142
    minEmotionValue = 0
    maxEmotionValue = 1
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
143 144


J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
145
class EmotionConversionPlugin(ConversionPlugin):
146
    pass
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
147 148 149 150 151


class ShelfMixin(object):
    @property
    def sh(self):
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
152
        if not hasattr(self, '_sh') or self._sh is None:
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
153
            self._sh = {}
154
            if os.path.isfile(self.shelf_file):
155
                try:
156
                    with self.open(self.shelf_file, 'rb') as p:
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
157
                        self._sh = pickle.load(p)
158
                except (IndexError, EOFError, pickle.UnpicklingError):
159 160 161
                    logger.warning('{} has a corrupted shelf file!'.format(self.id))
                    if not self.get('force_shelf', False):
                        raise
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
162 163 164 165
        return self._sh

    @sh.deleter
    def sh(self):
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
166 167
        if os.path.isfile(self.shelf_file):
            os.remove(self.shelf_file)
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
168
            del self._sh
169
        self.save()
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
170

J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
171 172 173 174
    @sh.setter
    def sh(self, value):
        self._sh = value

J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
175 176
    @property
    def shelf_file(self):
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
177
        if 'shelf_file' not in self or not self['shelf_file']:
178
            self.shelf_file = os.path.join(self.data_folder, self.name + '.p')
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
179
        return self['shelf_file']
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
180

181
    def save(self):
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
182
        logger.debug('saving pickle')
183
        if hasattr(self, '_sh') and self._sh is not None:
184
            with self.open(self.shelf_file, 'wb') as f:
185
                pickle.dump(self._sh, f)
186 187 188 189 190 191 192 193


def pfilter(plugins, **kwargs):
    """ Filter plugins by different criteria """
    if isinstance(plugins, models.Plugins):
        plugins = plugins.plugins
    elif isinstance(plugins, dict):
        plugins = plugins.values()
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
194
    ptype = kwargs.pop('plugin_type', AnalysisPlugin)
195 196 197
    logger.debug('#' * 100)
    logger.debug('ptype {}'.format(ptype))
    if ptype:
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
198 199
        if isinstance(ptype, PluginMeta):
            ptype = ptype.__name__
200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221
        try:
            ptype = ptype[0].upper() + ptype[1:]
            pclass = globals()[ptype]
            logger.debug('Class: {}'.format(pclass))
            candidates = filter(lambda x: isinstance(x, pclass),
                                plugins)
        except KeyError:
            raise models.Error('{} is not a valid type'.format(ptype))
    else:
        candidates = plugins

    logger.debug(candidates)

    def matches(plug):
        res = all(getattr(plug, k, None) == v for (k, v) in kwargs.items())
        logger.debug(
            "matching {} with {}: {}".format(plug.name, kwargs, res))
        return res

    if kwargs:
        candidates = filter(matches, candidates)
    return {p.name: p for p in candidates}
222 223 224


def validate_info(info):
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
225
    return all(x in info for x in ('name',))
226 227


228 229 230
def load_module(name, root=None):
    if root:
        sys.path.append(root)
231
    tmp = importlib.import_module(name)
232 233
    if root:
        sys.path.remove(root)
234 235 236 237 238 239 240 241 242 243 244
    return tmp


def log_subprocess_output(process):
    for line in iter(process.stdout.readline, b''):
        logger.info('%r', line)
    for line in iter(process.stderr.readline, b''):
        logger.error('%r', line)


def install_deps(*plugins):
245
    installed = False
246 247 248
    for info in plugins:
        requirements = info.get('requirements', [])
        if requirements:
249
            pip_args = [sys.executable, '-m', 'pip', 'install', '--use-wheel']
250 251 252 253 254 255 256 257
            for req in requirements:
                pip_args.append(req)
            logger.info('Installing requirements: ' + str(requirements))
            process = subprocess.Popen(pip_args,
                                       stdout=subprocess.PIPE,
                                       stderr=subprocess.PIPE)
            log_subprocess_output(process)
            exitcode = process.wait()
258
            installed = True
259 260
            if exitcode != 0:
                raise models.Error("Dependencies not properly installed")
261
    return installed
262 263


J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
264 265 266 267 268 269 270 271 272 273 274
def get_plugin_class(module):
    candidate = None
    for _, obj in inspect.getmembers(module):
        if inspect.isclass(obj) and inspect.getmodule(obj) == module:
            logger.debug(("Found plugin class:"
                          " {}@{}").format(obj, inspect.getmodule(obj)))
            candidate = obj
            break
    return candidate


275
def load_plugin_from_info(info, root=None, validator=validate_info, install=True, *args, **kwargs):
276 277
    if not root and '_path' in info:
        root = os.path.dirname(info['_path'])
278
    if not validator(info):
279
        raise ValueError('Plugin info is not valid: {}'.format(info))
280 281
    module = info["module"]

282 283 284 285 286 287 288
    try:
        tmp = load_module(module, root)
    except ImportError:
        if not install:
            raise
        install_deps(info)
        tmp = load_module(module, root)
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
289 290 291 292 293 294
    cls = None
    if '@type' not in info:
        cls = get_plugin_class(tmp)
    if not cls:
        raise Exception("No valid plugin for: {}".format(module))
    return cls(info=info, *args, **kwargs)
295 296


297
def parse_plugin_info(fpath):
298 299 300
    logger.debug("Loading plugin: {}".format(fpath))
    with open(fpath, 'r') as f:
        info = yaml.load(f)
301 302 303 304 305
    info['_path'] = fpath
    name = info['name']
    return name, info


306
def load_plugin(fpath, *args, **kwargs):
307
    name, info = parse_plugin_info(fpath)
308
    logger.debug("Info: {}".format(info))
309
    plugin = load_plugin_from_info(info, *args, **kwargs)
310
    return name, plugin
311 312


313
def load_plugins(folders, loader=load_plugin, *args, **kwargs):
314 315 316
    plugins = {}
    for search_folder in folders:
        for root, dirnames, filenames in os.walk(search_folder):
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
317 318
            # Do not look for plugins in hidden or special folders
            dirnames[:] = [d for d in dirnames if d[0] not in ['.', '_']]
319
            for filename in fnmatch.filter(filenames, '*.senpy'):
320
                fpath = os.path.join(root, filename)
321
                name, plugin = loader(fpath, *args, **kwargs)
322 323 324
                if plugin and name:
                    plugins[name] = plugin
    return plugins