__init__.py 8.9 KB
Newer Older
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
1 2
from future import standard_library
standard_library.install_aliases()
3

J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
4
import os.path
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
5
import os
6
import pickle
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
7
import logging
8
import tempfile
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
9
import copy
10 11 12 13 14 15 16

import fnmatch
import inspect
import sys
import subprocess
import importlib
import yaml
17
import threading
18

J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
19
from .. import models, utils
20
from ..api import API_PARAMS
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
21 22 23

logger = logging.getLogger(__name__)

24

J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
25
class Plugin(models.Plugin):
26
    def __init__(self, info=None):
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
27 28 29 30
        """
        Provides a canonical name for plugins and serves as base for other
        kinds of plugins.
        """
31
        if not info:
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
32 33
            raise models.Error(message=("You need to provide configuration"
                                        "information for the plugin."))
34
        logger.debug("Initialising {}".format(info))
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
35
        id = 'plugins/{}_{}'.format(info['name'], info['version'])
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
36
        super(Plugin, self).__init__(id=id, **info)
37
        self.is_activated = False
38
        self._lock = threading.Lock()
39

J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
40 41 42
    def get_folder(self):
        return os.path.dirname(inspect.getfile(self.__class__))

43 44 45 46 47 48
    def activate(self):
        pass

    def deactivate(self):
        pass

J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
49
    def test(self):
50 51 52 53
        if not hasattr(self, 'test_cases'):
            import inspect
            raise AttributeError(('Plugin {} [{}] does not have any defined '
                                  'test cases').format(self.id, inspect.getfile(self.__class__)))
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
54 55 56 57 58 59
        for case in self.test_cases:
            res = list(self.analyse_entry(models.Entry(case['entry']),
                                          case['params']))
            exp = case['expected']
            if not isinstance(exp, list):
                exp = [exp]
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
60
            utils.check_template(res, exp)
61 62
            for r in res:
                r.validate()
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
63

64

J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
65 66 67 68
SenpyPlugin = Plugin


class AnalysisPlugin(Plugin):
69

70
    def analyse(self, *args, **kwargs):
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
71 72 73 74 75 76 77 78 79 80
        raise NotImplemented(
            'Your method should implement either analyse or analyse_entry')

    def analyse_entry(self, entry, parameters):
        """ An implemented plugin should override this method.
        This base method is here to adapt old style plugins which only
        implement the *analyse* function.
        Note that this method may yield an annotated entry or a list of
        entries (e.g. in a tokenizer)
        """
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
81
        text = entry['nif:isString']
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
82 83 84 85 86
        params = copy.copy(parameters)
        params['input'] = text
        results = self.analyse(**params)
        for i in results.entries:
            yield i
87

J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
88 89 90 91 92 93
    def analyse_entries(self, entries, parameters):
        for entry in entries:
            logger.debug('Analysing entry with plugin {}: {}'.format(self, entry))
            for result in self.analyse_entry(entry, parameters):
                yield result

94

J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
95
class ConversionPlugin(Plugin):
96
    pass
97

98

99
class SentimentPlugin(models.SentimentPlugin, AnalysisPlugin):
100 101 102 103
    def __init__(self, info, *args, **kwargs):
        super(SentimentPlugin, self).__init__(info, *args, **kwargs)
        self.minPolarityValue = float(info.get("minPolarityValue", 0))
        self.maxPolarityValue = float(info.get("maxPolarityValue", 1))
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
104

105

106
class EmotionPlugin(models.EmotionPlugin, AnalysisPlugin):
107
    def __init__(self, info, *args, **kwargs):
108
        super(EmotionPlugin, self).__init__(info, *args, **kwargs)
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
109 110 111 112
        self.minEmotionValue = float(info.get("minEmotionValue", -1))
        self.maxEmotionValue = float(info.get("maxEmotionValue", 1))


113 114
class EmotionConversionPlugin(models.EmotionConversionPlugin, ConversionPlugin):
    pass
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
115 116 117 118 119


class ShelfMixin(object):
    @property
    def sh(self):
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
120
        if not hasattr(self, '_sh') or self._sh is None:
121 122
            self.__dict__['_sh'] = {}
            if os.path.isfile(self.shelf_file):
123 124
                try:
                    self.__dict__['_sh'] = pickle.load(open(self.shelf_file, 'rb'))
125
                except (IndexError, EOFError, pickle.UnpicklingError):
126 127 128
                    logger.warning('{} has a corrupted shelf file!'.format(self.id))
                    if not self.get('force_shelf', False):
                        raise
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
129 130 131 132
        return self._sh

    @sh.deleter
    def sh(self):
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
133 134
        if os.path.isfile(self.shelf_file):
            os.remove(self.shelf_file)
135 136
            del self.__dict__['_sh']
        self.save()
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
137

J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
138 139
    @property
    def shelf_file(self):
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
140
        if 'shelf_file' not in self or not self['shelf_file']:
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
141 142
            sd = os.environ.get('SENPY_DATA', tempfile.gettempdir())
            self.shelf_file = os.path.join(sd, self.name + '.p')
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
143
        return self['shelf_file']
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
144

145
    def save(self):
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
146
        logger.debug('saving pickle')
147 148 149
        if hasattr(self, '_sh') and self._sh is not None:
            with open(self.shelf_file, 'wb') as f:
                pickle.dump(self._sh, f)
150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186


default_plugin_type = API_PARAMS['plugin_type']['default']


def pfilter(plugins, **kwargs):
    """ Filter plugins by different criteria """
    if isinstance(plugins, models.Plugins):
        plugins = plugins.plugins
    elif isinstance(plugins, dict):
        plugins = plugins.values()
    ptype = kwargs.pop('plugin_type', default_plugin_type)
    logger.debug('#' * 100)
    logger.debug('ptype {}'.format(ptype))
    if ptype:
        try:
            ptype = ptype[0].upper() + ptype[1:]
            pclass = globals()[ptype]
            logger.debug('Class: {}'.format(pclass))
            candidates = filter(lambda x: isinstance(x, pclass),
                                plugins)
        except KeyError:
            raise models.Error('{} is not a valid type'.format(ptype))
    else:
        candidates = plugins

    logger.debug(candidates)

    def matches(plug):
        res = all(getattr(plug, k, None) == v for (k, v) in kwargs.items())
        logger.debug(
            "matching {} with {}: {}".format(plug.name, kwargs, res))
        return res

    if kwargs:
        candidates = filter(matches, candidates)
    return {p.name: p for p in candidates}
187 188 189 190 191 192


def validate_info(info):
    return all(x in info for x in ('name', 'module', 'description', 'version'))


193 194 195
def load_module(name, root=None):
    if root:
        sys.path.append(root)
196
    tmp = importlib.import_module(name)
197 198
    if root:
        sys.path.remove(root)
199 200 201 202 203 204 205 206 207 208 209 210 211 212
    return tmp


def log_subprocess_output(process):
    for line in iter(process.stdout.readline, b''):
        logger.info('%r', line)
    for line in iter(process.stderr.readline, b''):
        logger.error('%r', line)


def install_deps(*plugins):
    for info in plugins:
        requirements = info.get('requirements', [])
        if requirements:
213
            pip_args = [sys.executable, '-m', 'pip']
214 215 216 217 218 219 220 221 222 223 224 225 226 227
            pip_args.append('install')
            pip_args.append('--use-wheel')
            for req in requirements:
                pip_args.append(req)
            logger.info('Installing requirements: ' + str(requirements))
            process = subprocess.Popen(pip_args,
                                       stdout=subprocess.PIPE,
                                       stderr=subprocess.PIPE)
            log_subprocess_output(process)
            exitcode = process.wait()
            if exitcode != 0:
                raise models.Error("Dependencies not properly installed")


228 229 230
def load_plugin_from_info(info, root=None, validator=validate_info, install=True):
    if not root and '_path' in info:
        root = os.path.dirname(info['_path'])
231
    if not validator(info):
232
        raise ValueError('Plugin info is not valid: {}'.format(info))
233 234
    module = info["module"]

235 236 237 238 239 240 241
    try:
        tmp = load_module(module, root)
    except ImportError:
        if not install:
            raise
        install_deps(info)
        tmp = load_module(module, root)
242 243 244 245 246 247 248 249 250 251 252
    candidate = None
    for _, obj in inspect.getmembers(tmp):
        if inspect.isclass(obj) and inspect.getmodule(obj) == tmp:
            logger.debug(("Found plugin class:"
                          " {}@{}").format(obj, inspect.getmodule(obj)))
            candidate = obj
            break
    if not candidate:
        logger.debug("No valid plugin for: {}".format(module))
        return
    module = candidate(info=info)
253
    return module
254 255


256
def parse_plugin_info(fpath):
257 258 259
    logger.debug("Loading plugin: {}".format(fpath))
    with open(fpath, 'r') as f:
        info = yaml.load(f)
260 261 262 263 264 265 266
    info['_path'] = fpath
    name = info['name']
    return name, info


def load_plugin(fpath):
    name, info = parse_plugin_info(fpath)
267
    logger.debug("Info: {}".format(info))
268 269
    plugin = load_plugin_from_info(info)
    return name, plugin
270 271 272 273 274 275


def load_plugins(folders, loader=load_plugin):
    plugins = {}
    for search_folder in folders:
        for root, dirnames, filenames in os.walk(search_folder):
J. Fernando Sánchez's avatar
J. Fernando Sánchez committed
276 277
            # Do not look for plugins in hidden or special folders
            dirnames[:] = [d for d in dirnames if d[0] not in ['.', '_']]
278
            for filename in fnmatch.filter(filenames, '*.senpy'):
279 280
                fpath = os.path.join(root, filename)
                name, plugin = loader(fpath)
281 282 283
                if plugin and name:
                    plugins[name] = plugin
    return plugins