Commit e329e84e authored by J. Fernando Sánchez's avatar J. Fernando Sánchez

Merge branch '46-make-data-folder-configurable' into 'master'

Resolve "Make data folder configurable"

Closes #46 and #47

See merge request !19
parents 694201d8 778746c5
.. image:: img/header.png .. image:: img/header.png
:height: 6em :width: 100%
:target: http://demos.gsi.dit.upm.es/senpy :target: http://demos.gsi.dit.upm.es/senpy
.. image:: https://travis-ci.org/gsi-upm/senpy.svg?branch=master .. image:: https://travis-ci.org/gsi-upm/senpy.svg?branch=master
......
...@@ -25,6 +25,7 @@ from senpy.extensions import Senpy ...@@ -25,6 +25,7 @@ from senpy.extensions import Senpy
import logging import logging
import os import os
import sys
import argparse import argparse
import senpy import senpy
...@@ -74,6 +75,12 @@ def main(): ...@@ -74,6 +75,12 @@ def main():
action='store_true', action='store_true',
default=False, default=False,
help='Do not run a server, only install plugin dependencies') help='Do not run a server, only install plugin dependencies')
parser.add_argument(
'--data-folder',
'--data',
type=str,
default=None,
help='Where to look for data. It be set with the SENPY_DATA environment variable as well.')
parser.add_argument( parser.add_argument(
'--threaded', '--threaded',
action='store_false', action='store_false',
...@@ -88,13 +95,16 @@ def main(): ...@@ -88,13 +95,16 @@ def main():
args = parser.parse_args() args = parser.parse_args()
if args.version: if args.version:
print('Senpy version {}'.format(senpy.__version__)) print('Senpy version {}'.format(senpy.__version__))
print(sys.version)
exit(1) exit(1)
logging.basicConfig() logging.basicConfig()
rl = logging.getLogger() rl = logging.getLogger()
rl.setLevel(getattr(logging, args.level)) rl.setLevel(getattr(logging, args.level))
app = Flask(__name__) app = Flask(__name__)
app.debug = args.debug app.debug = args.debug
sp = Senpy(app, args.plugins_folder, default_plugins=args.default_plugins) sp = Senpy(app, args.plugins_folder,
default_plugins=args.default_plugins,
data_folder=args.data_folder)
sp.install_deps() sp.install_deps()
if args.only_install: if args.only_install:
return return
......
...@@ -15,6 +15,7 @@ from functools import partial ...@@ -15,6 +15,7 @@ from functools import partial
import os import os
import copy import copy
import errno
import logging import logging
import traceback import traceback
...@@ -27,6 +28,7 @@ class Senpy(object): ...@@ -27,6 +28,7 @@ class Senpy(object):
def __init__(self, def __init__(self,
app=None, app=None,
plugin_folder=".", plugin_folder=".",
data_folder=None,
default_plugins=False): default_plugins=False):
self.app = app self.app = app
self._search_folders = set() self._search_folders = set()
...@@ -42,6 +44,17 @@ class Senpy(object): ...@@ -42,6 +44,17 @@ class Senpy(object):
self.add_folder(os.path.join('plugins', 'conversion'), self.add_folder(os.path.join('plugins', 'conversion'),
from_root=True) from_root=True)
self.data_folder = data_folder or os.environ.get('SENPY_DATA',
os.path.join(os.getcwd(),
'senpy_data'))
try:
os.makedirs(self.data_folder)
except OSError as e:
if e.errno == errno.EEXIST:
print('Directory not created.')
else:
raise
if app is not None: if app is not None:
self.init_app(app) self.init_app(app)
...@@ -312,7 +325,8 @@ class Senpy(object): ...@@ -312,7 +325,8 @@ class Senpy(object):
def plugins(self): def plugins(self):
""" Return the plugins registered for a given application. """ """ Return the plugins registered for a given application. """
if self._outdated: if self._outdated:
self._plugin_list = plugins.load_plugins(self._search_folders) self._plugin_list = plugins.load_plugins(self._search_folders,
data_folder=self.data_folder)
self._outdated = False self._outdated = False
return self._plugin_list return self._plugin_list
......
...@@ -5,7 +5,6 @@ import os.path ...@@ -5,7 +5,6 @@ import os.path
import os import os
import pickle import pickle
import logging import logging
import tempfile
import copy import copy
import fnmatch import fnmatch
...@@ -16,6 +15,8 @@ import importlib ...@@ -16,6 +15,8 @@ import importlib
import yaml import yaml
import threading import threading
from contextlib import contextmanager
from .. import models, utils from .. import models, utils
from ..api import API_PARAMS from ..api import API_PARAMS
...@@ -23,7 +24,7 @@ logger = logging.getLogger(__name__) ...@@ -23,7 +24,7 @@ logger = logging.getLogger(__name__)
class Plugin(models.Plugin): class Plugin(models.Plugin):
def __init__(self, info=None): def __init__(self, info=None, data_folder=None):
""" """
Provides a canonical name for plugins and serves as base for other Provides a canonical name for plugins and serves as base for other
kinds of plugins. kinds of plugins.
...@@ -36,6 +37,7 @@ class Plugin(models.Plugin): ...@@ -36,6 +37,7 @@ class Plugin(models.Plugin):
super(Plugin, self).__init__(id=id, **info) super(Plugin, self).__init__(id=id, **info)
self.is_activated = False self.is_activated = False
self._lock = threading.Lock() self._lock = threading.Lock()
self.data_folder = data_folder or os.getcwd()
def get_folder(self): def get_folder(self):
return os.path.dirname(inspect.getfile(self.__class__)) return os.path.dirname(inspect.getfile(self.__class__))
...@@ -61,6 +63,13 @@ class Plugin(models.Plugin): ...@@ -61,6 +63,13 @@ class Plugin(models.Plugin):
for r in res: for r in res:
r.validate() r.validate()
@contextmanager
def open(self, fpath, *args, **kwargs):
if not os.path.isabs(fpath):
fpath = os.path.join(self.data_folder, fpath)
with open(fpath, *args, **kwargs) as f:
yield f
SenpyPlugin = Plugin SenpyPlugin = Plugin
...@@ -121,7 +130,8 @@ class ShelfMixin(object): ...@@ -121,7 +130,8 @@ class ShelfMixin(object):
self.__dict__['_sh'] = {} self.__dict__['_sh'] = {}
if os.path.isfile(self.shelf_file): if os.path.isfile(self.shelf_file):
try: try:
self.__dict__['_sh'] = pickle.load(open(self.shelf_file, 'rb')) with self.open(self.shelf_file, 'rb') as p:
self.__dict__['_sh'] = pickle.load(p)
except (IndexError, EOFError, pickle.UnpicklingError): except (IndexError, EOFError, pickle.UnpicklingError):
logger.warning('{} has a corrupted shelf file!'.format(self.id)) logger.warning('{} has a corrupted shelf file!'.format(self.id))
if not self.get('force_shelf', False): if not self.get('force_shelf', False):
...@@ -138,14 +148,13 @@ class ShelfMixin(object): ...@@ -138,14 +148,13 @@ class ShelfMixin(object):
@property @property
def shelf_file(self): def shelf_file(self):
if 'shelf_file' not in self or not self['shelf_file']: if 'shelf_file' not in self or not self['shelf_file']:
sd = os.environ.get('SENPY_DATA', tempfile.gettempdir()) self.shelf_file = os.path.join(self.data_folder, self.name + '.p')
self.shelf_file = os.path.join(sd, self.name + '.p')
return self['shelf_file'] return self['shelf_file']
def save(self): def save(self):
logger.debug('saving pickle') logger.debug('saving pickle')
if hasattr(self, '_sh') and self._sh is not None: if hasattr(self, '_sh') and self._sh is not None:
with open(self.shelf_file, 'wb') as f: with self.open(self.shelf_file, 'wb') as f:
pickle.dump(self._sh, f) pickle.dump(self._sh, f)
...@@ -207,12 +216,11 @@ def log_subprocess_output(process): ...@@ -207,12 +216,11 @@ def log_subprocess_output(process):
def install_deps(*plugins): def install_deps(*plugins):
installed = False
for info in plugins: for info in plugins:
requirements = info.get('requirements', []) requirements = info.get('requirements', [])
if requirements: if requirements:
pip_args = ['pip'] pip_args = [sys.executable, '-m', 'pip', 'install', '--use-wheel']
pip_args.append('install')
pip_args.append('--use-wheel')
for req in requirements: for req in requirements:
pip_args.append(req) pip_args.append(req)
logger.info('Installing requirements: ' + str(requirements)) logger.info('Installing requirements: ' + str(requirements))
...@@ -221,11 +229,13 @@ def install_deps(*plugins): ...@@ -221,11 +229,13 @@ def install_deps(*plugins):
stderr=subprocess.PIPE) stderr=subprocess.PIPE)
log_subprocess_output(process) log_subprocess_output(process)
exitcode = process.wait() exitcode = process.wait()
installed = True
if exitcode != 0: if exitcode != 0:
raise models.Error("Dependencies not properly installed") raise models.Error("Dependencies not properly installed")
return installed
def load_plugin_from_info(info, root=None, validator=validate_info, install=True): def load_plugin_from_info(info, root=None, validator=validate_info, install=True, *args, **kwargs):
if not root and '_path' in info: if not root and '_path' in info:
root = os.path.dirname(info['_path']) root = os.path.dirname(info['_path'])
if not validator(info): if not validator(info):
...@@ -249,7 +259,7 @@ def load_plugin_from_info(info, root=None, validator=validate_info, install=True ...@@ -249,7 +259,7 @@ def load_plugin_from_info(info, root=None, validator=validate_info, install=True
if not candidate: if not candidate:
logger.debug("No valid plugin for: {}".format(module)) logger.debug("No valid plugin for: {}".format(module))
return return
module = candidate(info=info) module = candidate(info=info, *args, **kwargs)
return module return module
...@@ -262,14 +272,14 @@ def parse_plugin_info(fpath): ...@@ -262,14 +272,14 @@ def parse_plugin_info(fpath):
return name, info return name, info
def load_plugin(fpath): def load_plugin(fpath, *args, **kwargs):
name, info = parse_plugin_info(fpath) name, info = parse_plugin_info(fpath)
logger.debug("Info: {}".format(info)) logger.debug("Info: {}".format(info))
plugin = load_plugin_from_info(info) plugin = load_plugin_from_info(info, *args, **kwargs)
return name, plugin return name, plugin
def load_plugins(folders, loader=load_plugin): def load_plugins(folders, loader=load_plugin, *args, **kwargs):
plugins = {} plugins = {}
for search_folder in folders: for search_folder in folders:
for root, dirnames, filenames in os.walk(search_folder): for root, dirnames, filenames in os.walk(search_folder):
...@@ -277,7 +287,7 @@ def load_plugins(folders, loader=load_plugin): ...@@ -277,7 +287,7 @@ def load_plugins(folders, loader=load_plugin):
dirnames[:] = [d for d in dirnames if d[0] not in ['.', '_']] dirnames[:] = [d for d in dirnames if d[0] not in ['.', '_']]
for filename in fnmatch.filter(filenames, '*.senpy'): for filename in fnmatch.filter(filenames, '*.senpy'):
fpath = os.path.join(root, filename) fpath = os.path.join(root, filename)
name, plugin = loader(fpath) name, plugin = loader(fpath, *args, **kwargs)
if plugin and name: if plugin and name:
plugins[name] = plugin plugins[name] = plugin
return plugins return plugins
...@@ -6,7 +6,7 @@ logger = logging.getLogger(__name__) ...@@ -6,7 +6,7 @@ logger = logging.getLogger(__name__)
class CentroidConversion(EmotionConversionPlugin): class CentroidConversion(EmotionConversionPlugin):
def __init__(self, info): def __init__(self, info, *args, **kwargs):
if 'centroids' not in info: if 'centroids' not in info:
raise Error('Centroid conversion plugins should provide ' raise Error('Centroid conversion plugins should provide '
'the centroids in their senpy file') 'the centroids in their senpy file')
...@@ -33,7 +33,7 @@ class CentroidConversion(EmotionConversionPlugin): ...@@ -33,7 +33,7 @@ class CentroidConversion(EmotionConversionPlugin):
ncentroids[aliases.get(k1, k1)] = nv1 ncentroids[aliases.get(k1, k1)] = nv1
info['centroids'] = ncentroids info['centroids'] = ncentroids
super(CentroidConversion, self).__init__(info) super(CentroidConversion, self).__init__(info, *args, **kwargs)
self.dimensions = set() self.dimensions = set()
for c in self.centroids.values(): for c in self.centroids.values():
......
from senpy.plugins import SentimentPlugin
class DummyPlugin(SentimentPlugin):
import noop
...@@ -49,14 +49,13 @@ class ExtensionsTest(TestCase): ...@@ -49,14 +49,13 @@ class ExtensionsTest(TestCase):
""" Installing a plugin """ """ Installing a plugin """
info = { info = {
'name': 'TestPip', 'name': 'TestPip',
'module': 'dummy', 'module': 'noop_plugin',
'description': None, 'description': None,
'requirements': ['noop'], 'requirements': ['noop'],
'version': 0 'version': 0
} }
root = os.path.join(self.dir, 'plugins', 'dummy_plugin') root = os.path.join(self.dir, 'plugins', 'noop')
module = plugins.load_plugin_from_info(info, root=root) module = plugins.load_plugin_from_info(info, root=root, install=True)
plugins.install_deps(info)
assert module.name == 'TestPip' assert module.name == 'TestPip'
assert module assert module
import noop import noop
......
...@@ -222,8 +222,9 @@ def make_mini_test(plugin_info): ...@@ -222,8 +222,9 @@ def make_mini_test(plugin_info):
def _add_tests(): def _add_tests():
root = os.path.dirname(__file__) root = os.path.join(os.path.dirname(__file__), '..')
plugs = plugins.load_plugins(os.path.join(root, ".."), loader=plugins.parse_plugin_info) print(root)
plugs = plugins.load_plugins([root, ], loader=plugins.parse_plugin_info)
for k, v in plugs.items(): for k, v in plugs.items():
pass pass
t_method = make_mini_test(v) t_method = make_mini_test(v)
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment