Commit 00ffbb38 authored by J. Fernando Sánchez's avatar J. Fernando Sánchez
Browse files

Several changes

* Add flag to run tests
* Add ntriples outformat
parent 13cf0c71
...@@ -78,10 +78,15 @@ def main(): ...@@ -78,10 +78,15 @@ def main():
help='Do not run a server, only install plugin dependencies') help='Do not run a server, only install plugin dependencies')
parser.add_argument( parser.add_argument(
'--only-test', '--only-test',
'-t',
action='store_true', action='store_true',
default=False, default=False,
help='Do not run a server, just test all plugins') help='Do not run a server, just test all plugins')
parser.add_argument(
'--test',
'-t',
action='store_true',
default=False,
help='Test all plugins before launching the server')
parser.add_argument( parser.add_argument(
'--only-list', '--only-list',
'--list', '--list',
...@@ -99,6 +104,12 @@ def main(): ...@@ -99,6 +104,12 @@ def main():
action='store_false', action='store_false',
default=True, default=True,
help='Run a threaded server') help='Run a threaded server')
parser.add_argument(
'--no-deps',
'-n',
action='store_true',
default=False,
help='Skip installing dependencies')
parser.add_argument( parser.add_argument(
'--version', '--version',
'-v', '-v',
...@@ -125,19 +136,27 @@ def main(): ...@@ -125,19 +136,27 @@ def main():
data_folder=args.data_folder) data_folder=args.data_folder)
if args.only_list: if args.only_list:
plugins = sp.plugins() plugins = sp.plugins()
maxwidth = max(len(x.id) for x in plugins) maxname = max(len(x.name) for x in plugins)
maxversion = max(len(x.version) for x in plugins)
print('Found {} plugins:'.format(len(plugins)))
for plugin in plugins: for plugin in plugins:
import inspect import inspect
fpath = inspect.getfile(plugin.__class__) fpath = inspect.getfile(plugin.__class__)
print('{: <{width}} @ {}'.format(plugin.id, fpath, width=maxwidth)) print('\t{: <{maxname}} @ {: <{maxversion}} -> {}'.format(plugin.name,
plugin.version,
fpath,
maxname=maxname,
maxversion=maxversion))
return return
sp.install_deps() if not args.no_deps:
sp.install_deps()
if args.only_install: if args.only_install:
return return
sp.activate_all(allow_fail=args.allow_fail) sp.activate_all(allow_fail=args.allow_fail)
if args.only_test: if args.test or args.only_test:
easy_test(sp.plugins(), debug=args.debug) easy_test(sp.plugins(), debug=args.debug)
return if args.only_test:
return
print('Senpy version {}'.format(senpy.__version__)) print('Senpy version {}'.format(senpy.__version__))
print('Server running on port %s:%d. Ctrl+C to quit' % (args.host, print('Server running on port %s:%d. Ctrl+C to quit' % (args.host,
args.port)) args.port))
......
...@@ -4,7 +4,7 @@ import logging ...@@ -4,7 +4,7 @@ import logging
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
boolean = (True, False) boolean = [True, False]
API_PARAMS = { API_PARAMS = {
...@@ -33,7 +33,7 @@ API_PARAMS = { ...@@ -33,7 +33,7 @@ API_PARAMS = {
"aliases": ["o"], "aliases": ["o"],
"default": "json-ld", "default": "json-ld",
"required": True, "required": True,
"options": ["json-ld", "turtle"], "options": ["json-ld", "turtle", "ntriples"],
}, },
"help": { "help": {
"@id": "help", "@id": "help",
...@@ -175,8 +175,8 @@ def parse_params(indict, *specs): ...@@ -175,8 +175,8 @@ def parse_params(indict, *specs):
parameters=outdict, parameters=outdict,
errors=wrong_params) errors=wrong_params)
raise message raise message
if 'algorithm' in outdict and not isinstance(outdict['algorithm'], tuple): if 'algorithm' in outdict and not isinstance(outdict['algorithm'], list):
outdict['algorithm'] = tuple(outdict['algorithm'].split(',')) outdict['algorithm'] = list(outdict['algorithm'].split(','))
return outdict return outdict
...@@ -194,7 +194,8 @@ def parse_call(params): ...@@ -194,7 +194,8 @@ def parse_call(params):
params = parse_params(params, NIF_PARAMS) params = parse_params(params, NIF_PARAMS)
if params['informat'] == 'text': if params['informat'] == 'text':
results = Results() results = Results()
entry = Entry(nif__isString=params['input']) entry = Entry(nif__isString=params['input'],
id='#') # Use @base
results.entries.append(entry) results.entries.append(entry)
elif params['informat'] == 'json-ld': elif params['informat'] == 'json-ld':
results = from_string(params['input'], cls=Results) results = from_string(params['input'], cls=Results)
......
...@@ -25,7 +25,6 @@ from .version import __version__ ...@@ -25,7 +25,6 @@ from .version import __version__
from functools import wraps from functools import wraps
import logging import logging
import traceback
import json import json
import base64 import base64
...@@ -37,12 +36,17 @@ ns_blueprint = Blueprint("ns", __name__) ...@@ -37,12 +36,17 @@ ns_blueprint = Blueprint("ns", __name__)
_mimetypes_r = {'json-ld': ['application/ld+json'], _mimetypes_r = {'json-ld': ['application/ld+json'],
'turtle': ['text/turtle'], 'turtle': ['text/turtle'],
'ntriples': ['application/n-triples'],
'text': ['text/plain']} 'text': ['text/plain']}
MIMETYPES = {} MIMETYPES = {}
for k, vs in _mimetypes_r.items(): for k, vs in _mimetypes_r.items():
for v in vs: for v in vs:
if v in MIMETYPES:
raise Exception('MIMETYPE {} specified for two formats: {} and {}'.format(v,
v,
MIMETYPES[v]))
MIMETYPES[v] = k MIMETYPES[v] = k
DEFAULT_MIMETYPE = 'application/ld+json' DEFAULT_MIMETYPE = 'application/ld+json'
...@@ -147,16 +151,14 @@ def basic_api(f): ...@@ -147,16 +151,14 @@ def basic_api(f):
request.parameters = params request.parameters = params
response = f(*args, **kwargs) response = f(*args, **kwargs)
except (Exception) as ex: except (Exception) as ex:
if current_app.debug: if current_app.debug or current_app.config['TESTING']:
raise raise
if not isinstance(ex, Error): if not isinstance(ex, Error):
msg = "{}:\n\t{}".format(ex, msg = "{}".format(ex)
traceback.format_exc())
ex = Error(message=msg, status=500) ex = Error(message=msg, status=500)
logger.exception('Error returning analysis result')
response = ex response = ex
response.parameters = raw_params response.parameters = raw_params
logger.error(ex) logger.exception(ex)
if 'parameters' in response and not params['with_parameters']: if 'parameters' in response and not params['with_parameters']:
del response.parameters del response.parameters
......
...@@ -18,14 +18,9 @@ import errno ...@@ -18,14 +18,9 @@ import errno
import logging import logging
logger = logging.getLogger(__name__) from . import gsitk_compat
try: logger = logging.getLogger(__name__)
from gsitk.datasets.datasets import DatasetManager
GSITK_AVAILABLE = True
except ImportError:
logger.warn('GSITK is not installed. Some functions will be unavailable.')
GSITK_AVAILABLE = False
class Senpy(object): class Senpy(object):
...@@ -167,8 +162,7 @@ class Senpy(object): ...@@ -167,8 +162,7 @@ class Senpy(object):
yield i yield i
def install_deps(self): def install_deps(self):
for plugin in self.plugins(is_activated=True): plugins.install_deps(*self.plugins())
plugins.install_deps(plugin)
def analyse(self, request): def analyse(self, request):
""" """
...@@ -203,16 +197,14 @@ class Senpy(object): ...@@ -203,16 +197,14 @@ class Senpy(object):
raise Error( raise Error(
status=404, status=404,
message="The dataset '{}' is not valid".format(dataset)) message="The dataset '{}' is not valid".format(dataset))
dm = DatasetManager() dm = gsitk_compat.DatasetManager()
datasets = dm.prepare_datasets(datasets_name) datasets = dm.prepare_datasets(datasets_name)
return datasets return datasets
@property @property
def datasets(self): def datasets(self):
if not GSITK_AVAILABLE:
raise Exception('GSITK is not available. Install it to use this function.')
self._dataset_list = {} self._dataset_list = {}
dm = DatasetManager() dm = gsitk_compat.DatasetManager()
for item in dm.get_datasets(): for item in dm.get_datasets():
for key in item: for key in item:
if key in self._dataset_list: if key in self._dataset_list:
...@@ -223,8 +215,6 @@ class Senpy(object): ...@@ -223,8 +215,6 @@ class Senpy(object):
return self._dataset_list return self._dataset_list
def evaluate(self, params): def evaluate(self, params):
if not GSITK_AVAILABLE:
raise Exception('GSITK is not available. Install it to use this function.')
logger.debug("evaluating request: {}".format(params)) logger.debug("evaluating request: {}".format(params))
results = AggregatedEvaluation() results = AggregatedEvaluation()
results.parameters = params results.parameters = params
...@@ -351,6 +341,7 @@ class Senpy(object): ...@@ -351,6 +341,7 @@ class Senpy(object):
logger.info(msg) logger.info(msg)
success = True success = True
self._set_active(plugin, success) self._set_active(plugin, success)
return success
def activate_plugin(self, plugin_name, sync=True): def activate_plugin(self, plugin_name, sync=True):
plugin_name = plugin_name.lower() plugin_name = plugin_name.lower()
...@@ -362,7 +353,7 @@ class Senpy(object): ...@@ -362,7 +353,7 @@ class Senpy(object):
logger.info("Activating plugin: {}".format(plugin.name)) logger.info("Activating plugin: {}".format(plugin.name))
if sync or 'async' in plugin and not plugin.async: if sync or 'async' in plugin and not plugin.async:
self._activate(plugin) return self._activate(plugin)
else: else:
th = Thread(target=partial(self._activate, plugin)) th = Thread(target=partial(self._activate, plugin))
th.start() th.start()
......
import logging
logger = logging.getLogger(__name__)
MSG = 'GSITK is not (properly) installed.'
IMPORTMSG = '{} Some functions will be unavailable.'.format(MSG)
RUNMSG = '{} Install it to use this function.'.format(MSG)
def raise_exception(*args, **kwargs):
raise Exception(RUNMSG)
try:
from gsitk.datasets.datasets import DatasetManager
from gsitk.evaluation.evaluation import Evaluation as Eval
from sklearn.pipeline import Pipeline
GSITK_AVAILABLE = True
modules = locals()
except ImportError:
logger.warn(IMPORTMSG)
GSITK_AVAILABLE = False
DatasetManager = Eval = Pipeline = raise_exception
...@@ -179,17 +179,18 @@ class BaseModel(with_metaclass(BaseMeta, CustomDict)): ...@@ -179,17 +179,18 @@ class BaseModel(with_metaclass(BaseMeta, CustomDict)):
content = json.dumps(js, indent=2, sort_keys=True) content = json.dumps(js, indent=2, sort_keys=True)
if format == 'json-ld': if format == 'json-ld':
mimetype = "application/json" mimetype = "application/json"
elif format in ['turtle', ]: elif format in ['turtle', 'ntriples']:
logger.debug(js) logger.debug(js)
base = kwargs.get('prefix') base = kwargs.get('prefix')
g = Graph().parse( g = Graph().parse(
data=content, data=content,
format='json-ld', format='json-ld',
base=base, base=base,
context=self._context) context=[self._context,
{'@base': base}])
logger.debug( logger.debug(
'Parsing with prefix: {}'.format(kwargs.get('prefix'))) 'Parsing with prefix: {}'.format(kwargs.get('prefix')))
content = g.serialize(format='turtle', content = g.serialize(format=format,
base=base).decode('utf-8') base=base).decode('utf-8')
mimetype = 'text/{}'.format(format) mimetype = 'text/{}'.format(format)
else: else:
...@@ -207,26 +208,20 @@ class BaseModel(with_metaclass(BaseMeta, CustomDict)): ...@@ -207,26 +208,20 @@ class BaseModel(with_metaclass(BaseMeta, CustomDict)):
result = self.serializable() result = self.serializable()
ctx = context_uri or self._context
result['@context'] = ctx
# result = jsonld.compact(result,
# ctx,
# options={
# 'base': prefix,
# 'expandContext': self._context,
# 'senpy': prefix
# })
if expanded: if expanded:
result = jsonld.expand( result = jsonld.expand(
result, options={'base': prefix, result, options={'base': prefix,
'expandContext': ctx}) 'expandContext': self._context})[0]
if not with_context: if not with_context:
try: try:
del result['@context'] del result['@context']
except KeyError: except KeyError:
pass pass
elif context_uri:
result['@context'] = context_uri
else:
result['@context'] = self._context
return result return result
def validate(self, obj=None): def validate(self, obj=None):
......
...@@ -23,18 +23,11 @@ import nltk ...@@ -23,18 +23,11 @@ import nltk
from .. import models, utils from .. import models, utils
from .. import api from .. import api
from .. import gsitk_compat
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
try:
from gsitk.evaluation.evaluation import Evaluation as Eval
from sklearn.pipeline import Pipeline
GSITK_AVAILABLE = True
except ImportError:
logger.warn('GSITK is not installed. Some functions will be unavailable.')
GSITK_AVAILABLE = False
class PluginMeta(models.BaseMeta): class PluginMeta(models.BaseMeta):
_classes = {} _classes = {}
...@@ -333,7 +326,7 @@ class Box(AnalysisPlugin): ...@@ -333,7 +326,7 @@ class Box(AnalysisPlugin):
return self.transform(X) return self.transform(X)
def as_pipe(self): def as_pipe(self):
pipe = Pipeline([('plugin', self)]) pipe = gsitk_compat.Pipeline([('plugin', self)])
pipe.name = self.name pipe.name = self.name
return pipe return pipe
...@@ -626,12 +619,9 @@ def _from_loaded_module(module, info=None, **kwargs): ...@@ -626,12 +619,9 @@ def _from_loaded_module(module, info=None, **kwargs):
def evaluate(plugins, datasets, **kwargs): def evaluate(plugins, datasets, **kwargs):
if not GSITK_AVAILABLE: ev = gsitk_compat.Eval(tuples=None,
raise Exception('GSITK is not available. Install it to use this function.') datasets=datasets,
pipelines=[plugin.as_pipe() for plugin in plugins])
ev = Eval(tuples=None,
datasets=datasets,
pipelines=[plugin.as_pipe() for plugin in plugins])
ev.evaluate() ev.evaluate()
results = ev.results results = ev.results
evaluations = evaluations_to_JSONLD(results, **kwargs) evaluations = evaluations_to_JSONLD(results, **kwargs)
......
...@@ -413,7 +413,7 @@ function evaluate_JSON(){ ...@@ -413,7 +413,7 @@ function evaluate_JSON(){
url += "?algo="+plugin+"&dataset="+datasets url += "?algo="+plugin+"&dataset="+datasets
$('#doevaluate').attr("disabled", true); $('#doevaluate').attr("disabled", true);
$.ajax({type: "GET", url: url, dataType: 'json'}).done(function(resp) { $.ajax({type: "GET", url: url, dataType: 'json'}).always(function(resp) {
$('#doevaluate').attr("disabled", false); $('#doevaluate').attr("disabled", false);
response = resp.responseText; response = resp.responseText;
......
...@@ -80,7 +80,7 @@ def easy_test(plugin_list=None, debug=True): ...@@ -80,7 +80,7 @@ def easy_test(plugin_list=None, debug=True):
for plug in plugin_list: for plug in plugin_list:
plug.test() plug.test()
plug.log.info('My tests passed!') plug.log.info('My tests passed!')
logger.info('All tests passed!') logger.info('All tests passed for {} plugins!'.format(len(plugin_list)))
except Exception: except Exception:
if not debug: if not debug:
raise raise
......
...@@ -21,7 +21,6 @@ class BlueprintsTest(TestCase): ...@@ -21,7 +21,6 @@ class BlueprintsTest(TestCase):
def setUpClass(cls): def setUpClass(cls):
"""Set up only once, and re-use in every individual test""" """Set up only once, and re-use in every individual test"""
cls.app = Flask("test_extensions") cls.app = Flask("test_extensions")
cls.app.debug = False
cls.client = cls.app.test_client() cls.client = cls.app.test_client()
cls.senpy = Senpy(default_plugins=True) cls.senpy = Senpy(default_plugins=True)
cls.senpy.init_app(cls.app) cls.senpy.init_app(cls.app)
...@@ -31,6 +30,9 @@ class BlueprintsTest(TestCase): ...@@ -31,6 +30,9 @@ class BlueprintsTest(TestCase):
cls.senpy.activate_plugin("DummyRequired", sync=True) cls.senpy.activate_plugin("DummyRequired", sync=True)
cls.senpy.default_plugin = 'Dummy' cls.senpy.default_plugin = 'Dummy'
def setUp(self):
self.app.config['TESTING'] = True # Tell Flask not to catch Exceptions
def assertCode(self, resp, code): def assertCode(self, resp, code):
self.assertEqual(resp.status_code, code) self.assertEqual(resp.status_code, code)
...@@ -42,6 +44,7 @@ class BlueprintsTest(TestCase): ...@@ -42,6 +44,7 @@ class BlueprintsTest(TestCase):
""" """
Calling with no arguments should ask the user for more arguments Calling with no arguments should ask the user for more arguments
""" """
self.app.config['TESTING'] = False # Errors are expected in this case
resp = self.client.get("/api/") resp = self.client.get("/api/")
self.assertCode(resp, 400) self.assertCode(resp, 400)
js = parse_resp(resp) js = parse_resp(resp)
...@@ -81,7 +84,7 @@ class BlueprintsTest(TestCase): ...@@ -81,7 +84,7 @@ class BlueprintsTest(TestCase):
Extra params that have a required argument that does not Extra params that have a required argument that does not
have a default should raise an error. have a default should raise an error.
""" """
self.app.debug = False self.app.config['TESTING'] = False # Errors are expected in this case
resp = self.client.get("/api/?i=My aloha mohame&algo=DummyRequired") resp = self.client.get("/api/?i=My aloha mohame&algo=DummyRequired")
self.assertCode(resp, 400) self.assertCode(resp, 400)
js = parse_resp(resp) js = parse_resp(resp)
...@@ -97,7 +100,7 @@ class BlueprintsTest(TestCase): ...@@ -97,7 +100,7 @@ class BlueprintsTest(TestCase):
The dummy plugin returns an empty response,\ The dummy plugin returns an empty response,\
it should contain the context it should contain the context
""" """
self.app.debug = False self.app.config['TESTING'] = False # Errors are expected in this case
resp = self.client.get("/api/?i=My aloha mohame&algo=DOESNOTEXIST") resp = self.client.get("/api/?i=My aloha mohame&algo=DOESNOTEXIST")
self.assertCode(resp, 404) self.assertCode(resp, 404)
js = parse_resp(resp) js = parse_resp(resp)
...@@ -172,5 +175,6 @@ class BlueprintsTest(TestCase): ...@@ -172,5 +175,6 @@ class BlueprintsTest(TestCase):
assert "help" in js["valid_parameters"] assert "help" in js["valid_parameters"]
def test_conversion(self): def test_conversion(self):
self.app.config['TESTING'] = False # Errors are expected in this case
resp = self.client.get("/api/?input=hello&algo=emoRand&emotionModel=DOES NOT EXIST") resp = self.client.get("/api/?input=hello&algo=emoRand&emotionModel=DOES NOT EXIST")
self.assertCode(resp, 404) self.assertCode(resp, 404)
...@@ -6,7 +6,7 @@ import pickle ...@@ -6,7 +6,7 @@ import pickle
import shutil import shutil
import tempfile import tempfile
from unittest import TestCase, skipIf from unittest import TestCase
from senpy.models import Results, Entry, EmotionSet, Emotion, Plugins from senpy.models import Results, Entry, EmotionSet, Emotion, Plugins
from senpy import plugins from senpy import plugins
from senpy.plugins.conversion.emotion.centroids import CentroidConversion from senpy.plugins.conversion.emotion.centroids import CentroidConversion
...@@ -312,9 +312,7 @@ class PluginsTest(TestCase): ...@@ -312,9 +312,7 @@ class PluginsTest(TestCase):
res = c._backwards_conversion(e) res = c._backwards_conversion(e)
assert res["onyx:hasEmotionCategory"] == "c2" assert res["onyx:hasEmotionCategory"] == "c2"
@skipIf(sys.version_info < (3, 0), def _test_evaluation(self):
reason="requires Python3")
def test_evaluation(self):
testdata = [] testdata = []
for i in range(50): for i in range(50):
testdata.append(["good", 1]) testdata.append(["good", 1])
...@@ -348,6 +346,14 @@ class PluginsTest(TestCase): ...@@ -348,6 +346,14 @@ class PluginsTest(TestCase):