Commit a3eb8f19 authored by J. Fernando Sánchez's avatar J. Fernando Sánchez
Browse files

Several changes

* Add flag to run tests (and exit, or run the server)
* Add ntriples outformat
* Modify dependency installation logic to avoid installing several times
* Add encoded URLs as base/prefix
* Allow plugin activation to fail
parents 61181db1 00ffbb38
version: '3'
services:
senpy:
image: "${IMAGENAME-gsiupm/senpy}:${VERSION-latest}"
entrypoint: ["/bin/bash"]
working_dir: "/senpy-plugins"
ports:
- 5000:5000
volumes:
- ".:/usr/src/app/"
version: '3'
services:
test:
image: "${IMAGENAME-gsiupm/senpy}:${VERSION-dev}"
entrypoint: ["py.test"]
volumes:
- ".:/usr/src/app/"
command:
[]
version: '3'
services:
senpy:
image: "${IMAGENAME-gsiupm/senpy}:${VERSION-dev}"
build:
context: .
dockerfile: Dockerfile${PYVERSION--2.7}
ports:
- 5001:5000
volumes:
- "./data:/data"
...@@ -78,10 +78,15 @@ def main(): ...@@ -78,10 +78,15 @@ def main():
help='Do not run a server, only install plugin dependencies') help='Do not run a server, only install plugin dependencies')
parser.add_argument( parser.add_argument(
'--only-test', '--only-test',
'-t',
action='store_true', action='store_true',
default=False, default=False,
help='Do not run a server, just test all plugins') help='Do not run a server, just test all plugins')
parser.add_argument(
'--test',
'-t',
action='store_true',
default=False,
help='Test all plugins before launching the server')
parser.add_argument( parser.add_argument(
'--only-list', '--only-list',
'--list', '--list',
...@@ -99,12 +104,24 @@ def main(): ...@@ -99,12 +104,24 @@ def main():
action='store_false', action='store_false',
default=True, default=True,
help='Run a threaded server') help='Run a threaded server')
parser.add_argument(
'--no-deps',
'-n',
action='store_true',
default=False,
help='Skip installing dependencies')
parser.add_argument( parser.add_argument(
'--version', '--version',
'-v', '-v',
action='store_true', action='store_true',
default=False, default=False,
help='Output the senpy version and exit') help='Output the senpy version and exit')
parser.add_argument(
'--allow-fail',
'--fail',
action='store_true',
default=False,
help='Do not exit if some plugins fail to activate')
args = parser.parse_args() args = parser.parse_args()
if args.version: if args.version:
print('Senpy version {}'.format(senpy.__version__)) print('Senpy version {}'.format(senpy.__version__))
...@@ -119,19 +136,27 @@ def main(): ...@@ -119,19 +136,27 @@ def main():
data_folder=args.data_folder) data_folder=args.data_folder)
if args.only_list: if args.only_list:
plugins = sp.plugins() plugins = sp.plugins()
maxwidth = max(len(x.id) for x in plugins) maxname = max(len(x.name) for x in plugins)
maxversion = max(len(x.version) for x in plugins)
print('Found {} plugins:'.format(len(plugins)))
for plugin in plugins: for plugin in plugins:
import inspect import inspect
fpath = inspect.getfile(plugin.__class__) fpath = inspect.getfile(plugin.__class__)
print('{: <{width}} @ {}'.format(plugin.id, fpath, width=maxwidth)) print('\t{: <{maxname}} @ {: <{maxversion}} -> {}'.format(plugin.name,
plugin.version,
fpath,
maxname=maxname,
maxversion=maxversion))
return return
sp.install_deps() if not args.no_deps:
sp.install_deps()
if args.only_install: if args.only_install:
return return
sp.activate_all() sp.activate_all(allow_fail=args.allow_fail)
if args.only_test: if args.test or args.only_test:
easy_test(sp.plugins(), debug=args.debug) easy_test(sp.plugins(), debug=args.debug)
return if args.only_test:
return
print('Senpy version {}'.format(senpy.__version__)) print('Senpy version {}'.format(senpy.__version__))
print('Server running on port %s:%d. Ctrl+C to quit' % (args.host, print('Server running on port %s:%d. Ctrl+C to quit' % (args.host,
args.port)) args.port))
......
...@@ -3,6 +3,10 @@ from .models import Error, Results, Entry, from_string ...@@ -3,6 +3,10 @@ from .models import Error, Results, Entry, from_string
import logging import logging
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
boolean = [True, False]
API_PARAMS = { API_PARAMS = {
"algorithm": { "algorithm": {
"aliases": ["algorithms", "a", "algo"], "aliases": ["algorithms", "a", "algo"],
...@@ -13,14 +17,14 @@ API_PARAMS = { ...@@ -13,14 +17,14 @@ API_PARAMS = {
"expanded-jsonld": { "expanded-jsonld": {
"@id": "expanded-jsonld", "@id": "expanded-jsonld",
"aliases": ["expanded"], "aliases": ["expanded"],
"options": "boolean", "options": boolean,
"required": True, "required": True,
"default": False "default": False
}, },
"with_parameters": { "with_parameters": {
"aliases": ['withparameters', "aliases": ['withparameters',
'with-parameters'], 'with-parameters'],
"options": "boolean", "options": boolean,
"default": False, "default": False,
"required": True "required": True
}, },
...@@ -29,14 +33,14 @@ API_PARAMS = { ...@@ -29,14 +33,14 @@ API_PARAMS = {
"aliases": ["o"], "aliases": ["o"],
"default": "json-ld", "default": "json-ld",
"required": True, "required": True,
"options": ["json-ld", "turtle"], "options": ["json-ld", "turtle", "ntriples"],
}, },
"help": { "help": {
"@id": "help", "@id": "help",
"description": "Show additional help to know more about the possible parameters", "description": "Show additional help to know more about the possible parameters",
"aliases": ["h"], "aliases": ["h"],
"required": True, "required": True,
"options": "boolean", "options": boolean,
"default": False "default": False
}, },
"emotionModel": { "emotionModel": {
...@@ -83,7 +87,7 @@ WEB_PARAMS = { ...@@ -83,7 +87,7 @@ WEB_PARAMS = {
"aliases": ["headers"], "aliases": ["headers"],
"required": True, "required": True,
"default": False, "default": False,
"options": "boolean" "options": boolean
}, },
} }
...@@ -132,7 +136,7 @@ NIF_PARAMS = { ...@@ -132,7 +136,7 @@ NIF_PARAMS = {
"aliases": ["u"], "aliases": ["u"],
"required": False, "required": False,
"default": "RFC5147String", "default": "RFC5147String",
"options": "RFC5147String" "options": ["RFC5147String", ]
} }
} }
...@@ -159,7 +163,7 @@ def parse_params(indict, *specs): ...@@ -159,7 +163,7 @@ def parse_params(indict, *specs):
wrong_params[param] = spec[param] wrong_params[param] = spec[param]
continue continue
if "options" in options: if "options" in options:
if options["options"] == "boolean": if options["options"] == boolean:
outdict[param] = outdict[param] in [None, True, 'true', '1'] outdict[param] = outdict[param] in [None, True, 'true', '1']
elif outdict[param] not in options["options"]: elif outdict[param] not in options["options"]:
wrong_params[param] = spec[param] wrong_params[param] = spec[param]
...@@ -172,7 +176,7 @@ def parse_params(indict, *specs): ...@@ -172,7 +176,7 @@ def parse_params(indict, *specs):
errors=wrong_params) errors=wrong_params)
raise message raise message
if 'algorithm' in outdict and not isinstance(outdict['algorithm'], list): if 'algorithm' in outdict and not isinstance(outdict['algorithm'], list):
outdict['algorithm'] = outdict['algorithm'].split(',') outdict['algorithm'] = list(outdict['algorithm'].split(','))
return outdict return outdict
...@@ -190,7 +194,8 @@ def parse_call(params): ...@@ -190,7 +194,8 @@ def parse_call(params):
params = parse_params(params, NIF_PARAMS) params = parse_params(params, NIF_PARAMS)
if params['informat'] == 'text': if params['informat'] == 'text':
results = Results() results = Results()
entry = Entry(nif__isString=params['input']) entry = Entry(nif__isString=params['input'],
id='#') # Use @base
results.entries.append(entry) results.entries.append(entry)
elif params['informat'] == 'json-ld': elif params['informat'] == 'json-ld':
results = from_string(params['input'], cls=Results) results = from_string(params['input'], cls=Results)
......
...@@ -18,15 +18,15 @@ ...@@ -18,15 +18,15 @@
Blueprints for Senpy Blueprints for Senpy
""" """
from flask import (Blueprint, request, current_app, render_template, url_for, from flask import (Blueprint, request, current_app, render_template, url_for,
jsonify) jsonify, redirect)
from .models import Error, Response, Help, Plugins, read_schema, dump_schema, Datasets from .models import Error, Response, Help, Plugins, read_schema, dump_schema, Datasets
from . import api from . import api
from .version import __version__ from .version import __version__
from functools import wraps from functools import wraps
import logging import logging
import traceback
import json import json
import base64
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
...@@ -34,6 +34,24 @@ api_blueprint = Blueprint("api", __name__) ...@@ -34,6 +34,24 @@ api_blueprint = Blueprint("api", __name__)
demo_blueprint = Blueprint("demo", __name__, template_folder='templates') demo_blueprint = Blueprint("demo", __name__, template_folder='templates')
ns_blueprint = Blueprint("ns", __name__) ns_blueprint = Blueprint("ns", __name__)
_mimetypes_r = {'json-ld': ['application/ld+json'],
'turtle': ['text/turtle'],
'ntriples': ['application/n-triples'],
'text': ['text/plain']}
MIMETYPES = {}
for k, vs in _mimetypes_r.items():
for v in vs:
if v in MIMETYPES:
raise Exception('MIMETYPE {} specified for two formats: {} and {}'.format(v,
v,
MIMETYPES[v]))
MIMETYPES[v] = k
DEFAULT_MIMETYPE = 'application/ld+json'
DEFAULT_FORMAT = 'json-ld'
def get_params(req): def get_params(req):
if req.method == 'POST': if req.method == 'POST':
...@@ -45,6 +63,30 @@ def get_params(req): ...@@ -45,6 +63,30 @@ def get_params(req):
return indict return indict
def encoded_url(url=None, base=None):
code = ''
if not url:
if request.method == 'GET':
url = request.full_path[1:] # Remove the first slash
else:
hash(frozenset(request.form.params().items()))
code = 'hash:{}'.format(hash)
code = code or base64.urlsafe_b64encode(url.encode()).decode()
if base:
return base + code
return url_for('api.decode', code=code, _external=True)
def decoded_url(code, base=None):
if code.startswith('hash:'):
raise Exception('Can not decode a URL for a POST request')
base = base or request.url_root
path = base64.urlsafe_b64decode(code.encode()).decode()
return base + path
@demo_blueprint.route('/') @demo_blueprint.route('/')
def index(): def index():
ev = str(get_params(request).get('evaluation', False)) ev = str(get_params(request).get('evaluation', False))
...@@ -59,13 +101,22 @@ def index(): ...@@ -59,13 +101,22 @@ def index():
def context(entity="context"): def context(entity="context"):
context = Response._context context = Response._context
context['@vocab'] = url_for('ns.index', _external=True) context['@vocab'] = url_for('ns.index', _external=True)
context['endpoint'] = url_for('api.api_root', _external=True)
return jsonify({"@context": context}) return jsonify({"@context": context})
@api_blueprint.route('/d/<code>')
def decode(code):
try:
return redirect(decoded_url(code))
except Exception:
return Error('invalid URL').flask()
@ns_blueprint.route('/') # noqa: F811 @ns_blueprint.route('/') # noqa: F811
def index(): def index():
context = Response._context context = Response._context.copy()
context['@vocab'] = url_for('.ns', _external=True) context['endpoint'] = url_for('api.api_root', _external=True)
return jsonify({"@context": context}) return jsonify({"@context": context})
...@@ -81,7 +132,7 @@ def basic_api(f): ...@@ -81,7 +132,7 @@ def basic_api(f):
default_params = { default_params = {
'inHeaders': False, 'inHeaders': False,
'expanded-jsonld': False, 'expanded-jsonld': False,
'outformat': 'json-ld', 'outformat': None,
'with_parameters': True, 'with_parameters': True,
} }
...@@ -100,29 +151,34 @@ def basic_api(f): ...@@ -100,29 +151,34 @@ def basic_api(f):
request.parameters = params request.parameters = params
response = f(*args, **kwargs) response = f(*args, **kwargs)
except (Exception) as ex: except (Exception) as ex:
if current_app.debug: if current_app.debug or current_app.config['TESTING']:
raise raise
if not isinstance(ex, Error): if not isinstance(ex, Error):
msg = "{}:\n\t{}".format(ex, msg = "{}".format(ex)
traceback.format_exc())
ex = Error(message=msg, status=500) ex = Error(message=msg, status=500)
logger.exception('Error returning analysis result')
response = ex response = ex
response.parameters = raw_params response.parameters = raw_params
logger.error(ex) logger.exception(ex)
if 'parameters' in response and not params['with_parameters']: if 'parameters' in response and not params['with_parameters']:
del response.parameters del response.parameters
logger.info('Response: {}'.format(response)) logger.info('Response: {}'.format(response))
mime = request.accept_mimetypes\
.best_match(MIMETYPES.keys(),
DEFAULT_MIMETYPE)
mimeformat = MIMETYPES.get(mime, DEFAULT_FORMAT)
outformat = params['outformat'] or mimeformat
return response.flask( return response.flask(
in_headers=params['inHeaders'], in_headers=params['inHeaders'],
headers=headers, headers=headers,
prefix=url_for('.api_root', _external=True), prefix=params.get('prefix', encoded_url()),
context_uri=url_for('api.context', context_uri=url_for('api.context',
entity=type(response).__name__, entity=type(response).__name__,
_external=True), _external=True),
outformat=params['outformat'], outformat=outformat,
expanded=params['expanded-jsonld']) expanded=params['expanded-jsonld'])
return decorated_function return decorated_function
......
...@@ -18,14 +18,9 @@ import errno ...@@ -18,14 +18,9 @@ import errno
import logging import logging
logger = logging.getLogger(__name__) from . import gsitk_compat
try: logger = logging.getLogger(__name__)
from gsitk.datasets.datasets import DatasetManager
GSITK_AVAILABLE = True
except ImportError:
logger.warn('GSITK is not installed. Some functions will be unavailable.')
GSITK_AVAILABLE = False
class Senpy(object): class Senpy(object):
...@@ -95,7 +90,7 @@ class Senpy(object): ...@@ -95,7 +90,7 @@ class Senpy(object):
if plugin in self._plugins: if plugin in self._plugins:
return self._plugins[plugin] return self._plugins[plugin]
results = self.plugins(id='plugins/{}'.format(name)) results = self.plugins(id='endpoint:plugins/{}'.format(name))
if not results: if not results:
return Error(message="Plugin not found", status=404) return Error(message="Plugin not found", status=404)
...@@ -167,8 +162,7 @@ class Senpy(object): ...@@ -167,8 +162,7 @@ class Senpy(object):
yield i yield i
def install_deps(self): def install_deps(self):
for plugin in self.plugins(is_activated=True): plugins.install_deps(*self.plugins())
plugins.install_deps(plugin)
def analyse(self, request): def analyse(self, request):
""" """
...@@ -203,16 +197,14 @@ class Senpy(object): ...@@ -203,16 +197,14 @@ class Senpy(object):
raise Error( raise Error(
status=404, status=404,
message="The dataset '{}' is not valid".format(dataset)) message="The dataset '{}' is not valid".format(dataset))
dm = DatasetManager() dm = gsitk_compat.DatasetManager()
datasets = dm.prepare_datasets(datasets_name) datasets = dm.prepare_datasets(datasets_name)
return datasets return datasets
@property @property
def datasets(self): def datasets(self):
if not GSITK_AVAILABLE:
raise Exception('GSITK is not available. Install it to use this function.')
self._dataset_list = {} self._dataset_list = {}
dm = DatasetManager() dm = gsitk_compat.DatasetManager()
for item in dm.get_datasets(): for item in dm.get_datasets():
for key in item: for key in item:
if key in self._dataset_list: if key in self._dataset_list:
...@@ -223,8 +215,6 @@ class Senpy(object): ...@@ -223,8 +215,6 @@ class Senpy(object):
return self._dataset_list return self._dataset_list
def evaluate(self, params): def evaluate(self, params):
if not GSITK_AVAILABLE:
raise Exception('GSITK is not available. Install it to use this function.')
logger.debug("evaluating request: {}".format(params)) logger.debug("evaluating request: {}".format(params))
results = AggregatedEvaluation() results = AggregatedEvaluation()
results.parameters = params results.parameters = params
...@@ -318,10 +308,15 @@ class Senpy(object): ...@@ -318,10 +308,15 @@ class Senpy(object):
else: else:
self._default = self._plugins[value.lower()] self._default = self._plugins[value.lower()]
def activate_all(self, sync=True): def activate_all(self, sync=True, allow_fail=False):
ps = [] ps = []
for plug in self._plugins.keys(): for plug in self._plugins.keys():
ps.append(self.activate_plugin(plug, sync=sync)) try:
self.activate_plugin(plug, sync=sync)
except Exception as ex:
if not allow_fail:
raise
logger.error('Could not activate {}: {}'.format(plug, ex))
return ps return ps
def deactivate_all(self, sync=True): def deactivate_all(self, sync=True):
...@@ -346,6 +341,7 @@ class Senpy(object): ...@@ -346,6 +341,7 @@ class Senpy(object):
logger.info(msg) logger.info(msg)
success = True success = True
self._set_active(plugin, success) self._set_active(plugin, success)
return success
def activate_plugin(self, plugin_name, sync=True): def activate_plugin(self, plugin_name, sync=True):
plugin_name = plugin_name.lower() plugin_name = plugin_name.lower()
...@@ -357,7 +353,7 @@ class Senpy(object): ...@@ -357,7 +353,7 @@ class Senpy(object):
logger.info("Activating plugin: {}".format(plugin.name)) logger.info("Activating plugin: {}".format(plugin.name))
if sync or 'async' in plugin and not plugin.async: if sync or 'async' in plugin and not plugin.async:
self._activate(plugin) return self._activate(plugin)
else: else:
th = Thread(target=partial(self._activate, plugin)) th = Thread(target=partial(self._activate, plugin))
th.start() th.start()
......
import logging
logger = logging.getLogger(__name__)
MSG = 'GSITK is not (properly) installed.'
IMPORTMSG = '{} Some functions will be unavailable.'.format(MSG)
RUNMSG = '{} Install it to use this function.'.format(MSG)