Commit b4ca5f4a authored by J. Fernando Sánchez's avatar J. Fernando Sánchez

Several fixes and changes

* Added interactive debugging
* Better exception logging
* More tests for errors
* Added ONBUILD to dockerfile
  Now creating new images based on senpy's is as easy as:
  ```from senpy:<version>```. This will automatically mount the code to
  /senpy-plugins and install all dependencies
* Added /data as a VOLUME
* Added `--use-wheel` to pip install both on the image and in the
  auto-install function.
* Closes #9

Break compatibilitity:

* Removed ability to (de)activate plugins through the web
parent 3311af21
......@@ -14,13 +14,20 @@ stages:
.test: &test_definition
variables:
PIP_CACHE_DIR: "$CI_PROJECT_DIR/.eggs"
PIP_CACHE_DIR: "$CI_PROJECT_DIR/pip-cache"
cache:
paths:
- .eggs/
- "$CI_PROJECT_DIR/pip-cache"
- .venv
key: "$CI_PROJECT_NAME"
stage: test
script:
- pip install --use-wheel -U pip setuptools virtualenv
- virtualenv .venv/$PYTHON_VERSION
- source .venv/$PYTHON_VERSION/bin/activate
- pip install --use-wheel -r requirements.txt
- pip install --use-wheel -r test-requirements.txt
- python setup.py test
test-3.5:
......
from python:2.7
RUN mkdir /cache/
ENV PIP_CACHE_DIR=/cache/
WORKDIR /usr/src/app
ADD requirements.txt /usr/src/app/
RUN pip install -r requirements.txt
RUN pip install --use-wheel -r requirements.txt
ADD . /usr/src/app/
RUN pip install .
ENTRYPOINT ["python", "-m", "senpy", "-f", ".", "--host", "0.0.0.0"]
VOLUME /data/
RUN mkdir /senpy-plugins/
WORKDIR /senpy-plugins/
ONBUILD ADD . /senpy-plugins/
ONBUILD RUN python -m senpy --only-install -f /senpy-plugins
ENTRYPOINT ["python", "-m", "senpy", "-f", "/senpy-plugins/", "--host", "0.0.0.0"]
\ No newline at end of file
from python:3.4
from python:3.4-slim
WORKDIR /usr/src/app
ADD requirements.txt /usr/src/app/
RUN pip install -r requirements.txt
RUN pip install --use-wheel -r requirements.txt
ADD . /usr/src/app/
RUN pip install .
RUN pip install --use-wheel .
ENTRYPOINT ["python", "-m", "senpy", "-f", ".", "--host", "0.0.0.0"]
VOLUME /data/
RUN mkdir /senpy-plugins/
WORKDIR /senpy-plugins/
ONBUILD ADD . /senpy-plugins/
ONBUILD RUN python -m senpy -f /senpy-plugins
ENTRYPOINT ["python", "-m", "senpy", "-f", "/senpy-plugins/", "--host", "0.0.0.0"]
\ No newline at end of file
from python:3.5
RUN mkdir /cache/
ENV PIP_CACHE_DIR=/cache/
WORKDIR /usr/src/app
ADD requirements.txt /usr/src/app/
RUN pip install -r requirements.txt
RUN pip install --use-wheel -r requirements.txt
ADD . /usr/src/app/
RUN pip install .
ENTRYPOINT ["python", "-m", "senpy", "-f", ".", "--host", "0.0.0.0"]
VOLUME /data/
RUN mkdir /senpy-plugins/
WORKDIR /senpy-plugins/
ONBUILD ADD . /senpy-plugins/
ONBUILD RUN python -m senpy --only-install -f /senpy-plugins
ENTRYPOINT ["python", "-m", "senpy", "-f", "/senpy-plugins/", "--host", "0.0.0.0"]
\ No newline at end of file
from python:{{PYVERSION}}
RUN mkdir /cache/
ENV PIP_CACHE_DIR=/cache/
WORKDIR /usr/src/app
ADD requirements.txt /usr/src/app/
RUN pip install -r requirements.txt
RUN pip install --use-wheel -r requirements.txt
ADD . /usr/src/app/
RUN pip install .
ENTRYPOINT ["python", "-m", "senpy", "-f", ".", "--host", "0.0.0.0"]
VOLUME /data/
RUN mkdir /senpy-plugins/
WORKDIR /senpy-plugins/
ONBUILD ADD . /senpy-plugins/
ONBUILD RUN python -m senpy --only-install -f /senpy-plugins
ENTRYPOINT ["python", "-m", "senpy", "-f", "/senpy-plugins/", "--host", "0.0.0.0"]
\ No newline at end of file
......@@ -26,6 +26,7 @@ from gevent.wsgi import WSGIServer
from gevent.monkey import patch_all
import logging
import os
import sys
import argparse
import senpy
......@@ -34,6 +35,22 @@ patch_all(thread=False)
SERVER_PORT = os.environ.get("PORT", 5000)
def info(type, value, tb):
if hasattr(sys, 'ps1') or not sys.stderr.isatty():
# we are in interactive mode or we don't have a tty-like
# device, so we call the default hook
sys.__excepthook__(type, value, tb)
else:
import traceback
import pdb
# we are NOT in interactive mode, print the exception...
traceback.print_exception(type, value, tb)
print
# ...then start the debugger in post-mortem mode.
# pdb.pm() # deprecated
pdb.post_mortem(tb) # more "modern"
def main():
parser = argparse.ArgumentParser(description='Run a Senpy server')
parser.add_argument(
......@@ -84,6 +101,8 @@ def main():
rl.setLevel(getattr(logging, args.level))
app = Flask(__name__)
app.debug = args.debug
if args.debug:
sys.excepthook = info
sp = Senpy(app, args.plugins_folder, default_plugins=args.default_plugins)
if args.only_install:
sp.install_deps()
......
......@@ -102,9 +102,8 @@ def plugins():
@api_blueprint.route('/plugins/<plugin>/', methods=['POST', 'GET'])
@api_blueprint.route('/plugins/<plugin>/<action>', methods=['POST', 'GET'])
@basic_api
def plugin(plugin=None, action="list"):
def plugin(plugin=None):
sp = current_app.senpy
if plugin == 'default' and sp.default_plugin:
response = sp.default_plugin
......@@ -113,11 +112,4 @@ def plugin(plugin=None, action="list"):
response = sp.plugins[plugin]
else:
return Error(message="Plugin not found", status=404)
if action == "list":
return response
method = "{}_plugin".format(action)
if (hasattr(sp, method)):
getattr(sp, method)(plugin)
return Response(message="Ok")
else:
return Error(message="action '{}' not allowed".format(action))
return response
......@@ -35,11 +35,3 @@ class Client(object):
code=response.status_code,
content=response.content))
raise ex
if __name__ == '__main__':
c = Client('http://senpy.cluster.gsi.dit.upm.es/api/')
resp = c.analyse('hello')
# print(resp)
print(resp.entries)
resp.validate()
......@@ -106,8 +106,12 @@ class Senpy(object):
resp = plug.analyse(**nif_params)
resp.analysis.append(plug)
logger.debug("Returning analysis result: {}".format(resp))
except Error as ex:
logger.exception('Error returning analysis result')
resp = ex
except Exception as ex:
resp = Error(message=str(ex), status=500)
logger.exception('Error returning analysis result')
return resp
@property
......@@ -120,10 +124,6 @@ class Senpy(object):
else:
return None
def parameters(self, algo):
return getattr(
self.plugins.get(algo) or self.default_plugin, "extra_params", {})
def activate_all(self, sync=False):
ps = []
for plug in self.plugins.keys():
......@@ -194,17 +194,6 @@ class Senpy(object):
th = Thread(target=deact)
th.start()
def reload_plugin(self, name):
logger.debug("Reloading {}".format(name))
plugin = self.plugins[name]
try:
del self.plugins[name]
nplug = self._load_plugin(plugin.module, plugin.path)
self.plugins[nplug.name] = nplug
except Exception as ex:
logger.error('Error reloading {}: {}'.format(name, ex))
self.plugins[name] = plugin
@classmethod
def validate_info(cls, info):
return all(x in info for x in ('name', 'module', 'version'))
......@@ -219,6 +208,7 @@ class Senpy(object):
if requirements:
pip_args = []
pip_args.append('install')
pip_args.append('--use-wheel')
for req in requirements:
pip_args.append(req)
logger.info('Installing requirements: ' + str(requirements))
......@@ -233,32 +223,27 @@ class Senpy(object):
name = info["name"]
sys.path.append(root)
(fp, pathname, desc) = imp.find_module(module, [root, ])
cls._install_deps(info)
tmp = imp.load_module(module, fp, pathname, desc)
sys.path.remove(root)
candidate = None
for _, obj in inspect.getmembers(tmp):
if inspect.isclass(obj) and inspect.getmodule(obj) == tmp:
logger.debug(("Found plugin class:"
" {}@{}").format(obj, inspect.getmodule(obj)))
candidate = obj
break
if not candidate:
logger.debug("No valid plugin for: {}".format(module))
return
module = candidate(info=info)
repo_path = root
try:
cls._install_deps(info)
tmp = imp.load_module(module, fp, pathname, desc)
sys.path.remove(root)
candidate = None
for _, obj in inspect.getmembers(tmp):
if inspect.isclass(obj) and inspect.getmodule(obj) == tmp:
logger.debug(("Found plugin class:"
" {}@{}").format(obj, inspect.getmodule(
obj)))
candidate = obj
break
if not candidate:
logger.debug("No valid plugin for: {}".format(module))
return
module = candidate(info=info)
repo_path = root
module._repo = Repo(repo_path)
except InvalidGitRepositoryError:
logger.debug("The plugin {} is not in a Git repository".format(
module))
module._repo = None
except Exception as ex:
logger.error("Exception importing {}: {}".format(module, ex))
logger.error("Trace: {}".format(traceback.format_exc()))
return None, None
return name, module
@classmethod
......
......@@ -115,7 +115,7 @@ class SenpyMixin(object):
return ser_or_down(self._plain_dict())
def jsonld(self, with_context=True, context_uri=None):
def jsonld(self, with_context=False, context_uri=None):
ser = self.serializable()
if with_context:
......@@ -230,6 +230,11 @@ def from_dict(indict):
return cls(**indict)
def from_json(injson):
indict = json.loads(injson)
return from_dict(indict)
def from_schema(name, schema_file=None, base_classes=None):
base_classes = base_classes or []
base_classes.append(BaseModel)
......@@ -275,6 +280,15 @@ class Error(SenpyMixin, BaseException):
self._error = _ErrorModel(message=message, *args, **kwargs)
self.message = message
def __getitem__(self, key):
return self._error[key]
def __setitem__(self, key, value):
self._error[key] = value
def __delitem__(self, key):
del self._error[key]
def __getattr__(self, key):
if key != '_error' and hasattr(self._error, key):
return getattr(self._error, key)
......
......@@ -50,6 +50,7 @@ class SentimentPlugin(SenpyPlugin, models.SentimentPlugin):
class EmotionPlugin(SentimentPlugin, models.EmotionPlugin):
def __init__(self, info, *args, **kwargs):
super(EmotionPlugin, self).__init__(info, *args, **kwargs)
self.minEmotionValue = float(info.get("minEmotionValue", 0))
self.maxEmotionValue = float(info.get("maxEmotionValue", 0))
self["@type"] = "onyx:EmotionAnalysis"
......
......@@ -13,7 +13,7 @@
"type": "list",
"items": {"type": "object"}
},
"code": {
"status": {
"type": "int"
},
"required": ["message"]
......
......@@ -12,8 +12,6 @@ except AttributeError:
install_reqs = parse_requirements("requirements.txt")
test_reqs = parse_requirements("test-requirements.txt")
# reqs is a list of requirement
# e.g. ['django==1.5.1', 'mezzanine==1.4.6']
install_reqs = [str(ir.req) for ir in install_reqs]
test_reqs = [str(ir.req) for ir in test_reqs]
......
import os
import logging
import json
from senpy.extensions import Senpy
from senpy import models
from flask import Flask
from unittest import TestCase
from gevent import sleep
from itertools import product
......@@ -14,7 +13,7 @@ def check_dict(indic, template):
def parse_resp(resp):
return json.loads(resp.data.decode('utf-8'))
return models.from_json(resp.data.decode('utf-8'))
class BlueprintsTest(TestCase):
......@@ -57,6 +56,17 @@ class BlueprintsTest(TestCase):
assert "@context" in js
assert "entries" in js
def test_error(self):
"""
The dummy plugin returns an empty response,\
it should contain the context
"""
resp = self.client.get("/api/?i=My aloha mohame&algo=DOESNOTEXIST")
self.assertCode(resp, 404)
js = parse_resp(resp)
logging.debug("Got response: %s", js)
assert isinstance(js, models.Error)
def test_list(self):
""" List the plugins """
resp = self.client.get("/api/plugins/")
......@@ -94,25 +104,6 @@ class BlueprintsTest(TestCase):
assert "@id" in js
assert js["@id"] == "Dummy_0.1"
def test_activate(self):
""" Activate and deactivate one plugin """
resp = self.client.get("/api/plugins/Dummy/deactivate")
self.assertCode(resp, 200)
sleep(0.5)
resp = self.client.get("/api/plugins/Dummy/")
self.assertCode(resp, 200)
js = parse_resp(resp)
assert "is_activated" in js
assert not js["is_activated"]
resp = self.client.get("/api/plugins/Dummy/activate")
self.assertCode(resp, 200)
sleep(0.5)
resp = self.client.get("/api/plugins/Dummy/")
self.assertCode(resp, 200)
js = parse_resp(resp)
assert "is_activated" in js
assert js["is_activated"]
def test_default(self):
""" Show only one plugin"""
resp = self.client.get("/api/plugins/default/")
......@@ -121,11 +112,6 @@ class BlueprintsTest(TestCase):
logging.debug(js)
assert "@id" in js
assert js["@id"] == "Dummy_0.1"
resp = self.client.get("/api/plugins/Dummy/deactivate")
self.assertCode(resp, 200)
sleep(0.5)
resp = self.client.get("/api/plugins/default/")
self.assertCode(resp, 404)
def test_context(self):
resp = self.client.get("/api/contexts/context.jsonld")
......
......@@ -2,6 +2,11 @@ from __future__ import print_function
import os
import logging
try:
from unittest import mock
except ImportError:
import mock
from functools import partial
from senpy.extensions import Senpy
from senpy.models import Error
......@@ -13,8 +18,9 @@ class ExtensionsTest(TestCase):
def setUp(self):
self.app = Flask("test_extensions")
self.dir = os.path.join(os.path.dirname(__file__))
self.senpy = Senpy(plugin_folder=self.dir, default_plugins=False)
self.senpy.init_app(self.app)
self.senpy = Senpy(plugin_folder=self.dir,
app=self.app,
default_plugins=False)
self.senpy.activate_plugin("Dummy", sync=True)
def test_init(self):
......@@ -69,7 +75,11 @@ class ExtensionsTest(TestCase):
def test_noplugin(self):
""" Don't analyse if there isn't any plugin installed """
self.senpy.deactivate_all(sync=True)
self.assertRaises(Error, partial(self.senpy.analyse, input="tupni"))
self.assertRaises(Error, partial(self.senpy.analyse,
input="tupni"))
self.assertRaises(Error, partial(self.senpy.analyse,
input="tupni",
algorithm='Dummy'))
def test_analyse(self):
""" Using a plugin """
......@@ -81,6 +91,18 @@ class ExtensionsTest(TestCase):
assert r1.analysis[0].id[:5] == "Dummy"
assert r2.analysis[0].id[:5] == "Dummy"
def test_analyse_error(self):
mm = mock.MagicMock()
mm.analyse.side_effect = Error('error on analysis', status=900)
self.senpy.plugins['MOCK'] = mm
resp = self.senpy.analyse(input='nothing', algorithm='MOCK')
assert resp['message'] == 'error on analysis'
assert resp['status'] == 900
mm.analyse.side_effect = Exception('generic exception on analysis')
resp = self.senpy.analyse(input='nothing', algorithm='MOCK')
assert resp['message'] == 'generic exception on analysis'
assert resp['status'] == 500
def test_filtering(self):
""" Filtering plugins """
assert len(self.senpy.filter_plugins(name="Dummy")) > 0
......@@ -90,3 +112,7 @@ class ExtensionsTest(TestCase):
assert not len(
self.senpy.filter_plugins(
name="Dummy", is_activated=True))
def test_load_default_plugins(self):
senpy = Senpy(plugin_folder=self.dir, default_plugins=True)
assert len(senpy.plugins) > 1
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment