Commit b4ca5f4a authored by J. Fernando Sánchez's avatar J. Fernando Sánchez
Browse files

Several fixes and changes

* Added interactive debugging
* Better exception logging
* More tests for errors
* Added ONBUILD to dockerfile
  Now creating new images based on senpy's is as easy as:
  ```from senpy:<version>```. This will automatically mount the code to
  /senpy-plugins and install all dependencies
* Added /data as a VOLUME
* Added `--use-wheel` to pip install both on the image and in the
  auto-install function.
* Closes #9

Break compatibilitity:

* Removed ability to (de)activate plugins through the web
parent 3311af21
...@@ -14,13 +14,20 @@ stages: ...@@ -14,13 +14,20 @@ stages:
.test: &test_definition .test: &test_definition
variables: variables:
PIP_CACHE_DIR: "$CI_PROJECT_DIR/.eggs" PIP_CACHE_DIR: "$CI_PROJECT_DIR/pip-cache"
cache: cache:
paths: paths:
- .eggs/ - .eggs/
- "$CI_PROJECT_DIR/pip-cache"
- .venv
key: "$CI_PROJECT_NAME" key: "$CI_PROJECT_NAME"
stage: test stage: test
script: script:
- pip install --use-wheel -U pip setuptools virtualenv
- virtualenv .venv/$PYTHON_VERSION
- source .venv/$PYTHON_VERSION/bin/activate
- pip install --use-wheel -r requirements.txt
- pip install --use-wheel -r test-requirements.txt
- python setup.py test - python setup.py test
test-3.5: test-3.5:
......
from python:2.7 from python:2.7
RUN mkdir /cache/
ENV PIP_CACHE_DIR=/cache/
WORKDIR /usr/src/app WORKDIR /usr/src/app
ADD requirements.txt /usr/src/app/ ADD requirements.txt /usr/src/app/
RUN pip install -r requirements.txt RUN pip install --use-wheel -r requirements.txt
ADD . /usr/src/app/ ADD . /usr/src/app/
RUN pip install . RUN pip install .
ENTRYPOINT ["python", "-m", "senpy", "-f", ".", "--host", "0.0.0.0"]
VOLUME /data/
RUN mkdir /senpy-plugins/
WORKDIR /senpy-plugins/
ONBUILD ADD . /senpy-plugins/
ONBUILD RUN python -m senpy --only-install -f /senpy-plugins
ENTRYPOINT ["python", "-m", "senpy", "-f", "/senpy-plugins/", "--host", "0.0.0.0"]
\ No newline at end of file
from python:3.4 from python:3.4-slim
WORKDIR /usr/src/app WORKDIR /usr/src/app
ADD requirements.txt /usr/src/app/ ADD requirements.txt /usr/src/app/
RUN pip install -r requirements.txt RUN pip install --use-wheel -r requirements.txt
ADD . /usr/src/app/ ADD . /usr/src/app/
RUN pip install . RUN pip install --use-wheel .
ENTRYPOINT ["python", "-m", "senpy", "-f", ".", "--host", "0.0.0.0"]
VOLUME /data/
RUN mkdir /senpy-plugins/
WORKDIR /senpy-plugins/
ONBUILD ADD . /senpy-plugins/
ONBUILD RUN python -m senpy -f /senpy-plugins
ENTRYPOINT ["python", "-m", "senpy", "-f", "/senpy-plugins/", "--host", "0.0.0.0"]
\ No newline at end of file
from python:3.5 from python:3.5
RUN mkdir /cache/
ENV PIP_CACHE_DIR=/cache/
WORKDIR /usr/src/app WORKDIR /usr/src/app
ADD requirements.txt /usr/src/app/ ADD requirements.txt /usr/src/app/
RUN pip install -r requirements.txt RUN pip install --use-wheel -r requirements.txt
ADD . /usr/src/app/ ADD . /usr/src/app/
RUN pip install . RUN pip install .
ENTRYPOINT ["python", "-m", "senpy", "-f", ".", "--host", "0.0.0.0"]
VOLUME /data/
RUN mkdir /senpy-plugins/
WORKDIR /senpy-plugins/
ONBUILD ADD . /senpy-plugins/
ONBUILD RUN python -m senpy --only-install -f /senpy-plugins
ENTRYPOINT ["python", "-m", "senpy", "-f", "/senpy-plugins/", "--host", "0.0.0.0"]
\ No newline at end of file
from python:{{PYVERSION}} from python:{{PYVERSION}}
RUN mkdir /cache/
ENV PIP_CACHE_DIR=/cache/
WORKDIR /usr/src/app WORKDIR /usr/src/app
ADD requirements.txt /usr/src/app/ ADD requirements.txt /usr/src/app/
RUN pip install -r requirements.txt RUN pip install --use-wheel -r requirements.txt
ADD . /usr/src/app/ ADD . /usr/src/app/
RUN pip install . RUN pip install .
ENTRYPOINT ["python", "-m", "senpy", "-f", ".", "--host", "0.0.0.0"]
VOLUME /data/
RUN mkdir /senpy-plugins/
WORKDIR /senpy-plugins/
ONBUILD ADD . /senpy-plugins/
ONBUILD RUN python -m senpy --only-install -f /senpy-plugins
ENTRYPOINT ["python", "-m", "senpy", "-f", "/senpy-plugins/", "--host", "0.0.0.0"]
\ No newline at end of file
...@@ -26,6 +26,7 @@ from gevent.wsgi import WSGIServer ...@@ -26,6 +26,7 @@ from gevent.wsgi import WSGIServer
from gevent.monkey import patch_all from gevent.monkey import patch_all
import logging import logging
import os import os
import sys
import argparse import argparse
import senpy import senpy
...@@ -34,6 +35,22 @@ patch_all(thread=False) ...@@ -34,6 +35,22 @@ patch_all(thread=False)
SERVER_PORT = os.environ.get("PORT", 5000) SERVER_PORT = os.environ.get("PORT", 5000)
def info(type, value, tb):
if hasattr(sys, 'ps1') or not sys.stderr.isatty():
# we are in interactive mode or we don't have a tty-like
# device, so we call the default hook
sys.__excepthook__(type, value, tb)
else:
import traceback
import pdb
# we are NOT in interactive mode, print the exception...
traceback.print_exception(type, value, tb)
print
# ...then start the debugger in post-mortem mode.
# pdb.pm() # deprecated
pdb.post_mortem(tb) # more "modern"
def main(): def main():
parser = argparse.ArgumentParser(description='Run a Senpy server') parser = argparse.ArgumentParser(description='Run a Senpy server')
parser.add_argument( parser.add_argument(
...@@ -84,6 +101,8 @@ def main(): ...@@ -84,6 +101,8 @@ def main():
rl.setLevel(getattr(logging, args.level)) rl.setLevel(getattr(logging, args.level))
app = Flask(__name__) app = Flask(__name__)
app.debug = args.debug app.debug = args.debug
if args.debug:
sys.excepthook = info
sp = Senpy(app, args.plugins_folder, default_plugins=args.default_plugins) sp = Senpy(app, args.plugins_folder, default_plugins=args.default_plugins)
if args.only_install: if args.only_install:
sp.install_deps() sp.install_deps()
......
...@@ -102,9 +102,8 @@ def plugins(): ...@@ -102,9 +102,8 @@ def plugins():
@api_blueprint.route('/plugins/<plugin>/', methods=['POST', 'GET']) @api_blueprint.route('/plugins/<plugin>/', methods=['POST', 'GET'])
@api_blueprint.route('/plugins/<plugin>/<action>', methods=['POST', 'GET'])
@basic_api @basic_api
def plugin(plugin=None, action="list"): def plugin(plugin=None):
sp = current_app.senpy sp = current_app.senpy
if plugin == 'default' and sp.default_plugin: if plugin == 'default' and sp.default_plugin:
response = sp.default_plugin response = sp.default_plugin
...@@ -113,11 +112,4 @@ def plugin(plugin=None, action="list"): ...@@ -113,11 +112,4 @@ def plugin(plugin=None, action="list"):
response = sp.plugins[plugin] response = sp.plugins[plugin]
else: else:
return Error(message="Plugin not found", status=404) return Error(message="Plugin not found", status=404)
if action == "list": return response
return response
method = "{}_plugin".format(action)
if (hasattr(sp, method)):
getattr(sp, method)(plugin)
return Response(message="Ok")
else:
return Error(message="action '{}' not allowed".format(action))
...@@ -35,11 +35,3 @@ class Client(object): ...@@ -35,11 +35,3 @@ class Client(object):
code=response.status_code, code=response.status_code,
content=response.content)) content=response.content))
raise ex raise ex
if __name__ == '__main__':
c = Client('http://senpy.cluster.gsi.dit.upm.es/api/')
resp = c.analyse('hello')
# print(resp)
print(resp.entries)
resp.validate()
...@@ -106,8 +106,12 @@ class Senpy(object): ...@@ -106,8 +106,12 @@ class Senpy(object):
resp = plug.analyse(**nif_params) resp = plug.analyse(**nif_params)
resp.analysis.append(plug) resp.analysis.append(plug)
logger.debug("Returning analysis result: {}".format(resp)) logger.debug("Returning analysis result: {}".format(resp))
except Error as ex:
logger.exception('Error returning analysis result')
resp = ex
except Exception as ex: except Exception as ex:
resp = Error(message=str(ex), status=500) resp = Error(message=str(ex), status=500)
logger.exception('Error returning analysis result')
return resp return resp
@property @property
...@@ -120,10 +124,6 @@ class Senpy(object): ...@@ -120,10 +124,6 @@ class Senpy(object):
else: else:
return None return None
def parameters(self, algo):
return getattr(
self.plugins.get(algo) or self.default_plugin, "extra_params", {})
def activate_all(self, sync=False): def activate_all(self, sync=False):
ps = [] ps = []
for plug in self.plugins.keys(): for plug in self.plugins.keys():
...@@ -194,17 +194,6 @@ class Senpy(object): ...@@ -194,17 +194,6 @@ class Senpy(object):
th = Thread(target=deact) th = Thread(target=deact)
th.start() th.start()
def reload_plugin(self, name):
logger.debug("Reloading {}".format(name))
plugin = self.plugins[name]
try:
del self.plugins[name]
nplug = self._load_plugin(plugin.module, plugin.path)
self.plugins[nplug.name] = nplug
except Exception as ex:
logger.error('Error reloading {}: {}'.format(name, ex))
self.plugins[name] = plugin
@classmethod @classmethod
def validate_info(cls, info): def validate_info(cls, info):
return all(x in info for x in ('name', 'module', 'version')) return all(x in info for x in ('name', 'module', 'version'))
...@@ -219,6 +208,7 @@ class Senpy(object): ...@@ -219,6 +208,7 @@ class Senpy(object):
if requirements: if requirements:
pip_args = [] pip_args = []
pip_args.append('install') pip_args.append('install')
pip_args.append('--use-wheel')
for req in requirements: for req in requirements:
pip_args.append(req) pip_args.append(req)
logger.info('Installing requirements: ' + str(requirements)) logger.info('Installing requirements: ' + str(requirements))
...@@ -233,32 +223,27 @@ class Senpy(object): ...@@ -233,32 +223,27 @@ class Senpy(object):
name = info["name"] name = info["name"]
sys.path.append(root) sys.path.append(root)
(fp, pathname, desc) = imp.find_module(module, [root, ]) (fp, pathname, desc) = imp.find_module(module, [root, ])
cls._install_deps(info)
tmp = imp.load_module(module, fp, pathname, desc)
sys.path.remove(root)
candidate = None
for _, obj in inspect.getmembers(tmp):
if inspect.isclass(obj) and inspect.getmodule(obj) == tmp:
logger.debug(("Found plugin class:"
" {}@{}").format(obj, inspect.getmodule(obj)))
candidate = obj
break
if not candidate:
logger.debug("No valid plugin for: {}".format(module))
return
module = candidate(info=info)
repo_path = root
try: try:
cls._install_deps(info)
tmp = imp.load_module(module, fp, pathname, desc)
sys.path.remove(root)
candidate = None
for _, obj in inspect.getmembers(tmp):
if inspect.isclass(obj) and inspect.getmodule(obj) == tmp:
logger.debug(("Found plugin class:"
" {}@{}").format(obj, inspect.getmodule(
obj)))
candidate = obj
break
if not candidate:
logger.debug("No valid plugin for: {}".format(module))
return
module = candidate(info=info)
repo_path = root
module._repo = Repo(repo_path) module._repo = Repo(repo_path)
except InvalidGitRepositoryError: except InvalidGitRepositoryError:
logger.debug("The plugin {} is not in a Git repository".format( logger.debug("The plugin {} is not in a Git repository".format(
module)) module))
module._repo = None module._repo = None
except Exception as ex:
logger.error("Exception importing {}: {}".format(module, ex))
logger.error("Trace: {}".format(traceback.format_exc()))
return None, None
return name, module return name, module
@classmethod @classmethod
......
...@@ -115,7 +115,7 @@ class SenpyMixin(object): ...@@ -115,7 +115,7 @@ class SenpyMixin(object):
return ser_or_down(self._plain_dict()) return ser_or_down(self._plain_dict())
def jsonld(self, with_context=True, context_uri=None): def jsonld(self, with_context=False, context_uri=None):
ser = self.serializable() ser = self.serializable()
if with_context: if with_context:
...@@ -230,6 +230,11 @@ def from_dict(indict): ...@@ -230,6 +230,11 @@ def from_dict(indict):
return cls(**indict) return cls(**indict)
def from_json(injson):
indict = json.loads(injson)
return from_dict(indict)
def from_schema(name, schema_file=None, base_classes=None): def from_schema(name, schema_file=None, base_classes=None):
base_classes = base_classes or [] base_classes = base_classes or []
base_classes.append(BaseModel) base_classes.append(BaseModel)
...@@ -275,6 +280,15 @@ class Error(SenpyMixin, BaseException): ...@@ -275,6 +280,15 @@ class Error(SenpyMixin, BaseException):
self._error = _ErrorModel(message=message, *args, **kwargs) self._error = _ErrorModel(message=message, *args, **kwargs)
self.message = message self.message = message
def __getitem__(self, key):
return self._error[key]
def __setitem__(self, key, value):
self._error[key] = value
def __delitem__(self, key):
del self._error[key]
def __getattr__(self, key): def __getattr__(self, key):
if key != '_error' and hasattr(self._error, key): if key != '_error' and hasattr(self._error, key):
return getattr(self._error, key) return getattr(self._error, key)
......
...@@ -50,6 +50,7 @@ class SentimentPlugin(SenpyPlugin, models.SentimentPlugin): ...@@ -50,6 +50,7 @@ class SentimentPlugin(SenpyPlugin, models.SentimentPlugin):
class EmotionPlugin(SentimentPlugin, models.EmotionPlugin): class EmotionPlugin(SentimentPlugin, models.EmotionPlugin):
def __init__(self, info, *args, **kwargs): def __init__(self, info, *args, **kwargs):
super(EmotionPlugin, self).__init__(info, *args, **kwargs)
self.minEmotionValue = float(info.get("minEmotionValue", 0)) self.minEmotionValue = float(info.get("minEmotionValue", 0))
self.maxEmotionValue = float(info.get("maxEmotionValue", 0)) self.maxEmotionValue = float(info.get("maxEmotionValue", 0))
self["@type"] = "onyx:EmotionAnalysis" self["@type"] = "onyx:EmotionAnalysis"
......
...@@ -13,7 +13,7 @@ ...@@ -13,7 +13,7 @@
"type": "list", "type": "list",
"items": {"type": "object"} "items": {"type": "object"}
}, },
"code": { "status": {
"type": "int" "type": "int"
}, },
"required": ["message"] "required": ["message"]
......
...@@ -12,8 +12,6 @@ except AttributeError: ...@@ -12,8 +12,6 @@ except AttributeError:
install_reqs = parse_requirements("requirements.txt") install_reqs = parse_requirements("requirements.txt")
test_reqs = parse_requirements("test-requirements.txt") test_reqs = parse_requirements("test-requirements.txt")
# reqs is a list of requirement
# e.g. ['django==1.5.1', 'mezzanine==1.4.6']
install_reqs = [str(ir.req) for ir in install_reqs] install_reqs = [str(ir.req) for ir in install_reqs]
test_reqs = [str(ir.req) for ir in test_reqs] test_reqs = [str(ir.req) for ir in test_reqs]
......
import os import os
import logging import logging
import json
from senpy.extensions import Senpy from senpy.extensions import Senpy
from senpy import models
from flask import Flask from flask import Flask
from unittest import TestCase from unittest import TestCase
from gevent import sleep
from itertools import product from itertools import product
...@@ -14,7 +13,7 @@ def check_dict(indic, template): ...@@ -14,7 +13,7 @@ def check_dict(indic, template):
def parse_resp(resp): def parse_resp(resp):
return json.loads(resp.data.decode('utf-8')) return models.from_json(resp.data.decode('utf-8'))
class BlueprintsTest(TestCase): class BlueprintsTest(TestCase):
...@@ -57,6 +56,17 @@ class BlueprintsTest(TestCase): ...@@ -57,6 +56,17 @@ class BlueprintsTest(TestCase):
assert "@context" in js assert "@context" in js
assert "entries" in js assert "entries" in js
def test_error(self):
"""
The dummy plugin returns an empty response,\
it should contain the context
"""
resp = self.client.get("/api/?i=My aloha mohame&algo=DOESNOTEXIST")
self.assertCode(resp, 404)
js = parse_resp(resp)
logging.debug("Got response: %s", js)
assert isinstance(js, models.Error)
def test_list(self): def test_list(self):
""" List the plugins """ """ List the plugins """
resp = self.client.get("/api/plugins/") resp = self.client.get("/api/plugins/")
...@@ -94,25 +104,6 @@ class BlueprintsTest(TestCase): ...@@ -94,25 +104,6 @@ class BlueprintsTest(TestCase):
assert "@id" in js assert "@id" in js
assert js["@id"] == "Dummy_0.1" assert js["@id"] == "Dummy_0.1"
def test_activate(self):
""" Activate and deactivate one plugin """
resp = self.client.get("/api/plugins/Dummy/deactivate")
self.assertCode(resp, 200)
sleep(0.5)
resp = self.client.get("/api/plugins/Dummy/")
self.assertCode(resp, 200)
js = parse_resp(resp)
assert "is_activated" in js
assert not js["is_activated"]
resp = self.client.get("/api/plugins/Dummy/activate")
self.assertCode(resp, 200)
sleep(0.5)
resp = self.client.get("/api/plugins/Dummy/")
self.assertCode(resp, 200)
js = parse_resp(resp)
assert "is_activated" in js
assert js["is_activated"]
def test_default(self): def test_default(self):
""" Show only one plugin""" """ Show only one plugin"""
resp = self.client.get("/api/plugins/default/") resp = self.client.get("/api/plugins/default/")
...@@ -121,11 +112,6 @@ class BlueprintsTest(TestCase): ...@@ -121,11 +112,6 @@ class BlueprintsTest(TestCase):
logging.debug(js) logging.debug(js)
assert "@id" in js assert "@id" in js
assert js["@id"] == "Dummy_0.1" assert js["@id"] == "Dummy_0.1"
resp = self.client.get("/api/plugins/Dummy/deactivate")
self.assertCode(resp, 200)
sleep(0.5)
resp = self.client.get("/api/plugins/default/")
self.assertCode(resp, 404)
def test_context(self): def test_context(self):
resp = self.client.get("/api/contexts/context.jsonld") resp = self.client.get("/api/contexts/context.jsonld")
......
...@@ -2,6 +2,11 @@ from __future__ import print_function ...@@ -2,6 +2,11 @@ from __future__ import print_function
import os import os
import logging import logging
try:
from unittest import mock
except ImportError:
import mock
from functools import partial from functools import partial
from senpy.extensions import Senpy from senpy.extensions import Senpy
from senpy.models import Error from senpy.models import Error
...@@ -13,8 +18,9 @@ class ExtensionsTest(TestCase): ...@@ -13,8 +18,9 @@ class ExtensionsTest(TestCase):
def setUp(self): def setUp(self):
self.app = Flask("test_extensions") self.app = Flask("test_extensions")
self.dir = os.path.join(os.path.dirname(__file__)) self.dir = os.path.join(os.path.dirname(__file__))
self.senpy = Senpy(plugin_folder=self.dir, default_plugins=False) self.senpy = Senpy(plugin_folder=self.dir,
self.senpy.init_app(self.app) app=self.app,
default_plugins=False)
self.senpy.activate_plugin("Dummy", sync=True) self.senpy.activate_plugin("Dummy", sync=True)
def test_init(self): def test_init(self):
...@@ -69,7 +75,11 @@ class ExtensionsTest(TestCase): ...@@ -69,7 +75,11 @@ class ExtensionsTest(TestCase):
def test_noplugin(self): def test_noplugin(self):
""" Don't analyse if there isn't any plugin installed """ """ Don't analyse if there isn't any plugin installed """
self.senpy.deactivate_all(sync=True) self.senpy.deactivate_all(sync=True)
self.assertRaises(Error, partial(self.senpy.analyse, input="tupni"))