Commit 908090f6 authored by J. Fernando Sánchez's avatar J. Fernando Sánchez
Browse files

Released v0.7

Bug-fixes and improvements:
* Closes #5
* Closes #1
* Adds Client (beta)
* Added several schemas
* Lighter string representation -> should avoid delays in the analysis
  with plugins that have 'heavy' attributes

Backwards-incompatible changes:
* Context in headers by default
* All schemas include a "@type" argument that is used for autodetection
  in the client

... And possibly many more, this is still <1.0
parent fbf03849
from python:2.7-slim
from python:2.7
WORKDIR /usr/src/app
ADD requirements.txt /usr/src/app/
from python:3.4-slim
from python:3.4
WORKDIR /usr/src/app
ADD requirements.txt /usr/src/app/
from python:3.5-slim
from python:3.5
WORKDIR /usr/src/app
ADD requirements.txt /usr/src/app/
from python:3.4
RUN pip install pytest
ADD requirements.txt /usr/src/app/
RUN pip install -r /usr/src/app/requirements.txt
from python:{{PYVERSION}}-slim
from python:{{PYVERSION}}
WORKDIR /usr/src/app
ADD requirements.txt /usr/src/app/
......@@ -4,6 +4,7 @@ NAME=senpy
VERSION=$(shell cat $(NAME)/VERSION)
TARNAME=$(NAME)-$(subst -,.,$(VERSION)).tar.gz
all: build run
......@@ -22,27 +23,24 @@ dockerfiles: $(addprefix Dockerfile-,$(PYVERSIONS))
Dockerfile-%: Dockerfile.template
sed "s/{{PYVERSION}}/$*/" Dockerfile.template > Dockerfile-$*
build: $(addprefix build-, $(PYMAIN))
quick_build: $(addprefix build-, $(PYMAIN))
buildall: $(addprefix build-, $(PYVERSIONS))
build: $(addprefix build-, $(PYVERSIONS))
build-%: Dockerfile-%
docker build -t '$(REPO)/$(NAME):$(VERSION)-python$*' -f Dockerfile-$* .;
docker build -t '$(IMAGENAME)-python$*' -f Dockerfile-$* .;
docker build -t '$(NAME)-debug' -f Dockerfile-debug-$* .;
quick_test: $(addprefix test-,$(PYMAIN))
test: $(addprefix test-,$(PYMAIN))
testall: $(addprefix test-,$(PYVERSIONS))
test: $(addprefix test-,$(PYVERSIONS))
docker run --rm -w /usr/src/app/ -v $$PWD:/usr/src/app --entrypoint=/bin/bash -ti $(NAME)-debug ;
(docker start $(NAME)-debug && docker attach $(NAME)-debug) || docker run -w /usr/src/app/ -v $$PWD:/usr/src/app --entrypoint=/bin/bash -ti --name $(NAME)-debug '$(IMAGENAME)-python$*'
debug: debug-$(PYMAIN)
test-%: build-%
docker run --rm -w /usr/src/app/ --entrypoint=/usr/local/bin/python -ti '$(REPO)/$(NAME):$(VERSION)-python$*' test --addopts "-vvv -s" ;
docker run --rm -w /usr/src/app/ --entrypoint=/usr/local/bin/python -ti '$(IMAGENAME)-python$*' test --addopts "-vvv -s" ;
docker run --rm -ti -v $$PWD:/usr/src/app/ -w /usr/src/app/ python:$(PYMAIN) python sdist;
......@@ -55,12 +53,13 @@ pip_test-%: sdist
pip_test: $(addprefix pip_test-,$(PYVERSIONS))
upload-%: test-%
docker push '$(REPO)/$(NAME):$(VERSION)-python$*'
docker push '$(IMAGENAME)-python$*'
upload: testall $(addprefix upload-,$(PYVERSIONS))
docker tag '$(REPO)/$(NAME):$(VERSION)-python$(PYMAIN)' '$(REPO)/$(NAME):$(VERSION)'
docker tag '$(REPO)/$(NAME):$(VERSION)-python$(PYMAIN)' '$(REPO)/$(NAME)'
docker push '$(REPO)/$(NAME):$(VERSION)'
upload: test $(addprefix upload-,$(PYVERSIONS))
docker tag '$(IMAGENAME)-python$(PYMAIN)' '$(IMAGENAME)'
docker tag '$(IMAGENAME)-python$(PYMAIN)' '$(REPO)/$(NAME)'
docker push '$(IMAGENAME)'
docker push '$(REPO)/$(NAME)'
@docker ps -a | awk '/$(REPO)\/$(NAME)/{ split($$2, vers, "-"); if(vers[1] != "${VERSION}"){ print $$1;}}' | xargs docker rm 2>/dev/null|| true
......@@ -78,6 +77,6 @@ pip_upload:
pip_test: $(addprefix pip_test-,$(PYVERSIONS))
run: build
docker run --rm -p 5000:5000 -ti '$(REPO)/$(NAME):$(VERSION)-python$(PYMAIN)'
docker run --rm -p 5000:5000 -ti '$(IMAGENAME)-python$(PYMAIN)'
.PHONY: test test-% build-% build test pip_test run yapf dev
import requests
import logging
from . import models
logger = logging.getLogger(__name__)
class Client(object):
def __init__(self, endpoint):
self.endpoint = endpoint
def analyse(self, input, method='GET', **kwargs):
return self.request('/', method=method, input=input, **kwargs)
def request(self, path=None, method='GET', **params):
url = '{}{}'.format(self.endpoint, path)
response = requests.request(method=method,
resp = models.from_dict(response.json())
return resp
except Exception as ex:
logger.error(('There seems to be a problem with the response:\n'
'\tURL: {url}\n'
'\tError: {error}\n'
'#### Response:\n'
'\tCode: {code}'
'\tContent: {content}'
raise ex
if __name__ == '__main__':
c = Client('')
resp = c.analyse('hello')
# print(resp)
......@@ -161,7 +161,7 @@ class Senpy(object):
self._set_active_plugin(plugin_name, success)
except Exception as ex:
msg = "Error activating plugin {} - {} : \n\t{}".format(, ex, ex.format_exc()), ex, traceback.format_exc())
raise Error(msg)
if sync:
......@@ -12,12 +12,15 @@ import time
import copy
import json
import os
import logging
import jsonref
import jsonschema
from flask import Response as FlaskResponse
import logging
logger = logging.getLogger(__name__)
DEFINITIONS_FILE = 'definitions.json'
CONTEXT_PATH = os.path.join(
os.path.dirname(os.path.realpath(__file__)), 'schemas', 'context.jsonld')
......@@ -40,7 +43,6 @@ def read_schema(schema_file, absolute=False):
base_schema = read_schema(DEFINITIONS_FILE)
class Context(dict):
......@@ -72,7 +74,7 @@ base_context = Context.load(CONTEXT_PATH)
class SenpyMixin(object):
context = base_context["@context"]
def flask(self, in_headers=False, headers=None, **kwargs):
def flask(self, in_headers=True, headers=None, **kwargs):
Return the values and error to be used in flask.
So far, it returns a fixed context. We should store/generate different
......@@ -151,14 +153,16 @@ class SenpyMixin(object):
return str(self.to_JSON())
class SenpyModel(SenpyMixin, dict):
class BaseModel(SenpyMixin, dict):
schema = base_schema
def __init__(self, *args, **kwargs): = kwargs.pop('id', '{}_{}'.format(
type(self).__name__, time.time()))
if 'id' in kwargs: = kwargs.pop('id')
elif kwargs.pop('_auto_id', True): = '_:{}_{}'.format(
type(self).__name__, time.time())
temp = dict(*args, **kwargs)
for obj in [self.schema, ] + self.schema.get('allOf', []):
......@@ -175,7 +179,11 @@ class SenpyModel(SenpyMixin, dict):
context = temp['context']
del temp['context']
self.__dict__['context'] = Context.load(context)
super(SenpyModel, self).__init__(temp)
temp['@type'] = getattr(self, '@type')
except AttributeError:
logger.warn('Creating an instance of an unknown model')
super(BaseModel, self).__init__(temp)
def _get_key(self, key):
key = key.replace("__", ":", 1)
......@@ -206,73 +214,80 @@ class SenpyModel(SenpyMixin, dict):
return d
class Response(SenpyModel):
schema = read_schema('response.json')
class Results(SenpyModel):
schema = read_schema('results.json')
class Entry(SenpyModel):
schema = read_schema('entry.json')
class Sentiment(SenpyModel):
schema = read_schema('sentiment.json')
class Analysis(SenpyModel):
schema = read_schema('analysis.json')
class EmotionSet(SenpyModel):
schema = read_schema('emotionSet.json')
class Emotion(SenpyModel):
schema = read_schema('emotion.json')
_subtypes = {}
class EmotionModel(SenpyModel):
schema = read_schema('emotionModel.json')
def register(rsubclass, rtype=None):
_subtypes[rtype or rsubclass.__name__] = rsubclass
class Suggestion(SenpyModel):
schema = read_schema('suggestion.json')
class PluginModel(SenpyModel):
schema = read_schema('plugin.json')
class EmotionPluginModel(SenpyModel):
schema = read_schema('plugin.json')
class SentimentPluginModel(SenpyModel):
schema = read_schema('plugin.json')
class Plugins(SenpyModel):
schema = read_schema('plugins.json')
def from_dict(indict):
target = indict.get('@type', None)
if target and target in _subtypes:
cls = _subtypes[target]
cls = BaseModel
return cls(**indict)
def from_schema(name, schema_file=None, base_classes=None):
base_classes = base_classes or []
schema_file = schema_file or '{}.json'.format(name)
class_name = '{}{}'.format(i[0].upper(), i[1:])
newclass = type(class_name, tuple(base_classes), {})
setattr(newclass, '@type', name)
setattr(newclass, 'schema', read_schema(schema_file))
register(newclass, name)
return newclass
def _add_from_schema(*args, **kwargs):
generatedClass = from_schema(*args, **kwargs)
globals()[generatedClass.__name__] = generatedClass
del generatedClass
for i in ['response',
_ErrorModel = from_schema('error')
class Error(SenpyMixin, BaseException):
def __init__(self,
super(Error, self).__init__(self, message, message)
self._error = _ErrorModel(message=message, *args, **kwargs)
self.message = message
self.status = status
self.params = params or {}
self.errors = errors or ""
def _plain_dict(self):
return self.__dict__
def __getattr__(self, key):
if key != '_error' and hasattr(self._error, key):
return getattr(self._error, key)
raise AttributeError(key)
def __str__(self):
return str(self.jsonld())
def __setattr__(self, key, value):
if key != '_error':
return setattr(self._error, key, value)
super(Error, self).__setattr__(key, value)
def __delattr__(self, key):
delattr(self._error, key)
register(Error, 'error')
......@@ -6,16 +6,16 @@ import os.path
import pickle
import logging
import tempfile
from .models import PluginModel, Error
from . import models
logger = logging.getLogger(__name__)
class SenpyPlugin(PluginModel):
class SenpyPlugin(models.Plugin):
def __init__(self, info=None):
if not info:
raise Error(message=("You need to provide configuration"
"information for the plugin."))
raise models.Error(message=("You need to provide configuration"
"information for the plugin."))
logger.debug("Initialising {}".format(info))
super(SenpyPlugin, self).__init__(info) = '{}_{}'.format(, self.version)
......@@ -40,7 +40,7 @@ class SenpyPlugin(PluginModel):
class SentimentPlugin(SenpyPlugin):
class SentimentPlugin(SenpyPlugin, models.SentimentPlugin):
def __init__(self, info, *args, **kwargs):
super(SentimentPlugin, self).__init__(info, *args, **kwargs)
self.minPolarityValue = float(info.get("minPolarityValue", 0))
......@@ -48,7 +48,7 @@ class SentimentPlugin(SenpyPlugin):
self["@type"] = "marl:SentimentAnalysis"
class EmotionPlugin(SenpyPlugin):
class EmotionPlugin(SentimentPlugin, models.EmotionPlugin):
def __init__(self, info, *args, **kwargs):
self.minEmotionValue = float(info.get("minEmotionValue", 0))
self.maxEmotionValue = float(info.get("maxEmotionValue", 0))
......@@ -71,10 +71,6 @@ class ShelfMixin(object):
del self.__dict__['_sh']
def __del__(self):
super(ShelfMixin, self).__del__()
def shelf_file(self):
if not hasattr(self, '_shelf_file') or not self._shelf_file:
......@@ -86,8 +82,7 @@ class ShelfMixin(object):
return self._shelf_file
def save(self):
logger.debug('closing pickle')
logger.debug('saving pickle')
if hasattr(self, '_sh') and self._sh is not None:
with open(self.shelf_file, 'wb') as f:
pickle.dump(self._sh, f)
del (self.__dict__['_sh'])
"$schema": "",
"description": "Senpy analysis",
"allOf": [{
"$ref": "atom.json"
"$schema": "",
"description": "Base schema for all Senpy objects",
"type": "object",
"properties": {
"@id": {
"type": "string"
"@type": {
"type": "string",
"description": "Type of the atom. e.g., 'onyx:EmotionAnalysis', 'nif:Entry'"
"required": ["@id", "@type"]
"$schema": "",
"type": "object",
"$allOf": [
"$ref": "plugin.json"
"properties": {
"onyx:usesEmotionModel": {
"type": "array",
"items": {
"$ref": "emotionModel.json"
......@@ -5,9 +5,6 @@
"@id": {
"type": "string"
"@type": {
"enum": [["nif:RFC5147String", "nif:Context"]]
"nif:isString": {
"description": "String contained in this Context",
"type": "string"
"$schema": "",
"description": "Base schema for all Senpy objects",
"type": "object",
"$allOf": [
{"$ref": "atom.json"},
"properties": {
"message": {
"type": "string"
"errors": {
"type": "list",
"items": {"type": "object"}
"code": {
"type": "int"
"required": ["message"]
......@@ -4,7 +4,12 @@
"required": ["@id", "extra_params"],
"properties": {
"@id": {
"type": "string"
"type": "string",
"description": "Unique identifier for the plugin, usually comprised of the name of the plugin and the version."
"name": {
"type": "string",
"description": "The name of the plugin, which will be used in the algorithm detection phase"
"extra_params": {
"type": "object",
"$schema": "",
"type": "object",
"$allOf": [
"$ref": "plugin.json"
"properties": {
"marl:minPolarityValue": {
"type": "number"
"marl:maxPolarityValue": {
"type": "number"
import logging
from functools import partial
from unittest.mock import patch
except ImportError:
from mock import patch
logger = logging.getLogger(__name__)
from unittest import TestCase
......@@ -11,9 +16,12 @@ from senpy.models import Error
class CLITest(TestCase):
def test_basic(self):
self.assertRaises(Error, partial(main_function, []))
res = main_function(['--input', 'test'])
assert 'entries' in res
res = main_function(['--input', 'test', '--algo', 'rand'])
assert 'entries' in res
assert 'analysis' in res
assert res['analysis'][0]['name'] == 'rand'
with patch('senpy.extensions.Senpy.analyse') as patched:
main_function(['--input', 'test'])
with patch('senpy.extensions.Senpy.analyse') as patched:
main_function(['--input', 'test', '--algo', 'rand'])
patched.assert_called_with(input='test', algo='rand')
from unittest import TestCase
from unittest.mock import patch
except ImportError:
from mock import patch
from senpy.client import Client
from senpy.models import Results, Error
class Call(dict):
def __init__(self, obj):
self.obj = obj.jsonld()
def json(self):
return self.obj
class ModelsTest(TestCase):
def setUp(self): = ''
self.port = 5000
def test_client(self):
endpoint = 'http://dummy/'
client = Client(endpoint)
success = Call(Results())
with patch('requests.request', return_value=success) as patched:
resp = client.analyse('hello')
assert isinstance(resp, Results)
patched.assert_called_with(url=endpoint + '/',
params={'input': 'hello'})
error = Call(Error('Nothing'))
with patch('requests.request', return_value=error) as patched:
resp = client.analyse(input='hello', algorithm='NONEXISTENT')
assert isinstance(resp, Error)
patched.assert_called_with(url=endpoint + '/',
params={'input': 'hello',
'algorithm': 'NONEXISTENT'})