Commit 908090f6 authored by J. Fernando Sánchez's avatar J. Fernando Sánchez
Browse files

Released v0.7

Bug-fixes and improvements:
* Closes #5
* Closes #1
* Adds Client (beta)
* Added several schemas
* Lighter string representation -> should avoid delays in the analysis
  with plugins that have 'heavy' attributes

Backwards-incompatible changes:
* Context in headers by default
* All schemas include a "@type" argument that is used for autodetection
  in the client

... And possibly many more, this is still <1.0
parent fbf03849
from python:2.7-slim from python:2.7
WORKDIR /usr/src/app WORKDIR /usr/src/app
ADD requirements.txt /usr/src/app/ ADD requirements.txt /usr/src/app/
......
from python:3.4-slim from python:3.4
WORKDIR /usr/src/app WORKDIR /usr/src/app
ADD requirements.txt /usr/src/app/ ADD requirements.txt /usr/src/app/
......
from python:3.5-slim from python:3.5
WORKDIR /usr/src/app WORKDIR /usr/src/app
ADD requirements.txt /usr/src/app/ ADD requirements.txt /usr/src/app/
......
from python:3.4
RUN pip install pytest
ADD requirements.txt /usr/src/app/
RUN pip install -r /usr/src/app/requirements.txt
from python:{{PYVERSION}}-slim from python:{{PYVERSION}}
WORKDIR /usr/src/app WORKDIR /usr/src/app
ADD requirements.txt /usr/src/app/ ADD requirements.txt /usr/src/app/
......
...@@ -4,6 +4,7 @@ NAME=senpy ...@@ -4,6 +4,7 @@ NAME=senpy
REPO=gsiupm REPO=gsiupm
VERSION=$(shell cat $(NAME)/VERSION) VERSION=$(shell cat $(NAME)/VERSION)
TARNAME=$(NAME)-$(subst -,.,$(VERSION)).tar.gz TARNAME=$(NAME)-$(subst -,.,$(VERSION)).tar.gz
IMAGENAME=$(REPO)/$(NAME):$(VERSION)
all: build run all: build run
...@@ -22,27 +23,24 @@ dockerfiles: $(addprefix Dockerfile-,$(PYVERSIONS)) ...@@ -22,27 +23,24 @@ dockerfiles: $(addprefix Dockerfile-,$(PYVERSIONS))
Dockerfile-%: Dockerfile.template Dockerfile-%: Dockerfile.template
sed "s/{{PYVERSION}}/$*/" Dockerfile.template > Dockerfile-$* sed "s/{{PYVERSION}}/$*/" Dockerfile.template > Dockerfile-$*
build: $(addprefix build-, $(PYMAIN)) quick_build: $(addprefix build-, $(PYMAIN))
buildall: $(addprefix build-, $(PYVERSIONS)) build: $(addprefix build-, $(PYVERSIONS))
build-%: Dockerfile-% build-%: Dockerfile-%
docker build -t '$(REPO)/$(NAME):$(VERSION)-python$*' -f Dockerfile-$* .; docker build -t '$(IMAGENAME)-python$*' -f Dockerfile-$* .;
build-debug-%: quick_test: $(addprefix test-,$(PYMAIN))
docker build -t '$(NAME)-debug' -f Dockerfile-debug-$* .;
test: $(addprefix test-,$(PYMAIN)) test: $(addprefix test-,$(PYVERSIONS))
testall: $(addprefix test-,$(PYVERSIONS))
debug-%: debug-%:
docker run --rm -w /usr/src/app/ -v $$PWD:/usr/src/app --entrypoint=/bin/bash -ti $(NAME)-debug ; (docker start $(NAME)-debug && docker attach $(NAME)-debug) || docker run -w /usr/src/app/ -v $$PWD:/usr/src/app --entrypoint=/bin/bash -ti --name $(NAME)-debug '$(IMAGENAME)-python$*'
debug: debug-$(PYMAIN) debug: debug-$(PYMAIN)
test-%: build-% test-%: build-%
docker run --rm -w /usr/src/app/ --entrypoint=/usr/local/bin/python -ti '$(REPO)/$(NAME):$(VERSION)-python$*' setup.py test --addopts "-vvv -s" ; docker run --rm -w /usr/src/app/ --entrypoint=/usr/local/bin/python -ti '$(IMAGENAME)-python$*' setup.py test --addopts "-vvv -s" ;
dist/$(TARNAME): dist/$(TARNAME):
docker run --rm -ti -v $$PWD:/usr/src/app/ -w /usr/src/app/ python:$(PYMAIN) python setup.py sdist; docker run --rm -ti -v $$PWD:/usr/src/app/ -w /usr/src/app/ python:$(PYMAIN) python setup.py sdist;
...@@ -55,12 +53,13 @@ pip_test-%: sdist ...@@ -55,12 +53,13 @@ pip_test-%: sdist
pip_test: $(addprefix pip_test-,$(PYVERSIONS)) pip_test: $(addprefix pip_test-,$(PYVERSIONS))
upload-%: test-% upload-%: test-%
docker push '$(REPO)/$(NAME):$(VERSION)-python$*' docker push '$(IMAGENAME)-python$*'
upload: testall $(addprefix upload-,$(PYVERSIONS)) upload: test $(addprefix upload-,$(PYVERSIONS))
docker tag '$(REPO)/$(NAME):$(VERSION)-python$(PYMAIN)' '$(REPO)/$(NAME):$(VERSION)' docker tag '$(IMAGENAME)-python$(PYMAIN)' '$(IMAGENAME)'
docker tag '$(REPO)/$(NAME):$(VERSION)-python$(PYMAIN)' '$(REPO)/$(NAME)' docker tag '$(IMAGENAME)-python$(PYMAIN)' '$(REPO)/$(NAME)'
docker push '$(REPO)/$(NAME):$(VERSION)' docker push '$(IMAGENAME)'
docker push '$(REPO)/$(NAME)'
clean: clean:
@docker ps -a | awk '/$(REPO)\/$(NAME)/{ split($$2, vers, "-"); if(vers[1] != "${VERSION}"){ print $$1;}}' | xargs docker rm 2>/dev/null|| true @docker ps -a | awk '/$(REPO)\/$(NAME)/{ split($$2, vers, "-"); if(vers[1] != "${VERSION}"){ print $$1;}}' | xargs docker rm 2>/dev/null|| true
...@@ -78,6 +77,6 @@ pip_upload: ...@@ -78,6 +77,6 @@ pip_upload:
pip_test: $(addprefix pip_test-,$(PYVERSIONS)) pip_test: $(addprefix pip_test-,$(PYVERSIONS))
run: build run: build
docker run --rm -p 5000:5000 -ti '$(REPO)/$(NAME):$(VERSION)-python$(PYMAIN)' docker run --rm -p 5000:5000 -ti '$(IMAGENAME)-python$(PYMAIN)'
.PHONY: test test-% build-% build test pip_test run yapf dev .PHONY: test test-% build-% build test pip_test run yapf dev
import requests
import logging
from . import models
logger = logging.getLogger(__name__)
class Client(object):
def __init__(self, endpoint):
self.endpoint = endpoint
def analyse(self, input, method='GET', **kwargs):
return self.request('/', method=method, input=input, **kwargs)
def request(self, path=None, method='GET', **params):
url = '{}{}'.format(self.endpoint, path)
response = requests.request(method=method,
url=url,
params=params)
try:
resp = models.from_dict(response.json())
resp.validate(resp)
return resp
except Exception as ex:
logger.error(('There seems to be a problem with the response:\n'
'\tURL: {url}\n'
'\tError: {error}\n'
'\t\n'
'#### Response:\n'
'\tCode: {code}'
'\tContent: {content}'
'\n').format(error=ex,
url=url,
code=response.status_code,
content=response.content))
raise ex
if __name__ == '__main__':
c = Client('http://senpy.cluster.gsi.dit.upm.es/api/')
resp = c.analyse('hello')
# print(resp)
print(resp.entries)
resp.validate()
...@@ -161,7 +161,7 @@ class Senpy(object): ...@@ -161,7 +161,7 @@ class Senpy(object):
self._set_active_plugin(plugin_name, success) self._set_active_plugin(plugin_name, success)
except Exception as ex: except Exception as ex:
msg = "Error activating plugin {} - {} : \n\t{}".format( msg = "Error activating plugin {} - {} : \n\t{}".format(
plugin.name, ex, ex.format_exc()) plugin.name, ex, traceback.format_exc())
logger.error(msg) logger.error(msg)
raise Error(msg) raise Error(msg)
if sync: if sync:
......
...@@ -12,12 +12,15 @@ import time ...@@ -12,12 +12,15 @@ import time
import copy import copy
import json import json
import os import os
import logging
import jsonref import jsonref
import jsonschema import jsonschema
from flask import Response as FlaskResponse from flask import Response as FlaskResponse
import logging
logger = logging.getLogger(__name__)
DEFINITIONS_FILE = 'definitions.json' DEFINITIONS_FILE = 'definitions.json'
CONTEXT_PATH = os.path.join( CONTEXT_PATH = os.path.join(
os.path.dirname(os.path.realpath(__file__)), 'schemas', 'context.jsonld') os.path.dirname(os.path.realpath(__file__)), 'schemas', 'context.jsonld')
...@@ -40,7 +43,6 @@ def read_schema(schema_file, absolute=False): ...@@ -40,7 +43,6 @@ def read_schema(schema_file, absolute=False):
base_schema = read_schema(DEFINITIONS_FILE) base_schema = read_schema(DEFINITIONS_FILE)
logging.debug(base_schema)
class Context(dict): class Context(dict):
...@@ -72,7 +74,7 @@ base_context = Context.load(CONTEXT_PATH) ...@@ -72,7 +74,7 @@ base_context = Context.load(CONTEXT_PATH)
class SenpyMixin(object): class SenpyMixin(object):
context = base_context["@context"] context = base_context["@context"]
def flask(self, in_headers=False, headers=None, **kwargs): def flask(self, in_headers=True, headers=None, **kwargs):
""" """
Return the values and error to be used in flask. Return the values and error to be used in flask.
So far, it returns a fixed context. We should store/generate different So far, it returns a fixed context. We should store/generate different
...@@ -151,14 +153,16 @@ class SenpyMixin(object): ...@@ -151,14 +153,16 @@ class SenpyMixin(object):
return str(self.to_JSON()) return str(self.to_JSON())
class SenpyModel(SenpyMixin, dict): class BaseModel(SenpyMixin, dict):
schema = base_schema schema = base_schema
def __init__(self, *args, **kwargs): def __init__(self, *args, **kwargs):
self.id = kwargs.pop('id', '{}_{}'.format( if 'id' in kwargs:
type(self).__name__, time.time())) self.id = kwargs.pop('id')
elif kwargs.pop('_auto_id', True):
self.id = '_:{}_{}'.format(
type(self).__name__, time.time())
temp = dict(*args, **kwargs) temp = dict(*args, **kwargs)
for obj in [self.schema, ] + self.schema.get('allOf', []): for obj in [self.schema, ] + self.schema.get('allOf', []):
...@@ -175,7 +179,11 @@ class SenpyModel(SenpyMixin, dict): ...@@ -175,7 +179,11 @@ class SenpyModel(SenpyMixin, dict):
context = temp['context'] context = temp['context']
del temp['context'] del temp['context']
self.__dict__['context'] = Context.load(context) self.__dict__['context'] = Context.load(context)
super(SenpyModel, self).__init__(temp) try:
temp['@type'] = getattr(self, '@type')
except AttributeError:
logger.warn('Creating an instance of an unknown model')
super(BaseModel, self).__init__(temp)
def _get_key(self, key): def _get_key(self, key):
key = key.replace("__", ":", 1) key = key.replace("__", ":", 1)
...@@ -206,73 +214,80 @@ class SenpyModel(SenpyMixin, dict): ...@@ -206,73 +214,80 @@ class SenpyModel(SenpyMixin, dict):
return d return d
class Response(SenpyModel): _subtypes = {}
schema = read_schema('response.json')
class Results(SenpyModel):
schema = read_schema('results.json')
class Entry(SenpyModel):
schema = read_schema('entry.json')
class Sentiment(SenpyModel):
schema = read_schema('sentiment.json')
class Analysis(SenpyModel):
schema = read_schema('analysis.json')
class EmotionSet(SenpyModel):
schema = read_schema('emotionSet.json')
class Emotion(SenpyModel):
schema = read_schema('emotion.json')
class EmotionModel(SenpyModel): def register(rsubclass, rtype=None):
schema = read_schema('emotionModel.json') _subtypes[rtype or rsubclass.__name__] = rsubclass
class Suggestion(SenpyModel): def from_dict(indict):
schema = read_schema('suggestion.json') target = indict.get('@type', None)
if target and target in _subtypes:
cls = _subtypes[target]
class PluginModel(SenpyModel): else:
schema = read_schema('plugin.json') cls = BaseModel
return cls(**indict)
class EmotionPluginModel(SenpyModel):
schema = read_schema('plugin.json') def from_schema(name, schema_file=None, base_classes=None):
base_classes = base_classes or []
base_classes.append(BaseModel)
class SentimentPluginModel(SenpyModel): schema_file = schema_file or '{}.json'.format(name)
schema = read_schema('plugin.json') class_name = '{}{}'.format(i[0].upper(), i[1:])
newclass = type(class_name, tuple(base_classes), {})
setattr(newclass, '@type', name)
class Plugins(SenpyModel): setattr(newclass, 'schema', read_schema(schema_file))
schema = read_schema('plugins.json') register(newclass, name)
return newclass
def _add_from_schema(*args, **kwargs):
generatedClass = from_schema(*args, **kwargs)
globals()[generatedClass.__name__] = generatedClass
del generatedClass
for i in ['response',
'results',
'entry',
'sentiment',
'analysis',
'emotionSet',
'emotion',
'emotionModel',
'suggestion',
'plugin',
'emotionPlugin',
'sentimentPlugin',
'plugins']:
_add_from_schema(i)
_ErrorModel = from_schema('error')
class Error(SenpyMixin, BaseException): class Error(SenpyMixin, BaseException):
def __init__(self, def __init__(self,
message, message,
status=500,
params=None,
errors=None,
*args, *args,
**kwargs): **kwargs):
super(Error, self).__init__(self, message, message)
self._error = _ErrorModel(message=message, *args, **kwargs)
self.message = message self.message = message
self.status = status
self.params = params or {}
self.errors = errors or ""
def _plain_dict(self): def __getattr__(self, key):
return self.__dict__ if key != '_error' and hasattr(self._error, key):
return getattr(self._error, key)
raise AttributeError(key)
def __str__(self): def __setattr__(self, key, value):
return str(self.jsonld()) if key != '_error':
return setattr(self._error, key, value)
else:
super(Error, self).__setattr__(key, value)
def __delattr__(self, key):
delattr(self._error, key)
register(Error, 'error')
...@@ -6,16 +6,16 @@ import os.path ...@@ -6,16 +6,16 @@ import os.path
import pickle import pickle
import logging import logging
import tempfile import tempfile
from .models import PluginModel, Error from . import models
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
class SenpyPlugin(PluginModel): class SenpyPlugin(models.Plugin):
def __init__(self, info=None): def __init__(self, info=None):
if not info: if not info:
raise Error(message=("You need to provide configuration" raise models.Error(message=("You need to provide configuration"
"information for the plugin.")) "information for the plugin."))
logger.debug("Initialising {}".format(info)) logger.debug("Initialising {}".format(info))
super(SenpyPlugin, self).__init__(info) super(SenpyPlugin, self).__init__(info)
self.id = '{}_{}'.format(self.name, self.version) self.id = '{}_{}'.format(self.name, self.version)
...@@ -40,7 +40,7 @@ class SenpyPlugin(PluginModel): ...@@ -40,7 +40,7 @@ class SenpyPlugin(PluginModel):
self.deactivate() self.deactivate()
class SentimentPlugin(SenpyPlugin): class SentimentPlugin(SenpyPlugin, models.SentimentPlugin):
def __init__(self, info, *args, **kwargs): def __init__(self, info, *args, **kwargs):
super(SentimentPlugin, self).__init__(info, *args, **kwargs) super(SentimentPlugin, self).__init__(info, *args, **kwargs)
self.minPolarityValue = float(info.get("minPolarityValue", 0)) self.minPolarityValue = float(info.get("minPolarityValue", 0))
...@@ -48,7 +48,7 @@ class SentimentPlugin(SenpyPlugin): ...@@ -48,7 +48,7 @@ class SentimentPlugin(SenpyPlugin):
self["@type"] = "marl:SentimentAnalysis" self["@type"] = "marl:SentimentAnalysis"
class EmotionPlugin(SenpyPlugin): class EmotionPlugin(SentimentPlugin, models.EmotionPlugin):
def __init__(self, info, *args, **kwargs): def __init__(self, info, *args, **kwargs):
self.minEmotionValue = float(info.get("minEmotionValue", 0)) self.minEmotionValue = float(info.get("minEmotionValue", 0))
self.maxEmotionValue = float(info.get("maxEmotionValue", 0)) self.maxEmotionValue = float(info.get("maxEmotionValue", 0))
...@@ -71,10 +71,6 @@ class ShelfMixin(object): ...@@ -71,10 +71,6 @@ class ShelfMixin(object):
del self.__dict__['_sh'] del self.__dict__['_sh']
self.save() self.save()
def __del__(self):
self.save()
super(ShelfMixin, self).__del__()
@property @property
def shelf_file(self): def shelf_file(self):
if not hasattr(self, '_shelf_file') or not self._shelf_file: if not hasattr(self, '_shelf_file') or not self._shelf_file:
...@@ -86,8 +82,7 @@ class ShelfMixin(object): ...@@ -86,8 +82,7 @@ class ShelfMixin(object):
return self._shelf_file return self._shelf_file
def save(self): def save(self):
logger.debug('closing pickle') logger.debug('saving pickle')
if hasattr(self, '_sh') and self._sh is not None: if hasattr(self, '_sh') and self._sh is not None:
with open(self.shelf_file, 'wb') as f: with open(self.shelf_file, 'wb') as f:
pickle.dump(self._sh, f) pickle.dump(self._sh, f)
del (self.__dict__['_sh'])
{
"$schema": "http://json-schema.org/draft-04/schema#",
"description": "Senpy analysis",
"allOf": [{
"$ref": "atom.json"
}]
}
{
"$schema": "http://json-schema.org/draft-04/schema#",
"description": "Base schema for all Senpy objects",
"type": "object",
"properties": {
"@id": {
"type": "string"
},
"@type": {
"type": "string",
"description": "Type of the atom. e.g., 'onyx:EmotionAnalysis', 'nif:Entry'"
}
},
"required": ["@id", "@type"]
}
{
"$schema": "http://json-schema.org/draft-04/schema#",
"type": "object",
"$allOf": [
{
"$ref": "plugin.json"
},
{
"properties": {
"onyx:usesEmotionModel": {
"type": "array",
"items": {
"$ref": "emotionModel.json"
}
}
}
}
]
}
...@@ -5,9 +5,6 @@ ...@@ -5,9 +5,6 @@
"@id": { "@id": {
"type": "string" "type": "string"
}, },
"@type": {
"enum": [["nif:RFC5147String", "nif:Context"]]
},
"nif:isString": { "nif:isString": {
"description": "String contained in this Context", "description": "String contained in this Context",
"type": "string" "type": "string"
......
{
"$schema": "http://json-schema.org/draft-04/schema#",
"description": "Base schema for all Senpy objects",
"type": "object",
"$allOf": [
{"$ref": "atom.json"},
{
"properties": {
"message": {
"type": "string"
},
"errors": {
"type": "list",
"items": {"type": "object"}
},
"code": {
"type": "int"
},
"required": ["message"]
}
}
]
}
...@@ -4,7 +4,12 @@ ...@@ -4,7 +4,12 @@