Python 3 compatible

There are also some slight changes to the JSON schemas and the use of
JSON-LD.

.gitignore

@@ -3,3 +3,4 @@
 *egg-info
 dist
 README.html
+__pycache__
\ No newline at end of file

requirements.txt

@@ -2,6 +2,10 @@ Flask>=0.10.1
 gunicorn>=19.0.0
 requests>=2.4.1
 GitPython>=0.3.2.RC1
-gevent>=1.0.1
+gevent>=1.1rc4
 PyLD>=0.6.5
 Flask-Testing>=0.4.2
+six
+future
+jsonschema
+jsonref

senpy/blueprints.py

@@ -17,8 +17,8 @@
 """
 Blueprints for Senpy
 """
-from flask import Blueprint, request, current_app, Flask, redirect, url_for, render_template
-from .models import Error, Response, Leaf
+from flask import Blueprint, request, current_app, render_template
+from .models import Error, Response
 from future.utils import iteritems
 
 import json
@@ -27,6 +27,7 @@ import logging
 logger = logging.getLogger(__name__)
 
 nif_blueprint = Blueprint("NIF Sentiment Analysis Server", __name__)
+demo_blueprint = Blueprint("Demo of the service. It includes an HTML+Javascript playground to test senpy", __name__)
 
 BASIC_PARAMS = {
     "algorithm": {
@@ -40,15 +41,6 @@ BASIC_PARAMS = {
     }
 }
 
-LIST_PARAMS = {
-    "params": {
-        "aliases": ["params", "with_params"],
-        "required": False,
-        "default": "0"
-    },
-}
-
-
 def get_params(req, params=BASIC_PARAMS):
     if req.method == 'POST':
         indict = req.form
@@ -76,12 +68,11 @@ def get_params(req, params=BASIC_PARAMS):
                    outdict[param] not in params[param]["options"]:
                     wrong_params[param] = params[param]
     if wrong_params:
-        message = Error({"status": 404,
-                         "message": "Missing or invalid parameters",
-                         "parameters": outdict,
-                         "errors": {param: error for param, error in
-                                    iteritems(wrong_params)}
-                         })
+        message = Error(status=404,
+                        message="Missing or invalid parameters",
+                        parameters=outdict,
+                        errors={param: error for param, error in
+                                iteritems(wrong_params)})
         raise Error(message=message)
     return outdict
 
@@ -107,12 +98,12 @@ def basic_analysis(params):
     return response
 
 
-@nif_blueprint.route('/')
+@demo_blueprint.route('/')
 def index():
     return render_template("index.html")
 
 
-@nif_blueprint.route('/api', methods=['POST', 'GET'])
+@nif_blueprint.route('/', methods=['POST', 'GET'])
 def api():
     try:
         params = get_params(request)
@@ -128,7 +119,7 @@ def api():
         return ex.message.flask()
 
 
-@nif_blueprint.route("/api/default")
+@nif_blueprint.route("/default")
 def default():
     # return current_app.senpy.default_plugin
     plug = current_app.senpy.default_plugin
@@ -139,9 +130,9 @@ def default():
         return error.flask()
 
 
-@nif_blueprint.route('/api/plugins/', methods=['POST', 'GET'])
-@nif_blueprint.route('/api/plugins/<plugin>', methods=['POST', 'GET'])
-@nif_blueprint.route('/api/plugins/<plugin>/<action>', methods=['POST', 'GET'])
+@nif_blueprint.route('/plugins/', methods=['POST', 'GET'])
+@nif_blueprint.route('/plugins/<plugin>/', methods=['POST', 'GET'])
+@nif_blueprint.route('/plugins/<plugin>/<action>', methods=['POST', 'GET'])
 def plugins(plugin=None, action="list"):
     filt = {}
     sp = current_app.senpy
@@ -151,21 +142,19 @@ def plugins(plugin=None, action="list"):
     if plugin and not plugs:
         return "Plugin not found", 400
     if action == "list":
-        with_params = get_params(request, LIST_PARAMS)["params"] == "1"
         in_headers = get_params(request, BASIC_PARAMS)["inHeaders"] != "0"
         if plugin:
             dic = plugs[plugin]
         else:
             dic = Response(
-                    {plug: plugs[plug].jsonld(with_params) for plug in plugs},
-                    frame={})
+                    {plug: plugs[plug].serializable() for plug in plugs})
         return dic.flask(in_headers=in_headers)
     method = "{}_plugin".format(action)
     if(hasattr(sp, method)):
         getattr(sp, method)(plugin)
-        return Leaf(message="Ok").flask()
+        return Response(message="Ok").flask()
     else:
-        return Error("action '{}' not allowed".format(action)).flask()
+        return Error(message="action '{}' not allowed".format(action)).flask()
 
 
 if __name__ == '__main__':

senpy/context.jsonld

-{
-    "dc": "http://purl.org/dc/terms/",
-    "dc:subject": {
-        "@type": "@id"
-    },
-    "xsd": "http://www.w3.org/2001/XMLSchema#",
-    "marl": "http://www.gsi.dit.upm.es/ontologies/marl/ns#",
-    "nif": "http://persistence.uni-leipzig.org/nlp2rdf/ontologies/nif-core#",
-    "onyx": "http://www.gsi.dit.upm.es/ontologies/onyx/ns#",
-    "emotions": {
-        "@container": "@set",
-        "@id": "onyx:hasEmotionSet"
-    },
-    "opinions": {
-        "@container": "@set",
-        "@id": "marl:hasOpinion"
-    },
-    "prov": "http://www.w3.org/ns/prov#",
-    "rdfs": "http://www.w3.org/2000/01/rdf-schema#",
-    "analysis": {
-        "@container": "@set",
-        "@id": "prov:wasInformedBy"
-    },
-    "entries": {
-        "@container": "@set",
-        "@id": "prov:generated"
-    },
-    "strings": {
-        "@container": "@set",
-        "@reverse": "nif:hasContext"
-    },
-    "date":
-    {
-        "@id": "dc:date",
-        "@type": "xsd:dateTime"
-    },
-    "text": { "@id": "nif:isString" },
-    "wnaffect": "http://www.gsi.dit.upm.es/ontologies/wnaffect#",
-    "xsd": "http://www.w3.org/2001/XMLSchema#",
-    "senpy": "http://www.gsi.dit.upm.es/ontologies/senpy/ns#",
-    "@vocab":  "http://www.gsi.dit.upm.es/ontologies/senpy/ns#"
-}

senpy/extensions.py

@@ -8,7 +8,7 @@ monkey.patch_all()
 
 from .plugins import SenpyPlugin, SentimentPlugin, EmotionPlugin
 from .models import Error
-from .blueprints import nif_blueprint
+from .blueprints import nif_blueprint, demo_blueprint
 
 from git import Repo, InvalidGitRepositoryError
 from functools import partial
@@ -57,7 +57,8 @@ class Senpy(object):
             app.teardown_appcontext(self.teardown)
         else:
             app.teardown_request(self.teardown)
-        app.register_blueprint(nif_blueprint)
+        app.register_blueprint(nif_blueprint, url_prefix="/api")
+        app.register_blueprint(demo_blueprint, url_prefix="/")
 
     def add_folder(self, folder):
         logger.debug("Adding folder: %s", folder)

senpy/models.py

+'''
+Senpy Models. 
+
+This implementation should mirror the JSON schema definition.
+For compatibility with Py3 and for easier debugging, this new version drops introspection
+and adds all arguments to the models.
+'''
 from __future__ import print_function
 from six import string_types
 
+import time
+import copy
 import json
 import os
 import logging
+import jsonref
+import jsonschema
 
-from collections import defaultdict
-from pyld import jsonld
 from flask import Response as FlaskResponse
 
 
-class Response(object):
+DEFINITIONS_FILE = 'definitions.json'
+CONTEXT_PATH = os.path.join(os.path.dirname(os.path.realpath(__file__)), 'schemas', 'context.jsonld')
 
-    @property
-    def context(self):
-        if not hasattr(self, '_context'):
-            self._context = None
-        return self._context
+def get_schema_path(schema_file, absolute=False):
+    if absolute:
+        return os.path.realpath(schema_file)
+    else:
+        return os.path.join(os.path.dirname(os.path.realpath(__file__)), 'schemas', schema_file)
+
+
+def read_schema(schema_file, absolute=False):
+    schema_path = get_schema_path(schema_file, absolute)
+    schema_uri = 'file://{}'.format(schema_path)
+    return jsonref.load(open(schema_path), base_uri=schema_uri)
+        
+
+base_schema = read_schema(DEFINITIONS_FILE)
+logging.debug(base_schema)
+
+class Context(dict):
 
     @staticmethod
-    def get_context(context):
-        if isinstance(context, list):
+    def load(context):
+        logging.debug('Loading context: {}'.format(context))
+        if not context:
+            return context
+        elif isinstance(context, list):
             contexts = []
             for c in context:
-                contexts.append(Response.get_context(c))
+                contexts.append(Context.load(c))
             return contexts
         elif isinstance(context, dict):
-            return context
+            return Context(context)
         elif isinstance(context, string_types):
             try:
                 with open(context) as f:
-                    return json.loads(f.read())
+                    return Context(json.loads(f.read()))
             except IOError:
                 return context
         else:
             raise AttributeError('Please, provide a valid context')                
 
-    def jsonld(self, frame=None, options=None,
-               context=None, removeContext=None):
-        if removeContext is None:
-            removeContext = Response._context  # Loop?
-        if frame is None:
-            frame = self._frame
-        if context is None:
-            context = self.context
-        else:
-            context = self.get_context(context)
-        # For some reason, this causes errors with pyld
-        # if options is None:
-            # options = {"expandContext": context.copy() }
-        js = self
-        if frame:
-            logging.debug("Framing: %s", json.dumps(self, indent=4))
-            logging.debug("Framing with %s", json.dumps(frame, indent=4))
-            js = jsonld.frame(js, frame, options)
-            logging.debug("Result: %s", json.dumps(js, indent=4))
-            logging.debug("Compacting with %s", json.dumps(context, indent=4))
-            js = jsonld.compact(js, context, options)
-            logging.debug("Result: %s", json.dumps(js, indent=4))
-        if removeContext == context:
-            del js["@context"]
-        return js
+base_context = Context.load(CONTEXT_PATH)
+
 
-    def to_JSON(self, removeContext=None):
-        return json.dumps(self.jsonld(removeContext=removeContext),
-                          default=lambda o: o.__dict__,
-                          sort_keys=True, indent=4)
+class SenpyMixin(object):
+    context = base_context
 
     def flask(self,
               in_headers=False,
@@ -73,46 +73,166 @@ class Response(object):
         """
         Return the values and error to be used in flask
         """
-        js = self.jsonld()
         headers = None
         if in_headers:
-            ctx = js["@context"]
             headers = {
                 "Link": ('<%s>;'
                          'rel="http://www.w3.org/ns/json-ld#context";'
                          ' type="application/ld+json"' % url)
             }
-            del js["@context"]
-        return FlaskResponse(json.dumps(js, indent=4),
-                             status=self.get("status", 200),
+        return FlaskResponse(self.to_JSON(with_context=not in_headers),
+                             status=getattr(self, "status", 200),
                              headers=headers,
                              mimetype="application/json")
-    
-class Entry(JSONLD):
-    pass
 
 
-class Sentiment(JSONLD):
-    pass
+    def serializable(self):
+        def ser_or_down(item):
+           if hasattr(item, 'serializable'):
+               return item.serializable()
+           elif isinstance(item, dict):
+               temp = dict()
+               for kp in item:
+                   vp = item[kp]
+                   temp[kp] = ser_or_down(vp)
+               return temp
+           elif isinstance(item, list):
+               return list(ser_or_down(i) for i in item)
+           else:
+               return item
+        return ser_or_down(self._plain_dict())
 
 
-class EmotionSet(JSONLD):
-    pass
+    def jsonld(self, context=None, with_context=False):
+        ser = self.serializable()
 
+        if  with_context:
+            ser["@context"] = self.context
 
-class Emotion(JSONLD):
-    pass
+        return ser
 
 
-class Suggestion(JSONLD):
-    pass
+    def to_JSON(self, *args, **kwargs):
+        js = json.dumps(self.jsonld(*args, **kwargs), indent=4,
+                        sort_keys=True)
+        return js
+
+    
+class SenpyModel(SenpyMixin, dict):
 
-class Error(BaseException, JSONLD):
-    # A better pattern would be this:
-    # htp://flask.pocoo.org/docs/0.10/patterns/apierrors/
-    _frame = {}
-    _context = {}
+    schema = base_schema
+    prefix = None
 
     def __init__(self, *args, **kwargs):
-        self.message = kwargs.get('message', None)
-        super(Error, self).__init__(*args)
+        temp = dict(*args, **kwargs)
+
+        reqs = self.schema.get('required', [])
+        for i in reqs:
+            if i not in temp:
+                prop = self.schema['properties'][i]
+                if 'default' in prop:
+                    temp[i] = copy.deepcopy(prop['default'])
+        if 'context' in temp:
+            context = temp['context']
+            del temp['context']
+            self.__dict__['context'] = Context.load(context)
+        super(SenpyModel, self).__init__(temp)
+
+
+    def _get_key(self, key):
+        key = key.replace("__", ":", 1)
+        return key
+
+    def __setitem__(self, key, value):
+        dict.__setitem__(self, key, value)
+
+
+    def __delitem__(self, key):
+        dict.__delitem__(self, key)
+
+    def __getattr__(self, key):
+        try:
+            return self.__getitem__(self._get_key(key))
+        except KeyError:
+            raise AttributeError(key)
+
+    def __setattr__(self, key, value):
+        self.__setitem__(self._get_key(key), value)
+
+    def __delattr__(self, key):
+        self.__delitem__(self._get_key(key))
+
+
+    def validate(self, obj=None):
+        if not obj:
+            obj = self
+        if hasattr(obj, "jsonld"):
+            obj = obj.jsonld()
+        jsonschema.validate(obj, self.schema)
+        
+    
+    @classmethod
+    def from_base(cls, name):
+        subschema = base_schema[name]
+        return warlock.model_factory(subschema, base_class=cls)
+
+    def _plain_dict(self):
+        d =  { k: v for (k,v) in self.items() if k[0] != "_"}
+        if hasattr(self, "id"):
+            d["@id"] = self.id
+        return d
+
+    @property
+    def id(self):
+        if not hasattr(self, '_id'):
+            self.__dict__["_id"] = '_:{}_{}'.format(type(self).__name__, time.time())
+        return self._id
+
+    @id.setter
+    def id(self, value):
+        self._id = value
+    
+
+class Response(SenpyModel):
+    schema = read_schema('response.json')
+
+class Results(SenpyModel):
+    schema = read_schema('results.json')
+
+    def jsonld(self, context=None, with_context=True):
+        return super(Results, self).jsonld(context, with_context)
+
+class Entry(SenpyModel):
+    schema = read_schema('entry.json')
+
+class Sentiment(SenpyModel):
+    schema = read_schema('sentiment.json')
+
+class Analysis(SenpyModel):
+    schema = read_schema('analysis.json')
+
+class EmotionSet(SenpyModel):
+    schema = read_schema('emotionSet.json')
+
+class Suggestion(SenpyModel):
+    schema = read_schema('suggestion.json')
+
+class PluginModel(SenpyModel):
+    schema = read_schema('plugin.json')
+
+class Plugins(SenpyModel):
+    schema = read_schema('plugins.json')
+
+class Error(SenpyMixin, BaseException ):
+
+    def __init__(self, message, status=500, params=None, errors=None, *args, **kwargs):
+        self.message = message
+        self.status = status
+        self.params = params or {}
+        self.errors = errors or ""
+
+    def _plain_dict(self):
+       return self.__dict__
+
+    def __str__(self):
+       return str(self.jsonld()) 

senpy/plugins.py

@@ -5,7 +5,7 @@ import inspect
 import os.path
 import shelve
 import logging
-from .models import Response, Leaf
+from .models import Response, PluginModel, Error
 
 logger = logging.getLogger(__name__)
 
@@ -58,36 +58,21 @@ PARAMS = {
 }
 
 
-class SenpyPlugin(Leaf):
-    _context = Leaf.get_context(Response._context)
-    _frame = {"@context": _context,
-              "name": {},
-              "extra_params": {"@container": "@index"},
-              "@explicit": True,
-              "version": {},
-              "repo": None,
-              "is_activated": {},
-              "params": None,
-              }
+class SenpyPlugin(PluginModel):
 
     def __init__(self, info=None):
         if not info:
             raise Error(message=("You need to provide configuration"
                                  "information for the plugin."))
         logger.debug("Initialising {}".format(info))
-        super(SenpyPlugin, self).__init__()
         self.name = info["name"]
         self.version = info["version"]
-        self.id = "{}_{}".format(self.name, self.version)
         self.params = info.get("params", PARAMS.copy())
         if "@id" not in self.params:
             self.params["@id"] = "params_%s" % self.id
-        self.extra_params = info.get("extra_params", {})
-        self.params.update(self.extra_params.copy())
-        if "@id" not in self.extra_params:
-            self.extra_params["@id"] = "extra_params_%s" % self.id
         self.is_activated = False
         self._info = info
+        super(SenpyPlugin, self).__init__()
 
     def get_folder(self):
         return os.path.dirname(inspect.getfile(self.__class__))
@@ -102,13 +87,6 @@ class SenpyPlugin(Leaf):
     def deactivate(self):
         pass
 
-    def jsonld(self, parameters=False, *args, **kwargs):
-        nframe = kwargs.pop("frame", self._frame)
-        if parameters:
-            nframe = nframe.copy()
-            nframe["params"] = {}
-        return super(SenpyPlugin, self).jsonld(frame=nframe, *args, **kwargs)
-
     @property
     def id(self):
         return "{}_{}".format(self.name, self.version)
@@ -123,6 +101,7 @@ class SentimentPlugin(SenpyPlugin):
         super(SentimentPlugin, self).__init__(info, *args, **kwargs)
         self.minPolarityValue = float(info.get("minPolarityValue", 0))
         self.maxPolarityValue = float(info.get("maxPolarityValue", 1))
+        self["@type"] = "marl:SentimentAnalysis"
 
 
 class EmotionPlugin(SenpyPlugin):
@@ -131,6 +110,7 @@ class EmotionPlugin(SenpyPlugin):
         resp = super(EmotionPlugin, self).__init__(info, *args, **kwargs)
         self.minEmotionValue = float(info.get("minEmotionValue", 0))
         self.maxEmotionValue = float(info.get("maxEmotionValue", 0))
+        self["@type"] = "onyx:EmotionAnalysis"
 
 
 class ShelfMixin(object):
@@ -145,6 +125,11 @@ class ShelfMixin(object):
     def sh(self):
         if os.path.isfile(self.shelf_file):
             os.remove(self.shelf_file)
+        self.close()
+
+    def __del__(self):
+        self.close()        
+        self.deactivate()
 
     @property
     def shelf_file(self):

senpy/plugins/rand/rand.py

@@ -2,7 +2,7 @@ import json
 import random
 
 from senpy.plugins import SentimentPlugin
-from senpy.models import Response, Opinion, Entry
+from senpy.models import Results, Sentiment, Entry
 
 
 class Sentiment140Plugin(SentimentPlugin):
@@ -10,22 +10,33 @@ class Sentiment140Plugin(SentimentPlugin):
         lang = params.get("language", "auto")
 
         p = params.get("prefix", None)
-        response = Response(prefix=p)
+        response = Results(prefix=p)
         polarity_value = max(-1, min(1, random.gauss(0.2, 0.2)))
         polarity = "marl:Neutral"
         if polarity_value > 0:
             polarity = "marl:Positive"
         elif polarity_value < 0:
             polarity = "marl:Negative"
-        entry = Entry(id="Entry0",
-                      text=params["input"],
-                      prefix=p)
-        opinion = Opinion(id="Opinion0",
-                          prefix=p,
-                          hasPolarity=polarity,
-                          polarityValue=polarity_value)
-        opinion["prov:wasGeneratedBy"] = self.id
-        entry.opinions</