added fuseki and luigi cron analysis

f078fdcf · Alberto Pascual · 275b8696 · f078fdcf · f078fdcf · f078fdcf
Commit f078fdcf authored Sep 22, 2017 by Alberto Pascual
--- a/.gitignore
+++ b/.gitignore
@@ -3,3 +3,12 @@
 *~
 bower_components
 elasticsearch
+/bower_components
+/luigi/luigi.cfg
+/luigi/__pycache__/
+*.pyc
+/elasticsearch/nodes/0/
+*.db
+.*
+/fuseki/*
+
--- a/Dockerfile
+++ b/Dockerfile
@@ -11,6 +11,11 @@ RUN npm install -g http-server bower

 WORKDIR /usr/src/app/

+RUN mkdir -p /etc/ssl/certs
+COPY lab.cluster.gsi.dit.upm.es.crt /usr/local/share/ca-certificates/
+COPY lab.cluster.gsi.dit.upm.es.pem /usr/local/share/ca-certificates/
+RUN update-ca-certificates
+
 ADD bower.json /usr/src/app/bower.json

 RUN bower link --allow-root

--- a/bower.json
+++ b/bower.json
@@ -31,7 +31,9 @@
    "paper-material": "paper-material#*",
    "yasgui-polymer": "yasgui-polymer#*",
    "paper-dialog":"paper-dialog#1.1.0",
-    "iron-pages": "PolymerElements/iron-pages#1.0.2"
+    "iron-pages": "PolymerElements/iron-pages#1.0.2",
+    "paper-item": "paper-item#1.0.2",
+    "paper-menu": "paper-menu#1.0.1"
  },
  "resolutions": {
    "webcomponentsjs": "0.7.24",

--- a/dashboard-reddit.html
+++ b/dashboard-reddit.html
@@ -8,10 +8,9 @@
 <link rel="import" href="/bower_components/comment-chart/comment-chart.html">
 <link rel="import" href="/bower_components/paper-tabs/paper-tabs.html">
 <link rel="import" href="/bower_components/yasgui-polymer/yasgui.html">
-<link rel = "import" href = "/bower_components/iron-pages/iron-pages.html">
-
-
-
+<link rel="import" href="/bower_components/iron-pages/iron-pages.html">
+<link rel="import" href="/bower_components/paper-item/paper-item.html">
+<link rel="import" href="/bower_components/paper-menu/paper-menu.html">


 <dom-module id="dashboard-reddit">
@@ -127,14 +126,11 @@
        </div>
        <div>
          <div class="chart_container" style="left: 0; float: left; width: 100%">
-            <comment-chart 
-				      datos = {{data}}
+            <news-chart 
+				      datos = "{{data}}"
 			        filters="{{filters}}"
-              topic= "about"
-              text= "text"
-              score= "datePublished"
              icon='social:people'>
-            </comment-chart>
+            </news-chart>
          </div>
          <!--<div class="chart_container" style="left: 0; float: right">
           <google-chart
@@ -155,7 +151,7 @@
    </div>
    <div>
        <yasgui-ui
-            endpoint="http://krusti.gsi:13030/gsicrawler/query"
+            endpoint="http://homer.gsi:13030/gsicrawler/query"
            queries="{{queries}}">
        </yasgui-ui>
    </div>

--- a/news-chart @ bcdae15b
+++ b/news-chart @ bcdae15b
+Subproject commit bcdae15bd6ee3320db2ef9ba55cce400b8521afd
--- a/demo/imports.html
+++ b/demo/imports.html
-<link rel="import" href="bower_components/polymer/polymer.html">
+<link rel="import" href="/bower_components/polymer/polymer.html">

-<link rel="import" href="bower_components/elastic-client/elastic-client.html">
-<link rel="import" href="bower_components/dashboard-reddit/dashboard-reddit.html">
+<link rel="import" href="/bower_components/elastic-client/elastic-client.html">
+<link rel="import" href="/bower_components/dashboard-reddit/dashboard-reddit.html">
+<link rel="import" href="/elements/news-chart/news-chart.html"
--- a/demo/index.html
+++ b/demo/index.html
@@ -18,7 +18,7 @@
    <template is="dom-bind">

    <elastic-client 
-      config='{"host": ""}' 
+      config='{"host": "localhost:19200"}' 
      client="{{client}}"
      cluster-status="{{myStatus}}">
    </elastic-client>

--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -8,6 +8,11 @@ services:
     - "8080:8080"
    volumes:
     - .:/usr/src/app
+    environment:
+     - ES_ENDPOINT_EXTERNAL=${ES_ENDPOINT_EXTERNAL}
+    networks:
+      - sefarad-network
+
  elasticsearch:
    image: "docker.elastic.co/elasticsearch/elasticsearch:5.5.2"
    ulimits:
@@ -21,14 +26,47 @@ services:
      - "xpack.security.enabled=false"
      - "http.cors.enabled=true"
      - 'http.cors.allow-origin=*'
-      - ES_ENDPOINT_EXTERNAL=${ES_ENDPOINT_EXTERNAL}
    volumes:
      - esdata:/usr/share/elasticsearch/data/

    ports:
-      - 9200:9200
-      - 9300:9300
+      - 19200:9200
+      - 19300:9300
+    networks:
+      - sefarad-network
+
+
+  fuseki:
+    image: stain/jena-fuseki
+    ports:
+      - "13030:3030"
+    volumes:
+     - ./fuseki:/fuseki
+    environment:
+      - ADMIN_PASSWORD=${FUSEKI_PASSWORD}
+    networks:
+      - sefarad-network
+
+  luigi:
+    build: ./luigi/
+    image: registry.cluster.gsi.dit.upm.es/sefarad/gsicrawler/luigi
+    volumes:
+      - ./luigi:/usr/src/app
+    ports:
+      - "8082:8082"
+    networks:
+      - sefarad-network
+    environment:
+      - ES_ENDPOINT=${ES_ENDPOINT}
+      - ES_PORT=${ES_PORT}
+      - FUSEKI_ENDPOINT_EXTERNAL=${FUSEKI_ENDPOINT_EXTERNAL}
+
+    depends_on:
+      - elasticsearch

+networks:
+  sefarad-network:
+    driver: bridge

 volumes:
  esdata:
--- a/queries.json
+++ b/queries.json
-[{"name":"Show the sentiment and emotion of posts and comments", "val":"PREFIX schema: <http://schema.org/>\nPREFIX marl: <http://www.gsi.dit.upm.es/ontologies/marl/ns#>\nPrefix me:    <http://www.mixedemotions-project.eu/ns/model#> \nPrefix onyx:  <http://www.gsi.dit.upm.es/ontologies/onyx/ns#> \nPREFIX wnaffect: <http://gsi.dit.upm.es/ontologies/wnaffect/ns#>\n\nSELECT ?blogpost ?sentiment ?emotion\nWHERE {\n  ?blogpost a schema:BlogPosting .\n  ?blogpost marl:hasOpinion ?opinion .\n  ?opinion marl:hasPolarity ?sentiment .\n  ?blogpost onyx:hasEmotionSet ?emotionlist .\n  ?emotionlist onyx:hasEmotionCategory ?emotion .\n}\nLIMIT 25\n"}]
-
+[{"name":"Show the sentiment and emotion of posts and comments", "val":"PREFIX schema: <http://schema.org/> \nPREFIX marl: <http://www.gsi.dit.upm.es/ontologies/marl/ns#> \nPREFIX me:    <http://www.mixedemotions-project.eu/ns/model#> \nPREFIX onyx:  <http://www.gsi.dit.upm.es/ontologies/onyx/ns#> \nPREFIX wnaffect: <http://gsi.dit.upm.es/ontologies/wnaffect/ns#>\n\nSELECT ?blogpost ?sentiment ?emotion\nWHERE {\n  ?blogpost a schema:BlogPosting .\n  ?blogpost marl:hasOpinion ?opinion .\n  ?opinion marl:hasPolarity ?sentiment .\n  ?blogpost onyx:hasEmotionSet ?emotionlist .\n  ?emotionlist onyx:hasEmotionCategory ?emotion .\n}\nLIMIT 25\n"},{"name":"How many opinions are positive, neutral or neutral on each tweet?","val":"PREFIX schema: <http://schema.org/>\nPREFIX marl: <http://www.gsi.dit.upm.es/ontologies/marl/ns#>\nPrefix me:    <http://www.mixedemotions-project.eu/ns/model#>\nPrefix onyx:  <http://www.gsi.dit.upm.es/ontologies/onyx/ns#> \nPREFIX wnaffect: <http://gsi.dit.upm.es/ontologies/wnaffect/ns#>\nPREFIX sc: <http://purl.org/science/owl/sciencecommons/>\nPREFIX rdf: <http://www.w3.org/1999/02/22-rdf-syntax-ns#>\nPREFIX sioc: <http://rdfs.org/sioc/ns#>\nPREFIX nif: <http://persistence.uni-leipzig.org/nlp2rdf/ontologies/nif-core#>\n SELECT ?blogpost\n  (COUNT(DISTINCT ?opinionTextPositive) AS ?positives) \n  (COUNT(DISTINCT ?opinionTextNegative) AS ?negatives) \n  (COUNT(DISTINCT ?opinionTextNeutral) AS ?neutral) \n WHERE {\n{\n ?blogpost a schema:BlogPosting .\n ?blogpost marl:hasOpinion ?opinion  .\n ?opinion marl:hasPolarity 'marl:Positive' .\n ?opinion nif:anchorOf ?opinionTextPositive .\n}\n UNION \n {\n ?blogpost a schema:BlogPosting .\n ?blogpost marl:hasOpinion ?opinion  .\n ?opinion marl:hasPolarity 'marl:Negative' .\n ?opinion nif:anchorOf ?opinionTextNegative .\n\n}\n  UNION \n {\n ?blogpost a schema:BlogPosting .\n ?blogpost marl:hasOpinion ?opinion  .\n ?opinion marl:hasPolarity 'marl:Neutral' .\n ?opinion nif:anchorOf ?opinionTextNeutral .\n}\n} GROUP BY ?blogpost"}]
\ No newline at end of file