Commit e366c8b9 authored by Manuel García-Amado's avatar Manuel García-Amado
Browse files

Merge branch 'master' into 'dbpedia'

# Conflicts:
#   .gitignore
#   bower.json
#   dashboard-reddit.env.html
#   dashboard-reddit.html
#   demo/elements/news-chart/news-chart.html
#   demo/index.html
#   docker-compose.yml
#   init.sh
#   luigi/analysistask.py
#   luigi/analyzers/emotions.py
#   luigi/analyzers/sentiments.py
#   luigi/scrapers/cnnScraper.py
#   luigi/scrapers/nytimesScraper.py
#   queries.json
parents 8f53d671 32d270e7
......@@ -12,3 +12,5 @@ elasticsearch
.*
/fuseki/*
dashboard-reddit.html
*.pem
*.crt
......@@ -26,14 +26,17 @@
"elasticjs-import": "DigElements/elasticjs-import#~1.0.1",
"elastic-client": "DigElements/elastic-client#~1.0.1",
"webcomponentsjs": "< 1.0.0",
"tweet-chart": "tweet-chart#*",
"tweet-chart": "tweet-chart#*",
"comment-chart": "comment-chart#*",
"paper-material": "paper-material#*",
"yasgui-polymer": "yasgui-polymer#*",
"paper-dialog":"paper-dialog#1.1.0",
"iron-pages": "PolymerElements/iron-pages#1.0.2",
"paper-item": "paper-item#1.0.2",
"paper-menu": "paper-menu#1.0.1"
"paper-menu": "paper-menu#1.0.1",
"iron-a11y-keys-behavior": "iron-a11y-keys-behavior#1.1.9",
"entities-chart": "entities-chart#*",
"iron-collapse": "iron-collapse#1.0.2"
},
"resolutions": {
"webcomponentsjs": "0.7.24",
......
<link rel="import" href="/bower_components/polymer/polymer.html">
<link rel="import" href="/bower_components/material-search/material-search.html">
<link rel="import" href="/bower_components/paper-material/paper-material.html">
<link rel="import" href="/bower_components/google-chart-elasticsearch/google-chart.html">
<link rel="import" href="/bower_components/paper-tabs/paper-tabs.html">
<link rel="import" href="/bower_components/paper-tabs/paper-tab.html">
<link rel="import" href="/bower_components/number-chart/number-chart.html">
<link rel="import" href="/bower_components/comment-chart/comment-chart.html">
<link rel="import" href="/bower_components/paper-tabs/paper-tabs.html">
<link rel="import" href="/bower_components/yasgui-polymer/yasgui.html">
<link rel="import" href="/bower_components/iron-pages/iron-pages.html">
<link rel="import" href="/bower_components/paper-item/paper-item.html">
<link rel="import" href="/bower_components/paper-menu/paper-menu.html">
<link rel="import" href="/elements/news-chart/news-chart.html">
<dom-module id="dashboard-reddit">
<link rel="import" type="css" href="dashboard-tourpedia.css" type="css">
<link rel="import" href="/styles/app-theme.html" type="css">
<template>
<iron-ajax auto
url="queries.json"
handle-as="json"
last-response="{{queries}}"></iron-ajax>
<paper-tabs selected="{{selected}}">
<paper-tab>Dashboard</paper-tab>
<paper-tab>Sparql Editor</paper-tab>
</paper-tabs>
<iron-pages selected="{{selected}}">
<div>
<material-search search-value="{{query}}"></material-search>
<br>
<div style="width: 100%; display: inline-block">
<div style="width: 32%; margin-right: 2%; float: left">
<number-chart
icon="info"
stylebg="bg-red"
data="{{data}}">
</number-chart>
</div>
<div style="width: 32%; margin-right: 2%; float: left">
<number-chart
data="{{data}}"
object="cnn"
aggKey="author"
title="CNN"
icon="icons:receipt"
stylebg="bg-yellow">
</number-chart>
</div>
<div style="width: 32%; float: left">
<number-chart
data="{{data}}"
object="The New York Times"
aggKey="author"
title="The New York Times"
icon="icons:receipt"
aggkey="location"
stylebg="bg-green">
</number-chart>
</div>
</div>
<div style="width: 100%; display: inline-block">
<div class="chart_container" style="right: 0; float: right">
<google-chart
data="{{data}}"
field="about"
id='pie-chart4'
extra-id='pie-chart4'
type='pie'
filters="{{filters}}"
icon='icons:list'
param='{{param}}'
options='{"title": "Article Sections"}'
cols='[{"label": "articlesection", "type": "string"},{"label": "Count", "type": "number"}]'
</google-chart>
</div>
<div class="chart_container" style="left: 0; float: left">
<google-chart
field="author"
data="{{data}}"
id='pie-chart3'
extra-id='pie-chart3'
type='pie'
filters="{{filters}}"
icon='icons:reorder'
options='{"title": "Newspapers"}'
cols='[{"label": "type", "type": "string"},{"label": "Count", "type": "number"}]'
</google-chart>
</div>
</div>
<div style="width: 100%">
<div class="chart_container" style="left: 0; float: left">
<google-chart
field="sentiment"
data="{{data}}"
id='pie-chart5'
extra-id='pie-chart5'
type='pie'
filters="{{filters}}"
icon='social:mood'
options='{"title": "Sentiments"}'
cols='[{"label": "Sentiment", "type": "string"},{"label": "Count", "type": "number"}]'
</google-chart>
</div>
<div class="chart_container" style="left: 0; float: right">
<google-chart
field="emotion"
data="{{data}}"
id='pie-chart5'
extra-id='pie-chart5'
type='pie'
filters="{{filters}}"
icon='social:sentiment-very-dissatisfied'
options='{"title": "Emotions"}'
cols='[{"label": "Sentiment", "type": "string"},{"label": "Count", "type": "number"}]'
</google-chart>
</div>
</div>
<div class="chart_container" style="left: 0; float: left; width: 100%">
<news-chart
datos = "{{data}}"
title="List of news"
filters="{{filters}}"
icon='icons:receipt'>
</news-chart>
</div>
</div>
</div>
<div>
<yasgui-ui
endpoint="http:///gsicrawler/query"
queries="{{queries}}">
</yasgui-ui>
</div>
</iron-pages>
</template>
<script>
var ready = false;
Polymer({
is: 'dashboard-reddit',
properties: {
selected: {
type: Number,
value: 0
},
ids:{
type: Array
},
data:{
type: Object
},
client: {
type: Object,
notify: true,
observer: '_clientChanged'
},
fields: {
type: Array,
value: function() { return []; }
},
filters: {
type: Array,
notify: true,
value: function() { return []; }
}
},
observers: [
'_filtersChange(filters.*)'
],
ready: function(){
console.log("ready");
},
_clientChanged: function() {
console.log("ClientChanged");
ready = true;
this._query();
},
_filtersChange: function() {
this._query();
},
_query: function() {
console.log("_filtersChangedash")
var that = this;
console.log("Ready?: ", ready);
if(ready){
this.client.search({
// undocumented params are appended to the query string
index: "gsicrawler",
body: {
size: 10,
query: {
bool: {
must: this.filters,
}
},
aggs: {
type: {
terms: {
field: "@type.keyword",
order: {
_count: "desc"
}
}
},
author: {
terms: {
field: "author.keyword",
order: {
_count: "desc"
}
}
},
about: {
terms: {
field: "about.keyword",
order: {
_count: "desc"
}
}
},
sentiment: {
terms: {
field: "sentiments.marl:hasPolarity.keyword",
order: {
_count: "desc"
}
}
},
emotion: {
terms: {
field: "emotions.onyx:hasEmotion.onyx:hasEmotionCategory.keyword",
order: {
_count: "desc"
}
}
}
}
}
}).then(function (resp) {
var myids = []
resp.hits.hits.forEach(function(entry){myids.push(entry._id)})
that.ids = myids;
//console.log(that.ids)
that.data = resp;
console.log(that.data);
});
}
}
});
</script>
</dom-module>
<link rel="import" href="/bower_components/polymer/polymer.html">
<link rel="import" href="/bower_components/material-search/material-search.html">
<link rel="import" href="/bower_components/paper-material/paper-material.html">
<link rel="import" href="/bower_components/google-chart-elasticsearch/google-chart.html">
<link rel="import" href="/elements/google-chart-elasticsearch/google-chart.html">
<link rel="import" href="/bower_components/paper-tabs/paper-tabs.html">
<link rel="import" href="/bower_components/paper-tabs/paper-tab.html">
<link rel="import" href="/bower_components/number-chart/number-chart.html">
<link rel="import" href="/elements/number-chart/number-chart.html">
<link rel="import" href="/bower_components/comment-chart/comment-chart.html">
<link rel="import" href="/bower_components/paper-tabs/paper-tabs.html">
<link rel="import" href="/bower_components/yasgui-polymer/yasgui.html">
<link rel="import" href="/bower_components/iron-pages/iron-pages.html">
<link rel="import" href="/bower_components/paper-item/paper-item.html">
<link rel="import" href="/bower_components/paper-menu/paper-menu.html">
<link rel="import" href="/elements/entities-chart/entities-chart.html">
<link rel="import" href="/elements/news-chart/news-chart.html">
......@@ -38,8 +39,9 @@
<div style="width: 100%; display: inline-block">
<div style="width: 32%; margin-right: 2%; float: left">
<number-chart
icon="info"
icon="/images/news.ico"
stylebg="bg-red"
title="Items selected"
data="{{data}}">
</number-chart>
</div>
......@@ -47,9 +49,9 @@
<number-chart
data="{{data}}"
object="cnn"
aggKey="creator"
aggKey="author"
title="CNN"
icon="icons:receipt"
icon="/images/cnn.png"
stylebg="bg-yellow">
</number-chart>
</div>
......@@ -57,9 +59,9 @@
<number-chart
data="{{data}}"
object="The New York Times"
aggKey="creator"
aggKey="author"
title="The New York Times"
icon="icons:receipt"
icon="/images/times.png"
aggkey="location"
stylebg="bg-green">
</number-chart>
......@@ -68,22 +70,19 @@
<div style="width: 100%; display: inline-block">
<div class="chart_container" style="right: 0; float: right">
<google-chart
data="{{data}}"
field="about"
id='pie-chart4'
extra-id='pie-chart4'
type='pie'
filters="{{filters}}"
icon='icons:list'
param='{{param}}'
options='{"title": "Article Sections"}'
cols='[{"label": "articlesection", "type": "string"},{"label": "Count", "type": "number"}]'
</google-chart>
</div>
<entities-chart
field="about"
data="{{data}}"
title="Entities founded"
icon="icons:list"
param="{{param}}"
id="entitieschart"
filters="{{filters}}">
</entities-chart>
</div>
<div class="chart_container" style="left: 0; float: left">
<google-chart
field="creator"
field="author"
data="{{data}}"
id='pie-chart3'
extra-id='pie-chart3'
......@@ -95,7 +94,7 @@
</google-chart>
</div>
</div>
<div style="width: 100%">
<div style="width: 100%">
<div class="chart_container" style="left: 0; float: left">
<google-chart
field="sentiment"
......@@ -108,7 +107,7 @@
options='{"title": "Sentiments"}'
cols='[{"label": "Sentiment", "type": "string"},{"label": "Count", "type": "number"}]'
</google-chart>
</div>
</div>
<div class="chart_container" style="left: 0; float: right">
<google-chart
field="emotion"
......@@ -121,23 +120,37 @@
options='{"title": "Emotions"}'
cols='[{"label": "Sentiment", "type": "string"},{"label": "Count", "type": "number"}]'
</google-chart>
</div>
</div>
</div>
<div class="chart_container" style="left: 0; float: left; width: 100%">
<google-chart
field="dataPublished"
data="{{data}}"
id='line-chart'
extra-id='line-chart'
type='area'
filters="{{filters}}"
icon='icons:trending-up'
options='{"title": "News rate"}'
cols='[{"label": "Date", "type": "string"},{"label": "Count", "type": "number"}]'
</google-chart>
</div>
<div class="chart_container" style="left: 0; float: left; width: 100%">
<news-chart
datos = "{{data}}"
title="List of news"
filters="{{filters}}"
icon='icons:receipt'>
</news-chart>
<news-chart
datos = "{{data}}"
title="List of news"
filters="{{filters}}"
icon='icons:receipt'>
</news-chart>
</div>
</div>
</div>
</div>
<div>
<yasgui-ui
endpoint="http://${FUSEKI_ENPOINT_DASHBOARD}/gsicrawler/query"
queries="{{queries}}">
endpoint="http://${FUSEKI_ENPOINT_DASHBOARD}/gsicrawler/query"
queries="{{queries}}">
</yasgui-ui>
</div>
</iron-pages>
......@@ -153,7 +166,9 @@
type: Number,
value: 0
},
query: {
type: String,
},
ids:{
type: Array
......@@ -181,29 +196,121 @@
observers: [
'_filtersChange(filters.*)'
],
behaviors: [
Polymer.IronA11yKeysBehavior
],
keyBindings: {
'enter': '_search'
},
ready: function(){
console.log("ready");
//console.log("ready");
},
_clientChanged: function() {
console.log("ClientChanged");
//console.log("ClientChanged");
ready = true;
this._query();
},
_search: function(){
//console.log("search fired")
//console.log(this.query.length)
if (this.query.length == 0){
//console.log("default search fired")
this._query()
}
else this._ESsearch()
},
_filtersChange: function() {
this._query();
},
_ESsearch: function() {
//console.log("_filtersChangedash")
var that = this;
//console.log("Ready?: ", ready);
if(ready){
this.client.search({
// undocumented params are appended to the query string
index: "gsicrawler",
body: {
size: 20,
query: {
multi_match:{
query: this.query,
fields: ['author', 'headline', 'sentiments', 'about']
}
},
aggs: {
type: {
terms: {
field: "@type.keyword",
order: {
_count: "desc"
}
}
},
author: {
terms: {
field: "author.keyword",
order: {
_count: "desc"
}
}
},
about: {
terms: {
field: "about.keyword",
order: {
_count: "desc"
}
}
},
sentiment: {
terms: {
field: "sentiments.marl:hasPolarity.keyword",
order: {
_count: "desc"
}
}
},
emotion: {
terms: {
field: "emotions.onyx:hasEmotion.onyx:hasEmotionCategory.keyword",
order: {
_count: "desc"
}
}
},
dataPublished: {
date_histogram : {
field : "dataPublished",
format: "MM-yyyy",
interval : "month"
}
}
}
}
}).then(function (resp) {
var myids = []
resp.hits.hits.forEach(function(entry){myids.push(entry._id)})
that.ids = myids;
//console.log(that.ids)
that.data = resp;
//console.log(that.data);
});
}
},
_query: function() {
console.log("_filtersChangedash")
//console.log("_query")
var that = this;
console.log("Ready?: ", ready);
//console.log("Ready?: ", ready);
if(ready){
this.client.search({
// undocumented params are appended to the query string
index: "gsicrawler",
body: {
size: 10,
size: 20,
query: {
bool: {
must: this.filters,
......@@ -218,9 +325,9 @@
}
}
},
creator: {
author: {
terms: {
field: "creator.keyword",
field: "author.keyword",
order: {
_count: "desc"
}
......@@ -234,31 +341,38 @@
}
}
},
sentiment: {
terms: {
field: "sentiments.marl:hasPolarity.keyword",
order: {
_count: "desc"
}
}
},
emotion: {
terms: {
field: "emotions.onyx:hasEmotion.onyx:hasEmotionCategory.keyword",
order: {
_count: "desc"
sentiment: {
terms: {
field: "sentiments.marl:hasPolarity.keyword",
order: {
_count: "desc"
}
}
},
emotion: {
terms: {
field: "emotions.onyx:hasEmotion.onyx:hasEmotionCategory.keyword",
order: {