Commit decea9e4 authored by Manuel García-Amado's avatar Manuel García-Amado
Browse files

Merge branch 'fuseki' into 'master'

Fuseki to master

See merge request !3
parents f5171111 0c456c13
......@@ -5,3 +5,4 @@
/elasticsearch/nodes/0/
*.db
.*
/fuseki/*
from node:7.10.0
ENV NODE_PATH=/tmp/node_modules APP_NAME=gsi-crawler
# Install dependencies first to use cache
RUN apt-get update && apt-get install -y gettext
RUN npm install -g http-server bower
WORKDIR /usr/src/app/
ADD bower.json /usr/src/app/bower.json
# Install the application, move the resulting libraries to /usr/src (so it doesn't get overwritten when
# mounting it as a volue, and link to it.
RUN bower install --allow-root && mv bower_components .. && ln -s /usr/src/bower_components /usr/src/app/
ADD . /usr/src/app
CMD ["/usr/src/app/init.sh"]
<link rel="import" href="../../../bower_components/iron-ajax/iron-ajax.html">
<link rel="import" href="../../../bower_components/iron-image/iron-image.html">
<link rel="import" href="../../../bower_components/paper-button/paper-button.html">
<!--<link rel="import" href="../../bower_components/paper-progress/paper-progress.html">-->
<link rel="import" href="../../../bower_components/paper-spinner/paper-spinner.html">
<link rel="import" href="../../../bower_components/neon-animation/neon-animation-runner-behavior.html">
<link rel="import" href="sentiment-review-view.html">
<link rel="import" href="emotion-review-view.html">
<dom-module id="analysis-card">
<style is="custom-style">
div.card-content{
text-align: center;
padding: 20px;
margin-bottom: 20px;
min-height: 100px;
}
img.platform{
position: absolute;
top: 20px;
left: 15px;
max-width: 80px;
}
paper-progress {
display: block;
width: 100%;
margin: 20px 0;
}
paper-progress.blue {
--paper-progress-active-color: var(--paper-light-blue-500);
--paper-progress-secondary-color: var(--paper-light-blue-100);
}
#result{
margin-top: 40px;
}
#loading{
margin-top: 40px;
}
#error{
position: absolute;
top: 50%;
left: 50%;
transform: translate(-50%, -50%);
color: #FF5252;
}
div.analysis-info {
margin-bottom: 50px;
}
span.grade{
font-size: 300%;
}
paper-button.red {
color: #F44336;
--paper-button-flat-focus-color: #FFEBEE;
}
paper-button.red:hover {
background: #FFEBEE;
}
paper-button.blue {
color: var(--paper-light-blue-500);
--paper-button-flat-focus-color: var(--paper-light-blue-50);
}
paper-button.blue:hover {
background: var(--paper-light-blue-50);
}
</style>
<template>
<paper-material elevation="1">
<div class="card-content">
<div id="error" style="display:none;">
<iron-icon icon="fa:times-circle"></iron-icon> <span>{{error}}</span>
</div>
<div id="loading" style="display:none;">
<span id="progressmessage">{{progressmessage}}</span></br></br>
<span>{{url}}</span></br></br>
<paper-spinner active></paper-spinner>
<!--<paper-progress id="loadingprogressbar" value="{{progress}}" max="100" class="blue"></paper-progress>-->
</div>
<div id="result" style="display:none;">
<div class = "analysisInfo">
<iron-image preload src="{{ image }}" sizing="contain" style="width:256px; height:256px;"></iron-image>
<h3><strong>Name:</strong></h3> <span>{{ name }}</span>
<h3><strong>Total Comments:</strong></h3> <span>{{ number_total_reviews }}</span>
</div>
<div id="emotionsAnalysisInfo" class="analysisInfo">
<h3><strong>Emotion: </strong><i id="disgusted" class="twa twa-2x twa-unamused"></i><i id="angry" class="twa twa-2x twa-angry"></i><i id="afraid" class="twa twa-2x twa-confounded"></i><i id="sad" class="twa twa-2x twa-cry"></i><i id="happy" class="twa twa-2x twa-smiley"></i><i id="surprised" class="twa twa-2x twa-open-mouth"></i></h3> <h1><span id="emotionsGrade" class="grade">{{ predominantEmotion }}</span></h1>
<paper-button id="emotions-reviews-button" class="reviews-button" on-tap="showEmotionsReviews" class="blue"><iron-icon icon="fa:commenting-o"></iron-icon> Reviews</paper-button>
</div>
<br>
<div id="sentimentsAnalysisInfo" class="analysisInfo">
<h3><strong>Average Polarity: </strong><span id="symbol"></span></h3> <h1><span id="sentimentsGrade" class="grade">{{ averagePolarity }}</span></h1>
<paper-button id="sentiments-reviews-button" class="reviews-button" on-tap="showSentimentsReviews" class="red"><iron-icon icon="fa:commenting-o"></iron-icon> Reviews</paper-button>
</div>
</div>
<img class="platform" src="{{ image_logo }}"></img></div>
</div>
</paper-material>
<iron-ajax
id="scrapurl"
url="{{ luigiurl }}"
handle-as="json"
on-response="scrapStarted"
last-response="{{response}}"
method="POST"
header="Content-type:application/json">
</iron-ajax>
<iron-ajax
id="infor"
url="{{getURL(response)}}"
handle-as="json"
on-response="infoReceived"
method="GET"></iron-ajax>
</template>
</dom-module>
<script>
AnalysisCard = Polymer({
is: 'analysis-card',
behaviors: [
Polymer.NeonAnimationRunnerBehavior
],
properties:{
error: {
type: String,
value: null
},
loaded: {
type: Boolean,
value: false
},
url: String,
webpage: String,
analysis_type: String,
uuid: String,
image_logo: String,
image: String,
name: String,
grade: String,
reviews: Array,
number_total_reviews: Number,
percent: Number,
progress: String,
response: Object,
progressmessage: String,
averagePolarity: Number,
luigiurl: {
type: String,
value: "http://localhost:8000/startAnalysis"
},
elasticsearchurl: {
type: String,
value: "http://localhost:9200"
},
predominantEmotion: String,
animationConfig: {
type: Object,
value: function() {
return {
'entry': [{
name: 'scale-up-animation',
node: this,
transformOrigin: '0 0'
}],
'exit': [{
name: 'fade-out-animation',
node: this
}]
}
}
}
},
show: function() {
this.style.display = 'block';
this.playAnimation('entry');
},
factoryImpl: function(url, webpage, analysis_type, luigiurl, elasticsearchurl){
this.webpage = webpage;
this.url = url;
this.analysis_type = analysis_type;
this.luigiurl = luigiurl;
this.elasticsearchurl = elasticsearchurl;
this.loaded = false;
this.progress = '0';
this.progressmessage = 'Crawling...';
this.selectLogo();
this.$.scrapurl.params={"url":this.url, "analysisType":this.analysis_type, "website":this.webpage};
this.$.scrapurl.generateRequest();
this.showLoading(this.progressmessage);
},
ready: function(){
if(this.url == undefined) return;
if(!this.loaded){
this.$.scrapurl.params={"url":this.url, "analysisType":this.analysis_type, "website":this.webpage};
this.$.scrapurl.generateRequest();
this.showLoading(this.progressmessage);
}else{
this.$.error.style.display='none';
this.$.loading.style.display='none';
this.$.result.style.display='block';
}
this.selectLogo();
},
scrapStarted: function(event){
response = event.detail.response;
if(response == null){
this.showError('Connection error. Please check your internet connection.');
this.showErrorToast('An error ocurred while analyzing: Connection error. Please check your internet connection.');
return;
}
console.log(response);
this.error = response.error;
if(this.error == null){
this.uuid = response.id;
this.loaded = !response.loading;
this.progress = response.progress;
this.progressmessage = 'Processing...';
this.retrieveInfoDelayed(1000);
this.showLoading(this.progressmessage);
}else{
this.loaded = true;
this.showError(this.error);
this.showErrorToast('An error occurred while analyzing: ' + this.error);
}
},
getURL: function(response){
if(response == null){
this.showError('Connection error. Please check your Luigi endpoint.');
this.showErrorToast('An error ocurred while analyzing: Connection error. Please check your Luigi endpoint.');
return;
}
else return this.elasticsearchurl + '/'+response['index']+'/_search?q=id:'+response['id']
},
retrieveInfoDelayed: function(delay){
this.async(function(){
//this.$.infor.params = {"id":this.id,"analysis_type":this.analysisType,"website":this.website};
this.$.infor.generateRequest();
}, delay);
},
infoReceived: function(event){
elasticresponse = event.detail.response;
try{
response = elasticresponse.hits.hits[0]["_source"]
}catch(err){
//console.log(err)
this.showError('Crawling error.. Please try again later.');
this.showErrorToast('An error ocurred while analyzing: Crawling error. Please try again later.');
return;
}
if(response == null){
this.showError('Connection error. Please check your internet connection.');
this.showErrorToast('An error ocurred while analyzing: Connection error. Please check your internet connection.');
return;
}
this.loaded = !response.loading;
this.error = response.error;
this.progress = response.progress;
if(this.error != null){
this.showError(this.error);
this.showErrorToast('An error occurred while analyzing: ' + this.error);
}else if(!this.loaded){
this.retrieveInfoDelayed(2000);
if(!response.scraping){
this.progressmessage = 'Analyzing...';
this.showLoading(this.progressmessage);
}
}else{
console.log(response)
this.name = response.name;
this.image = response.image;
this.number_total_reviews = response.reviews.length;
this.reviews = response.reviews;
if(response.containsSentimentsAnalysis){
this.averagePolarity = 0;
for(i=0; i<response.reviews.length; i++){
//this.averagePolarity += response.reviews[i].sentimentAnalysis.entries[0].opinions[0]['marl:hasPolarityValue'];
this.averagePolarity += parseInt(response.reviews[i]['polarity']);
//console.log(parseInt(response.reviews[i]['polarity']));
}
this.averagePolarity /= this.number_total_reviews;
this.averagePolarity = Math.round(this.averagePolarity * 100) / 100
this.applyColorBySentimentGrade();
}else{
this.$.sentimentsAnalysisInfo.style="display:none;";
}
if(response.containsEmotionsAnalysis){
var joy = 0;
var surprised = 0;
var disgust = 0;
var sad = 0;
var angry = 0;
for(i=0; i<response.reviews.length; i++){
//var emotion = this.reviews[i].emotionAnalysis.entries[0]['nif:emotionSets'].emotions[0]['onyx:hasEmotionCategory'];
var emotion = this.reviews[i]['emotion'];
if(emotion.indexOf('joy') >= 0)
joy++;
else if(emotion.indexOf('neutral-emotion') >= 0)
surprised++;
else if(emotion.indexOf('anger') >= 0)
angry++;
else if(emotion.indexOf('sadness') >= 0)
sad++;
else if(emotion.indexOf('disgust') >= 0)
disgust++;
else console.log(emotion);
}
console.log(joy + " " + surprised + " " + disgust + " " + sad + " " + angry);
if(joy >= disgust && joy >= sad && joy >= angry && joy >= surprised){
this.predominantEmotion = "Joy";
}else if(surprised >= disgust && surprised >= sad && surprised >= angry){
this.predominantEmotion = "Surprised";
}else if(disgust >= sad && disgust >= angry){
this.predominantEmotion = "Disgust";
}else if(sad >= angry){
this.predominantEmotion = "Sad";
}else{
this.predominantEmotion = "Angry";
}
this.applyColorByEmotionGrade();
}else{
this.$.emotionsAnalysisInfo.style="display:none;";
}
this.showResult();
this.showOkToast(this.name +' analysis finished!');
}
},
applyColorBySentimentGrade: function(){
if(this.averagePolarity >= 0.6){
this.$.sentimentsGrade.style.color= '#00E676';
this.$.symbol.style.color = '#00E676';
this.$.symbol.innerHTML='Positive +';
}else if(this.averagePolarity > 0.4 && this.averagePolarity < 0.6){
this.$.sentimentsGrade.style.color= '#AAAAAA';
this.$.symbol.style.color = '#AAAAAA';
this.$.symbol.innerHTML='Neutral ~';
}else{
this.$.sentimentsGrade.style.color = '#FF1744';
this.$.symbol.style.color = '#FF1744';
this.$.symbol.innerHTML='Negative -';
}
},
applyColorByEmotionGrade: function(){
if(this.predominantEmotion == "Angry"){
this.$.emotionsGrade.style.color= '#C81F1D';
//this.$.angry.style.display="none";
this.$.afraid.style.display="none";
this.$.surprised.style.display="none";
this.$.happy.style.display="none";
this.$.sad.style.display="none";
this.$.disgusted.style.display="none";
}else if(this.predominantEmotion == "Disgust"){
this.$.emotionsGrade.style.color= '#E0901B';
this.$.angry.style.display="none";
//this.$.afraid.style.display="none";
this.$.surprised.style.display="none";
this.$.happy.style.display="none";
this.$.sad.style.display="none";
this.$.disgusted.style.display="none";
}else if(this.predominantEmotion == "Surprised"){
this.$.emotionsGrade.style.color= '#F7EA2A';
this.$.angry.style.display="none";
this.$.afraid.style.display="none";
//this.$.surprised.style.display="none";
this.$.happy.style.display="none";
this.$.sad.style.display="none";
this.$.disgusted.style.display="none";
}else if(this.predominantEmotion == "Joy"){
this.$.emotionsGrade.style.color= '#75B84C';
this.$.angry.style.display="none";
this.$.afraid.style.display="none";
this.$.surprised.style.display="none";
//this.$.happy.style.display="none";
this.$.sad.style.display="none";
this.$.disgusted.style.display="none";
}else if(this.predominantEmotion == "Sad"){
this.$.emotionsGrade.style.color= '#395EAD';
this.$.angry.style.display="none";
this.$.afraid.style.display="none";
this.$.surprised.style.display="none";
this.$.happy.style.display="none";
//this.$.sad.style.display="none";
this.$.disgusted.style.display="none";
}else{
this.$.emotionsGrade.style.color= '#562C8F';
this.$.angry.style.display="none";
this.$.afraid.style.display="none";
this.$.surprised.style.display="none";
this.$.happy.style.display="none";
this.$.sad.style.display="none";
//this.$.disgusted.style.display="none";
}
},
showLoading: function(message){
this.$.result.style.display='none';
this.$.error.style.display='none';
this.$.loading.style.display='block';
//this.$.loadingprogressbar.value=this.progress;
this.$.progressmessage.innerHTML = message;
},
showResult: function(){
this.$.error.style.display='none';
this.$.loading.style.display='none';
this.$.result.style.display='block';
},
showError: function(message){
this.error = message;
this.$.loading.style.display='none';
this.$.result.style.display='none';
this.$.error.style.display='block';
},
selectLogo: function(){
if(this.webpage == "yelp")
this.image_logo = "/images/yelp_logo.png";
else if(this.webpage == "amazon")
this.image_logo = "/images/amazon_logo.png";
else if(this.webpage == "facebook")
this.image_logo = "/images/facebook_logo.png";
else if(this.webpage == "youtube")
this.image_logo = "/images/youtube_logo.png";
else if(this.webpage == "tripadvisor")
this.image_logo = "/images/tripadvisor_logo.png";
else if(this.webpage == "foursquare")
this.image_logo = "/images/foursquare_logo.png";
else if(this.webpage == "twitter")
this.image_logo = "/images/twitter_logo.png";
},
showOkToast: function(message){
this.showToast(message, '#00E676');
},
showErrorToast: function(message) {
this.showToast(message, '#F44336');
},
showToast: function(message, color) {
toast = document.getElementById('toast');
toast.style.zIndex = 10000;
toast.style.background = color;
toast.text = message;
toast.show();
},
showSentimentsReviews: function() {
this.showSentimentsDialog('Reviews', this.reviews);
},
showSentimentsDialog: function(title, reviews){
document.getElementById('scrollable-dialog-title').innerHTML=title;
var scrollableContent = document.getElementById('scrollable-content');
scrollableContent.innerHTML = "";
for(i=0; i<reviews.length; i++){
var review_object = reviews[i];
var review = new SentimentReviewView(review_object);
scrollableContent.appendChild(review);
var hr = document.createElement('hr');
scrollableContent.appendChild(hr);
}
document.getElementById('scrollable-dialog').open();
},
showEmotionsReviews: function() {
this.showEmotionsDialog('Reviews', this.reviews);
},
showEmotionsDialog: function(title, reviews){
document.getElementById('scrollable-dialog-title').innerHTML=title;
var scrollableContent = document.getElementById('scrollable-content');
scrollableContent.innerHTML = "";
for(i=0; i<reviews.length; i++){
var review_object = reviews[i];
var review = new EmotionReviewView(review_object);
scrollableContent.appendChild(review);
var hr = document.createElement('hr');
scrollableContent.appendChild(hr);
}
document.getElementById('scrollable-dialog').open();
}
});
</script>
{
"name": "gsi-crawler",
"authors": [
"gsi-upm"
],
"description": "Dashboard to use scrapy to extract data from websites",
"main": "crawler.html",
"keywords": [
"dashboard",
"web-component",
"elasticsearch",
"polymer"
],
"version":"1.0.0",
"dependencies": {
"polymer": "polymer#1.7.0",
"paper-button": "paper-button#1.0.3",
"paper-fab-speed-dial": "paper-fab-speed-dial#1.0.1",
"paper-fab-transitions":"paper-fab-transitions#0.1.0",
"iron-icons": "PolymerElements/iron-icons#^1.0.0",
"page": "page#1.6.3",
"paper-item": "paper-item#1.0.2",
"paper-menu": "paper-menu#1.0.1",
"paper-dialog": "paper-dialog#1.0.1",
"paper-dialog-scrollable":"paper-dialog-scrollable#1.0.1",
"paper-icon-button": "PolymerElements/paper-icon-button#^1.0.0",
"webcomponentsjs": "< 1.0.0",
"paper-input": "paper-input#1.1.24",
"iron-ajax": "iron-ajax#1.4.4",
"iron-image": "iron-image#1.2.6",
"paper-spinner":"paper-spinner#1.2.0",
"paper-toast": "paper-toast#1.0.0",
"paper-card": "PolymerElements/paper-card#1.1.4"
},
"resolutions": {
"webcomponentsjs": "0.7.24",
"elastic.js": "^1.2.0"
},
"license": "Apache-2.0",
"homepage": "https://lab.cluster.gsi.dit.upm.es/sefarad/dashboard-tourpedia"
}
<!--
@license
Copyright (c) 2015 The Polymer Project Authors. All rights reserved.
This code may only be used under the BSD style license found at http://polymer.github.io/LICENSE.txt
The complete set of authors may be found at http://polymer.github.io/AUTHORS.txt
The complete set of contributors may be found at http://polymer.github.io/CONTRIBUTORS.txt
Code distributed by Google as part of the polymer project is also
subject to an additional IP rights grant found at http://polymer.github.io/PATENTS.txt
-->
<link rel="import" href="../../bower_components/polymer/polymer.html">
<link rel="import" href="../../bower_components/paper-fab-transitions/paper-fab-speed-dial.html">
<link rel="import" href="../../bower_components/paper-fab/paper-fab.html">
<link rel="import" href="new-analysis-dialog.html">
<link rel="import" href="font-awesome.html">
<dom-module id="gsicrawler-greeting">
<template>
<style include="shared-styles">
#dialogs-container{
background:#fff
}
#scrollable-dialog{
background:#fff
}
body{
background: #fafafa;
font-family: 'Roboto', 'Helvetica Neue', Helvetica, Arial, sans-serif;
color: #333;
}
#analysis-menu {
position:fixed;
bottom: 20px;
right: 20px;
}
#no-analysis-message{
text-align: center;
color: #aaaaaa;