Commit e409acfe authored by Alberto Pascual's avatar Alberto Pascual
Browse files

environment variables set, k8s first configuration

parent f535593d
deploy:
stage: deploy
environment: test
script:
- make -e deploy
only:
- master
\ No newline at end of file
......@@ -3,6 +3,9 @@ from node:7.10.0
ENV NODE_PATH=/tmp/node_modules APP_NAME=gsi-crawler
# Install dependencies first to use cache
RUN apt-get update && apt-get install -y gettext
RUN npm install -g http-server bower
ADD bower.json /usr/src/bower.json
......
NAME=gsicrawler
DEVPORT=8080
KUBE_CA_PEM_FILE=""
KUBE_URL=""
KUBE_TOKEN=""
KUBE_NAMESPACE=$(NAME)
KUBECTL=docker run --rm -v $(KUBE_CA_PEM_FILE):/tmp/ca.pem -v $$PWD:/tmp/cwd/ -i lachlanevenson/k8s-kubectl --server="$(KUBE_URL)" --token="$(KUBE_TOKEN)" --certificate-authority="/tmp/ca.pem" -n $(KUBE_NAMESPACE)
CI_REGISTRY=docker.io
CI_REGISTRY_USER=gitlab
CI_BUILD_TOKEN=""
CI_COMMIT_REF_NAME=master
deploy:
@$(KUBECTL) delete secret $(CI_REGISTRY) || true
@$(KUBECTL) create secret docker-registry $(CI_REGISTRY) --docker-server=$(CI_REGISTRY) --docker-username=$(CI_REGISTRY_USER) --docker-email=$(CI_REGISTRY_USER) --docker-password=$(CI_BUILD_TOKEN)
@$(KUBECTL) apply -f /tmp/cwd/k8s/
.PHONY:
deploy
......@@ -115,7 +115,7 @@
</paper-material>
<iron-ajax
id="scrapurl"
url="http://localhost:8000/startAnalysis"
url="http://luigi:8000/startAnalysis"
handle-as="json"
on-response="scrapStarted"
last-response="{{response}}"
......@@ -238,7 +238,12 @@
}
},
getURL: function(response){
return 'http://localhost:9200/'+response['index']+'/_search?q=id:'+response['id']
if(response == null){
this.showError('Connection error. Please check your Luigi endpoint.');
this.showErrorToast('An error ocurred while analyzing: Connection error. Please check your Luigi endpoint.');
return;
}
else return 'http://elasticsearch:9200/'+response['index']+'/_search?q=id:'+response['id']
},
retrieveInfoDelayed: function(delay){
this.async(function(){
......@@ -248,7 +253,14 @@
},
infoReceived: function(event){
elasticresponse = event.detail.response;
response = elasticresponse.hits.hits[0]["_source"]
try{
response = elasticresponse.hits.hits[0]["_source"]
}catch(err){
//console.log(err)
this.showError('Crawling error. Please try again later.');
this.showErrorToast('An error ocurred while analyzing: Crawling error. Please try again later.');
return;
}
if(response == null){
this.showError('Connection error. Please check your internet connection.');
this.showErrorToast('An error ocurred while analyzing: Connection error. Please check your internet connection.');
......
<link rel="import" href="../../../bower_components/iron-ajax/iron-ajax.html">
<link rel="import" href="../../../bower_components/iron-image/iron-image.html">
<link rel="import" href="../../../bower_components/paper-button/paper-button.html">
<!--<link rel="import" href="../../bower_components/paper-progress/paper-progress.html">-->
<link rel="import" href="../../../bower_components/paper-spinner/paper-spinner.html">
<link rel="import" href="../../../bower_components/neon-animation/neon-animation-runner-behavior.html">
<link rel="import" href="fake-review-view.html">
<link rel="import" href="sentiment-review-view.html">
<link rel="import" href="emotion-review-view.html">
<dom-module id="analysis-card">
<style is="custom-style">
div.card-content{
text-align: center;
padding: 20px;
margin-bottom: 20px;
min-height: 100px;
}
img.platform{
position: absolute;
top: 20px;
left: 15px;
max-width: 80px;
}
paper-progress {
display: block;
width: 100%;
margin: 20px 0;
}
paper-progress.blue {
--paper-progress-active-color: var(--paper-light-blue-500);
--paper-progress-secondary-color: var(--paper-light-blue-100);
}
#result{
margin-top: 40px;
}
#loading{
margin-top: 40px;
}
#error{
position: absolute;
top: 50%;
left: 50%;
transform: translate(-50%, -50%);
color: #FF5252;
}
div.analysis-info {
margin-bottom: 50px;
}
span.grade{
font-size: 300%;
}
paper-button.red {
color: #F44336;
--paper-button-flat-focus-color: #FFEBEE;
}
paper-button.red:hover {
background: #FFEBEE;
}
paper-button.blue {
color: var(--paper-light-blue-500);
--paper-button-flat-focus-color: var(--paper-light-blue-50);
}
paper-button.blue:hover {
background: var(--paper-light-blue-50);
}
</style>
<template>
<paper-material elevation="1">
<div class="card-content">
<div id="error" style="display:none;">
<iron-icon icon="fa:times-circle"></iron-icon> <span>{{error}}</span>
</div>
<div id="loading" style="display:none;">
<span id="progressmessage">{{progressmessage}}</span></br></br>
<span>{{url}}</span></br></br>
<paper-spinner active></paper-spinner>
<!--<paper-progress id="loadingprogressbar" value="{{progress}}" max="100" class="blue"></paper-progress>-->
</div>
<div id="result" style="display:none;">
<div class = "analysisInfo">
<iron-image preload src="{{ image }}" sizing="contain" style="width:256px; height:256px;"></iron-image>
<h3><strong>Name:</strong></h3> <span>{{ name }}</span>
<h3><strong>Total Comments:</strong></h3> <span>{{ number_total_reviews }}</span>
</div>
<div id="fakeAnalysisInfo" class="analysisInfo">
<h3><strong>Fake Comments:</strong></h3> <span>{{ number_fake_reviews }}</span>
<h3><strong>Grade:</strong></h3> <h1><span id="fakeGrade" class="grade">{{ grade }}</span></h1>
(<strong><span>{{ percent }}</span>%</strong> fake comments detected)
<br>
<paper-button id="fake-reviews-button" on-tap="showFakeReviews" class="red"><iron-icon icon="fa:commenting-o"></iron-icon> Reviews</paper-button>
</div>
<div id="emotionsAnalysisInfo" class="analysisInfo">
<h3><strong>Emotion: </strong><i id="disgusted" class="twa twa-2x twa-unamused"></i><i id="angry" class="twa twa-2x twa-angry"></i><i id="afraid" class="twa twa-2x twa-confounded"></i><i id="sad" class="twa twa-2x twa-cry"></i><i id="happy" class="twa twa-2x twa-smiley"></i><i id="surprised" class="twa twa-2x twa-open-mouth"></i></h3> <h1><span id="emotionsGrade" class="grade">{{ predominantEmotion }}</span></h1>
<paper-button id="emotions-reviews-button" class="reviews-button" on-tap="showEmotionsReviews" class="blue"><iron-icon icon="fa:commenting-o"></iron-icon> Reviews</paper-button>
</div>
<br>
<div id="sentimentsAnalysisInfo" class="analysisInfo">
<h3><strong>Average Polarity: </strong><span id="symbol"></span></h3> <h1><span id="sentimentsGrade" class="grade">{{ averagePolarity }}</span></h1>
<paper-button id="sentiments-reviews-button" class="reviews-button" on-tap="showSentimentsReviews" class="red"><iron-icon icon="fa:commenting-o"></iron-icon> Reviews</paper-button>
</div>
</div>
<img class="platform" src="{{ image_logo }}"></img></div>
</div>
</paper-material>
<iron-ajax
id="scrapurl"
url="http://${LUIGI_ENDPOINT}/startAnalysis"
handle-as="json"
on-response="scrapStarted"
last-response="{{response}}"
method="POST"
header="Content-type:application/json">
</iron-ajax>
<iron-ajax
id="infor"
url="{{getURL(response)}}"
handle-as="json"
on-response="infoReceived"
method="GET"></iron-ajax>
</template>
</dom-module>
<script>
AnalysisCard = Polymer({
is: 'analysis-card',
behaviors: [
Polymer.NeonAnimationRunnerBehavior
],
properties:{
error: {
type: String,
value: null
},
loaded: {
type: Boolean,
value: false
},
url: String,
webpage: String,
analysis_type: String,
uuid: String,
image_logo: String,
image: String,
name: String,
grade: String,
reviews: Array,
number_total_reviews: Number,
number_fake_reviews: Number,
percent: Number,
progress: String,
response: Object,
progressmessage: String,
averagePolarity: Number,
predominantEmotion: String,
animationConfig: {
type: Object,
value: function() {
return {
'entry': [{
name: 'scale-up-animation',
node: this,
transformOrigin: '0 0'
}],
'exit': [{
name: 'fade-out-animation',
node: this
}]
}
}
}
},
show: function() {
this.style.display = 'block';
this.playAnimation('entry');
},
factoryImpl: function(url, webpage, analysis_type){
this.webpage = webpage;
this.url = url;
this.analysis_type = analysis_type;
this.loaded = false;
this.progress = '0';
this.progressmessage = 'Crawling...';
this.selectLogo();
this.$.scrapurl.params={"url":this.url, "analysisType":this.analysis_type, "website":this.webpage};
this.$.scrapurl.generateRequest();
this.showLoading(this.progressmessage);
},
ready: function(){
if(this.url == undefined) return;
if(!this.loaded){
this.$.scrapurl.params={"url":this.url, "analysisType":this.analysis_type, "website":this.webpage};
this.$.scrapurl.generateRequest();
this.showLoading(this.progressmessage);
}else{
this.$.error.style.display='none';
this.$.loading.style.display='none';
this.$.result.style.display='block';
}
this.selectLogo();
},
scrapStarted: function(event){
response = event.detail.response;
if(response == null){
this.showError('Connection error. Please check your internet connection.');
this.showErrorToast('An error ocurred while analyzing: Connection error. Please check your internet connection.');
return;
}
console.log(response);
this.error = response.error;
if(this.error == null){
this.uuid = response.id;
this.loaded = !response.loading;
this.progress = response.progress;
this.progressmessage = 'Processing...';
this.retrieveInfoDelayed(1000);
this.showLoading(this.progressmessage);
}else{
this.loaded = true;
this.showError(this.error);
this.showErrorToast('An error occurred while analyzing: ' + this.error);
}
},
getURL: function(response){
if(response == null){
this.showError('Connection error. Please check your Luigi endpoint.');
this.showErrorToast('An error ocurred while analyzing: Connection error. Please check your Luigi endpoint.');
return;
}
else return 'http://${ES_ENDPOINT}/'+response['index']+'/_search?q=id:'+response['id']
},
retrieveInfoDelayed: function(delay){
this.async(function(){
//this.$.infor.params = {"id":this.id,"analysis_type":this.analysisType,"website":this.website};
this.$.infor.generateRequest();
}, delay);
},
infoReceived: function(event){
elasticresponse = event.detail.response;
try{
response = elasticresponse.hits.hits[0]["_source"]
}catch(err){
//console.log(err)
this.showError('Crawling error.. Please try again later.');
this.showErrorToast('An error ocurred while analyzing: Crawling error. Please try again later.');
return;
}
if(response == null){
this.showError('Connection error. Please check your internet connection.');
this.showErrorToast('An error ocurred while analyzing: Connection error. Please check your internet connection.');
return;
}
this.loaded = !response.loading;
this.error = response.error;
this.progress = response.progress;
if(this.error != null){
this.showError(this.error);
this.showErrorToast('An error occurred while analyzing: ' + this.error);
}else if(!this.loaded){
this.retrieveInfoDelayed(2000);
if(!response.scraping){
this.progressmessage = 'Analyzing...';
this.showLoading(this.progressmessage);
}
}else{
console.log(response)
this.name = response.name;
this.image = response.image;
this.number_total_reviews = response.reviews.length;
this.reviews = response.reviews;
if(response.containsFakeAnalysis){
this.number_fake_reviews = 0;
for(i=0; i < response.reviews.length; i++){
if(response.reviews[i]['fake'])
this.number_fake_reviews++;
}
this.percent = (this.number_fake_reviews / this.number_total_reviews)*100;
this.percent = this.percent.toFixed(2);
this.grade = 'F';
if(this.percent < 20) this.grade = 'A';
else if(this.percent < 40) this.grade = 'B';
else if(this.percent < 60) this.grade = 'C';
else if(this.percent < 80) this.grade = 'D';
this.applyColorByFakeGrade();
}else{
this.$.fakeAnalysisInfo.style="display:none;";
}
if(response.containsSentimentsAnalysis){
this.averagePolarity = 0;
for(i=0; i<response.reviews.length; i++){
//this.averagePolarity += response.reviews[i].sentimentAnalysis.entries[0].opinions[0]['marl:hasPolarityValue'];
this.averagePolarity += parseInt(response.reviews[i]['polarity']);
//console.log(parseInt(response.reviews[i]['polarity']));
}
this.averagePolarity /= this.number_total_reviews;
this.averagePolarity = Math.round(this.averagePolarity * 100) / 100
this.applyColorBySentimentGrade();
}else{
this.$.sentimentsAnalysisInfo.style="display:none;";
}
if(response.containsEmotionsAnalysis){
var joy = 0;
var surprised = 0;
var disgust = 0;
var sad = 0;
var angry = 0;
for(i=0; i<response.reviews.length; i++){
//var emotion = this.reviews[i].emotionAnalysis.entries[0]['nif:emotionSets'].emotions[0]['onyx:hasEmotionCategory'];
var emotion = this.reviews[i]['emotion'];
if(emotion.indexOf('joy') >= 0)
joy++;
else if(emotion.indexOf('neutral-emotion') >= 0)
surprised++;
else if(emotion.indexOf('anger') >= 0)
angry++;
else if(emotion.indexOf('sadness') >= 0)
sad++;
else if(emotion.indexOf('disgust') >= 0)
disgust++;
else console.log(emotion);
}
console.log(joy + " " + surprised + " " + disgust + " " + sad + " " + angry);
if(joy >= disgust && joy >= sad && joy >= angry && joy >= surprised){
this.predominantEmotion = "Joy";
}else if(surprised >= disgust && surprised >= sad && surprised >= angry){
this.predominantEmotion = "Surprised";
}else if(disgust >= sad && disgust >= angry){
this.predominantEmotion = "Disgust";
}else if(sad >= angry){
this.predominantEmotion = "Sad";
}else{
this.predominantEmotion = "Angry";
}
this.applyColorByEmotionGrade();
}else{
this.$.emotionsAnalysisInfo.style="display:none;";
}
this.showResult();
this.showOkToast(this.name +' analysis finished!');
}
},
applyColorByFakeGrade: function(){
if(this.grade == 'A')
this.$.fakeGrade.style.color= '#00E676';
else if(this.grade == 'B')
this.$.fakeGrade.style.color = '#4CAF50';
else if(this.grade == 'C')
this.$.fakeGrade.style.color = '#FFC107';
else if(this.grade == 'D')
this.$.fakeGrade.style.color = '#FF5252';
else if(this.grade == 'F')
this.$.fakeGrade.style.color = '#FF1744';
},
applyColorBySentimentGrade: function(){
if(this.averagePolarity >= 0.6){
this.$.sentimentsGrade.style.color= '#00E676';
this.$.symbol.style.color = '#00E676';
this.$.symbol.innerHTML='Positive +';
}else if(this.averagePolarity > 0.4 && this.averagePolarity < 0.6){
this.$.sentimentsGrade.style.color= '#AAAAAA';
this.$.symbol.style.color = '#AAAAAA';
this.$.symbol.innerHTML='Neutral ~';
}else{
this.$.sentimentsGrade.style.color = '#FF1744';
this.$.symbol.style.color = '#FF1744';
this.$.symbol.innerHTML='Negative -';
}
},
applyColorByEmotionGrade: function(){
if(this.predominantEmotion == "Angry"){
this.$.emotionsGrade.style.color= '#C81F1D';
//this.$.angry.style.display="none";
this.$.afraid.style.display="none";
this.$.surprised.style.display="none";
this.$.happy.style.display="none";
this.$.sad.style.display="none";
this.$.disgusted.style.display="none";
}else if(this.predominantEmotion == "Disgust"){
this.$.emotionsGrade.style.color= '#E0901B';
this.$.angry.style.display="none";
//this.$.afraid.style.display="none";
this.$.surprised.style.display="none";
this.$.happy.style.display="none";
this.$.sad.style.display="none";
this.$.disgusted.style.display="none";
}else if(this.predominantEmotion == "Surprised"){
this.$.emotionsGrade.style.color= '#F7EA2A';
this.$.angry.style.display="none";
this.$.afraid.style.display="none";
//this.$.surprised.style.display="none";
this.$.happy.style.display="none";
this.$.sad.style.display="none";
this.$.disgusted.style.display="none";
}else if(this.predominantEmotion == "Joy"){
this.$.emotionsGrade.style.color= '#75B84C';
this.$.angry.style.display="none";
this.$.afraid.style.display="none";
this.$.surprised.style.display="none";
//this.$.happy.style.display="none";
this.$.sad.style.display="none";
this.$.disgusted.style.display="none";
}else if(this.predominantEmotion == "Sad"){
this.$.emotionsGrade.style.color= '#395EAD';
this.$.angry.style.display="none";
this.$.afraid.style.display="none";
this.$.surprised.style.display="none";
this.$.happy.style.display="none";
//this.$.sad.style.display="none";
this.$.disgusted.style.display="none";
}else{
this.$.emotionsGrade.style.color= '#562C8F';
this.$.angry.style.display="none";
this.$.afraid.style.display="none";
this.$.surprised.style.display="none";
this.$.happy.style.display="none";
this.$.sad.style.display="none";
//this.$.disgusted.style.display="none";
}
},
showLoading: function(message){
this.$.result.style.display='none';
this.$.error.style.display='none';
this.$.loading.style.display='block';
//this.$.loadingprogressbar.value=this.progress;
this.$.progressmessage.innerHTML = message;
},
showResult: function(){
this.$.error.style.display='none';
this.$.loading.style.display='none';
this.$.result.style.display='block';
},
showError: function(message){
this.error = message;
this.$.loading.style.display='none';
this.$.result.style.display='none';
this.$.error.style.display='block';
},
selectLogo: function(){
if(this.webpage == "yelp")
this.image_logo = "/images/yelp_logo.png";
else if(this.webpage == "amazon")
this.image_logo = "/images/amazon_logo.png";
else if(this.webpage == "facebook")
this.image_logo = "/images/facebook_logo.png";
else if(this.webpage == "youtube")
this.image_logo = "/images/youtube_logo.png";
else if(this.webpage == "tripadvisor")
this.image_logo = "/images/tripadvisor_logo.png";
else if(this.webpage == "foursquare")
this.image_logo = "/images/foursquare_logo.png";
else if(this.webpage == "twitter")
this.image_logo = "/images/twitter_logo.png";
},
showOkToast: function(message){
this.showToast(message, '#00E676');
},
showErrorToast: function(message) {
this.showToast(message, '#F44336');
},
showToast: function(message, color) {
toast = document.getElementById('toast');
toast.style.zIndex = 10000;
toast.style.background = color;
toast.text = message;
toast.show();
},
showFakeReviews: function() {
this.showFakeDialog('Reviews', this.reviews);
},
showFakeDialog: function(title, reviews){
document.getElementById('scrollable-dialog-title').innerHTML=title;
var scrollableContent = document.getElementById('scrollable-content');
scrollableContent.innerHTML = "";
for(i=0; i<reviews.length; i++){
var review_object = reviews[i];
var review = new FakeReviewView(review_object);
scrollableContent.appendChild(review);
var hr = document.createElement('hr');
scrollableContent.appendChild(hr);
}
document.getElementById('scrollable-dialog').open();
},
showSentimentsReviews: function() {
this.showSentimentsDialog('Reviews', this.reviews);
},
showSentimentsDialog: function(title, reviews){
document.getElementById('scrollable-dialog-title').innerHTML=title;
var scrollableContent = document.getElementById('scrollable-content');
scrollableContent.innerHTML = "";
for(i=0; i<reviews.length; i++){
var review_object = reviews[i];
var review = new SentimentReviewView(review_object);
scrollableContent.appendChild(review);
var hr = document.createElement('hr');
scrollableContent.appendChild(hr);
}
document.getElementById('scrollable-dialog').open();
},