Commit 212807ff authored by J. Fernando Sánchez's avatar J. Fernando Sánchez
Browse files

Fixed configuration of analysis-card

Envsubst can be used on final files (e.g., `demo/index.html`) but should
not be used on files that are meant to be reused (such as polymer
definition files).
This commit adds the ES and LUIGI configuration parameters to the
components so they acn be reused.

It also changes the ES endpoint to the global sefarad
endpoint (sefarad/sefarad) and adds building the image to the CI
pipeline.
This last step is likely to fail, tho, as it is untested.
parent ed73aeec
variables:
LUIGI_IMAGE: $CI_REGISTRY_IMAGE/luigi
WEB_IMAGE: $CI_REGISTRY_IMAGE/web
before_script:
- docker login -u gitlab-ci-token -p $CI_BUILD_TOKEN $CI_REGISTRY
stages:
- build
- push
- deploy
build-luigi:
stage: build
script:
- make -e build-luigi
build-web:
stage: build
script:
- make -e build-web
push-web:
stage: push
script:
- make -e push-web
push-luigi:
stage: push
script:
- make -e push-luigi
deploy:
stage: deploy
environment: test
......
......@@ -17,5 +17,17 @@ deploy:
@$(KUBECTL) create secret docker-registry $(CI_REGISTRY) --docker-server=$(CI_REGISTRY) --docker-username=$(CI_REGISTRY_USER) --docker-email=$(CI_REGISTRY_USER) --docker-password=$(CI_BUILD_TOKEN)
@$(KUBECTL) apply -f /tmp/cwd/k8s/
build-luigi:
docker build luigi -t $(LUIGI_IMAGE)
push-luigi: build-luigi
docker push $(LUIGI_IMAGE)
build-web:
docker build . -t $(WEB_IMAGE)
push-web: build-web
docker push $(WEB_IMAGE)
.PHONY:
deploy
......@@ -115,7 +115,7 @@
</paper-material>
<iron-ajax
id="scrapurl"
url="http://localhost:8000/startAnalysis"
url="{{ luigiurl }}"
handle-as="json"
on-response="scrapStarted"
last-response="{{response}}"
......@@ -164,6 +164,14 @@
response: Object,
progressmessage: String,
averagePolarity: Number,
luigiurl: {
type: String,
value: "http://localhost:8000/startAnalysis"
},
elasticsearchurl: {
type: String,
value: "http://localhost:9200"
},
predominantEmotion: String,
......@@ -190,10 +198,12 @@
this.style.display = 'block';
this.playAnimation('entry');
},
factoryImpl: function(url, webpage, analysis_type){
factoryImpl: function(url, webpage, analysis_type, luigiurl, elasticsearchurl){
this.webpage = webpage;
this.url = url;
this.analysis_type = analysis_type;
this.luigiurl = luigiurl;
this.elasticsearchurl = elasticsearchurl;
this.loaded = false;
this.progress = '0';
this.progressmessage = 'Crawling...';
......@@ -243,7 +253,7 @@
this.showErrorToast('An error ocurred while analyzing: Connection error. Please check your Luigi endpoint.');
return;
}
else return 'http://localhost:9200/'+response['index']+'/_search?q=id:'+response['id']
else return this.elasticsearchurl + '/'+response['index']+'/_search?q=id:'+response['id']
},
retrieveInfoDelayed: function(delay){
this.async(function(){
......
......@@ -119,19 +119,32 @@ subject to an additional IP rights grant found at http://polymer.github.io/PATEN
},
elasticresponse:{
type:String
}
},
luigiurl: {
type: String,
value: 'LUIGI!',
notify: true
},
elasticsearchurl: {
type: String,
value: 'ES!',
notify: true
},
},
ready: function(){
console.log('LUIGI: '+this.luigiurl);
console.log('ES: '+this.elasticsearchurl);
console.log('Greeting: ' + this.greeting);
},
getURL: function(){
return 'http://localhost:8000/startAnalysis'
return this.luigiurl + '/startAnalysis';
},
getelasticURL: function(response){
//console.log(response)
return 'http://localhost:9200/'+response['index']+'/_search?q=id:'+response['id']
return this.elasticsearchurl + '/'+response['index']+'/_search?q=id:'+response['id']
},
......@@ -148,15 +161,16 @@ subject to an additional IP rights grant found at http://polymer.github.io/PATEN
},
showNewAnalysisDialog: function(webpage){
console.log('ShowNewAnalysisDialog '+this.getURL());
var container = document.getElementById('dialogs-container');
container.innerHTML = "";
var new_analysis = null;
if(webpage == 'foursquare'){
new_analysis = new NewAnalysisDialog(webpage, 'https://es.foursquare.com/v/cafeter%C3%ADa-hd/4b5b0ca9f964a520d0e028e3');
new_analysis = new NewAnalysisDialog(webpage, 'https://es.foursquare.com/v/cafeter%C3%ADa-hd/4b5b0ca9f964a520d0e028e3', this.getURL(), this.elasticsearchurl);
}else if(webpage == 'amazon'){
new_analysis = new NewAnalysisDialog(webpage, "http://www.amazon.com/HP-H3C52AA-ABA-K1500-Keyboard/dp/B00E4TOWR0/");
new_analysis = new NewAnalysisDialog(webpage, "http://www.amazon.com/HP-H3C52AA-ABA-K1500-Keyboard/dp/B00E4TOWR0/", this.getURL(), this.elasticsearchurl);
}else if(webpage == 'reddit'){
new_analysis = new NewAnalysisDialog(webpage, "politics");
new_analysis = new NewAnalysisDialog(webpage, "politics", this.getURL(), this.elasticsearchurl());
}
new_analysis.style.width = '80%';
......
<html>
<head>
<link rel="stylesheet" href="https://maxcdn.bootstrapcdn.com/bootstrap/3.3.7/css/bootstrap.min.css" integrity="sha384-BVYiiSIFeK1dGmJRAkycuHAHRg32OmUcww7on3RYdg4Va+PmSTsz/K68vbdEjh4u" crossorigin="anonymous">
<script src="/bower_components/webcomponentsjs/webcomponents-lite.js"></script>
<script src="../bower_components/webcomponentsjs/webcomponents-lite.js"></script>
<link rel="import" href="imports.html"></link>
</head>
<body>
<template is="dom-bind">
<gsicrawler-greeting></gsicrawler-greeting>
<gsicrawler-greeting
luigiURL="${LUIGI_ENDPOINT}"
elasticsearchURL="${ES_ENDPOINT}"></gsicrawler-greeting>
<!-- <button id="databutton" onclick="changedata()">Click to change data</button> -->
</template>
</body>
......
......@@ -12,14 +12,14 @@ services:
depends_on:
- elasticsearch
environment:
- LUIGI_ENDPOINT=localhost:8000
- ES_ENDPOINT=localhost:9200
- LUIGI_ENDPOINT=http://localhost:18000
- ES_ENDPOINT=http://localhost:19200
elasticsearch:
image: elasticsearch
ports:
- "9200:9200"
- "9300:9300"
- "19200:9200"
- "19300:9300"
volumes:
- ./elasticsearch/nodes:/usr/share/elasticsearch/data/nodes
- ./elasticsearch/config:/usr/share/elasticsearch/config
......@@ -32,8 +32,7 @@ services:
volumes:
- ./luigi:/usr/src/app
ports:
- "8082:8082"
- "8000:8000"
- "18000:8000"
networks:
- sefarad-network
environment:
......
......@@ -5,8 +5,8 @@ if [ -f /.dockerenv ]; then
cp -a /usr/src/bower_components /usr/src/app/;
fi
envsubst < /usr/src/app/analysis-cardenv.html > /usr/src/app/analysis-card.html;
envsubst < /usr/src/app/demo/indexenv.html > /usr/src/app/demo/index.html;
bower link --allow-root
bower link $APP_NAME --allow-root
http-server .
\ No newline at end of file
http-server .
......@@ -26,7 +26,7 @@ spec:
- name: LUIGI_ENDPOINT
value: "gsicrawler-luigi.cluster.gsi.dit.upm.es"
- name: ES_ENDPOINT
value: "gsicrawler-elasticsearch.cluster.gsi.dit.upm.es"
value: "sefarad-elasticsearch.cluster.gsi.dit.upm.es"
---
apiVersion: extensions/v1beta1
kind: Deployment
......@@ -49,48 +49,12 @@ spec:
memory: "512Mi"
cpu: "1000m"
ports:
- name: luigi
- name: luigiproxy
containerPort: 8000
- name: luigi2
- name: luigiweb
containerPort: 8082
env:
- name: ES_ENDPOINT
value: "gsicrawler-es"
- name: ES_PORT
value: "9200"
---
apiVersion: extensions/v1beta1
kind: Deployment
metadata:
name: gsicrawler-es
spec:
replicas: 1
template:
metadata:
labels:
role: gsicrawler-es
app: test
spec:
containers:
- name: gsicrawler-elasticsearch-latest
image: docker.elastic.co/elasticsearch/elasticsearch:5.4.0
imagePullPolicy: Always
resources:
limits:
memory: "2048Mi"
cpu: "1000m"
env:
- name: ES_JAVA_OPTS
value: -Xms1g -Xmx1g
command: ["bin/elasticsearch"]
args: ['-Ehttp.host=0.0.0.0',
'-Etransport.host=127.0.0.1',
'-Ehttp.cors.enabled=true',
'-Ehttp.cors.allow-origin="*"',
'-Ehttp.cors.allow-methods=OPTIONS, HEAD, GET, POST, PUT, DELETE',
'-Ehttp.cors.allow-headers=X-Requested-With, X-Auth-Token, Content-Type, Content-Length"']
ports:
- name: elasticsearch
containerPort: 9200
- name: elasticsearch-2
containerPort: 9300
\ No newline at end of file
value: "9200"
\ No newline at end of file
......@@ -3,6 +3,8 @@ apiVersion: extensions/v1beta1
kind: Ingress
metadata:
name: gsicrawler-ingress
annotations:
ingress.kubernetes.io/rewrite-target: /
spec:
rules:
- host: gsicrawler.cluster.gsi.dit.upm.es
......@@ -12,13 +14,6 @@ spec:
backend:
serviceName: gsicrawler-web
servicePort: 8080
- host: gsicrawler-elasticsearch.cluster.gsi.dit.upm.es
http:
paths:
- path: /
backend:
serviceName: gsicrawler-es
servicePort: 9200
- host: gsicrawler-luigi.cluster.gsi.dit.upm.es
http:
paths:
......@@ -26,3 +21,7 @@ spec:
backend:
serviceName: gsicrawler-luigi
servicePort: 8000
- path: /tasks/
backend:
serviceName: gsicrawler-luigi
servicePort: 8082
\ No newline at end of file
......@@ -18,19 +18,11 @@ metadata:
spec:
type: ClusterIP
ports:
- port: 8000
- name: luigiproxy
port: 8000
protocol: TCP
selector:
role: gsicrawler-luigi
---
apiVersion: v1
kind: Service
metadata:
name: gsicrawler-es
spec:
type: ClusterIP
ports:
- port: 9200
- name: luigiweb
port: 8082
protocol: TCP
selector:
role: gsicrawler-es
\ No newline at end of file
role: gsicrawler-luigi
\ No newline at end of file
......@@ -180,6 +180,9 @@ class AnalysisTask(luigi.Task):
if random.random() < probFake:
review["fake"] = True
else: review["fake"] = False
output.write(json.dumps(i))
#print(i)
output.write('\n')
def output(self):
"""
......
......@@ -95,12 +95,16 @@
webpage: {
type: String,
observer: '_webpageChanged'
}
},
luigiurl: String,
elasticsearchurl: String,
},
factoryImpl: function(webpage, field_value){
factoryImpl: function(webpage, field_value, luigiurl, elasticsearchurl){
this.webpage = webpage;
this.field_value = field_value;
this._webpageChanged(webpage);
this.luigiurl = luigiurl;
this.elasticsearchurl = elasticsearchurl;
},
behaviors: [
Polymer.PaperDialogBehavior
......@@ -189,36 +193,27 @@
toast.text = message;
toast.show();
},
addCard: function(atype) {
var ctype;
if(this.webpage == 'reddit'){
ctype = RedditAnalysisCard;
}
else {
ctype = AnalysisCard;
}
analysis = new ctype(this.url, this.webpage, atype, this.luigiurl, this.elasticsearchurl);
this.appendAnalysisOnTop(analysis);
},
addFakeAnalysis: function() {
if(this.webpage == 'reddit'){
var analysis = new RedditAnalysisCard(this.url, this.webpage, 'fake')
}else{
var analysis = new AnalysisCard(this.url, this.webpage, 'fake');
} this.appendAnalysisOnTop(analysis);
this.addCard('fake');
},
addSentimentAnalysis: function() {
if(this.webpage == 'reddit'){
var analysis = new RedditAnalysisCard(this.url, this.webpage, 'sentiments')
}else{
var analysis = new AnalysisCard(this.url, this.webpage, 'sentiments');
}
this.appendAnalysisOnTop(analysis);
this.addCard('sentiments');
},
addEmotionAnalysis: function() {
if(this.webpage == 'reddit'){
var analysis = new RedditAnalysisCard(this.url, this.webpage, 'emotions')
}else{
var analysis = new AnalysisCard(this.url, this.webpage, 'emotions');
}
this.appendAnalysisOnTop(analysis);
},
addAllAnalysis: function(){
if(this.webpage == 'reddit'){
var analysis = new RedditAnalysisCard(this.url, this.webpage, 'fake,emotions,sentiments')
}else{
var analysis = new AnalysisCard(this.url, this.webpage, 'fake,emotions,sentiments');
}
this.appendAnalysisOnTop(analysis);
this.addCard('fake,emotions,sentiments');
},
appendAnalysisOnTop: function(analysis) {
var analysis_list = document.getElementById('analysis-list');
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment