Commit 9448d6b5 authored by Alberto Pascual's avatar Alberto Pascual
Browse files

senpy added

parent 95014d6e
......@@ -98,6 +98,7 @@ class AnalysisTask(luigi.Task):
output.write('\n')
except:
pass
self.set_status_message("Dabiq DONE!")
for article in rumiyaharticles:
#print(article)
if article["@type"] == "schema:Article":
......@@ -108,28 +109,38 @@ class AnalysisTask(luigi.Task):
output.write('\n')
except:
pass
self.set_status_message("rumiyah DONE!")
for newsitem in alj:
i = myAnalysis(newsitem)
output.write(json.dumps(i))
output.write('\n')
self.set_status_message("aljazeera DONE!")
for newsitem in cnn:
i = myAnalysis(newsitem)
output.write(json.dumps(i))
output.write('\n')
self.set_status_message("CNN DONE!")
for newsitem in nyt:
i = myAnalysis(newsitem)
output.write(json.dumps(i))
output.write('\n')
self.set_status_message("NYT DONE!")
for tweet in tweets:
i = myAnalysis(tweet)
output.write(json.dumps(i))
output.write('\n')
self.set_status_message("twitter DONE!")
"""
for tweet in tweets:
......
......@@ -45,9 +45,9 @@ def semanticAnalysis(i):
for k in range(0,number_of_requests+1):
if i_len - int(REQUEST_LONG*(k+1)) > 0:
r = requests.post('http://senpy.cluster.gsi.dit.upm.es//api/', data={'algo':'sentiment-vader', 'apiKey':API_KEY_MEANING_CLOUD, 'i':i["http://schema.org/articleBody"][0]["@value"][REQUEST_LONG*k:REQUEST_LONG*k+REQUEST_LONG]})
r = requests.post('http://localhost:5000/api/', data={'algo':'sentiment-vader', 'apiKey':API_KEY_MEANING_CLOUD, 'i':i["http://schema.org/articleBody"][0]["@value"][REQUEST_LONG*k:REQUEST_LONG*k+REQUEST_LONG]})
else:
r = requests.post('http://senpy.cluster.gsi.dit.upm.es/api/', data={'algo':'sentiment-vader', 'apiKey':API_KEY_MEANING_CLOUD, 'i':i["http://schema.org/articleBody"][0]["@value"][REQUEST_LONG*k:-1]})
r = requests.post('http://localhost:5000/api/', data={'algo':'sentiment-vader', 'apiKey':API_KEY_MEANING_CLOUD, 'i':i["http://schema.org/articleBody"][0]["@value"][REQUEST_LONG*k:-1]})
time.sleep(1)
r = r.json()
......@@ -190,10 +190,10 @@ def myAnalysis(i):
for k in range(0,number_of_requests+1):
if i_len - int(REQUEST_LONG*(k+1)) > 0:
r = requests.post('http://senpy.cluster.gsi.dit.upm.es/api/', data={'algo':'sentiment-vader', 'apiKey':API_KEY_MEANING_CLOUD, 'i':i["schema:articleBody"][REQUEST_LONG*k:REQUEST_LONG*k+REQUEST_LONG]})
r = requests.post('http://senpy:5000/api/', data={'algo':'sentiment-vader', 'apiKey':API_KEY_MEANING_CLOUD, 'i':i["schema:articleBody"][REQUEST_LONG*k:REQUEST_LONG*k+REQUEST_LONG]})
else:
r = requests.post('http://senpy.cluster.gsi.dit.upm.es/api/', data={'algo':'sentiment-vader', 'apiKey':API_KEY_MEANING_CLOUD, 'i':i["schema:articleBody"][REQUEST_LONG*k:-1]})
time.sleep(1)
r = requests.post('http://senpy:5000/api/', data={'algo':'sentiment-vader', 'apiKey':API_KEY_MEANING_CLOUD, 'i':i["schema:articleBody"][REQUEST_LONG*k:-1]})
#time.sleep(1)
r = r.json()
if not 'entries' in r:
......
......@@ -29,7 +29,8 @@ services:
# - sefarad-network
senpy:
image: gsiupm/senpy
build: ./senpy_vader
image: registry.cluster.gsi.dit.upm.es/sefarad/gsicrawler/senpy:v0.5.2
command: --default-plugins
ports:
- "8000:5000"
......@@ -49,7 +50,7 @@ services:
gsicrawler:
build: .
image: registry.cluster.gsi.dit.upm.es/sefarad/gsicrawler
image: registry.cluster.gsi.dit.upm.es/sefarad/gsicrawler:v0.5.2
volumes:
- .:/usr/src/app
ports:
......
......@@ -4,40 +4,52 @@ kind: ConfigMap
metadata:
name: gsicrawler-config
data:
ES_ENDPOINT: "$ES_ENDPOINT"
ES_PORT: "$ES_PORT"
TWITTER_CONSUMER_KEY: "$TWITTER_CONSUMER_KEY"
TWITTER_CONSUMER_SECRET: "$TWITTER_CONSUMER_SECRET"
TWITTER_ACCESS_TOKEN: "$TWITTER_ACCESS_TOKEN"
TWITTER_ACCESS_TOKEN_SECRET: "$TWITTER_ACCESS_TOKEN_SECRET"
FUSEKI_ENDPOINT: "$FUSEKI_ENDPOINT"
FUSEKI_PORT: "$FUSEKI_PORT"
API_KEY_MEANING_CLOUD: "$API_KEY_MEANING_CLOUD"
ES_ENDPOINT: "sefarad-es"
ES_PORT: "9200"
TWITTER_CONSUMER_KEY: "aXyLS0LMM69OBfpslDNe4oZxL"
TWITTER_CONSUMER_SECRET: "nWJIjYoyWT3vm282CDiEGzIQ1ZlSTF2IwPFnoTbUXih95u2fY8"
TWITTER_ACCESS_TOKEN: "377869454-OB1GQt1ycK5EGrdgHYoHdmJ8WbgTciezCKHXjzH2"
TWITTER_ACCESS_TOKEN_SECRET: "qP5kXmi85SFvasv7RdECVwYc0gpch19mMJkISTcvjKc4x"
FUSEKI_ENDPOINT: "sefarad-fuseki"
FUSEKI_PORT: "3030"
API_KEY_MEANING_CLOUD: "9eee4626ccd5bd8df5b10cf86a811081"
---
apiVersion: extensions/v1beta1
kind: Deployment
metadata:
name: ${NAME}
name: gsicrawler-trivalent
spec:
replicas: 1
template:
metadata:
labels:
role: gsicrawler-luigi
app: ${NAME}
role: gsicrawler-trivalent
app: gsicrawler-trivalent
spec:
containers:
- name: gsicrawler
image: ${CI_REGISTRY_IMAGE}:${VERSION}
image: registry.cluster.gsi.dit.upm.es/sefarad/gsicrawler:latest
imagePullPolicy: Always
resources:
limits:
memory: "512Mi"
memory: "256Mi"
cpu: "200m"
ports:
- name: luigiweb
- name: gsicrawler
containerPort: 8082
envFrom:
- configMapRef:
name: gsicrawler-config
- name: senpy
image: registry.cluster.gsi.dit.upm.es/sefarad/gsicrawler/senpy:latest
imagePullPolicy: Always
resources:
limits:
memory: "256Mi"
cpu: "200m"
ports:
- name: senpy
containerPort: 5000
envFrom:
- configMapRef:
name: gsicrawler-config
FROM gsiupm/senpy:0.10.8-python2.7
RUN git clone https://github.com/gsi-upm/senpy-plugins-community && rm -r senpy-plugins-community/emotion-* && rm -r senpy-plugins-community/sentiment-basic
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment