minor fixes

gsicrawler/scrapers/facebook.py

@@ -2,8 +2,8 @@ from facebook_scraper import get_posts
 import os
 import json
 
-user = os.environ["FACEBOOK_USER"]
-password = os.environ["FACEBOOK_PASSWORD"]
+user = os.environ.get("FACEBOOK_USER")
+password = os.environ.get("FACEBOOK_PASSWORD")
 
 def getFBPageFeedData (page_id, num_status=10):
 

gsicrawler/scrapers/twitter.py

@@ -20,7 +20,7 @@ def search(api, query, count):
 
     while len(searched_tweets) < max_tweets:
         count = max_tweets - len(searched_tweets)
-        new_tweets = api.search(q=query, count=count, max_id=str(last_id - 1))
+        new_tweets = api.search_tweets(q=query, count=count, max_id=str(last_id - 1))
         if not new_tweets:
             break
         searched_tweets.extend(new_tweets)
@@ -35,7 +35,11 @@ def timeline(api, query, count):
 def retrieveTweets(querytype, query, count=0, keep=False, library="tweepy", before=None, after=None):
     if library == "snscrape":
         
-        full_query = '{} since:{} until:{} '.format(query, after, before)
+        full_query = '{}'
+        if before:
+            full_query = full_query + ' since:{}'.format(before)
+        if after:
+            full_query = full_query + ' after:{}'.format(after)
         scraper = twitter.TwitterSearchScraper(full_query)
         tweet_list = []
 

requirements.txt

@@ -19,3 +19,4 @@ tornado
 discord
 psaw
 facebook_scraper
+snscrape

supervisor/noflower.conf

+[include]
+files = common.conf partial/celery.conf partial/web.conf partial/listener.conf