Merge pull request #46 from uclatommy/develop

Fixing: #39 and #45.

requirements.txt

@@ -8,6 +8,6 @@ pytz>=2016.10
 requests>=2.13.0
 requests-oauthlib>=0.8.0
 six>=1.10.0
-tweepy>=3.5.0
+-e git://github.com/tweepy/tweepy.git@2efe385fc69385b57733f747ee62e6be12a1338b#egg=tweepy
 twython>=3.4.0
 nose

test/test_data.py

@@ -68,7 +68,7 @@ def test_dates(self):
         df = self.feels_db.tweet_dates
         timebox = timedelta(seconds=60)
         second = timedelta(seconds=1)
-        df = df.groupby(pd.TimeGrouper(freq=f'{int(timebox/second)}S')).size()
+        df = df.groupby(pd.Grouper(freq=f'{int(timebox/second)}S')).size()
         df = df[df != 0]
         print(df)
         self.assertEqual(len(df), 3)

tweetfeels/tweetdata.py

@@ -68,6 +68,9 @@ def __init__(self, file='feels.sqlite'):
             self.make_feels_db(self._db)
         self._debug = False
         self.fields = self._fields
+        self.default_fetch = (
+            "SELECT * FROM tweets WHERE created_at > ? AND created_at <= ?"
+            )
 
     @property
     def _fields(self):
@@ -122,7 +125,7 @@ def all(self):
         return df
 
     def fetchbin(self, start=None, end=None, binsize=timedelta(seconds=60),
-                 empty=False):
+                 empty=False, fetch_q=None, fetch_params=None):
         """
         Returns a generator that can be used to iterate over the tweet data
         based on ``binsize``.
@@ -139,20 +142,24 @@ def fetchbin(self, start=None, end=None, binsize=timedelta(seconds=60),
         :rtype: tuple
         """
         second = timedelta(seconds=1)
-        if start is None: start=self.start-second
-        if end is None: end=self.end
-        if start == self.start: start = start-second
+        if start is None:
+            start = self.start-second
+        if end is None:
+            end = self.end
+        if start == self.start:
+            start = start-second
         df = self.tweet_dates
-        df = df.groupby(pd.TimeGrouper(freq=f'{int(binsize/second)}S')).size()
+        df = df.groupby(pd.Grouper(freq=f'{int(binsize/second)}S')).size()
         df = df[df.index > start - binsize]
-        if not empty: df = df[df != 0]
+        if not empty:
+            df = df[df != 0]
         conn = sqlite3.connect(self._db, detect_types=sqlite3.PARSE_DECLTYPES)
         c = conn.cursor()
-        c.execute(
-            "SELECT * FROM tweets WHERE created_at > ? AND created_at <= ?",
-            (start, end)
-            )
-        for i in range(0,len(df)):
+        if fetch_q is None:
+            fetch_q = self.default_fetch
+            fetch_params = (start, end)
+        c.execute(fetch_q, fetch_params)
+        for i in range(0, len(df)):
             frame = []
             if df.iloc[i] > 0:
                 frame = pd.DataFrame.from_records(
@@ -161,7 +168,8 @@ def fetchbin(self, start=None, end=None, binsize=timedelta(seconds=60),
                     )
             left = df.index[i].to_pydatetime()
             right = left + binsize
-            if len(frame)>0 or empty: yield TweetBin(frame, left, right)
+            if len(frame) > 0 or empty:
+                yield TweetBin(frame, left, right)
         c.close()
 
     def tweets_since(self, dt):

tweetfeels/tweetfeels.py

@@ -155,15 +155,10 @@ def self_update():
         if len(self.tracking) == 0:
             print('Nothing to track!')
         else:
-            self._stream.filter(
-                track=self.tracking, languages=self.lang, async=True
-                )
-#  This does not work due to upstream bug in tweepy 3.5.0. They have fixed it in
-#  https://github.com/tweepy/tweepy/pull/783
-#            self._stream.filter(
-#               track=self.tracking, languages=self.lang, async=True,
-#               filter_level=self._filter_level
-#               )
+           self._stream.filter(
+              track=self.tracking, languages=self.lang, is_async=True,
+              filter_level=self._filter_level
+              )
         if seconds is not None:
             t = Thread(target=delayed_stop)
             t.start()