import tweepy
from tweepy import OAuthHandler
from tweepy import Stream
from tweepy.streaming import StreamListener

consumer_key = ''
consumer_secret = ''
access_token = ''
access_secret = ''

auth = OAuthHandler(consumer_key, consumer_secret)
auth.set_access_token(access_token, access_secret)

class TweetsListener( StreamListener ):

  def on_data(self, data):
      try:
          with open('mytweets.json', 'a', newline='') as f:
              f.write(data)
              return True
          num_tweets = num_tweets + 1
      except BaseException as e:
          print("Error on_data: %s" % str(e))
      return True

  def on_error(self, status):
      print(status)
      return True

import os

#Delete any existing file. If you do not want to delete the earlier file. Then you can commend the line below
os.remove( 'mytweets.json' )
try:
  twitter_stream = Stream( auth, TweetsListener() )
  twitter_stream.filter( track=['sultan'] )
except BaseException as e:
  print( "Streaming Stopped")

# read the entire file into a python array
with open('mytweets.json', 'r') as f:
  data = f.readlines()

# remove the trailing "\n" from each line
data = map(lambda x: x.rstrip(), data)

# Load all json messages as one python list
data_json_str = "[" + ','.join(data) + "]"

# Now the list of json message can be converted into dataframe. Python support loading 
# json messages as dataframe
import pandas as pd
tweets_df = pd.read_json(data_json_str)

tweets_df[0:2]

tweets_df_text = tweets_df[['text']]

pd.set_option('max_colwidth', 800)
tweets_df_text

# Tokenize the tweets message 
# Remove stop words
# Calculate the occurance of words
import numpy as np
from sklearn.feature_extraction.text import CountVectorizer

vect = CountVectorizer( stop_words = "english" )
vect.fit( tweets_df_text.text )
tweets_vec = vect.fit_transform( tweets_df_text.text )
word_freq_df = pd.DataFrame({'term': vect.get_feature_names(),
                           'tf':np.asarray( tweets_vec.sum( axis=0 ) ).ravel().tolist()})
word_freq_df.head()

# Print the words in descending order
word_freq_df.sort_values( "tf", ascending = False ).head( 10 )