# twitter loader
import sys
sys.path.append('[...]/GetOldTweets-python/')
import got3

# misc packages
from collections import defaultdict
import pandas as pd
import numpy as np
import pickle as pkl
from math import isnan
import os
from time import time

# Packages for scraping
from fake_useragent import UserAgent
import re
import requests as req
from bs4 import BeautifulSoup
from urllib.parse import urljoin

# Packages for cleaning text
from string import digits, punctuation
from nltk.corpus import stopwords
from textwrap import wrap

# Packages for unsupervised learning
from sklearn.feature_extraction.text import  TfidfVectorizer, CountVectorizer
from sklearn.decomposition import PCA, NMF, TruncatedSVD
from sklearn.preprocessing import MinMaxScaler

# Packages for plotting
import matplotlib.pyplot as plt
%matplotlib inline
import seaborn as sns
sns.set()

def recent_tweets(username=False, query=False, max_tweets=10):
    '''
    Pulling recent tweets from a Twitter username and query custom combination.
    
    Input: username, query (e.g. a hashtag), number of tweets to pull
    Output: List of tweet text and metadata
    
    '''

    d_tweets = defaultdict(list)
    tweetCriteria = got3.manager.TweetCriteria().setMaxTweets(max_tweets)
    
    if username and query:
        tweetCriteria = tweetCriteria.setUsername(username).setQuerySearch(query)
    elif username and query==False:
        tweetCriteria = tweetCriteria.setUsername(username)
    elif username==False and query:
        tweetCriteria = tweetCriteria.setQuerySearch(query)
    else:
        return False
    tweets = got3.manager.TweetManager.getTweets(tweetCriteria)
    for i in range(max_tweets):
        #print(f'Retrieving tweet #{i+1} of {max_tweets}')
        try:
            tweet_current = tweets[i]
            d_tweets[tweet_current.id] = [tweet_current.permalink, tweet_current.username, tweet_current.date, 
                                  tweet_current.text, tweet_current.retweets, tweet_current.favorites, 
                                  tweet_current.mentions, tweet_current.hashtags, tweet_current.geo]
        except:
            return False
    return d_tweets

def tweet_dict_into_df(d_tweets):
    '''
    Create a dataframe from pulled tweets
    '''
    
    df = pd.DataFrame.from_dict(d_tweets, orient='index', dtype=None).reset_index()
    df.columns=['id','permalink','username','date','text','retweets','favorites','mentions','hashtags','geo']
    return df

def tweet_loader(current_username=False,current_query=False,current_tweets=10):
    
    '''
    A function that pulls together pulling the tweet, putting it into a dataframe then saving it one-by-one 
    (in case internet collapses)
    '''
    
    if current_query and current_username:
        naming = current_query + '_' + current_username
    elif current_query and current_username==False:
        naming = current_query
    elif current_query==False and current_username:
        naming = current_username
    
    print(f'Downloading {current_tweets} tweets for {naming}')
    d_tweets = recent_tweets(username=current_username, query=current_query, max_tweets=current_tweets)
    
    if d_tweets:
        print(f'Saving to df')
        df = tweet_dict_into_df(d_tweets)

        filename = time.strftime("%Y%m%d-%H%M%S") + '_' + naming + '_recent_' + str(current_tweets) + '_tweets.pkl'
        print(f'Saving dataframe as {filename}')
        df.to_pickle(filename)
    
        return df
    
    else:
        return False

def twitter_account_search():
    
    '''
    As a subject of this project, I decided to use a list of Fortune 500 companies for clustering. 
    1. I first obtained the website urls' of the companies via BeautifulSoup
    2. Then I opened the website of each company and looked for twitter ids using RegEx
    3. Saved each companies' Twitter ID where it was found
    
    '''
    
    ua = UserAgent()
    header = {'User-Agent': ua.random}
    url_fortune = 'https://www.zyxware.com/articles/4344/list-of-fortune-500-companies-and-their-websites'
    r_fortunelist = req.get(url_fortune, headers=header)
    soup_fortune = BeautifulSoup(r_fortunelist.content,"lxml")

    d_twitter = defaultdict(list)
    for a in soup_fortune.find('table', {'class':'data-table'}).tbody.find_all('a'):
        url_company = a.attrs['href']
        if not d_twitter[url_company]:
            try:
                site_content = req.get(url_company, headers=header).content
                print(f'Gathering {url_company}')
            except:
                print(f'Website not found for {url_company}')
            try:
                twitter_account = re.search(r'twitter.com/([\w\d]+)',str(site_content).lower()).group(1)
                print(f'Twitter found: {twitter_account}')
                d_twitter[url_company] = twitter_account
            except:
                d_twitter[url_company] = np.NaN
                print(f'Twitter not found')
        else:
            print(f'Twitter already stored for {url_company}')
    
    pkl.dump(d_twitter, open('twitter_ids.pkl', 'wb'))
    return d_twitter

def own_read_pickle(filename):
    
    '''
    I built this unpickler as the built-in one in Pandas seemed to not be able to handle the task efficiently
    '''

    import pickle as pkl
    objects = []
    print(f'Pickle load of {filename} started')
    with (open(filename, 'rb')) as openfile:
        while True:
            try:
                objects.append(pkl.load(openfile))
            except EOFError:
                break

    df_output = objects[0]
    print(f'Pickle load of {filename} finished')
    return df_output

def download_tweets():
    
    '''
    Download Fortune 500 companies' most recent 1000 tweets into pickle databases
    '''
    
    d_twitter = own_read_pickle('twitter_ids.pkl')
    do=False
    for web, handler in [(k,v) for k,v in d_twitter.items() if (v is np.nan or v != v)==False]:
        if handler=='chemours':
            do=True

        if do:
            print('*'*50)
            print(f'Collecting tweets for {handler}')
            df_tweets = tweet_loader(current_username=handler,current_query=False,current_tweets=1000)
            #dfs.append(df_tweets)

    #pkl.dump(dfs, open('twitter_dfs.pkl', 'wb'))
    print('-*-'*50)
    print('Operation finished')

def clean_tweet(df):
    # Collapsing entire df.text into a string
    tweet_docs = df.text.tolist()
    print('Extracting company names')
    tweet_ids = df.permalink.str.extract(r'twitter\.com\/([\w\d]+)')[0].tolist()
    print('Extracting company names finished')
    
    for i, doc in enumerate(tweet_docs):
        
        doc = str(doc).lower()
        
        # Removing numbers
        remove_digits = str.maketrans('', '', digits)
        doc = doc.translate(remove_digits)

        # Removing links from the text
        doc = re.sub(r'http\S+', '', doc, flags=re.MULTILINE)
        doc = re.sub(r'pic.twitter.com\S+', '', doc, flags=re.MULTILINE)
        
        # Removing punctuation
        remove_punct = str.maketrans('', '', punctuation.replace('@',''))
        doc = doc.translate(remove_punct)
        
        # Removing twitter @someone references
        doc = re.sub(r'@\s\S+', '', doc, flags=re.MULTILINE)
        
        # Removing all other non-word characters
        doc = re.sub(r'[^a-zA-Z0-9-\s_.]', '', doc, flags=re.MULTILINE)
        
        # Removing leading whitespaces, \n, double-spaces
        doc = doc.lstrip()
        doc = re.sub(r'\s{2}', ' ', doc, flags=re.MULTILINE)
                
        # Removing manual stopwords
        stopwords = ['dm','metlife','auto','quote','warner','time','twc','cable',
                     'thanks','sharing','thank','hello','hi','able','need','want',
                     'email','im','new','great','thats','day','nb','hope','work',
                     'hear','happy','know','let','share','like','wed','sounds','youd',
                     'look','looks','id','info','information','link','follow','glad',
                     'youre','welcome','enjoyed','enjoy','chris','team', 'details'
                     'private','additional','socialmediaprudentialcom','free','feel',
                     'think','insurance','home','dont','today','spun','company','isnt',
                     'prudential','southern','plc','regarding','way','customerrelationsautonationcom',
                     'better','having','using','world','check','scaleforgood','goal','mcdonalds',
                     'come','social','use','reaching','ok','use','tweet','john','deere','provide','visit',
                     'visiting','state','memberservicesmolinahealthcarecom','ph','make','sure','helpanthemcom',
                     'person','pls','retweet','best','oh','ty'
                    ]
        doc = ' '.join([word for word in doc.split() if word not in stopwords])
        
        # Modifying tweet_docs one-by-one to return document corpus
        tweet_docs[i]=doc
    
    cleaned_tweets = list(zip(tweet_ids,tweet_docs))
    
    return cleaned_tweets

def print_top_words(model, feature_names, n_top_words):
    for topic_idx, topic in enumerate(model.components_):
        message = "Topic #%d: " % topic_idx
        message += " ".join([feature_names[i]
                             for i in topic.argsort()[:-n_top_words - 1:-1]])
        print(message)
    print()

def topic_models(tweet_id_string,nmf_n_components=40,lsi_n_components=300):    

    n_features = 1000
    n_top_words = 20
    tweet_string = [t for _,t in tweet_id_string]

    # Use tf-idf features for NMF.
    print('Vectorizing...')
    tfidf_vectorizer = TfidfVectorizer(max_df=0.3, min_df=2,
                                       stop_words='english')
    
    tfidf = tfidf_vectorizer.fit_transform(tweet_string)
    print('Vectorizing done')
    
    if lsi_n_components>0:
        
        modelLSI = TruncatedSVD(n_components=lsi_n_components, 
                             algorithm='randomized',
                             n_iter=10, random_state=42)
        print('Fitting LSI model')
        lsi = modelLSI.fit(tfidf)
        lsi_doc_topics = lsi.transform(tfidf)
        print('Finished fitting LSI model')
        
    else:
        lsi = None
        lsi_doc_topics = None

    # Set up the NMF
    modelNMF = NMF(n_components=nmf_n_components)
    print('Fitting NMF model')
    nmf = modelNMF.fit(tfidf)
    nmf_doc_topics = nmf.transform(tfidf)
    print('Finished fitting NMF model')
    
    import time
    filename = time.strftime("%Y%m%d-%H%M%S") + '_nmf_' + str(nmf_n_components) + '.sav'
    print(f'Saving nmf model as {filename}')
    pkl.dump(nmf, open(filename, 'wb'))
    print(f'nmf model saved as {filename}')
    
    if lsi_n_components>0:
        filename = time.strftime("%Y%m%d-%H%M%S") + '_lsi_' + str(lsi_n_components) + '.sav'
        print(f'Saving lsi model as {filename}')
        pkl.dump(lsi, open(filename, 'wb'))
        print(f'lsi model saved as {filename}')
    
    print('*'*20,'Printing NMF topics','*'*20)
    tfidf_feature_names = tfidf_vectorizer.get_feature_names()
    print_top_words(nmf, tfidf_feature_names, n_top_words)
    print('*'*20,'Print finished','*'*20)
    
    return tfidf_vectorizer, nmf, nmf_doc_topics, lsi, lsi_doc_topics

def read_and_clean(nmf_n_components=40,lsi_n_components=300):
    print('Reading data')
    df = pd.read_pickle('all_comp_twitter_df.pkl')
    print('Finished')
    
    print('Cleaning tweets')
    tweet_string = clean_tweet(df)
    print('Cleaning tweets finished')
    
    print('Starting topic model algorithm')
    tfidf_vectorizer, nmf, nmf_doc_topics, lsi, lsi_doc_topics=topic_models(tweet_string,nmf_n_components,lsi_n_components)
    print('Topic model algorithm finished')
    
    return tweet_string, tfidf_vectorizer, nmf, nmf_doc_topics, lsi, lsi_doc_topics

tweet_string, tfidf_vectorizer, nmf, nmf_doc_topics, lsi, lsi_doc_topics = read_and_clean(nmf_n_components=15,
                                                                                              lsi_n_components=0)

topic_names=['0-Problem_send_more_info','1-Enjoy_Our_Savings','2-Issue_Shoutout','3-Wrong_Company','4-CustomerCare_Will_Help','5-Future',
             '6-Problem_Do_A_Claim','7-Apologies','8-Appreciation','9-Media_Queries','10-Store_Feedback',
             '11-Payments','12-Community','13-Problem_Followup_on_Claim','14-Problem_Direct_Message'
            ]

def read_from_saved_model(saved_model='20181109-211034_lsi_300.sav'):
    df = pd.read_pickle('all_comp_twitter_df.pkl')
    tweet_string = clean_tweet(df)
    tweet_string_noid = [t for _,t in tweet_string]

    tfidf_vectorizer = TfidfVectorizer(max_df=0.3, min_df=2,
                                           stop_words='english')

    tfidf = tfidf_vectorizer.fit_transform(tweet_string_noid)
    
    model = own_read_pickle(saved_model)
    doc_topics = model.transform(tfidf)
    
    tweet_id_topic_vector = pd.DataFrame(list(zip([c for c,_ in tweet_string],np.array(doc_topics),np.argmax(np.array(doc_topics),axis=1))), columns=['Company','Tweet_Topics','Hard_Classification'])
    company_topic_matrix = (tweet_id_topic_vector
                            .groupby(['Company'])['Tweet_Topics']
                            .apply(lambda x: np.mean(np.array(x),axis=0))
                            .reset_index()
                           )
    
    return tweet_string, doc_topics, company_topic_matrix

# Topic frequency
#unique, counts = np.unique(np.argmax(np.array(doc_topics),axis=1), return_counts=True)
#total = np.sum(counts)
#print(np.asarray((unique, counts/total*100)).T)

# Finding hard classification of each document
#print(list(zip(tweet_string[:1000],np.argmax(np.array(doc_topics),axis=1)[:1000])))

# Map topic vectors to doc ids
tweet_id_topic_vector = pd.DataFrame(list(zip([c for c,_ in tweet_string],np.array(nmf_doc_topics),np.argmax(np.array(nmf_doc_topics),axis=1))), columns=['Company','Tweet_Topics','Hard_Classification'])
company_topic_matrix = (tweet_id_topic_vector
                        .groupby(['Company'])['Tweet_Topics']
                        .apply(lambda x: np.mean(np.array(x),axis=0))
                        .reset_index()
                       )

from sklearn.preprocessing import MinMaxScaler

x = pd.DataFrame(company_topic_matrix.Tweet_Topics.tolist(),columns=topic_names).values #returns a numpy array
min_max_scaler = MinMaxScaler()
x_scaled = min_max_scaler.fit_transform(x)
df = pd.DataFrame(x_scaled,columns=topic_names)
pd.concat([company_topic_matrix.Company,df],axis=1)

from scipy.cluster.hierarchy import dendrogram, linkage

def fancy_dendrogram(*args, **kwargs):
    max_d = kwargs.pop('max_d', None)
    if max_d and 'color_threshold' not in kwargs:
        kwargs['color_threshold'] = max_d
    annotate_above = kwargs.pop('annotate_above', 0)

    ddata = dendrogram(*args, **kwargs)

    if not kwargs.get('no_plot', False):
        plt.title('Hierarchical Clustering Dendrogram (truncated)')
        plt.xlabel('sample index or (cluster size)')
        plt.ylabel('distance')
        for i, d, c in zip(ddata['icoord'], ddata['dcoord'], ddata['color_list']):
            x = 0.5 * sum(i[1:3])
            y = d[1]
            if y > annotate_above:
                plt.plot(x, y, 'o', c=c)
                plt.annotate("%.3g" % y, (x, y), xytext=(0, -5),
                             textcoords='offset points',
                             va='top', ha='center')
        if max_d:
            plt.axhline(y=max_d, c='k')
    return ddata

def plot_hierarchical_cluster(company_topic_matrix,max_d):
    data=np.matrix(company_topic_matrix.Tweet_Topics.tolist())
    Z = linkage(data, 'ward')

    fancy_dendrogram(
        Z,
        truncate_mode='lastp',
        p=30,
        leaf_rotation=90.,
        leaf_font_size=12.,
        show_contracted=True,
        annotate_above=10,
        max_d=max_d,
    )
    plt.show()

def hierarchy_freq_table(clusters):
    df_c = pd.DataFrame(clusters,columns=['Cluster']).Cluster
    df_c = df_c.value_counts().sort_index().reset_index().merge(df_c.value_counts(normalize=True).mul(100).round(1).reset_index(),on='index')
    df_c.columns=['clusters','#_comp','%_comp']
    return df_c

max_d = 0.1
plt.rcParams['figure.figsize'] = [7, 5]
plot_hierarchical_cluster(company_topic_matrix,max_d)
main_clusters = hierarchical_cluster(company_topic_matrix,max_d)
hierarchy_freq_table(main_clusters)

max_d = 0.0185
c = 1

df_dive = cluster_dip(company_topic_matrix, main_clusters, c)
plot_hierarchical_cluster(df_dive,max_d)
hierarchy_freq_table(hierarchical_cluster(df_dive,max_d))

max_d = 0.035
c = 2

df_dive = cluster_dip(company_topic_matrix, main_clusters, c)
plot_hierarchical_cluster(df_dive,max_d)
hierarchy_freq_table(hierarchical_cluster(df_dive,max_d))

def cluster_dip(company_topic_matrix, clusters, cluster_num):
    company_topic_cluster = pd.concat([company_topic_matrix,pd.DataFrame(clusters,columns=['Cluster'])],axis=1)
    company_topic_cluster = company_topic_cluster[company_topic_cluster.Cluster==cluster_num].drop(['Cluster'],axis=1)
    return company_topic_cluster

def hierarchical_cluster(company_topic_matrix,max_d):
    from scipy.cluster.hierarchy import fcluster

    data=np.matrix(company_topic_matrix.Tweet_Topics.tolist())
    Z = linkage(data, 'ward')

    clusters = fcluster(Z, max_d, criterion='distance')
    print(f'Found {np.unique(clusters).shape[0]} clusters')

    return clusters

def cluster_deep_dive(array_doc_topics, array_companies, array_clusters, list_topic_names, list_dive_into_clusters=None, list_dive_into_maxd=None, bool_rescale=True):
    
    # Create dataframe from companies and topic model output 'topics'
    # Then groupby company and average topic vectors into 1 list per company
    list_company_topic_matrix = (pd.DataFrame(list(zip(array_companies,
                                                       array_doc_topics)),
                                              columns=['Company','Tweet_Topics'])
                            .groupby(['Company'])['Tweet_Topics']
                            .apply(lambda x: np.mean(np.array(x),axis=0))
                            .reset_index()
                           )

    # Explode the 1 list per company into a matrix style dataframe
    array_topic_matrix = pd.DataFrame(list_company_topic_matrix.Tweet_Topics.tolist()).values
    
    # If bool_rescale is True, rescale vectors (0,1)
    if bool_rescale:
        min_max_scaler = MinMaxScaler()
        array_topic_matrix = min_max_scaler.fit_transform(array_topic_matrix)
    
    # Create company-topics-cluster dataframe with named topics
    df_company_topic_cluster = pd.concat([list_company_topic_matrix.Company,
                    pd.DataFrame(array_topic_matrix,columns=topic_names),
                    pd.DataFrame(array_clusters,columns=['Cluster'])
                   ],axis=1)
    #df_company_topic_cluster['Orig_Clusters'] = df_company_topic_cluster['Cluster']
    
    # Re-cluster selected clusters
    if list_dive_into_clusters is not None:
        
        # Loop through each cluster to be broken up
        for i, cluster_num in enumerate(list_dive_into_clusters):
            
            print('*'*50)
            print(f'Working on passed cluster_id {cluster_num}')
            
            # Extract max_d
            max_d = list_dive_into_maxd[i]
            
            # Take data for deep-dive cluster and re-cluster
            df_deepdive_company_topics_cluster = cluster_dip(list_company_topic_matrix, array_clusters, cluster_num)
            array_deep_dive_clusters = hierarchical_cluster(df_deepdive_company_topics_cluster,max_d)
            df_deepdivecompany_newtopic = pd.concat([df_deepdive_company_topics_cluster.reset_index(),
                                                     pd.DataFrame(array_deep_dive_clusters,
                                                                  columns=['Cluster'])],axis=1)
            
            # Append new cluster to original
            df_company_topic_cluster = (df_company_topic_cluster
                                         .merge(df_deepdivecompany_newtopic, how='left',left_index=True,right_on='index')
                                         .drop(['index','Company_y','Tweet_Topics'],axis=1)
                                        )
            # Rename weird merge names
            df_company_topic_cluster.rename({'Company_x': 'Company', 
                        'Cluster_x': 'Cluster_Main', 
                        'Cluster_y': 'Cluster_deepdive'}, 
                       axis = 1, inplace=True)
            
            # Create temporary merged clusters
            df_company_topic_cluster['Cluster_Main'] = np.where(
                np.isnan(df_company_topic_cluster['Cluster_deepdive'])==False, 
                df_company_topic_cluster['Cluster_Main'].astype(str)+df_company_topic_cluster['Cluster_deepdive'].astype(str),
                df_company_topic_cluster['Cluster_Main'].astype(str))
            
            # Create mapping table for new clusters
            d=defaultdict(str)
            new_c = range(1,df_company_topic_cluster.Cluster_Main.nunique()+1)
            d_i = 0
            for c in df_company_topic_cluster.Cluster_Main.unique():
                if not d[c]:
                    d[c]=new_c[d_i]
                    d_i += 1
            
            # Re-map clusters based on mapping table
            df_company_topic_cluster['Cluster'] = df_company_topic_cluster['Cluster_Main'].map(d)
            
            # Re-set index
            df_company_topic_cluster.reset_index(inplace=True)
            
            # Delete new columns to make table look like in the beginning
            del df_company_topic_cluster['Cluster_deepdive']
            del df_company_topic_cluster['Cluster_Main']
            del df_company_topic_cluster['index']
            #print(df_company_topic_cluster.index)
                
    return df_company_topic_cluster

tweet_string, nmf_doc_topics, company_topic_matrix = read_from_saved_model(saved_model='20181113-140709_nmf_15.sav')

array_doc_topics = nmf_doc_topics
array_companies = [c for c,_ in tweet_string]
array_clusters = hierarchical_cluster(company_topic_matrix,0.12)
list_topic_names = ['0-Problem_send_more_info','1-Enjoy_Our_Savings','2-Issue_Shoutout','3-Wrong_Company','4-CustomerCare_Will_Help','5-Future',
                    '6-Problem_Do_A_Claim','7-Apologies','8-Appreciation','9-Media_Queries','10-Store_Feedback',
                    '11-Payments','12-Community','13-Problem_Followup_on_Claim','14-Problem_Direct_Message'
                   ]
list_dive_into_clusters = [1,2]
list_dive_into_maxd = [0.0185,0.035]
bool_rescale = True

df = cluster_deep_dive(array_doc_topics, array_companies, array_clusters, list_topic_names, list_dive_into_clusters, list_dive_into_maxd, bool_rescale)

ax = hierarchy_freq_table(df.Cluster)['%_comp'].plot(kind='bar',figsize=(15, 5))
ax.set_xlabel('Clusters')
ax.set_ylabel('% shares')

from sklearn.manifold import TSNE
plt.rcParams['figure.figsize'] = [10, 5]

time_start = time()

data=np.matrix(company_topic_matrix.Tweet_Topics.tolist())

tsne = TSNE(n_components=2, verbose=1, perplexity=70, n_iter=5000)
tsne_pca_results = tsne.fit_transform(data)

df_tsne = None
df_tsne = pd.DataFrame(df.Cluster)
df_tsne['x-tsne-pca'] = tsne_pca_results[:,0]
df_tsne['y-tsne-pca'] = tsne_pca_results[:,1]

print('t-SNE done! Time elapsed: {} seconds'.format(time()-time_start))

plt.scatter(df_tsne['x-tsne-pca'], df_tsne['y-tsne-pca'], c=df_tsne['Cluster'], cmap=plt.cm.get_cmap("jet", df_tsne['Cluster'].nunique()))
plt.colorbar(ticks=range(1,df_tsne['Cluster'].nunique()+1))
plt.clim(0.5, df_tsne['Cluster'].nunique()+0.5)
plt.show()

for name, values in df[['Company','Cluster']].groupby('Cluster'):
    v=', '.join(values.Company.tolist())
    print(f'Cluster: {name}: {v}')
    print()

grouped = df.groupby('Cluster')
plt.rcParams['figure.figsize'] = [16, 5]
font = {'size'   : 22}
plt.rc('font', **font)

cluster_average = grouped[df.columns[1:-1]].agg('mean').reset_index()
cluster_average_cleaned = cluster_average.drop(['Cluster'],axis=1).T


for name, group in grouped:
    
    print('*'*54,f'Cluster {name}','*'*54)
    #print(np.array(group.Company))
    group.drop(['Cluster'],axis=1).T.iloc[1:].plot(legend=None)
    #print(group.T.index[1:-1])
    plt.xticks(range(15),['\n'.join(wrap(l, 7)) for l in group.T.index[1:-1]])
    plt.title('Cluster company split tweet topics')
    #plt.legend(group.Company)

    # Put a legend below current axis
    plt.legend(group.Company,loc='upper center', bbox_to_anchor=(0.5, -0.2),ncol=8, fancybox=True)
    
    plt.show()
    
    i = name-1
    cluster_average_cleaned[i].plot(legend=None)
    plt.xticks(range(15),['\n'.join(wrap(l, 7)) for l in cluster_average.T.index[1:]])
    plt.title('Cluster average tweet topics')
    #plt.legend(cluster_average.Cluster[i:],loc='upper center', bbox_to_anchor=(0.5, -0.2),ncol=8, fancybox=True)
    plt.show()

# Clustering: KMeans
from sklearn.cluster import MiniBatchKMeans
km = MiniBatchKMeans(n_clusters=13)
km.fit(np.matrix(company_topic_matrix.Tweet_Topics.tolist()))

unique, counts = np.unique(km.labels_, return_counts=True)
total = np.sum(counts)
print(np.asarray((unique, counts/total*100)).T)

def n_clusters_check(data):
    SSEs = []
    Sil_coefs = []
    for k in range(2,40):
        km = MiniBatchKMeans(n_clusters=k, random_state=1)
        km.fit(data)
        labels = km.labels_
        Sil_coefs.append(metrics.silhouette_score(data, labels, metric='euclidean'))
        SSEs.append(km.inertia_) # The SSE is just inertia, we
                                                # could have just said km.inertia_
    return SSEs, Sil_coefs

def plot_n_clusters(data):
    
    SSEs, Sil_coefs = n_clusters_check(data)
    
    fig, (ax1, ax2) = plt.subplots(1,2, figsize=(15,5), sharex=True)
    k_clusters = range(2,40)
    ax1.plot(k_clusters, Sil_coefs)
    ax1.set_xlabel('number of clusters')
    ax1.set_ylabel('silhouette coefficient')

    # plot here on ax2
    ax2.plot(k_clusters, SSEs)
    ax2.set_xlabel('number of clusters')
    ax2.set_ylabel('SSE');

from sklearn import metrics

data = np.matrix(company_topic_matrix.Tweet_Topics.tolist())
plot_n_clusters(data)

def show_variance_explained_plots(model):
    
    var_exp_array = model.explained_variance_ratio_
    n_comps = var_exp_array.shape[0] 
    
    fig, ax = plt.subplots(1,2,figsize=(10,4))
    
    ax[0].fill_between(range(n_comps), var_exp_array)
    ax[0].set_title('Variance Explained by Nth Principal Component')
    
    ax[1].fill_between(range(n_comps), np.cumsum(var_exp_array))
    ax[1].set_title('Cumulative Variance Explained by N Components')
    
    plt.show()

show_variance_explained_plots(lsi_model)

from __future__ import print_function

from sklearn.datasets import make_blobs
from sklearn.cluster import KMeans
from sklearn.metrics import silhouette_samples, silhouette_score

import matplotlib.pyplot as plt
import matplotlib.cm as cm
import numpy as np

# Generating the sample data from make_blobs
# This particular setting has one distinct cluster and 3 clusters placed close
# together.
X = data

range_n_clusters = range(2,15)

for n_clusters in range_n_clusters:
    # Create a subplot with 1 row and 2 columns
    fig, ax1 = plt.subplots(1, 1)
    fig.set_size_inches(18, 7)

    # The 1st subplot is the silhouette plot
    # The silhouette coefficient can range from -1, 1 but in this example all
    # lie within [-0.1, 1]
    ax1.set_xlim([-0.1, 1])
    # The (n_clusters+1)*10 is for inserting blank space between silhouette
    # plots of individual clusters, to demarcate them clearly.
    ax1.set_ylim([0, len(X) + (n_clusters + 1) * 10])

    # Initialize the clusterer with n_clusters value and a random generator
    # seed of 10 for reproducibility.
    clusterer = KMeans(n_clusters=n_clusters, random_state=10)
    cluster_labels = clusterer.fit_predict(X)

    # The silhouette_score gives the average value for all the samples.
    # This gives a perspective into the density and separation of the formed
    # clusters
    silhouette_avg = silhouette_score(X, cluster_labels)
    print("For n_clusters =", n_clusters,
          "The average silhouette_score is :", silhouette_avg)

    # Compute the silhouette scores for each sample
    sample_silhouette_values = silhouette_samples(X, cluster_labels)

    y_lower = 10
    for i in range(n_clusters):
        # Aggregate the silhouette scores for samples belonging to
        # cluster i, and sort them
        ith_cluster_silhouette_values = \
            sample_silhouette_values[cluster_labels == i]

        ith_cluster_silhouette_values.sort()

        size_cluster_i = ith_cluster_silhouette_values.shape[0]
        y_upper = y_lower + size_cluster_i

        color = cm.nipy_spectral(float(i) / n_clusters)
        ax1.fill_betweenx(np.arange(y_lower, y_upper),
                          0, ith_cluster_silhouette_values,
                          facecolor=color, edgecolor=color, alpha=0.7)

        # Label the silhouette plots with their cluster numbers at the middle
        ax1.text(-0.05, y_lower + 0.5 * size_cluster_i, str(i))

        # Compute the new y_lower for next plot
        y_lower = y_upper + 10  # 10 for the 0 samples

    ax1.set_title("The silhouette plot for the various clusters.")
    ax1.set_xlabel("The silhouette coefficient values")
    ax1.set_ylabel("Cluster label")

    # The vertical line for average silhouette score of all the values
    ax1.axvline(x=silhouette_avg, color="red", linestyle="--")

    ax1.set_yticks([])  # Clear the yaxis labels / ticks
    ax1.set_xticks([-0.1, 0, 0.2, 0.4, 0.6, 0.8, 1])

    plt.suptitle(("Silhouette analysis for KMeans clustering on sample data "
                  "with n_clusters = %d" % n_clusters),
                 fontsize=14, fontweight='bold')

plt.show()