In [57]:
%%shell
jupyter nbconvert --to html k-means-clusters.ipynb
Out[57]:
In [45]:
## %%shell
## jupyter nbconvert --to html k-means-clusters.ipynb
In [46]:
import os
import pandas as pd
import numpy as np
from scipy import spatial
import matplotlib.pyplot as plt
from sklearn.manifold import TSNE
In [48]:
## Collecting embedded data
## path = "/content/drive/MyDrive/Colab Notebooks/CA2data/"
embeddings_dict1 = {}
with open("/content/drive/MyDrive/Colab Notebooks/CA2data/fruits", 'r') as f1:
for line in f1:
values = line.split()
word = values[0]
vector = np.asarray(values[1:], "float32")
embeddings_dict1[word] = vector
f1.close()
embeddings_dict2 = {}
with open("/content/drive/MyDrive/Colab Notebooks/CA2data/animals", 'r') as f2:
for line in f2:
values = line.split()
word = values[0]
vector = np.asarray(values[1:], "float32")
embeddings_dict2[word] = vector
f2.close()
embeddings_dict3 = {}
with open("/content/drive/MyDrive/Colab Notebooks/CA2data/veggies", 'r') as f3:
for line in f3:
values = line.split()
word = values[0]
vector = np.asarray(values[1:], "float32")
embeddings_dict3[word] = vector
f3.close()
embeddings_dict4 = {}
with open("/content/drive/MyDrive/Colab Notebooks/CA2data/countries", 'r') as f4:
for line in f4:
values = line.split()
word = values[0]
vector = np.asarray(values[1:], "float32")
embeddings_dict4[word] = vector
f4.close()
In [49]:
## Defining merge function
def Merge(a, b, c, d):
merged = {**a, **b, **c, **d}
return merged;
In [50]:
## Merging the data
all_data = Merge(embeddings_dict1, embeddings_dict2, embeddings_dict3, embeddings_dict4)
In [51]:
# Checking data
# all_data
In [52]:
lists = sorted(all_data.items()) # sorted by key, return a list of tuples
x, y = zip(*lists) # unpack a list of pairs into two tuples
ax = plt.gca()
ax.set_xticks(ax.get_xticks()[::10])
plt.plot(x, y)
plt.show()
In [ ]:
## plt.scatter(embeddings_dict1[:,0], embeddings_dict1[:,1], c=labels)
In [ ]:
def thefunction(filedata, centroids, labels, cmap_name='seaice_2'):
df = pd.read_table('filedata',sep=' ', index_col= 0,
).iloc[:, 0:].dropna()
values = df.values.reshape((len(df),2))
centroids,labels = vq.kmeans2(values, 3, minit='points')
plt.plot()
plt.xlim([0, 10])
plt.ylim([0, 10])
plt.title('Dataset')
plt.scatter(centroids, labels)
plt.show()
# create new plot and data
plt.plot()
X = np.array(list(zip(centroids, labels))).reshape(len(centroids), 2)
colors = ['b', 'g', 'r']
markers = ['o', 'v', 's']
# KMeans algorithm
K = 3
kmeans_model = KMeans(n_clusters=K).fit(X)
plt.plot()
for i, l in enumerate(kmeans_model.labels_):
plt.plot(centroids[i], labels(df1)[i], color=colors[l], marker=markers[l],ls='None')
plt.xlim([0, 10])
plt.ylim([0, 10])
plt.show()
In [ ]:
#calculate 2d indicators
def indic(data):
#alternatively you can calulate any other indicators
max = np.max(data, axis=1)
min = np.min(data, axis=1)
return max, min
x,y = indic(zip(*lists)) # unpack a list of pairs into two tuples
plt.scatter(x, y, marker='x')
plt.show()
In [14]:
pd.DataFrame(all_data).T.plot()
plt.show()
In [ ]:
Data Processing
In [15]:
# Required library
from sklearn.cluster import KMeans
In [16]:
models = KMeans(n_clusters=4)
models
Out[16]:
In [20]:
df = pd.DataFrame(columns=["test_items"])
df["values"] = all_data.values()
In [ ]:
vectors = [np.array(f) for f in df["values"]]
vectors
In [22]:
models.fit(vectors)
Out[22]:
In [23]:
models.inertia_
Out[23]:
In [24]:
from gensim.models import Word2Vec
from nltk.cluster import KMeansClusterer
import nltk
from sklearn import cluster
from sklearn import metrics
In [ ]:
# Example
# training data
## sentences = [['this', 'is', 'the', 'good', 'machine', 'learning', 'book'],
# ['this', 'is', 'another', 'book'],
# ['one', 'more', 'book'],
# ['this', 'is', 'the', 'new', 'post'],
# ['this', 'is', 'about', 'machine', 'learning', 'post'],
# ['and', 'this', 'is', 'the', 'last', 'post']]
# # training model
# model = Word2Vec(sentences, min_count=1)
# # get vector data
# X = model[model.wv.vocab]
# print (X)
In [26]:
NUM_CLUSTERS=4
kclusterer = KMeansClusterer(NUM_CLUSTERS, distance=nltk.cluster.util.cosine_distance, repeats=25)
assigned_clusters = kclusterer.cluster(all_data.values(), assign_clusters=True)
centroids = kclusterer._centroid
centroids
print (assigned_clusters)
In [27]:
words = list(all_data)
for i, word in enumerate(words):
print (word + ":" + str(assigned_clusters[i]))
In [31]:
labels = models.labels_
labels
Out[31]:
In [ ]:
In [32]:
df["test_items"] = all_data.keys()
df["clusters"] = assigned_clusters
df["centroids"] = df['clusters'].apply(lambda x: kclusterer.means()[x])
In [33]:
df
Out[33]:
In [34]:
feature_matrix = df["values"] .to_numpy()
centroid = df['centroids'].to_numpy()
def nltk_inertia(feature_matrix, centroid):
sum_ = []
for i in range(feature_matrix.shape[0]):
sum_.append(np.sum((feature_matrix[i] - centroid[i])**2)) #here implementing inertia as given in the docs of scikit i.e sum of squared distance..
return sum(sum_)
nltk_inertia(feature_matrix, centroid)
Out[34]:
Data Visualisation
In [35]:
x = df["test_items"]
y = df["clusters"]
ax = plt.gca()
ax.set_xticks(ax.get_xticks()[::10])
plt.scatter(x, y, marker='x')
Out[35]:
In [36]:
df1 = df[df.clusters==0]
df2 = df[df.clusters==1]
df3 = df[df.clusters==2]
df4 = df[df.clusters==3]
In [37]:
ax = plt.gca()
plt.scatter(df1.test_items,df1.clusters,color='green')
plt.scatter(df2.test_items,df2.clusters,color='red')
plt.scatter(df3.test_items, df3.clusters,color='blue')
plt.scatter(df4.test_items, df4.clusters,color='black')
plt.xlabel('Age')
ax.set_xticks(ax.get_xticks()[::70])
## plt.axes(xscale='log', yscale='log')## plt.xticks(["veggie", "animals", "countries"])
## plt.locator_params(axis='x', nbins=5)
plt.ylabel('Income ($)')
plt.legend()
Out[37]:
In [38]:
sum_ = []
k_rng = range(feature_matrix.shape[0])
for i in k_rng:
sum_.append(np.sum((feature_matrix[i] - centroid[i])**2)) #here implementing inertia as given in the docs of scikit i.e sum of squared distance..
In [ ]:
sum_
In [40]:
plt.xlabel('K')
plt.ylabel('Sum of squared error')
plt.plot(k_rng,sum_)
Out[40]:
