Eunseo

.Rhistory

@@ -0,0 +1,51 @@
+library(httr)
+library(rvest)
+url = 'https://quotes.toscrape.com/'
+quote = GET(url)
+print(quote)
+source("~/gobunsil/512_crawling.R", echo=TRUE)
+quote_html = read_html(quote)
+quote_div = quote_html %>%
+html_nodes('.quote') %>%
+html_nodes('.text')
+print(quote_div)
+quote_div = quote_html %>%
+html_nodes('.quote') %>%
+html_nodes('.text')
+install.packages(rvest)
+install.packages('rvest')
+install.packages('httr')
+install.packages("httr")
+quote_html = read_html(quote)
+quote_div = quote_html %>%
+html_nodes('.quote') %>%
+html_nodes('.text')
+library(dplyr)
+quote_html = read_html(quote)
+quote_div = quote_html %>%
+html_nodes('.quote') %>%
+html_nodes('.text')
+quote = GET(url)
+print(quote_div)
+quote_html = read_html(quote)
+library(rvest)
+library(httr)
+library(rvest)
+url = 'https://quotes.toscrape.com/'
+quote = GET(url)
+print(quote)
+quote_html = read_html(quote)
+print(quote_html)
+quote_div = quote_html %>%
+html_nodes('.quote') %>%
+html_nodes('.text')
+print(quote_div)
+library(magrittr)
+library(stringr)
+data = Sys.time()
+write.csv(data, paste0('C:/Users/leebi/Dropbox/lecture/hyu_data_analysis/',Sys.time() %>%
+str_replace_all("[[:punct:]]", "_"), '.csv'))
+write.csv(data, paste0('C:\Users\윤은서\Documents\gobunsil/',Sys.time() %>%
+str_replace_all("[[:punct:]]", "_"), '.csv'))
+write.csv(data, paste0('C:\Users\윤은서\Documents\gobunsil\',Sys.time() %>%
+str_replace_all("[[:punct:]]", "_"), '.csv'))

Clean_Korea.csv

@@ -0,0 +1,50 @@
+Year,Industry,Aggriculture,Manufacturing,Services,GDP
+1972,24.87756871,26.50608056,18.73374417,41.32670994,10862327878
+1973,27.62469922,24.60695095,21.4878874,40.40309012,13876531432
+1974,26.61100569,24.17204301,20.85895003,40.85768501,19544094741
+1975,26.6853826,24.48594408,20.13354073,39.58230585,21784297521
+1976,27.64081588,23.04668067,21.44920126,39.18592118,29902479339
+1977,28.4639485,21.78997318,21.07200628,39.33502077,38446487603
+1978,30.2112942,19.91691347,21.13657596,39.04748653,51972107438
+1979,31.5585622,18.53633847,21.64692013,39.19999506,66946900826
+1980,32.08022132,14.26931587,21.98584774,42.84142771,65398646758
+1981,31.27252375,15.02542793,21.834797,43.03641247,72933350954
+1982,31.68245279,14.00327476,21.83582199,43.62922568,78358866335
+1983,32.97968129,12.62527523,22.97102971,43.0354832,87760360941
+1984,34.5128532,11.87256096,24.57243995,43.05540181,97510235986
+1985,33.9458775,11.75369938,24.22838158,44.2117697,1.01E+11
+1986,34.87480798,10.4282338,25.44506136,44.80617716,1.17E+11
+1987,35.94773283,9.309206904,26.72579126,45.02283525,1.48E+11
+1988,36.59023239,9.221019667,27.60381028,44.76909093,2.00E+11
+1989,36.17361211,8.541523675,26.59500681,46.30745866,2.47E+11
+1990,36.32183897,7.606496334,25.01897224,46.48652099,2.83E+11
+1991,37.14771997,6.824490651,25.18901473,47.14862313,3.31E+11
+1992,35.88131907,6.60760508,24.41078933,48.50843552,3.56E+11
+1993,36.21407742,5.914881371,24.64149999,49.12226709,3.93E+11
+1994,36.24273611,5.658221058,25.30211274,49.15517966,4.64E+11
+1995,36.49125561,5.328672039,25.80146219,49.26055313,5.67E+11
+1996,35.44805561,4.960650984,24.70505809,50.27848579,6.10E+11
+1997,35.39646916,4.47516964,24.66161551,50.75853992,5.70E+11
+1998,35.30828329,4.225940698,25.46710788,52.19709863,3.83E+11
+1999,34.66586525,4.249754418,25.67377289,51.65035937,4.98E+11
+2000,34.75531678,3.857792038,26.44837351,51.61843819,5.76E+11
+2001,33.2209652,3.565819021,24.87906659,53.08394528,5.48E+11
+2002,32.81274988,3.206802038,24.48564896,53.60723336,6.27E+11
+2003,33.11125973,2.964597278,24.12400805,53.87127477,7.03E+11
+2004,34.73483971,2.958018544,26.11841277,53.09698216,7.93E+11
+2005,34.15033175,2.619975731,25.73718379,53.87891643,9.35E+11
+2006,33.517273,2.497470419,25.30537196,54.69918253,1.05E+12
+2007,33.45375008,2.283409085,25.48295331,55.07869334,1.17E+12
+2008,32.51021797,2.142249743,25.62215148,56.18724044,1.05E+12
+2009,32.90536001,2.240556812,25.77490296,55.99857203,9.44E+11
+2010,34.11677597,2.144016322,27.43876659,54.69835731,1.14E+12
+2011,34.45096726,2.209199955,28.23679861,54.64985026,1.25E+12
+2012,34.13469264,2.186185041,27.8318955,55.07320475,1.28E+12
+2013,34.44782919,2.099047114,27.78585374,55.21360303,1.37E+12
+2014,34.09167237,2.05667705,27.04245216,55.64499447,1.48E+12
+2015,34.14993567,2.003907793,26.60603573,55.57650557,1.47E+12
+2016,34.29779968,1.859034883,26.35776522,55.35863931,1.50E+12
+2017,34.76630854,1.850756295,26.94587269,54.84779034,1.62E+12
+2018,34.04580231,1.746403394,26.63850865,55.69169325,1.72E+12
+2019,32.67914892,1.667930979,25.22222287,57.24215056,1.65E+12
+2020,32.58263032,1.832292167,24.81069263,57.12124404,1.64E+12

Cluster_Analysis.R

@@ -0,0 +1,196 @@
+rm(list=ls())
+
+
+library(ggplot2)
+library(NbClust)
+library(factoextra)
+library(fpc)
+library(dbscan)
+library(dplyr)
+library(cluster)
+
+# change the path to where you saved the file
+Clean_Korea <- read.csv("~/Documents/Classes/depaul/Korea/Data/WorldBank_GDP/Korea_GDPMacro_Data/Clean_Korea.csv")
+
+rownames(Clean_Korea) = Clean_Korea$Year
+
+# Remove X, Year, GDP
+data = Clean_Korea[c(-1,-2, -7)]
+head(data)
+
+kmean_clust = NbClust(data, min.nc=3, max.nc=5, method='kmeans')
+
+##### Plot Similarity #####
+
+distance = get_dist(data, method='euclidian')
+fviz_dist(dist(data), gradient=list(low='#00AFBB', mid='white', high='#FC4E07'))
+
+##### K-Means Clustering #####
+
+clust_1 = NbClust(data, min.nc=2, max.nc=10, method='kmeans')
+
+# Based on majority rule, 2 clusters is ideal. 
+
+# Using Elbow method
+fviz_nbclust(data, FUNcluster=kmeans, method='wss')
+
+# With the elbow, we should use 3 or 4 clusters
+
+##### 3 Clusters #####
+km3 = kmeans(data, 3)
+
+## Evaluate Quality of 3 Clusters ##
+km3_stats = cluster.stats(dist(data), km3$cluster, silhouette = TRUE)
+
+km3_stats$cluster.size
+
+km3$centers
+# notice how only aggriculture has a significant difference between the clusters
+#  Industry and Manufacturing have the smallest differences
+
+# Silhouettes above or near 0.50 indicate that the clusters are strong. 
+km3_stats$clus.avg.silwidths
+
+# Within cluster average distance by cluster
+km3_stats$average.distance
+
+# Between cluster average distnace by cluster
+km3_stats$separation
+
+## Evaluate Overall quality of the clustering
+
+# Between cluster average distance
+km3_stats$average.between
+
+# Within cluster average distance
+km3_stats$average.within
+
+# Within cluster sum of squares
+km3_stats$within.cluster.ss
+
+# between cluster sum of squares
+km3$betweenss
+
+# Silhouette metric
+km3_stats$avg.silwidth
+
+## View Assignment of clusters
+table(rownames(data), Cluster=km3$cluster)
+
+
+##### kluster overview function #####
+cluster_overview = function(data, kcluster){
+  k_stats = cluster.stats(dist(data), kcluster$cluster, silhouette = TRUE)
+  print('Cluster Centers')
+  print(kcluster$centers)
+  print('Cluster size')
+  print(k_stats$cluster.size)
+  print('Average Sil Widths')
+  print(k_stats$clus.avg.silwidths)
+  print('Within cluster average distance by cluster')
+  print(k_stats$average.distance)
+  print('Between cluster average distnace by cluster')
+  print(k_stats$separation)
+  print(' Evaluate Overall quality of the clustering')
+  print( 'Between cluster average distance')
+  print(k_stats$average.between)
+  print(' Within cluster average distance')
+  print(k_stats$average.within)
+  print(' Within cluster sum of squares')
+  print(k_stats$within.cluster.ss)
+  print(' between cluster sum of squares')
+  print(kcluster$betweenss)
+  print('Silhouette metric')
+  print(k_stats$avg.silwidth)
+  print(' View Assignment of clusters ')
+  print(table(rownames(data), Cluster=kcluster$cluster))
+}
+
+cluster_overview(data, km3)
+
+##### Different Clusters #####
+km4 = kmeans(data, 4)
+cluster_overview(data, km4)
+
+km5 = kmeans(data, 5)
+cluster_overview(data, km5)
+
+km8 = kmeans(data, 8)
+cluster_overview(data, km8)
+
+
+##### K Medoids #####
+
+# k medoids recommends 3 clusters
+kmedoids = NbClust(data, min.nc=2, max.nc=8, method='median')
+
+# Elbow 
+
+fviz_nbclust(data, pam, method='wss')
+
+km3 = pam(data, 3, diss=FALSE, metric='euclidean')
+cluster_overview(data, km3)
+
+km4 = pam(data, 4, diss=FALSE, metric='euclidean')
+cluster_overview(data, km4)
+
+##### Hierarchical #####
+nc = NbClust(data, min.nc=2, max.nc=10, method='complete')
+
+hc = hclust(dist(data), method='complete')
+
+plot(hc)
+
+rect.hclust(hc, k=4)
+
+hc$cluster = cutree(hc, k=4)
+
+cluster_overview(data, hc)
+
+
+##### Box Plots #####
+# sets plots to print 2x1 per window
+par(mfrow=c(2,1))
+for (i in 1:(ncol(data))){
+  boxplot(data[[i]]~hc$cluster, xlab='cluster', main=names(data)[i])
+}
+
+# resets plot to print 1 per window
+par(mfrow=c(1,1))
+
+
+##### Plotting #####
+
+library(plotly)
+
+Industry = data$Industry
+Manufacturing = data$Manufacturing
+Services = data$Services
+Aggriculture = data$Aggriculture
+
+plot_ly(x=Industry, y=Manufacturing, z=Services, mode='markers', color=Aggriculture)
+
+#
+
+
+
+
+
+#
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+#
+
+#