Install Packages
install.packages("bindrcpp")
install.packages("gridExtra")
install.packages("reshape2")
install.packages("topicmodels")
install.packages("maps")
install.packages("ggraph")
install.packages("igraph")
install.packages("tm")
install.packages("NLP")
install.packages("wordcloud")
install.packages("RColorBrewer")
install.packages("SnowballC")
install.packages("tidytext")
install.packages("tidyquant")
install.packages("quantmod")
install.packages("TTR")
install.packages("PerformanceAnalytics")
install.packages("xts")
install.packages("zoo")
install.packages("lubridate")
install.packages("forcats")
install.packages("stringr")
install.packages("dplyr")
install.packages("purrr")
install.packages("readr")
install.packages("tidyr")
install.packages("tibble")
install.packages("ggplot2")
install.packages("tidyverse")
install.packages("rtweet")
Load Packages
library(rtweet)
library(tidyverse)
library(tidyquant)
library(ggplot2)
library(tidytext)
library(SnowballC)
library(wordcloud)
library(tm)
library(igraph)
library(ggraph)
library(maps)
library(topicmodels)
library(reshape2)
library(grid)
library(gridExtra)
create lat/lng variables using all available tweet and profile geo-location data
irvine_tweets_latlon <- lat_lng(irvine_tweets)
plot state boundaries
par(mar = c(0, 0, 0, 0))
maps::map("county", regions="california,orange", lwd = .25)
with(irvine_tweets_latlon, points(lng, lat, pch = 20, cex = .75, col = rgb(0, .3, .7, .75)))
Get friends
friends <- get_friends("sociolegaltech")
What languages do my friends speak?
friends %>%
count(lang) %>%
droplevels() %>%
ggplot(aes(x = reorder(lang, desc(n)), y = n)) +
geom_bar(stat = "identity", color = palette_light()[1], fill = palette_light()[1], alpha = 0.8) +
theme_tq() +
theme(axis.text.x = element_text(angle = 45, vjust = 1, hjust = 1)) +
labs(x = "language ISO 639-1 code",
y = "number of friends")
Who are my most “influential” friends (i.e. friends with the biggest network)?
friends %>%
ggplot(aes(x = log2(friends_count))) +
geom_density(color = palette_light()[1], fill = palette_light()[1], alpha = 0.8) +
theme_tq() +
labs(x = "log2 of number of friends",
y = "density")
Tiday Text Analysis
data(stop_words)
Unnest Words
friends_descr <- friends %>%
unnest_tokens(word, description) %>%
mutate(word_stem = wordStem(word)) %>%
anti_join(stop_words, by = "word") %>%
filter(!grepl("\\.|http", word))
Most Commonly Used Words
friends_descr %>%
count(word_stem, sort = TRUE) %>%
filter(n > 20) %>%
ggplot(aes(x = reorder(word_stem, n), y = n)) +
geom_col(color = palette_light()[1], fill = palette_light()[1], alpha = 0.8) +
coord_flip() +
theme_tq() +
labs(x = "",
y = "count of word stem in all followers' descriptions")
Word Cloud
friends_descr %>%
count(word_stem) %>%
mutate(word_stem = removeNumbers(word_stem)) %>%
with(wordcloud(word_stem, n, max.words = 100, colors = palette_light()))
Word Pairs
friends_descr_ngrams <- friends %>%
unnest_tokens(bigram, description, token = "ngrams", n = 2, collapse = FALSE) %>%
filter(!grepl("\\.|http", bigram)) %>%
separate(bigram, c("word1", "word2"), sep = " ") %>%
filter(!word1 %in% stop_words$word) %>%
filter(!word2 %in% stop_words$word)
Count of Words
bigram_counts <- friends_descr_ngrams %>%
count(word1, word2, sort = TRUE)
Graph
bigram_counts %>%
filter(n > 10) %>%
ggplot(aes(x = reorder(word1, -n), y = reorder(word2, -n), fill = n)) +
geom_tile(alpha = 0.8, color = "white") +
scale_fill_gradientn(colours = c(palette_light()[[1]], palette_light()[[2]])) +
coord_flip() +
theme_tq() +
theme(legend.position = "right") +
theme(axis.text.x = element_text(angle = 45, vjust = 1, hjust = 1)) +
labs(x = "first word in pair",
y = "second word in pair")
Graph Word Pairs
bigram_graph <- bigram_counts %>%
filter(n > 5) %>%
graph_from_data_frame()
set.seed(1)
a <- grid::arrow(type = "closed", length = unit(.15, "inches"))
ggraph(bigram_graph, layout = "fr") +
geom_edge_link(aes(edge_alpha = n), show.legend = FALSE,
arrow = a, end_cap = circle(.07, 'inches')) +
geom_node_point(color = palette_light()[1], size = 5, alpha = 0.8) +
geom_node_text(aes(label = name), vjust = 1, hjust = 0.5) +
theme_void()
Negated Meanings
bigrams_separated <- friends %>%
unnest_tokens(bigram, description, token = "ngrams", n = 2, collapse = FALSE) %>%
filter(!grepl("\\.|http", bigram)) %>%
separate(bigram, c("word1", "word2"), sep = " ") %>%
filter(word1 == "not" | word1 == "no") %>%
filter(!word2 %in% stop_words$word)
not_words <- bigrams_separated %>%
filter(word1 == "not") %>%
inner_join(get_sentiments("afinn"), by = c(word2 = "word")) %>%
count(word2, score, sort = TRUE) %>%
ungroup()
not_words %>%
mutate(contribution = n * score) %>%
arrange(desc(abs(contribution))) %>%
head(20) %>%
mutate(word2 = reorder(word2, contribution)) %>%
ggplot(aes(word2, n * score, fill = n * score > 0)) +
geom_col(show.legend = FALSE) +
scale_fill_manual(values = palette_light()) +
labs(x = "",
y = "Sentiment score * number of occurrences",
title = "Words preceded by \"not\"") +
coord_flip() +
theme_tq()
Sentiment Analysis
What is the predominatant sentiment
friends_descr_sentiment <- friends_descr %>%
left_join(select(bigrams_separated, word1, word2), by = c("word" = "word2")) %>%
inner_join(get_sentiments("nrc"), by = "word") %>%
inner_join(get_sentiments("bing"), by = "word") %>%
rename(nrc = sentiment.x, bing = sentiment.y) %>%
mutate(nrc = ifelse(!is.na(word1), NA, nrc),
bing = ifelse(!is.na(word1) & bing == "positive", "negative",
ifelse(!is.na(word1) & bing == "negative", "positive", bing)))
friends_descr_sentiment %>%
filter(nrc != "positive") %>%
filter(nrc != "negative") %>%
gather(x, y, nrc, bing) %>%
count(x, y, sort = TRUE) %>%
filter(n > 10) %>%
ggplot(aes(x = reorder(y, n), y = n)) +
facet_wrap(~ x, scales = "free") +
geom_col(color = palette_light()[1], fill = palette_light()[1], alpha = 0.8) +
coord_flip() +
theme_tq() +
labs(x = "",
y = "count of sentiment in followers' descriptions")
Are followers’ descriptions mostly positive or negative?
friends_descr_sentiment %>%
count(screen_name, word, bing) %>%
group_by(screen_name, bing) %>%
summarise(sum = sum(n)) %>%
spread(bing, sum, fill = 0) %>%
mutate(sentiment = positive - negative) %>%
ggplot(aes(x = sentiment)) +
geom_density(color = palette_light()[1], fill = palette_light()[1], alpha = 0.8) +
theme_tq()
Word Cloud
friends_descr_sentiment %>%
count(word, bing, sort = TRUE) %>%
acast(word ~ bing, value.var = "n", fill = 0) %>%
comparison.cloud(colors = palette_light()[1:2],
max.words = 100)
Topic Modeling
dtm_words_count <- friends_descr %>%
mutate(word_stem = removeNumbers(word_stem)) %>%
count(screen_name, word_stem, sort = TRUE) %>%
ungroup() %>%
filter(word_stem != "") %>%
cast_dtm(screen_name, word_stem, n)
# set a seed so that the output of the model is predictable
dtm_lda <- LDA(dtm_words_count, k = 5, control = list(seed = 1234))
topics_beta <- tidy(dtm_lda, matrix = "beta")
p1 <- topics_beta %>%
filter(grepl("[a-z]+", term)) %>% # some words are Chinese, etc. I don't want these because ggplot doesn't plot them correctly
group_by(topic) %>%
top_n(10, beta) %>%
ungroup() %>%
arrange(topic, -beta) %>%
mutate(term = reorder(term, beta)) %>%
ggplot(aes(term, beta, color = factor(topic), fill = factor(topic))) +
geom_col(show.legend = FALSE, alpha = 0.8) +
scale_color_manual(values = palette_light()) +
scale_fill_manual(values = palette_light()) +
facet_wrap(~ topic, ncol = 5) +
coord_flip() +
theme_tq() +
labs(x = "",
y = "beta (~ occurrence in topics 1-5)",
title = "The top 10 most characteristic words describe topic categories.")
user_topic <- tidy(dtm_lda, matrix = "gamma") %>%
arrange(desc(gamma)) %>%
group_by(document) %>%
top_n(1, gamma)
p2 <- user_topic %>%
group_by(topic) %>%
top_n(10, gamma) %>%
ggplot(aes(x = reorder(document, -gamma), y = gamma, color = factor(topic))) +
facet_wrap(~ topic, scales = "free", ncol = 5) +
geom_point(show.legend = FALSE, size = 4, alpha = 0.8) +
scale_color_manual(values = palette_light()) +
scale_fill_manual(values = palette_light()) +
theme_tq() +
coord_flip() +
labs(x = "",
y = "gamma\n(~ affiliation with topics 1-5)")
Map Your Grids
grid.arrange(p1, p2, ncol = 1, heights = c(0.7, 0.3))
sessionInfo()
LS0tCnRpdGxlOiAiTGVnYWwgQW5hbHl0aWNzIE1vZHVsZSBUd2l0dGVyIgpvdXRwdXQ6CiAgaHRtbF9ub3RlYm9vawotLS0KSW5zdGFsbCBQYWNrYWdlcwpgYGAKaW5zdGFsbC5wYWNrYWdlcygiYmluZHJjcHAiKQppbnN0YWxsLnBhY2thZ2VzKCJncmlkRXh0cmEiKQppbnN0YWxsLnBhY2thZ2VzKCJyZXNoYXBlMiIpCmluc3RhbGwucGFja2FnZXMoInRvcGljbW9kZWxzIikKaW5zdGFsbC5wYWNrYWdlcygibWFwcyIpCmluc3RhbGwucGFja2FnZXMoImdncmFwaCIpCmluc3RhbGwucGFja2FnZXMoImlncmFwaCIpCmluc3RhbGwucGFja2FnZXMoInRtIikKaW5zdGFsbC5wYWNrYWdlcygiTkxQIikKaW5zdGFsbC5wYWNrYWdlcygid29yZGNsb3VkIikKaW5zdGFsbC5wYWNrYWdlcygiUkNvbG9yQnJld2VyIikKaW5zdGFsbC5wYWNrYWdlcygiU25vd2JhbGxDIikKaW5zdGFsbC5wYWNrYWdlcygidGlkeXRleHQiKQppbnN0YWxsLnBhY2thZ2VzKCJ0aWR5cXVhbnQiKQppbnN0YWxsLnBhY2thZ2VzKCJxdWFudG1vZCIpCmluc3RhbGwucGFja2FnZXMoIlRUUiIpCmluc3RhbGwucGFja2FnZXMoIlBlcmZvcm1hbmNlQW5hbHl0aWNzIikKaW5zdGFsbC5wYWNrYWdlcygieHRzIikKaW5zdGFsbC5wYWNrYWdlcygiem9vIikKaW5zdGFsbC5wYWNrYWdlcygibHVicmlkYXRlIikKaW5zdGFsbC5wYWNrYWdlcygiZm9yY2F0cyIpCmluc3RhbGwucGFja2FnZXMoInN0cmluZ3IiKQppbnN0YWxsLnBhY2thZ2VzKCJkcGx5ciIpCmluc3RhbGwucGFja2FnZXMoInB1cnJyIikKaW5zdGFsbC5wYWNrYWdlcygicmVhZHIiKQppbnN0YWxsLnBhY2thZ2VzKCJ0aWR5ciIpCmluc3RhbGwucGFja2FnZXMoInRpYmJsZSIpCmluc3RhbGwucGFja2FnZXMoImdncGxvdDIiKQppbnN0YWxsLnBhY2thZ2VzKCJ0aWR5dmVyc2UiKQppbnN0YWxsLnBhY2thZ2VzKCJydHdlZXQiKQpgYGAKTG9hZCBQYWNrYWdlcwpgYGB7cn0KbGlicmFyeShiaW5kcmNwcCkKbGlicmFyeShncmlkRXh0cmEpCmxpYnJhcnkocmVzaGFwZTIpCmxpYnJhcnkodG9waWNtb2RlbHMpCmxpYnJhcnkobWFwcykKbGlicmFyeShnZ3JhcGgpCmxpYnJhcnkoaWdyYXBoKQpsaWJyYXJ5KHRtKQpsaWJyYXJ5KE5MUCkKbGlicmFyeSh3b3JkY2xvdWQpCmxpYnJhcnkoUkNvbG9yQnJld2VyKQpsaWJyYXJ5KFNub3diYWxsQykKbGlicmFyeSh0aWR5dGV4dCkKbGlicmFyeSh0aWR5cXVhbnQpCmxpYnJhcnkocXVhbnRtb2QpCmxpYnJhcnkoVFRSKQpsaWJyYXJ5KFBlcmZvcm1hbmNlQW5hbHl0aWNzKQpsaWJyYXJ5KHh0cykKbGlicmFyeSh6b28pCmxpYnJhcnkobHVicmlkYXRlKQpsaWJyYXJ5KGZvcmNhdHMpCmxpYnJhcnkoc3RyaW5ncikKbGlicmFyeShkcGx5cikKbGlicmFyeShwdXJycikKbGlicmFyeShyZWFkcikKbGlicmFyeSh0aWR5cikKbGlicmFyeSh0aWJibGUpCmxpYnJhcnkoZ2dwbG90MikKbGlicmFyeSh0aWR5dmVyc2UpCmxpYnJhcnkocnR3ZWV0KQpgYGAKIyMgSW50cm9kdWN0aW9uIHRvIHJ0d2VldAoKY3JlYXRlIGEgdG9rZW4gd2l0aCB0aGUgd2ViIGJyb3dzZXIgbWV0aG9kOiBjcmVhdGUgdG9rZW4gYW5kIHNhdmUgaXQgYXMgYW4gZW52aXJvbm1lbnQgdmFyaWFibGUKCkNyZWF0ZSBhbiBBY2Nlc3MgVG9rZW4gVHV0b3JpYWwKW2h0dHA6Ly9ydHdlZXQuaW5mby9hcnRpY2xlcy9hdXRoLmh0bWxdKGh0dHA6Ly9ydHdlZXQuaW5mby9hcnRpY2xlcy9hdXRoLmh0bWwpCgpgYGB7cn0KY3JlYXRlX3Rva2VuKAogIGFwcCA9ICJzb2Npb2xlZ2FsdGVjaF9SX2FuYWx5c2lzIiwKICBjb25zdW1lcl9rZXkgPSAicnBSWFBTU0hEd0JlakJNeTJTREk4aWtMSiIsCiAgY29uc3VtZXJfc2VjcmV0ID0gIjRWblo2S29yT0VHaTdtS2M4VXVwSVlGZThVVnJrT2dLdDE1T2xpZEh6Y0o5VlRqeVhLIikKYGBgCgogU2VhcmNoIGZvciB0d2VldHMgdXNpbmcgYSBoYXNodGFnLgogYGBgCiBuYW1lLW9mLWRhdGFmcmFtZSA8LSBzZWFyY2hfdHdlZXRzKCAic2VhcmNodGVybSIsIG4gPSBudW1iZXJvZnR3ZWV0cywgaW5jbHVkZV9ydHMgPSBmYWxzZW9ydHJ1ZSkKIGBgYAogCmBgYHtyfQpzZWFyY2hlZF90d2VldHMgPC0gc2VhcmNoX3R3ZWV0cygKICAiI2JpdGNvaW4iLCBuID0gMTgwMDAsIGluY2x1ZGVfcnRzID0gRkFMU0UKKQpgYGAKCllvdSB3aWxsIG5vdyBoYXZlIGEgZGF0YSBmcmFtZSB3aXRoIHR3ZWV0cyB0aGF0IGhhdmUgYmVlbiBzY3JhcGVkIGZyb20gdHdpdHRlci4gCgpZb3UgY2FuIGxvb2sgYXQgdGhlIGRhdGFmcmFtZSBieSBlbnRlcmluZyBgdmlldyhzZWFyY2hlZF90d2VldHMpYAoKSWYgeW91IGRvIG5vdCBoYXZlIGFueSBvYnNlcnZhdGlvbnMsIHNvbWV0aGluZyBkaWQgbm90IHdvcmsgY29ycmVjdGx5LgoKWW91IGNhbiBpbnRlcmFjdCB3aXRoIHRoZSBkYXRhIGZyYW1lLCBzdWNoIGFzIHBsb3R0aW5nIGEgdGltZSBzZXJpZXMuCgpgYGAKdHNfcGxvdChuYW1lb2ZkYXRhZnJhbWUsICJ0aW1lX2ludGVydmFsIikgKwpvdGhlciBvcHRpb25zIHRvIG1ha2UgdGhlIHBsb3QKYGBgCgojIyBwbG90IHRpbWUgc2VyaWVzIG9mIHR3ZWV0cwpgYGB7cn0KdHNfcGxvdChzZWFyY2hlZF90d2VldHMsICIzIGhvdXJzIikgKwogIGdncGxvdDI6OnRoZW1lX21pbmltYWwoKSArCiAgZ2dwbG90Mjo6dGhlbWUocGxvdC50aXRsZSA9IGdncGxvdDI6OmVsZW1lbnRfdGV4dChmYWNlID0gImJvbGQiKSkgKwogIGdncGxvdDI6OmxhYnMoCiAgICB4ID0gTlVMTCwgeSA9IE5VTEwsCiAgICB0aXRsZSA9ICJGcmVxdWVuY3kgb2YgI2xlZ2FsdGVjaCBUd2l0dGVyIHN0YXR1c2VzIGZyb20gcGFzdCA5IGRheXMiLAogICAgc3VidGl0bGUgPSAiVHdpdHRlciBzdGF0dXMgKHR3ZWV0KSBjb3VudHMgYWdncmVnYXRlZCB1c2luZyB0aHJlZS1ob3VyIGludGVydmFscyIsCiAgICBjYXB0aW9uID0gIlxuU291cmNlOiBEYXRhIGNvbGxlY3RlZCBmcm9tIFR3aXR0ZXIncyBSRVNUIEFQSSB2aWEgcnR3ZWV0IgogICkKYGBgCgojIyBzZWFyY2ggZm9yIDEwLDAwMCB0d2VldHMgc2VudCBmcm9tIHRoZSBVUwp5b3UgY2FuIHVzZSB0aGUgc2VhcmNoX3R3ZWV0cyBmdWNudGlvbiBmb3IgdHdlZXRzIHdpdGhpbiBhIHNwZWNpZmllZCBnZW9ncmFwaGljIGFyZWEsIHVzaW5nIGEgR29vZ2xlIEFQSSBrZXksIHRvIGxvb2sgdXAgY29vcmRpbmF0ZXMuCmBgYHtyfQppcnZpbmVfdHdlZXRzIDwtIHNlYXJjaF90d2VldHMoCiAgImxhbmc6ZW4iLCBnZW9jb2RlID0gbG9va3VwX2Nvb3JkcygiaXJ2aW5lLCBDQSIsICJjb3VudHJ5OlVTIiwgYXBpa2V5ID0iQUl6YVN5QTdBdS1YaVVTNkJEMVlsMXlsRWZMaEo5enU4V2MxLW53IiksIG4gPSAxMDAwMAopCmBgYAoKYGBge3J9Cmluc3RhbGwucGFja2FnZXMoImpzb25saXRlIikKbGlicmFyeShqc29ubGl0ZSkKYml0Y29pbl90d2VldHMgPC0gZnJvbUpTT04oJ2h0dHBzOi8vZ2l0aHViLmNvbS9zb2Npb2xlZ2FsdGVjaC9zb2Npb2xlZ2FsdGVjaC5naXRodWIuaW8vcmF3L21hc3Rlci9iaXRjb2luX3R3ZWV0cy5qc29uJykKYGBgCgoKCiMjIGNyZWF0ZSBsYXQvbG5nIHZhcmlhYmxlcyB1c2luZyBhbGwgYXZhaWxhYmxlIHR3ZWV0IGFuZCBwcm9maWxlIGdlby1sb2NhdGlvbiBkYXRhCmBgYHtyfQppcnZpbmVfdHdlZXRzX2xhdGxvbiA8LSBsYXRfbG5nKGlydmluZV90d2VldHMpCmBgYAojIyBwbG90IHN0YXRlIGJvdW5kYXJpZXMKYGBge3J9CnBhcihtYXIgPSBjKDAsIDAsIDAsIDApKQptYXBzOjptYXAoImNvdW50eSIsIHJlZ2lvbnM9ImNhbGlmb3JuaWEsb3JhbmdlIiwgbHdkID0gLjI1KQp3aXRoKGlydmluZV90d2VldHNfbGF0bG9uLCBwb2ludHMobG5nLCBsYXQsIHBjaCA9IDIwLCBjZXggPSAuNzUsIGNvbCA9IHJnYigwLCAuMywgLjcsIC43NSkpKQpgYGAKCiMjIEdldCBUd2VldHMgb2YgQSBVc2VyCmBgYHtyfQp0aW1lbGluZV90d2VldHMgPC0gZ2V0X3RpbWVsaW5lKCJzb2Npb2xlZ2FsdGVjaCIsIGluY2x1ZGVSdHM9VFJVRSkKYGBgCgojIyBHZXQgVHdlZXRzIG9mIEEgVXNlcgpgYGB7cn0KbWVudGlvbnNfdHdlZXRzIDwtIGdldF9tZW50aW9ucygic29jaW9sZWdhbHRlY2giKQpgYGAKCiMjIEdldCBmcmllbmRzCmBgYHtyfQpmcmllbmRzIDwtIGdldF9mcmllbmRzKCJzb2Npb2xlZ2FsdGVjaCIpCmBgYGAKCiMjIEdldCBJbmZvcm1hdGlvbiBvbiBGcmllbmRzCmBgYHtyfQpmcmllbmRzIDwtIGxvb2t1cF91c2VycyhmcmllbmRzJHVzZXJfaWQpCmBgYAoKIyMgV2hhdCBsYW5ndWFnZXMgZG8gbXkgZnJpZW5kcyBzcGVhaz8KYGBge3J9CmZyaWVuZHMgJT4lCiAgY291bnQobGFuZykgJT4lCiAgZHJvcGxldmVscygpICU+JQogIGdncGxvdChhZXMoeCA9IHJlb3JkZXIobGFuZywgZGVzYyhuKSksIHkgPSBuKSkgKwogICAgZ2VvbV9iYXIoc3RhdCA9ICJpZGVudGl0eSIsIGNvbG9yID0gcGFsZXR0ZV9saWdodCgpWzFdLCBmaWxsID0gcGFsZXR0ZV9saWdodCgpWzFdLCBhbHBoYSA9IDAuOCkgKwogICAgdGhlbWVfdHEoKSArCiAgICB0aGVtZShheGlzLnRleHQueCA9IGVsZW1lbnRfdGV4dChhbmdsZSA9IDQ1LCB2anVzdCA9IDEsIGhqdXN0ID0gMSkpICsKICAgIGxhYnMoeCA9ICJsYW5ndWFnZSBJU08gNjM5LTEgY29kZSIsCiAgICAgICAgIHkgPSAibnVtYmVyIG9mIGZyaWVuZHMiKQpgYGAKIyMgV2hvIGFyZSBteSBtb3N0IOKAnGluZmx1ZW50aWFs4oCdIGZyaWVuZHMgKGkuZS4gZnJpZW5kcyB3aXRoIHRoZSBiaWdnZXN0IG5ldHdvcmspPwpgYGB7cn0KZnJpZW5kcyAlPiUKICBnZ3Bsb3QoYWVzKHggPSBsb2cyKGZyaWVuZHNfY291bnQpKSkgKwogICAgZ2VvbV9kZW5zaXR5KGNvbG9yID0gcGFsZXR0ZV9saWdodCgpWzFdLCBmaWxsID0gcGFsZXR0ZV9saWdodCgpWzFdLCBhbHBoYSA9IDAuOCkgKwogICAgdGhlbWVfdHEoKSArCiAgICBsYWJzKHggPSAibG9nMiBvZiBudW1iZXIgb2YgZnJpZW5kcyIsCiAgICAgICAgIHkgPSAiZGVuc2l0eSIpCmBgYAoKIyMgSG93IGFjdGl2ZSBhcmUgbXkgZm9sbG93ZXJzIChpLmUuIGhvdyBvZnRlbiBkbyB0aGV5IHR3ZWV0KQpgYGB7cn0KZnJpZW5kcyAlPiUKICBtdXRhdGUoZGF0ZSA9IGFzLkRhdGUoYWNjb3VudF9jcmVhdGVkX2F0LCBmb3JtYXQgPSAiJVktJW0tJWQiKSwKICAgICAgICAgdG9kYXkgPSBhcy5EYXRlKCIyMDE4LTA2LTIyIiwgZm9ybWF0ID0gIiVZLSVtLSVkIiksCiAgICAgICAgIGRheXMgPSBhcy5udW1lcmljKHRvZGF5IC0gZGF0ZSksCiAgICAgICAgIHN0YXR1c2VzQ291bnRfcERheSA9IHN0YXR1c2VzX2NvdW50IC8gZGF5cykgJT4lCiAgZ2dwbG90KGFlcyh4ID0gbG9nMihzdGF0dXNlc0NvdW50X3BEYXkpKSkgKwogICAgZ2VvbV9kZW5zaXR5KGNvbG9yID0gcGFsZXR0ZV9saWdodCgpWzFdLCBmaWxsID0gcGFsZXR0ZV9saWdodCgpWzFdLCBhbHBoYSA9IDAuOCkgKwogICAgdGhlbWVfdHEoKQpgYGAKCiMjIFRpZGF5IFRleHQgQW5hbHlzaXMKCmBgYHtyfQpkYXRhKHN0b3Bfd29yZHMpCmBgYAoKIyMgVW5uZXN0IFdvcmRzCmBgYHtyfQpmcmllbmRzX2Rlc2NyIDwtIGZyaWVuZHMgJT4lCiAgdW5uZXN0X3Rva2Vucyh3b3JkLCBkZXNjcmlwdGlvbikgJT4lCiAgbXV0YXRlKHdvcmRfc3RlbSA9IHdvcmRTdGVtKHdvcmQpKSAlPiUKICBhbnRpX2pvaW4oc3RvcF93b3JkcywgYnkgPSAid29yZCIpICU+JQogIGZpbHRlcighZ3JlcGwoIlxcLnxodHRwIiwgd29yZCkpCmBgYAoKIyMgTW9zdCBDb21tb25seSBVc2VkIFdvcmRzCmBgYHtyfQpmcmllbmRzX2Rlc2NyICU+JQogIGNvdW50KHdvcmRfc3RlbSwgc29ydCA9IFRSVUUpICU+JQogIGZpbHRlcihuID4gMjApICU+JQogIGdncGxvdChhZXMoeCA9IHJlb3JkZXIod29yZF9zdGVtLCBuKSwgeSA9IG4pKSArCiAgICBnZW9tX2NvbChjb2xvciA9IHBhbGV0dGVfbGlnaHQoKVsxXSwgZmlsbCA9IHBhbGV0dGVfbGlnaHQoKVsxXSwgYWxwaGEgPSAwLjgpICsKICAgIGNvb3JkX2ZsaXAoKSArCiAgICB0aGVtZV90cSgpICsKICAgIGxhYnMoeCA9ICIiLAogICAgICAgICB5ID0gImNvdW50IG9mIHdvcmQgc3RlbSBpbiBhbGwgZm9sbG93ZXJzJyBkZXNjcmlwdGlvbnMiKQpgYGAKCiMjIFdvcmQgQ2xvdWQKYGBge3J9CmZyaWVuZHNfZGVzY3IgJT4lCiAgY291bnQod29yZF9zdGVtKSAlPiUKICBtdXRhdGUod29yZF9zdGVtID0gcmVtb3ZlTnVtYmVycyh3b3JkX3N0ZW0pKSAlPiUKICB3aXRoKHdvcmRjbG91ZCh3b3JkX3N0ZW0sIG4sIG1heC53b3JkcyA9IDEwMCwgY29sb3JzID0gcGFsZXR0ZV9saWdodCgpKSkKYGBgCgojIyBXb3JkIFBhaXJzCmBgYHtyfQpmcmllbmRzX2Rlc2NyX25ncmFtcyA8LSBmcmllbmRzICU+JQogIHVubmVzdF90b2tlbnMoYmlncmFtLCBkZXNjcmlwdGlvbiwgdG9rZW4gPSAibmdyYW1zIiwgbiA9IDIsIGNvbGxhcHNlID0gRkFMU0UpICU+JQogIGZpbHRlcighZ3JlcGwoIlxcLnxodHRwIiwgYmlncmFtKSkgJT4lCiAgc2VwYXJhdGUoYmlncmFtLCBjKCJ3b3JkMSIsICJ3b3JkMiIpLCBzZXAgPSAiICIpICU+JQogIGZpbHRlcighd29yZDEgJWluJSBzdG9wX3dvcmRzJHdvcmQpICU+JQogIGZpbHRlcighd29yZDIgJWluJSBzdG9wX3dvcmRzJHdvcmQpCmBgYAoKIyMgQ291bnQgb2YgV29yZHMKYGBge3J9CmJpZ3JhbV9jb3VudHMgPC0gZnJpZW5kc19kZXNjcl9uZ3JhbXMgJT4lCiAgY291bnQod29yZDEsIHdvcmQyLCBzb3J0ID0gVFJVRSkKYGBgCgojIyBHcmFwaCAKYGBge3J9CmJpZ3JhbV9jb3VudHMgJT4lCiAgZmlsdGVyKG4gPiAxMCkgJT4lCiAgZ2dwbG90KGFlcyh4ID0gcmVvcmRlcih3b3JkMSwgLW4pLCB5ID0gcmVvcmRlcih3b3JkMiwgLW4pLCBmaWxsID0gbikpICsKICAgIGdlb21fdGlsZShhbHBoYSA9IDAuOCwgY29sb3IgPSAid2hpdGUiKSArCiAgICBzY2FsZV9maWxsX2dyYWRpZW50bihjb2xvdXJzID0gYyhwYWxldHRlX2xpZ2h0KClbWzFdXSwgcGFsZXR0ZV9saWdodCgpW1syXV0pKSArCiAgICBjb29yZF9mbGlwKCkgKwogICAgdGhlbWVfdHEoKSArCiAgICB0aGVtZShsZWdlbmQucG9zaXRpb24gPSAicmlnaHQiKSArCiAgICB0aGVtZShheGlzLnRleHQueCA9IGVsZW1lbnRfdGV4dChhbmdsZSA9IDQ1LCB2anVzdCA9IDEsIGhqdXN0ID0gMSkpICsKICAgIGxhYnMoeCA9ICJmaXJzdCB3b3JkIGluIHBhaXIiLAogICAgICAgICB5ID0gInNlY29uZCB3b3JkIGluIHBhaXIiKQpgYGAKCiMgR3JhcGggV29yZCBQYWlycwpgYGB7cn0KYmlncmFtX2dyYXBoIDwtIGJpZ3JhbV9jb3VudHMgJT4lCiAgZmlsdGVyKG4gPiA1KSAlPiUKICBncmFwaF9mcm9tX2RhdGFfZnJhbWUoKQoKc2V0LnNlZWQoMSkKCmEgPC0gZ3JpZDo6YXJyb3codHlwZSA9ICJjbG9zZWQiLCBsZW5ndGggPSB1bml0KC4xNSwgImluY2hlcyIpKQoKZ2dyYXBoKGJpZ3JhbV9ncmFwaCwgbGF5b3V0ID0gImZyIikgKwogIGdlb21fZWRnZV9saW5rKGFlcyhlZGdlX2FscGhhID0gbiksIHNob3cubGVnZW5kID0gRkFMU0UsCiAgICAgICAgICAgICAgICAgYXJyb3cgPSBhLCBlbmRfY2FwID0gY2lyY2xlKC4wNywgJ2luY2hlcycpKSArCiAgZ2VvbV9ub2RlX3BvaW50KGNvbG9yID0gIHBhbGV0dGVfbGlnaHQoKVsxXSwgc2l6ZSA9IDUsIGFscGhhID0gMC44KSArCiAgZ2VvbV9ub2RlX3RleHQoYWVzKGxhYmVsID0gbmFtZSksIHZqdXN0ID0gMSwgaGp1c3QgPSAwLjUpICsKICB0aGVtZV92b2lkKCkKYGBgCgojIE5lZ2F0ZWQgTWVhbmluZ3MKYGBge3J9CmJpZ3JhbXNfc2VwYXJhdGVkIDwtIGZyaWVuZHMgJT4lCiAgdW5uZXN0X3Rva2VucyhiaWdyYW0sIGRlc2NyaXB0aW9uLCB0b2tlbiA9ICJuZ3JhbXMiLCBuID0gMiwgY29sbGFwc2UgPSBGQUxTRSkgJT4lCiAgZmlsdGVyKCFncmVwbCgiXFwufGh0dHAiLCBiaWdyYW0pKSAlPiUKICBzZXBhcmF0ZShiaWdyYW0sIGMoIndvcmQxIiwgIndvcmQyIiksIHNlcCA9ICIgIikgJT4lCiAgZmlsdGVyKHdvcmQxID09ICJub3QiIHwgd29yZDEgPT0gIm5vIikgJT4lCiAgZmlsdGVyKCF3b3JkMiAlaW4lIHN0b3Bfd29yZHMkd29yZCkKCm5vdF93b3JkcyA8LSBiaWdyYW1zX3NlcGFyYXRlZCAlPiUKICBmaWx0ZXIod29yZDEgPT0gIm5vdCIpICU+JQogIGlubmVyX2pvaW4oZ2V0X3NlbnRpbWVudHMoImFmaW5uIiksIGJ5ID0gYyh3b3JkMiA9ICJ3b3JkIikpICU+JQogIGNvdW50KHdvcmQyLCBzY29yZSwgc29ydCA9IFRSVUUpICU+JQogIHVuZ3JvdXAoKQpgYGAKCmBgYHtyfQpub3Rfd29yZHMgJT4lCiAgbXV0YXRlKGNvbnRyaWJ1dGlvbiA9IG4gKiBzY29yZSkgJT4lCiAgYXJyYW5nZShkZXNjKGFicyhjb250cmlidXRpb24pKSkgJT4lCiAgaGVhZCgyMCkgJT4lCiAgbXV0YXRlKHdvcmQyID0gcmVvcmRlcih3b3JkMiwgY29udHJpYnV0aW9uKSkgJT4lCiAgZ2dwbG90KGFlcyh3b3JkMiwgbiAqIHNjb3JlLCBmaWxsID0gbiAqIHNjb3JlID4gMCkpICsKICAgIGdlb21fY29sKHNob3cubGVnZW5kID0gRkFMU0UpICsKICAgIHNjYWxlX2ZpbGxfbWFudWFsKHZhbHVlcyA9IHBhbGV0dGVfbGlnaHQoKSkgKwogICAgbGFicyh4ID0gIiIsCiAgICAgICAgIHkgPSAiU2VudGltZW50IHNjb3JlICogbnVtYmVyIG9mIG9jY3VycmVuY2VzIiwKICAgICAgICAgdGl0bGUgPSAiV29yZHMgcHJlY2VkZWQgYnkgXCJub3RcIiIpICsKICAgIGNvb3JkX2ZsaXAoKSArCiAgICB0aGVtZV90cSgpCmBgYAoKIyMgU2VudGltZW50IEFuYWx5c2lzCiMjIFdoYXQgaXMgdGhlIHByZWRvbWluYXRhbnQgc2VudGltZW50CmBgYHtyfQpmcmllbmRzX2Rlc2NyX3NlbnRpbWVudCA8LSBmcmllbmRzX2Rlc2NyICU+JQogIGxlZnRfam9pbihzZWxlY3QoYmlncmFtc19zZXBhcmF0ZWQsIHdvcmQxLCB3b3JkMiksIGJ5ID0gYygid29yZCIgPSAid29yZDIiKSkgJT4lCiAgaW5uZXJfam9pbihnZXRfc2VudGltZW50cygibnJjIiksIGJ5ID0gIndvcmQiKSAlPiUKICBpbm5lcl9qb2luKGdldF9zZW50aW1lbnRzKCJiaW5nIiksIGJ5ID0gIndvcmQiKSAlPiUKICByZW5hbWUobnJjID0gc2VudGltZW50LngsIGJpbmcgPSBzZW50aW1lbnQueSkgJT4lCiAgbXV0YXRlKG5yYyA9IGlmZWxzZSghaXMubmEod29yZDEpLCBOQSwgbnJjKSwKICAgICAgICAgYmluZyA9IGlmZWxzZSghaXMubmEod29yZDEpICYgYmluZyA9PSAicG9zaXRpdmUiLCAibmVnYXRpdmUiLAogICAgICAgICAgICAgICAgICAgICAgIGlmZWxzZSghaXMubmEod29yZDEpICYgYmluZyA9PSAibmVnYXRpdmUiLCAicG9zaXRpdmUiLCBiaW5nKSkpCmBgYAoKYGBge3J9CmZyaWVuZHNfZGVzY3Jfc2VudGltZW50ICU+JQogIGZpbHRlcihucmMgIT0gInBvc2l0aXZlIikgJT4lCiAgZmlsdGVyKG5yYyAhPSAibmVnYXRpdmUiKSAlPiUKICBnYXRoZXIoeCwgeSwgbnJjLCBiaW5nKSAlPiUKICBjb3VudCh4LCB5LCBzb3J0ID0gVFJVRSkgJT4lCiAgZmlsdGVyKG4gPiAxMCkgJT4lCiAgZ2dwbG90KGFlcyh4ID0gcmVvcmRlcih5LCBuKSwgeSA9IG4pKSArCiAgICBmYWNldF93cmFwKH4geCwgc2NhbGVzID0gImZyZWUiKSArCiAgICBnZW9tX2NvbChjb2xvciA9IHBhbGV0dGVfbGlnaHQoKVsxXSwgZmlsbCA9IHBhbGV0dGVfbGlnaHQoKVsxXSwgYWxwaGEgPSAwLjgpICsKICAgIGNvb3JkX2ZsaXAoKSArCiAgICB0aGVtZV90cSgpICsKICAgIGxhYnMoeCA9ICIiLAogICAgICAgICB5ID0gImNvdW50IG9mIHNlbnRpbWVudCBpbiBmb2xsb3dlcnMnIGRlc2NyaXB0aW9ucyIpCmBgYAoKIyMgQXJlIGZvbGxvd2Vyc+KAmSBkZXNjcmlwdGlvbnMgbW9zdGx5IHBvc2l0aXZlIG9yIG5lZ2F0aXZlPwpgYGB7cn0KZnJpZW5kc19kZXNjcl9zZW50aW1lbnQgJT4lCiAgY291bnQoc2NyZWVuX25hbWUsIHdvcmQsIGJpbmcpICU+JQogIGdyb3VwX2J5KHNjcmVlbl9uYW1lLCBiaW5nKSAlPiUKICBzdW1tYXJpc2Uoc3VtID0gc3VtKG4pKSAlPiUKICBzcHJlYWQoYmluZywgc3VtLCBmaWxsID0gMCkgJT4lCiAgbXV0YXRlKHNlbnRpbWVudCA9IHBvc2l0aXZlIC0gbmVnYXRpdmUpICU+JQogIGdncGxvdChhZXMoeCA9IHNlbnRpbWVudCkpICsKICAgIGdlb21fZGVuc2l0eShjb2xvciA9IHBhbGV0dGVfbGlnaHQoKVsxXSwgZmlsbCA9IHBhbGV0dGVfbGlnaHQoKVsxXSwgYWxwaGEgPSAwLjgpICsKICAgIHRoZW1lX3RxKCkKYGBgCiMjIFdvcmQgQ2xvdWQKYGBge3J9CmZyaWVuZHNfZGVzY3Jfc2VudGltZW50ICU+JQogIGNvdW50KHdvcmQsIGJpbmcsIHNvcnQgPSBUUlVFKSAlPiUKICBhY2FzdCh3b3JkIH4gYmluZywgdmFsdWUudmFyID0gIm4iLCBmaWxsID0gMCkgJT4lCiAgY29tcGFyaXNvbi5jbG91ZChjb2xvcnMgPSBwYWxldHRlX2xpZ2h0KClbMToyXSwKICAgICAgICAgICAgICAgICAgIG1heC53b3JkcyA9IDEwMCkKYGBgCgojIFRvcGljIE1vZGVsaW5nCmBgYHtyfQpkdG1fd29yZHNfY291bnQgPC0gZnJpZW5kc19kZXNjciAlPiUKICBtdXRhdGUod29yZF9zdGVtID0gcmVtb3ZlTnVtYmVycyh3b3JkX3N0ZW0pKSAlPiUKICBjb3VudChzY3JlZW5fbmFtZSwgd29yZF9zdGVtLCBzb3J0ID0gVFJVRSkgJT4lCiAgdW5ncm91cCgpICU+JQogIGZpbHRlcih3b3JkX3N0ZW0gIT0gIiIpICU+JQogIGNhc3RfZHRtKHNjcmVlbl9uYW1lLCB3b3JkX3N0ZW0sIG4pCgojIHNldCBhIHNlZWQgc28gdGhhdCB0aGUgb3V0cHV0IG9mIHRoZSBtb2RlbCBpcyBwcmVkaWN0YWJsZQpkdG1fbGRhIDwtIExEQShkdG1fd29yZHNfY291bnQsIGsgPSA1LCBjb250cm9sID0gbGlzdChzZWVkID0gMTIzNCkpCgp0b3BpY3NfYmV0YSA8LSB0aWR5KGR0bV9sZGEsIG1hdHJpeCA9ICJiZXRhIikKYGBgCgpgYGB7cn0KcDEgPC0gdG9waWNzX2JldGEgJT4lCiAgZmlsdGVyKGdyZXBsKCJbYS16XSsiLCB0ZXJtKSkgJT4lICMgc29tZSB3b3JkcyBhcmUgQ2hpbmVzZSwgZXRjLiBJIGRvbid0IHdhbnQgdGhlc2UgYmVjYXVzZSBnZ3Bsb3QgZG9lc24ndCBwbG90IHRoZW0gY29ycmVjdGx5CiAgZ3JvdXBfYnkodG9waWMpICU+JQogIHRvcF9uKDEwLCBiZXRhKSAlPiUKICB1bmdyb3VwKCkgJT4lCiAgYXJyYW5nZSh0b3BpYywgLWJldGEpICU+JQogIG11dGF0ZSh0ZXJtID0gcmVvcmRlcih0ZXJtLCBiZXRhKSkgJT4lCiAgZ2dwbG90KGFlcyh0ZXJtLCBiZXRhLCBjb2xvciA9IGZhY3Rvcih0b3BpYyksIGZpbGwgPSBmYWN0b3IodG9waWMpKSkgKwogICAgZ2VvbV9jb2woc2hvdy5sZWdlbmQgPSBGQUxTRSwgYWxwaGEgPSAwLjgpICsKICAgIHNjYWxlX2NvbG9yX21hbnVhbCh2YWx1ZXMgPSBwYWxldHRlX2xpZ2h0KCkpICsKICAgIHNjYWxlX2ZpbGxfbWFudWFsKHZhbHVlcyA9IHBhbGV0dGVfbGlnaHQoKSkgKwogICAgZmFjZXRfd3JhcCh+IHRvcGljLCBuY29sID0gNSkgKwogICAgY29vcmRfZmxpcCgpICsKICAgIHRoZW1lX3RxKCkgKwogICAgbGFicyh4ID0gIiIsCiAgICAgICAgIHkgPSAiYmV0YSAofiBvY2N1cnJlbmNlIGluIHRvcGljcyAxLTUpIiwKICAgICAgICAgdGl0bGUgPSAiVGhlIHRvcCAxMCBtb3N0IGNoYXJhY3RlcmlzdGljIHdvcmRzIGRlc2NyaWJlIHRvcGljIGNhdGVnb3JpZXMuIikKYGBgCgpgYGB7cn0KdXNlcl90b3BpYyA8LSB0aWR5KGR0bV9sZGEsIG1hdHJpeCA9ICJnYW1tYSIpICU+JQogIGFycmFuZ2UoZGVzYyhnYW1tYSkpICU+JQogIGdyb3VwX2J5KGRvY3VtZW50KSAlPiUKICB0b3BfbigxLCBnYW1tYSkKYGBgCgpgYGB7cn0KcDIgPC0gdXNlcl90b3BpYyAlPiUKICBncm91cF9ieSh0b3BpYykgJT4lCiAgdG9wX24oMTAsIGdhbW1hKSAlPiUKICBnZ3Bsb3QoYWVzKHggPSByZW9yZGVyKGRvY3VtZW50LCAtZ2FtbWEpLCB5ID0gZ2FtbWEsIGNvbG9yID0gZmFjdG9yKHRvcGljKSkpICsKICAgIGZhY2V0X3dyYXAofiB0b3BpYywgc2NhbGVzID0gImZyZWUiLCBuY29sID0gNSkgKwogICAgZ2VvbV9wb2ludChzaG93LmxlZ2VuZCA9IEZBTFNFLCBzaXplID0gNCwgYWxwaGEgPSAwLjgpICsKICAgIHNjYWxlX2NvbG9yX21hbnVhbCh2YWx1ZXMgPSBwYWxldHRlX2xpZ2h0KCkpICsKICAgIHNjYWxlX2ZpbGxfbWFudWFsKHZhbHVlcyA9IHBhbGV0dGVfbGlnaHQoKSkgKwogICAgdGhlbWVfdHEoKSArCiAgICBjb29yZF9mbGlwKCkgKwogICAgbGFicyh4ID0gIiIsCiAgICAgICAgIHkgPSAiZ2FtbWFcbih+IGFmZmlsaWF0aW9uIHdpdGggdG9waWNzIDEtNSkiKQpgYGAKCiMjIE1hcCBZb3VyIEdyaWRzCmBgYHtyfQpncmlkLmFycmFuZ2UocDEsIHAyLCBuY29sID0gMSwgaGVpZ2h0cyA9IGMoMC43LCAwLjMpKQpgYGAKCmBgYHtyfQpzZXNzaW9uSW5mbygpCmBgYA==