# get link to all posts and their filename
posts <- gh::gh("/repos/:owner/:repo/contents/:path", 
                owner = "lockedatapublished",
                repo = "blog",
                path = "content/posts")

gh_posts <- tibble::tibble(name = purrr::map_chr(posts, "name"),
                           path = purrr::map_chr(posts, "path"),
                           raw = purrr::map_chr(posts, "download_url"))

library("magrittr")
gh_posts %>%
  head() %>%
  knitr::kable()

get_pics <- function(path){
    message(path)
    file <- gh::gh("/repos/:owner/:repo/contents/:path", 
                   owner = "lockedatapublished",
                   repo = "blog",
                   path = path)
    content <- rawToChar(base64enc::base64decode(file$content))
    # get links to imgs
    img <- stringr::str_match(content, 'src=\\\".*?\\/img\\/(.*?)\"' )[,2]
    tibble::tibble(path = path,
                   img = img)
}

get_pics("content/posts/2013-05-11-setting-up-wordpress-on-azure.md") %>%
  knitr::kable()

pics <- purrr::map_df(gh_posts$path, get_pics)
gh_pics <- dplyr::left_join(gh_posts, pics, by = "path")
gh_pics <- dplyr::filter(gh_pics, !is.na(img))
readr::write_csv(gh_pics, path = "data/gh_imgs.csv")

get_one_yaml <- function(path){
  print(path)
  file <- gh::gh("/repos/:owner/:repo/contents/:path", 
                 owner = "lockedatapublished",
                 repo = "blog",
                 path = path)
  content <- rawToChar(base64enc::base64decode(file$content))
  
  # the yaml function didn't like this
  content <- stringr::str_replace_all(content, "“", "")
  content <- stringr::str_replace_all(content, "â€\u009d", "")
  write(content, "temporary.md")
  data <- rmarkdown::yaml_front_matter("temporary.md")
  file.remove("temporary.md")
  
  # is this an elegant solution? No :-)
  # but this way I'll get both categories and tags
  # and won't get issues if several categories/tags
  categories <- data$categories
  data$categories <- NULL
  data <- dplyr::as_data_frame(data)
  
  
  if("tags" %in% names(data)){
    
    data <- dplyr::mutate(data, value = TRUE)
    data <- tidyr::spread(data, tags, value, fill = FALSE)
    data <- dplyr::mutate(data, path = path)
  }
  
  
  if(!is.null(categories)){
    categories <- dplyr::tibble(categories = paste0("cat_", categories))
    categories <- dplyr::mutate(categories, value = TRUE)
    categories <- tidyr::spread(categories, categories, value, fill = FALSE)
  }
  
  data <- cbind(data, categories)
  data
}

get_one_yaml("content/posts/2013-05-11-setting-up-wordpress-on-azure.md") %>%
  knitr::kable()

info <- purrr::map_df(gh_posts$path, get_one_yaml)
gh_posts <- dplyr::left_join(gh_posts, info, by = "path")

readr::write_csv(gh_posts, path = "data/gh_posts.csv")

library("magrittr")

# get links and tags
sitemap <- xml2::read_xml("https://itsalocke.com/blog/sitemap.xml") %>% 
  xml2::as_list() %>%
  .$urlset

# probably re-inventing the wheel
get_one <- function(element, what){
  one <- unlist(element[[what]])
  if(is.null(one)){
    one <- ""
  }
  
  one
}

# tibble with everything
sitemap <- tibble::tibble(url = purrr::map_chr(sitemap, get_one, "loc"),
                       date = purrr::map_chr(sitemap, get_one, "lastmod"))

# only blog posts
blog <- dplyr::filter(sitemap, !stringr::str_detect(url, "tags\\/"))
blog <- dplyr::filter(blog, !stringr::str_detect(url, "categories\\/"))
blog <- dplyr::filter(blog, !stringr::str_detect(url, "statuses\\/"))
blog <- dplyr::filter(blog, url != "https://itsalocke.com/blog/stuff-i-read-this-week/")
blog <- dplyr::filter(blog, !stringr::str_detect(url, "https://itsalocke.com/blog/.*?\\/.*?\\/"))
blog <- dplyr::filter(blog, url != "https://itsalocke.com/blog/")
blog <- dplyr::filter(blog, url != "https://itsalocke.com/blog/posts/")

head(blog) %>%
  knitr::kable()

gh_info <- readr::read_csv("data/gh_posts.csv")
gh_info <- dplyr::filter(gh_info, !stringr::str_detect(name, "\\.Rmd"))

unique(gh_info$slug)

# https://github.com/rstudio/blogdown/blob/0c4c30dbfb3ae77b27594685902873d63c2894ad/R/utils.R#L277
dash_filename = function(string, pattern = '[^[:alnum:]^\\.]+') {
  tolower(string) %>%
    stringr::str_replace_all("â", "") %>%
    stringr::str_replace_all("DataOps.*? it.*?s a thing (honest)",
                             "dataops--its-a-thing-honest") %>%
    stringr::str_replace_all(pattern, '-') %>%
    stringr::str_replace_all('^-+|-+$', '') 
    
}
gh_info <- dplyr::mutate(gh_info, 
                         base = ifelse(!is.na(slug), slug, title),
                         base = dash_filename(base),
                         false_url = paste0("https://itsalocke.com/blog/", 
                                      base, "/"))

tail(gh_info$false_url)

all_info <- fuzzyjoin::stringdist_left_join(blog, gh_info, 
                                            by = c("url" = "false_url"),
                                            max_dist = 3)
all_info$url[(is.na(all_info$raw))]

all_info$title[duplicated(all_info$raw)]

readr::write_csv(all_info, path = "data/all_info_about_posts.csv")

head(all_info) %>%
  knitr::kable()

library("ggplot2")
all_info <- dplyr::mutate(all_info, date = anytime::anytime(date.x))
ggplot(all_info) +
  geom_point(aes(date), y = 0.5, col = "#2165B6", size = 0.9) +
  hrbrthemes::theme_ipsum(grid = "Y") 

all_info <- dplyr::select(all_info, - base, - false_url)
categories_info <- all_info %>%
  tidyr::gather("category", "value", 11:ncol(all_info)) %>%
  dplyr::filter(!is.na(value)) %>%
  dplyr::filter(stringr::str_detect(category, "cat\\_")) %>%
  dplyr::mutate(category = stringr::str_replace(category, "cat\\_", ""))
categories <- categories_info %>%
  dplyr::count(category, sort = TRUE) 

knitr::kable(categories)

categories_info <- dplyr::mutate(categories_info, date = anytime::anytime(date.x))
ggplot(categories_info) +
  geom_point(aes(date, category), col = "#2165B6", size = 0.9) +
  hrbrthemes::theme_ipsum(grid = "Y")