################################################################################
##                                                                            ##
##                            BStats - LAB 3                                  ##
##                      Data Manipulation with dplyr                          ##
##                                                                            ##
##                         Author: Miguel Salema                              ##
##                                                                            ##
################################################################################

##-- Introduction --------------------------------------------------------------

# Lab Overview
# - Loading .csv data files
# - Talk about the "pipe" %>% 
# - Main Goal: Introduction to data wrangling in R

##-- Setup ---------------------------------------------------------------------

# Clean the work environment
rm(list = ls()) # USE with CAUTION: this will delete everything in your environment

# Load packages
library(tidyverse) # loads the core tidyverse packages

# See the working directory
getwd()

##-- Loading Data --------------------------------------------------------------
# See Lab 1

# Load .csv Data
df <- read_csv("../data/qp_sample.csv")

##-- Piping --------------------------------------------------------------------
# See Lab 1

# %>% 

##-- The Basics ----------------------------------------------------------------

# Table info
# Print the names of the variables in the dataset
names(df)

# Count number of observations (rows)
nrow(df)

# or:
df %>% 
  summarize(n_row = n())

# Look at the first rows in the data
df %>% 
  slice(1:3)

# or
head(df)

# select columns
df %>% 
  select(rem_extra, rembase) 

# Print first 3 rows of the variables "salary" and "age"
df %>% 
  select(rembase, age) %>% 
  slice(1:3)

##-- Ordering Data --------------------------------------------------------------

# Ascending order
df %>% 
  arrange(rembase)

# Descending order
df %>% 
  arrange(desc(rembase))


##-- Subsetting Data ------------------------------------------------------------

# Basic subsetting
df %>% 
  filter(age <= 45)

# Create a new tibble with filtered data
df_young <-  df %>% 
  filter(age <= 45)

# Multiple conditions
df %>% 
  filter(age <= 25 & rembase > 10000)

df %>% 
  filter(age < 35 | male == 1)

df %>% 
  filter(nut2_firm == "Lisboa" | nut2_firm == "Alentejo")

# Boolean logic example
TRUE & TRUE
TRUE & FALSE
FALSE & FALSE
TRUE | TRUE
TRUE | FALSE
FALSE | FALSE

##-- Unique Values --------------------------------------------------------------

# Seeing unique values
df %>% 
  distinct(educ)

##-- Exercises ------------------------------------------------------------------

# Exercise 1: How many workers are not between 35 and 50 years old?


##-- Creating New Variables -----------------------------------------------------

# Add new variables that are functions of existing variables
df <- df %>% 
  mutate(total_wage = rembase + rem_extra)

df <- df %>% 
  mutate(age2 = age^2)

# Create variables with specific values
df <- df %>% 
  mutate(one = 1)

# Deleting a variable
df <- df %>% 
  select(-one)

##-- Grouping Data --------------------------------------------------------------

# Count number of workers by education level
df %>% 
  group_by(educ) %>%  
  summarize(n = n()) %>% 
  ungroup()

# Average wage by gender
df %>% 
  group_by(male) %>%  
  summarize(av_wage = mean(total_wage)) %>% 
  ungroup()

# Calculating percentages
df %>% 
  group_by(educ) %>%  
  summarize(n = n()) %>% 
  ungroup() %>% 
  mutate(prop = n/sum(n)) %>% 
  arrange(desc(prop))

##-- Resources ------------------------------------------------------------------

# Introduction to the Tidyverse: https://learn.datacamp.com/courses/introduction-to-the-tidyverse
# Data transformation with dplyr cheatsheet: https://www.rstudio.com/resources/cheatsheets/
# Data tidying with tidyr cheatsheet: https://www.rstudio.com/resources/cheatsheets/
# R reference card: http://cran.r-project.org/doc/contrib/Short-refcard.pdf