How to scrape the data

The data from the “Professionally Speaking” website is split by month across many years.

Professionally Speaking Website

I decided to make the urls from a couple of small functions:

# Function to loop through years
ps_year <- function(start.year, end.year) {
        yr <- list()
        n <- 1
        for (i in start.year:end.year) {
                yr[n:(n + 3)] <- rep(i, 4)
                n <- n + 4
                
        }
        return(yr)
}

# Function to loop through months
ps_month <- function() {
        mon <- c("March", "June", "September", "December")
        return(mon)

}

# Function to build professionally speaking URL
ps_url <- function(start.year, end.year) {
        url <- paste0("http://professionallyspeaking.oct.ca/hearings/hearings-",
                      ps_year(start.year, end.year), "-",
                      ps_month(), ".aspx")
        return(url)
        
}

With our functions made, we can now scrape all of the available “Hearings” data:

start.date <- 2012
end.date <- 2018

url <- ps_url(start.date, end.date)
# Use possibly to process urls not yet available
read_url_possibly <- possibly(read_html, otherwise = NULL)
nodes_possibly <- possibly(html_nodes, otherwise = NULL)
text_possibly <- possibly(html_text, otherwise = NULL)

url.data <- url %>%
        map(read_url_possibly) %>%
        map(nodes_possibly, "p") %>%
        map(text_possibly)
        
# View results
head(url.data[[1]], 2)

## [1] "Three-member panels of the Discipline  Committee conduct public hearings  into cases of alleged incompetence or  professional misconduct. The panels  are a mix of elected and appointed  Council members. Members found  guilty of incompetence or professional  misconduct may have their certificate  revoked, suspended or limited. In  professional misconduct matters only,  the committee may also reprimand,  admonish or counsel the member,  impose a fine, order the member to  pay costs, or publish the order in  Professionally Speaking."
## [2] "Member: C. Robert Clements Registration No: 266425Decision: Revoked"

How to clean the data

Ok good. It worked. Let’s clean it up some by removing that first paragraph, and segregating it by case rather than by line:

url.data.clean <- list()

# Remove preamble
for (i in 1:length(url.data)) {
        url.data.clean[i] <- list(i = url.data[[i]][-1])
        
}

# Separate by case, instead of by sentence/line
url.data.clean <- url.data.clean %>%
        map(paste, collapse = " <br><br> ") %>%
        textclean::replace_white() %>%
        map(strsplit, split = "(?<=.)(?=Member: )", perl = TRUE)

# Label list by year-month
names(url.data.clean) <- paste(ps_year(start.date, end.date), 
                               ps_month(),"",
                               sep = "_")

And now, let’s put everything in a data frame instead of working with lists, because I’m not as familiar with lists…

# Establish number of entries
n <- url.data.clean %>%
        unlist() %>%
        length()

# Skeleton data frame
url.data.df <- tibble(id = rep(NA, n),
                      name = NA,
                      gender = NA,
                      text = NA)

# Label date
url.data.df$id <- url.data.clean %>%
        unlist() %>%
        names()

# Extract year
url.data.df$year <- url.data.df$id %>%
        str_extract("20[0-9]+")

# Extract month
url.data.df$month <- url.data.df$id %>%
        str_extract("[A-Za-z]+")

# Place full text into `text` column
url.data.df$text <- url.data.clean %>%
        map(1) %>%
        unlist() %>%
        textclean::replace_white()

Let’s take a look to see what our data frame looks like right now:

Perfect. That’s exactly what we are looking for (namely a structured view of the unstructured entries in Professionally Speaking).

Let’s continue our processing steps to extract the name of the teacher:

## Extract name
# First, we need to add spaces so all words are separated by a space
url.data.df$text <- url.data.df$text %>%
        map(str_replace_all, "([0-9a-zA-Z])([D|R])", "\\1 \\2")

# Next, we add a [space] after ":"
url.data.df$text <- url.data.df$text %>%
        map(str_replace_all, "([:])(.)", "\\1 \\2")


# Looking for all words after "Member: " and before "Registration|Decision"
url.data.df$name <- url.data.df$text %>%
        str_extract("(?s)(?<=Member: )(.*?)(?= (Reg|Dec))") %>%
        textclean::replace_white()

Let’s now extract the “Decision”:

## Extract Decision
# Looking for all words after "Decision: " and before "A panel/etc."
url.data.df$decision <- url.data.df$text %>%
        str_extract("(?s)(?<=Decision: )(.*?)(?= <br>)")

Let’s find the gender of the teacher:

# If "he" is mentioned than gender = male, otherwise gender = female
is_male <- function(txt) {
        tok <- tokenizers::tokenize_words(txt) %>%
                unlist()
        sum.he <- sum(tok == "he" | tok == "him" | tok == "himself")
        sum.she <- sum(tok == "she" | tok == "her" | tok == "herself")
        
        if (sum.he > sum.she) {
                return("male")
        } else {
                return("female")
        }

}

url.data.df$gender <- url.data.df$text %>%
        map(is_male)