
label <- c(
    "Raw Ingestion",
    "Date mutation",
    "Selection",
    "Filtering",
    "Col drop",
    "Read First", 
    "Time Window", 
    "UA Agent Pre",
    "UA AGENT",      
    "Asset heuristic",
    "Aticle filtering",  
    "Rate heuristic", 
    "Read time heuristic",
    "ASN Enrichment",
    "ASN filtering 1",   
    "ASN filtering 2",  
    "IP Exclusion",  
    "HONEY POTS",     
    "KPI MEDIAN READTIME",
    "READTIME STATS"
)

cat("\n #### FREAD + DATATABLE #### \n\n")

data_datatable <- read.table("data_table.result", 
                             sep = ",", 
                             header = FALSE
                            )

seconds <- as.data.frame(matrix(data_datatable$V1, 
                            ncol = length(label),
                            byrow=TRUE
                           )
                     )

n_rows <- as.data.frame(matrix(data_datatable$V2, 
                            ncol = length(label),
                            byrow=TRUE
                           )
                     )
n_rows$V1 = data.table::shift(n_rows$V1, 
                              type="lag", 
                              fill=725832)

colnames(seconds) <- label
tot <- 0

for (i in 1:length(label)) {

    val1 <- median(seconds[, i])
    val2 <- n_rows[1, i]

    cat(paste(label[i], 
              val1,
              "secs for",
              val2,
              "rows",
              sep = " "))
    tot <- tot + val1
    cat("\n")

}

cat (paste("TOT:", tot, sep=" "))
cat("\n")
cat(paste("Ingestion time:", 
          median(seconds[, 1]), "\n", 
          sep = " "))
cat(paste("Data manipulation time:", 
          median(
                 rowSums(seconds[, 2:length(label)])
                ), 
          "\n", 
          sep = " "))


cat("\n")

cat("\n #### READR + DPLYR #### \n\n")

data <- read.table("dplyr_readr.result", 
                             sep = ",", 
                             header = FALSE
                        )

cat("\n")

seconds <- as.data.frame(matrix(data$V1, 
                        ncol = length(label),
                        byrow=TRUE
                       )
                 )

n_rows <- as.data.frame(matrix(data$V2, 
                        ncol = length(label),
                        byrow=TRUE
                       )
                 )

n_rows$V1 = data.table::shift(n_rows$V1, 
                              type="lag", 
                              fill=725832)

colnames(seconds) <- label
tot <- 0

for (i in 1:length(label)) {

    val1 <- median(seconds[, i])
    val2 <- n_rows[1, i]

    cat(paste(label[i], 
              val1,
              "secs for",
              val2,
              "rows",
              sep = " "))
    tot <- tot + val1
    cat("\n")

}

cat (paste("TOT:", tot, sep=" "))
cat("\n")
cat(paste("Ingestion time:", 
          median(seconds[, 1]), "\n", 
          sep = " "))
cat(paste("Data manipulation time:", 
          median(
                 rowSums(seconds[, 2:length(label)])
                ), 
          "\n", 
          sep = " "))


cat("\n\n")






