Updated parsers

Miguel Tuñón
1 parent ddea4061
Showing 17 changed files with 267 additions and 72 deletions
ISO27001effectiveness/DESCRIPTION
ISO27001effectiveness/NAMESPACE
ISO27001effectiveness/R/Hackmageddon_Parser.R
ISO27001effectiveness/R/ISOSurvey_Parser.R
ISO27001effectiveness/R/Util.R
ISO27001effectiveness/man/FilterMultiCountry.Rd
ISO27001effectiveness/man/GetCountryAbrev.Rd
ISO27001effectiveness/man/GetDefaultAttacksData.Rd
ISO27001effectiveness/man/GetISOSurveyCertsPerCountry.Rd
ISO27001effectiveness/man/GetISOSurveyCertsPerSector.Rd
ISO27001effectiveness/man/GetISOSurveySitesPerCountry.Rd
ISO27001effectiveness/man/LoadParserLibraries.Rd
ISO27001effectiveness/man/ParseHMExcel.Rd
ISO27001effectiveness/man/ParseHMFolder.Rd
ISO27001effectiveness/man/ProccesISOSurveyByCountryRaw.Rd
ISO27001effectiveness/man/ProccesISOSurveyRaw.Rd
ISO27001effectiveness/man/ProcessHMRaw.Rd
 Package: ISO27001effectiveness
 Type: Package
-Title: What the Package Does (Title Case)
+Title: Study about how ISO27001 certifications affect cyberattacks
 Version: 0.1.0
 Author: Who wrote it
 Maintainer: The package maintainer <yourself@somewhere.net>
-Description: More about what it does (maybe more than one line)
-    Use four spaces when indenting paragraphs within the Description.
+Description: Compare the cyberattacks reported in hackmaggedon website with data
+    obtained from the official ISO survey to 27001. Working around countries, 
+    industrial sectors, etc.
 License: What license is it under?
 Encoding: UTF-8
 LazyData: true
 RoxygenNote: 5.0.1
 Imports: xlsx,
-    ggplot2
+    ggplot2,
+    countrycode
 # Generated by roxygen2: do not edit by hand
  
+export(GetDefaultAttacksData)
+export(GetISOSurveyCertsPerCountry)
+export(GetISOSurveyCertsPerSector)
+export(GetISOSurveySitesPerCountry)
+export(ParseHMExcel)
+export(ParseHMFolder)
+export(ProccesISOSurveyByCountryRaw)
+export(ProccesISOSurveyRaw)
+export(ProcessHMRaw)
@@ -49,16 +49,31 @@ ProcessHMRaw &lt;- function(dataset.raw, dateOffset){
   #Standar names to the columns
   dataset <- setNames(dataset.raw, c("Date", "Attack", "Target", "Country"))
  
-  #Remove rows with Date NA
+  #Data frame changes to standarize values and make easier the joins
   dataset <- dataset[!is.na(dataset$Date),]
   dataset <- dataset[!is.na(dataset$Country),]
-  dataset <- dataset[!dataset$Country == ">1",]
-  dataset <- dataset[!dataset$Country == ">A",]
+  dataset$Country <- toupper(dataset$Country)
+
   dataset <- dataset[!dataset$Country == "INT",]
   dataset <- dataset[!grepl(">",dataset$Country),]
+  dataset <- dataset[dataset$Country != "N/A",]
   dataset$Country <- gsub("\n"," ",dataset$Country)
   dataset <- FilterMultiCountry(dataset)
-  dataset <- dataset[!dataset$Country == "",]
+  dataset <- dataset[dataset$Country != "",]
+
+  dataset <- dataset[dataset$Country != "H",]
+  dataset <- dataset[dataset$Country != "W",]
+  dataset <- dataset[dataset$Country != "14",]
+  dataset <- dataset[dataset$Country != "EU",]
+  dataset <- dataset[dataset$Country != "UN",]
+  dataset <- dataset[dataset$Country != "TI",]
+  dataset <- dataset[dataset$Country != ".TI",]
+  dataset$Country <- gsub("G8","GI",dataset$Country)
+  dataset$Country <- gsub("UK","GB",dataset$Country)
+  dataset$Country <- gsub("EN","GB",dataset$Country)
+  dataset$Country <- gsub("UAE","AE",dataset$Country)
+  dataset$Country <- gsub("CB","KH",dataset$Country)
+
  
   #Format properly the date
   dataset$Date <- as.POSIXct(dataset$Date*86400, tz = "GMT", origin = dateOffset)
@@ -66,28 +81,43 @@ ProcessHMRaw &lt;- function(dataset.raw, dateOffset){
   dataset
 }
  
+#' Look for rows with more than one country target and split into multiple
+#'
+#' @param dataset.pre data.frame to process
+#'
+#' @return data.frame
 FilterMultiCountry <- function(dataset.pre) {
+
+  #data.frame with multiple taget country rows
   multi <- dataset.pre[grepl(" ",dataset.pre$Country),]
  
-  dataset <- dataset.pre[!grepl(" ",dataset.pre$Country),]
+  if (nrow(multi) == 0) { #Ignore if there are not multiple target rows
  
-  for (i in 1:nrow(multi)) {
-    crow <- multi[i,]
+    dataset.pre
+  } else {
  
-    country_s <- strsplit(toString(crow$Country), " ")
+    #data.frame with every rows except multi ones
+    dataset <- dataset.pre[!grepl(" ",dataset.pre$Country),]
  
-    for (j in 1:length(country_s)) {
-      Date <- crow[1]
-      Attack <- crow[2]
-      Target <- crow[3]
-      Country <- country_s[[1]][j]
-      new.row <- data.frame(Date, Attack, Target, Country)
-      print(new.row)
-      dataset <- rbind(dataset, new.row)
+    #Iterate over multi
+    for (i in 1:nrow(multi)) {
+      crow <- multi[i,] #current row
+
+      country_s <- strsplit(toString(crow$Country), " ")[[1]] #each country target
+
+      #Iterate over each country target
+      for (j in 1:length(country_s)) {
+        Date <- crow[1]
+        Attack <- crow[2]
+        Target <- crow[3]
+        Country <- country_s[j]
+        new.row <- data.frame(Date, Attack, Target, Country)
+        dataset <- rbind(dataset, new.row) #Append new row to output data.frame
+      }
     }
-  }
  
-  dataset
+    dataset
+    }
 }
  
 #' Parse every excel file into a folder
@@ -69,6 +69,7 @@ ProccesISOSurveyRaw &lt;- function(dataset.raw, years){
 #' @param years List of years to return, c("X2006", "X2010", ...)
 #'
 #' @return data.frame
+#' @export
 #'
 #' @examples
 #'
@@ -79,8 +80,24 @@ ProccesISOSurveyByCountryRaw &lt;- function(dataset.raw, years){
   #Standard proccess
   dataset <- ProccesISOSurveyRaw(dataset.raw, years)
  
+  dataset$Country <- gsub("Bolivia","Bolivia, Plurinational State of",dataset$Country)
+  dataset$Country <- gsub("Cape Verde","Cabo Verde",dataset$Country)
+  dataset$Country <- gsub("Congo, Republic of","Congo, the Democratic Republic of the",dataset$Country)
+  dataset$Country <- gsub("Côte D'ivoire","Cote d'Ivoire",dataset$Country)
+  dataset$Country <- gsub("Gibraltar (UK)","Gibraltar",dataset$Country)
+  dataset$Country <- gsub("Hong Kong, China","Hong Kong",dataset$Country)
+  dataset$Country <- gsub("Macau, China","Macao",dataset$Country)
+  dataset$Country <- gsub("Palestine","Palestine, State of",dataset$Country)
+  dataset$Country <- gsub("San Marino, Republic of","San Marino",dataset$Country)
+  dataset$Country <- gsub("Taipei, Chinese","Taiwan, Province of China",dataset$Country)
+  dataset$Country <- gsub("The Former Yugoslav Republic of Macedonia","Macedonia, the former Yugoslav Republic of",dataset$Country)
+  dataset$Country <- gsub("United States of America","United States",dataset$Country)
+  dataset$Country <- gsub("Venezuela","Venezuela, Bolivarian Republic of",dataset$Country)
+
+
   #Translate country names to 2 letter code
-  CountryNames <- GetCountryAbrev()
+  CountryNames <- data.frame(countrycode::countrycode_data$country.name, countrycode::countrycode_data$iso2c)
+  CountryNames <- setNames(CountryNames, c("Country","country_short"))
   dataset <- merge(x = dataset, y = CountryNames, by = "Country", all.x = TRUE)
  
   dataset
-#-----------------------------Util functions--------------------------------------------------
-
-#' Return the 2 letter code of a country relation
-#'
-#' Relation of country names included in the ISO Survey input file with 2 letter code
-#' included on the hackmaggedon input files
-#'
-#' @return data.frame
-GetCountryAbrev <- function(){
-  Country <- c("Afghanistan", "Albania", "Algeria", "American Samoa", "Andorra", "Angola", "Anguilla", "Antarctica", "Antigua and Barbuda", "Argentina", "Armenia", "Aruba", "Australia", "Austria", "Azerbaijan", "Bahamas", "Bahrain", "Bangladesh", "Barbados", "Belarus", "Belgium", "Belize", "Benin", "Bermuda", "Bhutan", "Bolivia", "Bonaire", "Bosnia and Herzegovina", "Botswana", "Bouvet Island", "Brazil", "British Indian Ocean Territory", "Brunei Darussalam", "Bulgaria", "Burkina Faso", "Burundi", "Cambodia", "Cameroon", "Canada", "Cape Verde", "Cayman Islands", "Central African Republic", "Chad", "Chile", "China", "Christmas Island", "Cocos (Keeling) Islands", "Colombia", "Comoros", "Congo", "Congo, Republic of", "Cook Islands", "Costa Rica", "Croatia", "Cuba", "CuraÃ§ao", "Cyprus", "Czech Republic", "Côte D'ivoire", "Denmark", "Djibouti", "Dominica", "Dominican Republic", "Ecuador", "Egypt", "El Salvador", "Equatorial Guinea", "Eritrea", "Estonia", "Ethiopia", "Falkland Islands (Malvinas)", "Faroe Islands", "Fiji", "Finland", "France", "French Guiana", "French Polynesia", "French Southern Territories", "Gabon", "Gambia", "Georgia", "Germany", "Ghana", "Gibraltar (UK)", "Greece", "Greenland", "Grenada", "Guadeloupe", "Guam", "Guatemala", "Guernsey", "Guinea", "Guinea-Bissau", "Guyana", "Haiti", "Heard Island and McDonald Mcdonald Islands", "Holy See (Vatican City State)", "Honduras", "Hong Kong, China", "Hungary", "Iceland", "India", "Indonesia", "Iran, Islamic Republic of", "Iraq", "Ireland", "Isle of Man", "Israel", "Italy", "Jamaica", "Japan", "Jersey", "Jordan", "Kazakhstan", "Kenya", "Kiribati", "Korea, Democratic People's Republic of", "Korea, Republic of", "Kuwait", "Kyrgyzstan", "Lao People's Democratic Republic", "Latvia", "Lebanon", "Lesotho", "Liberia", "Libya", "Liechtenstein", "Lithuania", "Luxembourg", "Macau, China", "The Former Yugoslav Republic of Macedonia", "Madagascar", "Malawi", "Malaysia", "Maldives", "Mali", "Malta", "Marshall Islands", "Martinique", "Mauritania", "Mauritius", "Mayotte", "Mexico", "Micronesia, Federated States of", "Moldova, Republic of", "Monaco", "Mongolia", "Montenegro", "Montserrat", "Morocco", "Mozambique", "Myanmar", "Namibia", "Nauru", "Nepal", "Netherlands", "New Caledonia", "New Zealand", "Nicaragua", "Niger", "Nigeria", "Niue", "Norfolk Island", "Northern Mariana Islands", "Norway", "Oman", "Pakistan", "Palau", "Palestine", "Panama", "Papua New Guinea", "Paraguay", "Peru", "Philippines", "Pitcairn", "Poland", "Portugal", "Puerto Rico", "Qatar", "Romania", "Russian Federation", "Rwanda", "Reunion", "Saint Barthelemy", "Saint Helena", "Saint Kitts and Nevis", "Saint Lucia", "Saint Martin (French part)", "Saint Pierre and Miquelon", "Saint Vincent and the Grenadines", "Samoa", "San Marino, Republic of", "Sao Tome and Principe", "Saudi Arabia", "Senegal", "Serbia", "Seychelles", "Sierra Leone", "Singapore", "Sint Maarten (Dutch part)", "Slovakia", "Slovenia", "Solomon Islands", "Somalia", "South Africa", "South Georgia and the South Sandwich Islands", "South Sudan", "Spain", "Sri Lanka", "Sudan", "Suriname", "Svalbard and Jan Mayen", "Swaziland", "Sweden", "Switzerland", "Syrian Arab Republic", "Taiwan, Province of China", "Tajikistan", "Tanzania, United Republic of", "Thailand", "Timor-Leste", "Togo", "Tokelau", "Tonga", "Trinidad and Tobago", "Tunisia", "Turkey", "Turkmenistan", "Turks and Caicos Islands", "Tuvalu", "Uganda", "Ukraine", "United Arab Emirates", "United Kingdom", "United States of America", "United States Minor Outlying Islands", "Uruguay", "Uzbekistan", "Vanuatu", "Venezuela", "Viet Nam", "British Virgin Islands", "US Virgin Islands", "Wallis and Futuna", "Western Sahara", "Yemen", "Zambia", "Zimbabwe", "Aland Islands", "Taipei, Chinese")
-  country_short <- c("AF", "AL", "DZ", "AS", "AD", "AO", "AI", "AQ", "AG", "AR", "AM", "AW", "AU", "AT", "AZ", "BS", "BH", "BD", "BB", "BY", "BE", "BZ", "BJ", "BM", "BT", "BO", "BQ", "BA", "BW", "BV", "BR", "IO", "BN", "BG", "BF", "BI", "KH", "CM", "CA", "CV", "KY", "CF", "TD", "CL", "CN", "CX", "CC", "CO", "KM", "CG", "CD", "CK", "CR", "HR", "CU", "CW", "CY", "CZ", "CI", "DK", "DJ", "DM", "DO", "EC", "EG", "SV", "GQ", "ER", "EE", "ET", "FK", "FO", "FJ", "FI", "FR", "GF", "PF", "TF", "GA", "GM", "GE", "DE", "GH", "GI", "GR", "GL", "GD", "GP", "GU", "GT", "GG", "GN", "GW", "GY", "HT", "HM", "VA", "HN", "HK", "HU", "IS", "IN", "ID", "IR", "IQ", "IE", "IM", "IL", "IT", "JM", "JP", "JE", "JO", "KZ", "KE", "KI", "KP", "KR", "KW", "KG", "LA", "LV", "LB", "LS", "LR", "LY", "LI", "LT", "LU", "MO", "MK", "MG", "MW", "MY", "MV", "ML", "MT", "MH", "MQ", "MR", "MU", "YT", "MX", "FM", "MD", "MC", "MN", "ME", "MS", "MA", "MZ", "MM", "NA", "NR", "NP", "NL", "NC", "NZ", "NI", "NE", "NG", "NU", "NF", "MP", "NO", "OM", "PK", "PW", "PS", "PA", "PG", "PY", "PE", "PH", "PN", "PL", "PT", "PR", "QA", "RO", "RU", "RW", "RE", "BL", "SH", "KN", "LC", "MF", "PM", "VC", "WS", "SM", "ST", "SA", "SN", "RS", "SC", "SL", "SG", "SX", "SK", "SI", "SB", "SO", "ZA", "GS", "SS", "ES", "LK", "SD", "SR", "SJ", "SZ", "SE", "CH", "SY", "TW", "TJ", "TZ", "TH", "TL", "TG", "TK", "TO", "TT", "TN", "TR", "TM", "TC", "TV", "UG", "UA", "AE", "GB", "US", "UM", "UY", "UZ", "VU", "VE", "VN", "VG", "VI", "WF", "EH", "YE", "ZM", "ZW", "AX", "CN")
-
-  dataset <- data.frame(Country, country_short)
-
-  dataset
-}
+% Generated by roxygen2: do not edit by hand
+% Please edit documentation in R/Hackmageddon_Parser.R
+\name{FilterMultiCountry}
+\alias{FilterMultiCountry}
+\title{Look for rows with more than one country target and split into multiple}
+\usage{
+FilterMultiCountry(dataset.pre)
+}
+\arguments{
+\item{dataset.pre}{data.frame to process}
+}
+\value{
+data.frame
+}
+\description{
+Look for rows with more than one country target and split into multiple
+}
+
-% Generated by roxygen2: do not edit by hand
-% Please edit documentation in R/Util.R
-\name{GetCountryAbrev}
-\alias{GetCountryAbrev}
-\title{Return the 2 letter code of a country relation}
-\usage{
-GetCountryAbrev()
-}
-\value{
-data.frame
-}
-\description{
-Relation of country names included in the ISO Survey input file with 2 letter code
-included on the hackmaggedon input files
-}
-
+% Generated by roxygen2: do not edit by hand
+% Please edit documentation in R/Hackmageddon_Parser.R
+\name{GetDefaultAttacksData}
+\alias{GetDefaultAttacksData}
+\title{Parse the default data from the package from hackmaggedon (2012-2016)}
+\usage{
+GetDefaultAttacksData()
+}
+\value{
+data.frame
+}
+\description{
+Parse the default data from the package from hackmaggedon (2012-2016)
+}
+\examples{
+Attacks <- GetDefaultAttacksData()
+}
+
+% Generated by roxygen2: do not edit by hand
+% Please edit documentation in R/ISOSurvey_Parser.R
+\name{GetISOSurveyCertsPerCountry}
+\alias{GetISOSurveyCertsPerCountry}
+\title{Get data of certificates per year and country from IS27001}
+\usage{
+GetISOSurveyCertsPerCountry()
+}
+\value{
+data.frame
+}
+\description{
+Get data of certificates per year and country from IS27001
+}
+\examples{
+Cert_PerCountry <- GetISOSurveyCertsPerCountry()
+}
+
+% Generated by roxygen2: do not edit by hand
+% Please edit documentation in R/ISOSurvey_Parser.R
+\name{GetISOSurveyCertsPerSector}
+\alias{GetISOSurveyCertsPerSector}
+\title{Get data of certificates per year and sector from IS27001}
+\usage{
+GetISOSurveyCertsPerSector()
+}
+\value{
+data.frame
+}
+\description{
+Get data of certificates per year and sector from IS27001
+}
+\examples{
+Cert_PerSector <- GetISOSurveyCertsPerSector()
+}
+
+% Generated by roxygen2: do not edit by hand
+% Please edit documentation in R/ISOSurvey_Parser.R
+\name{GetISOSurveySitesPerCountry}
+\alias{GetISOSurveySitesPerCountry}
+\title{Get data of sites per year and country from IS27001}
+\usage{
+GetISOSurveySitesPerCountry()
+}
+\value{
+data.frame
+}
+\description{
+Get data of sites per year and country from IS27001
+}
+\examples{
+Sites_PerCountry <- GetISOSurveySitesPerCountry()
+}
+
-% Generated by roxygen2: do not edit by hand
-% Please edit documentation in R/Util.R
-\name{LoadParserLibraries}
-\alias{LoadParserLibraries}
-\title{Install and load required libraries}
-\usage{
-LoadParserLibraries()
-}
-\description{
-This function checks if every required library is installed to be loaded, if not they will be installed and then loaded.
-Libraries installed:
- xlsx to parse excel files like ISO survey source format
-}
-
+% Generated by roxygen2: do not edit by hand
+% Please edit documentation in R/Hackmageddon_Parser.R
+\name{ParseHMExcel}
+\alias{ParseHMExcel}
+\title{Parse an excel raw data file from armaggedon}
+\usage{
+ParseHMExcel(file, cols)
+}
+\arguments{
+\item{file}{path to the excel file}
+
+\item{cols}{list of columns index to read}
+}
+\value{
+data.frame
+}
+\description{
+Parse an excel raw data file from armaggedon
+}
+\examples{
+data.raw <- ParseHMExcel("./data/hackmaggedon/file.xls", c(2, 3, 6, 5))
+}
+
+% Generated by roxygen2: do not edit by hand
+% Please edit documentation in R/Hackmageddon_Parser.R
+\name{ParseHMFolder}
+\alias{ParseHMFolder}
+\title{Parse every excel file into a folder}
+\usage{
+ParseHMFolder(folder, cols, dateOffset)
+}
+\arguments{
+\item{folder}{path to the folder to iterate}
+
+\item{cols}{columns to parse into each file}
+
+\item{dateOffset}{origin to calc the dates into each file}
+}
+\value{
+data.frame
+}
+\description{
+Parse every excel file into a folder
+}
+\examples{
+data.pro <- ProcessHMRaw("./data/hackmaggedon/", c(1, 5, 3) "1899-12-30")
+}
+
@@ -2,7 +2,7 @@
 % Please edit documentation in R/ISOSurvey_Parser.R
 \name{ProccesISOSurveyByCountryRaw}
 \alias{ProccesISOSurveyByCountryRaw}
-\title{Process raw data from ISO survey}
+\title{Join data from ISOSurvey with 2 letter code countries plus process raw.data}
 \usage{
 ProccesISOSurveyByCountryRaw(dataset.raw, years)
 }
@@ -15,12 +15,11 @@ ProccesISOSurveyByCountryRaw(dataset.raw, years)
 data.frame
 }
 \description{
-Proccess the raw data from ISO survey to replace NAs, normalizate country names and filter years
+Join data from ISOSurvey with 2 letter code countries plus process raw.data
 }
 \examples{
  
 Cert_PerCountry <- ProccesISOSurveyRaw(Cert_PerCountry, c("X2010", "X2011", "X2012", "X2013", "X2014", "X2015"))
 Sites_PerCountry <- ProccesISOSurveyRaw(Sites_PerCountry, c("X2010", "X2011", "X2012", "X2013", "X2014", "X2015"))
-Cert_PerSector <- ProccesISOSurveyRaw(Cert_PerSector, c("X2010", "X2011", "X2012", "X2013", "X2014", "X2015"))
 }
  
+% Generated by roxygen2: do not edit by hand
+% Please edit documentation in R/ISOSurvey_Parser.R
+\name{ProccesISOSurveyRaw}
+\alias{ProccesISOSurveyRaw}
+\title{PRocess raw data parsed from excel file ISOSurvey27001}
+\usage{
+ProccesISOSurveyRaw(dataset.raw, years)
+}
+\arguments{
+\item{dataset.raw}{data.frame with raw data}
+
+\item{years}{list of years to include preceded with a X}
+}
+\value{
+data.frame
+}
+\description{
+PRocess raw data parsed from excel file ISOSurvey27001
+}
+\examples{
+Cert_PerSector <- ProccesISOSurveyRaw(Cert_PerSector, c("X2010", "X2011", "X2012", "X2013", "X2014", "X2015"))
+}
+
+% Generated by roxygen2: do not edit by hand
+% Please edit documentation in R/Hackmageddon_Parser.R
+\name{ProcessHMRaw}
+\alias{ProcessHMRaw}
+\title{Prepare raw data from hackmaggedon's excel to use it}
+\usage{
+ProcessHMRaw(dataset.raw, dateOffset)
+}
+\arguments{
+\item{dataset.raw}{data.frame with raw data}
+
+\item{dateOffset}{origin to add the numeric date}
+}
+\value{
+data.frame
+}
+\description{
+Prepare raw data from hackmaggedon's excel to use it
+}
+\examples{
+data.pro <- ProcessHMRaw(data.raw, "1899-12-30")
+}
+