diff --git a/.buildlibrary b/.buildlibrary index cdf5c89a..3abf568a 100644 --- a/.buildlibrary +++ b/.buildlibrary @@ -1,4 +1,4 @@ -ValidationKey: '32694786' +ValidationKey: '32721040' AcceptedWarnings: - 'Warning: package ''.*'' was built under R version' - 'Warning: namespace ''.*'' is not available and has been replaced' diff --git a/CITATION.cff b/CITATION.cff index 4b852439..8722d18c 100644 --- a/CITATION.cff +++ b/CITATION.cff @@ -2,8 +2,8 @@ cff-version: 1.2.0 message: If you use this software, please cite it using the metadata from this file. type: software title: 'mrremind: MadRat REMIND Input Data Package' -version: 0.167.1 -date-released: '2023-07-28' +version: 0.167.2 +date-released: '2023-08-01' abstract: The mrremind packages contains data preprocessing for the REMIND model. authors: - family-names: Baumstark diff --git a/DESCRIPTION b/DESCRIPTION index 5e9eb77b..f7f0eaa9 100644 --- a/DESCRIPTION +++ b/DESCRIPTION @@ -1,8 +1,8 @@ Type: Package Package: mrremind Title: MadRat REMIND Input Data Package -Version: 0.167.1 -Date: 2023-07-28 +Version: 0.167.2 +Date: 2023-08-01 Authors@R: c( person("Lavinia", "Baumstark", , "lavinia@pik-potsdam.de", role = c("aut", "cre")), person("Renato", "Rodrigues", role = "aut"), @@ -41,7 +41,7 @@ Depends: madrat (>= 2.5.1), magclass (>= 3.17), mrdrivers (>= 1.0.0), - mrcommons (>= 1.26.6), + mrcommons (>= 1.33.3), edgeTransport (>= 0.17.0) Imports: assertr, @@ -57,7 +57,6 @@ Imports: magrittr, mgsub, nnls, - openxlsx, plyr, purrr, quitte (>= 0.3105.0), diff --git a/NAMESPACE b/NAMESPACE index 0aca73f2..1016649c 100644 --- a/NAMESPACE +++ b/NAMESPACE @@ -31,7 +31,6 @@ export(calcpm_delta_kap_industry) export(convertADVANCE_WP2) export(convertAGEB) export(convertEDGAR7Fgases) -export(convertEU_ReferenceScenario) export(convertEmber) export(convertHRE) export(convertMueller) @@ -44,7 +43,6 @@ export(readADVANCE_WP2) export(readAGEB) export(readAR6GWP) export(readEDGAR7Fgases) -export(readEU_ReferenceScenario) export(readEmber) export(readHRE) export(readIEA_WEIO_2014) @@ -213,7 +211,6 @@ importFrom(magclass,write.magpie) importFrom(magclass,write.report) importFrom(magrittr,"%<>%") importFrom(magrittr,"%>%") -importFrom(openxlsx,read.xlsx) importFrom(plyr,round_any) importFrom(purrr,map) importFrom(purrr,reduce) diff --git a/R/convertEU_ReferenceScenario.R b/R/convertEU_ReferenceScenario.R deleted file mode 100644 index 9f133591..00000000 --- a/R/convertEU_ReferenceScenario.R +++ /dev/null @@ -1,83 +0,0 @@ -#' Convert EU Reference Scenario -#' -#' Converts EU Reference Scenario magpie object into appropriate form for the REMIND model -#' -#' @author Renato Rodrigues, Falk Benke, Robin Hasse -#' @param x EU Reference Scenario magpie object derived from readEU_ReferenceScenario function -#' @param subtype data subtype. Either "techAssump.*", "2016" or "2020" -#' @return converted EU Reference Scenario magpie object -#' @examples -#' \dontrun{ -#' test <- readSource("EU_ReferenceScenario", subtype = "2020", convert = TRUE) -#' } -#' @importFrom madrat toolGetMapping toolCountryFill toolAggregate -#' @importFrom magclass getItems<- getItems getSets<- setItems mselect -#' @importFrom utils read.csv2 -#' @export - -convertEU_ReferenceScenario <- function(x, subtype) { - - EU_27 <- c( - "ALA", "AUT", "BEL", "BGR", "HRV", "CYP", "CZE", "DNK", "EST", - "FRO", "FIN", "FRA", "DEU", "GIB", "GRC", "GGY", "HUN", "IRL", - "IMN", "ITA", "JEY", "LVA", "LTU", "LUX", "MLT", "NLD", "POL", - "PRT", "ROU", "SVK", "SVN", "ESP", "SWE" - ) - EU_28 <- c(EU_27, "GBR") - - - - # Technology assumptions ----------------------------------------------------- - - if (grepl("^techAssump\\..+$", subtype)) { - - subsubtype <- sub("^techAssump\\.", "", subtype) - - # map to EU 28 countries - mapping <- toolGetMapping(name = "regionmappingEU_ReferenceScenario.csv", - type = "regional", - where = "mappingfolder") - mapping <- mapping[mapping[["CountryCode"]] %in% EU_28, ] - if (subsubtype %in% c("Domestic", "Renovation Costs")) { - xReg <- x["EUR", , invert = TRUE] - xReg <- toolAggregate(x = xReg, - rel = mapping, - from = gsub(" ", ".", subsubtype), - to = "CountryCode") - if (subsubtype == "Renovation Costs") { - x <- xReg - } else { - xEur <- mselect(x, region = "EUR") - xEur <- do.call(mbind, lapply(EU_28, setItems, x = xEur, dim = 1)) - x <- xEur - x[!is.na(xReg)] <- xReg[!is.na(xReg)] - } - } else { - x <- do.call(mbind, lapply(EU_28, setItems, x = x, dim = 1)) - getSets(x)[1] <- "region" - } - - # harmonise units - getItems(x, "unit") <- sub("Euro", "EUR", getItems(x, "unit")) - getItems(x, "unit") <- sub("^NA$", "EUR/appliance", getItems(x, "unit")) - - } else { - - # results ------------------------------------------------------------------ - - # fill up zero countries - iso3 <- read.csv2("isotwo2iso3Mapping.csv", stringsAsFactors = FALSE) - getItems(x, dim = 1) <- sapply(getItems(x, dim = 1), function(y) iso3[which(iso3[, 1] == y), 2]) - - } - - x <- toolCountryFill(x, fill = NA, verbosity = 2) - - EU <- switch(subtype, "2016" = EU_28, "2020" = EU_27) - - x.eu <- x[EU, , ] - x.eu[is.na(x.eu)] <- 0 - x[EU, , ] <- x.eu[EU, , ] - - return(x) -} diff --git a/R/readEU_ReferenceScenario.R b/R/readEU_ReferenceScenario.R deleted file mode 100644 index 31d6215e..00000000 --- a/R/readEU_ReferenceScenario.R +++ /dev/null @@ -1,251 +0,0 @@ -#' Read EU Reference Scenario -#' -#' Read EU Reference Scenario .xlsx file as magpie object -#' -#' @return magpie object of EU reference scenario data by country. Units follow -#' REMIND report conventions and conversion factor is defined in -#' EUReferenceScenario2REMIND.xlsx file. -#' @param subtype data subtype. Either "techAssump.*", "2016" or "2020" -#' @author Renato Rodrigues, Falk Benke, Robin Hasse -#' @examples -#' \dontrun{ -#' test <- readSource("EU_ReferenceScenario", subtype = "2020", convert = FALSE) -#' } -#' @importFrom readxl excel_sheets read_excel -#' @importFrom openxlsx read.xlsx -#' @importFrom reshape2 melt -#' @importFrom stats aggregate -#' @importFrom quitte as.quitte -#' @importFrom magclass as.magpie collapseDim -#' @importFrom tidyr pivot_longer -#' @export - -readEU_ReferenceScenario <- function(subtype) { - - - # Technology assumptions ----------------------------------------------------- - - if (grepl("^techAssump\\..+$", subtype)) { - - subsubtype <- sub("^techAssump\\.", "", subtype) - - # read excel sheet - rawdata <- read.xlsx( - file.path("ref2020_technology_assumptions", - "REF2020_Technology Assumptions_Energy.xlsx"), - sheet = subsubtype, startRow = 2, colNames = FALSE, skipEmptyRows = TRUE, - fillMergedCells = TRUE) - - # remove notes at the bottom - data <- rawdata[cumsum(grepl("^Notes", rawdata[, 1])) == 0, ] - - - - if (subsubtype %in% c("Domestic", "Industry", "Power&Heat")) { - - ## Sectors ==== - - # split subsector tables (Residential & Services) - splitLines <- unlist(apply(data, 1, function(l) length(unique(l)) == 1)) - subsectors <- data[splitLines, 1] - if (any(splitLines)) { - splitLines <- seq_along(splitLines)[which(splitLines)] - data <- lapply(seq_along(splitLines), function(i) { - start <- splitLines[i] + 1 - end <- if (i == length(splitLines)) nrow(data) else splitLines[i + 1] - 1 - data[start:end, ] - }) - names(data) <- subsectors - } else { - data <- list(subsubtype = data) - names(data) <- subsubtype - } - - # pivot variables to long format - data <- lapply(data, function(chunk) { - chunk[1, ] <- trimws(gsub("(\n.*$)|,", "", chunk[1, ])) - variables <- unique(unlist(chunk[1, 2:ncol(chunk)])) - varCols <- lapply(variables, function(var) which(chunk[1, ] == var)) - names(varCols) <- variables - - if (subsubtype %in% c("Domestic", "Industry")) { - # identify group lines (end uses and units) - groupLines <- do.call(cbind, lapply(varCols, function(cols) { - data.frame( - var = apply(chunk[, cols], 1, function(l) length(unique(l)))) - })) - groupLines <- as.vector(apply(groupLines, 1, function(l) all(l == 1))) - groupLines[1:3] <- FALSE - - # add end use column - enduses <- chunk[groupLines, 1] - enduseIndex <- cumsum(groupLines) - enduseIndex[enduseIndex == 0] <- NA - chunk[["enduse"]] <- enduses[enduseIndex] - - # save units - if (subsubtype == "Domestic") { - units <- do.call(cbind, lapply(varCols, function(cols) { - dfCol <- data.frame( - var = apply(chunk[groupLines, cols], 1, function(l) unique(l))) - colnames(dfCol) <- chunk[1, head(cols, 1)] - return(dfCol) - })) - units[["enduse"]] <- chunk[groupLines, 1] - units <- pivot_longer(units, -"enduse", - names_to = "variable", - values_to = "unit") - units[["unit"]] <- sub("^in ", "", units[["unit"]]) - } - chunk <- chunk[!groupLines,] - - # treat regional heat pump data - chunk <- chunk[!grepl("Heat pump air", chunk[, 1]), ] - hpLines <- grepl("in .* countries", chunk[, 1], ignore.case = TRUE) - chunk[["region"]] <- ifelse(hpLines, sub("in ", "", chunk[, 1]), NA) - chunk[["region"]] <- sub("Countries", "countries", chunk[["region"]]) - chunk[, 1] <- ifelse(hpLines, "Heat pump air", chunk[, 1]) - } - - # pivot to long format - fixCols <- c("technology", "enduse", "region") - colnames <- apply(chunk[1:3, ], 2, paste, collapse = "_") - colnames[which(colnames(chunk) %in% fixCols)] <- - colnames(chunk)[which(colnames(chunk) %in% fixCols)] - colnames[1] <- "technology" - chunk <- chunk[4:nrow(chunk), ] - colnames(chunk) <- colnames - headerCols <- if (subsubtype == "Power&Heat") { - c("variable", "unit", "period") - } else { - c("variable", "pointInTime", "level") - } - chunk <- pivot_longer(chunk, -intersect(colnames(chunk), fixCols), - names_to = headerCols, - names_sep = "_") - chunk[["value"]] <- as.numeric(chunk[["value"]]) - if ("level" %in% colnames(chunk)) { - chunk[chunk[["level"]] == "NA", "level"] <- "central" - } - if (subsubtype == "Power&Heat") { - chunk[["period"]] = as.numeric(chunk[["period"]]) - } - - # add unit column - if (subsubtype == "Domestic") { - chunk <- left_join(chunk, units, by = c("variable", "enduse")) - } else if (subsubtype == "Industry") { - units <- "EUR\\/kW|equal to 1 in 2015" - chunk[["unit"]] <- sub(paste0(".*(", units, ").*"), "\\1", - chunk[["variable"]]) - chunk[["variable"]] <- trimws(gsub(paste0(units, "|\\(|\\)"), "", - chunk[["variable"]])) - } - - return(chunk) - }) - - # merge to one df with subsector column - data <- do.call(rbind, lapply(names(data), function(subsector) { - chunk <- data[[subsector]] - chunk[["subsector"]] <- subsector - return(chunk) - })) - - } else if (subsubtype == "Renovation Costs") { - - ## Renovation Costs ==== - - colnames(data) <- data[1, ] - colnames(data)[1:2] <- c("region", "renovation") - data <- data[2:nrow(data), ] - - data <- pivot_longer(data, -c("region", "renovation"), - names_to = "variable", - values_to = "value") - data[["unit"]] <- sub(".*\\((.*)\\)$", "\\1", data[["variable"]]) - data[["variable"]] <- trimws(sub("\\(.*\\)$", "", data[["variable"]])) - data[["value"]] <- as.numeric(data[["value"]]) - - } else { - - stop("Invalid type of technology assumption: ", subsubtype) - - } - - # convert to magpie object - if ("region" %in% colnames(data)) { - data[is.na(data[["region"]]), "region"] <- "EUR" - } - x <- collapseDim(as.magpie(as.quitte(data))) - - return(x) - - } - - - - if (!subtype %in% c("2016", "2020")) { - stop("Invalid subtype. Must be either 'techAssump.*', '2016' or '2020'") - } - - - - # Results -------------------------------------------------------------------- - - mapping <- NULL - - # load mapping and data - mapping$A <- suppressMessages(read_excel(paste0("EUReferenceScenario2REMIND_", subtype, ".xlsx"), sheet = "A")) - mapping$B <- suppressMessages(read_excel(paste0("EUReferenceScenario2REMIND_", subtype, ".xlsx"), sheet = "B")) - - if (subtype == "2016") { - source_file <- "AppendixRefSce.xls" - sheets <- excel_sheets(source_file) - sheets <- sheets[-c(1, 2, 3, length(sheets))] - no_rows <- 12 - columns <- c("REMIND") - } else { - source_file <- "ref2020_energy-transport-ghg.xlsx" - sheets <- excel_sheets(source_file) - sheets <- sheets[-c(1, 2, 3)] - no_rows <- 11 - columns <- c("REMIND", "REMIND_2") - } - - data <- NULL - # looping through regions, filtering and converting values - for (sheet in sheets) { - for (column in columns) { - type <- substr(sheet, nchar(sheet), nchar(sheet)) # A or B - region <- substr(sheet, start = 1, stop = 2) - countrySheet <- suppressMessages(read_excel(source_file, sheet = sheet, skip = 1)) - # cleaning sheet - countrySheet <- countrySheet[, seq(1, no_rows)] - # replace with remind mapping - countrySheet$REMIND <- mapping[[type]][[column]][-1] - # removing extra name column - countrySheet <- countrySheet[, -1] - # making sure the data is numeric - countrySheet[, -length(colnames(countrySheet))] <- sapply(countrySheet[, -length(colnames(countrySheet))], as.numeric) - # converting unit to REMIND unit - countrySheet[, -length(colnames(countrySheet))] <- countrySheet[, -length(colnames(countrySheet))] * mapping[[type]]$factor[-1] - # remove empty rows - countrySheet <- countrySheet[-which(is.na(countrySheet$REMIND)), ] - countrySheet <- cbind(region, countrySheet) - countrySheet[is.na(countrySheet)] <- 0 - # merge repeated items - countrySheet <- aggregate(. ~ REMIND + region, data = countrySheet, FUN = sum) - data <- rbind(data, countrySheet) - } - } - - data <- aggregate(. ~ REMIND + region, data = data, FUN = sum) - - # long format - data <- melt(data, id.vars = 1:2) - colnames(data) <- c("variable", "region", "period", "value") - # dump contents into magpie - x <- as.magpie(data, spatial = 2, datacol = 4, temporal = 3) - return(x) -} diff --git a/README.md b/README.md index 59e02c5e..5e633fd3 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # MadRat REMIND Input Data Package -R package **mrremind**, version **0.167.1** +R package **mrremind**, version **0.167.2** [![CRAN status](https://www.r-pkg.org/badges/version/mrremind)](https://cran.r-project.org/package=mrremind) [![R build status](https://github.com/pik-piam/mrremind/workflows/check/badge.svg)](https://github.com/pik-piam/mrremind/actions) [![codecov](https://codecov.io/gh/pik-piam/mrremind/branch/master/graph/badge.svg)](https://app.codecov.io/gh/pik-piam/mrremind) [![r-universe](https://pik-piam.r-universe.dev/badges/mrremind)](https://pik-piam.r-universe.dev/builds) @@ -39,7 +39,7 @@ In case of questions / problems please contact Lavinia Baumstark . +Baumstark L, Rodrigues R, Levesque A, Oeser J, Bertram C, Mouratiadou I, Malik A, Schreyer F, Soergel B, Rottoli M, Mishra A, Dirnaichner A, Pehl M, Giannousakis A, Klein D, Strefler J, Feldhaus L, Brecha R, Rauner S, Dietrich J, Bi S, Benke F, Weigmann P, Richters O, Hasse R, Fuchs S, Mandaroux R (2023). _mrremind: MadRat REMIND Input Data Package_. R package version 0.167.2, . A BibTeX entry for LaTeX users is @@ -48,7 +48,7 @@ A BibTeX entry for LaTeX users is title = {mrremind: MadRat REMIND Input Data Package}, author = {Lavinia Baumstark and Renato Rodrigues and Antoine Levesque and Julian Oeser and Christoph Bertram and Ioanna Mouratiadou and Aman Malik and Felix Schreyer and Bjoern Soergel and Marianna Rottoli and Abhijeet Mishra and Alois Dirnaichner and Michaja Pehl and Anastasis Giannousakis and David Klein and Jessica Strefler and Lukas Feldhaus and Regina Brecha and Sebastian Rauner and Jan Philipp Dietrich and Stephen Bi and Falk Benke and Pascal Weigmann and Oliver Richters and Robin Hasse and Sophie Fuchs and Rahel Mandaroux}, year = {2023}, - note = {R package version 0.167.1}, + note = {R package version 0.167.2}, url = {https://github.com/pik-piam/mrremind}, } ``` diff --git a/man/convertEU_ReferenceScenario.Rd b/man/convertEU_ReferenceScenario.Rd deleted file mode 100644 index b4e6b622..00000000 --- a/man/convertEU_ReferenceScenario.Rd +++ /dev/null @@ -1,27 +0,0 @@ -% Generated by roxygen2: do not edit by hand -% Please edit documentation in R/convertEU_ReferenceScenario.R -\name{convertEU_ReferenceScenario} -\alias{convertEU_ReferenceScenario} -\title{Convert EU Reference Scenario} -\usage{ -convertEU_ReferenceScenario(x, subtype) -} -\arguments{ -\item{x}{EU Reference Scenario magpie object derived from readEU_ReferenceScenario function} - -\item{subtype}{data subtype. Either "techAssump.*", "2016" or "2020"} -} -\value{ -converted EU Reference Scenario magpie object -} -\description{ -Converts EU Reference Scenario magpie object into appropriate form for the REMIND model -} -\examples{ -\dontrun{ -test <- readSource("EU_ReferenceScenario", subtype = "2020", convert = TRUE) -} -} -\author{ -Renato Rodrigues, Falk Benke, Robin Hasse -} diff --git a/man/readEU_ReferenceScenario.Rd b/man/readEU_ReferenceScenario.Rd deleted file mode 100644 index 8e3cb022..00000000 --- a/man/readEU_ReferenceScenario.Rd +++ /dev/null @@ -1,27 +0,0 @@ -% Generated by roxygen2: do not edit by hand -% Please edit documentation in R/readEU_ReferenceScenario.R -\name{readEU_ReferenceScenario} -\alias{readEU_ReferenceScenario} -\title{Read EU Reference Scenario} -\usage{ -readEU_ReferenceScenario(subtype) -} -\arguments{ -\item{subtype}{data subtype. Either "techAssump.*", "2016" or "2020"} -} -\value{ -magpie object of EU reference scenario data by country. Units follow - REMIND report conventions and conversion factor is defined in - EUReferenceScenario2REMIND.xlsx file. -} -\description{ -Read EU Reference Scenario .xlsx file as magpie object -} -\examples{ -\dontrun{ -test <- readSource("EU_ReferenceScenario", subtype = "2020", convert = FALSE) -} -} -\author{ -Renato Rodrigues, Falk Benke, Robin Hasse -}