diff --git a/.buildlibrary b/.buildlibrary index b22570e..d2636c9 100644 --- a/.buildlibrary +++ b/.buildlibrary @@ -1,4 +1,4 @@ -ValidationKey: '429330' +ValidationKey: '586110' AutocreateReadme: yes AcceptedWarnings: - 'Warning: package ''.*'' was built under R version' diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index d9409d2..9c3b069 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -15,7 +15,7 @@ repos: - id: mixed-line-ending - repo: https://github.com/lorenzwalthert/precommit - rev: v0.3.2.9007 + rev: v0.3.2.9013 hooks: - id: parsable-R - id: deps-in-desc diff --git a/CITATION.cff b/CITATION.cff index 44c900a..3b64607 100644 --- a/CITATION.cff +++ b/CITATION.cff @@ -2,8 +2,8 @@ cff-version: 1.2.0 message: If you use this software, please cite it using the metadata from this file. type: software title: 'mredgebuildings: Prepare data to be used by the EDGE-Buildings model' -version: 0.2.2 -date-released: '2023-06-07' +version: 0.3.0 +date-released: '2023-06-29' abstract: Prepare data to be used by the EDGE-Buildings model. authors: - family-names: Hasse diff --git a/DESCRIPTION b/DESCRIPTION index 2155bbf..2f0c6f7 100644 --- a/DESCRIPTION +++ b/DESCRIPTION @@ -1,8 +1,8 @@ Type: Package Package: mredgebuildings Title: Prepare data to be used by the EDGE-Buildings model -Version: 0.2.2 -Date: 2023-06-07 +Version: 0.3.0 +Date: 2023-06-29 Authors@R: c( person("Robin", "Hasse", , "robin.hasse@pik-potsdam.de", role = c("aut", "cre"), comment = c(ORCID = "0000-0003-1818-3186")), diff --git a/Makefile b/Makefile index c1e0c68..38e857f 100644 --- a/Makefile +++ b/Makefile @@ -11,8 +11,9 @@ HELP_PARSING = 'm <- readLines("Makefile");\ help: ## Show this help. @Rscript -e $(HELP_PARSING) -build: ## Build the package using lucode2::buildLibrary(). - Rscript -e 'lucode2::buildLibrary()' +build: ## Build the package using lucode2::buildLibrary(). You can pass the + ## updateType with 'make build u=3' + Rscript -e 'lucode2::buildLibrary(updateType = "$(u)")' check: ## Build documentation and vignettes, run testthat tests, ## and check if code etiquette is followed using lucode2::check(). diff --git a/NAMESPACE b/NAMESPACE index e34b317..b05c8ad 100644 --- a/NAMESPACE +++ b/NAMESPACE @@ -5,6 +5,7 @@ export(calcFloorspacePast) export(calcHDDCDD) export(calcHouseholdSize) export(calcIEAPFU) +export(calcMatchingReference) export(calcRenovationAlternatives) export(calcShareETP) export(calcShareOdyssee) @@ -14,6 +15,7 @@ export(convertCensusHub) export(convertDaioglou) export(convertEEAfloorspace) export(convertEUBuildingsDB) +export(convertEnergiforsk2016) export(convertEurObservER) export(convertEuropeanCommissionRenovation) export(convertEurostatBuildings) @@ -28,6 +30,7 @@ export(readDaioglou) export(readEEAfloorspace) export(readEHI) export(readEUBuildingsDB) +export(readEnergiforsk2016) export(readEurObservER) export(readEuropeanCommissionRenovation) export(readEurostatBuildings) @@ -43,6 +46,7 @@ import(magclass) import(mrcommons) import(mrdrivers) importFrom(dplyr,"%>%") +importFrom(dplyr,.data) importFrom(dplyr,across) importFrom(dplyr,all_of) importFrom(dplyr,anti_join) @@ -111,6 +115,7 @@ importFrom(stats,predict) importFrom(stats,pweibull) importFrom(tidyr,complete) importFrom(tidyr,gather) +importFrom(tidyr,pivot_longer) importFrom(tidyr,replace_na) importFrom(tidyr,separate) importFrom(tidyr,spread) diff --git a/R/calcBuildingStock.R b/R/calcBuildingStock.R index 824a08b..68da3cc 100644 --- a/R/calcBuildingStock.R +++ b/R/calcBuildingStock.R @@ -726,7 +726,8 @@ calcBuildingStock <- function(subtype = c("residential", "commercial")) { group_by(across(all_of(c("region", "period")))) %>% mutate(value = proportions(.data[["value"]])) stock <- stock %>% - left_join(heatingShares, by = c("region", "period")) %>% + left_join(heatingShares, by = c("region", "period"), + relationship = "many-to-many") %>% mutate(value = .data[["value.x"]] * .data[["value.y"]]) %>% select(-"value.x", -"value.y") %>% rename(heating = "carrier") diff --git a/R/calcMatchingReference.R b/R/calcMatchingReference.R new file mode 100644 index 0000000..0128224 --- /dev/null +++ b/R/calcMatchingReference.R @@ -0,0 +1,342 @@ +#' Calculate Reference data for input matching +#' +#' @author Robin Hasse +#' +#' @param subtype character, matching reference +#' +#' @importFrom magclass mbind as.magpie collapseDim +#' @importFrom madrat readSource toolCountryFill toolGetMapping +#' @importFrom quitte as.quitte interpolate_missing_periods +#' @importFrom dplyr group_by filter mutate .data +#' @export +#' +calcMatchingReference <- function(subtype) { + + periods <- 2000:2020 + + weight <- NULL + minVal <- 0 + maxVal <- NULL + description <- NULL + + # reference mapping + refMap <- toolGetMapping( + name = paste0("refMap_", subtype, ".csv"), + type = "sectoral", + where = "mappingfolder", + returnPathOnly = TRUE) %>% + read.csv(comment.char = "#") + + + + # READ DATA ------------------------------------------------------------------ + + if (grepl("Odyssee", subtype)) { + readOdysseeData <- function(subsector) { + readSource("Odyssee", subsector) %>% + as.quitte(na.rm = TRUE) %>% + select("region", "period", "variable", "value") %>% + mutate(variable = sub("_.*$", "", .data[["variable"]])) + } + households <- readOdysseeData("households") + services <- readOdysseeData("services") + } else if (grepl("EUBDB", subtype)) { + eubdbCateories <- c( + "BuildingStockCharacteristics", + "TechnicalBuildingSystems" + ) + # labels cannot be longer than 63 characters in gams -> shorten + eubdb <- do.call(rbind, lapply(eubdbCateories, function(c) { + readSource("EUBuildingsDB", c) %>% + as.quitte(na.rm = TRUE) + })) %>% + mutate(variable = sub("_.*$", "", .data[["variable"]]), + variable = sub("Number", "No", .data[["variable"]]), + variable = sub("non.residential", "com", .data[["variable"]]), + variable = sub("dwelling", "dwel", .data[["variable"]])) %>% + select("region", "period", "variable", "value") + + } else if (grepl("mredgebuildings", subtype)) { + stockHist <- calcOutput("BuildingStock", aggregate = FALSE) %>% + as.quitte(na.rm = TRUE) %>% + filter(.data[["variable"]] == "floor") %>% + select(-"model", -"scenario", -"variable", -"unit") + } + + aggStock <- function(stockHist, dim) { + data <- stockHist %>% + group_by(across(all_of(c("region", "period", dim)))) %>% + summarise(value = sum(.data[["value"]]) / 1E6, + .groups = "drop") + colnames(data)[3] <- "variable" + return(as.quitte(data)) + } + + + + # PREPARE DATA --------------------------------------------------------------- + + # nolint start: todo_comment_linter. + + switch(subtype, + + ## mredgebuildings_buildingType ==== + + mredgebuildings_buildingType = { + + data <- aggStock(stockHist, "buildingType") + + unit <- "million m2" + description <- "stock of residential buildings" + + }, + + ## mredgebuildings_location ==== + + mredgebuildings_location = { + + data <- aggStock(stockHist, "location") + + unit <- "million m2" + description <- "stock of residential buildings" + + }, + + ## mredgebuildings_vintage ==== + + mredgebuildings_vintage = { + + data <- aggStock(stockHist, "vintage") + + unit <- "million m2" + description <- "stock of residential buildings" + + }, + + ## mredgebuildings_heating ==== + + mredgebuildings_heating = { + + data <- aggStock(stockHist, "heating") + + unit <- "million m2" + description <- "stock of residential buildings" + + }, + + ## IDEES_heatingShare ==== + + IDEES_heatingShare = { + + data <- c("Residential", "Tertiary") %>% + lapply(readSource, type = "JRC_IDEES") %>% + do.call(what = mbind) %>% + as.quitte(na.rm = TRUE) %>% + select(-"scenario", -"model", -"unit") + + data <- refMap %>% + select("variable", ".variableOriginal", ".variableTotal") %>% + unique() %>% + left_join(data, + by = c(.variableOriginal = "variable")) %>% + group_by(across(-all_of(c(".variableOriginal", "value")))) %>% + summarise(value = sum(.data[["value"]]), .groups = "drop") %>% + left_join(data, + by = c(.variableTotal = "variable", "region", "period")) %>% + mutate(value = .data[["value.x"]] / .data[["value.y"]], + unit = "1") %>% + select(-"value.x", -"value.y", -".variableTotal") %>% + as.quitte() + + unit <- "1" + description <- "share of heating systems in the stock" + + }, + + ## Odyssee_stock ==== + + Odyssee_stock = { + + vars <- list( + households = unique(refMap[refMap[[".dataSet"]] == "households", "variable"]), + services = unique(refMap[refMap[[".dataSet"]] == "services", "variable"]) + ) + households <- households %>% + filter(.data[["variable"]] %in% vars$households) %>% + mutate(value = .data[["value"]] / 1E6, + unit = "million") + services <- services %>% + filter(.data[["variable"]] %in% c(vars$services)) %>% + mutate(value = .data[["value"]] / 1E6, # m2 -> million m2 + unit = "million m2") + + data <- rbind(households, services) %>% + as.quitte() + + unit <- "million m2 or million dwellings" + description <- "stock of dwellings and service floor space" + + }, + + + ## Odyssee_dwelSize ==== + + Odyssee_dwelSize = { + + data <- refMap %>% + select("variable") %>% + unique() %>% + left_join(households, by = "variable") + + # TODO: weight should be floor area -> circle dependency + + unit <- "m2/dwel" + description <- "average dwelling size of the stock and new construction" + + }, + + + ## Odyssee_construction ==== + + Odyssee_construction = { + + data <- refMap %>% + select("variable") %>% + unique() %>% + left_join(households, by = "variable") %>% + mutate(value = .data[["value"]] / 1E6, # dwel/yr -> million dwel/yr + unit = "million dwel/yr") %>% + as.quitte() + + unit <- "million dwel/yr" + description <- "flow of newly constructed buildings" + + }, + + + ## Odyssee_constructionFloor ==== + + Odyssee_constructionFloor = { + + data <- refMap %>% + select("variable") %>% + unique() %>% + separate("variable", c("dwel", "size"), sep = "_") %>% + left_join(households, by = c(dwel = "variable")) %>% + left_join(households, by = c(size = "variable", "region", "period")) %>% + mutate(value = .data[["value.x"]] * .data[["value.y"]] / 1E6, # m2/yr -> million m2/yr + unit = "million m2/yr") %>% + select(-"value.x", -"value.y") %>% + unite("variable", "dwel", "size", sep = "_") %>% + as.quitte() %>% + interpolate_missing_periods(periods, expand.values = TRUE) # TODO: remove extrapolation + + unit <- "million m2/yr" + description <- "flow of newly constructed buildings" + + }, + + + + ## Odyssee_heatingShare ==== + + Odyssee_heatingShare = { + data <- rbind(households, services) + + data <- refMap %>% + select("variable") %>% + unique() %>% + separate("variable", c("heating", "total"), sep = "_") %>% + left_join(households, by = c(heating = "variable")) %>% + left_join(households, by = c(total = "variable", "region", "period")) %>% + mutate(value = .data[["value.x"]] / .data[["value.y"]], + unit = "1") %>% + select(-"value.x", -"value.y") %>% + unite("variable", "heating", "total", sep = "_") %>% + as.quitte() %>% + interpolate_missing_periods(periods) # TODO: remove extrapolation + + # TODO: weights + + unit <- "1" + description <- "share of heating systems in the stock" + + }, + + + ## EUBDB_vintage ==== + + EUBDB_vintage = { + + data <- refMap %>% + select("variable") %>% + unique() %>% + left_join(eubdb, by = "variable") %>% + mutate(unit = "1") %>% + as.quitte() + + unit <- "1" + description <- "share of dwellings in vintage cohort" + + }, + + + ## EUBDB_stock ==== + + EUBDB_stock = { + + data <- refMap %>% + select("variable", "qty") %>% + unique() %>% + left_join(eubdb, by = "variable") %>% + mutate(value = .data[["value"]] / 1E6, # qty/yr-> million qty/yr + unit = c(dwel = "million dwel", area = "million m2")[.data[["qty"]]]) %>% + select(-"qty") %>% + as.quitte() + + unit <- "million dwel/yr" + description <- "stock of dwellings and service floor space" + }, + + + ## EuropeanCommissionRenovation ==== + + EuropeanCommissionRenovation = { + + # The data represents the average between 2014 and 2017. But we assume + # this value for all historic periods temporarily + data <- readSource("EuropeanCommissionRenovation") %>% + as.quitte(na.rm = TRUE) %>% + unite("variable", "variable", "renovation", "subsector", sep = "|") %>% + right_join(refMap %>% + select("variable") %>% + unique(), + by = "variable") %>% + select(-"model", -"scenario") %>% + group_by(across(all_of(c("region", "variable", "unit", "value")))) %>% + complete(period = periods) %>% + as.quitte() + + unit <- "1/yr" + description <- "stock of dwellings and service floor space" + }, + + stop("The subtype '", subtype, "' is an invalid matching reference.") + ) + + # nolint end: todo_comment_linter. + + data <- data %>% + filter(.data[["period"]] %in% periods) %>% + as.magpie() %>% + collapseDim(keepdim = c("region", "period", "variable")) %>% + toolCountryFill(NA, verbosity = 2) + + + return(list(x = data, + weight = weight, + unit = unit, + min = minVal, + max = maxVal, + description = description)) +} diff --git a/R/convertEnergiforsk2016.R b/R/convertEnergiforsk2016.R new file mode 100644 index 0000000..16270b5 --- /dev/null +++ b/R/convertEnergiforsk2016.R @@ -0,0 +1,26 @@ +#' Convert European district heating prices +#' +#' We should consider prper extrapolation +#' +#' @author Robin Hasse +#' +#'@param x MAgPIE object with original district heating prices +#' +#' @importFrom madrat toolCountry2isocode toolCountryFill +#' @importFrom magclass getItems<- getItems collapseDim +#' @export + +convertEnergiforsk2016 <- function(x) { + + data <- x + + # rename regions: ISO2 -> ISO3 + getItems(data, 1) <- toolCountry2isocode(getItems(data, 1)) + + # fill missing countries with NA + data <- data %>% + collapseDim() %>% + toolCountryFill(verbosity = 2) + + return(data) +} diff --git a/R/readEHI.R b/R/readEHI.R index 49d56be..ba39c29 100644 --- a/R/readEHI.R +++ b/R/readEHI.R @@ -1,14 +1,5 @@ #' Read EHI Heating Market Report #' -#' PDF market reports on Renewables in Europe -#' -#' @source https://www.eurobserv-er.org/ -#' @note I download the report and snip a png of the table. Then I extract the -#' data with [this online tool](https://extracttable.com/) and save it as a -#' tabular csv in the source folder. The Tool is not fully reliable. This -#' function throws an error if values don't sum up. You then have to correct the -#' csv file manually. -#' #' @param subtype . #' @returns magpie object #' diff --git a/R/readEnergiforsk2016.R b/R/readEnergiforsk2016.R new file mode 100644 index 0000000..94eb037 --- /dev/null +++ b/R/readEnergiforsk2016.R @@ -0,0 +1,21 @@ +#' Read European district heating prices +#' +#' @author Robin Hasse +#' +#' @importFrom utils read.csv +#' @importFrom tidyr pivot_longer +#' @importFrom dplyr select matches mutate rename +#' @importFrom quitte as.quitte +#' @importFrom magclass as.magpie +#' @export + +readEnergiforsk2016 <- function() { + file <- "european-district-heating-price-series-energiforskrapport-2016-316.csv" + read.csv(file, skip = 1) %>% + select(-matches("All\\.countries")) %>% + pivot_longer(-"Year", names_to = "region", values_to = "value") %>% + mutate(region = gsub("\\.", " ", .data[["region"]])) %>% + rename(period = "Year") %>% + as.quitte() %>% + as.magpie() +} diff --git a/README.md b/README.md index 66326fb..92c1bd3 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # Prepare data to be used by the EDGE-Buildings model -R package **mredgebuildings**, version **0.2.2** +R package **mredgebuildings**, version **0.3.0** [![CRAN status](https://www.r-pkg.org/badges/version/mredgebuildings)](https://cran.r-project.org/package=mredgebuildings) [![R build status](https://github.com/pik-piam/mredgebuildings/workflows/check/badge.svg)](https://github.com/pik-piam/mredgebuildings/actions) [![codecov](https://codecov.io/gh/pik-piam/mredgebuildings/branch/master/graph/badge.svg)](https://app.codecov.io/gh/pik-piam/mredgebuildings) [![r-universe](https://pik-piam.r-universe.dev/badges/mredgebuildings)](https://pik-piam.r-universe.dev/builds) @@ -38,7 +38,7 @@ In case of questions / problems please contact Robin Hasse . +Hasse R, Führlich P, Levesque A, Tockhorn H (2023). _mredgebuildings: Prepare data to be used by the EDGE-Buildings model_. R package version 0.3.0, . A BibTeX entry for LaTeX users is @@ -47,7 +47,7 @@ A BibTeX entry for LaTeX users is title = {mredgebuildings: Prepare data to be used by the EDGE-Buildings model}, author = {Robin Hasse and Pascal Führlich and Antoine Levesque and Hagen Tockhorn}, year = {2023}, - note = {R package version 0.2.2}, + note = {R package version 0.3.0}, url = {https://github.com/pik-piam/mredgebuildings}, } ``` diff --git a/man/calcMatchingReference.Rd b/man/calcMatchingReference.Rd new file mode 100644 index 0000000..f039b00 --- /dev/null +++ b/man/calcMatchingReference.Rd @@ -0,0 +1,17 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/calcMatchingReference.R +\name{calcMatchingReference} +\alias{calcMatchingReference} +\title{Calculate Reference data for input matching} +\usage{ +calcMatchingReference(subtype) +} +\arguments{ +\item{subtype}{character, matching reference} +} +\description{ +Calculate Reference data for input matching +} +\author{ +Robin Hasse +} diff --git a/man/convertEnergiforsk2016.Rd b/man/convertEnergiforsk2016.Rd new file mode 100644 index 0000000..08f998c --- /dev/null +++ b/man/convertEnergiforsk2016.Rd @@ -0,0 +1,17 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/convertEnergiforsk2016.R +\name{convertEnergiforsk2016} +\alias{convertEnergiforsk2016} +\title{Convert European district heating prices} +\usage{ +convertEnergiforsk2016(x) +} +\arguments{ +\item{x}{MAgPIE object with original district heating prices} +} +\description{ +We should consider prper extrapolation +} +\author{ +Robin Hasse +} diff --git a/man/readEHI.Rd b/man/readEHI.Rd index 1c61f00..4e4448b 100644 --- a/man/readEHI.Rd +++ b/man/readEHI.Rd @@ -3,9 +3,6 @@ \name{readEHI} \alias{readEHI} \title{Read EHI Heating Market Report} -\source{ -https://www.eurobserv-er.org/ -} \usage{ readEHI(subtype) } @@ -16,14 +13,7 @@ readEHI(subtype) magpie object } \description{ -PDF market reports on Renewables in Europe -} -\note{ -I download the report and snip a png of the table. Then I extract the -data with \href{https://extracttable.com/}{this online tool} and save it as a -tabular csv in the source folder. The Tool is not fully reliable. This -function throws an error if values don't sum up. You then have to correct the -csv file manually. +Read EHI Heating Market Report } \author{ Robin Hasse diff --git a/man/readEnergiforsk2016.Rd b/man/readEnergiforsk2016.Rd new file mode 100644 index 0000000..0f95d4f --- /dev/null +++ b/man/readEnergiforsk2016.Rd @@ -0,0 +1,14 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/readEnergiforsk2016.R +\name{readEnergiforsk2016} +\alias{readEnergiforsk2016} +\title{Read European district heating prices} +\usage{ +readEnergiforsk2016() +} +\description{ +Read European district heating prices +} +\author{ +Robin Hasse +}