diff --git a/.Rbuildignore b/.Rbuildignore index 1c67824..058b808 100755 --- a/.Rbuildignore +++ b/.Rbuildignore @@ -3,3 +3,5 @@ ^docs$ ^\.github$ ^\.ipynb +.lintr +.gitignore diff --git a/.github/workflows/R-CMD-check.yaml b/.github/workflows/R-CMD-check.yaml index b39aee0..5c813d4 100755 --- a/.github/workflows/R-CMD-check.yaml +++ b/.github/workflows/R-CMD-check.yaml @@ -31,13 +31,13 @@ jobs: config: - {os: macOS-latest, r: 'release'} - {os: windows-latest, r: 'release'} - - {os: windows-latest, r: 'oldrel', rspm: "https://packagemanager.rstudio.com/cran/latest"} - - {os: ubuntu-22.04, r: 'devel', rspm: "https://packagemanager.rstudio.com/cran/__linux__/bionic/latest"} - - {os: ubuntu-22.04, r: 'release', rspm: "https://packagemanager.rstudio.com/cran/__linux__/bionic/latest"} - - {os: ubuntu-22.04, r: 'oldrel', rspm: "https://packagemanager.rstudio.com/cran/__linux__/bionic/latest"} + - {os: ubuntu-24.04, r: 'devel', rspm: "https://packagemanager.rstudio.com/cran/__linux__/bionic/latest"} + - {os: ubuntu-24.04, r: 'release', rspm: "https://packagemanager.rstudio.com/cran/__linux__/bionic/latest"} + - {os: ubuntu-24.04, r: 'oldrel', rspm: "https://packagemanager.rstudio.com/cran/__linux__/bionic/latest"} # - {os: ubuntu-18.04, r: '3.6', rspm: "https://packagemanager.rstudio.com/cran/__linux__/bionic/latest"} # - {os: ubuntu-18.04, r: '3.5', rspm: "https://packagemanager.rstudio.com/cran/__linux__/bionic/latest"} # - {os: ubuntu-18.04, r: '3.4', rspm: "https://packagemanager.rstudio.com/cran/__linux__/bionic/latest"} + # - {os: windows-latest, r: 'oldrel', rspm: "https://packagemanager.rstudio.com/cran/latest"} env: RSPM: ${{ matrix.config.rspm }} diff --git a/.github/workflows/make_doc.R b/.github/workflows/make_doc.R index cd26c23..ed4348d 100755 --- a/.github/workflows/make_doc.R +++ b/.github/workflows/make_doc.R @@ -1,8 +1,8 @@ -pkgs<-c("pkgdown","devtools","rjson","xml2","data.table") +pkgs<-c("pkgdown","devtools","jsonlite","xml2","data.table","pak") pkgs_to_install<-pkgs[!(pkgs %in% installed.packages()[,"Package"])] if (length(pkgs_to_install)>0) install.packages(pkgs_to_install) -devtools::install_github("eurostat/restatapi") +pak::pak("eurostat/restatapi") cat(getwd()) cat(Sys.info()['release']) message(getwd()) diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..24fcebd --- /dev/null +++ b/.gitignore @@ -0,0 +1,7 @@ +.Rproj.user +.Rhistory +.RData +.Ruserdata +*.Rproj +.lintr +.gitignore diff --git a/DESCRIPTION b/DESCRIPTION index d983ce6..f5c86ce 100755 --- a/DESCRIPTION +++ b/DESCRIPTION @@ -1,8 +1,8 @@ Package: restatapi Type: Package Title: Search and Retrieve Data from Eurostat Database -Date: 2026-03-27 -Version: 0.25.0 +Date: 2026-10-08 +Version: 0.30.0 Encoding: UTF-8 Authors@R: c(person("Mátyás", "Mészáros", email = "matyas.meszaros@ec.europa.eu", role = c("aut", "cre")), person("Sebastian", "Weinand", role = "ctb")) @@ -12,9 +12,9 @@ Description: Eurostat is the statistical office of the European Union and provid () to search and download data from the Eurostat database using the SDMX standard. License: EUPL -Imports: data.table, rjson, xml2 -Suggests: chron, knitr, rmarkdown, tinytest, remotes +Imports: data.table, jsonlite, xml2 +Suggests: tinytest NeedsCompilation: no URL: https://github.com/eurostat/restatapi BugReports: https://github.com/eurostat/restatapi/issues -RoxygenNote: 7.3.2 +Config/roxygen2/version: 8.1.0 diff --git a/NEWS.md b/NEWS.md index ae4e9ee..446aec1 100755 --- a/NEWS.md +++ b/NEWS.md @@ -1,3 +1,19 @@ +# restatapi 0.30.0 + +Major revision of the package including: +- replaced the `rjson` package with `jsonlite` for parsing the configuration file +- removed the `chron`, `remotes`, `knitr` and `rmarkdown` packages from the suggested dependencies +- shortened the verbose debug messages so they no longer include the source code of functions from the call stack +- fixed the `get_eurostat_data()` function so that the `mode="csv"` and `mode="xml"` downloads return the same result: the empty `CONF_STATUS` column is dropped and a non-empty value is added into the `flags` column +- aligned the `CONF_STATUS`/flag handling in the `get_eurostat_bulk()` function with the `get_eurostat_data()` function, folding the `CONF_STATUS` value into the `flags` column +- the `extract_data()` function now also extracts the `CONF_STATUS` value from the XML so the confidentiality information is available in the `flags` column +- added regression tests checking the `mode="csv"`/`mode="xml"` parity for datasets with the `CONF_STATUS` column +- new `dsd_order` parameter in the `get_eurostat_data()` function to have a standard order of columns independently from the download mode +- `load_cfg()` now accepts a path to a local JSON configuration file in the `cfg_file` parameter +- removal of the `check_toc` option from the `extract_data()` function as the TOC does not have the link anymore to the XML format +- additional tests +- CRAN release + # restatapi 0.25.0 - additional debugging for `get_eurostat_toc()` function diff --git a/R/create_filter_table.R b/R/create_filter_table.R index 9a174cd..ab415d2 100755 --- a/R/create_filter_table.R +++ b/R/create_filter_table.R @@ -186,7 +186,7 @@ create_filter_table <- function(filters,date_filter=FALSE,dsd=NULL,exact_match=T } } else{ concepts<-names(filters) - ft<-data.table::rbindlist(lapply(1:length(filters),function (x,f,d){ + ft<-data.table::rbindlist(lapply(seq_along(filters),function (x,f,d){ do.call(rbind,lapply(unlist(f[x]),restatapi::search_eurostat_dsd,dsd=d[toupper(d$concept)==toupper(concepts[x]),],exact_match=exact_match,...)) },f=filters,d=dsd)) } diff --git a/R/extract_data.R b/R/extract_data.R index 3b0ade0..f31642e 100755 --- a/R/extract_data.R +++ b/R/extract_data.R @@ -1,16 +1,15 @@ -#' @title Extract data values from SDMX XML +#' @title Extract data values from SDMX XML #' @description Extracts the data values from the SDMX XML data file #' @param xml_lf an input XML leaf with data series from an SDMX XML file to extract the value and its dimensions from it #' @param keep_flags a logical value if to extract the observation status (flag) information from the XML file. The default value is \code{FALSE} -#' @param stringsAsFactors a logical value. If \code{TRUE} the columns are converted to factors. The default is \code{FALSE}, +#' @param stringsAsFactors a logical value. If \code{TRUE} the columns are converted to factors. The default is \code{FALSE}, #' in this case the strings are returned as characters. -#' @param bulk a logical value with default value \code{TRUE} if the input SDMX XML file is from the bulk download facility containing all the observations. -#' If the input file has pre-filtered values then the value \code{FALSE} should be used. -#' @param check_toc if the data file was downloaded using the URL from the TOC or not. The default is FALSE means not the TOC link is used. -#' @export +#' @param bulk a logical value with default value \code{TRUE} if the input SDMX XML file is from the bulk download facility containing all the observations. +#' If the input file has pre-filtered values then the value \code{FALSE} should be used. +#' @export #' @details It is a sub-function to use in the \code{\link{get_eurostat_data}} and the \code{\link{get_eurostat_raw}} functions. #' @return a data frame containing the values of an SDMX node: the dimensions, value and the optional flag(s) -#' @examples +#' @examples #' \dontshow{ #' if (parallel::detectCores()<=2){ #' options(restatapi_cores=1) @@ -29,9 +28,9 @@ #' extract_data(xml_leafs[1]) #' options(timeout=60) #' } -#' +#' -extract_data<-function(xml_lf,keep_flags=FALSE,stringsAsFactors=FALSE,bulk=TRUE,check_toc=FALSE){ +extract_data<-function(xml_lf,keep_flags=FALSE,stringsAsFactors=FALSE,bulk=TRUE){ # if (getOption("restatapi_verbose",FALSE)) {message("extract_data - API version:",get("rav",envir=restatapi::.restatapi_env))} rav<-get("rav",envir=restatapi::.restatapi_env) prefix<-switch(rav,"1"="generic:","2"="g:") @@ -39,30 +38,29 @@ extract_data<-function(xml_lf,keep_flags=FALSE,stringsAsFactors=FALSE,bulk=TRUE, if (Sys.info()[['sysname']]=='Windows'){ xml_lf<-gsub(prefix,"",xml_lf) xml_lf<-xml2::as_xml_document(xml_lf) - } + } bd<-t(as.data.frame(xml2::xml_attrs(xml_lf),stringsAsFactors=FALSE)) rownames(bd)<-NULL dv<-xml2::xml_attrs(xml2::xml_children(xml_lf)) if (keep_flags){ - flagc<-switch(rav,"1"="OBS_STATUS","2"="OBS_FLAG") - # if (check_toc) {flagc<-"OBS_STATUS"} + flagc<-switch(rav,"1"="OBS_STATUS","2"=c("OBS_FLAG","CONF_STATUS")) cn<-c("TIME_PERIOD","OBS_VALUE",flagc) } else { cn<-c("TIME_PERIOD","OBS_VALUE") } df<-do.call(rbind, lapply(lapply(dv, unlist),"[", cn)) colnames(df)<-cn - out<-data.frame(bd,df,stringsAsFactors=FALSE) + out<-data.frame(bd,df,stringsAsFactors=FALSE) } else { # if (Sys.info()[['sysname']]=='Windows'){ # xml_lf<-xml2::as_xml_document(xml_lf) - # } + # } tmp<-as.data.frame(xml2::xml_attrs(xml2::xml_children(xml2::xml_find_all(xml_lf,paste0(".//",prefix,"SeriesKey")))),stringsAsFactors=FALSE) bd<-as.data.frame(tmp[2,],stringsAsFactors=FALSE) colnames(bd)<-tmp[1,] rownames(bd)<-NULL obs<-xml2::xml_find_all(xml_lf,paste0(".//",prefix,"Obs")) - cn<-c("obsTime","obsValue") + cn<-c("obsTime","obsValue") df<-data.table::rbindlist(lapply(obs, function(x) {dr<-as.data.frame(t(unlist(xml2::xml_attrs(xml2::xml_children(x)))),stringsAsFactors=FALSE) rownames(dr)<-NULL if (ncol(dr)<2) {dr$nc<-""} @@ -73,7 +71,7 @@ extract_data<-function(xml_lf,keep_flags=FALSE,stringsAsFactors=FALSE,bulk=TRUE, dr$OBS_STATUS<-"" } else{ f<-gsub("na","",f) - f<-tolower(f) + #f<-tolower(f) f<-gsub("@","",f) dr$OBS_STATUS<-paste0(f,collapse="") } @@ -84,7 +82,7 @@ extract_data<-function(xml_lf,keep_flags=FALSE,stringsAsFactors=FALSE,bulk=TRUE, # df$obsValue<-as.numeric(df$obsValue) df$obsValue[df$obsValue=="NaN"]<-NA if (keep_flags){ - out<-data.frame(bd,df,stringsAsFactors=stringsAsFactors) + out<-data.frame(bd,df,stringsAsFactors=stringsAsFactors) } else{ out<-data.frame(bd,df[,1:2],stringsAsFactors=stringsAsFactors) } diff --git a/R/extract_dsd.R b/R/extract_dsd.R index a7c9e15..c97cb5b 100755 --- a/R/extract_dsd.R +++ b/R/extract_dsd.R @@ -1,20 +1,20 @@ -#' @title Extract the Data Structure Definition content from SDMX XML +#' @title Extract the Data Structure Definition content from SDMX XML #' @description Extracts values from the XML Data Structure Definition (DSD) file -#' @param concept a character vector with a concept id +#' @param concept a character vector with a concept id #' @param dsd_xml an XML file with DSD content -#' @param lang a character string either \code{en}, \code{de} or \code{fr} to define the language version for the name column +#' @param lang a character string either \code{en}, \code{de} or \code{fr} to define the language version for the name column #' of the DSD. It is used only in the new API. The default is \code{en} - English. -#' @export +#' @export #' @details It is a sub-function to use in the \code{\link{get_eurostat_dsd}} function. -#' @return a matrix with 3 columns if the provided \code{concept} has a code list in the DSD file. The first column is the provided \code{concept}. The second column +#' @return a matrix with 3 columns if the provided \code{concept} has a code list in the DSD file. The first column is the provided \code{concept}. The second column #' is the possible codes under the given \code{concept}. The last column is the name/description for the code in the second column, which can be used as labels. -#' @examples +#' @examples #' \dontshow{ #' if (parallel::detectCores()<=2){ #' options(restatapi_cores=1) #' }else{ #' options(restatapi_cores=2) -#' } +#' } #' } #' \donttest{ #' id<-"med_rd6" @@ -32,28 +32,28 @@ #' ) #' if (!(grepl("amzn|-aws|-azure ",Sys.info()['release']))) options(timeout=2) #' tryCatch({ -#' dsd_xml<-xml2::read_xml(dsd_url)}, +#' dsd_xml<-xml2::read_xml(dsd_url)}, #' error=function(e){ -#' message("Unable to download the xml file.\n",e)}, +#' message("Unable to download the xml file.\n",e)}, #' warning=function(w){ -#' message("Unable to download the xml file.\n",w)}) -#' if (exists("dsd_xml")) extract_dsd("FREQ",dsd_xml) +#' message("Unable to download the xml file.\n",w)}) +#' if (exists("dsd_xml")) extract_dsd("FREQ",dsd_xml) #' options(timeout=60) #' } extract_dsd<-function(concept=NULL,dsd_xml=NULL,lang="en"){ rav<-get("rav",envir=restatapi::.restatapi_env) if (getOption("restatapi_verbose",FALSE)) {message("\nextract_dsd - API version:",rav)} - + xml_string_clc<-switch(rav, "1" = paste0('//str:Codelist[@id="CL_',concept,'"]/str:Code'), "2" = paste0('//s:Codelist[@id="',toupper(concept),'"]/s:Code') - + ) xml_string_cln<-switch(rav, "1" = paste0('//str:Codelist[@id="CL_',concept,'"]/str:Code/com:Name'), "2" = paste0('//s:Codelist[@id="',toupper(concept),'"]/s:Code/c:Name[@xml:lang="',lang,'"]') - + ) if (getOption("restatapi_verbose",FALSE)) {message("extract_dsd - xml_string_clc:",xml_string_clc,"\nextract_dsd - xml_string_cln:",xml_string_cln)} @@ -67,5 +67,5 @@ extract_dsd<-function(concept=NULL,dsd_xml=NULL,lang="en"){ xml_cln<-xml2::xml_text(xml2::xml_find_all(dsd_xml,xml_string_cln)) cbind(concept,xml_clc,xml_cln) } - } -} \ No newline at end of file + } +} diff --git a/R/filter_raw_data.R b/R/filter_raw_data.R index 607aed4..fa0b2bc 100755 --- a/R/filter_raw_data.R +++ b/R/filter_raw_data.R @@ -60,7 +60,7 @@ filter_raw_data<-function(raw_data=NULL,filter_table=NULL,date_filter=FALSE){ raw_data[grepl("\\dQ",time),ft:=paste0(substr(time,1,4),"-",lapply(substr(time,6,6),function(x){if (x<"4"){"0"}else{""}}),as.character((as.numeric(substr(time,6,6))-1)*3+1),"-01")] raw_data[grepl("-Q",time),ft:=paste0(substr(time,1,4),"-",lapply(substr(time,7,7),function(x){if (x<"4"){"0"}else{""}}),as.character((as.numeric(substr(time,7,7))-1)*3+1),"-01")] raw_data[grepl("S",time),ft:=paste0(substr(time,1,4),"-0",as.character((as.numeric(substr(time,6,6))-1)*6+1),"-01")] - data_out<-data.table::rbindlist(lapply(1:nrow(filter_table),function (x){raw_data[(ft>=filter_table$sd[x] & ft<=filter_table$ed[x])]})) + data_out<-data.table::rbindlist(lapply(seq_len(nrow(filter_table)),function (x){raw_data[(ft>=filter_table$sd[x] & ft<=filter_table$ed[x])]})) data_out<-data_out[,!'ft'][] }else{ filter_table$concept<-toupper(filter_table$concept) diff --git a/R/get_compressed_sdmx.R b/R/get_compressed_sdmx.R index 896421d..5a4c2f8 100755 --- a/R/get_compressed_sdmx.R +++ b/R/get_compressed_sdmx.R @@ -34,11 +34,11 @@ get_compressed_sdmx<-function(url=NULL,verbose=FALSE,format="gz"){ if (verbose) { tryCatch({utils::download.file(url,temp,dmethod)}, error = function(e) { - message("get_compressed_sdmx - Error by the download of the SDMX file:",'\n',paste(unlist(e),collapse="\n")) + message("get_compressed_sdmx - Error by the download of the SDMX file:",'\n',conditionMessage(e)) tbc<-FALSE }, warning = function(w) { - message("get_compressed_sdmx - Warning during the download of the SDMX file:",'\n',paste(unlist(w),collapse="\n")) + message("get_compressed_sdmx - Warning during the download of the SDMX file:",'\n',conditionMessage(w)) tbc<-FALSE }) } else { @@ -62,10 +62,10 @@ get_compressed_sdmx<-function(url=NULL,verbose=FALSE,format="gz"){ if (verbose) { tryCatch({xml_fajl<-utils::unzip(temp,paste0(fajl,".xml"),exdir=tmpdir)}, error = function(e) { - message("get_compressed_sdmx - Error during the unzip of the SDMX file:",'\n',paste(unlist(e),collapse="\n")) + message("get_compressed_sdmx - Error during the unzip of the SDMX file:",'\n',conditionMessage(e)) }, warning = function(w) { - message("get_compressed_sdmx - Warning by the unzip of the SDMX file:",'\n',paste(unlist(w),collapse="\n")) + message("get_compressed_sdmx - Warning by the unzip of the SDMX file:",'\n',conditionMessage(w)) }) } else { tryCatch({xml_fajl<-utils::unzip(temp,paste0(fajl,".xml"),exdir=tmpdir)}, @@ -82,10 +82,10 @@ get_compressed_sdmx<-function(url=NULL,verbose=FALSE,format="gz"){ xml<-xml2::read_xml(cid) close(cid)}, error = function(e) { - message("get_compressed_sdmx - Error during retrieval and extraction of the SDMX file:",'\n',paste(unlist(e),collapse="\n")) + message("get_compressed_sdmx - Error during retrieval and extraction of the SDMX file:",'\n',conditionMessage(e)) }, warning = function(w) { - message("get_compressed_sdmx - Warning by the retrieval and extraction of the SDMX file:",'\n',paste(unlist(w),collapse="\n")) + message("get_compressed_sdmx - Warning by the retrieval and extraction of the SDMX file:",'\n',conditionMessage(w)) }) } else { tryCatch({ diff --git a/R/get_eurostat_bulk.R b/R/get_eurostat_bulk.R index 95bef56..90643b9 100755 --- a/R/get_eurostat_bulk.R +++ b/R/get_eurostat_bulk.R @@ -5,53 +5,53 @@ #' @param cache a logical value whether to do caching. Default is \code{TRUE}. #' @param update_cache a logical value with a default value \code{FALSE}, whether to update cache. Can be set also with #' \code{options(restatapi_update=TRUE)}. -#' @param cache_dir a path to a cache directory. The \code{NULL} (default) uses the memory as cache. -#' If the folder \code{cache_dir} directory does not exist it saves in the 'restatapi' directory +#' @param cache_dir a path to a cache directory. The \code{NULL} (default) uses the memory as cache. +#' If the folder \code{cache_dir} directory does not exist it saves in the 'restatapi' directory #' under the temporary directory from \code{tempdir()}. Directory can also be set with #' \code{option(restatapi_cache_dir=...)}. #' @param compress_file a logical value whether to compress the #' RDS-file in caching. Default is \code{TRUE}. -#' @param stringsAsFactors a logical value with the default \code{TRUE}. In this case the columns are converted to factors. If \code{FALSE}, +#' @param stringsAsFactors a logical value with the default \code{TRUE}. In this case the columns are converted to factors. If \code{FALSE}, #' the strings are returned as characters. #' @param select_freq a character symbol for a time frequency when a dataset has multiple time #' frequencies. Possible values are: -#' A = annual, S = semi-annual, H = half-year, Q = quarterly, M = monthly, W = weekly, D = daily. -#' The default is \code{NULL} as most datasets have only one time frequency. +#' A = annual, S = semi-annual, H = half-year, Q = quarterly, M = monthly, W = weekly, D = daily. +#' The default is \code{NULL} as most datasets have only one time frequency. #' In case if there are multiple frequencies and \code{select_freq=NULL}, then only the most common frequency kept. #' If all the frequencies needed the \code{\link{get_eurostat_raw}} function can be used. #' @param keep_flags a logical value whether the observation status (flags) - e.g. "confidential", #' "provisional", etc. - should be kept in a separate column or if they -#' can be removed. Default is \code{FALSE}. For flag values see: +#' can be removed. Default is \code{FALSE}. For flag values see: #' \url{https://ec.europa.eu/eurostat/api/dissemination/sdmx/2.1/codelist/ESTAT/OBS_STATUS/?compressed=false&format=TSV&lang=en}. -#' @param cflags a logical value whether the missing observations with flag 'c' - "confidential" -#' should be kept or not. Default is \code{FALSE}, in this case these observations dropped from the dataset. If this parameter -#' \code{TRUE} then all the flags and the suppressed observations with missing values are kept. In this case the parameter provided in \code{keep_flags} is set to \code{TRUE}. -#' @param check_toc a logical value whether to check the provided \code{id} in the Table of Contents (TOC) or not. The default value -#' \code{FALSE}, in this case the base URL for the download link is retrieved from the configuration file. -#' If the value is \code{TRUE} then the TOC is downloaded and the \code{id} is checked in it. If it found there then the download link -#' is retrieved form the TOC. +#' @param cflags a logical value whether the missing observations with flag 'C' - "confidential" +#' should be kept or not. Default is \code{FALSE}, in this case these observations dropped from the dataset. If this parameter +#' \code{TRUE} then all the flags and the suppressed observations with missing values are kept. In this case the parameter provided in \code{keep_flags} is set to \code{TRUE}. +#' @param check_toc a logical value whether to check the provided \code{id} in the Table of Contents (TOC) or not. The default value +#' \code{FALSE}, in this case the base URL for the download link is retrieved from the configuration file. +#' If the value is \code{TRUE} then the TOC is downloaded and the \code{id} is checked in it. If it found there then the download link +#' is retrieved form the TOC. #' @param verbose a logical value with default \code{FALSE}, so detailed messages (for debugging) will not printed. #' Can be set also with \code{options(restatapi_verbose=TRUE)}. -#' @param ... other parameter(s) to pass on the \code{\link{load_cfg}} function +#' @param ... other parameter(s) to pass on the \code{\link{load_cfg}} function #' @export -#' -#' @details Data sets are downloaded from \href{https://ec.europa.eu/eurostat/web/user-guides/data-browser/api-data-access/api-migrating/bulkdownload}{the Eurostat bulk download facility} +#' +#' @details Data sets are downloaded from \href{https://ec.europa.eu/eurostat/web/user-guides/data-browser/api-data-access/api-migrating/bulkdownload}{the Eurostat bulk download facility} #' in TSV format as in this case smaller file has to be downloaded and processed. If there is more then one frequency then #' the dataset is filtered for a unique time frequency. #' If no frequency is selected and there are multiple frequencies in the dataset, then the most common value is used used for frequency. -#' -#' Compared to the ouptut of the \code{\link{get_eurostat_raw}} function, the frequency (FREQ) and time format (TIME_FORMAT) columns are not included in the bulk data -#' and the column names for the time period, observation values and status have standardised names: "time", "values" and "flags" +#' +#' Compared to the ouptut of the \code{\link{get_eurostat_raw}} function, the frequency (FREQ) and time format (TIME_FORMAT) columns are not included in the bulk data +#' and the column names for the time period, observation values and status have standardised names: "time", "values" and "flags" #' independently if the data was downloaded previously in SDMX or TSV format. -#' -#' By default all datasets cached as they are often rather large. +#' +#' By default all datasets cached as they are often rather large. #' The datasets cached in memory (default) or can be stored in a temporary directory if \code{cache_dir} or \code{option(restatpi_cache_dir)} is defined. #' The cache can be emptied with \code{\link{clean_restatapi_cache}}. -#' +#' #' The \code{id}, is a value from the \code{code} column of the table of contents (\code{\link{get_eurostat_toc}}), and can be searched for it with the \code{\link{search_eurostat_toc}} function. The id value can be retrieved from the \href{https://ec.europa.eu/eurostat/data/database}{Eurostat database} #' as well. The Eurostat database gives codes in the Data Navigation Tree after every dataset #' in parenthesis. -#' @return a data.table with the following columns: +#' @return a data.table with the following columns: #' \tabular{ll}{ #' dimension names \tab One column for each dimension in the data\cr #' \code{time} \tab A column for the time dimension\cr @@ -62,13 +62,13 @@ #' The data.table does not include all missing values. The missing values are dropped if both the value and the flag is missing #' on a particular time. #' @seealso \code{\link{get_eurostat_data}}, \code{\link{get_eurostat_raw}} -#' @examples +#' @examples #' \dontshow{ #' if (parallel::detectCores()<=2){ #' options(restatapi_cores=1) #' }else{ #' options(restatapi_cores=2) -#' } +#' } #' } #' \donttest{ #' if (!(grepl("amzn|-aws|-azure ",Sys.info()['release']))) options(timeout=2) @@ -93,8 +93,8 @@ get_eurostat_bulk <- function(id, cflags=FALSE, check_toc=FALSE, verbose=FALSE,...){ - - .datatable.aware=TRUE + + .datatable.aware=TRUE FREQ<-N<-restat_bulk<-NULL tbc<-TRUE #to be continued for the next steps verbose<-verbose|getOption("restatapi_verbose",FALSE) @@ -104,21 +104,20 @@ get_eurostat_bulk <- function(id, if((!exists(".restatapi_env")|(length(list(...))>0))){ if ((length(list(...))>0)) { if (all(names(list(...)) %in% c("api_version","load_toc","parallel","max_cores","verbose"))){ - load_cfg(...) + load_cfg(...) } else { load_cfg() } } else { load_cfg() - } + } } - # if (verbose) {message("get_eurostat_bulk - API version:",get("rav",envir=restatapi::.restatapi_env))} - + if (!is.null(id)){id<-tolower(trimws(id))} else { tbc<-FALSE message("The dataset 'id' is missing.") } - + if (tbc) { if (check_toc){ toc<-restatapi::get_eurostat_toc(verbose=verbose) @@ -140,7 +139,7 @@ get_eurostat_bulk <- function(id, udate<-format(Sys.Date(),"%Y.%m.%d") } } - + if (tbc) { if ((cache)&(!update_cache)) { nm<-paste0("b_",id,"-",udate,"-",sum(keep_flags),"-",sum(cflags),sub("-$","",paste0("-",select_freq),perl=TRUE)) @@ -153,21 +152,21 @@ get_eurostat_bulk <- function(id, " - ",class(check_toc)," - ",class(TRUE)," - ",class(verbose))} restat_bulk<-restatapi::get_eurostat_raw(id,"txt",cache,update_cache,cache_dir,compress_file,stringsAsFactors,keep_flags,check_toc,melt=TRUE,verbose=verbose) } - } - + } + if (!is.null(restat_bulk)){ - restat_bulk[] + #restat_bulk[] drop<-NULL if ("freq" %in% colnames(restat_bulk)) {data.table::setnames(restat_bulk,"freq","FREQ")} if ("FREQ" %in% colnames(restat_bulk)) {drop=c("FREQ")} - if ("TIME_FORMAT" %in% colnames(restat_bulk)) {drop<-c(drop,"TIME_FORMAT")} + if ("TIME_FORMAT" %in% colnames(restat_bulk)) {drop<-c(drop,"TIME_FORMAT")} if (is.null(select_freq)){ if (length(unique(restat_bulk$FREQ))>1){ st<-data.table::setorder(restat_bulk[,.N,by=FREQ],-N)[1,1] if (is.factor(st$FREQ)){select_freq<-as.character(levels(st$FREQ)[st$FREQ[1]])}else{select_freq<-as.character(st$FREQ)} message("There are multiple frequencies in the dataset. The '", select_freq, "' is selected as it is the most common frequency.") - } - } + } + } if ((!(is.null(select_freq)))&("FREQ" %in% colnames(restat_bulk))){restat_bulk<-restat_bulk[restat_bulk$FREQ==select_freq,][]} if ("OBS_VALUE" %in% colnames(restat_bulk)) { if (keep_flags){ @@ -176,19 +175,22 @@ get_eurostat_bulk <- function(id, data.table::setnames(restat_bulk,cn,"flags")[] } else { if ("OBS_STATUS" %in% colnames(restat_bulk)) {drop<-c(drop,"OBS_STATUS")} - if ("OBS_FLAG" %in% colnames(restat_bulk)) {drop<-c(drop,"OBS_FLAG")} + if ("OBS_FLAG" %in% colnames(restat_bulk)) {drop<-c(drop,"OBS_FLAG")} } data.table::setnames(restat_bulk,c("TIME_PERIOD","OBS_VALUE"),c("time","values"))[] } restat_bulk$time<-gsub('[MD]',"-",restat_bulk$time) restat_bulk$time<-gsub('([0-9]{4})([Q|S])',"\\1-\\2",restat_bulk$time,perl=TRUE) if (keep_flags) { + if ("CONF_STATUS" %in% colnames(restat_bulk)) {restat_bulk$flags<-paste0(restat_bulk$flags,restat_bulk$CONF_STATUS)} restat_bulk$flags<-as.character(restat_bulk$flags) restat_bulk$flags[is.na(restat_bulk$flags)]<-"" - if (!cflags) {restat_bulk<-restat_bulk[restat_bulk$flags!="c"][]} + restat_bulk$flags<-gsub("@|NA","",restat_bulk$flags) + if (!cflags) {restat_bulk<-restat_bulk[!grepl("C|@C",restat_bulk$flags)][]} } else if ("flags" %in% colnames(restat_bulk)){ drop<-c(drop,"flags") } + if ("CONF_STATUS" %in% colnames(restat_bulk)) {drop<-c(drop,"CONF_STATUS")} if(!is.null(drop)) {restat_bulk[,(drop):=NULL][]} if (any(sapply(restat_bulk,is.factor))&(!stringsAsFactors)) { col_conv<-colnames(restat_bulk)[!(colnames(restat_bulk) %in% c("values"))] @@ -196,22 +198,22 @@ get_eurostat_bulk <- function(id, } if (any(!sapply(restat_bulk[],is.factor))&(stringsAsFactors)) { restat_bulk<-data.table::data.table(restat_bulk,stringsAsFactors=stringsAsFactors)[] - } + } if (is.factor(restat_bulk$values)){ if (any(grepl('\\d+\\:\\d+',restat_bulk$values[!is.na(restat_bulk$values)],perl=TRUE))){ restat_bulk$values<-as.character(levels(restat_bulk$values))[restat_bulk$values] restat_bulk$values[grepl('^\\:$',restat_bulk$values,perl=TRUE)]<-NA } else { - restat_bulk$values<-suppressWarnings(as.numeric(levels(restat_bulk$values))[restat_bulk$values]) + restat_bulk$values<-suppressWarnings(as.numeric(levels(restat_bulk$values))[restat_bulk$values]) } - } - } - + } + } + if ((!is.null(restat_bulk))&cache&(all(!grepl("get_eurostat_data",as.character(sys.calls()),perl=TRUE)))){ oname<-paste0("b_",id,"-",udate,"-",sum(keep_flags),"-",sum(cflags),sub("-$","",paste0("-",select_freq),perl=TRUE)) pl<-restatapi::put_eurostat_cache(restat_bulk,oname,update_cache,cache_dir,compress_file) if ((!is.null(pl))&(verbose)) {message("get_eurostat_bulk - The bulk data was cached ",pl,"." )} } - + return(restat_bulk) } diff --git a/R/get_eurostat_codelist.R b/R/get_eurostat_codelist.R index 6f0f5fd..1e07f20 100755 --- a/R/get_eurostat_codelist.R +++ b/R/get_eurostat_codelist.R @@ -71,11 +71,11 @@ get_eurostat_codelist <- function(id, message("\nget_eurostat_codelist - Trying to download the codelist from: ",cls_endpoint) tryCatch({utils::download.file(cls_endpoint,temp,dmethod)}, error = function(e) { - message("get_eurostat_codelist - Error by the download of the codelist file:",'\n',paste(unlist(e),collapse="\n")) + message("get_eurostat_codelist - Error by the download of the codelist file:",'\n',conditionMessage(e)) cls_xml<-NULL }, warning = function(w) { - message("get_eurostat_codelist - Warning by the download of the codelist file:",'\n',paste(unlist(w),collapse="\n")) + message("get_eurostat_codelist - Warning by the download of the codelist file:",'\n',conditionMessage(w)) tbc<-FALSE cls_xml<-NULL }) @@ -84,11 +84,11 @@ get_eurostat_codelist <- function(id, message("Trying to extract the codelist from: ",temp) tryCatch({cls_xml<-xml2::read_xml(temp)}, error = function(e) { - message("get_eurostat_codelist - Error during the extraction of the XML from the downloaded codelist file:",'\n',paste(unlist(e),collapse="\n")) + message("get_eurostat_codelist - Error during the extraction of the XML from the downloaded codelist file:",'\n',conditionMessage(e)) cls_xml<-NULL }, warning = function(w) { - message("get_eurostat_codelist - There is warning by the extraction of the XML from the downloaded codelist file:",'\n',paste(unlist(w),collapse="\n")) + message("get_eurostat_codelist - There is warning by the extraction of the XML from the downloaded codelist file:",'\n',conditionMessage(w)) cls_xml<-NULL }) } else { diff --git a/R/get_eurostat_data.R b/R/get_eurostat_data.R index 28ef97d..13367a6 100755 --- a/R/get_eurostat_data.R +++ b/R/get_eurostat_data.R @@ -3,32 +3,32 @@ #' @param id A code name for the dataset of interest. #' See \code{\link{search_eurostat_toc}} for details how to get an id. #' @param filters a string, a character vector or named list containing words to filter by the different concepts or geographical location. -#' If filter applied only part of the dataset is downloaded through the API. The words can be -#' any word, Eurostat variable code, and value which are in the DSD \code{\link{search_eurostat_dsd}}. +#' If filter applied only part of the dataset is downloaded through the API. The words can be +#' any word, Eurostat variable code, and value which are in the DSD \code{\link{search_eurostat_dsd}}. #' If a named list is used, then the name of the list elements should be the concepts from the DSD and the provided values will be used to filter the dataset for the given concept. #' The default is \code{NULL}, in this case the whole dataset is returned via the bulk download. To filter by time see \code{date_filter} below. -#' If after filtering still the dataset has more observations than the limit per query via the API, then the raw download is used to retrieve the whole dataset and apply the filter on the local computer. This option can be disabled with the \code{local_filter=FALSE} parameter. +#' If after filtering still the dataset has more observations than the limit per query via the API, then the raw download is used to retrieve the whole dataset and apply the filter on the local computer. This option can be disabled with the \code{local_filter=FALSE} parameter. #' @param lang a character string either \code{en}, \code{de} or \code{fr} to define the language version for the DSD to search in for the \code{filters}. The default is \code{en} - English. -#' @param exact_match a boolean with the default value \code{TRUE}, if the strings provided in \code{filters} shall be matched exactly as it is or as a pattern. -#' @param date_filter a vector which can be numeric or character containing dates to filter the dataset. If date is defined as character string it should follow the format yyyy[-mm][-dd], where the month and the day part is optional. -#' If date filter applied only part of the dataset is downloaded through the API. +#' @param exact_match a boolean with the default value \code{TRUE}, if the strings provided in \code{filters} shall be matched exactly as it is or as a pattern. +#' @param date_filter a vector which can be numeric or character containing dates to filter the dataset. If date is defined as character string it should follow the format yyyy[-mm][-dd], where the month and the day part is optional. +#' If date filter applied only part of the dataset is downloaded through the API. #' The default is \code{NULL}, in this case the whole dataset is returned via the bulk download. -#' If after filtering still the dataset has more observations than the limit per query via the API, then the raw download is used to retrieve the data and apply the filter on the local computer. -#' This option can be disabled with the \code{local_filter=FALSE} parameter. +#' If after filtering still the dataset has more observations than the limit per query via the API, then the raw download is used to retrieve the data and apply the filter on the local computer. +#' This option can be disabled with the \code{local_filter=FALSE} parameter. #' @param label a boolean with the default \code{FALSE}. If it is \code{TRUE} then the code values are replaced by the name from the Data Structure Definition (DSD) \code{\link{get_eurostat_dsd}}. #' For example instead of "D1110A", "Raw cows' milk from farmtype" is used or "HU32" is replaced by "Észak-Alföld". #' @param select_freq a character symbol for a time frequency when a dataset has multiple time #' frequencies. Possible values are: -#' A = annual, S = semi-annual, H = half-year, Q = quarterly, M = monthly, W = weekly, D = daily. +#' A = annual, S = semi-annual, H = half-year, Q = quarterly, M = monthly, W = weekly, D = daily. #' The default is \code{NULL} as most datasets have just one time #' frequency and in case there are multiple frequencies, then only the most common frequency kept. #' If all the frequencies needed the \code{\link{get_eurostat_raw}} can be used. -#' @param cache a logical whether to do caching. Default is \code{TRUE}. Affects +#' @param cache a logical whether to do caching. Default is \code{TRUE}. Affects #' only queries without filtering. If \code{filters} or \code{date_filter} is used then there is no caching. #' @param update_cache a logical with a default value \code{FALSE}, whether to update the data in the cache. Can be set also with #' \code{options(restatapi_update=TRUE)} -#' @param cache_dir a path to a cache directory. The \code{NULL} (default) uses the memory as cache. -#' If the folder \code{cache_dir} directory does not exist it saves in the 'restatapi' directory +#' @param cache_dir a path to a cache directory. The \code{NULL} (default) uses the memory as cache. +#' If the folder \code{cache_dir} directory does not exist it saves in the 'restatapi' directory #' under the temporary directory from \code{tempdir()}. Directory can also be set with #' \code{option(restatapi_cache_dir=...)}. #' @param compress_file a logical whether to compress the @@ -37,60 +37,62 @@ #' converted to factors. If the value \code{FALSE} they are returned as characters. #' @param keep_flags a logical whether the observation status (flags) - e.g. "confidential", #' "provisional", etc. - should be kept in a separate column or if they -#' can be removed. Default is \code{FALSE}. For flag values see: +#' can be removed. Default is \code{FALSE}. For flag values see: #' \url{https://ec.europa.eu/eurostat/api/dissemination/sdmx/2.1/codelist/ESTAT/OBS_STATUS/?compressed=false&format=TSV&lang=en}. #' @param cflags a logical whether the missing observations with flag 'c' - "confidential" -#' should be kept or not. Default is \code{FALSE}, in this case these observations dropped from the dataset. If this parameter +#' should be kept or not. Default is \code{FALSE}, in this case these observations dropped from the dataset. If this parameter #' \code{TRUE} then the flags are kept and the parameter provided in \code{keep_flags} is not taken into account. -#' @param check_toc a boolean whether to check the provided \code{id} in the Table of Contents (TOC) or not. The default value -#' \code{FALSE}, in this case the base URL for the download link is retrieved from the configuration file. -#' If the value is \code{TRUE} then the TOC is downloaded and the \code{id} is checked in it. If it found then the download link -#' is retrieved form the TOC. -#' @param local_filter a boolean whether do the filtering on the local computer or not in case after filtering still the dataset has more observations -#' than the limit per query via the API would allow to download. The default is \code{TRUE}, in this case if the response footer contains information -#' that the result cannot be downloaded becuase it is too large, then the whole raw dataset is downloaded and filtered on the local computer. -#' @param force_local_filter a boolean with the default value \code{FALSE}. In case, if there are existing filter conditions, then it will do the filtering on the local -#' computer and not requesting through the REST API. It can be useful, if the values are not numeric as these are provided as NaN (Not a Number) through the REST API, -#' but it is fully listed in the raw dataset. -#' @param mode defines the format of the dataset response from the API. It can be -#' \code{csv} for SDMX-CSV or \code{xml} for the SDMX-ML version. -#' @param verbose A boolean with default \code{FALSE}, so detailed messages (for debugging) will not printed. +#' @param check_toc a boolean whether to check the provided \code{id} in the Table of Contents (TOC) or not. The default value +#' \code{FALSE}, in this case the base URL for the download link is retrieved from the configuration file. +#' If the value is \code{TRUE} then the TOC is downloaded and the \code{id} is checked in it. If it found then the download link +#' is retrieved form the TOC. +#' @param local_filter a boolean whether do the filtering on the local computer or not in case after filtering still the dataset has more observations +#' than the limit per query via the API would allow to download. The default is \code{TRUE}, in this case if the response footer contains information +#' that the result cannot be downloaded because it is too large, then the whole raw dataset is downloaded and filtered on the local computer. +#' @param force_local_filter a boolean with the default value \code{FALSE}. In case, if there are existing filter conditions, then it will do the filtering on the local +#' computer and not requesting through the REST API. It can be useful, if the values are not numeric as these are provided as NaN (Not a Number) through the REST API, +#' but it is fully listed in the raw dataset. +#' @param mode defines the format of the dataset response from the API. It can be \code{csv} for SDMX-CSV or \code{xml} for the SDMX-ML version. +#' This parameter is ignored if \code{force_local_filter} is \code{TRUE} as it will download first the whole dataset in TSV format. +#' @param dsd_order a boolean with default \code{FALSE} defines if the output column should be ordered as it is in the DSD or not as +#' the order can be different if the \code{mode} parameter \code{csv} or \code{xml} can provide different column order. If it is \code{TRUE} the retrieval time is a bit longer as it needs to get the DSD and order the columns as well. +#' @param verbose a boolean with default \code{FALSE}, so detailed messages (for debugging) will not printed. #' Can be set also with \code{options(restatapi_verbose=TRUE)} -#' @param ... further arguments to the for \code{\link{search_eurostat_dsd}} function, e.g.: \code{ignore.case} or \code{name}. -#' The \code{ignore.case} has the default value \code{FALSE}, then the strings provided in \code{filters} are matched as is, +#' @param ... further arguments to the for \code{\link{search_eurostat_dsd}} function, e.g.: \code{ignore.case} or \code{name}. +#' The \code{ignore.case} has the default value \code{FALSE}, then the strings provided in \code{filters} are matched as is, #' otherwise the case of the letters is ignored. If the \code{name=FALSE} then the pattern(s) provided in the \code{filters} -#' argument is only searched in the code column of the DSD, and the names of the codes will not be searched. +#' argument is only searched in the code column of the DSD, and the names of the codes will not be searched. #' @export -#' -#' @details Data sets are downloaded from the Eurostat Web Services -#' \href{https://ec.europa.eu/eurostat/web/user-guides/data-browser/api-data-access/api-detailed-guidelines/sdmx2-1}{SDMX API} if there is a filter otherwise the +#' +#' @details Data sets are downloaded from the Eurostat Web Services +#' \href{https://ec.europa.eu/eurostat/web/user-guides/data-browser/api-data-access/api-detailed-guidelines/sdmx2-1}{SDMX API} if there is a filter otherwise the #' \href{https://ec.europa.eu/eurostat/web/user-guides/data-browser/api-data-access/api-migrating/bulkdownload}{the Eurostat bulk download facility} is used. #' If only the table \code{id} is given, the whole table is downloaded from the #' bulk download facility. If also \code{filters} or \code{date_filter} is defined then the SDMX REST API is #' used. In case after filtering the dataset has more rows than the limitation of the SDMX REST API (1 million values at one time) then the bulk download is used to retrieve the whole dataset . -#' -#' By default all datasets cached as they are often rather large. +#' +#' By default all datasets cached as they are often rather large. #' The datasets cached in memory (default) or can be stored in a temporary directory if \code{cache_dir} or \code{option(restatpi_cache_dir)} is defined. #' The cache can be emptied with \code{\link{clean_restatapi_cache}}. -#' -#' The \code{id}, is a value from the \code{code} column of the table of contents (\code{\link{get_eurostat_toc}}), and can be searched +#' +#' The \code{id}, is a value from the \code{code} column of the table of contents (\code{\link{get_eurostat_toc}}), and can be searched #' for with the \code{\link{search_eurostat_toc}} function. The id value can be retrieved from the \href{https://ec.europa.eu/eurostat/web/main/data/database}{Eurostat database} #' as well. The Eurostat database gives codes in the Data Navigation Tree after every dataset in parenthesis. -#' -#' Filtering can be done by the codes as described in the API documentation providing in the correct order and connecting with "." and "+". -#' If we do not know the codes we can filter based on words or by the mix of the two putting in a vector like \code{c("AT$","Belgium","persons","Total")}. -#' Be careful that the filter is case sensitive, if you do not know the code or label exactly you can use the option \code{ignore.case=TRUE} and \code{exact_match=FALSE}, -#' but in this case the results may include unwanted elements as well. In the \code{filters} parameter regular expressions can be used as well. -#' We do not have to worry about the correct order of the filter, it will be put in the correct place based on the DSD. -#' -#' +#' +#' Filtering can be done by the codes as described in the API documentation providing in the correct order and connecting with "." and "+". +#' If we do not know the codes we can filter based on words or by the mix of the two putting in a vector like \code{c("AT$","Belgium","persons","Total")}. +#' Be careful that the filter is case sensitive, if you do not know the code or label exactly you can use the option \code{ignore.case=TRUE} and \code{exact_match=FALSE}, +#' but in this case the results may include unwanted elements as well. In the \code{filters} parameter regular expressions can be used as well. +#' We do not have to worry about the correct order of the filter, it will be put in the correct place based on the DSD. +#' +#' #' The \code{date_filter} shall be a string in the format yyyy[-mm][-dd]. The month and the day part is optional, but if we use the years and we have monthly frequency then all the data for the given year is retrieved. #' The string can be extended by adding the "<" or ">" to the beginning or to the end of the string. In this case the date filter is treated as range, and the date is used as a starting or end date. The data will include the observation of the start/end date. -#' A single date range can be defined as well by concatenating two dates with the ":", e.g. \code{"2016-08:2017-03-15"}. As seen in the example the dates can have different length: one defined only at year/month level, the other by day level. +#' A single date range can be defined as well by concatenating two dates with the ":", e.g. \code{"2016-08:2017-03-15"}. As seen in the example the dates can have different length: one defined only at year/month level, the other by day level. #' If a date range is defined with ":", it is not possible to use the "<" or ">" characters in the date filter. -#' If there are multiple dates which is not a continuous range, it can be put in vector in any order like \code{c("2016-08",2013:2015,"2017-07-01")}. In this case, as well, it is not possible to use the "<" or ">" characters. -#' -#' @return a data.table with the following columns: +#' If there are multiple dates which is not a continuous range, it can be put in vector in any order like \code{c("2016-08",2013:2015,"2017-07-01")}. In this case, as well, it is not possible to use the "<" or ">" characters. +#' +#' @return a data.table with the following columns: #' \tabular{ll}{ #' \code{freq} \tab A column for the frequency of the data in case there are multiple frequencies, for #' single frequency this columns is dropped from the data table\cr @@ -100,28 +102,28 @@ #' \code{flags} \tab A column for flags if the \code{keep_flags=TRUE} or \code{cflags=TRUE} otherwise this column #' is not included in the data table #' } -#' +#' #' The data.table does not include all missing values. The missing values are dropped if the value and flag are missing #' on a particular time. -#' -#' In case the provided \code{filters} can be found in the DSD, then it is used to query the API or applied locally. If the applied \code{filters} with combination of \code{date_filter} +#' +#' In case the provided \code{filters} can be found in the DSD, then it is used to query the API or applied locally. If the applied \code{filters} with combination of \code{date_filter} #' and \code{select_freq} has no observation in the data set then the fucntion returns the data.table with 0 row. -#' +#' #' In case none of the provided \code{filters}, \code{date_filter} or \code{select_freq} can be parsed or found in the DSD then the whole dataset downloaded through the bulk download with a warning message. -#' -#' In case the \code{id} is not exist then the function returns the value \code{NULL}. -#' +#' +#' In case the \code{id} is not exist then the function returns the value \code{NULL}. +#' #' @seealso \code{\link{search_eurostat_toc}}, \code{\link{search_eurostat_dsd}}, \code{\link{get_eurostat_bulk}} -#' @examples +#' @examples #' load_cfg() #' eu<-get("cc",envir=restatapi::.restatapi_env) -#' +#' #' \dontshow{ #' if (parallel::detectCores()<=2){ #' options(restatapi_cores=1) #' }else{ #' options(restatapi_cores=2) -#' } +#' } #' } #' \donttest{ #' if (!(grepl("amzn|-aws|-azure ",Sys.info()['release']))) options(timeout=2) @@ -156,7 +158,7 @@ #' select_freq="M") #' dt<-get_eurostat_data("htec_cis3", #' filters="lu", -#' ignore.case=TRUE) +#' ignore.case=TRUE) #' dt<-get_eurostat_data("bop_its6_det", #' filters=list(bop_item="SC", #' currency="MIO_EUR", @@ -168,7 +170,7 @@ #' select_freq="A", #' label=TRUE, #' name=FALSE) -#' clean_restatapi_cache("/tmp",verbose=TRUE) +#' clean_restatapi_cache("/tmp",verbose=TRUE) #' options(timeout=60) #' } @@ -190,8 +192,9 @@ get_eurostat_data <- function(id, local_filter=TRUE, force_local_filter=FALSE, mode="xml", + dsd_order=FALSE, verbose=FALSE,...) { - + .datatable.aware=TRUE restat<-rdat<-drop<-concept<-code<-freq<-N<-values<-flags<-ft<-dft<-to_add<-filters_url<-NULL verbose<-verbose|getOption("restatapi_verbose",FALSE) @@ -199,15 +202,15 @@ get_eurostat_data <- function(id, dmethod<-getOption("restatapi_dmethod",get("dmethod",envir=restatapi::.restatapi_env)) if (getOption("restatapi_cores",1L)>=parallel::detectCores()) options(restatapi_cores=parallel::detectCores()-1) # if (verbose) {message("\nget_eurostat_data - API version:",get("rav",envir=restatapi::.restatapi_env))} - tbc<-cr<-TRUE # to be continued for the next steps / cache result data.table + tbc<-cr<-TRUE # to be continued for the next steps / cache result data.table if (verbose) {message("get_eurostat_data - footer code option value at start:",paste(getOption("code_opt",NULL),collapse=", "))} options(code_opt=NULL) if (verbose) {message("get_eurostat_data - footer code option value after reset:",paste(getOption("code_opt",NULL),collapse=", "))} if(cflags){keep_flags<-cflags} - + if (!(exists(".restatapi_env"))) {load_cfg()} - - cfg<-get("cfg",envir=restatapi::.restatapi_env) + + cfg<-get("cfg",envir=restatapi::.restatapi_env) rav<-get("rav",envir=restatapi::.restatapi_env) if (!is.null(id)){id<-tolower(trimws(id))} else { tbc<-FALSE @@ -215,7 +218,7 @@ get_eurostat_data <- function(id, message("The dataset 'id' is missing.") } if (verbose) {message("get_eurostat_data - id: ",id)} - + if (check_toc){ toc<-restatapi::get_eurostat_toc(verbose=verbose) if (is.null(toc)){ @@ -233,31 +236,31 @@ get_eurostat_data <- function(id, }else{ udate<-format(Sys.Date(),"%Y.%m.%d") } - + if (tbc){ if (verbose) {message("get_eurostat_data - select_freq: ",select_freq)} - if(!is.null(select_freq)){ + if(!is.null(select_freq)){ if (verbose) {message("get_eurostat_data - not NULL select_freq: ",select_freq)} append_sf<-FALSE if (is.null(filters)|(length(filters)>1)) # no filter or there is already several filters defined => the select_freq is appended to the the filters { if (verbose) {message("get_eurostat_data - select_freq with NULL filters: ",select_freq)} - append_sf<-TRUE + append_sf<-TRUE } else if (!is.null(filters)) #there are filters - { + { if (verbose) {message("get_eurostat_data - select_freq with filters: ",select_freq)} if (length(filters)==1) #there is a single string filter { if (verbose) {message("get_eurostat_data - select_freq with 1 filter: ",select_freq)} if (grepl("\\.",filters,perl=TRUE)) #filter is given as a string for the REST API - { + { if (verbose) {message("get_eurostat_data - select_freq with string filter with '.'at the beginning: ",select_freq)} if (grepl("^\\.",filters,perl=TRUE)) # no FREQ value is given in the filter - { + { filters<-paste0(select_freq,filters) } else{ filters<-paste0(select_freq,"+",filters) - } + } }else{ append_sf<-TRUE } @@ -266,7 +269,7 @@ get_eurostat_data <- function(id, append_sf<-TRUE } } - + if (append_sf){ to_add<-switch(select_freq, A=c("^Annual$","^A$"), @@ -290,17 +293,17 @@ get_eurostat_data <- function(id, } } if ((!is.null(filters))|(!is.null(date_filter))) #there is filter/date_filter defined - { + { if (!is.null(filters))#filter defined => create filter table and filter url - { + { dsd<-get_eurostat_dsd(id,lang=lang,verbose=verbose) if (is.null(dsd)){ message("Could not download the DSD. The filter is ignored") filters_url<-NULL } else { - if(rav==1) {dsdorder<-unique(dsd$concept)[1:(length(unique(dsd$concept))-2)]} - if(rav==2) {dsdorder<-unique(dsd$concept)[1:(length(unique(dsd$concept)))]} - + if(rav==1) {dsdorder<-unique(dsd$concept)[seq_len(length(unique(dsd$concept))-2)]} + if(rav==2) {dsdorder<-unique(dsd$concept)[seq_along(unique(dsd$concept))]} + if (length(gregexpr("\\.",filters,perl=TRUE)[[1]])!=(length(dsdorder)-1) | length(filters)>1){ if (!(is.null(names(filters)))) { if ("time" %in% names(filters)) date_filter<-c(date_filter, unlist(filters["time"])) @@ -309,7 +312,7 @@ get_eurostat_data <- function(id, if (nrow(ft)>0){ ft<-unique(ft[ft$code!=FALSE,2:3]) ft<-ft[order(match(ft$concept, dsdorder)),] - filters_url<-paste0(sapply(dsdorder,gen_ft,ft),collapse=".") + filters_url<-paste0(sapply(dsdorder,gen_ft,ft),collapse=".") } else { filters_url<-NULL } @@ -320,33 +323,33 @@ get_eurostat_data <- function(id, filters_url<-NULL } if (!is.null(date_filter)) #date filter defined => create date filter table and url - { + { if (verbose) {message("get_eurostat_data - date_filter: ",match.call()$date_filter)} dft<-restatapi::create_filter_table(filters=date_filter,date_filter=TRUE,verbose=verbose) if (verbose) {message("get_eurostat_data - date_filter: ",paste(date_filter,collapse=", ")," nrow dft: ",nrow(dft))} if(!is.null(dft)){ if(nrow(dft)>0){ - date_filter<-apply(dft,1,function (x) {paste0("?",if(x[1]!=0){paste0("startPeriod=",x[1])},if(x[1]!=0&x[2]!=Inf){paste0("&endPeriod=",x[2])},if(x[1]==0&x[2]!=Inf){paste0("endPeriod=",x[2])})}) + date_filter<-apply(dft,1,function (x) {paste0("?",if(x[1]!=0){paste0("startPeriod=",x[1])},if(x[1]!=0&x[2]!=Inf){paste0("&endPeriod=",x[2])},if(x[1]==0&x[2]!=Inf){paste0("endPeriod=",x[2])})}) } else { date_filter<-NULL - } + } } else{ date_filter<-NULL } }else{dft<-NULL} if (verbose){message(filters_url,"-",date_filter)} - + # # finished filter creation start data retrieval # - + if (is.null(filters_url)&(is.null(date_filter))) #after parsing there is no valid filter or it is missing => bulk download - { + { message("None of the filter could be applied. The whole dataset will be retrieved through bulk download.") restat<-restatapi::get_eurostat_bulk(id,cache,update_cache,cache_dir,compress_file,stringsAsFactors,select_freq,keep_flags,cflags,check_toc,verbose=verbose) if (!is.null(restat) & (verbose)) {message("get_eurostat_data - bulk restat - nrow:",nrow(restat),";ncol:",ncol(restat),";colnames:",paste(colnames(restat),collapse="/"));message("cflags:",cflags)} } else if (force_local_filter) #there is valid filter but want to filter locally, not using the API and filter url => raw download and filtering - { + { message("Forcing to apply filter locally. The whole dataset will be downloaded through the raw download and then the filters are applied locally.") restat_raw<-restatapi::get_eurostat_raw(id,"txt",cache,update_cache,cache_dir,compress_file,stringsAsFactors,keep_flags,check_toc,melt=TRUE,verbose) if (!is.null(restat_raw) & (verbose)) {message("get_eurostat_data - raw restat - nrow:",nrow(restat_raw),";ncol:",ncol(restat_raw),";colnames:",paste(colnames(restat_raw),collapse="/"))} @@ -363,8 +366,8 @@ get_eurostat_data <- function(id, cr<-FALSE restat<-restat_raw[] if (!is.null(restat) & (verbose)) {message("get_eurostat_data - local filtered restat - nrow:",nrow(restat),";ncol:",ncol(restat),";colnames:",paste(colnames(restat),collapse="/"))} - } else #there is valid filter url => use the REST API with SDMX - { + } else #there is valid filter url => use the REST API with SDMX + { base_url<-eval(parse(text=paste0("cfg$QUERY_BASE_URL$'",rav,"'$ESTAT$data$'2.1'$data"))) if (mode=="xml"){ data_endpoint<-sub("\\/\\/(?=\\?)","/",paste0(base_url,"/",id,"/",filters_url,"/",date_filter),perl=TRUE) @@ -375,10 +378,10 @@ get_eurostat_data <- function(id, } else { data_endpoint<-gsub("\\/\\/(?=\\?)","/",paste0(base_url,"/",id,"/",filters_url,"/",date_filter,"?format=SDMX-CSV&detail=dataonly&compressed=true"),perl=TRUE) } - } else { + } else { message("Incorrect mode:",mode,"\n It should be either 'csv' or 'xml' (default)" ) tbc<-FALSE - } + } if (tbc) { if (mode=="csv"){ restat<-data.table::rbindlist(lapply(data_endpoint, function(x) { @@ -387,23 +390,23 @@ get_eurostat_data <- function(id, if(max(utils::sessionInfo()$otherPkgs$data.table$Version,utils::sessionInfo()$loadedOnly$data.table$Version)>"1.11.7"){ tryCatch({rdat<-data.table::fread(text=readLines(gzcon(url(x))),sep=',',sep2=',',colClasses='character',header=TRUE,stringsAsFactors=stringsAsFactors)}, error = function(e) { - if (verbose){message("get_eurostat_data - Error by the reading in with data.table the downloaded CSV file:",'\n',paste(unlist(e),collapse="\n"))} + if (verbose){message("get_eurostat_data - Error by the reading in with data.table the downloaded CSV file:",'\n',conditionMessage(e))} else {message("There is an error by the reading of the CSV file. Run the same command with verbose=TRUE option to get more info on the issue.")} tbc<-FALSE }, warning = function(w) { - if (verbose){message("get_eurostat_data - Warning by the reading in with data.table the downloaded CSV file:",'\n',paste(unlist(w),collapse="\n"))} + if (verbose){message("get_eurostat_data - Warning by the reading in with data.table the downloaded CSV file:",'\n',conditionMessage(w))} else {message("There is a warning by the reading of the CSV file. Run the same command with verbose=TRUE option to get more info on the issue.")} }) } else{ tryCatch({rdat<-data.table::fread(paste(readLines(gzcon(url(x))),collapse="\n"),sep=',',sep2=',',colClasses='character',header=TRUE,stringsAsFactors=stringsAsFactors)}, error = function(e) { - if (verbose){message("get_eurostat_data - Error by the reading in with data.table the downloaded CSV file:",'\n',paste(unlist(e),collapse="\n"))} + if (verbose){message("get_eurostat_data - Error by the reading in with data.table the downloaded CSV file:",'\n',conditionMessage(e))} else {message("There is an error by the reading of the CSV file. Run the same command with verbose=TRUE option to get more info on the issue.")} tbc<-FALSE }, warning = function(w) { - if (verbose){message("get_eurostat_rdat - Warning by the reading in with data.table the downloaded CSV file:",'\n',paste(unlist(w),collapse="\n"))} + if (verbose){message("get_eurostat_rdat - Warning by the reading in with data.table the downloaded CSV file:",'\n',conditionMessage(w))} else {message("There is a warning by the reading of the CSV file. Run the same command with verbose=TRUE option to get more info on the issue.")} }) } @@ -414,9 +417,9 @@ get_eurostat_data <- function(id, if (any(grepl(paste0(id, ".* does not exist"),rdat))){ message("The file ",gsub(".*/","",x)," does not exist or is not readable on the server. Try to download with the check_toc=TRUE option.") tbc<-FALSE - } + } } - } + } if (!is.null(rdat)){ data.table::as.data.table(rdat,stringsAsFactors=stringsAsFactors) rdat[, c("DATAFLOW", "LAST UPDATE") := NULL] @@ -429,12 +432,12 @@ get_eurostat_data <- function(id, temp <- tempfile() tryCatch({utils::download.file(x,temp,dmethod,quiet=!verbose)}, error = function(e) { - if (verbose) {message("get_eurostat_data - Error by the download the xml file:",'\n',paste(unlist(e),collapse="\n"))} + if (verbose) {message("get_eurostat_data - Error by the download the xml file:",'\n',conditionMessage(e))} else {message("There is an error by the download of the XML file. Run the same command with verbose=TRUE option to get more info on the issue.")} tbc<-FALSE }, warning = function(w) { - if(verbose){message("get_eurostat_data - Warning by the download the xml file:",'\n',paste(unlist(w),collapse="\n"))} + if(verbose){message("get_eurostat_data - Warning by the download the xml file:",'\n',conditionMessage(w))} else {message("There is a warning by the download of the XML file. Run the same command with verbose=TRUE option to get more info on the issue.")} tbc<-FALSE }) @@ -443,16 +446,16 @@ get_eurostat_data <- function(id, xml_foot<-NULL tryCatch({xml_foot<-xml2::xml_find_all(xml2::read_xml(temp),".//footer:Message")}, error = function(e) { - if (verbose) {message("get_eurostat_data - Error by the extraction of the footer from the xml:",'\n',paste(unlist(e),collapse="\n"))} + if (verbose) {message("get_eurostat_data - Error by the extraction of the footer from the xml:",'\n',conditionMessage(e))} else {message("There is an error by the reading of the downloaded XML file. Run the same command with verbose=TRUE option to get more info on the issue.")} tbc<-FALSE }, warning = function(w) { - if(verbose){message("get_eurostat_data - Warning by the extraction of the footer from the xml:",'\n',paste(unlist(w),collapse="\n"))} + if(verbose){message("get_eurostat_data - Warning by the extraction of the footer from the xml:",'\n',conditionMessage(w))} else {message("There is a warning by the reading of the downloaded XML file. Run the same command with verbose=TRUE option to get more info on the issue.")} tbc<-FALSE }) - } + } if (tbc & !is.null(xml_foot)){ if (length(xml_foot)>0){ code<-xml2::xml_attr(xml_foot,"code") @@ -473,10 +476,10 @@ get_eurostat_data <- function(id, tbc<-FALSE tryCatch({xml_fault<-xml2::xml_find_all(xml2::read_xml(temp),".//S:Fault")}, error = function(e) { - if (verbose) {message("get_eurostat_data - Error by the extraction of the faultcode from the xml:",'\n',paste(unlist(e),collapse="\n"))} + if (verbose) {message("get_eurostat_data - Error by the extraction of the faultcode from the xml:",'\n',conditionMessage(e))} }, warning = function(w) { - if(verbose){message("get_eurostat_data - Warning by the extraction of the faultcode from the xml:",'\n',paste(unlist(w),collapse="\n"))} + if(verbose){message("get_eurostat_data - Warning by the extraction of the faultcode from the xml:",'\n',conditionMessage(w))} }) message("Problem by the extraction of the footer information from the xml_file.") } @@ -484,15 +487,15 @@ get_eurostat_data <- function(id, xml_mark<-switch(rav,"1" = ".//generic:Series","2" = ".//g:Series") tryCatch({xml_leafs<-xml2::xml_find_all(xml2::read_xml(temp),xml_mark)}, error = function(e) { - if (verbose) {message("get_eurostat_data - Error by the reading of data from the downloaded XML:",'\n',paste(unlist(e),collapse="\n"))} + if (verbose) {message("get_eurostat_data - Error by the reading of data from the downloaded XML:",'\n',conditionMessage(e))} else {message("There is an error by the reading of the downloaded XML file. Run the same command with verbose=TRUE option to get more info on the issue.")} }, - warning = function(w) { - if(verbose){message("get_eurostat_data - Warning by reading of data from the downloaded XML file:",'\n',paste(unlist(w),collapse="\n"))} + warning = function(w) { + if(verbose){message("get_eurostat_data - Warning by reading of data from the downloaded XML file:",'\n',conditionMessage(w))} else {message("There is a warning by the downloaded of the XML file. Run the same command with verbose=TRUE option to get more info on the issue.")} } ) - tryCatch({ + tryCatch({ if (verbose) {message("get_eurostat_data - class(xml_leafs):",class(xml_leafs), "\nget_eurostat_data - number of nodes: ",length(xml_leafs), "\nget_eurostat_data - number of cores: ",getOption("restatapi_cores",1L))} @@ -505,11 +508,11 @@ get_eurostat_data <- function(id, cl<-parallel::makeCluster(min(2,getOption("restatapi_cores",1L))) parallel::clusterEvalQ(cl,require(xml2)) parallel::clusterExport(cl,c("extract_data")) - rdat<-data.table::rbindlist(parallel::parLapply(cl,xml_leafs,extract_data,keep_flags=keep_flags,stringsAsFactors=stringsAsFactors,bulk=FALSE)) + rdat<-data.table::rbindlist(parallel::parLapply(cl,xml_leafs,extract_data,keep_flags=keep_flags,stringsAsFactors=stringsAsFactors,bulk=FALSE)) parallel::stopCluster(cl) - } + } }else{ - rdat<-data.table::rbindlist(parallel::mclapply(xml_leafs,extract_data,keep_flags=keep_flags,stringsAsFactors=stringsAsFactors,bulk=FALSE,mc.cores=getOption("restatapi_cores",1L))) + rdat<-data.table::rbindlist(parallel::mclapply(xml_leafs,extract_data,keep_flags=keep_flags,stringsAsFactors=stringsAsFactors,bulk=FALSE,mc.cores=getOption("restatapi_cores",1L))) } }, error = function(e){if (verbose){message("get_eurostat_data - ",e);rdat<-NULL}}, @@ -525,16 +528,16 @@ get_eurostat_data <- function(id, if (verbose) {message("get_eurostat_data - footer code option value after retrieval:",paste(getOption("code_opt",NULL),collapse=", "))} if (!is.null(restat)) #at least one url provided a valid result => check if all the queries with data downloaded { - if ((nrow(restat)==0)) # if there is no data in the results - { + if ((nrow(restat)==0)) # if there is no data in the results + { if (all(getOption("code_opt",NULL)==404)) # no data for all the filters => stop processing by restat<-NULL - { + { message("404 - No data retrived with the given filter(s)") restat<-NULL } else if (any(getOption("code_opt",NULL)==413)) #there is some data but was not downloaded - { + { if (local_filter) #apply filter locally - { + { message("No data retrieved for the given filter(s), because the results are too big to download immediately through the REST API. The whole dataset is downloaded through the raw download and the filters are applied locally.") if (verbose) {message("get_eurostat_data - cache:",cache," update cache:",update_cache," cache dir:",cache_dir)} restat_raw<-restatapi::get_eurostat_raw(id,"txt",cache,update_cache,cache_dir,compress_file,stringsAsFactors,keep_flags,check_toc,melt=TRUE,verbose) @@ -549,8 +552,8 @@ get_eurostat_data <- function(id, } restat<-restat_raw if (!is.null(restat) & (verbose)) {message("get_eurostat_data - local filtered restat - nrow:",nrow(restat),";ncol:",ncol(restat),";colnames:",paste(colnames(restat),collapse="/"))} - } else #inform that there is data but could not be downloaded immediately - { + } else #inform that there is data but could not be downloaded immediately + { message("No data retrieved for the given filter(s), because the results are too big to download immediately through the REST API. You may want to download the whole dataset and apply the filter(s) locally.") } } else if (any(getOption("code_opt",NULL)>=500)) #if there is some data but for some of the filters there is a warning that there is "internal application error" or "exception while getting all data and footnotes slice" or "Cannot connect to Comext service." @@ -559,11 +562,11 @@ get_eurostat_data <- function(id, if (!verbose) {notification<-paste(notification,"You can check the details rerunning the request with the 'verbose=TRUE' parameter.")} message(notification) restat<-NULL - } + } } else if (any(getOption("code_opt",NULL)==413)) #if there is some data but for some of the filters there is a warning that could not be downloaded immediately - { + { if (local_filter) #apply filter locally and replace the data from the REST API - { + { message("One or some of the filter(s) resulted too large datatset to download through the REST API. The whole dataset is downloaded through the raw download and the filters are applied locally.") restat_raw<-restatapi::get_eurostat_raw(id,"txt",cache,update_cache,cache_dir,compress_file,stringsAsFactors,keep_flags,check_toc,melt=TRUE,verbose) if (!is.null(restat_raw) & (verbose)) {message("get_eurostat_data - raw restat - nrow:",nrow(restat_raw),";ncol:",ncol(restat_raw),";colnames:",paste(colnames(restat_raw),collapse="/"))} @@ -573,8 +576,8 @@ get_eurostat_data <- function(id, if (nrow(dft)>0){restat_raw<-restatapi::filter_raw_data(restat_raw,dft,TRUE)} restat<-restat_raw if (!is.null(restat) & (verbose)) {message("get_eurostat_data - local filtered restat - nrow:",nrow(restat),";ncol:",ncol(restat),";colnames:",paste(colnames(restat),collapse="/"))} - } else #inform that there is more data but could not be downloaded immediately - { + } else #inform that there is more data but could not be downloaded immediately + { if (nrow(restat)>0){message("The retrived dataset is partial!!!")} message("One or some of the filter(s) resulted too large datatset to download through the REST API. You may want to download the whole dataset and apply the filter(s) locally.") } @@ -584,7 +587,7 @@ get_eurostat_data <- function(id, if (!verbose) {notification<-paste(notification,"You can check the details rerunning the request with the 'verbose=TRUE' parameter.")} message(notification) restat<-NULL - } + } cr<-FALSE # do not cache filtered data only bulk datasets } } @@ -609,6 +612,7 @@ get_eurostat_data <- function(id, if (nrow(restat)>0) #there is data, set column names and format data and flags { data.table::setnames(restat,colnames(restat),tolower(colnames(restat))) + if ("freq" %in% colnames(restat)){ if (length(unique(restat$freq))==1){restat[,"freq":=NULL]} else{ if (cr){ @@ -618,19 +622,19 @@ get_eurostat_data <- function(id, if (stringsAsFactors){select_freq<-as.character(levels(st$freq)[st$freq[1]])}else{as.character(st$freq)} warning("There are multiple frequencies in the dataset. The '", select_freq, "' is selected as it is the most common frequency.") } - } + } restat<-restat[restat$freq==select_freq] restat[,"freq":=NULL] } } } - if ("time_format" %in% colnames(restat)) {drop<-c(drop,"time_format")} + if ("time_format" %in% colnames(restat)) {drop<-c(drop,"time_format")} if ("obstime" %in% colnames(restat)){data.table::setnames(restat,c("obstime","obsvalue"),c("time","values"))} if ("time_period" %in% colnames(restat)) {data.table::setnames(restat,"time_period","time")} if ("obs_value" %in% colnames(restat)) {data.table::setnames(restat,"obs_value","values")} restat$time<-gsub('[MD]',"-",restat$time) restat$time<-gsub('([0-9]{4})Q',"\\1-Q",restat$time,perl=TRUE) - + if (keep_flags){ if ("obs_status" %in% colnames(restat)){ restat$obs_status<-as.character(restat$obs_status) @@ -648,25 +652,20 @@ get_eurostat_data <- function(id, } } else if ("obs_flag" %in% colnames(restat)) { data.table::setnames(restat,"obs_flag","flags") - } + } + if ("conf_status" %in% colnames(restat)){ + conf_val<-as.character(restat$conf_status) + restat$flags<-paste0(restat$flags,restat$conf_status) + drop<-c(drop,"conf_status") + } + restat$flags<-gsub("@|NA","",restat$flags) } else { if ("obs_status" %in% colnames(restat)){drop<-c(drop,"obs_status")} if ("obs_flag" %in% colnames(restat)){drop<-c(drop,"obs_flag")} + if ("conf_status" %in% colnames(restat)){drop<-c(drop,"conf_status")} } if(!is.null(drop)) {restat[,(drop):=NULL]} - if ("flags" %in% colnames(restat)){ - restat<-restat[!(is.na(restat$values)&(is.na(restat$flags)|restat$flags==""))] - if (keep_flags) { - restat$flags<-as.character(restat$flags) - restat[is.na(flags),flags:=""] - if (!cflags) {restat<-restat[restat$flags!="c"]} - } else{ - restat[,"flags":=NULL] - } - } else { - restat<-restat[!(is.na(restat$values))] - } - restat<-data.table::data.table(restat,key=names(restat),stringsAsFactors=stringsAsFactors) + if (is.factor(restat$values)) {restat[,values:=as.character(values)]} if (any(sapply(restat,is.factor))&(!stringsAsFactors)) { col_conv<-colnames(restat)[!(colnames(restat) %in% c("values"))] @@ -680,7 +679,7 @@ get_eurostat_data <- function(id, restat$values<-as.character(levels(restat$values))[restat$values] restat$values[grepl('^\\:$',restat$values)]<-NA } else { - restat$values<-suppressWarnings(as.numeric(levels(restat$values))[restat$values]) + restat$values<-suppressWarnings(as.numeric(levels(restat$values))[restat$values]) } } if (any(is.character(restat$values))){ @@ -689,11 +688,27 @@ get_eurostat_data <- function(id, restat$values<-suppressWarnings(as.numeric(restat$values)) } } + + if ("flags" %in% colnames(restat)){ + restat<-restat[!(is.na(restat$values)&(is.na(restat$flags)|restat$flags==""))] + if (keep_flags) { + restat$flags<-as.character(restat$flags) + restat[is.na(flags),flags:=""] + if (!cflags) {restat<-restat[!grepl("C",restat$flags)]} + } else{ + restat[,"flags":=NULL] + } + } else { + restat<-restat[!(is.na(restat$values))] + } + restat<-data.table::data.table(restat,key=names(restat),stringsAsFactors=stringsAsFactors) restat<-unique(restat)[] - # dsd<-get_eurostat_dsd(id,verbose=verbose) - # dsdorder<-tolower(unique(dsd$concept)[1:length(unique(dsd$concept))]) - # co<-dsdorder[dsdorder %in% colnames(restat)] - # data.table::setcolorder(restat,co) + if (dsd_order){ + dsd<-get_eurostat_dsd(id,verbose=verbose) + dsdorder<-tolower(unique(dsd$concept)[1:length(unique(dsd$concept))]) + co<-dsdorder[dsdorder %in% colnames(restat)] + data.table::setcolorder(restat,co) + } } } if (verbose) {message("get_eurostat_data - processed restat - nrow:",nrow(restat),"; ncol:",ncol(restat),"; colnames:",paste(colnames(restat),collapse="/"))} @@ -710,7 +725,7 @@ get_eurostat_data <- function(id, if (!is.null(dsd)){ if (verbose) {message("get_eurostat_data - dsd - nrow:",nrow(dsd),";ncol:",ncol(dsd))} cn<-colnames(restat)[!(colnames(restat) %in% c("time","values","flags"))] - restat<-data.table::data.table(restat,stringsAsFactors=TRUE) + restat<-data.table::data.table(restat,stringsAsFactors=TRUE) if (verbose) {message("get_eurostat_data - restat - nrow:",nrow(restat),";ncol:",ncol(restat),";colnames:",paste(cn,collapse="/"))} sub_dsd<-dsd[dsd$code %in% as.character(levels(unique(unlist(as.list(restat[,(cn),with=FALSE]))))),] sub_dsd<-data.table::setorder(sub_dsd,concept,code) @@ -721,11 +736,11 @@ get_eurostat_data <- function(id, col_conv<-colnames(restat)[!(colnames(restat) %in% c("values"))] if (is.factor(restat$values)) {col_conv<-c(col_conv,"values")} restat[,col_conv]<-restat[,lapply(.SD,as.character),.SDcols=col_conv] - } + } } else { message("Could not download the DSD. No label is applied.") } - } + } } if (getOption("restatapi_verbose",FALSE)) {message("get_eurostat_data - restat NULL?:",is.null(restat),"\n\t id: ",id,"\n\t filters: ",filters_url,"\n\t date filter: ",date_filter)} return(restat) diff --git a/R/get_eurostat_dsd.R b/R/get_eurostat_dsd.R index 478cb15..ca6c16c 100755 --- a/R/get_eurostat_dsd.R +++ b/R/get_eurostat_dsd.R @@ -82,11 +82,11 @@ get_eurostat_dsd <- function(id, message("\nget_eurostat_dsd - Trying to download the DSD from: ",dsd_endpoint) tryCatch({utils::download.file(dsd_endpoint,temp,dmethod)}, error = function(e) { - message("get_eurostat_dsd - Error by the download of the DSD file:",'\n',paste(unlist(e),collapse="\n")) + message("get_eurostat_dsd - Error by the download of the DSD file:",'\n',conditionMessage(e)) dsd_xml<-NULL }, warning = function(w) { - message("get_eurostat_dsd - Warning by the download of the DSD file:",'\n',paste(unlist(w),collapse="\n")) + message("get_eurostat_dsd - Warning by the download of the DSD file:",'\n',conditionMessage(w)) tbc<-FALSE dsd_xml<-NULL }) @@ -96,11 +96,11 @@ get_eurostat_dsd <- function(id, message("Trying to extract the DSD from: ",temp) tryCatch({dsd_xml<-xml2::read_xml(temp)}, error = function(e) { - message("get_eurostat_dsd - Error during the extraction of the XML from the downloaded DSD file:",'\n',paste(unlist(e),collapse="\n")) + message("get_eurostat_dsd - Error during the extraction of the XML from the downloaded DSD file:",'\n',conditionMessage(e)) dsd_xml<-NULL }, warning = function(w) { - message("get_eurostat_dsd - There is warning by the extraction of the XML from the downloaded DSD file:",'\n',paste(unlist(w),collapse="\n")) + message("get_eurostat_dsd - There is warning by the extraction of the XML from the downloaded DSD file:",'\n',conditionMessage(w)) dsd_xml<-NULL }) } else { @@ -167,11 +167,11 @@ get_eurostat_dsd <- function(id, message("get_eurostat_dsd - Trying to download the CC from: ",cc_endpoint) tryCatch({utils::download.file(cc_endpoint,temp,dmethod)}, error = function(e) { - message("get_eurostat_dsd - Error by the download of the CC file:",'\n',paste(unlist(e),collapse="\n")) + message("get_eurostat_dsd - Error by the download of the CC file:",'\n',conditionMessage(e)) cc_xml<-NULL }, warning = function(w) { - message("get_eurostat_dsd - Warning by the download of the CC file:",'\n',paste(unlist(w),collapse="\n")) + message("get_eurostat_dsd - Warning by the download of the CC file:",'\n',conditionMessage(w)) tbc<-FALSE cc_xml<-NULL }) @@ -180,11 +180,11 @@ get_eurostat_dsd <- function(id, message("get_eurostat_dsd - Trying to extract the CC from: ",temp) tryCatch({cc_xml<-xml2::read_xml(temp)}, error = function(e) { - message("get_eurostat_dsd - Error during the extraction of the XML from the downloaded CC file:",'\n',paste(unlist(e),collapse="\n")) + message("get_eurostat_dsd - Error during the extraction of the XML from the downloaded CC file:",'\n',conditionMessage(e)) cc_xml<-NULL }, warning = function(w) { - message("get_eurostat_dsd - There is warning by the extraction of the XML from the downloaded CC file:",'\n',paste(unlist(w),collapse="\n")) + message("get_eurostat_dsd - There is warning by the extraction of the XML from the downloaded CC file:",'\n',conditionMessage(w)) cc_xml<-NULL }) } else { diff --git a/R/get_eurostat_raw.R b/R/get_eurostat_raw.R index eaa3f83..0144767 100755 --- a/R/get_eurostat_raw.R +++ b/R/get_eurostat_raw.R @@ -1,14 +1,14 @@ -#' @title Get Eurostat data as it is +#' @title Get Eurostat data as it is #' @description Download data sets from \href{https://ec.europa.eu/eurostat}{Eurostat} database . #' @param id A code name for the dataset of interest. #' See \code{\link{search_eurostat_toc}} for details how to get an id. -#' @param mode defines the format of the downloaded dataset. It can be \code{txt} (the default value) for -#' Tab Separated Values (TSV), or \code{csv} for SDMX-CSV, or \code{xml} for the SDMX-ML version. +#' @param mode defines the format of the downloaded dataset. It can be \code{txt} (the default value) for +#' Tab Separated Values (TSV), or \code{csv} for SDMX-CSV, or \code{xml} for the SDMX-ML version. #' @param cache a logical whether to do caching. Default is \code{TRUE}. #' @param update_cache a logical with a default value \code{FALSE}, whether to update cache. Can be set also with #' \code{options(restatapi_update=TRUE)} -#' @param cache_dir a path to a cache directory. The \code{NULL} (default) uses the memory as cache. -#' If the folder if the \code{cache_dir} directory does not exist it saves in the 'restatapi' directory +#' @param cache_dir a path to a cache directory. The \code{NULL} (default) uses the memory as cache. +#' If the folder if the \code{cache_dir} directory does not exist it saves in the 'restatapi' directory #' under the temporary directory from \code{tempdir()}. Directory can also be set with #' \code{option(restatapi_cache_dir=...)}. #' @param compress_file a logical whether to compress the @@ -17,29 +17,29 @@ #' converted to factors. The default value \code{FALSE}, in this case they are returned as characters. #' @param keep_flags a logical whether the observation status (flags) - e.g. "confidential", #' "provisional", etc. - should be kept in a separate column or if they -#' can be removed. Default is \code{FALSE}. For flag values see: +#' can be removed. Default is \code{FALSE}. For flag values see: #' \url{https://ec.europa.eu/eurostat/api/dissemination/sdmx/2.1/codelist/ESTAT/OBS_STATUS/?compressed=false&format=TSV&lang=en}. -#' @param check_toc a boolean whether to check the provided \code{id} in the Table of Contents (TOC) or not. The default value -#' \code{FALSE}, in this case the base URL for the download link is retrieved from the configuration file. -#' If the value is \code{TRUE} then the TOC is downloaded and the \code{id} is checked in it. If it found then the download link -#' is retrieved form the TOC. -#' @param melt a boolean with default value \code{TRUE} and used only if the \code{mode="txt"}. In case it is \code{FALSE}, -#' the downloaded tsv file is not melted, the time dimension remains in columns and it does not process the flags. +#' @param check_toc a boolean whether to check the provided \code{id} in the Table of Contents (TOC) or not. The default value +#' \code{FALSE}, in this case the base URL for the download link is retrieved from the configuration file. +#' If the value is \code{TRUE} then the TOC is downloaded and the \code{id} is checked in it. If it found then the download link +#' is retrieved form the TOC. +#' @param melt a boolean with default value \code{TRUE} and used only if the \code{mode="txt"}. In case it is \code{FALSE}, +#' the downloaded tsv file is not melted, the time dimension remains in columns and it does not process the flags. #' @param verbose A boolean with default \code{FALSE}, so detailed messages (for debugging) will not printed. -#' Can be set also with \code{options(restatapi_verbose=TRUE)} +#' Can be set also with \code{options(restatapi_verbose=TRUE)} #' @param ... further argument for the \code{\link{load_cfg}} function #' @export -#' -#' @details Data sets are downloaded from \href{https://ec.europa.eu/eurostat/web/user-guides/data-browser/api-data-access/api-migrating/bulkdownload}{the Eurostat bulk download facility} +#' +#' @details Data sets are downloaded from \href{https://ec.europa.eu/eurostat/web/user-guides/data-browser/api-data-access/api-migrating/bulkdownload}{the Eurostat bulk download facility} #' in CSV, TSV or SDMX format. -#' -#' +#' +#' #' The \code{id}, should be a value from the \code{code} column of the table of contents (\code{\link{get_eurostat_toc}}), and can be searched for with the \code{\link{search_eurostat_toc}} function. The id value can be retrieved from the \href{https://ec.europa.eu/eurostat/data/database}{Eurostat database} #' as well. The Eurostat database gives codes in the Data Navigation Tree after every dataset in parenthesis. -#' By default all datasets downloaded in TSV format and cached as they are often rather large. +#' By default all datasets downloaded in TSV format and cached as they are often rather large. #' The datasets cached in memory (default) or can be stored in a temporary directory if \code{cache_dir} or \code{option(restatpi_cache_dir)} is defined. #' The cache can be emptied with \code{\link{clean_restatapi_cache}}. -#' If the \code{id} is checked in TOC then the data will saved in the cache with the date from the "lastUpdate" column from the TOC, otherwise it is saved with the current date. +#' If the \code{id} is checked in TOC then the data will saved in the cache with the date from the "lastUpdate" column from the TOC, otherwise it is saved with the current date. #' @return a data.table with the following columns if the default \code{melt=TRUE} is used: #' \tabular{ll}{ #' \code{FREQ} \tab The frequency of the data (\strong{A}nnual, \strong{S}emi-annual, \strong{H}alf-year, \strong{Q}uarterly, \strong{M}onthly, @@ -55,18 +55,18 @@ #' in the data table, and the name of the column depends on the source file (TSV/SDMX-ML) #' } #' The data does not include all missing values. The missing values are dropped if the value and flags are missing -#' on a particular time. -#' -#' In case \code{melt=FALSE} the results is a data.table where the first column contains the comma separated values of the various dimensions, and the columns contains the observations for each time dimension. -#' +#' on a particular time. +#' +#' In case \code{melt=FALSE} the results is a data.table where the first column contains the comma separated values of the various dimensions, and the columns contains the observations for each time dimension. +#' #' @seealso \code{\link{get_eurostat_data}}, \code{\link{get_eurostat_bulk}} -#' @examples +#' @examples #' \dontshow{ #' if (parallel::detectCores()<=2){ #' options(restatapi_cores=1) #' }else{ #' options(restatapi_cores=2) -#' } +#' } #' } #' \donttest{ #' if (!(grepl("amzn|-aws|-azure ",Sys.info()['release']))) options(timeout=2) @@ -82,9 +82,9 @@ #' options(timeout=60) #' } -get_eurostat_raw <- function(id, +get_eurostat_raw <- function(id, mode="txt", - cache=TRUE, + cache=TRUE, update_cache=FALSE, cache_dir=NULL, compress_file=TRUE, @@ -93,26 +93,26 @@ get_eurostat_raw <- function(id, check_toc=FALSE, melt=TRUE, verbose=FALSE,...){ - + restat_raw<-raw<-NULL verbose<-verbose|getOption("restatapi_verbose",FALSE) update_cache<-update_cache|getOption("restatapi_update", FALSE) dmethod<-getOption("restatapi_dmethod",get("dmethod",envir=restatapi::.restatapi_env)) if (getOption("restatapi_cores",1L)>=parallel::detectCores()) options(restatapi_cores=parallel::detectCores()-1) - tbc<-TRUE #to be continued to the next steps + tbc<-TRUE #to be continued to the next steps if (verbose) {message("\nget_eurostat_raw - API version:",get("rav",envir=restatapi::.restatapi_env))} if((!exists(".restatapi_env")|(length(list(...))>0))){ if ((length(list(...))>0)) { if (all(names(list(...)) %in% c("api_version","load_toc","parallel","max_cores","verbose"))){ - load_cfg(...) + load_cfg(...) } else { load_cfg() } } else { load_cfg() - } + } } - cfg<-get("cfg",envir=restatapi::.restatapi_env) + cfg<-get("cfg",envir=restatapi::.restatapi_env) rav<-get("rav",envir=restatapi::.restatapi_env) # if (verbose) {message("get_eurostat_raw - API version:",rav)} if (!is.null(id)){id<-tolower(trimws(id))} else { @@ -125,7 +125,7 @@ get_eurostat_raw <- function(id, message("It is not possible to have raw casted table with mode='csv' or mode='xml'.") } } - + if (tbc){ if (check_toc){ toc<-restatapi::get_eurostat_toc(verbose=verbose) @@ -187,9 +187,9 @@ get_eurostat_raw <- function(id, } } } - - - + + + if (tbc){ if ((cache)&(!update_cache)) { restat_raw<-data.table::copy(restatapi::get_eurostat_cache(paste0("r_",id,"-",udate,"-",sum(keep_flags)),cache_dir,verbose=verbose)) @@ -199,23 +199,23 @@ get_eurostat_raw <- function(id, if(max(utils::sessionInfo()$otherPkgs$data.table$Version,utils::sessionInfo()$loadedOnly$data.table$Version)>"1.11.7"){ tryCatch({raw<-data.table::fread(text=readLines(gzcon(url(bulk_url))),sep=',',sep2=',',colClasses='character',header=TRUE,stringsAsFactors=stringsAsFactors)}, error = function(e) { - if (verbose){message("get_eurostat_raw - Error by the reading in with data.table the downloaded CSV file:",'\n',paste(unlist(e),collapse="\n"))} + if (verbose){message("get_eurostat_raw - Error by the reading in with data.table the downloaded CSV file:",'\n',conditionMessage(e))} else {message("There is an error by the download of the CSV file. Run the same command with verbose=TRUE option to get more info on the issue.")} tbc<-FALSE }, warning = function(w) { - if (verbose){message("get_eurostat_raw - Warning by the reading in with data.table the downloaded CSV file:",'\n',paste(unlist(w),collapse="\n"))} + if (verbose){message("get_eurostat_raw - Warning by the reading in with data.table the downloaded CSV file:",'\n',conditionMessage(w))} else {message("There is a warning by the download of the CSV file. Run the same command with verbose=TRUE option to get more info on the issue.")} }) } else{ tryCatch({raw<-data.table::fread(paste(readLines(gzcon(url(bulk_url))),collapse="\n"),sep=',',sep2=',',colClasses='character',header=TRUE,stringsAsFactors=stringsAsFactors)}, error = function(e) { - if (verbose){message("get_eurostat_raw - Error by the reading in with data.table the downloaded CSV file:",'\n',paste(unlist(e),collapse="\n"))} + if (verbose){message("get_eurostat_raw - Error by the reading in with data.table the downloaded CSV file:",'\n',conditionMessage(e))} else {message("There is an error by the download of the CSV file. Run the same command with verbose=TRUE option to get more info on the issue.")} tbc<-FALSE }, warning = function(w) { - if (verbose){message("get_eurostat_raw - Warning by the reading in with data.table the downloaded CSV file:",'\n',paste(unlist(w),collapse="\n"))} + if (verbose){message("get_eurostat_raw - Warning by the reading in with data.table the downloaded CSV file:",'\n',conditionMessage(w))} else {message("There is a warning by the download of the CSV file. Run the same command with verbose=TRUE option to get more info on the issue.")} }) } @@ -226,9 +226,9 @@ get_eurostat_raw <- function(id, if (any(grepl(paste0(id, ".* does not exist"),raw))){ message("The file ",gsub(".*/","",bulk_url)," does not exist or is not readable on the server. Try to download with the check_toc=TRUE option.") tbc<-FALSE - } + } } - } + } if (tbc){ restat_raw<-data.table::copy(raw) restat_raw[, c("DATAFLOW", "LAST UPDATE") := NULL] @@ -239,23 +239,23 @@ get_eurostat_raw <- function(id, if(max(utils::sessionInfo()$otherPkgs$data.table$Version,utils::sessionInfo()$loadedOnly$data.table$Version)>"1.11.7"){ tryCatch({raw<-data.table::fread(text=readLines(gzcon(url(bulk_url))),sep='\t',sep2=',',colClasses='character',header=TRUE,stringsAsFactors=stringsAsFactors)}, error = function(e) { - if (verbose){message("get_eurostat_raw - Error by the reading in with data.table the downloaded TSV file:",'\n',paste(unlist(e),collapse="\n"))} + if (verbose){message("get_eurostat_raw - Error by the reading in with data.table the downloaded TSV file:",'\n',conditionMessage(e))} else {message("There is an error by the download of the TSV file. Run the same command with verbose=TRUE option to get more info on the issue.")} tbc<-FALSE }, warning = function(w) { - if (verbose){message("get_eurostat_raw - Warning by the reading in with data.table the downloaded TSV file:",'\n',paste(unlist(w),collapse="\n"))} + if (verbose){message("get_eurostat_raw - Warning by the reading in with data.table the downloaded TSV file:",'\n',conditionMessage(w))} else {message("There is a warning by the download of the TSV file. Run the same command with verbose=TRUE option to get more info on the issue.")} }) } else{ tryCatch({raw<-data.table::fread(paste(readLines(gzcon(url(bulk_url))),collapse="\n"),sep='\t',sep2=',',colClasses='character',header=TRUE,stringsAsFactors=stringsAsFactors)}, error = function(e) { - if (verbose){message("get_eurostat_raw - Error by the reading in with data.table the downloaded TSV file:",'\n',paste(unlist(e),collapse="\n"))} + if (verbose){message("get_eurostat_raw - Error by the reading in with data.table the downloaded TSV file:",'\n',conditionMessage(e))} else {message("There is an error by the download of the TSV file. Run the same command with verbose=TRUE option to get more info on the issue.")} tbc<-FALSE }, warning = function(w) { - if (verbose){message("get_eurostat_raw - Warning by the reading in with data.table the downloaded TSV file:",'\n',paste(unlist(w),collapse="\n"))} + if (verbose){message("get_eurostat_raw - Warning by the reading in with data.table the downloaded TSV file:",'\n',conditionMessage(w))} else {message("There is a warning by the download of the TSV file. Run the same command with verbose=TRUE option to get more info on the issue.")} }) } @@ -267,12 +267,12 @@ get_eurostat_raw <- function(id, message("The file ",gsub(".*/","",bulk_url)," does not exist or is not readable on the server. Try to download with the check_toc=TRUE option.") tbc<-FALSE } - } + } } if (tbc) { if(melt) { - cname<-colnames(raw)[1] + cname<-colnames(raw)[1] if (is.character(cname)){ cnames<-utils::head(unlist(strsplit(cname,(',|\\\\'))),-1) rname<-switch(rav, "1" = utils::tail(unlist(strsplit(cname,(',|\\\\'))),1),"2"="time") @@ -283,20 +283,20 @@ get_eurostat_raw <- function(id, data.table::setnames(raw_melted,2:3,c(rname,"values")) raw_melted<-raw_melted[raw_melted$values!=":",] restat_raw<-data.table::as.data.table(data.table::tstrsplit(raw_melted$bdown,",",fixed=TRUE),stringsAsFactors=stringsAsFactors) - data.table::setnames(restat_raw,cnames) + data.table::setnames(restat_raw,cnames) restat_raw<-data.table::data.table(restat_raw,raw_melted[,2:3],stringsAsFactors=stringsAsFactors) if (keep_flags) {restat_raw$flags<-gsub('[0-9\\.\\-\\s\\:]',"",restat_raw$values,perl=TRUE)} restat_raw$values<-gsub('^\\:$',"",restat_raw$values,perl=TRUE) restat_raw$values<-gsub('[^0-9\\.\\-\\:]',"",restat_raw$values,perl=TRUE) - restat_raw<-data.table::data.table(restat_raw,stringsAsFactors=stringsAsFactors) + restat_raw<-data.table::data.table(restat_raw,stringsAsFactors=stringsAsFactors) } else { message("The file download was not successful. Try again later.") - } + } } else { restat_raw<-raw cache<-update_cache<-FALSE } - } + } # } # } } else if (mode=="xml"){ @@ -318,11 +318,11 @@ get_eurostat_raw <- function(id, cl<-parallel::makeCluster(getOption("restatapi_cores",1L)) parallel::clusterEvalQ(cl,require(xml2)) parallel::clusterExport(cl,c("extract_data")) - restat_raw<-data.table::rbindlist(parallel::parLapply(cl,xml_leafs,extract_data,keep_flags=keep_flags,stringsAsFactors=stringsAsFactors)) - parallel::stopCluster(cl) + restat_raw<-data.table::rbindlist(parallel::parLapply(cl,xml_leafs,extract_data,keep_flags=keep_flags,stringsAsFactors=stringsAsFactors)) + parallel::stopCluster(cl) } }else{ - restat_raw<-data.table::rbindlist(parallel::mclapply(xml_leafs,extract_data,keep_flags=keep_flags,stringsAsFactors=stringsAsFactors,mc.cores=getOption("restatapi_cores",1L))) + restat_raw<-data.table::rbindlist(parallel::mclapply(xml_leafs,extract_data,keep_flags=keep_flags,stringsAsFactors=stringsAsFactors,mc.cores=getOption("restatapi_cores",1L))) } } else{ message("Could not download the SDMX file, use the verbose option to see the exact cause of the error.") @@ -340,12 +340,18 @@ get_eurostat_raw <- function(id, } if ((!keep_flags) & ("OBS_STATUS" %in% colnames(restat_raw))) {restat_raw$OBS_STATUS<-NULL} } - if (verbose) {message("get_eurostat_raw - caching in raw: ",all(!grepl("get_eurostat_bulk|get_eurostat_data",as.character(sys.calls()),perl=TRUE)), + call_names<-vapply(sys.calls(),function(cc) paste(deparse(cc[[1]]),collapse=""),character(1)) + this_call<-sys.call() + this_call_str<-paste0(vapply(as.list(this_call), + function(a) if (is.function(a)) "" else paste0(as.character(a),collapse=""), + character(1)), + collapse="") + if (verbose) {message("get_eurostat_raw - caching in raw: ",all(!grepl("get_eurostat_bulk|get_eurostat_data",call_names,perl=TRUE)), "\nget_eurostat_raw - local filter: ",exists("local_filter",envir=sys.parent(1)), - "\nget_eurostat_raw - called from: ",as.character(sys.call()), - "\nget_eurostat_raw - get_eurostat_raw in sys.call(): ", grepl("^get_eurostat_rawidtxt",paste0(as.character(sys.call()),collapse="")))} + "\nget_eurostat_raw - called from: ",paste(deparse(this_call[[1]]),collapse=""), + "\nget_eurostat_raw - get_eurostat_raw in sys.call(): ", grepl("^get_eurostat_rawidtxt",this_call_str))} #check if the function was called from the get_eurostat_data function - if (grepl("^get_eurostat_rawidtxt",paste0(as.character(sys.call()),collapse=""))&any(grepl("get_eurostat_data",as.character(sys.calls())))){ + if (grepl("^get_eurostat_rawidtxt",this_call_str)&any(grepl("get_eurostat_data",call_names))){ #if yes get the value of local_filter and force_local_filter from the call if (exists("local_filter",envir=sys.parent(1))) {plf<-get("local_filter",envir=sys.parent(1))} else {plf<-FALSE} if (exists("force_local_filter",envir=sys.parent(1))) {pflf<-get("force_local_filter",envir=sys.parent(1))} else {pflf<-FALSE} @@ -354,8 +360,8 @@ get_eurostat_raw <- function(id, } else { child_cache<-FALSE } - - if ((!is.null(restat_raw))&cache&(all(!grepl("get_eurostat_bulk|get_eurostat_data",as.character(sys.calls()),perl=TRUE))|child_cache)){ + + if ((!is.null(restat_raw))&cache&(all(!grepl("get_eurostat_bulk|get_eurostat_data",call_names,perl=TRUE))|child_cache)){ oname<-paste0("r_",id,"-",udate,"-",sum(keep_flags)) pl<-restatapi::put_eurostat_cache(restat_raw,oname,update_cache,cache_dir,compress_file) if ((!is.null(pl))&(verbose)) {message("get_eurostat_raw - The raw data was cached ",pl,"." )} diff --git a/R/get_eurostat_toc.R b/R/get_eurostat_toc.R index 28d959a..c09846c 100755 --- a/R/get_eurostat_toc.R +++ b/R/get_eurostat_toc.R @@ -1,7 +1,7 @@ #' @title Download the Table of Contents of Eurostat datasets -#' @description Download Table of Contents (TOC) of Eurostat datasets if it is not cached previously. +#' @description Download Table of Contents (TOC) of Eurostat datasets if it is not cached previously. #' @param mode a character string either \code{xml} or \code{txt} defining the download mode. Depending on the mode the 'xml' version or the 'text' version of the TOC is downloaded. -#' The deafult value is \code{xml} as it provides more information (e.g. number of values, short description and download links in different formats (SDMX, TSV)) +#' The deafult value is \code{xml} as it provides more information (e.g. number of values, short description and download links in different formats (SDMX, TSV)) #' @param cache a boolean whether to load/save the TOC from/in the cache or not. The default value is \code{TRUE}, so that the TOC is checked first in the cache and if does not exist then downloaded from Eurostat and cached. #' @param update_cache a boolean to update cache or not. The default value is \code{FALSE}, so the cache is not updated. Can be set also with \code{options(restatapi_update=TRUE)} #' @param cache_dir a path to a cache directory. The default is \code{NULL}, in this case the TOC is cached in the memory (in the '.restatapi_env'). Otherwise if the \code{cache_dir} directory does not exist it creates the 'restatapi' directory in the temporary directory from \code{tempdir()} to save the RDS- file. Directory can also be set with \code{option(restatapi_cache_dir=...)}. @@ -36,14 +36,14 @@ #' @seealso \code{\link{search_eurostat_toc}}, \code{\link{get_eurostat_dsd}}, \code{\link{get_eurostat_raw}}, \code{\link{get_eurostat_bulk}}, \code{\link{get_eurostat_data}}. #' @details The TOC is downloaded from Eurostat websites through the REST API for the \code{xml} (default) version or from the bulk download facilities for \code{txt} version. #' From the downloaded TOC the values in the 'code' column can be used as \code{id} in the \code{\link{get_eurostat_dsd}}, \code{\link{get_eurostat_raw}}, \code{\link{get_eurostat_bulk}}, and \code{\link{get_eurostat_data}} functions. -#' @references For more technical information see the detailed documentation of the \href{https://ec.europa.eu/eurostat/data/web-services}{API}. -#' @examples +#' @references For more technical information see the detailed documentation of the \href{https://ec.europa.eu/eurostat/data/web-services}{API}. +#' @examples #' \dontshow{ #' if (parallel::detectCores()<=2){ #' options(restatapi_cores=1) #' }else{ #' options(restatapi_cores=2) -#' } +#' } #' } #' \donttest{ #' if (!(grepl("amzn|-aws|-azure ",Sys.info()['release']))) options(timeout=2) @@ -71,22 +71,22 @@ get_eurostat_toc<-function(mode="xml", if((!exists(".restatapi_env")|(length(list(...))>0))){ if ((length(list(...))>0)) { if (all(names(list(...)) %in% c("api_version","load_toc","parallel","max_cores","verbose"))){ - load_cfg(...) + load_cfg(...) } else { load_cfg() } } else { load_cfg() - } + } } - + if(any(grepl("get_eurostat_bulk|get_eurostat_data|get_eurostat_raw",as.character(sys.calls()),perl=TRUE))) {update_cache<-FALSE} - + if ((cache) & (!update_cache)) { toc<-restatapi::get_eurostat_cache(paste0("toc.",mode,".",lang),cache_dir,verbose=verbose) } if ((!cache)|(is.null(toc))|(update_cache)){ - cfg<-get("cfg",envir=restatapi::.restatapi_env) + cfg<-get("cfg",envir=restatapi::.restatapi_env) rav<-get("rav",envir=restatapi::.restatapi_env) if(mode=="txt"){ toc_endpoint<-eval(parse(text=paste0("cfg$TOC_ENDPOINT$'",rav,"'$ESTAT$txt$",lang))) @@ -94,31 +94,31 @@ get_eurostat_toc<-function(mode="xml", if (verbose) {message("get_eurostat_toc - Downloading ",toc_endpoint)} tryCatch({utils::download.file(toc_endpoint,temp,dmethod,quiet=!verbose)}, error = function(e) { - if (verbose) {message("get_eurostat_toc - Error during the download of the tsv version of the TOC file:",'\n',paste(unlist(e),collapse="\n"))} + if (verbose) {message("get_eurostat_toc - Error during the download of the tsv version of the TOC file:",'\n',conditionMessage(e))} else {message("There is an error by the download of the txt TOC file. Run the same command with verbose=TRUE option to get more info on the issue.")} tbc<-FALSE }, warning = function(w) { - if (verbose) {message("get_eurostat_toc - Warning by the download of the tsv version of the TOC file:",'\n',paste(unlist(w),collapse="\n"))} + if (verbose) {message("get_eurostat_toc - Warning by the download of the tsv version of the TOC file:",'\n',conditionMessage(w))} else {message("There is a warning by the download of the txt TOC file. Run the same command with verbose=TRUE option to get more info on the issue.")} tbc<-FALSE }) if (tbc) { tryCatch({toc<-data.table::fread(temp,header=TRUE,sep="\t",stringsAsFactors=FALSE,fill=Inf)}, error = function(e) { - if (verbose) {message("get_eurostat_toc - Error during the reading of the tsv version of the TOC file:",'\n',paste(unlist(e),collapse="\n"))} + if (verbose) {message("get_eurostat_toc - Error during the reading of the tsv version of the TOC file:",'\n',conditionMessage(e))} else {message("There is an error by the reading of the downloaded txt TOC file. Run the same command with verbose=TRUE option to get more info on the issue.")} tbc<-FALSE }, warning = function(w) { - if (verbose) {message("get_eurostat_toc - Warning by the reading of the tsv version of the TOC file:",'\n',paste(unlist(w),collapse="\n"))} + if (verbose) {message("get_eurostat_toc - Warning by the reading of the tsv version of the TOC file:",'\n',conditionMessage(w))} else {message("There is a warning by the reading of the downloaded txt TOC file. Run the same command with verbose=TRUE option to get more info on the issue.")} tbc<-FALSE }) if (tbc) { if (!is.null(toc)) { cnames<-c("title","code","type","lastUpdate","lastModified","dataStart","dataEnd","values") - if (ncol(toc)>8) cnames<-c(cnames,paste0("X",1:(ncol(toc)-8))) + if (ncol(toc)>8) cnames<-c(cnames,paste0("X",1:(ncol(toc)-8))) names(toc)<-cnames toc$code<-sub("^\\s*","",toc$code) toc<-toc[toc$type!="folder"&toc$code!="",1:8] @@ -132,12 +132,12 @@ get_eurostat_toc<-function(mode="xml", if (verbose) {message("get_eurostat_toc - Downloading ",toc_endpoint)} tryCatch({xml_leafs<-xml2::xml_find_all(xml2::read_xml(toc_endpoint,verbose=verbose),".//nt:leaf")}, error = function(e) { - if (verbose) {message("get_eurostat_toc - Error during the download of the xml version of the TOC file:",'\n',paste(unlist(e),collapse="\n"))} + if (verbose) {message("get_eurostat_toc - Error during the download of the xml version of the TOC file:",'\n',conditionMessage(e))} else {message("There is an error by the download of the XML TOC file. Run the same command with verbose=TRUE option to get more info on the issue.")} tbc<-FALSE }, warning = function(w) { - if (verbose) {message("get_eurostat_toc - Warning by the download of the xml version of the TOC file:",'\n',paste(unlist(w),collapse="\n"))} + if (verbose) {message("get_eurostat_toc - Warning by the download of the xml version of the TOC file:",'\n',conditionMessage(w))} else {message("There is a warning by the download of the XML TOC file. Run the same command with verbose=TRUE option to get more info on the issue.")} tbc<-FALSE }) @@ -158,19 +158,19 @@ get_eurostat_toc<-function(mode="xml", leafs<-parallel::parLapply(cl,as.character(xml_leafs),extract_toc) parallel::stopCluster(cl)}, error = function(e) { - if (verbose) {message("get_eurostat_toc - Error during the launch of the parallel processing:",'\n',paste(unlist(e),collapse="\n"))} + if (verbose) {message("get_eurostat_toc - Error during the launch of the parallel processing:",'\n',conditionMessage(e))} }, warning = function(w) { - if (verbose) {message("get_eurostat_toc - Warning during the launch of the parallel processing:",'\n',paste(unlist(w),collapse="\n"))} - }) + if (verbose) {message("get_eurostat_toc - Warning during the launch of the parallel processing:",'\n',conditionMessage(w))} + }) } }else{ tryCatch({leafs<-parallel::mclapply(xml_leafs,extract_toc,mc.cores=getOption("restatapi_cores",1L))}, error = function(e) { - if (verbose) {message("get_eurostat_toc - Error during the launch of the parallel processing:",'\n',paste(unlist(e),collapse="\n"))} + if (verbose) {message("get_eurostat_toc - Error during the launch of the parallel processing:",'\n',conditionMessage(e))} }, warning = function(w) { - if (verbose) {message("get_eurostat_toc - Warning during the launch of the parallel processing:",'\n',paste(unlist(w),collapse="\n"))} + if (verbose) {message("get_eurostat_toc - Warning during the launch of the parallel processing:",'\n',conditionMessage(w))} }) } if (exists("leafs")){ @@ -182,29 +182,29 @@ get_eurostat_toc<-function(mode="xml", keep<-c(paste0("title.",lang),"code","type","lastUpdate","lastModified","dataStart","dataEnd","values",paste0("unit.",lang),paste0("source.",lang),paste0("shortDescription.",lang),"metadata.html","metadata.sdmx","downloadLink.tsv") if (verbose) {message("get_eurostat_toc - cols to keep: ",paste(keep,collapse = ","))} toc<-toc[,keep,with=FALSE] - names(toc)<-c("title","code","type","lastUpdate","lastModified","dataStart","dataEnd","values","unit","source","shortDescription","metadata.html","metadata.sdmx","downloadLink.tsv") + names(toc)<-c("title","code","type","lastUpdate","lastModified","dataStart","dataEnd","values","unit","source","shortDescription","metadata.html","metadata.sdmx","downloadLink.tsv") } - } + } } } } else { warning('Incorrect mode is given. It should be either "xml" or "txt".') - } + } if (!is.null(toc)){ toc<-toc[!duplicated(toc[,c(1:8)]),] toc$values<-as.numeric(toc$values) toc$lastUpdate<-format(as.Date(toc$lastUpdate,"%d.%m.%Y"),"%Y.%m.%d") - toc$lastModified<-format(as.Date(toc$lastModified,"%d.%m.%Y"),"%Y.%m.%d") + toc$lastModified<-format(as.Date(toc$lastModified,"%d.%m.%Y"),"%Y.%m.%d") } else{ if (verbose) {message("get_eurostat_toc - The TOC is empty. Please check the download link form the line above in a web browser.")} } - } + } if (!is.null(toc)&cache){ name<-paste0("toc.",mode,".",lang) pl<-restatapi::put_eurostat_cache(toc,name,update_cache,cache_dir,compress_file) if (verbose){message("get_eurostat_toc - The TOC was cached ",pl,".")} } - return(toc) + return(toc) } diff --git a/R/load_cfg.R b/R/load_cfg.R index 5b750db..ad6bd5b 100755 --- a/R/load_cfg.R +++ b/R/load_cfg.R @@ -1,7 +1,8 @@ #' @title Load configuration data from JSON #' @description Load the configuration information to the '.restatapi_env' from the JSON configuration file. #' @param api_version It can be either "old", "new", "test" or "current". The default value is "current" which defined by the DEFAULT_API_VERSION value of the config file. -#' @param cfg_file The location of the config file. It can be either "github" (the default value) or "local". +#' @param cfg_file The location of the config file. It can be either "github" (the default value), "local", or a path to a local JSON configuration file. +#' If a file path is provided, the configuration is loaded from that file. If the file does not exist or cannot be parsed, the preinstalled file in the package is used. #' @param load_toc The default value \code{FALSE}, which means that the XML version of the Table of contents (TOC) will not be downloaded and #' cached automatically in the '.restatapi_env' when the package is loaded. #' @param parallel A boolean with the default value \code{TRUE}. If there are multiple cores/logical processors then part of the data @@ -49,17 +50,26 @@ load_cfg<-function(api_version="default",cfg_file="github",load_toc=FALSE,parall if (cfg_file=="github"){ cfg_source<-"GitHub" tryCatch( - {assign("cfg",rjson::fromJSON(file="https://raw.githubusercontent.com/eurostat/restatapi/master/inst/extdata/rest_api_config.json"),envir=.restatapi_env)}, + {assign("cfg",jsonlite::fromJSON("https://raw.githubusercontent.com/eurostat/restatapi/master/inst/extdata/rest_api_config.json",simplifyVector=TRUE,simplifyDataFrame=FALSE,simplifyMatrix=FALSE),envir=.restatapi_env)}, error = function(e) {if (verbose) {warning("\nload_cfg - The configuration file could not be downloaded from GitHub, the preinstalled file in the package is used.")} - assign("cfg",rjson::fromJSON(file=system.file("extdata","rest_api_config.json",package="restatapi")),envir=.restatapi_env) - cfg_source<-"the file installed locally"}) + assign("cfg",jsonlite::fromJSON(system.file("extdata","rest_api_config.json",package="restatapi"),simplifyVector=TRUE,simplifyDataFrame=FALSE,simplifyMatrix=FALSE),envir=.restatapi_env) + cfg_source<<-"the file installed locally"}) } else if (cfg_file=="local"){ if (verbose) {message("\nload_cfg - The preinstalled file in the package is used.")} - assign("cfg",rjson::fromJSON(file=system.file("extdata","rest_api_config.json",package="restatapi")),envir=.restatapi_env) + assign("cfg",jsonlite::fromJSON(system.file("extdata","rest_api_config.json",package="restatapi"),simplifyVector=TRUE,simplifyDataFrame=FALSE,simplifyMatrix=FALSE),envir=.restatapi_env) cfg_source<-"the file installed locally" + } else if (file.exists(cfg_file)){ + cfg_source<-paste0("the file '",cfg_file,"'") + tryCatch( + {assign("cfg",jsonlite::fromJSON(cfg_file,simplifyVector=TRUE,simplifyDataFrame=FALSE,simplifyMatrix=FALSE),envir=.restatapi_env) + if (verbose) {message("\nload_cfg - The configuration file '",cfg_file,"' is used.")}}, + error = function(e) + {if (verbose) {warning("\nload_cfg - The configuration file '",cfg_file,"' could not be parsed, the preinstalled file in the package is used.")} + assign("cfg",jsonlite::fromJSON(system.file("extdata","rest_api_config.json",package="restatapi"),simplifyVector=TRUE,simplifyDataFrame=FALSE,simplifyMatrix=FALSE),envir=.restatapi_env) + cfg_source<<-"the file installed locally"}) } else { - message('Incorrect value for the cfg_file parameter. It should take the value either "github" or "local".') + message('Incorrect value for the cfg_file parameter. It should take the value either "github", "local", or a path to an existing JSON configuration file.') tbc<-FALSE } if (tbc) { diff --git a/README.md b/README.md index 6a4d6b9..d9bc4ab 100755 --- a/README.md +++ b/README.md @@ -22,11 +22,8 @@ An R package to search and retrieve data from Eurostat database using SDMX install.packages("restatapi") ``` -or use the development version from GitHub +or use the development version from GitHub. -```R -remotes::install_github("eurostat/restatapi") -``` ## background This package is similar to other packages like the [eurodata](https://cran.r-project.org/package=eurodata), [eurostat](https://cran.r-project.org/package=eurostat), [rdbnomics](https://cran.r-project.org/package=rdbnomics), [RJSDMX](https://cran.r-project.org/package=RJSDMX) or [TSsdmx](https://cran.r-project.org/package=TSsdmx) which can be used to download data from Eurostat database. The difference is that `restatapi` is based on SDMX (Statistical Data and Metadata eXchange) and XML to search and retrieve filtered datasets and use the TSV (tab separated values) bulk download facility to get whole data tables. The code was written in a way that the number of dependencies on other packages should be very small. The `restatapi` package provides flexible filtering options, data caching, and uses the `parallel` and `data.table` package to handle large dataset in an efficient way. @@ -101,16 +98,16 @@ Finally, download only the quarterly data (`select_freq="Q"`) for several time p Before 2022, in the old dissemination chain the value was assigned to *the first day* of the month, quarter and year, so it was enough to filter for one day to get the value for the whole period. Under the current API the value belongs to the full period. If a date range does not cover the whole period no value is returned. For example, to get the value of the whole quarter the date filter should start at least on the first date of the quarter and end at least on the last day of the quarter. With exact numerical example to get the value for 2022/Q3, the `startDate` should be 2022-07-01 or earlier and the `endDate` should be 2022-09-30 or later. In the old version of the API it was enough if the period included the day 2022-07-01 only. ```R -dt<-get_eurostat_data("avia_par_me",select_freq="A",cache=FALSE) -dt<-get_eurostat_data("avia_par_me", +get_eurostat_data("avia_par_me",select_freq="A",cache=FALSE) +get_eurostat_data("avia_par_me", filters="Q...ME_LYPG_HU_LHBP+ME_LYTV_UA_UKKK", date_filter=c("2016-08","2017-07-01"), select_freq="M") -dt<-get_eurostat_data("avia_par_me", +get_eurostat_data("avia_par_me", filters=c("HU","Quarterly","Monthly"), date_filter=c("2016-08","2014-03-01"), label=TRUE) -dt<-get_eurostat_data("avia_par_me", +get_eurostat_data("avia_par_me", filters="HU", exact_match=FALSE, date_filter=c("2017-03",2016,"2017-07-01",2012:2014), @@ -120,40 +117,31 @@ dt<-get_eurostat_data("avia_par_me", name=FALSE) ``` -**Example 7:** Download from the Time Use Survey (TUS) data (`tus_00age`) for 2010 (`date_filter=2010`) in Hungary how much time spent with travel on average. If someone does not know the exact codes then the filter patterns (`filters=c("total","time spent","HU",'travel')`) can be searched in the labels (`label=TRUE`) non case sensitive (`ignore.case=TRUE`) without forcing exact match of the patterns (`exact_match=FALSE`). The first function call will result an empty data table as the SDMX webservice will provide NaN (Not a Number) response for the time values (hh:mm). But these values are included in the bulk download files, and can be retrieved by forcing the filtering on the local computer (`force_local_filter=TRUE`). Then the retrieved values can be summed using the [chron](https://cran.r-project.org/package=chron) package. +**Example 7:** Download from the Time Use Survey (TUS) data (`tus_00age`) for 2010 (`date_filter=2010`) in Hungary how much time spent with travel on average. If someone does not know the exact codes then the filter patterns (`filters=c("total","time spent","HU",'travel')`) can be searched in the labels (`label=TRUE`) non case sensitive (`ignore.case=TRUE`) without forcing exact match of the patterns (`exact_match=FALSE`). ```R -dt<-get_eurostat_data("tus_00age", - filters=c("HU","total","time spent",'travel'), - date_filter=2010, - exact_match=FALSE, - ignore.case=TRUE, - label=TRUE) -dt<-get_eurostat_data("tus_00age", +get_eurostat_data("tus_00age", filters=c("HU","total","time spent",'travel'), date_filter=2010, - force_local_filter=TRUE, exact_match=FALSE, ignore.case=TRUE, label=TRUE) -dt -dt[acl00!="Total",sum(chron::times(paste0(values,":00")))] ``` **Example 8:** Download the data on the production of cow's milk on farms by NUTS 2 regions (`agr_r_milkpr`) first only for the new Member States joined in 2004 and keeping only the period between March 2009 and 5 June 2011 (`date_filter="2009-03:2011-06-05"`). The country code of the member states can be loaded from the `.restatapi env` (`eu<-get("cc",envir=.restatapi_env)`) and it can be used in the query (`filters=eu$NMS10`). Then get all the data for Belgium for all [NUTS](https://ec.europa.eu/eurostat/en/web/products-manuals-and-guidelines/-/ks-gq-20-092) level from the same data set before July 2009 (`date_filter="<2009-07"`) with the labels (`label=TRUE`) and the observation status (so called *flags*) information (`flags=TRUE`). In this case for filter (`filters="BE"`) the exact matching of pattern should be turned off (`exact_match=FALSE`) to get not just at country (NUTS0) level. Finally, get the data at for Hungary at NUTS2 level after 19 May 2017 (`date_filter="2017-05-19<`) and keeping the lines which were removed due to confidentiality (`cflags=TRUE`). For this we do not have to know the exact code, name or number of the NUTS2 regions as we can provide regular expression in the filter (`filters=c("^HU..")`) and providing the option that the expression is Perl compatible (`perl=TRUE`). ```R eu<-get("cc",envir=.restatapi_env) -dt<-get_eurostat_data("agr_r_milkpr", +get_eurostat_data("agr_r_milkpr", filters=eu$NMS10, date_filter="2009-03:2011-06-05") -dt<-get_eurostat_data("agr_r_milkpr", +get_eurostat_data("agr_r_milkpr", filters="BE", date_filter="<2009-07", keep_flags=TRUE, exact_match=FALSE, label=TRUE) -dt<-get_eurostat_data("agr_r_milkpr", +get_eurostat_data("agr_r_milkpr", filters=c("^HU.."), date_filter="2017-05-19<", cflags=TRUE, @@ -164,7 +152,7 @@ dt<-get_eurostat_data("agr_r_milkpr", **Example 9:** Download the balance (`stk_flow="BAL"`) from the international trade in services dataset (`bop_its6_det`) for 2020 (`date_filter=2020`), transport services (`bop_item="SC"`), with reporting countries Hungary and the EU (`geo=c("EU27_2020","HU")`) and trading partner outside EU (`partner="EXT_EU27_2020"`). In order to avoid that Hungary shows up in the partner countries the filter should be defined as a named list (`filters=list(bop_item="SC",partner="EXT_EU27_2020",geo=c("EU27_2020","HU"),stk_flow="BAL")`) and do not search for the terms in the labels (`name=FALSE`). In this case the filter patterns only searched where the concept equals to the name. ```R -dt<-get_eurostat_data("bop_its6_det", +get_eurostat_data("bop_its6_det", filters=list(bop_item="SC", partner="EXT_EU27_2020", geo=c("EU27_2020","HU"), diff --git a/docs/404.html b/docs/404.html index b385e72..f8a1292 100755 --- a/docs/404.html +++ b/docs/404.html @@ -32,7 +32,7 @@ restatapi - 0.24.2 + 0.30.0 @@ -88,7 +88,7 @@

Page not found (404)

-

Site built with pkgdown 2.1.1.

+

Site built with pkgdown 2.2.1.

diff --git a/docs/authors.html b/docs/authors.html index 4351bd7..da5a0ec 100755 --- a/docs/authors.html +++ b/docs/authors.html @@ -17,7 +17,7 @@ restatapi - 0.24.2 + 0.30.0 @@ -65,15 +65,15 @@

Citation

-

Mészáros M (2025). +

Mészáros M (2026). restatapi: Search and Retrieve Data from Eurostat Database. -R package version 0.24.2, https://github.com/eurostat/restatapi. +R package version 0.30.0, https://github.com/eurostat/restatapi. 

@Manual{,
   title = {restatapi: Search and Retrieve Data from Eurostat Database},
   author = {Mátyás Mészáros},
-  year = {2025},
-  note = {R package version 0.24.2},
+  year = {2026},
+  note = {R package version 0.30.0},
   url = {https://github.com/eurostat/restatapi},
 }
@@ -88,7 +88,7 @@

Citation

-

Site built with pkgdown 2.1.1.

+

Site built with pkgdown 2.2.1.

diff --git a/docs/index.html b/docs/index.html index e77a5bd..86ad2df 100755 --- a/docs/index.html +++ b/docs/index.html @@ -12,7 +12,7 @@ - +