mmatyi · Oct 9, 2026 · Jan 30, 2026 · Jan 30, 2026 · Jan 30, 2026 · Jan 30, 2026
diff --git a/.Rbuildignore b/.Rbuildignore
@@ -3,3 +3,5 @@
 ^docs$
 ^\.github$
 ^\.ipynb
+.lintr
+.gitignore
diff --git a/.github/workflows/R-CMD-check.yaml b/.github/workflows/R-CMD-check.yaml
@@ -31,13 +31,13 @@ jobs:
         config:
           - {os: macOS-latest,   r: 'release'}
           - {os: windows-latest, r: 'release'}
-          - {os: windows-latest, r: 'oldrel', rspm: "https://packagemanager.rstudio.com/cran/latest"}
-          - {os: ubuntu-22.04,   r: 'devel', rspm: "https://packagemanager.rstudio.com/cran/__linux__/bionic/latest"}
-          - {os: ubuntu-22.04,   r: 'release', rspm: "https://packagemanager.rstudio.com/cran/__linux__/bionic/latest"}
-          - {os: ubuntu-22.04,   r: 'oldrel',  rspm: "https://packagemanager.rstudio.com/cran/__linux__/bionic/latest"}
+          - {os: ubuntu-24.04,   r: 'devel', rspm: "https://packagemanager.rstudio.com/cran/__linux__/bionic/latest"}
+          - {os: ubuntu-24.04,   r: 'release', rspm: "https://packagemanager.rstudio.com/cran/__linux__/bionic/latest"}
+          - {os: ubuntu-24.04,   r: 'oldrel',  rspm: "https://packagemanager.rstudio.com/cran/__linux__/bionic/latest"}
     #      - {os: ubuntu-18.04,   r: '3.6',     rspm: "https://packagemanager.rstudio.com/cran/__linux__/bionic/latest"}
     #      - {os: ubuntu-18.04,   r: '3.5',     rspm: "https://packagemanager.rstudio.com/cran/__linux__/bionic/latest"}
     #      - {os: ubuntu-18.04,   r: '3.4',     rspm: "https://packagemanager.rstudio.com/cran/__linux__/bionic/latest"}
+    #      - {os: windows-latest, r: 'oldrel', rspm: "https://packagemanager.rstudio.com/cran/latest"}
 
     env:
       RSPM: ${{ matrix.config.rspm }}

diff --git a/.github/workflows/make_doc.R b/.github/workflows/make_doc.R
@@ -1,8 +1,8 @@
-pkgs<-c("pkgdown","devtools","rjson","xml2","data.table")
+pkgs<-c("pkgdown","devtools","jsonlite","xml2","data.table","pak")
 pkgs_to_install<-pkgs[!(pkgs %in% installed.packages()[,"Package"])]
 if (length(pkgs_to_install)>0) install.packages(pkgs_to_install)
 
-devtools::install_github("eurostat/restatapi")
+pak::pak("eurostat/restatapi")
 cat(getwd())
 cat(Sys.info()['release'])
 message(getwd())

diff --git a/.gitignore b/.gitignore
@@ -0,0 +1,7 @@
+.Rproj.user
+.Rhistory
+.RData
+.Ruserdata
+*.Rproj
+.lintr
+.gitignore
diff --git a/DESCRIPTION b/DESCRIPTION
@@ -1,8 +1,8 @@
 Package: restatapi
 Type: Package
 Title: Search and Retrieve Data from Eurostat Database
-Date: 2026-03-27
-Version: 0.25.0
+Date: 2026-10-08
+Version: 0.30.0
 Encoding: UTF-8
 Authors@R: c(person("Mátyás", "Mészáros", email = "matyas.meszaros@ec.europa.eu", role = c("aut", "cre")),
             person("Sebastian", "Weinand", role = "ctb"))
@@ -12,9 +12,9 @@ Description: Eurostat is the statistical office of the European Union and provid
              (<https://ec.europa.eu/eurostat/web/user-guides/data-browser/api-data-access/api-detailed-guidelines/sdmx2-1>) to search and download data from 
              the Eurostat database using the SDMX standard. 
 License: EUPL
-Imports: data.table, rjson, xml2
-Suggests: chron, knitr, rmarkdown, tinytest, remotes
+Imports: data.table, jsonlite, xml2
+Suggests: tinytest
 NeedsCompilation: no
 URL: https://github.com/eurostat/restatapi
 BugReports: https://github.com/eurostat/restatapi/issues
-RoxygenNote: 7.3.2
+Config/roxygen2/version: 8.1.0
diff --git a/NEWS.md b/NEWS.md
@@ -1,3 +1,19 @@
+# restatapi 0.30.0
+
+Major revision of the package including:
+- replaced the `rjson` package with `jsonlite` for parsing the configuration file
+- removed the `chron`, `remotes`, `knitr` and `rmarkdown` packages from the suggested dependencies
+- shortened the verbose debug messages so they no longer include the source code of functions from the call stack
+- fixed the `get_eurostat_data()` function so that the `mode="csv"` and `mode="xml"` downloads return the same result: the empty `CONF_STATUS` column is dropped and a non-empty value is added into the `flags` column
+- aligned the `CONF_STATUS`/flag handling in the `get_eurostat_bulk()` function with the `get_eurostat_data()` function, folding the `CONF_STATUS` value into the `flags` column
+- the `extract_data()` function now also extracts the `CONF_STATUS` value from the XML so the confidentiality information is available in the `flags` column
+- added regression tests checking the `mode="csv"`/`mode="xml"` parity for datasets with  the `CONF_STATUS` column
+- new `dsd_order` parameter in the `get_eurostat_data()` function to have a standard order of columns independently from the download mode
+- `load_cfg()` now accepts a path to a local JSON configuration file in the `cfg_file` parameter
+- removal of the `check_toc` option from the `extract_data()` function as the TOC does not have the link anymore to the XML format
+- additional tests
+- CRAN release
+
 # restatapi 0.25.0
 
 - additional debugging for `get_eurostat_toc()` function

diff --git a/R/create_filter_table.R b/R/create_filter_table.R
@@ -186,7 +186,7 @@ create_filter_table <- function(filters,date_filter=FALSE,dsd=NULL,exact_match=T
         }
       } else{
         concepts<-names(filters)
-        ft<-data.table::rbindlist(lapply(1:length(filters),function (x,f,d){
+        ft<-data.table::rbindlist(lapply(seq_along(filters),function (x,f,d){
           do.call(rbind,lapply(unlist(f[x]),restatapi::search_eurostat_dsd,dsd=d[toupper(d$concept)==toupper(concepts[x]),],exact_match=exact_match,...))
         },f=filters,d=dsd))
       }

diff --git a/R/extract_data.R b/R/extract_data.R
@@ -1,16 +1,15 @@
-#' @title Extract data values from SDMX XML 
+#' @title Extract data values from SDMX XML
 #' @description Extracts the data values from the SDMX XML data file
 #' @param xml_lf an input XML leaf with data series from an SDMX XML file to extract the value and its dimensions from it
 #' @param keep_flags a logical value if to extract the observation status (flag) information from the XML file. The default value is \code{FALSE}
-#' @param stringsAsFactors a logical value. If \code{TRUE} the columns are converted to factors. The default is \code{FALSE}, 
+#' @param stringsAsFactors a logical value. If \code{TRUE} the columns are converted to factors. The default is \code{FALSE},
 #'        in this case the strings are returned as characters.
-#' @param bulk a logical value with default value \code{TRUE} if the input SDMX XML file is from the bulk download facility containing all the observations. 
-#'        If the input file has pre-filtered values then the value \code{FALSE} should be used.  
-#' @param check_toc if the data file was downloaded using the URL from the TOC or not. The default is FALSE means not the TOC link is used.         
-#' @export 
+#' @param bulk a logical value with default value \code{TRUE} if the input SDMX XML file is from the bulk download facility containing all the observations.
+#'        If the input file has pre-filtered values then the value \code{FALSE} should be used.
+#' @export
 #' @details It is a sub-function to use in the \code{\link{get_eurostat_data}} and the \code{\link{get_eurostat_raw}} functions.
 #' @return a data frame containing the values of an SDMX node: the dimensions, value and the optional flag(s)
-#' @examples 
+#' @examples
 #' \dontshow{
 #' if (parallel::detectCores()<=2){
 #'    options(restatapi_cores=1)
@@ -29,40 +28,39 @@
 #'   extract_data(xml_leafs[1])
 #' options(timeout=60)
 #' }
-#' 
+#'
 
-extract_data<-function(xml_lf,keep_flags=FALSE,stringsAsFactors=FALSE,bulk=TRUE,check_toc=FALSE){
+extract_data<-function(xml_lf,keep_flags=FALSE,stringsAsFactors=FALSE,bulk=TRUE){
   # if (getOption("restatapi_verbose",FALSE))  {message("extract_data - API version:",get("rav",envir=restatapi::.restatapi_env))}
   rav<-get("rav",envir=restatapi::.restatapi_env)
   prefix<-switch(rav,"1"="generic:","2"="g:")
   if(bulk){
     if (Sys.info()[['sysname']]=='Windows'){
       xml_lf<-gsub(prefix,"",xml_lf)
       xml_lf<-xml2::as_xml_document(xml_lf)
-    } 
+    }
     bd<-t(as.data.frame(xml2::xml_attrs(xml_lf),stringsAsFactors=FALSE))
     rownames(bd)<-NULL
     dv<-xml2::xml_attrs(xml2::xml_children(xml_lf))
     if (keep_flags){
-      flagc<-switch(rav,"1"="OBS_STATUS","2"="OBS_FLAG")
-      # if (check_toc) {flagc<-"OBS_STATUS"}
+      flagc<-switch(rav,"1"="OBS_STATUS","2"=c("OBS_FLAG","CONF_STATUS"))
       cn<-c("TIME_PERIOD","OBS_VALUE",flagc)
     } else {
       cn<-c("TIME_PERIOD","OBS_VALUE")
     }
     df<-do.call(rbind, lapply(lapply(dv, unlist),"[", cn))
     colnames(df)<-cn
-    out<-data.frame(bd,df,stringsAsFactors=FALSE)  
+    out<-data.frame(bd,df,stringsAsFactors=FALSE)
   } else {
     # if (Sys.info()[['sysname']]=='Windows'){
     #   xml_lf<-xml2::as_xml_document(xml_lf)
-    # } 
+    # }
     tmp<-as.data.frame(xml2::xml_attrs(xml2::xml_children(xml2::xml_find_all(xml_lf,paste0(".//",prefix,"SeriesKey")))),stringsAsFactors=FALSE)
     bd<-as.data.frame(tmp[2,],stringsAsFactors=FALSE)
     colnames(bd)<-tmp[1,]
     rownames(bd)<-NULL
     obs<-xml2::xml_find_all(xml_lf,paste0(".//",prefix,"Obs"))
-    cn<-c("obsTime","obsValue") 
+    cn<-c("obsTime","obsValue")
     df<-data.table::rbindlist(lapply(obs, function(x) {dr<-as.data.frame(t(unlist(xml2::xml_attrs(xml2::xml_children(x)))),stringsAsFactors=FALSE)
                                                 rownames(dr)<-NULL
                                                 if (ncol(dr)<2) {dr$nc<-""}
@@ -73,7 +71,7 @@ extract_data<-function(xml_lf,keep_flags=FALSE,stringsAsFactors=FALSE,bulk=TRUE,
                                                     dr$OBS_STATUS<-""
                                                   } else{
                                                     f<-gsub("na","",f)
-                                                    f<-tolower(f)
+                                                    #f<-tolower(f)
                                                     f<-gsub("@","",f)
                                                     dr$OBS_STATUS<-paste0(f,collapse="")
                                                   }
@@ -84,7 +82,7 @@ extract_data<-function(xml_lf,keep_flags=FALSE,stringsAsFactors=FALSE,bulk=TRUE,
     # df$obsValue<-as.numeric(df$obsValue)
     df$obsValue[df$obsValue=="NaN"]<-NA
     if (keep_flags){
-      out<-data.frame(bd,df,stringsAsFactors=stringsAsFactors)  
+      out<-data.frame(bd,df,stringsAsFactors=stringsAsFactors)
     } else{
       out<-data.frame(bd,df[,1:2],stringsAsFactors=stringsAsFactors)
     }

diff --git a/R/extract_dsd.R b/R/extract_dsd.R
@@ -1,20 +1,20 @@
-#' @title Extract the Data Structure Definition content from SDMX XML 
+#' @title Extract the Data Structure Definition content from SDMX XML
 #' @description Extracts values from the XML Data Structure Definition (DSD) file
-#' @param concept a character vector with a concept id  
+#' @param concept a character vector with a concept id
 #' @param dsd_xml an XML file with DSD content
-#' @param lang a character string either \code{en}, \code{de} or \code{fr} to define the language version for the name column 
+#' @param lang a character string either \code{en}, \code{de} or \code{fr} to define the language version for the name column
 #' of the DSD. It is used only in the new API. The default is \code{en} - English.
-#' @export 
+#' @export
 #' @details It is a sub-function to use in the \code{\link{get_eurostat_dsd}} function.
-#' @return  a matrix with 3 columns if the provided \code{concept} has a code list in the DSD file. The first column is the provided \code{concept}. The second column 
+#' @return  a matrix with 3 columns if the provided \code{concept} has a code list in the DSD file. The first column is the provided \code{concept}. The second column
 #'         is the possible codes under the given \code{concept}. The last column is the name/description for the code in the second column, which can be used as labels.
-#' @examples 
+#' @examples
 #' \dontshow{
 #' if (parallel::detectCores()<=2){
 #'    options(restatapi_cores=1)
 #' }else{
 #'    options(restatapi_cores=2)
-#' }    
+#' }
 #' }
 #' \donttest{
 #' id<-"med_rd6"
@@ -32,28 +32,28 @@
 #'  )
 #' if (!(grepl("amzn|-aws|-azure ",Sys.info()['release']))) options(timeout=2)
 #' tryCatch({
-#'   dsd_xml<-xml2::read_xml(dsd_url)}, 
+#'   dsd_xml<-xml2::read_xml(dsd_url)},
 #'   error=function(e){
-#'   message("Unable to download the xml file.\n",e)}, 
+#'   message("Unable to download the xml file.\n",e)},
 #'   warning=function(w){
-#'   message("Unable to download the xml file.\n",w)}) 
-#' if (exists("dsd_xml")) extract_dsd("FREQ",dsd_xml) 
+#'   message("Unable to download the xml file.\n",w)})
+#' if (exists("dsd_xml")) extract_dsd("FREQ",dsd_xml)
 #' options(timeout=60)
 #' }
 
 extract_dsd<-function(concept=NULL,dsd_xml=NULL,lang="en"){
   rav<-get("rav",envir=restatapi::.restatapi_env)
   if (getOption("restatapi_verbose",FALSE))  {message("\nextract_dsd - API version:",rav)}
-  
+
   xml_string_clc<-switch(rav,
                       "1" = paste0('//str:Codelist[@id="CL_',concept,'"]/str:Code'),
                       "2" = paste0('//s:Codelist[@id="',toupper(concept),'"]/s:Code')
-    
+
   )
   xml_string_cln<-switch(rav,
                       "1" = paste0('//str:Codelist[@id="CL_',concept,'"]/str:Code/com:Name'),
                       "2" = paste0('//s:Codelist[@id="',toupper(concept),'"]/s:Code/c:Name[@xml:lang="',lang,'"]')
-                        
+
   )
   if (getOption("restatapi_verbose",FALSE))  {message("extract_dsd - xml_string_clc:",xml_string_clc,"\nextract_dsd - xml_string_cln:",xml_string_cln)}
 
@@ -67,5 +67,5 @@ extract_dsd<-function(concept=NULL,dsd_xml=NULL,lang="en"){
       xml_cln<-xml2::xml_text(xml2::xml_find_all(dsd_xml,xml_string_cln))
       cbind(concept,xml_clc,xml_cln)
     }
-  } 
-}  
+  }
+}
diff --git a/R/filter_raw_data.R b/R/filter_raw_data.R
@@ -60,7 +60,7 @@ filter_raw_data<-function(raw_data=NULL,filter_table=NULL,date_filter=FALSE){
       raw_data[grepl("\\dQ",time),ft:=paste0(substr(time,1,4),"-",lapply(substr(time,6,6),function(x){if (x<"4"){"0"}else{""}}),as.character((as.numeric(substr(time,6,6))-1)*3+1),"-01")]
       raw_data[grepl("-Q",time),ft:=paste0(substr(time,1,4),"-",lapply(substr(time,7,7),function(x){if (x<"4"){"0"}else{""}}),as.character((as.numeric(substr(time,7,7))-1)*3+1),"-01")]
       raw_data[grepl("S",time),ft:=paste0(substr(time,1,4),"-0",as.character((as.numeric(substr(time,6,6))-1)*6+1),"-01")]
-      data_out<-data.table::rbindlist(lapply(1:nrow(filter_table),function (x){raw_data[(ft>=filter_table$sd[x] & ft<=filter_table$ed[x])]}))
+      data_out<-data.table::rbindlist(lapply(seq_len(nrow(filter_table)),function (x){raw_data[(ft>=filter_table$sd[x] & ft<=filter_table$ed[x])]}))
       data_out<-data_out[,!'ft'][]
     }else{
       filter_table$concept<-toupper(filter_table$concept)

diff --git a/R/get_compressed_sdmx.R b/R/get_compressed_sdmx.R
@@ -34,11 +34,11 @@ get_compressed_sdmx<-function(url=NULL,verbose=FALSE,format="gz"){
         if (verbose) {
           tryCatch({utils::download.file(url,temp,dmethod)},
                    error = function(e) {
-                     message("get_compressed_sdmx - Error by the download of the SDMX file:",'\n',paste(unlist(e),collapse="\n"))
+                     message("get_compressed_sdmx - Error by the download of the SDMX file:",'\n',conditionMessage(e))
                      tbc<-FALSE
                    },
                    warning = function(w) {
-                     message("get_compressed_sdmx - Warning during the download of the SDMX file:",'\n',paste(unlist(w),collapse="\n"))
+                     message("get_compressed_sdmx - Warning during the download of the SDMX file:",'\n',conditionMessage(w))
                      tbc<-FALSE
                    })
         } else {
@@ -62,10 +62,10 @@ get_compressed_sdmx<-function(url=NULL,verbose=FALSE,format="gz"){
         if (verbose) {
           tryCatch({xml_fajl<-utils::unzip(temp,paste0(fajl,".xml"),exdir=tmpdir)},
                    error = function(e) {
-                     message("get_compressed_sdmx - Error during the unzip of the SDMX file:",'\n',paste(unlist(e),collapse="\n"))
+                     message("get_compressed_sdmx - Error during the unzip of the SDMX file:",'\n',conditionMessage(e))
                    },
                    warning = function(w) {
-                     message("get_compressed_sdmx - Warning by the unzip of the SDMX file:",'\n',paste(unlist(w),collapse="\n"))
+                     message("get_compressed_sdmx - Warning by the unzip of the SDMX file:",'\n',conditionMessage(w))
                    })
         } else {
           tryCatch({xml_fajl<-utils::unzip(temp,paste0(fajl,".xml"),exdir=tmpdir)},
@@ -82,10 +82,10 @@ get_compressed_sdmx<-function(url=NULL,verbose=FALSE,format="gz"){
                  xml<-xml2::read_xml(cid)
                  close(cid)},
                  error = function(e) {
-                   message("get_compressed_sdmx - Error during retrieval and extraction of the SDMX file:",'\n',paste(unlist(e),collapse="\n"))
+                   message("get_compressed_sdmx - Error during retrieval and extraction of the SDMX file:",'\n',conditionMessage(e))
                  },
                  warning = function(w) {
-                   message("get_compressed_sdmx - Warning by the retrieval and extraction of the SDMX file:",'\n',paste(unlist(w),collapse="\n"))
+                   message("get_compressed_sdmx - Warning by the retrieval and extraction of the SDMX file:",'\n',conditionMessage(w))
                  })
         } else {
           tryCatch({