From 2b32eb262229428f4da26e2b24a5c28284f03f19 Mon Sep 17 00:00:00 2001 From: Sander Devisscher Date: Mon, 23 Jun 2025 15:59:34 +0200 Subject: [PATCH 01/28] Create get_wekeo_data.R #111 --- R/get_wekeo_data.R | 180 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 180 insertions(+) create mode 100644 R/get_wekeo_data.R diff --git a/R/get_wekeo_data.R b/R/get_wekeo_data.R new file mode 100644 index 0000000..6656ae9 --- /dev/null +++ b/R/get_wekeo_data.R @@ -0,0 +1,180 @@ +#' Get Wekeo data using Earthkit +#' +#' @author Martijn Bollen & Sander Devisscher +#' +#' This function retrieves data from the Wekeo service using the Earthkit library. +#' +#' @param dataset_id The dataset identifier. +#' @param api_request The API request in JSON format. +#' @param bbox A bounding box for the data request (optional). +#' @param productType The type of product to request (optional). +#' @param resolution The resolution of the data (optional). +#' @param startdate The start date for the data request (optional). +#' @param enddate The end date for the data request (optional). +#' @param itemsPerPage The number of items per page (default is 200). +#' @param startIndex The starting index for pagination (default is 0). +#' +#' @details +#' This function retrieves data from the [Wekeo service](https://wekeo.copernicus.eu/) using the Earthkit library. +#' It constructs an API request based on the provided parameters and downloads the data. +#' This function is a wrapper around the `earthkit_download` from python function. +#' To get this function to work, you need to have the `earthkit` python package installed +#' which depends on the `Microsoft C++ Build Tools`. +#' This toolkit is required to compile the `earthkit` package. +#' To install the toolkit please place ict helpdesk call at `ict.helpdesk@inbo.be` +#' +#' When no API request is provided the function constructs a custom API request. +#' When the API request is provided, it will be used directly, ignoring the other parameters (except dataset_id). +#' +#' @return A terra SpatRaster object containing the requested data. +#' +#' @examples +#' \dontrun{ +#' +#' # Example usage with a custom API request +#' api_request <- '{ +#' "dataset_id": "EO:CLMS:DAT:CLMS_GLOBAL_NDVI_300M_V1_10DAILY_NETCDF", +#' "productType": "NDVI300", +#' "resolution": "300", +#' "startdate": "2021-01-01T00:00:00.000Z", +#' "enddate": "2021-01-29T23:59:59.999Z", +#' "itemsPerPage": 200, +#' "startIndex": 0 +#' }' +#' +#' Call the function with the custom API request +#' data <- get_wekeo_data( +#' dataset_id = "EO:CLMS:DAT:CLMS_GLOBAL_NDVI_300M_V1_10DAILY_NETCDF", +#' api_request = api_request +#' ) +#' } + + +get_wekeo_data <- function(dataset_id, + api_request, + bbox = NULL, + productType, + resolution, + startdate, + enddate, + itemsPerPage = 200, + startIndex = 0){ + + # Check if dataset_id is provided + if(missing(dataset_id)) { + stop("dataset_id is required") + } + + # Check if api_request is provided or construct it + if(!missing(api_request)){ + cat(paste0("Using provided API request:\n", api_request, "\n")) + }else{ + cat("No API request provided, constructing default request.\n") + + # Construct the API request + # dataset_id is mandatory + # productType, resolution, startdate, enddate and bbox are optional + # itemsPerPage and startIndex are used for pagination + # The base API request exists of dataset_id + api_request <- paste0('{ + "dataset_id": "', dataset_id, '"') + + # Add optional parameters to the API request + if(!missing(productType)) { + api_request <- paste0(api_request, ', "productType": "', productType, '"') + } + + if(!missing(resolution)) { + api_request <- paste0(api_request, ', "resolution": "', resolution, '"') + } + + if(!missing(startdate)) { + api_request <- paste0(api_request, ', "startdate": "', startdate, '"') + } + + if(!missing(enddate)) { + api_request <- paste0(api_request, ', "enddate": "', enddate, '"') + } + + if(!missing(bbox)) { + bbox_str <- paste(bbox, collapse = ",") + api_request <- paste0(api_request, ', "bbox": [', bbox_str, ']') + } + + # Add pagination parameters + # These should allways be added to the API request + api_request <- paste0(api_request, ', + "itemsPerPage": ', itemsPerPage, ', + "startIndex": ', startIndex, ' + }') + } + + # Download the data using the earthkit_download function + r <- earthkit_download( + dataset_id = dataset_id, + api_request = api_request + ) |> terra::rast() +} + + +#' Wrapper function for ekd.from_source +#' @param source The data source, e.g., "wekeo". +#' @param dataset_id The dataset identifier. +#' @param api_request The API request in JSON format. +#' @param cache_dir Directory to store cached files. +#' @return The path to the downloaded file. + +earthkit_download <- function( + source = "wekeo", + dataset_id, + api_request, + cache_dir = file.path(Sys.getenv("USERPROFILE"), ".earthkit_cache") +) { + #create the cache directory if it doesn't exist + dir.create(cache_dir, recursive = TRUE, showWarnings = FALSE) + + # create a virtual environment if it doesn't exist + if (!reticulate::virtualenv_exists("myenv")) { + message("Creating virtual environment 'myenv'...") + reticulate::virtualenv_create(envname = "myenv") + }else{ + cat("Virtual environment 'myenv' already exists.\n") + } + + # Install earthkit data in the virtual environment ifnot allready installed + if (!reticulate::py_module_available("earthkit")) { + message("Installing earthkit in the virtual environment 'myenv'...") + reticulate::virtualenv_install(packages = "earthkit", envname = "myenv") + } + + # Use the virtual environment + reticulate::use_virtualenv("myenv", required = TRUE) + + + pyconfig <- reticulate::py_config() + + # Install earthkit.data if not already installed + reticulate::py_install(packages = "earthkit", + envname = pyconfig$python) + + # Create a temporary variable name to capture the path + reticulate::py_run_string("ekd_path_out = None") + + python_code <- paste( + "import earthkit.data as ekd", + "import warnings", + "warnings.filterwarnings('ignore')", + sprintf("ekd.settings.set({'cache-policy': 'user', 'user-cache-directory': r'%s'})", cache_dir), + "print('πŸ“ Using Earthkit cache at:', ekd.cache.directory())", + sprintf("request = %s", api_request), + sprintf("ds = ekd.from_source('%s', '%s', request=request)", source, dataset_id), + "path = ds.path", # cached file path (usually a .nc or .grib file) + "ekd_path_out = path", + sep = "\n" + ) + + reticulate::py_run_string(python_code) + + # Return the cached file path back to R + return(py$ekd_path_out) +} From 99297c2fc368d2411fb0b22d807f9e86c55c7322 Mon Sep 17 00:00:00 2001 From: Sander Devisscher Date: Mon, 23 Jun 2025 15:59:41 +0200 Subject: [PATCH 02/28] Create get_wekeo_data.Rd #111 --- man/get_wekeo_data.Rd | 81 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 81 insertions(+) create mode 100644 man/get_wekeo_data.Rd diff --git a/man/get_wekeo_data.Rd b/man/get_wekeo_data.Rd new file mode 100644 index 0000000..55534f9 --- /dev/null +++ b/man/get_wekeo_data.Rd @@ -0,0 +1,81 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/get_wekeo_data.R +\name{get_wekeo_data} +\alias{get_wekeo_data} +\title{Get Wekeo data using Earthkit} +\usage{ +get_wekeo_data( + dataset_id, + api_request, + bbox = NULL, + productType, + resolution, + startdate, + enddate, + itemsPerPage = 200, + startIndex = 0 +) +} +\arguments{ +\item{dataset_id}{The dataset identifier.} + +\item{api_request}{The API request in JSON format.} + +\item{bbox}{A bounding box for the data request (optional).} + +\item{productType}{The type of product to request (optional).} + +\item{resolution}{The resolution of the data (optional).} + +\item{startdate}{The start date for the data request (optional).} + +\item{enddate}{The end date for the data request (optional).} + +\item{itemsPerPage}{The number of items per page (default is 200).} + +\item{startIndex}{The starting index for pagination (default is 0).} +} +\value{ +A terra SpatRaster object containing the requested data. +} +\description{ +Get Wekeo data using Earthkit +} +\details{ +This function retrieves data from the \href{https://wekeo.copernicus.eu/}{Wekeo service} using the Earthkit library. +It constructs an API request based on the provided parameters and downloads the data. +This function is a wrapper around the \code{earthkit_download} from python function. +To get this function to work, you need to have the \code{earthkit} python package installed +which depends on the \verb{Microsoft C++ Build Tools}. +This toolkit is required to compile the \code{earthkit} package. +To install the toolkit please place ict helpdesk call at \code{ict.helpdesk@inbo.be} + +When no API request is provided the function constructs a custom API request. +When the API request is provided, it will be used directly, ignoring the other parameters (except dataset_id). +} +\examples{ +\dontrun{ + +# Example usage with a custom API request +api_request <- '{ +"dataset_id": "EO:CLMS:DAT:CLMS_GLOBAL_NDVI_300M_V1_10DAILY_NETCDF", +"productType": "NDVI300", +"resolution": "300", +"startdate": "2021-01-01T00:00:00.000Z", +"enddate": "2021-01-29T23:59:59.999Z", +"itemsPerPage": 200, +"startIndex": 0 +}' + +Call the function with the custom API request +data <- get_wekeo_data( + dataset_id = "EO:CLMS:DAT:CLMS_GLOBAL_NDVI_300M_V1_10DAILY_NETCDF", + api_request = api_request + ) +} +} +\author{ +Martijn Bollen & Sander Devisscher + +This function retrieves data from the Wekeo service using the Earthkit library. +} From 7149b2cbec884a7a08d2a42d0c1e13d8be80bab4 Mon Sep 17 00:00:00 2001 From: Sander Devisscher Date: Mon, 23 Jun 2025 16:10:00 +0200 Subject: [PATCH 03/28] remove non ascii #111 --- R/get_wekeo_data.R | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/R/get_wekeo_data.R b/R/get_wekeo_data.R index 6656ae9..750ef38 100644 --- a/R/get_wekeo_data.R +++ b/R/get_wekeo_data.R @@ -27,6 +27,9 @@ #' When the API request is provided, it will be used directly, ignoring the other parameters (except dataset_id). #' #' @return A terra SpatRaster object containing the requested data. +#' @export +#' +#' @family download #' #' @examples #' \dontrun{ @@ -165,7 +168,7 @@ earthkit_download <- function( "import warnings", "warnings.filterwarnings('ignore')", sprintf("ekd.settings.set({'cache-policy': 'user', 'user-cache-directory': r'%s'})", cache_dir), - "print('πŸ“ Using Earthkit cache at:', ekd.cache.directory())", + "print('Using Earthkit cache at:', ekd.cache.directory())", sprintf("request = %s", api_request), sprintf("ds = ekd.from_source('%s', '%s', request=request)", source, dataset_id), "path = ds.path", # cached file path (usually a .nc or .grib file) From b72612b09e62b98825ea9365dca7a38ba8a36305 Mon Sep 17 00:00:00 2001 From: Sander Devisscher Date: Mon, 23 Jun 2025 16:10:08 +0200 Subject: [PATCH 04/28] Update NAMESPACE #111 --- NAMESPACE | 1 + 1 file changed, 1 insertion(+) diff --git a/NAMESPACE b/NAMESPACE index 3ab9c44..c70c55b 100644 --- a/NAMESPACE +++ b/NAMESPACE @@ -13,6 +13,7 @@ export(connect_to_bucket) export(download_dep_media) export(download_gdrive_if_missing) export(download_seq_media) +export(get_wekeo_data) export(install_sp) export(label_converter) export(label_selecter) From 8c18dbc7a33923234e8408e9404f37e5b039c048 Mon Sep 17 00:00:00 2001 From: Sander Devisscher Date: Mon, 23 Jun 2025 16:10:14 +0200 Subject: [PATCH 05/28] Update get_wekeo_data.Rd #111 --- man/get_wekeo_data.Rd | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/man/get_wekeo_data.Rd b/man/get_wekeo_data.Rd index 55534f9..4a4df10 100644 --- a/man/get_wekeo_data.Rd +++ b/man/get_wekeo_data.Rd @@ -74,8 +74,15 @@ data <- get_wekeo_data( ) } } +\seealso{ +Other download: +\code{\link{download_dep_media}()}, +\code{\link{download_gdrive_if_missing}()}, +\code{\link{download_seq_media}()} +} \author{ Martijn Bollen & Sander Devisscher This function retrieves data from the Wekeo service using the Earthkit library. } +\concept{download} From e4f5e8d0f79ad682bebb84ce5b4aa8376a4dfdb8 Mon Sep 17 00:00:00 2001 From: Sander Devisscher Date: Mon, 23 Jun 2025 16:10:22 +0200 Subject: [PATCH 06/28] Create earthkit_download.Rd #111 --- man/earthkit_download.Rd | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) create mode 100644 man/earthkit_download.Rd diff --git a/man/earthkit_download.Rd b/man/earthkit_download.Rd new file mode 100644 index 0000000..f40066f --- /dev/null +++ b/man/earthkit_download.Rd @@ -0,0 +1,28 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/get_wekeo_data.R +\name{earthkit_download} +\alias{earthkit_download} +\title{Wrapper function for ekd.from_source} +\usage{ +earthkit_download( + source = "wekeo", + dataset_id, + api_request, + cache_dir = file.path(Sys.getenv("USERPROFILE"), ".earthkit_cache") +) +} +\arguments{ +\item{source}{The data source, e.g., "wekeo".} + +\item{dataset_id}{The dataset identifier.} + +\item{api_request}{The API request in JSON format.} + +\item{cache_dir}{Directory to store cached files.} +} +\value{ +The path to the downloaded file. +} +\description{ +Wrapper function for ekd.from_source +} From 5251ed99f71d9a30f255fb0a594a0f55ebbd2382 Mon Sep 17 00:00:00 2001 From: Sander Devisscher Date: Mon, 23 Jun 2025 16:10:30 +0200 Subject: [PATCH 07/28] Update download_seq_media.Rd #111 --- man/download_seq_media.Rd | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/man/download_seq_media.Rd b/man/download_seq_media.Rd index b0e4531..ae81848 100644 --- a/man/download_seq_media.Rd +++ b/man/download_seq_media.Rd @@ -43,7 +43,8 @@ download_seq_media(dataset = drg, \seealso{ Other download: \code{\link{download_dep_media}()}, -\code{\link{download_gdrive_if_missing}()} +\code{\link{download_gdrive_if_missing}()}, +\code{\link{get_wekeo_data}()} } \author{ Lynn Pallemaerts From 3c08a5881deddcef0dfd8f3d8aed4186b781f2e1 Mon Sep 17 00:00:00 2001 From: Sander Devisscher Date: Mon, 23 Jun 2025 16:10:37 +0200 Subject: [PATCH 08/28] Update download_gdrive_if_missing.Rd #111 --- man/download_gdrive_if_missing.Rd | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/man/download_gdrive_if_missing.Rd b/man/download_gdrive_if_missing.Rd index cec862d..8a89d79 100644 --- a/man/download_gdrive_if_missing.Rd +++ b/man/download_gdrive_if_missing.Rd @@ -63,7 +63,8 @@ download_gdrive_if_missing(gfileID = "1FX8DDyREKMH1M3iW9ijWjVjO_tBH8PXi", \seealso{ Other download: \code{\link{download_dep_media}()}, -\code{\link{download_seq_media}()} +\code{\link{download_seq_media}()}, +\code{\link{get_wekeo_data}()} } \author{ Sander Devisscher From dab50461391e6c2c54b41192a6a84af31a37b49c Mon Sep 17 00:00:00 2001 From: Sander Devisscher Date: Mon, 23 Jun 2025 16:10:44 +0200 Subject: [PATCH 09/28] Update download_dep_media.Rd #111 --- man/download_dep_media.Rd | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/man/download_dep_media.Rd b/man/download_dep_media.Rd index 5744005..355db95 100644 --- a/man/download_dep_media.Rd +++ b/man/download_dep_media.Rd @@ -63,7 +63,8 @@ download_dep_media(dataset = drg, \seealso{ Other download: \code{\link{download_gdrive_if_missing}()}, -\code{\link{download_seq_media}()} +\code{\link{download_seq_media}()}, +\code{\link{get_wekeo_data}()} } \author{ Lynn Pallemaerts From ecdc8faf3fd00c6d45293c57d373d70b0b23371c Mon Sep 17 00:00:00 2001 From: Sander Devisscher Date: Mon, 23 Jun 2025 16:10:50 +0200 Subject: [PATCH 10/28] Update DESCRIPTION #111 --- DESCRIPTION | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/DESCRIPTION b/DESCRIPTION index bbb645d..9e81585 100644 --- a/DESCRIPTION +++ b/DESCRIPTION @@ -39,4 +39,6 @@ Imports: osmdata (>= 0.2.5), readr (>= 2.1.5), tcltk, - aws.s3 (>= 0.3.21) + aws.s3 (>= 0.3.21), + reticulate (>= 1.42.0), + terra (>= 1.8.50) From 66c34db98be4d048782e1bde9dae93ed7fbdc48a Mon Sep 17 00:00:00 2001 From: Sander Devisscher Date: Mon, 23 Jun 2025 16:36:07 +0200 Subject: [PATCH 11/28] add py_env variable #111 --- R/get_wekeo_data.R | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/R/get_wekeo_data.R b/R/get_wekeo_data.R index 750ef38..c7b47cc 100644 --- a/R/get_wekeo_data.R +++ b/R/get_wekeo_data.R @@ -13,6 +13,7 @@ #' @param enddate The end date for the data request (optional). #' @param itemsPerPage The number of items per page (default is 200). #' @param startIndex The starting index for pagination (default is 0). +#' @param py_env The name of the Python virtual environment to use (default is "myenv"). #' #' @details #' This function retrieves data from the [Wekeo service](https://wekeo.copernicus.eu/) using the Earthkit library. @@ -61,7 +62,8 @@ get_wekeo_data <- function(dataset_id, startdate, enddate, itemsPerPage = 200, - startIndex = 0){ + startIndex = 0, + py_env = "myenv") { # Check if dataset_id is provided if(missing(dataset_id)) { @@ -125,41 +127,39 @@ get_wekeo_data <- function(dataset_id, #' @param dataset_id The dataset identifier. #' @param api_request The API request in JSON format. #' @param cache_dir Directory to store cached files. +#' @param py_env The name of the Python virtual environment to use (default is "myenv"). #' @return The path to the downloaded file. earthkit_download <- function( source = "wekeo", dataset_id, api_request, - cache_dir = file.path(Sys.getenv("USERPROFILE"), ".earthkit_cache") + cache_dir = file.path(Sys.getenv("USERPROFILE"), ".earthkit_cache"), + py_env = "myenv" ) { #create the cache directory if it doesn't exist dir.create(cache_dir, recursive = TRUE, showWarnings = FALSE) # create a virtual environment if it doesn't exist - if (!reticulate::virtualenv_exists("myenv")) { - message("Creating virtual environment 'myenv'...") - reticulate::virtualenv_create(envname = "myenv") + if (!reticulate::virtualenv_exists(py_env)) { + message(paste0("Creating virtual environment '", py_env, "'...")) + reticulate::virtualenv_create(envname = py_env) }else{ - cat("Virtual environment 'myenv' already exists.\n") + cat(paste0("Virtual environment '", py_env, "' already exists.\n")) } # Install earthkit data in the virtual environment ifnot allready installed if (!reticulate::py_module_available("earthkit")) { - message("Installing earthkit in the virtual environment 'myenv'...") - reticulate::virtualenv_install(packages = "earthkit", envname = "myenv") + message(paste0("Installing earthkit in the virtual environment '", py_env, "'...")) + reticulate::virtualenv_install(packages = "earthkit", envname = py_env) } # Use the virtual environment - reticulate::use_virtualenv("myenv", required = TRUE) + reticulate::use_virtualenv(py_env, required = TRUE) pyconfig <- reticulate::py_config() - # Install earthkit.data if not already installed - reticulate::py_install(packages = "earthkit", - envname = pyconfig$python) - # Create a temporary variable name to capture the path reticulate::py_run_string("ekd_path_out = None") From aff2530de3fc08fcdd7c7b38d47ed73915363e06 Mon Sep 17 00:00:00 2001 From: Sander Devisscher Date: Mon, 23 Jun 2025 16:47:53 +0200 Subject: [PATCH 12/28] Update earthkit_download.Rd #111 --- man/earthkit_download.Rd | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/man/earthkit_download.Rd b/man/earthkit_download.Rd index f40066f..d36ee79 100644 --- a/man/earthkit_download.Rd +++ b/man/earthkit_download.Rd @@ -8,7 +8,8 @@ earthkit_download( source = "wekeo", dataset_id, api_request, - cache_dir = file.path(Sys.getenv("USERPROFILE"), ".earthkit_cache") + cache_dir = file.path(Sys.getenv("USERPROFILE"), ".earthkit_cache"), + py_env = "myenv" ) } \arguments{ @@ -19,6 +20,8 @@ earthkit_download( \item{api_request}{The API request in JSON format.} \item{cache_dir}{Directory to store cached files.} + +\item{py_env}{The name of the Python virtual environment to use (default is "myenv").} } \value{ The path to the downloaded file. From c6a60e76a6a41321a53113288d801bb8993a277c Mon Sep 17 00:00:00 2001 From: Sander Devisscher Date: Mon, 23 Jun 2025 16:47:59 +0200 Subject: [PATCH 13/28] Update get_wekeo_data.Rd #111 --- man/get_wekeo_data.Rd | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/man/get_wekeo_data.Rd b/man/get_wekeo_data.Rd index 4a4df10..0f9b705 100644 --- a/man/get_wekeo_data.Rd +++ b/man/get_wekeo_data.Rd @@ -13,7 +13,8 @@ get_wekeo_data( startdate, enddate, itemsPerPage = 200, - startIndex = 0 + startIndex = 0, + py_env = "myenv" ) } \arguments{ @@ -34,6 +35,8 @@ get_wekeo_data( \item{itemsPerPage}{The number of items per page (default is 200).} \item{startIndex}{The starting index for pagination (default is 0).} + +\item{py_env}{The name of the Python virtual environment to use (default is "myenv").} } \value{ A terra SpatRaster object containing the requested data. From 385f67049fd602ca9ff5e43449a30c3fdce7f453 Mon Sep 17 00:00:00 2001 From: Sander Devisscher Date: Thu, 26 Jun 2025 16:31:07 +0200 Subject: [PATCH 14/28] Revert "add py_env variable" This reverts commit 66c34db98be4d048782e1bde9dae93ed7fbdc48a. --- R/get_wekeo_data.R | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/R/get_wekeo_data.R b/R/get_wekeo_data.R index c7b47cc..750ef38 100644 --- a/R/get_wekeo_data.R +++ b/R/get_wekeo_data.R @@ -13,7 +13,6 @@ #' @param enddate The end date for the data request (optional). #' @param itemsPerPage The number of items per page (default is 200). #' @param startIndex The starting index for pagination (default is 0). -#' @param py_env The name of the Python virtual environment to use (default is "myenv"). #' #' @details #' This function retrieves data from the [Wekeo service](https://wekeo.copernicus.eu/) using the Earthkit library. @@ -62,8 +61,7 @@ get_wekeo_data <- function(dataset_id, startdate, enddate, itemsPerPage = 200, - startIndex = 0, - py_env = "myenv") { + startIndex = 0){ # Check if dataset_id is provided if(missing(dataset_id)) { @@ -127,39 +125,41 @@ get_wekeo_data <- function(dataset_id, #' @param dataset_id The dataset identifier. #' @param api_request The API request in JSON format. #' @param cache_dir Directory to store cached files. -#' @param py_env The name of the Python virtual environment to use (default is "myenv"). #' @return The path to the downloaded file. earthkit_download <- function( source = "wekeo", dataset_id, api_request, - cache_dir = file.path(Sys.getenv("USERPROFILE"), ".earthkit_cache"), - py_env = "myenv" + cache_dir = file.path(Sys.getenv("USERPROFILE"), ".earthkit_cache") ) { #create the cache directory if it doesn't exist dir.create(cache_dir, recursive = TRUE, showWarnings = FALSE) # create a virtual environment if it doesn't exist - if (!reticulate::virtualenv_exists(py_env)) { - message(paste0("Creating virtual environment '", py_env, "'...")) - reticulate::virtualenv_create(envname = py_env) + if (!reticulate::virtualenv_exists("myenv")) { + message("Creating virtual environment 'myenv'...") + reticulate::virtualenv_create(envname = "myenv") }else{ - cat(paste0("Virtual environment '", py_env, "' already exists.\n")) + cat("Virtual environment 'myenv' already exists.\n") } # Install earthkit data in the virtual environment ifnot allready installed if (!reticulate::py_module_available("earthkit")) { - message(paste0("Installing earthkit in the virtual environment '", py_env, "'...")) - reticulate::virtualenv_install(packages = "earthkit", envname = py_env) + message("Installing earthkit in the virtual environment 'myenv'...") + reticulate::virtualenv_install(packages = "earthkit", envname = "myenv") } # Use the virtual environment - reticulate::use_virtualenv(py_env, required = TRUE) + reticulate::use_virtualenv("myenv", required = TRUE) pyconfig <- reticulate::py_config() + # Install earthkit.data if not already installed + reticulate::py_install(packages = "earthkit", + envname = pyconfig$python) + # Create a temporary variable name to capture the path reticulate::py_run_string("ekd_path_out = None") From 6edff016e96dbd99788f0d1eccc054f6dcc0300b Mon Sep 17 00:00:00 2001 From: Sander Devisscher Date: Thu, 26 Jun 2025 16:36:11 +0200 Subject: [PATCH 15/28] remove python setup #111 @martijn-bollen het is blijkbaar best practice om deze setup buiten de functie te doen. --- R/get_wekeo_data.R | 24 ------------------------ 1 file changed, 24 deletions(-) diff --git a/R/get_wekeo_data.R b/R/get_wekeo_data.R index 750ef38..a3fce5c 100644 --- a/R/get_wekeo_data.R +++ b/R/get_wekeo_data.R @@ -136,30 +136,6 @@ earthkit_download <- function( #create the cache directory if it doesn't exist dir.create(cache_dir, recursive = TRUE, showWarnings = FALSE) - # create a virtual environment if it doesn't exist - if (!reticulate::virtualenv_exists("myenv")) { - message("Creating virtual environment 'myenv'...") - reticulate::virtualenv_create(envname = "myenv") - }else{ - cat("Virtual environment 'myenv' already exists.\n") - } - - # Install earthkit data in the virtual environment ifnot allready installed - if (!reticulate::py_module_available("earthkit")) { - message("Installing earthkit in the virtual environment 'myenv'...") - reticulate::virtualenv_install(packages = "earthkit", envname = "myenv") - } - - # Use the virtual environment - reticulate::use_virtualenv("myenv", required = TRUE) - - - pyconfig <- reticulate::py_config() - - # Install earthkit.data if not already installed - reticulate::py_install(packages = "earthkit", - envname = pyconfig$python) - # Create a temporary variable name to capture the path reticulate::py_run_string("ekd_path_out = None") From a60c4f46c1af7d8c750fb333e65df0e129642e07 Mon Sep 17 00:00:00 2001 From: Sander Devisscher Date: Thu, 26 Jun 2025 16:36:28 +0200 Subject: [PATCH 16/28] update rd's #112 --- man/earthkit_download.Rd | 5 +---- man/get_wekeo_data.Rd | 5 +---- 2 files changed, 2 insertions(+), 8 deletions(-) diff --git a/man/earthkit_download.Rd b/man/earthkit_download.Rd index d36ee79..f40066f 100644 --- a/man/earthkit_download.Rd +++ b/man/earthkit_download.Rd @@ -8,8 +8,7 @@ earthkit_download( source = "wekeo", dataset_id, api_request, - cache_dir = file.path(Sys.getenv("USERPROFILE"), ".earthkit_cache"), - py_env = "myenv" + cache_dir = file.path(Sys.getenv("USERPROFILE"), ".earthkit_cache") ) } \arguments{ @@ -20,8 +19,6 @@ earthkit_download( \item{api_request}{The API request in JSON format.} \item{cache_dir}{Directory to store cached files.} - -\item{py_env}{The name of the Python virtual environment to use (default is "myenv").} } \value{ The path to the downloaded file. diff --git a/man/get_wekeo_data.Rd b/man/get_wekeo_data.Rd index 0f9b705..4a4df10 100644 --- a/man/get_wekeo_data.Rd +++ b/man/get_wekeo_data.Rd @@ -13,8 +13,7 @@ get_wekeo_data( startdate, enddate, itemsPerPage = 200, - startIndex = 0, - py_env = "myenv" + startIndex = 0 ) } \arguments{ @@ -35,8 +34,6 @@ get_wekeo_data( \item{itemsPerPage}{The number of items per page (default is 200).} \item{startIndex}{The starting index for pagination (default is 0).} - -\item{py_env}{The name of the Python virtual environment to use (default is "myenv").} } \value{ A terra SpatRaster object containing the requested data. From 350cab90645671af6ac60de9c5f2a370446e272f Mon Sep 17 00:00:00 2001 From: Sander Devisscher Date: Thu, 26 Jun 2025 18:03:03 +0200 Subject: [PATCH 17/28] add example & authentication #111 --- R/get_wekeo_data.R | 75 +++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 71 insertions(+), 4 deletions(-) diff --git a/R/get_wekeo_data.R b/R/get_wekeo_data.R index a3fce5c..5d33d2a 100644 --- a/R/get_wekeo_data.R +++ b/R/get_wekeo_data.R @@ -34,22 +34,46 @@ #' @examples #' \dontrun{ #' +#' # Set up the virtual environment and install required python packages +#' py_env <- "myenv" # Change this to your desired virtual environment name +#' # 0. Create the virtualenv if needed +#' if (!py_env %in% reticulate::virtualenv_list()) { +#' reticulate::virtualenv_create(py_env) +#' } +#' +#' # 1. Activate the environment for the session +#' reticulate::use_virtualenv(py_env, required = TRUE) +#' +#' # 2. Install Python packages if not already installed +#' if (!reticulate::py_module_available("hda")) { +#' message("Installing Python module 'hda'...") +#' reticulate::virtualenv_install(py_env,"hda") +#' } +#' if (!reticulate::py_module_available("earthkit")) { +#' message("Installing Python module 'earthkit'...") +#' reticulate::virtualenv_install(py_env,"earthkit") +#' } +#' #' # Example usage with a custom API request #' api_request <- '{ #' "dataset_id": "EO:CLMS:DAT:CLMS_GLOBAL_NDVI_300M_V1_10DAILY_NETCDF", -#' "productType": "NDVI300", #' "resolution": "300", +#' "bbox": [ +#' 1.6063443461063671, +#' 48.05229722213296, +#' 8.35299059975311, +#' 51.7736550957488 +#' ], #' "startdate": "2021-01-01T00:00:00.000Z", -#' "enddate": "2021-01-29T23:59:59.999Z", +#' "enddate": "2021-01-01T23:59:59.999Z", #' "itemsPerPage": 200, #' "startIndex": 0 #' }' #' -#' Call the function with the custom API request #' data <- get_wekeo_data( #' dataset_id = "EO:CLMS:DAT:CLMS_GLOBAL_NDVI_300M_V1_10DAILY_NETCDF", #' api_request = api_request -#' ) +#' ) #' } @@ -136,6 +160,9 @@ earthkit_download <- function( #create the cache directory if it doesn't exist dir.create(cache_dir, recursive = TRUE, showWarnings = FALSE) + # setup HDA credentials + hda_client <- setup_hda_credentials() + # Create a temporary variable name to capture the path reticulate::py_run_string("ekd_path_out = None") @@ -157,3 +184,43 @@ earthkit_download <- function( # Return the cached file path back to R return(py$ekd_path_out) } + +#' Setup HDA credentials for accessing Wekeo data +#' @description +#' This function sets up the HDA (Harmonized Data Access) credentials required to access Wekeo data. +#' It checks if the HDA Python module is available, installs it if necessary, +#' and prompts the user for their HDA username and password if they are not already stored in the `.hdarc` file. +#' +#' @examples +#' \dontrun{ +#' # Setup HDA credentials +#' hda_client <- setup_hda_credentials() +#' } +#' +#' @return A Python client object for accessing Wekeo data. + +setup_hda_credentials <- function() { + # Prompt in R + username <- readline("Enter your username: ") + password <- getPass::getPass("Enter your password: ") # getPass package for masked input + + # Write .hdarc file in R + reticulate::py_run_string("import pathlib; pyhome = str(pathlib.Path.home())") + py_home <- reticulate::py$pyhome + hdarc_path <- file.path(py_home, ".hdarc") + writeLines( + c( + paste0("user:", username), + paste0("password:", password) + ), + con = hdarc_path + ) + + # Now create the client in Python + reticulate::py_run_string(" +from hda import Client +hda_client = Client() + ") + return(py$hda_client) +} + From b482861c1aca12092c4fd3c38f9d7e704ed791f1 Mon Sep 17 00:00:00 2001 From: Sander Devisscher Date: Thu, 26 Jun 2025 18:05:13 +0200 Subject: [PATCH 18/28] Update DESCRIPTION #111 --- DESCRIPTION | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/DESCRIPTION b/DESCRIPTION index 9e81585..f935d8c 100644 --- a/DESCRIPTION +++ b/DESCRIPTION @@ -41,4 +41,5 @@ Imports: tcltk, aws.s3 (>= 0.3.21), reticulate (>= 1.42.0), - terra (>= 1.8.50) + terra (>= 1.8.50), + getPass (>= 0.2.4) From 1d04ede90767845c0e49822c250386ea3708c164 Mon Sep 17 00:00:00 2001 From: Sander Devisscher Date: Thu, 26 Jun 2025 18:05:19 +0200 Subject: [PATCH 19/28] Update get_wekeo_data.Rd #111 --- man/get_wekeo_data.Rd | 32 ++++++++++++++++++++++++++++---- 1 file changed, 28 insertions(+), 4 deletions(-) diff --git a/man/get_wekeo_data.Rd b/man/get_wekeo_data.Rd index 4a4df10..3ca9ef1 100644 --- a/man/get_wekeo_data.Rd +++ b/man/get_wekeo_data.Rd @@ -56,22 +56,46 @@ When the API request is provided, it will be used directly, ignoring the other p \examples{ \dontrun{ +# Set up the virtual environment and install required python packages +py_env <- "myenv" # Change this to your desired virtual environment name +# 0. Create the virtualenv if needed +if (!py_env \%in\% reticulate::virtualenv_list()) { + reticulate::virtualenv_create(py_env) +} + +# 1. Activate the environment for the session +reticulate::use_virtualenv(py_env, required = TRUE) + +# 2. Install Python packages if not already installed +if (!reticulate::py_module_available("hda")) { + message("Installing Python module 'hda'...") + reticulate::virtualenv_install(py_env,"hda") +} +if (!reticulate::py_module_available("earthkit")) { + message("Installing Python module 'earthkit'...") + reticulate::virtualenv_install(py_env,"earthkit") +} + # Example usage with a custom API request api_request <- '{ "dataset_id": "EO:CLMS:DAT:CLMS_GLOBAL_NDVI_300M_V1_10DAILY_NETCDF", -"productType": "NDVI300", "resolution": "300", +"bbox": [ + 1.6063443461063671, + 48.05229722213296, + 8.35299059975311, + 51.7736550957488 +], "startdate": "2021-01-01T00:00:00.000Z", -"enddate": "2021-01-29T23:59:59.999Z", +"enddate": "2021-01-01T23:59:59.999Z", "itemsPerPage": 200, "startIndex": 0 }' -Call the function with the custom API request data <- get_wekeo_data( dataset_id = "EO:CLMS:DAT:CLMS_GLOBAL_NDVI_300M_V1_10DAILY_NETCDF", api_request = api_request - ) +) } } \seealso{ From e3366fc6a097a1f3657e33176257c72e4283fd21 Mon Sep 17 00:00:00 2001 From: Sander Devisscher Date: Thu, 26 Jun 2025 18:05:24 +0200 Subject: [PATCH 20/28] Create setup_hda_credentials.Rd #111 --- man/setup_hda_credentials.Rd | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) create mode 100644 man/setup_hda_credentials.Rd diff --git a/man/setup_hda_credentials.Rd b/man/setup_hda_credentials.Rd new file mode 100644 index 0000000..23eaeb7 --- /dev/null +++ b/man/setup_hda_credentials.Rd @@ -0,0 +1,23 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/get_wekeo_data.R +\name{setup_hda_credentials} +\alias{setup_hda_credentials} +\title{Setup HDA credentials for accessing Wekeo data} +\usage{ +setup_hda_credentials() +} +\value{ +A Python client object for accessing Wekeo data. +} +\description{ +This function sets up the HDA (Harmonized Data Access) credentials required to access Wekeo data. +It checks if the HDA Python module is available, installs it if necessary, +and prompts the user for their HDA username and password if they are not already stored in the \code{.hdarc} file. +} +\examples{ +\dontrun{ +# Setup HDA credentials +hda_client <- setup_hda_credentials() +} + +} From 9cf9dd7180656e98d8e2b1515b50863bc4628800 Mon Sep 17 00:00:00 2001 From: Sander Devisscher Date: Thu, 26 Jun 2025 18:14:53 +0200 Subject: [PATCH 21/28] Apply suggestions from code review Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- R/get_wekeo_data.R | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/R/get_wekeo_data.R b/R/get_wekeo_data.R index 5d33d2a..e3c8828 100644 --- a/R/get_wekeo_data.R +++ b/R/get_wekeo_data.R @@ -129,7 +129,7 @@ get_wekeo_data <- function(dataset_id, } # Add pagination parameters - # These should allways be added to the API request + # These should always be added to the API request api_request <- paste0(api_request, ', "itemsPerPage": ', itemsPerPage, ', "startIndex": ', startIndex, ' From 2e274d5acf73a04e72e25a89f25e930eaf0b2210 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Thu, 26 Jun 2025 16:19:15 +0000 Subject: [PATCH 22/28] Increment version [skip ci] --- DESCRIPTION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/DESCRIPTION b/DESCRIPTION index f935d8c..39c1f1c 100644 --- a/DESCRIPTION +++ b/DESCRIPTION @@ -1,6 +1,6 @@ Package: fistools Title: Tools & data used for wildlife management & invasive species in Flanders -Version: 1.2.19 +Version: 1.2.20 Authors@R: c( person(given = "Sander", middle = "", family = "Devisscher", "sander.devisscher@inbo.be", role = c("aut", "cre"), comment = c(ORCID = "0000-0003-2015-5731")), From e1c68583007f234087ffb259b861fb9c9459c05c Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Thu, 26 Jun 2025 16:55:57 +0000 Subject: [PATCH 23/28] Build pkgdown site [skip ci] --- docs/404.html | 4 +- docs/CODE_OF_CONDUCT.html | 4 +- docs/LICENSE-text.html | 4 +- docs/LICENSE.html | 4 +- docs/authors.html | 8 +- docs/index.html | 14 +- docs/reference/CRS_extracter.html | 4 +- docs/reference/UUID_List.html | 4 +- docs/reference/apply_grtsdb.html | 4 +- docs/reference/boswachterijen.html | 4 +- .../reference/calculate_polygon_centroid.html | 4 +- docs/reference/check.html | 4 +- docs/reference/cleanup_sqlite.html | 4 +- docs/reference/col_content_compare.html | 4 +- docs/reference/colcompare.html | 4 +- docs/reference/collect_osm_features.html | 4 +- docs/reference/connect_to_bucket.html | 6 +- docs/reference/download_dep_media.html | 7 +- .../reference/download_gdrive_if_missing.html | 7 +- docs/reference/download_seq_media.html | 7 +- docs/reference/drg_example.html | 4 +- docs/reference/earthkit_download.html | 92 +++++++++ docs/reference/get_wekeo_data.html | 188 ++++++++++++++++++ docs/reference/index.html | 22 +- docs/reference/install_sp.html | 4 +- docs/reference/label_converter.html | 4 +- docs/reference/label_selecter.html | 4 +- docs/reference/lib_crs.html | 4 +- docs/reference/qd_pci1.html | 4 +- docs/reference/qd_pci2.html | 4 +- docs/reference/qd_pci2_D.html | 4 +- docs/reference/rename_ct_files.html | 4 +- docs/reference/setup_hda_credentials.html | 82 ++++++++ docs/reference/sunsetter.html | 4 +- docs/reference/sunsetter2.html | 4 +- docs/search.json | 2 +- docs/sitemap.xml | 3 + 37 files changed, 462 insertions(+), 76 deletions(-) create mode 100644 docs/reference/earthkit_download.html create mode 100644 docs/reference/get_wekeo_data.html create mode 100644 docs/reference/setup_hda_credentials.html diff --git a/docs/404.html b/docs/404.html index 4837a40..3ff45a1 100644 --- a/docs/404.html +++ b/docs/404.html @@ -20,7 +20,7 @@ fistools - 1.2.19 + 1.2.20 + + + + + +
+
+
+ +

Wrapper function for ekd.from_source

+ +
earthkit_download.Rd
+
+ +
+

Wrapper function for ekd.from_source

+
+ +
+

Usage

+ 
earthkit_download(
+  source = "wekeo",
+  dataset_id,
+  api_request,
+  cache_dir = file.path(Sys.getenv("USERPROFILE"), ".earthkit_cache")
+)
+
+ +
+

Arguments

+ + +
source
+

The data source, e.g., "wekeo".

+ + +
dataset_id
+

The dataset identifier.

+ + +
api_request
+

The API request in JSON format.

+ + +
cache_dir
+

Directory to store cached files.

+ +
+
+

Value

+

The path to the downloaded file.

+
+ +
+ + +
+

Developed by Sander Devisscher, Lynn Pallemaerts, Soria Delva, Emma Cartuyvels, Martijn Bollen.

+
+ +
+

Site built with pkgdown 2.1.3.

+
+ +
+ + + + + + + diff --git a/docs/reference/get_wekeo_data.html b/docs/reference/get_wekeo_data.html new file mode 100644 index 0000000..7e1939e --- /dev/null +++ b/docs/reference/get_wekeo_data.html @@ -0,0 +1,188 @@ + +Get Wekeo data using Earthkit β€” get_wekeo_data β€’ fistools + Skip to contents + + +
+
+
+ +

Get Wekeo data using Earthkit

+ +
get_wekeo_data.Rd
+
+ +
+

Get Wekeo data using Earthkit

+
+ +
+

Usage

+ 
get_wekeo_data(
+  dataset_id,
+  api_request,
+  bbox = NULL,
+  productType,
+  resolution,
+  startdate,
+  enddate,
+  itemsPerPage = 200,
+  startIndex = 0
+)
+
+ +
+

Arguments

+ + +
dataset_id
+

The dataset identifier.

+ + +
api_request
+

The API request in JSON format.

+ + +
bbox
+

A bounding box for the data request (optional).

+ + +
productType
+

The type of product to request (optional).

+ + +
resolution
+

The resolution of the data (optional).

+ + +
startdate
+

The start date for the data request (optional).

+ + +
enddate
+

The end date for the data request (optional).

+ + +
itemsPerPage
+

The number of items per page (default is 200).

+ + +
startIndex
+

The starting index for pagination (default is 0).

+ +
+
+

Value

+

A terra SpatRaster object containing the requested data.

+
+
+

Details

+

This function retrieves data from the Wekeo service using the Earthkit library. +It constructs an API request based on the provided parameters and downloads the data. +This function is a wrapper around the earthkit_download from python function. +To get this function to work, you need to have the earthkit python package installed +which depends on the Microsoft C++ Build Tools. +This toolkit is required to compile the earthkit package. +To install the toolkit please place ict helpdesk call at ict.helpdesk@inbo.be

+

When no API request is provided the function constructs a custom API request. +When the API request is provided, it will be used directly, ignoring the other parameters (except dataset_id).

+
+
+

See also

+

Other download: +download_dep_media(), +download_gdrive_if_missing(), +download_seq_media()

+
+
+

Author

+

Martijn Bollen & Sander Devisscher

+

This function retrieves data from the Wekeo service using the Earthkit library.

+
+ +
+

Examples

+ 
if (FALSE) { # \dontrun{
+
+# Set up the virtual environment and install required python packages
+py_env <- "myenv" # Change this to your desired virtual environment name
+# 0. Create the virtualenv if needed
+if (!py_env %in% reticulate::virtualenv_list()) {
+  reticulate::virtualenv_create(py_env)
+}
+
+# 1. Activate the environment for the session
+reticulate::use_virtualenv(py_env, required = TRUE)
+
+# 2. Install Python packages if not already installed
+if (!reticulate::py_module_available("hda")) {
+  message("Installing Python module 'hda'...")
+  reticulate::virtualenv_install(py_env,"hda")
+}
+if (!reticulate::py_module_available("earthkit")) {
+  message("Installing Python module 'earthkit'...")
+  reticulate::virtualenv_install(py_env,"earthkit")
+}
+
+# Example usage with a custom API request
+api_request <- '{
+"dataset_id": "EO:CLMS:DAT:CLMS_GLOBAL_NDVI_300M_V1_10DAILY_NETCDF",
+"resolution": "300",
+"bbox": [
+  1.6063443461063671,
+  48.05229722213296,
+  8.35299059975311,
+  51.7736550957488
+],
+"startdate": "2021-01-01T00:00:00.000Z",
+"enddate": "2021-01-01T23:59:59.999Z",
+"itemsPerPage": 200,
+"startIndex": 0
+}'
+
+data <- get_wekeo_data(
+  dataset_id = "EO:CLMS:DAT:CLMS_GLOBAL_NDVI_300M_V1_10DAILY_NETCDF",
+  api_request = api_request
+)
+} # }
+
+
+
+ + +
+

Developed by Sander Devisscher, Lynn Pallemaerts, Soria Delva, Emma Cartuyvels, Martijn Bollen.

+
+ +
+

Site built with pkgdown 2.1.3.

+
+ +
+ + + + + + + diff --git a/docs/reference/index.html b/docs/reference/index.html index 6ca1f97..1a8362c 100644 --- a/docs/reference/index.html +++ b/docs/reference/index.html @@ -7,7 +7,7 @@ fistools - 1.2.19 + 1.2.20 + + + + + +
+
+
+ +

Setup HDA credentials for accessing Wekeo data

+ +
setup_hda_credentials.Rd
+
+ +
+

This function sets up the HDA (Harmonized Data Access) credentials required to access Wekeo data. +It checks if the HDA Python module is available, installs it if necessary, +and prompts the user for their HDA username and password if they are not already stored in the .hdarc file.

+
+ +
+

Usage

+ 
setup_hda_credentials()
+
+ +
+

Value

+

A Python client object for accessing Wekeo data.

+
+ +
+

Examples

+ 
if (FALSE) { # \dontrun{
+# Setup HDA credentials
+hda_client <- setup_hda_credentials()
+} # }
+
+
+
+
+ + +
+

Developed by Sander Devisscher, Lynn Pallemaerts, Soria Delva, Emma Cartuyvels, Martijn Bollen.

+
+ +
+

Site built with pkgdown 2.1.3.

+
+ +
+ + + + + + + diff --git a/docs/reference/sunsetter.html b/docs/reference/sunsetter.html index bebcf65..f7f8070 100644 --- a/docs/reference/sunsetter.html +++ b/docs/reference/sunsetter.html @@ -9,7 +9,7 @@ fistools - 1.2.19 + 1.2.20