From f44c18d4ba9eee3dd06c945e660391e34c2f3372 Mon Sep 17 00:00:00 2001 From: James Lamb Date: Sun, 12 Jan 2020 18:03:38 -0600 Subject: [PATCH 1/4] [R-package] clarified error messages and documentation for lgb.get.eval.result() --- R-package/R/lgb.Booster.R | 46 ++++++++--- R-package/man/lgb.get.eval.result.Rd | 21 +++-- R-package/tests/testthat/test_lgb.Booster.R | 90 +++++++++++++++++++++ 3 files changed, 143 insertions(+), 14 deletions(-) create mode 100644 R-package/tests/testthat/test_lgb.Booster.R diff --git a/R-package/R/lgb.Booster.R b/R-package/R/lgb.Booster.R index 33c5755fab83..182354106825 100644 --- a/R-package/R/lgb.Booster.R +++ b/R-package/R/lgb.Booster.R @@ -884,16 +884,19 @@ lgb.dump <- function(booster, num_iteration = NULL) { #' @name lgb.get.eval.result #' @title Get record evaluation result from booster -#' @description Get record evaluation result from booster -#' @param booster Object of class \code{lgb.Booster} -#' @param data_name name of dataset -#' @param eval_name name of evaluation -#' @param iters iterations, NULL will return all +#' @description Given a (\code{lgb.Booster}), return evaluation results for a +#' particular metric on a particular dataset. +#' @param booster Object of class \code{\link{lgb.Booster}} +#' @param data_name Name of the dataset to return evaluation results for. +#' @param eval_name Name of the evaluation metric to return results for. +#' @param iters An integer vector of iterations you want to get evaluation results for. If NULL +#' (the default), evaluation results for all iterations will be returned. #' @param is_err TRUE will return evaluation error instead #' #' @return vector of evaluation result #' #' @examples +#' # train a regression model #' library(lightgbm) #' data(agaricus.train, package = "lightgbm") #' train <- agaricus.train @@ -912,6 +915,14 @@ lgb.dump <- function(booster, num_iteration = NULL) { #' , learning_rate = 1.0 #' , early_stopping_rounds = 5L #' ) +#' +#' # Examine valid data_name values +#' print(setdiff(names(model$record_evals), "start_iter")) +#' +#' # Examine valid eval_name values for dataset "test" +#' print(names(model$record_evals[["test"]])) +#' +#' # Get L2 values for "test" dataset #' lgb.get.eval.result(model, "test", "l2") #' @export lgb.get.eval.result <- function(booster, data_name, eval_name, iters = NULL, is_err = FALSE) { @@ -926,13 +937,30 @@ lgb.get.eval.result <- function(booster, data_name, eval_name, iters = NULL, is_ stop("lgb.get.eval.result: data_name and eval_name should be characters") } - # Check if recorded evaluation is existing - if (is.null(booster$record_evals[[data_name]])) { - stop("lgb.get.eval.result: wrong data name") + # NOTE: "start_iter" exists in booster$record_evals but is not a valid data_name + data_names <- setdiff(names(booster$record_evals), "start_iter") + if (!(data_name %in% data_names)) { + stop(paste0( + "lgb.get.eval.result: data_name " + , shQuote(data_name) + , " not found. Only the followwing datasets exist in record evals: [" + , paste(data_names, collapse = ",") + , "]" + )) } # Check if evaluation result is existing - if (is.null(booster$record_evals[[data_name]][[eval_name]])) { + eval_names <- names(booster$record_evals[[data_name]]) + if (!(eval_name %in% eval_names)) { + stop(paste0( + "lgb.get.eval.result: eval_name " + , shQuote(eval_name) + , " not found. Only the followwing eval_names exist for dataset " + , shQuote(data_name) + , ": [" + , paste(eval_names, collapse = ",") + , "]" + )) stop("lgb.get.eval.result: wrong eval name") } diff --git a/R-package/man/lgb.get.eval.result.Rd b/R-package/man/lgb.get.eval.result.Rd index e049ff53ebb2..a03e59b7bb25 100644 --- a/R-package/man/lgb.get.eval.result.Rd +++ b/R-package/man/lgb.get.eval.result.Rd @@ -13,13 +13,14 @@ lgb.get.eval.result( ) } \arguments{ -\item{booster}{Object of class \code{lgb.Booster}} +\item{booster}{Object of class \code{\link{lgb.Booster}}} -\item{data_name}{name of dataset} +\item{data_name}{Name of the dataset to return evaluation results for.} -\item{eval_name}{name of evaluation} +\item{eval_name}{Name of the evaluation metric to return results for.} -\item{iters}{iterations, NULL will return all} +\item{iters}{An integer vector of iterations you want to get evaluation results for. If NULL +(the default), evaluation results for all iterations will be returned.} \item{is_err}{TRUE will return evaluation error instead} } @@ -27,9 +28,11 @@ lgb.get.eval.result( vector of evaluation result } \description{ -Get record evaluation result from booster +Given a (\code{lgb.Booster}), return evaluation results for a + particular metric on a particular dataset. } \examples{ +# train a regression model library(lightgbm) data(agaricus.train, package = "lightgbm") train <- agaricus.train @@ -48,5 +51,13 @@ model <- lgb.train( , learning_rate = 1.0 , early_stopping_rounds = 5L ) + +# Examine valid data_name values +print(setdiff(names(model$record_evals), "start_iter")) + +# Examine valid eval_name values for dataset "test" +print(names(model$record_evals[["test"]])) + +# Get L2 values for "test" dataset lgb.get.eval.result(model, "test", "l2") } diff --git a/R-package/tests/testthat/test_lgb.Booster.R b/R-package/tests/testthat/test_lgb.Booster.R new file mode 100644 index 000000000000..a6960b6bb45d --- /dev/null +++ b/R-package/tests/testthat/test_lgb.Booster.R @@ -0,0 +1,90 @@ +context("lgb.get.eval.result") + +test_that("lgb.get.eval.result() should throw an informative error if booster is not an lgb.Booster", { + bad_inputs <- list( + matrix(1.0:10.0, 2L, 5L) + , TRUE + , c("a", "b") + , NA + , 10L + , lgb.Dataset( + data = matrix(1.0:10.0, 2L, 5L) + , params = list() + ) + ) + for (bad_input in bad_inputs) { + expect_error({ + lgb.get.eval.result( + booster = bad_input + , data_name = "test" + , eval_name = "l2" + ) + }, regexp = "Can only use", fixed = TRUE) + } +}) + +test_that("lgb.get.eval.result() should throw an informative error for incorrect data_name", { + data(agaricus.train, package = "lightgbm") + data(agaricus.test, package = "lightgbm") + dtrain <- lgb.Dataset( + agaricus.train$data + , label = agaricus.train$label + ) + model <- lgb.train( + params = list( + objective = "regression" + , metric = "l2" + ) + , data = dtrain + , nrounds = 5L + , valids = list( + "test" = lgb.Dataset.create.valid( + dtrain + , agaricus.test$data + , label = agaricus.test$label + ) + ) + , min_data = 1L + , learning_rate = 1.0 + ) + expect_error({ + eval_results <- lgb.get.eval.result( + booster = model + , data_name = "testing" + , eval_name = "l2" + ) + }, regexp = "Only the followwing datasets exist in record evals: [test]", fixed = TRUE) +}) + +test_that("lgb.get.eval.result() should throw an informative error for incorrect eval_name", { + data(agaricus.train, package = "lightgbm") + data(agaricus.test, package = "lightgbm") + dtrain <- lgb.Dataset( + agaricus.train$data + , label = agaricus.train$label + ) + model <- lgb.train( + params = list( + objective = "regression" + , metric = "l2" + ) + , data = dtrain + , nrounds = 5L + , valids = list( + "test" = lgb.Dataset.create.valid( + dtrain + , agaricus.test$data + , label = agaricus.test$label + ) + ) + , min_data = 1L + , learning_rate = 1.0 + ) + expect_error({ + eval_results <- lgb.get.eval.result( + booster = model + , data_name = "test" + , eval_name = "l1" + ) + }, regexp = "Only the followwing eval_names exist for dataset 'test': [l2]", fixed = TRUE) +}) From 7a0b450df29c9b3ee91d08f3b5e6ebe6905ba76c Mon Sep 17 00:00:00 2001 From: James Lamb Date: Sun, 12 Jan 2020 21:56:44 -0600 Subject: [PATCH 2/4] fixed bad link --- R-package/R/lgb.Booster.R | 4 ++-- R-package/man/lgb.get.eval.result.Rd | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/R-package/R/lgb.Booster.R b/R-package/R/lgb.Booster.R index 182354106825..8005448480be 100644 --- a/R-package/R/lgb.Booster.R +++ b/R-package/R/lgb.Booster.R @@ -884,9 +884,9 @@ lgb.dump <- function(booster, num_iteration = NULL) { #' @name lgb.get.eval.result #' @title Get record evaluation result from booster -#' @description Given a (\code{lgb.Booster}), return evaluation results for a +#' @description Given a \code{lgb.Booster}, return evaluation results for a #' particular metric on a particular dataset. -#' @param booster Object of class \code{\link{lgb.Booster}} +#' @param booster Object of class \code{lgb.Booster} #' @param data_name Name of the dataset to return evaluation results for. #' @param eval_name Name of the evaluation metric to return results for. #' @param iters An integer vector of iterations you want to get evaluation results for. If NULL diff --git a/R-package/man/lgb.get.eval.result.Rd b/R-package/man/lgb.get.eval.result.Rd index a03e59b7bb25..23c59b584e79 100644 --- a/R-package/man/lgb.get.eval.result.Rd +++ b/R-package/man/lgb.get.eval.result.Rd @@ -13,7 +13,7 @@ lgb.get.eval.result( ) } \arguments{ -\item{booster}{Object of class \code{\link{lgb.Booster}}} +\item{booster}{Object of class \code{gb.Booster}} \item{data_name}{Name of the dataset to return evaluation results for.} From c8d61fb657b64bb948b7cb10175004e87fce2b39 Mon Sep 17 00:00:00 2001 From: James Lamb Date: Mon, 20 Jan 2020 23:37:43 -0600 Subject: [PATCH 3/4] Update R-package/man/lgb.get.eval.result.Rd Co-Authored-By: Nikita Titov --- R-package/man/lgb.get.eval.result.Rd | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/R-package/man/lgb.get.eval.result.Rd b/R-package/man/lgb.get.eval.result.Rd index 23c59b584e79..4a570556a248 100644 --- a/R-package/man/lgb.get.eval.result.Rd +++ b/R-package/man/lgb.get.eval.result.Rd @@ -13,7 +13,7 @@ lgb.get.eval.result( ) } \arguments{ -\item{booster}{Object of class \code{gb.Booster}} +\item{booster}{Object of class \code{lgb.Booster}} \item{data_name}{Name of the dataset to return evaluation results for.} From 0e0b1cdaa0db4d763fa93627ca008e4928501ba5 Mon Sep 17 00:00:00 2001 From: James Lamb Date: Mon, 20 Jan 2020 23:43:25 -0600 Subject: [PATCH 4/4] [R-package] added fixes from codereview back in --- R-package/R/lgb.Booster.R | 8 ++++---- R-package/man/lgb.get.eval.result.Rd | 2 +- R-package/tests/testthat/test_lgb.Booster.R | 4 ++-- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/R-package/R/lgb.Booster.R b/R-package/R/lgb.Booster.R index 8005448480be..de0bd4c72d45 100644 --- a/R-package/R/lgb.Booster.R +++ b/R-package/R/lgb.Booster.R @@ -943,8 +943,8 @@ lgb.get.eval.result <- function(booster, data_name, eval_name, iters = NULL, is_ stop(paste0( "lgb.get.eval.result: data_name " , shQuote(data_name) - , " not found. Only the followwing datasets exist in record evals: [" - , paste(data_names, collapse = ",") + , " not found. Only the following datasets exist in record evals: [" + , paste(data_names, collapse = ", ") , "]" )) } @@ -955,10 +955,10 @@ lgb.get.eval.result <- function(booster, data_name, eval_name, iters = NULL, is_ stop(paste0( "lgb.get.eval.result: eval_name " , shQuote(eval_name) - , " not found. Only the followwing eval_names exist for dataset " + , " not found. Only the following eval_names exist for dataset " , shQuote(data_name) , ": [" - , paste(eval_names, collapse = ",") + , paste(eval_names, collapse = ", ") , "]" )) stop("lgb.get.eval.result: wrong eval name") diff --git a/R-package/man/lgb.get.eval.result.Rd b/R-package/man/lgb.get.eval.result.Rd index 4a570556a248..c5473825e61a 100644 --- a/R-package/man/lgb.get.eval.result.Rd +++ b/R-package/man/lgb.get.eval.result.Rd @@ -28,7 +28,7 @@ lgb.get.eval.result( vector of evaluation result } \description{ -Given a (\code{lgb.Booster}), return evaluation results for a +Given a \code{lgb.Booster}, return evaluation results for a particular metric on a particular dataset. } \examples{ diff --git a/R-package/tests/testthat/test_lgb.Booster.R b/R-package/tests/testthat/test_lgb.Booster.R index a6960b6bb45d..a3e7a9dfc63d 100644 --- a/R-package/tests/testthat/test_lgb.Booster.R +++ b/R-package/tests/testthat/test_lgb.Booster.R @@ -53,7 +53,7 @@ test_that("lgb.get.eval.result() should throw an informative error for incorrect , data_name = "testing" , eval_name = "l2" ) - }, regexp = "Only the followwing datasets exist in record evals: [test]", fixed = TRUE) + }, regexp = "Only the following datasets exist in record evals: [test]", fixed = TRUE) }) test_that("lgb.get.eval.result() should throw an informative error for incorrect eval_name", { @@ -86,5 +86,5 @@ test_that("lgb.get.eval.result() should throw an informative error for incorrect , data_name = "test" , eval_name = "l1" ) - }, regexp = "Only the followwing eval_names exist for dataset 'test': [l2]", fixed = TRUE) + }, regexp = "Only the following eval_names exist for dataset 'test': [l2]", fixed = TRUE) })