% Generated by roxygen2: do not edit by hand
% Please edit documentation in R/relativeOverfitting.R
\name{estimateRelativeOverfitting}
\alias{estimateRelativeOverfitting}
\title{Estimate relative overfitting.}
\usage{
estimateRelativeOverfitting(
  predish,
  measures,
  task,
  learner = NULL,
  pred.train = NULL,
  iter = 1
)
}
\arguments{
\item{predish}{(\link{ResampleDesc} | \link{ResamplePrediction} | \link{Prediction})\cr
Resampling strategy or resampling prediction or test predictions.}

\item{measures}{(\link{Measure} | list of \link{Measure})\cr
Performance measure(s) to evaluate.
Default is the default measure for the task, see here \link{getDefaultMeasure}.}

\item{task}{(\link{Task})\cr
The task.}

\item{learner}{(\link{Learner} | \code{character(1)})\cr
The learner.
If you pass a string the learner will be created via \link{makeLearner}.}

\item{pred.train}{(\link{Prediction})\cr
Training predictions. Only needed if test predictions are passed.}

\item{iter}{(\link{integer})\cr
Iteration number. Default 1, usually you don't need to specify this. Only needed if test predictions are passed.}
}
\value{
(\link{data.frame}). Relative overfitting estimate(s), named by measure(s), for each resampling iteration.
}
\description{
Estimates the relative overfitting of a model as the ratio of the difference in test and train performance to the difference of test performance in the no-information case and train performance.
In the no-information case the features carry no information with respect to the prediction. This is simulated by permuting features and predictions.
}
\details{
Currently only support for classification and regression tasks is implemented.
}
\examples{
\dontshow{ if (requireNamespace("class")) \{ }
task = makeClassifTask(data = iris, target = "Species")
rdesc = makeResampleDesc("CV", iters = 2)
estimateRelativeOverfitting(rdesc, acc, task, makeLearner("classif.knn"))
estimateRelativeOverfitting(rdesc, acc, task, makeLearner("classif.lda"))
rpred = resample("classif.knn", task, rdesc)$pred
estimateRelativeOverfitting(rpred, acc, task)
\dontshow{ \} }
}
\references{
Bradley Efron and Robert Tibshirani; Improvements on Cross-Validation: The .632+ Bootstrap Method, Journal of the American Statistical Association, Vol. 92, No. 438. (Jun., 1997), pp. 548-560.
}
\seealso{
Other performance: 
\code{\link{ConfusionMatrix}},
\code{\link{calculateConfusionMatrix}()},
\code{\link{calculateROCMeasures}()},
\code{\link{makeCostMeasure}()},
\code{\link{makeCustomResampledMeasure}()},
\code{\link{makeMeasure}()},
\code{\link{measures}},
\code{\link{performance}()},
\code{\link{setAggregation}()},
\code{\link{setMeasurePars}()}
}
\concept{performance}
