Using the atomic layers

Overview

uniml is arranged as a stack of small layers. The top-level run_workflow() function wires them together, but every layer can also be called directly. This vignette walks through the layers from bottom to top with the bundled example cohort.

Data and specification

The first step is to declare the binary outcome and the positive class. Identifier and excluded columns are kept out of the modelling matrix.

library(uniml)

data <- read.csv(
  system.file("extdata", "uniml-cvd-mortality.csv", package = "uniml")
)
set.seed(123)
data <- data[1:400, ]

spec <- uniml_spec(
  task_type = "binary",
  outcome = "died",
  positive = "yes",
  id = "id"
)

data <- as_uniml_data(data, spec)
names(data)
#>  [1] "data"         "features"     "outcome"      "outcome_name" "positive"    
#>  [6] "id"           "exclude"      "role"         "types"        "spec"        
#> [11] "warnings"     "meta"

Preprocessing

fit_preprocessor() learns imputation, scaling, and encoding parameters from the training data. apply_preprocessor() applies those stored parameters to new data without refitting.

spec <- set_preprocess(
  spec,
  impute = "median_mode",
  scale = "zscore",
  encode = "onehot"
)

preprocessor <- fit_preprocessor(data, spec = spec)
processed <- apply_preprocessor(preprocessor, data$data)

dim(processed)
#> [1] 400  62
head(names(processed), 8)
#> [1] "id"               "age"              "bmi"              "killip_class"    
#> [5] "sbp"              "dbp"              "heart_rate"       "respiratory_rate"

Use scale = "minmax" for range scaling, encode = "onehot_drop" to drop the first dummy level, and balance = "smote" or "rose" when the training set needs class balancing. Optional imputation methods include "mice", "missForest", and "missRanger".

Feature selection

Multiple selection methods can be combined. combine controls whether the methods are intersected or unioned, and remove_collinear() prunes redundant predictors by correlation or VIF.

spec <- set_feature_selection(
  spec,
  methods = c("univariate", "lasso"),
  combine = "intersect",
  p_threshold = 0.2,
  cor_threshold = 0.9
)

selection <- select_features(processed, spec)
#> Warning: from glmnet C++ code (error code -89); Convergence for 89th lambda
#> value not reached after maxit=100000 iterations; solutions for larger lambdas
#> returned
#> Warning: from glmnet C++ code (error code -95); Convergence for 95th lambda
#> value not reached after maxit=100000 iterations; solutions for larger lambdas
#> returned
#> Warning: from glmnet C++ code (error code -92); Convergence for 92th lambda
#> value not reached after maxit=100000 iterations; solutions for larger lambdas
#> returned
selection$selected_features
#> [1] "creatinine" "spo2"
head(selection$details, 4)
#>                   feature selected      p_value     method method_selected
#> univariate.1           id    FALSE 2.764950e-01 univariate           FALSE
#> univariate.2          age     TRUE 3.407849e-09 univariate            TRUE
#> univariate.3          bmi     TRUE 6.950311e-02 univariate            TRUE
#> univariate.4 killip_class     TRUE 9.237249e-06 univariate            TRUE
#>              combined_selected coefficient
#> univariate.1             FALSE          NA
#> univariate.2             FALSE          NA
#> univariate.3             FALSE          NA
#> univariate.4             FALSE          NA

Individual methods are also exported: select_univariate(), select_multivariate(), select_lasso(), select_boruta(), select_rf_importance(), and select_rfe().

Training and tuning

The selected features are turned into an mlr3 task. Learner aliases come from the specification, and tune_model() returns the best parameters together with the cross-validated performance.

features <- intersect(selection$selected_features, names(processed))
task <- make_task(processed[, c(features, "died"), drop = FALSE], spec)

spec <- set_learners(spec, "cart", tune = TRUE)
learners <- resolve_learners(spec, task = task, n_train = task$nrow)

tuned <- tune_model(
  task,
  learners[["cart"]],
  resampling = mlr3::rsmp("cv", folds = 3),
  tuner = "random_search",
  terminator = mlr3tuning::trm("evals", n_evals = 4),
  seed = 123
)

tuned$status
#> [1] "tuned"
tuned$best_params
#> $xval
#> [1] 10
#> 
#> $minsplit
#> [1] 34
#> 
#> $minbucket
#> [1] 12
#> 
#> $cp
#> [1] 0.003984816
#> 
#> $maxdepth
#> [1] 7
model <- train_model(task, tuned$learner, params = tuned$best_params)
model
#> 
#> ── <LearnerClassifRpart> (classif.rpart): Classification Tree ──────────────────
#> • Model: rpart
#> • Parameters: cp=0.003985, maxdepth=7, minbucket=12, minsplit=34, xval=10
#> • Packages: mlr3 and rpart
#> • Predict Types: response and [prob]
#> • Feature Types: logical, integer, numeric, factor, and ordered
#> • Encapsulation: none (fallback: -)
#> • Properties: importance, missings, multiclass, selected_features, twoclass,
#> and weights
#> • Other settings: use_weights = 'use', predict_raw = 'FALSE'

Metrics and confidence intervals

Predictions can be evaluated with a fixed threshold or with the threshold helpers. Bootstrap confidence intervals resample paired probabilities and labels.

prediction <- model$predict(task)
metrics <- calc_metrics(prediction, threshold = 0.5)
unlist(metrics[c("auc", "accuracy", "sensitivity", "specificity", "brier")])
#>         auc    accuracy sensitivity specificity       brier 
#>  0.67185100  0.94250000  0.35714286  0.98655914  0.05122944

ci <- calc_ci(
  prediction$prob[, "yes"],
  prediction$truth,
  method = "bootstrap",
  R = 50,
  threshold = 0.5,
  positive = "yes",
  seed = 123
)
ci[, c("metric", "mean", "lower", "upper")]
#>               metric         mean        lower        upper
#> 1                auc  0.676015499  0.600589042   0.76887399
#> 2             pr_auc  0.334062362  0.171790091   0.53385843
#> 3              brier  0.051134999  0.033380553   0.06713001
#> 4           accuracy  0.943000000  0.923062500   0.96637500
#> 5        sensitivity  0.364899576  0.214655172   0.54772727
#> 6        specificity  0.987131421  0.976525732   0.99464268
#> 7                ppv  0.681950053  0.435416667   0.87098214
#> 8                npv  0.953190082  0.935590782   0.97335557
#> 9                 f1  0.469604942  0.302812500   0.65877193
#> 10                f2  0.400094438  0.244212170   0.58727297
#> 11            youden  0.352030997  0.201178083   0.53774799
#> 12               plr 34.437683672 13.971995074  98.64303519
#> 13               nlr  0.643367873  0.456841403   0.79607358
#> 14               dor 57.436423352 18.061997930 150.44831933
#> 15 balanced_accuracy  0.676015499  0.600589042   0.76887399
#> 16             kappa  0.442956132  0.274255119   0.63586410
#> 17       calib_slope  1.038547493  0.780269668   1.35204953
#> 18   calib_intercept  0.093297117 -0.800165103   0.96364277
#> 19       calib_large  0.001415334 -0.407688637   0.31052877
#> 20               ece  0.010995931  0.001055411   0.02493041
#> 21               mce  0.082086563  0.004883519   0.24227941

The evaluation layer also covers calibration, DeLong and Hosmer-Lemeshow tests, NRI/IDI, decision-curve analysis, and threshold selection.

calibrated <- calibrate_probabilities(
  prediction$prob[, "yes"],
  labels = prediction$truth,
  method = "platt",
  positive = "yes"
)
calibrated$method
#> [1] "platt"
calibrated$metrics_before
#>        brier          ece          mce calib_slope calib_intercept calib_r2   n
#> 1 0.05122944 6.678685e-18 6.938894e-18           1    2.220446e-17        1 400
calibrated$metrics_after
#>        brier        ece       mce calib_slope calib_intercept  calib_r2   n
#> 1 0.05304062 0.01267565 0.4166667   0.9301357     0.004850608 0.7369441 400

Plots

The plotting functions accept a single prediction, a collection of predictions, or a uniml_result.

plot_roc(prediction)
ROC curve of a single mlr3 prediction.

ROC curve of a single mlr3 prediction.

plot_confusion(prediction, threshold = 0.5)
Confusion matrix at a fixed threshold.

Confusion matrix at a fixed threshold.

All performance plots share theme_uniml() and the palette helpers, so a comparison figure can be styled once and reused after assemble_result() or run_workflow().