{ "_id": "6a3d89e0a319ef941447c771", "Package": "mixedsubjectsirt", "Title": "Item Response Theory Calibration with a Mixed Subjects Design", "Version": "1.0.0", "Authors@R": "person(\"Klint\", \"Kanopka\", , \"klint.kanopka@nyu.edu\", role = c(\"aut\", \"cre\"),\ncomment = c(ORCID = \"0000-0003-3196-9538\"))", "Description": "Integrates large language model generated item responses\ninto psychometric calibration studies through a mixed-subjects\ndesign for unidimensional two-parameter and one-parameter\nlogistic item response theory models. Human pilot responses are\naugmented with model-generated responses using a\nprediction-powered inference estimator (Angelopoulos, Bates,\nFannjiang, Jordan and Zrnic (2023)\n; Angelopoulos, Duchi and Zrnic\n(2023) ) adapted to marginal\nmaximum-likelihood estimation, following the mixed-subjects\ndesign of Broska, Howes and van Loon (2025)\n. The estimator is anchored to\nthe human responses and is asymptotically unbiased for the\nhuman item parameters at any tuning weight; the weight on the\nsynthetic responses is chosen to minimize propagated\nability-score risk, down-weighting uninformative or biased\ngenerated responses. Louis-corrected sandwich standard errors,\nability scoring, cross-fitted tuning, and scale linking are\nalso provided.", "License": "MIT + file LICENSE", "Encoding": "UTF-8", "Language": "en-US", "RoxygenNote": "7.3.3", "VignetteBuilder": "knitr", "Config/testthat/edition": "3", "URL": "https://klintkanopka.com/mixedsubjectsirt/,\nhttps://github.com/klintkanopka/mixedsubjectsirt", "BugReports": "https://github.com/klintkanopka/mixedsubjectsirt/issues", "NeedsCompilation": "no", "Packaged": { "Date": "2026-06-25 20:00:39 UTC", "User": "root" }, "Author": "Klint Kanopka [aut, cre] (ORCID:\n)", "Maintainer": "Klint Kanopka ", "Repository": "https://cran.r-universe.dev", "Date/Publication": "2026-06-25 15:50:09 UTC", "RemoteUrl": "https://github.com/cran/mixedsubjectsirt", "RemoteRef": "HEAD", "RemoteSha": "2e77546e86fd98746edc454a11c8d8112e7b2aa4", "MD5sum": "489e09421b22ae9557e6ab859851637d", "_user": "cran", "_type": "src", "_file": "mixedsubjectsirt_1.0.0.tar.gz", "_fileid": "973c88075dff9047177fe8474c5cba7e0f27de2113d895311ac72b4573c3fe7a", "_filesize": 925418, "_sha256": "973c88075dff9047177fe8474c5cba7e0f27de2113d895311ac72b4573c3fe7a", "_created": "2026-06-25T20:00:39.000Z", "_published": "2026-06-25T20:04:48.164Z", "_distro": "resolute", "_jobs": [ { "job": 83526199785, "time": 214, "config": "linux-devel-x86_64", "r": "4.7.0", "check": "OK", "artifact": "7889516675" }, { "job": 83526199793, "time": 210, "config": "linux-release-x86_64", "r": "4.6.0", "check": "OK", "artifact": "7889515266" }, { "job": 83525176953, "time": 314, "config": "source", "r": "4.6.0", "check": "OK", "artifact": "7889433392" }, { "job": 83526199799, "time": 157, "config": "wasm-release", "r": "4.6.0", "check": "OK", "artifact": "7889494734" } ], "_buildurl": "https://github.com/r-universe/cran/actions/runs/28196656638", "_status": "success", "_host": "GitHub-Actions", "_upstream": "https://github.com/cran/mixedsubjectsirt", "_commit": { "id": "2e77546e86fd98746edc454a11c8d8112e7b2aa4", "author": "Klint Kanopka ", "committer": "cran-robot ", "message": "version 1.0.0\n", "time": 1782402609 }, "_maintainer": { "name": "Klint Kanopka", "email": "klint.kanopka@nyu.edu", "login": "klintkanopka", "bluesky": "@klint.bsky.social", "orcid": "0000-0003-3196-9538", "twitter": "@KlintKanopka", "description": "Assitant Professor of Applied Statistics at NYU ASH. Interested in measurement, psychometrics, networks, and machine learning.", "uuid": 28941934 }, "_registered": true, "_dependencies": [ { "package": "mirt", "role": "Imports" }, { "package": "rmutil", "role": "Imports" }, { "package": "ggplot2", "role": "Suggests" }, { "package": "knitr", "role": "Suggests" }, { "package": "rmarkdown", "role": "Suggests" }, { "package": "testthat", "version": ">= 3.0.0", "role": "Suggests" } ], "_owner": "cran", "_selfowned": false, "_usedby": 0, "_updates": [ { "week": "2026-26", "n": 1 } ], "_tags": [ { "name": "1.0.0", "date": "2026-06-25" } ], "_stars": 0, "_contributors": [ { "user": "klintkanopka", "count": 1, "uuid": 28941934 } ], "_userbio": { "uuid": 6899542, "type": "organization", "name": "cran", "followers": 611, "description": "Unofficial read-only mirror of all CRAN R packages" }, "_downloads": { "count": 0, "source": "https://cranlogs.r-pkg.org/downloads/total/last-month/mixedsubjectsirt" }, "_devurl": "https://github.com/klintkanopka/mixedsubjectsirt", "_pkgdown": "https://klintkanopka.com/mixedsubjectsirt/", "_searchresults": 0, "_rbuild": "4.6.0", "_assets": [ "extra/citation.cff", "extra/citation.html", "extra/citation.json", "extra/citation.txt", "extra/contents.json", "extra/mixedsubjectsirt.html", "extra/NEWS.html", "extra/NEWS.txt", "extra/readme.html", "extra/readme.md", "LICENSE", "manual.pdf" ], "_cranurl": false, "_releases": [ { "version": "1.0.0", "date": "2026-06-25" } ], "_exports": [ "ability_gradient", "ability_gradient_1pl", "ability_risk", "ability_risk_1pl", "diagnose_lambda_grid", "fit_1pl", "fit_2pl", "fit_mixed_subjects", "fit_mixed_subjects_1pl", "fit_mixed_subjects_from_quadrature", "fit_mixed_subjects_iterative", "fit_mixed_subjects_mml", "fit_mixed_subjects_mml_1pl", "fit_mixed_subjects_split", "link_item_parameters", "make_quadrature", "mixed_subjects_loss", "mixed_subjects_quadrature", "posterior_weights_2pl", "score_theta", "simulate_2pl", "summarize_expected_counts", "tune_lambda_ability_risk", "tune_lambda_ability_risk_1pl", "tune_lambda_ability_risk_crossfit", "tune_lambda_ability_risk_item", "tune_lambda_ppi_score", "tune_lambda_ppi_score_1pl", "tune_lambda_ppi_score_item", "vcov_mixed_subjects", "vcov_mixed_subjects_1pl", "vcov_mixed_subjects_mml" ], "_help": [ { "page": "ability_gradient", "title": "Gradient of ML ability scores with respect to item parameters", "topics": [ "ability_gradient" ] }, { "page": "ability_gradient_1pl", "title": "Gradient of ML ability scores w.r.t. 1PL item parameters", "topics": [ "ability_gradient_1pl" ] }, { "page": "ability_risk", "title": "Propagated ability risk from item-parameter uncertainty", "topics": [ "ability_risk" ] }, { "page": "ability_risk_1pl", "title": "Propagated ability risk for a 1PL fit", "topics": [ "ability_risk_1pl" ] }, { "page": "diagnose_lambda_grid", "title": "Diagnose lambda values over a grid", "topics": [ "diagnose_lambda_grid" ] }, { "page": "fit_1pl", "title": "Fit a 1PL (one-parameter logistic) model", "topics": [ "fit_1pl" ] }, { "page": "fit_2pl", "title": "Fit a unidimensional 2PL IRT model", "topics": [ "fit_2pl" ] }, { "page": "fit_mixed_subjects", "title": "Fit a mixed-subjects 2PL calibration", "topics": [ "fit_mixed_subjects" ] }, { "page": "fit_mixed_subjects_1pl", "title": "Fit a mixed-subjects 1PL calibration (frozen expected-count)", "topics": [ "fit_mixed_subjects_1pl" ] }, { "page": "fit_mixed_subjects_from_quadrature", "title": "Fit from precomputed quadrature summaries", "topics": [ "fit_mixed_subjects_from_quadrature" ] }, { "page": "fit_mixed_subjects_iterative", "title": "Fit a mixed-subjects 2PL calibration with iterative EM", "topics": [ "fit_mixed_subjects_iterative" ] }, { "page": "fit_mixed_subjects_mml", "title": "Fit a mixed-subjects 2PL calibration via marginal maximum likelihood", "topics": [ "fit_mixed_subjects_mml" ] }, { "page": "fit_mixed_subjects_mml_1pl", "title": "Fit a mixed-subjects 1PL calibration via marginal maximum likelihood", "topics": [ "fit_mixed_subjects_mml_1pl" ] }, { "page": "fit_mixed_subjects_split", "title": "Fit a split-sample mixed-subjects 2PL calibration", "topics": [ "fit_mixed_subjects_split" ] }, { "page": "link_item_parameters", "title": "Link item parameters onto a target scale", "topics": [ "link_item_parameters" ] }, { "page": "make_quadrature", "title": "Create a standard-normal Gauss-Hermite quadrature grid", "topics": [ "make_quadrature" ] }, { "page": "mixed_subjects_loss", "title": "Mixed-subjects objective function", "topics": [ "mixed_subjects_loss" ] }, { "page": "mixed_subjects_quadrature", "title": "Convert responses to quadrature form", "topics": [ "mixed_subjects_quadrature" ] }, { "page": "posterior_weights_2pl", "title": "Compute posterior quadrature weights for a 2PL model", "topics": [ "posterior_weights_2pl" ] }, { "page": "score_theta", "title": "Estimate ability scores from a 2PL calibration", "topics": [ "score_theta" ] }, { "page": "simulate_2pl", "title": "Simulate 2PL item responses", "topics": [ "simulate_2pl" ] }, { "page": "summarize_expected_counts", "title": "Summarize response data as expected quadrature counts", "topics": [ "summarize_expected_counts" ] }, { "page": "tune_lambda_ability_risk", "title": "Tune lambda by downstream ability-score risk", "topics": [ "tune_lambda_ability_risk" ] }, { "page": "tune_lambda_ability_risk_1pl", "title": "Tune lambda by downstream ability-score risk for a 1PL model", "topics": [ "tune_lambda_ability_risk_1pl" ] }, { "page": "tune_lambda_ability_risk_crossfit", "title": "Cross-fit ability-score-risk lambda tuning", "topics": [ "tune_lambda_ability_risk_crossfit" ] }, { "page": "tune_lambda_ability_risk_item", "title": "Per-item ability-risk lambda tuning via coordinate descent", "topics": [ "tune_lambda_ability_risk_item" ] }, { "page": "tune_lambda_ppi_score", "title": "Plug-in PPI++ optimal tuning parameter", "topics": [ "tune_lambda_ppi_score" ] }, { "page": "tune_lambda_ppi_score_1pl", "title": "Plug-in PPI++ optimal tuning parameter for a 1PL model", "topics": [ "tune_lambda_ppi_score_1pl" ] }, { "page": "tune_lambda_ppi_score_item", "title": "Per-item PPI++ optimal tuning parameters", "topics": [ "tune_lambda_ppi_score_item" ] }, { "page": "vcov_mixed_subjects", "title": "Sandwich covariance for a mixed-subjects fit", "topics": [ "vcov_mixed_subjects" ] }, { "page": "vcov_mixed_subjects_1pl", "title": "Sandwich covariance for a 1PL mixed-subjects fit", "topics": [ "vcov_mixed_subjects_1pl" ] }, { "page": "vcov_mixed_subjects_mml", "title": "Marginal-MML sandwich covariance for a mixed-subjects fit", "topics": [ "vcov_mixed_subjects_mml" ] } ], "_readme": "https://github.com/cran/mixedsubjectsirt/raw/HEAD/README.md", "_rundeps": [ "audio", "beepr", "brio", "callr", "class", "cli", "clipr", "cluster", "codetools", "crayon", "dcurver", "Deriv", "desc", "diffobj", "digest", "dplyr", "e1071", "evaluate", "fs", "future", "future.apply", "generics", "globals", "glue", "GPArotation", "gridExtra", "gtable", "jsonlite", "lattice", "lifecycle", "listenv", "magrittr", "MASS", "Matrix", "mgcv", "mirai", "mirt", "nanonext", "nlme", "otel", "parallelly", "pbapply", "permute", "pillar", "pkgbuild", "pkgconfig", "pkgload", "praise", "processx", "progressr", "proxy", "ps", "qs2", "R.methodsS3", "R.oo", "R.utils", "R6", "Rcpp", "RcppArmadillo", "RcppParallel", "rlang", "rmutil", "rprojroot", "sessioninfo", "SimDesign", "splines2", "stringfish", "testthat", "tibble", "tidyselect", "utf8", "vctrs", "vegan", "waldo", "withr" ], "_vignettes": [ { "source": "weakly-informative-llm.Rmd", "filename": "weakly-informative-llm.html", "title": "Calibrating with a Weakly-Informative, Biased LLM", "engine": "knitr::rmarkdown", "headings": [ "The setup", "Naive pooling inherits the bias", "$\\lambda$ moves efficiency, not bias", "Choosing $\\lambda$", "Takeaways", "Reproducing" ], "created": "2026-06-25 15:50:09", "modified": "2026-06-25 15:50:09", "commits": 1 }, { "source": "lambda-tuning.Rmd", "filename": "lambda-tuning.html", "title": "Choosing Lambda in Mixed-Subjects IRT", "engine": "knitr::rmarkdown", "headings": [ "Two objectives, two estimators", "Example data", "Ability-risk tuning: Minimizing $\\mathbb{E}[g'\\Sigma_\\gamma g]$", "Cross-fit $\\lambda$ tuning (recommended workflow)", "Frozen expected-count estimator (fast approximation)", "Minimizing $\\text{Tr}\\big[\\Sigma_\\gamma\\big]$ (diagnostic only)", "Choosing a procedure" ], "created": "2026-06-25 15:50:09", "modified": "2026-06-25 15:50:09", "commits": 1 }, { "source": "linking-comparison.Rmd", "filename": "linking-comparison.html", "title": "IRT Linking and Gradient Asymmetry: Diagnostic Guide", "engine": "knitr::rmarkdown", "headings": [ "Background", "Background (frozen expected-count estimator)", "Linking implementations", "Simulation", "Fitting human and LLM models", "Applying the three methods", "Parameter alignment after linking", "TCC alignment", "Gradient asymmetry: what linking fixes and what it does not", "Lambda sweep: how $\\lambda$ interacts with linking quality", "The role of power tuning", "Validation: what does $\\lambda^*$ measure?", "Test A — Perfect paired surrogate ($F = Y$)", "Test B — Partially overlapping predictions", "Test C — Stochastic LLM predictions (practical baseline)", "Summary: PPI++ score vs. ability risk", "Summary of findings", "Recommendation", "The marginal-MML fix" ], "created": "2026-06-25 15:50:09", "modified": "2026-06-25 15:50:09", "commits": 1 }, { "source": "mixed-subjects-1pl.Rmd", "filename": "mixed-subjects-1pl.html", "title": "Mixed-Subjects 1PL Calibration", "engine": "knitr::rmarkdown", "headings": [ "Simulate a 1PL test", "Step 1: Fit the 1PL baseline", "Step 2: Fit mixed-subjects MML (1PL)", "Step 3: Correct covariance — $(J+1) \\times (J+1)$ sandwich", "Step 4: Ability-score risk and lambda tuning", "Step 5: Verify — F = Y gives lambda > 0", "Compare 1PL and 2PL", "Ability-score risk: 1PL vs 2PL parameterization" ], "created": "2026-06-25 15:50:09", "modified": "2026-06-25 15:50:09", "commits": 1 }, { "source": "mixed-subjects-workflow.Rmd", "filename": "mixed-subjects-workflow.html", "title": "Mixed-Subjects IRT Calibration", "engine": "knitr::rmarkdown", "headings": [ "Simulate example data", "Step 1: Fit the human baseline", "Step 2: Fit the MML mixed-subjects model", "Step 3: Select $\\lambda$ by ability-score risk", "Step 3b (recommended workflow): cross-fit $\\lambda$ tuning", "Step 4: Inspect the covariance", "Compare calibrations", "When the LLM is uninformative", "Validation" ], "created": "2026-06-25 15:50:09", "modified": "2026-06-25 15:50:09", "commits": 1 }, { "source": "lambda-tuning-item.Rmd", "filename": "lambda-tuning-item.html", "title": "Per-Item Lambda (Experimental)", "engine": "knitr::rmarkdown", "headings": [ "Why per-item lambda?", "Simulate a heterogeneous test", "Step 1: Fit 2PL baseline and get global scalar lambda", "Step 2: PPI++ score per item (fast diagnostic)", "Step 3: Per-item ability-risk tuning", "Step 4: Compare scalar vs. per-item parameter recovery", "Important note on initialization", "Approximation caveat" ], "created": "2026-06-25 15:50:09", "modified": "2026-06-25 15:50:09", "commits": 1 }, { "source": "simulation-validation.Rmd", "filename": "simulation-validation.html", "title": "Simulation Validation of the Mixed-Subjects MML Estimator", "engine": "knitr::rmarkdown", "headings": [ "Design", "Does $\\lambda$-selection track predictor quality?", "Do standard errors achieve appropriate coverage?", "Does the method improve downstream scoring?", "What is the role of cross-fitting?", "Is coverage valid at the tuned $\\lambda$?", "Summary", "Reproducing these results" ], "created": "2026-06-25 15:50:09", "modified": "2026-06-25 15:50:09", "commits": 1 }, { "source": "understanding-ability-risk.Rmd", "filename": "understanding-ability-risk.html", "title": "Understanding Ability-Risk Tuning", "engine": "knitr::rmarkdown", "headings": [ "Why this vignette exists", "Key Intuition", "The three response matrices", "1. Observed human responses: $O$", "2. Paired LLM-predicted human responses: $P$", "3. Additional LLM-generated responses: $G$", "The mixed-subjects IRT objective", "What lambda is learning", "$$L_O^", "Ability-risk tuning", "The approximate target is$$\\widehat R(\\lambda)", "Why row alignment matters", "Case A: perfect paired prediction", "$$\\lambda_", "\\frac", "Case B: row-shuffled perfect predictions", "Case C: same DGP, fresh Bernoulli draw", "$$\\operatorname{Cov}(O_{ij},P_{ij})", "$$\\operatorname{Var}(P_{ij})", "What kind of LLM data produces higher lambda?", "One approach to row alignment: leave-one-item-out prediction", "Another approach: covariate-based prediction", "Something that probably won't work: item-text-only generation", "How to generate $G$", "Summary", "Technical Explanation", "Overview: four objects, one objective", "1. The estimator and its estimating equation", "2. The sandwich covariance of $\\hat\\gamma$", "3. Ability scoring and the implicit gradient", "4. Delta-method propagation and the risk", "5. Why this differs from the PPI++ trace objective" ], "created": "2026-06-25 15:50:09", "modified": "2026-06-25 15:50:09", "commits": 1 } ], "_score": 3.6020599913279625, "_indexed": true, "_nocasepkg": "mixedsubjectsirt", "_universes": [ "cran", "klintkanopka" ], "_binaries": [ { "r": "4.7.0", "os": "linux", "version": "1.0.0", "date": "2026-06-25T20:03:05.000Z", "distro": "resolute", "commit": "2e77546e86fd98746edc454a11c8d8112e7b2aa4", "fileid": "7b873a7291527feab1a39a8c67a8f152fd927a3b3d08e48bac5b73ddf25d1368", "status": "success", "check": "OK", "buildurl": "https://github.com/r-universe/cran/actions/runs/28196656638" }, { "r": "4.6.0", "os": "linux", "version": "1.0.0", "date": "2026-06-25T20:03:03.000Z", "distro": "resolute", "commit": "2e77546e86fd98746edc454a11c8d8112e7b2aa4", "fileid": "69218fb057e51fad6317043a498b7ba1fe5f6259589e6cf5f153a92145f937ff", "status": "success", "check": "OK", "buildurl": "https://github.com/r-universe/cran/actions/runs/28196656638" }, { "r": "4.6.0", "os": "wasm", "version": "1.0.0", "date": "2026-06-25T20:03:33.000Z", "commit": "2e77546e86fd98746edc454a11c8d8112e7b2aa4", "fileid": "8bbd9234dee91e984a76a43ca78737ee9d91d8c2fb7fb70caf873f512ddf1af1", "status": "success", "buildurl": "https://github.com/r-universe/cran/actions/runs/28196656638" } ] }