## ----setup, include=FALSE-----------------------------------------------------
knitr::opts_chunk$set(
  collapse = TRUE,
  comment = "#>"
)

cmdstan_available <- identical(Sys.getenv("PAIRWISELLM_RUN_CMDSTAN_VIGNETTES"), "true") &&
  requireNamespace("cmdstanr", quietly = TRUE) &&
  tryCatch(
    !is.null(cmdstanr::cmdstan_version()),
    error = function(e) FALSE
  )

knitr::opts_chunk$set(eval = cmdstan_available)

## ----cmdstan-install, eval=FALSE----------------------------------------------
# install.packages(
#   "cmdstanr",
#   repos = c("https://stan-dev.r-universe.dev", getOption("repos"))
# )
# cmdstanr::check_cmdstan_toolchain(fix = TRUE)
# cmdstanr::install_cmdstan()

## ----cmdstan-check, eval=TRUE-------------------------------------------------
cmdstan_available

if (cmdstan_available) {
  cmdstanr::cmdstan_version()
}

## ----example-setup, eval=TRUE-------------------------------------------------
library(pairwiseLLM)

data("example_writing_samples", package = "pairwiseLLM")
samples <- example_writing_samples[, c("ID", "text", "quality_score")]

trait <- trait_description("overall_quality")
prompt_template <- set_prompt_template()

deterministic_judge <- function(A, B, state, ...) {
  a_wins <- A$quality_score[[1]] >= B$quality_score[[1]]
  list(
    is_valid = TRUE,
    Y = as.integer(a_wins),
    invalid_reason = NA_character_
  )
}

## ----adaptive-run-------------------------------------------------------------
# session_dir <- tempfile("pairwisellm-adaptive-")
# 
# out <- adaptive_rank(
#   data = samples,
#   id_col = "ID",
#   text_col = "text",
#   judge = deterministic_judge,
#   n_steps = 22L,
#   session_dir = session_dir,
#   persist_item_log = TRUE,
#   resume = FALSE,
#   seed = 42L,
#   progress = "none"
# )

## ----adaptive-results---------------------------------------------------------
# out$summary
# 
# out$items[, c(
#   "item_id", "theta_raw_eap", "theta_raw_sd", "rank_raw", "degree"
# )] |>
#   head()
# 
# out$refits[, c(
#   "refit_id", "total_pairs_done", "diagnostics_pass",
#   "reliability_EAP", "stop_decision", "stop_reason"
# )]

## ----adaptive-logs------------------------------------------------------------
# names(out$logs)
# 
# step_log <- adaptive_step_log(out$state)
# step_log[, c(
#   "step_id", "pair_id", "A_id", "B_id", "Y", "status",
#   "round_stage", "fallback_used", "starvation_reason"
# )] |>
#   head()
# 
# summarize_adaptive(out$state)
# summarize_refits(out$state, last_n = 1L, include_optional = FALSE)
# summarize_items(out$state, top_n = 5L)
# 
# result_history <- adaptive_results_history(out$state)
# head(result_history)

## ----lower-level-lifecycle----------------------------------------------------
# low_level_items <- samples[1:5, ]
# low_level_items$item_id <- as.character(low_level_items$ID)
# 
# low_level_state <- adaptive_rank_start(low_level_items, seed = 17L)
# low_level_state <- adaptive_rank_run_live(
#   state = low_level_state,
#   judge = deterministic_judge,
#   n_steps = 3L,
#   btl_config = list(refit_pairs_target = 5000L),
#   progress = "none"
# )
# 
# summarize_adaptive(low_level_state)

## ----adaptive-persistence-----------------------------------------------------
# session_metadata <- validate_session_dir(session_dir)
# session_metadata[c("schema_version", "package_version", "n_items")]
# 
# loaded_state <- load_adaptive_session(session_dir)
# summarize_adaptive(loaded_state)
# 
# resumed <- adaptive_rank(
#   data = samples,
#   id_col = "ID",
#   text_col = "text",
#   judge = deterministic_judge,
#   n_steps = 2L,
#   session_dir = session_dir,
#   persist_item_log = TRUE,
#   resume = TRUE,
#   progress = "none"
# )
# 
# c(
#   before = nrow(out$logs$step_log),
#   after = nrow(resumed$logs$step_log)
# )

## ----adaptive-manual-save-----------------------------------------------------
# snapshot_dir <- tempfile("pairwisellm-snapshot-")
# save_adaptive_session(resumed$state, snapshot_dir, overwrite = TRUE)
# validate_session_dir(snapshot_dir)
# snapshot <- load_adaptive_session(snapshot_dir)

## ----invalid-judgment, eval=TRUE----------------------------------------------
invalid_once_judge <- function(A, B, state, ...) {
  if (nrow(state$step_log) == 0L) {
    return(list(
      is_valid = FALSE,
      Y = NA_integer_,
      invalid_reason = "simulated_parse_failure"
    ))
  }
  deterministic_judge(A, B, state)
}

invalid_demo <- adaptive_rank(
  data = samples[1:4, ],
  id_col = "ID",
  text_col = "text",
  judge = invalid_once_judge,
  n_steps = 2L,
  resume = FALSE,
  seed = 7L,
  progress = "none"
)

adaptive_step_log(invalid_demo$state)[, c(
  "step_id", "pair_id", "status", "judge_valid", "judge_invalid_reason"
)]

## ----live-openai, eval=FALSE, purl=TRUE---------------------------------------
# library(pairwiseLLM)
# 
# stopifnot(nzchar(Sys.getenv("OPENAI_API_KEY")))
# 
# real_samples <- utils::read.csv(
#   "writing-samples.csv",
#   stringsAsFactors = FALSE
# )
# stopifnot(nrow(real_samples) >= 2L)
# stopifnot(all(c("ID", "text") %in% names(real_samples)))
# stopifnot(!anyNA(real_samples$ID), !anyDuplicated(real_samples$ID))
# stopifnot(!anyNA(real_samples$text), all(nzchar(real_samples$text)))
# 
# live_trait <- trait_description("overall_quality")
# live_prompt <- set_prompt_template()
# live_session <- "adaptive-live-session"
# 
# live <- adaptive_rank(
#   data = real_samples,
#   id_col = "ID",
#   text_col = "text",
#   backend = "openai",
#   model = "gpt-5.6-luna",
#   endpoint = "responses",
#   trait_name = live_trait$name,
#   trait_description = live_trait$description,
#   prompt_template = live_prompt,
#   judge_args = list(reasoning = "none"),
#   n_steps = 200L,
#   session_dir = live_session,
#   checkpoint_every_steps = 10L,
#   persist_item_log = TRUE,
#   resume = FALSE,
#   seed = 20260904L,
#   progress = "refits",
#   save_outputs = TRUE
# )

## ----live-openai-inspection, eval=FALSE, purl=TRUE----------------------------
# live$summary
# 
# live$items[, c(
#   "item_id", "theta_raw_eap", "theta_raw_sd", "rank_raw", "degree"
# )] |>
#   head(10L)
# 
# live$refits[, c(
#   "refit_id", "total_pairs_done", "diagnostics_pass",
#   "reliability_EAP", "stop_decision", "stop_reason"
# )] |>
#   tail()

## ----live-openai-audit, eval=FALSE, purl=TRUE---------------------------------
# live_steps <- live$logs$step_log[, c(
#   "step_id", "pair_id", "A_id", "B_id", "Y", "status",
#   "judge_backend", "judge_model", "judge_endpoint", "judge_invalid_reason",
#   "llm_status_code", "llm_error_message",
#   "prompt_tokens", "completion_tokens", "total_tokens"
# )]
# 
# tail(live_steps)
# 
# colSums(live_steps[, c(
#   "prompt_tokens", "completion_tokens", "total_tokens"
# )], na.rm = TRUE)
# 
# live_steps[live_steps$status == "invalid", ]

## ----live-openai-resume, eval=FALSE, purl=TRUE--------------------------------
# live <- adaptive_rank(
#   data = real_samples,
#   id_col = "ID",
#   text_col = "text",
#   backend = "openai",
#   model = "gpt-5.6-luna",
#   endpoint = "responses",
#   trait_name = live_trait$name,
#   trait_description = live_trait$description,
#   prompt_template = live_prompt,
#   judge_args = list(reasoning = "none"),
#   n_steps = 100L,
#   session_dir = live_session,
#   checkpoint_every_steps = 10L,
#   persist_item_log = TRUE,
#   resume = TRUE,
#   progress = "refits",
#   save_outputs = TRUE
# )