Module neuralops.config

Unified configuration system for the Efficient Transformer Toolkit.

Load a complete configuration from YAML + CLI: from neuralops.config import load_config cfg = load_config("config.yaml", cli_args=args)

Sub-modules

neuralops.config.compression_config

neuralops.config.config

Complete configuration dataclasses for all framework components …

neuralops.config.data_config

neuralops.config.experiment_config

neuralops.config.hardware_config

neuralops.config.loader

Configuration loader: YAML + CLI overrides → Config dataclass.

neuralops.config.method_config

neuralops.config.model_config

neuralops.config.sweep_config

Functions

def load_config(yaml_path: str | None = None, cli_args: argparse.Namespace | None = None) ‑> Config

Expand source code

def load_config(
    yaml_path: str | None = None,
    cli_args: argparse.Namespace | None = None,
) -> Config:
    """
    Load configuration from optional YAML file + CLI overrides.
    Always returns a fully populated Config dataclass with defaults.
    """
    cfg_dict: dict[str, Any] = {}
    if yaml_path:
        cfg_dict = load_yaml(yaml_path)
    if cli_args:
        cfg_dict = merge_cli_args(cfg_dict, cli_args)

    # Convert the flat dict to nested Config using the top‑level Config dataclass
    return _dict_to_dataclass(Config, cfg_dict)

Load configuration from optional YAML file + CLI overrides. Always returns a fully populated Config dataclass with defaults.

Classes

class CompressionConfig (attn_type: str = 'full',
dropout: float = 0.1,
rank: int = 32,
svd_rank: int = 32,
svd_energy: float = 0.95,
min_rank: int = 4,
max_rank: int = 64,
lora_r: int = 8,
lora_alpha: int = 16,
lora_dropout: float = 0.1,
rdrop_alpha: float = 5.0,
r3f_epsilon: float = 1e-05,
llrd_decay: float = 0.95,
mixout_p: float = 0.9,
dislora_beta: float = 0.5,
dislora_capacity_factor: float = 1.0,
fisher_num_batches: int = 20,
fisher_sample_size: int = 200,
fisher_scale_max: float = 2.0,
gelora_margin: float = 1.0,
gelora_min_rank: int = 2,
gelora_max_rank: int = 32,
block_size: int = 64,
local_window: int = 64,
top_k_inter: int = 4,
num_landmarks: int = 16,
feature_dim: int = 128,
rff_features: int = 128,
k_early: int = 32,
k_mid: int = 16,
k_late: int = 12,
use_gate: bool = True,
progressive: bool = True,
max_seq_len: int = 512,
low_rank_cache_size: int = 16,
cur_rank: int = 16,
lsh_rounds: int = 4,
lsh_bucket_size: int = 32,
window_size: int = 128,
compression_rank: int = 64,
use_adaptive_gate: bool = True,
gate_bias: float = 2.0,
softpick_temperature: float = 1.0,
rectify_interval: int = 4,
aks_enabled: bool = True,
rel_pos_max_distance: int = 128,
use_rel_pos_bias: bool = True,
adaptive_span: bool = True,
max_span: float = 512.0,
num_kernel_heads: int = 8,
num_full_heads: int = 4,
soa_rank: int = 32,
svd_rank_qkv: int = 32,
svd_rank_out: int = 32,
num_local_heads: int = 8,
num_global_heads: int = 4,
num_experts: int = 8,
top_k: int = 2,
ratio: int = 9,
n_heads: int | None = None,
n_layers: int | None = None,
use_full_attention: bool = False,
use_fp8: bool = False,
use_lion: bool = False,
gradient_checkpointing: bool = False,
early_stopping_patience: int | None = None,
max_grad_norm: float = 1.0)

Expand source code

@dataclass
class CompressionConfig:
    """
    All parameters related to attention injection, low‑rank compression,
    sparsity, hybrid heads, etc.
    These fields are read by the injection functions via getattr(comp_config, …).
    """

    # ── General ────────────────────────────────────────
    attn_type: str = "full"
    dropout: float = 0.1
    rank: int = 32  # generic rank (used by many modules)
    svd_rank: int = 32  # SVD rank
    svd_energy: float = 0.95  # energy retention for SVD truncation
    min_rank: int = 4
    max_rank: int = 64

    # ── LoRA / method‑specific ─────────────────────────
    lora_r: int = 8
    lora_alpha: int = 16
    lora_dropout: float = 0.1
    rdrop_alpha: float = 5.0
    r3f_epsilon: float = 1e-5
    llrd_decay: float = 0.95
    mixout_p: float = 0.9
    dislora_beta: float = 0.5
    dislora_capacity_factor: float = 1.0
    fisher_num_batches: int = 20
    fisher_sample_size: int = 200
    fisher_scale_max: float = 2.0
    gelora_margin: float = 1.0
    gelora_min_rank: int = 2
    gelora_max_rank: int = 32

    # ── Attention common fields ────────────────────────
    block_size: int = 64
    local_window: int = 64
    top_k_inter: int = 4
    num_landmarks: int = 16
    feature_dim: int = 128
    rff_features: int = 128
    k_early: int = 32
    k_mid: int = 16
    k_late: int = 12
    use_gate: bool = True
    progressive: bool = True
    max_seq_len: int = 512
    low_rank_cache_size: int = 16  # LoLA
    cur_rank: int = 16  # PHANTOM
    lsh_rounds: int = 4
    lsh_bucket_size: int = 32
    window_size: int = 128
    compression_rank: int = 64
    use_adaptive_gate: bool = True
    gate_bias: float = 2.0
    softpick_temperature: float = 1.0
    rectify_interval: int = 4
    aks_enabled: bool = True
    rel_pos_max_distance: int = 128
    use_rel_pos_bias: bool = True
    adaptive_span: bool = True
    max_span: float = 512.0
    num_kernel_heads: int = 8
    num_full_heads: int = 4
    soa_rank: int = 32
    svd_rank_qkv: int = 32
    svd_rank_out: int = 32
    num_local_heads: int = 8
    num_global_heads: int = 4
    num_experts: int = 8
    top_k: int = 2
    ratio: int = 9  # hybrid attention / Mamba ratio
    n_heads: int | None = None
    n_layers: int | None = None
    use_full_attention: bool = False
    use_fp8: bool = False

    # Additional fields used by some compressors
    use_lion: bool = False
    gradient_checkpointing: bool = False
    early_stopping_patience: int | None = None
    max_grad_norm: float = 1.0

All parameters related to attention injection, low‑rank compression, sparsity, hybrid heads, etc. These fields are read by the injection functions via getattr(comp_config, …).

Instance variables

var adaptive_span : bool

The type of the None singleton.

var aks_enabled : bool

The type of the None singleton.

var attn_type : str

The type of the None singleton.

var block_size : int

The type of the None singleton.

var compression_rank : int

The type of the None singleton.

var cur_rank : int

The type of the None singleton.

var dislora_beta : float

The type of the None singleton.

var dislora_capacity_factor : float

The type of the None singleton.

var dropout : float

The type of the None singleton.

var early_stopping_patience : int | None

The type of the None singleton.

var feature_dim : int

The type of the None singleton.

var fisher_num_batches : int

The type of the None singleton.

var fisher_sample_size : int

The type of the None singleton.

var fisher_scale_max : float

The type of the None singleton.

var gate_bias : float

The type of the None singleton.

var gelora_margin : float

The type of the None singleton.

var gelora_max_rank : int

The type of the None singleton.

var gelora_min_rank : int

The type of the None singleton.

var gradient_checkpointing : bool

The type of the None singleton.

var k_early : int

The type of the None singleton.

var k_late : int

The type of the None singleton.

var k_mid : int

The type of the None singleton.

var llrd_decay : float

The type of the None singleton.

var local_window : int

The type of the None singleton.

var lora_alpha : int

The type of the None singleton.

var lora_dropout : float

The type of the None singleton.

var lora_r : int

The type of the None singleton.

var low_rank_cache_size : int

The type of the None singleton.

var lsh_bucket_size : int

The type of the None singleton.

var lsh_rounds : int

The type of the None singleton.

var max_grad_norm : float

The type of the None singleton.

var max_rank : int

The type of the None singleton.

var max_seq_len : int

The type of the None singleton.

var max_span : float

The type of the None singleton.

var min_rank : int

The type of the None singleton.

var mixout_p : float

The type of the None singleton.

var n_heads : int | None

The type of the None singleton.

var n_layers : int | None

The type of the None singleton.

var num_experts : int

The type of the None singleton.

var num_full_heads : int

The type of the None singleton.

var num_global_heads : int

The type of the None singleton.

var num_kernel_heads : int

The type of the None singleton.

var num_landmarks : int

The type of the None singleton.

var num_local_heads : int

The type of the None singleton.

var progressive : bool

The type of the None singleton.

var r3f_epsilon : float

The type of the None singleton.

var rank : int

The type of the None singleton.

var ratio : int

The type of the None singleton.

var rdrop_alpha : float

The type of the None singleton.

var rectify_interval : int

The type of the None singleton.

var rel_pos_max_distance : int

The type of the None singleton.

var rff_features : int

The type of the None singleton.

var soa_rank : int

The type of the None singleton.

var softpick_temperature : float

The type of the None singleton.

var svd_energy : float

The type of the None singleton.

var svd_rank : int

The type of the None singleton.

var svd_rank_out : int

The type of the None singleton.

var svd_rank_qkv : int

The type of the None singleton.

var top_k : int

The type of the None singleton.

var top_k_inter : int

The type of the None singleton.

var use_adaptive_gate : bool

The type of the None singleton.

var use_fp8 : bool

The type of the None singleton.

var use_full_attention : bool

The type of the None singleton.

var use_gate : bool

The type of the None singleton.

var use_lion : bool

The type of the None singleton.

var use_rel_pos_bias : bool

The type of the None singleton.

var window_size : int

The type of the None singleton.

class Config (model: ModelConfig = <factory>,
data: DataConfig = <factory>,
training: TrainingConfig = <factory>,
compression: CompressionConfig = <factory>,
method: MethodConfig = <factory>,
hardware: HardwareConfig = <factory>,
sweep: SweepConfig = <factory>,
experiment: ExperimentConfig = <factory>)

Expand source code

@dataclass
class Config:
    """Master configuration that holds all sub‑configs."""

    model: ModelConfig = field(default_factory=ModelConfig)
    data: DataConfig = field(default_factory=DataConfig)
    training: TrainingConfig = field(default_factory=TrainingConfig)
    compression: CompressionConfig = field(default_factory=CompressionConfig)
    method: MethodConfig = field(default_factory=MethodConfig)
    hardware: HardwareConfig = field(default_factory=HardwareConfig)
    sweep: SweepConfig = field(default_factory=SweepConfig)
    experiment: ExperimentConfig = field(default_factory=ExperimentConfig)

Master configuration that holds all sub‑configs.

Instance variables

var compression : CompressionConfig

The type of the None singleton.

var data : DataConfig

The type of the None singleton.

var experiment : ExperimentConfig

The type of the None singleton.

var hardware : HardwareConfig

The type of the None singleton.

var method : MethodConfig

The type of the None singleton.

var model : ModelConfig

The type of the None singleton.

var sweep : SweepConfig

The type of the None singleton.

var training : TrainingConfig

The type of the None singleton.

class DataConfig (local_glue_path: str = '/Users/tahamajs/Documents/uni/datasets/glue',
max_seq_length: int = 128,
max_train_samples: int | None = None,
max_val_samples: int | None = None,
num_proc: int = 4,
streaming: bool = False,
remove_columns: list[str] = <factory>,
split_mapping: dict[str, str] = <factory>)

Expand source code

@dataclass
class DataConfig:
    """Data loading and preprocessing configuration."""

    local_glue_path: str = str(get_glue_dir())
    max_seq_length: int = 128
    max_train_samples: int | None = None
    max_val_samples: int | None = None
    num_proc: int = 4
    streaming: bool = False
    remove_columns: list[str] = field(default_factory=lambda: ["label"])
    split_mapping: dict[str, str] = field(
        default_factory=lambda: {
            "train": "train",
            "validation": "validation",
            "validation_matched": "validation_matched",
            "validation_mismatched": "validation_mismatched",
        }
    )

Data loading and preprocessing configuration.

Instance variables

var local_glue_path : str

The type of the None singleton.

var max_seq_length : int

The type of the None singleton.

var max_train_samples : int | None

The type of the None singleton.

var max_val_samples : int | None

The type of the None singleton.

var num_proc : int

The type of the None singleton.

var remove_columns : list[str]

The type of the None singleton.

var split_mapping : dict[str, str]

The type of the None singleton.

var streaming : bool

The type of the None singleton.

class DatasetConfig (name: str,
path: str,
task_type: str = 'single',
num_choices: int = 0,
num_labels: int = 2)

Expand source code

@dataclass
class DatasetConfig:
    """Description of a single dataset for an experiment sweep."""

    name: str
    path: str
    task_type: str = "single"  # single, pair, regression, mc
    num_choices: int = 0  # for multi‑choice
    num_labels: int = 2

Description of a single dataset for an experiment sweep.

Instance variables

var name : str

The type of the None singleton.

var num_choices : int

The type of the None singleton.

var num_labels : int

The type of the None singleton.

var path : str

The type of the None singleton.

var task_type : str

The type of the None singleton.

class DisLoRAConfig (beta: float = 0.5, capacity_factor: float = 1.0)

Expand source code

@dataclass
class DisLoRAConfig:
    beta: float = 0.5
    capacity_factor: float = 1.0

DisLoRAConfig(beta: float = 0.5, capacity_factor: float = 1.0)

Instance variables

var beta : float

The type of the None singleton.

var capacity_factor : float

The type of the None singleton.

class ExperimentConfig (base_models: dict[str, str] = <factory>,
svd_ranks: list[int] = <factory>,
datasets: list[DatasetConfig] = <factory>,
min_samples: int = 10,
max_samples: int = 200,
batch_size: int = 4,
epochs: int = 1,
learning_rate: float = 2e-05,
max_length: int = 128,
val_split: float = 0.2,
seed: int = 42,
gradient_accumulation_steps: int = 1,
save_checkpoint_interval: int = 10,
output_dir: str = './results',
plots_dir: str = './plots/comprehensive',
log_dir: str = './logs',
selected_models: list[str] = <factory>,
selected_datasets: list[str] = <factory>,
selected_ranks: list[int] = <factory>)

Expand source code

@dataclass
class ExperimentConfig:
    """Multi‑run experiment grid (SVD ranks, datasets, models)."""

    base_models: dict[str, str] = field(default_factory=lambda: {"bert": "bert-base-uncased"})
    svd_ranks: list[int] = field(default_factory=lambda: [8, 16, 32])
    datasets: list[DatasetConfig] = field(default_factory=list)
    min_samples: int = 10
    max_samples: int = 200
    batch_size: int = 4
    epochs: int = 1
    learning_rate: float = 2e-5
    max_length: int = 128
    val_split: float = 0.2
    seed: int = 42
    gradient_accumulation_steps: int = 1
    save_checkpoint_interval: int = 10
    output_dir: str = "./results"
    plots_dir: str = "./plots/comprehensive"
    log_dir: str = "./logs"
    selected_models: list[str] = field(default_factory=list)
    selected_datasets: list[str] = field(default_factory=list)
    selected_ranks: list[int] = field(default_factory=list)

Multi‑run experiment grid (SVD ranks, datasets, models).

Instance variables

var base_models : dict[str, str]

The type of the None singleton.

var batch_size : int

The type of the None singleton.

var datasets : list[DatasetConfig]

The type of the None singleton.

var epochs : int

The type of the None singleton.

var gradient_accumulation_steps : int

The type of the None singleton.

var learning_rate : float

The type of the None singleton.

var log_dir : str

The type of the None singleton.

var max_length : int

The type of the None singleton.

var max_samples : int

The type of the None singleton.

var min_samples : int

The type of the None singleton.

var output_dir : str

The type of the None singleton.

var plots_dir : str

The type of the None singleton.

var save_checkpoint_interval : int

The type of the None singleton.

var seed : int

The type of the None singleton.

var selected_datasets : list[str]

The type of the None singleton.

var selected_models : list[str]

The type of the None singleton.

var selected_ranks : list[int]

The type of the None singleton.

var svd_ranks : list[int]

The type of the None singleton.

var val_split : float

The type of the None singleton.

class FisherConfig (num_batches: int = 20, sample_size: int = 200, scale_max: float = 2.0)

Expand source code

@dataclass
class FisherConfig:
    num_batches: int = 20
    sample_size: int = 200
    scale_max: float = 2.0

FisherConfig(num_batches: int = 20, sample_size: int = 200, scale_max: float = 2.0)

Instance variables

var num_batches : int

The type of the None singleton.

var sample_size : int

The type of the None singleton.

var scale_max : float

The type of the None singleton.

class GeLoRAConfig (margin: float = 1.0, min_rank: int = 2, max_rank: int = 32)

Expand source code

@dataclass
class GeLoRAConfig:
    margin: float = 1.0
    min_rank: int = 2
    max_rank: int = 32

GeLoRAConfig(margin: float = 1.0, min_rank: int = 2, max_rank: int = 32)

Instance variables

var margin : float

The type of the None singleton.

var max_rank : int

The type of the None singleton.

var min_rank : int

The type of the None singleton.

class HardwareConfig (device: str = 'auto',
force_cpu_for_svd: bool = True,
enable_fp16: bool = False,
enable_bf16: bool = False,
num_workers: int = 4,
pin_memory: bool = False,
torch_compile: bool = False,
empty_cache_every_step: bool = False)

Expand source code

@dataclass
class HardwareConfig:
    device: str = "auto"  # auto, cuda, mps, cpu
    force_cpu_for_svd: bool = True
    enable_fp16: bool = False
    enable_bf16: bool = False
    num_workers: int = 4
    pin_memory: bool = False
    torch_compile: bool = False
    empty_cache_every_step: bool = False

HardwareConfig(device: str = 'auto', force_cpu_for_svd: bool = True, enable_fp16: bool = False, enable_bf16: bool = False, num_workers: int = 4, pin_memory: bool = False, torch_compile: bool = False, empty_cache_every_step: bool = False)

Instance variables

var device : str

The type of the None singleton.

var empty_cache_every_step : bool

The type of the None singleton.

var enable_bf16 : bool

The type of the None singleton.

var enable_fp16 : bool

The type of the None singleton.

var force_cpu_for_svd : bool

The type of the None singleton.

var num_workers : int

The type of the None singleton.

var pin_memory : bool

The type of the None singleton.

var torch_compile : bool

The type of the None singleton.

class LLRDConfig (decay: float = 0.95)

Expand source code

@dataclass
class LLRDConfig:
    decay: float = 0.95

LLRDConfig(decay: float = 0.95)

Instance variables

var decay : float

The type of the None singleton.

class LoRAConfig (r: int = 8, alpha: int = 16, dropout: float = 0.1)

Expand source code

@dataclass
class LoRAConfig:
    r: int = 8
    alpha: int = 16
    dropout: float = 0.1

LoRAConfig(r: int = 8, alpha: int = 16, dropout: float = 0.1)

Instance variables

var alpha : int

The type of the None singleton.

var dropout : float

The type of the None singleton.

var r : int

The type of the None singleton.

class MethodConfig (method: str = 'baseline',
lora: LoRAConfig = <factory>,
rdrop: RDropConfig = <factory>,
r3f: R3FConfig = <factory>,
fisher: FisherConfig = <factory>,
gelora: GeLoRAConfig = <factory>,
llrd: LLRDConfig = <factory>,
mixout: MixoutConfig = <factory>,
dislora: DisLoRAConfig = <factory>)

Expand source code

@dataclass
class MethodConfig:
    """Training method selection and its sub‑configs."""

    method: str = (
        "baseline"  # baseline, lora, dora, dislora, rdrop, r3f, fisher, gelora, llrd, mixout
    )
    lora: LoRAConfig = field(default_factory=LoRAConfig)
    rdrop: RDropConfig = field(default_factory=RDropConfig)
    r3f: R3FConfig = field(default_factory=R3FConfig)
    fisher: FisherConfig = field(default_factory=FisherConfig)
    gelora: GeLoRAConfig = field(default_factory=GeLoRAConfig)
    llrd: LLRDConfig = field(default_factory=LLRDConfig)
    mixout: MixoutConfig = field(default_factory=MixoutConfig)
    dislora: DisLoRAConfig = field(default_factory=DisLoRAConfig)

Training method selection and its sub‑configs.

Instance variables

var dislora : DisLoRAConfig

The type of the None singleton.

var fisher : FisherConfig

The type of the None singleton.

var gelora : GeLoRAConfig

The type of the None singleton.

var llrd : LLRDConfig

The type of the None singleton.

var lora : LoRAConfig

The type of the None singleton.

var method : str

The type of the None singleton.

var mixout : MixoutConfig

The type of the None singleton.

var r3f : R3FConfig

The type of the None singleton.

var rdrop : RDropConfig

The type of the None singleton.

class MixoutConfig (p: float = 0.9)

Expand source code

@dataclass
class MixoutConfig:
    p: float = 0.9

MixoutConfig(p: float = 0.9)

Instance variables

var p : float

The type of the None singleton.

class ModelConfig (name: str = 'distilbert-base-uncased',
num_labels: int = 2,
attention_type: str = 'full',
use_fast_tokenizer: bool = True,
trust_remote_code: bool = False,
cache_dir: str = './cache',
fix_layernorm_naming: bool = True,
variants: list[str] = <factory>)

Expand source code

@dataclass
class ModelConfig:
    """Model loading configuration."""

    name: str = "distilbert-base-uncased"
    num_labels: int = 2
    attention_type: str = "full"  # registry key
    use_fast_tokenizer: bool = True
    trust_remote_code: bool = False
    cache_dir: str = "./cache"
    fix_layernorm_naming: bool = True
    variants: list[str] = field(default_factory=list)  # e.g., ["standard", "compressed"]

Model loading configuration.

Instance variables

var attention_type : str

The type of the None singleton.

var cache_dir : str

The type of the None singleton.

var fix_layernorm_naming : bool

The type of the None singleton.

var name : str

The type of the None singleton.

var num_labels : int

The type of the None singleton.

var trust_remote_code : bool

The type of the None singleton.

var use_fast_tokenizer : bool

The type of the None singleton.

var variants : list[str]

The type of the None singleton.

class R3FConfig (epsilon: float = 1e-05)

Expand source code

@dataclass
class R3FConfig:
    epsilon: float = 1e-5

R3FConfig(epsilon: float = 1e-05)

Instance variables

var epsilon : float

The type of the None singleton.

class RDropConfig (alpha: float = 5.0)

Expand source code

@dataclass
class RDropConfig:
    alpha: float = 5.0

RDropConfig(alpha: float = 5.0)

Instance variables

var alpha : float

The type of the None singleton.

class SweepConfig (name: str = 'sweep',
algorithm: str = 'tpe',
direction: str = 'maximize',
metric: str = 'eval_accuracy',
max_trials: int = 20,
parallel_workers: int = 1,
pruner: str = 'hyperband',
search_space: dict[str, typing.Any] = <factory>,
base_config: dict[str, typing.Any] = <factory>)

Expand source code

@dataclass
class SweepConfig:
    """Hyperparameter sweep configuration."""

    name: str = "sweep"
    algorithm: str = "tpe"  # tpe, random, grid, cmaes
    direction: str = "maximize"
    metric: str = "eval_accuracy"
    max_trials: int = 20
    parallel_workers: int = 1
    pruner: str = "hyperband"
    search_space: dict[str, Any] = field(default_factory=dict)
    base_config: dict[str, Any] = field(default_factory=dict)

Hyperparameter sweep configuration.

Instance variables

var algorithm : str

The type of the None singleton.

var base_config : dict[str, typing.Any]

The type of the None singleton.

var direction : str

The type of the None singleton.

var max_trials : int

The type of the None singleton.

var metric : str

The type of the None singleton.

var name : str

The type of the None singleton.

var parallel_workers : int

The type of the None singleton.

var pruner : str

The type of the None singleton.

var search_space : dict[str, typing.Any]

The type of the None singleton.

class TrainingConfig (batch_size: int = 16,
eval_batch_size: int = 32,
epochs: int = 3,
learning_rate: float = 2e-05,
weight_decay: float = 0.01,
warmup_ratio: float = 0.1,
warmup_steps: int = 0,
gradient_accumulation_steps: int = 1,
max_grad_norm: float = 1.0,
lr_scheduler_type: str = 'linear',
optimizer: str = 'adamw',
early_stopping_patience: int | None = None,
early_stopping_threshold: float = 0.001,
save_strategy: str = 'epoch',
save_steps: int = 500,
save_total_limit: int = 2,
save_only_model: bool = True,
eval_strategy: str = 'epoch',
eval_steps: int = 500,
metric_for_best_model: str = 'eval_loss',
greater_is_better: bool = False,
fp16: bool = False,
bf16: bool = False,
report_to: str = 'none')

Expand source code

@dataclass
class TrainingConfig:
    """Training hyperparameters."""

    batch_size: int = 16
    eval_batch_size: int = 32
    epochs: int = 3
    learning_rate: float = 2e-5
    weight_decay: float = 0.01
    warmup_ratio: float = 0.1
    warmup_steps: int = 0
    gradient_accumulation_steps: int = 1
    max_grad_norm: float = 1.0
    lr_scheduler_type: str = "linear"
    optimizer: str = "adamw"
    early_stopping_patience: int | None = None
    early_stopping_threshold: float = 0.001
    save_strategy: str = "epoch"
    save_steps: int = 500
    save_total_limit: int = 2
    save_only_model: bool = True
    eval_strategy: str = "epoch"
    eval_steps: int = 500
    metric_for_best_model: str = "eval_loss"
    greater_is_better: bool = False
    fp16: bool = False
    bf16: bool = False
    report_to: str = "none"

Training hyperparameters.

Instance variables

var batch_size : int

The type of the None singleton.

var bf16 : bool

The type of the None singleton.

var early_stopping_patience : int | None

The type of the None singleton.

var early_stopping_threshold : float

The type of the None singleton.

var epochs : int

The type of the None singleton.

var eval_batch_size : int

The type of the None singleton.

var eval_steps : int

The type of the None singleton.

var eval_strategy : str

The type of the None singleton.

var fp16 : bool

The type of the None singleton.

var gradient_accumulation_steps : int

The type of the None singleton.

var greater_is_better : bool

The type of the None singleton.

var learning_rate : float

The type of the None singleton.

var lr_scheduler_type : str

The type of the None singleton.

var max_grad_norm : float

The type of the None singleton.

var metric_for_best_model : str

The type of the None singleton.

var optimizer : str

The type of the None singleton.

var report_to : str

The type of the None singleton.

var save_only_model : bool

The type of the None singleton.

var save_steps : int

The type of the None singleton.

var save_strategy : str

The type of the None singleton.

var save_total_limit : int

The type of the None singleton.

var warmup_ratio : float

The type of the None singleton.

var warmup_steps : int

The type of the None singleton.

var weight_decay : float

The type of the None singleton.