Struct MultiHeadAttention 

pub struct MultiHeadAttention<B>
where
    B: Backend,
{ /* private fields */ }

Multi-Head Self-Attention module.

Examples

let attn = MultiHeadAttention::<CpuBackend>::new(512, 8, DType::F64, &dev)?;
let x = CpuTensor::rand((2, 10, 512), DType::F64, &dev)?;
let y = attn.forward(&x)?; // [2, 10, 512]

Implementations

impl<B> MultiHeadAttention<B>

where B: Backend,

pub fn new( d_model: usize, num_heads: usize, dtype: DType, device: &<B as Backend>::Device, ) -> Result<MultiHeadAttention<B>, Error>

Create a new Multi-Head Attention module.

Arguments

• d_model: total model dimension (must be divisible by num_heads)
• num_heads: number of attention heads
• dtype: data type for parameters
• device: device to create parameters on

pub fn with_causal(self, causal: bool) -> MultiHeadAttention<B>

Enable causal (autoregressive) masking.

pub fn num_heads(&self) -> usize

pub fn d_model(&self) -> usize

pub fn head_dim(&self) -> usize

Trait Implementations

impl<B> Module<B> for MultiHeadAttention<B>

where B: Backend,

fn forward(&self, x: &Tensor<B>) -> Result<Tensor<B>, Error>

Forward pass: self-attention on input x.

Input: [batch, seq_len, d_model] Output: [batch, seq_len, d_model]

fn parameters(&self) -> Vec<Tensor<B>>

Return all trainable parameters of this module. The optimizer uses these to update weights during training.

fn named_parameters(&self) -> Vec<(String, Tensor<B>)>

Return all trainable parameters with human-readable names.

fn set_training(&self, _training: bool)

Set training or evaluation mode.

fn is_training(&self) -> bool

Whether the module is in training mode (default: true).

fn train(&self)

Convenience: set training mode.

fn eval(&self)

Convenience: set evaluation mode.

fn num_parameters(&self) -> usize

Total number of scalar parameters in this module.

fn trainable_params_count(&self) -> usize

Number of trainable (variable) parameters.

fn frozen_parameters(&self) -> Vec<Tensor<B>>

Freeze all parameters: returns new parameter tensors with is_variable = false, preventing gradient accumulation.

fn state_dict(&self) -> Vec<(String, Tensor<B>)>

Returns a state_dict-style map of parameter name → tensor.