fastvideo.v1.configs.models.encoders.llama

Contents

`fastvideo.v1.configs.models.encoders.llama`#

Module Contents#

Classes#

`LlamaArchConfig`
`LlamaConfig`

API#

class fastvideo.v1.configs.models.encoders.llama.LlamaArchConfig[source]#

Bases: fastvideo.v1.configs.models.encoders.base.TextEncoderArchConfig

attention_bias: bool[source]#: False

attention_dropout: float[source]#: 0.0

bos_token_id: int[source]#: 1

eos_token_id: int[source]#: 2

head_dim: Optional[int][source]#: None

hidden_act: str[source]#: ‘silu’

hidden_size: int[source]#: 4096

hidden_state_skip_layer: int[source]#: 2

initializer_range: float[source]#: 0.02

intermediate_size: int[source]#: 11008

max_position_embeddings: int[source]#: 2048

mlp_bias: bool[source]#: False

num_attention_heads: int[source]#: 32

num_hidden_layers: int[source]#: 32

num_key_value_heads: Optional[int][source]#: None

pad_token_id: int[source]#: 0

pretraining_tp: int[source]#: 1

rms_norm_eps: float[source]#: 1e-06

rope_scaling: Optional[float][source]#: None

rope_theta: float[source]#: 10000.0

text_len: int[source]#: 256

tie_word_embeddings: bool[source]#: False

use_cache: bool[source]#: True

vocab_size: int[source]#: 32000

class fastvideo.v1.configs.models.encoders.llama.LlamaConfig[source]#

Bases: fastvideo.v1.configs.models.encoders.base.TextEncoderConfig

arch_config: fastvideo.v1.configs.models.encoders.base.TextEncoderArchConfig[source]#: ‘field(…)’

prefix: str#: ‘llama’