micro_batch_size: 4 global_batch_size: 8 tensor_model_parallel_size: 1 pipeline_model_parallel_size: 1 virtual_pipeline_model_parallel_size: null encoder_seq_length: 4096 max_position_embeddings: 4096 num_layers: 24 hidden_size: 2048 ffn_hidden_size: 5440 num_attention_heads: 16 init_method_std: 0.014 use_scaled_init_method: true hidden_dropout: 0.0 attention_dropout: 0.0 ffn_dropout: 0.0 kv_channels: null apply_query_key_layer_scaling: true normalization: layernorm1p layernorm_epsilon: 1.0e-05 do_layer_norm_weight_decay: false make_vocab_size_divisible_by: 128 pre_process: true post_process: true persist_layer_norm: true bias: false activation: fast-swiglu headscale: false transformer_block_type: pre_ln openai_gelu: false normalize_attention_scores: true position_embedding_type: rope rotary_percentage: 0.5 attention_type: multihead share_embeddings_and_output_weights: false tokenizer: library: sentencepiece type: null model: nemo:2b31eecaff7e49149027e10688ca05aa_2053796188904e679f7e2754a2a1f280_mt_nlg_plus_multilingual_ja_zh_the_stack_frac_015_256k.model delimiter: null vocab_file: null merge_file: null sentencepiece_legacy: false tokenizer_model: nemo:a919114446344e349e73a0d807d9af98_mt_nlg_plus_multilingual_ja_zh_the_stack_frac_015_256k.model native_amp_init_scale: 4294967296 native_amp_growth_interval: 1000 hysteresis: 2 fp32_residual_connection: false fp16_lm_cross_entropy: false megatron_amp_O2: true grad_allreduce_chunk_size_mb: 125 grad_div_ar_fusion: true gradient_accumulation_fusion: false bias_activation_fusion: false bias_dropout_add_fusion: false masked_softmax_fusion: true seed: 1234 resume_from_checkpoint: null use_cpu_initialization: false onnx_safe: false apex_transformer_log_level: 30 gradient_as_bucket_view: true sync_batch_comm: false activations_checkpoint_granularity: null activations_checkpoint_method: null activations_checkpoint_num_layers: null num_micro_batches_with_partial_activation_checkpoints: null activations_checkpoint_layers_per_pipeline: null sequence_parallel: false transformer_engine: false fp8: false fp8_e4m3: false fp8_hybrid: false fp8_margin: 0 fp8_interval: 1 fp8_amax_history_len: 1 fp8_amax_compute_algo: most_recent reduce_amax: true use_emha: false optim: name: fused_adam lr: '9e-5' weight_decay: 0.1 betas: - 0.9 - 0.98 sched: name: CosineAnnealing warmup_steps: 500 constant_steps: 50000 min_lr: '9e-6' data: data_impl: mmap splits_string: 98,20 seq_length: 4096 skip_warmup: true num_workers: 2 dataloader_type: single reset_position_ids: false reset_attention_mask: false eod_mask_loss: false index_mapping_dir: null data_prefix: - 0.0333 - /datasets/the_pile/train/my-gpt3_00_text_document - 0.0333 - /datasets/the_pile/train/my-gpt3_01_text_document - 0.0333 - /datasets/the_pile/train/my-gpt3_02_text_document - 0.0333 - /datasets/the_pile/train/my-gpt3_03_text_document - 0.0333 - /datasets/the_pile/train/my-gpt3_04_text_document - 0.0333 - /datasets/the_pile/train/my-gpt3_05_text_document - 0.0333 - /datasets/the_pile/train/my-gpt3_06_text_document - 0.0333 - /datasets/the_pile/train/my-gpt3_07_text_document - 0.0333 - /datasets/the_pile/train/my-gpt3_08_text_document - 0.0333 - /datasets/the_pile/train/my-gpt3_09_text_document - 0.0333 - /datasets/the_pile/train/my-gpt3_10_text_document - 0.0333 - /datasets/the_pile/train/my-gpt3_11_text_document - 0.0333 - /datasets/the_pile/train/my-gpt3_12_text_document - 0.0333 - /datasets/the_pile/train/my-gpt3_13_text_document - 0.0333 - /datasets/the_pile/train/my-gpt3_14_text_document - 0.0333 - /datasets/the_pile/train/my-gpt3_15_text_document - 0.0333 - /datasets/the_pile/train/my-gpt3_16_text_document - 0.0333 - /datasets/the_pile/train/my-gpt3_17_text_document - 0.0333 - /datasets/the_pile/train/my-gpt3_18_text_document - 0.0333 - /datasets/the_pile/train/my-gpt3_19_text_document - 0.0333 - /datasets/the_pile/train/my-gpt3_20_text_document - 0.0333 - /datasets/the_pile/train/my-gpt3_21_text_document - 0.0333 - /datasets/the_pile/train/my-gpt3_22_text_document - 0.0333 - /datasets/the_pile/train/my-gpt3_23_text_document - 0.0333 - /datasets/the_pile/train/my-gpt3_24_text_document - 0.0333 - /datasets/the_pile/train/my-gpt3_25_text_document - 0.0333 - /datasets/the_pile/train/my-gpt3_26_text_document - 0.0333 - /datasets/the_pile/train/my-gpt3_27_text_document - 0.0333 - /datasets/the_pile/train/my-gpt3_28_text_document - 0.0334 - /datasets/the_pile/train/my-gpt3_29_text_document precision: bf16 target: nemo.collections.nlp.models.language_modeling.megatron_gpt_model.MegatronGPTModel nemo_version: 1.15.0rc0 mcore_gpt: true