From b889d5079b730bc4d55b76549c85452dfd477ed1 Mon Sep 17 00:00:00 2001 From: Chen Cui Date: Fri, 2 Oct 2026 13:06:40 -0700 Subject: [PATCH 1/5] feat(recipe): add Qwen3.5 text MXFP8 long-context SFT Signed-off-by: Chen Cui --- docs/recipe-usage.md | 7 + examples/models/qwen/qwen35_text/README.md | 138 +++++++++++++++ src/megatron/bridge/recipes/qwen/__init__.py | 2 + .../bridge/recipes/qwen/gb200/__init__.py | 2 + .../bridge/recipes/qwen/gb200/qwen35.py | 155 ++++++++++++++++- .../recipes/test_qwen35_text_long_context.py | 158 ++++++++++++++++++ 6 files changed, 459 insertions(+), 3 deletions(-) create mode 100644 examples/models/qwen/qwen35_text/README.md create mode 100644 tests/unit_tests/recipes/test_qwen35_text_long_context.py diff --git a/docs/recipe-usage.md b/docs/recipe-usage.md index 3fd192cc45..01f1e77f0f 100644 --- a/docs/recipe-usage.md +++ b/docs/recipe-usage.md @@ -336,3 +336,10 @@ Cluster fabric and orchestration settings do not belong in recipes. Keep credent - [torchrun Documentation](https://docs.pytorch.org/docs/stable/elastic/run.html) - [PyTorch Multinode Training documentation](https://docs.pytorch.org/tutorials/intermediate/ddp_series_multinode.html) - [NeMo-Run documentation](https://docs.nvidia.com/nemo-framework/user-guide/latest/nemorun/index.html#) + +### Qwen3.5 text-only 128K MXFP8 SFT + +The 16-GB200 recipe uses a text-only checkpoint with MTP, CoderForge offline +packs and MXFP8 parameter gather. See the +[checkpoint, data and launch steps](../examples/models/qwen/qwen35_text/README.md) +for runtime prerequisites and the bounded validation results. diff --git a/examples/models/qwen/qwen35_text/README.md b/examples/models/qwen/qwen35_text/README.md new file mode 100644 index 0000000000..3af6cba3e5 --- /dev/null +++ b/examples/models/qwen/qwen35_text/README.md @@ -0,0 +1,138 @@ +# Qwen3.5 35B-A3B text-only 128K MXFP8 SFT + +`qwen35_text_35b_a3b_sft_long_context_16gpu_gb200_fp8mx_config` trains the +text decoder and one pretrained MTP layer on 16 GB200 GPUs within one NVLink +domain: TP1, PP1, CP8, EP16, ETP1, global batch 32 and microbatch 1. +It enables MXFP8 compute and parameter gather, CuTeDSL grouped MLP, single +expert-weight storage, FP32 gradient reduction, learned expert routing, +HybridEP, and selective `gdn_norm_out`/`moe` recomputation. + +## 1. Prepare the text checkpoint + +Download `Qwen/Qwen3.5-35B-A3B` at revision +`59d61f3ce65a6d9863b86d2e96597125219dc754` into a local HF snapshot. Extract its +text weights, then import them into Megatron format: + +```bash +uv run python scripts/conversion/extract_qwen35_text_checkpoint.py \ + --source "$HF_SNAPSHOT" --output "$TEXT_HF" + +./scripts/conversion/convert.sh import \ + --executor local --device cpu \ + --hf-model "$TEXT_HF" --megatron-path "$TEXT_MEGATRON" \ + --torch-dtype bfloat16 +``` + +The extractor retains decoder, output-head and MTP weights and removes the +vision tower. CPU conversion needs enough host memory for the full text model; +use the conversion launcher's distributed GPU backend when necessary. + +## 2. Prepare the data + +The default builder selects `togethercomputer/CoderForge-Preview`, configuration +`trajectories`, split `SWE_Rebench`, pinned to revision +`060fca96cf723b2ebab3181e9e59fafd273df3cb`. It normalizes messages and tools, +derives a seeded 5% validation split, applies the checkpoint's chat template, +and packs conversations into 131072-token budgets. Conversations are aligned +to 16 tokens for CP8; their attention boundaries remain separate. Assistant +responses are supervised; system/user/tool content and padding are masked. +A 128K pack can contain several shorter conversations. + +Set `NEMO_HOME` to shared storage for materialized and packed data. Do not pass +`--dataset coderforge` when retaining the recipe's pinned source and validation +settings: a dataset preset replaces them. Prebuilt packs must use the same +checkpoint tokenizer and CP8 alignment. Their exact paths can be supplied as +shown in the validation overrides below. + +## 3. Launch training + +The validated runtime is NeMo 26.10.rc3 with MCore revision +`b5581a9e874fc5be2b91e74cdd3ccdc64a1b5204` from the pending +[MTP padding-mask fix #7611](https://github.com/NVIDIA/Megatron-LM/pull/7611). +That fix is required for packed MTP correctness and is **not included** in this +recipe PR or the current MCore main pin. Older TE/cuDNN stacks may lack the +grouped-tensor API or padded THD attention support for 256-wide heads. + +For four GPUs per node, request four nodes within one NVLink domain using your +cluster's topology settings. Export `WANDB_API_KEY` before launching: + +```bash +./scripts/training/train.sh \ + --nodes 4 --gpus-per-node 4 \ + --account "$ACCOUNT" --partition "$PARTITION" \ + --container-image "$CONTAINER_IMAGE" \ + --mount "$WORKSPACE" --env HF_HOME --env NEMO_HOME --env WANDB_API_KEY \ + --mount "$PWD/src:/opt/Megatron-Bridge/src" \ + --mount "$PWD/scripts:/opt/Megatron-Bridge/scripts" \ + --mount "$MCORE_SOURCE:/opt/Megatron-Bridge/3rdparty/Megatron-LM" \ + --recipe qwen35_text_35b_a3b_sft_long_context_16gpu_gb200_fp8mx_config \ + --pretrained_checkpoint "$TEXT_MEGATRON" \ + --save_dir "$WORKSPACE/results/qwen35-text-mxfp8" \ + "logger.wandb_project=$WANDB_PROJECT" \ + "logger.wandb_entity=$WANDB_ENTITY" +``` + +Run from this Bridge checkout and set `MCORE_SOURCE` to a separate MCore +checkout containing the revision above. All mounted paths must be accessible +on the allocated nodes. The recipe sets the CuTeDSL and HybridEP environment flags before imports +through the public runner. It runs 500 updates, with warmup 200 and LR decay +horizon 300000. A 100-update check remains within warmup and demonstrates +short-run stability, not completed-schedule convergence. + +The Python recipe accepts `seq_length`, `hf_path` and `hf_revision`. For an +extracted local HF config use `hf_path=TEXT_HF, hf_revision=None`. Config and +tokenizer share a source. Context lengths must be positive multiples of 16 and +fit the model's position limit. Resizing CP/TP/EP requires revalidating packing +alignment, batch divisibility and dispatcher layout. + +## Validation scope + +The reference launcher and migrated recipe each completed 100 updates on +16 GB200 GPUs. Every update had finite LM, MTP and router losses, zero skipped +or NaN iterations, and evaluations at steps 50 and 100. All four loss histories +matched within `1e-6 + 0.01 * abs(reference)` at every update. + +| Measurement | Reference launcher | Bridge recipe | +| --- | ---: | ---: | +| Final LM loss | 0.1486210 | 0.1485086 | +| Final MTP loss | 0.1949851 | 0.1948212 | +| Last 10 updates, seconds/update | 28.9344 | 28.5919 | +| Last 10 updates, model TFLOP/s/GPU | 332.21 | 336.19 | + +Runs: [reference](https://wandb.ai/nvidia/qwen35-text-sft-mcore/runs/qczs0k0o) +and [Bridge](https://wandb.ai/nvidia/qwen35-text-sft-mcore/runs/9hdu7ypa). +These used Bridge base `00ca266c4abc5f8435821823a1dbe05d0c3d6033` plus this +recipe's precursor, with the execution settings now made recipe defaults. +The published change has separate construction and configuration checks on +its newer main base; the 100-step evidence predates that rebase. + +The comparison reused the reference's prebuilt data: a seeded 99:1 split of +77169 trajectories, first 20% of the training split (15279 trajectories), and +all 772 validation trajectories. `first_fit_shuffle` packing produced 6491 +training packs and 333 validation packs. Seed 1234 and the full 300000-step LR +horizon were preserved. This does not qualify the default builder's 5% split. +For the same bounded workload, append these overrides to the launch command: + +```bash +train.train_iters=100 \ +validation.eval_global_batch_size=32 validation.eval_micro_batch_size=1 \ +dataset.hf_dataset=null dataset.hf_validation_proportion=null \ +"dataset.dataset_root=$MATERIALIZED_DATA" \ +"dataset.offline_packing_specs.packed_train_data_path=$PACKED_DATA/training.parquet" \ +"dataset.offline_packing_specs.packed_val_data_path=$PACKED_DATA/validation.parquet" \ +"dataset.offline_packing_specs.packed_metadata_path=$PACKED_DATA/metadata.jsonl" \ +"tokenizer.tokenizer_model=$TEXT_HF" '++tokenizer.hf_tokenizer_kwargs.revision=null' \ +"checkpoint.hf_source_path=$TEXT_HF" \ +checkpoint.load=null checkpoint.load_optim=false checkpoint.load_rng=false \ +checkpoint.save=null checkpoint.save_interval=null checkpoint.async_save=false \ +logger.log_interval=1 logger.log_throughput=true +``` + +The recipe initializes from pretrained weights with FP32 master-parameter +loading, and disables optimizer/RNG loading. For a full training resume, set +`checkpoint.load_main_params_from_ckpt=false`, `checkpoint.load_optim=true`, +`checkpoint.load_rng=true` and an explicit `checkpoint.load`. + +Checkpoint saving was disabled for both comparison runs; checkpoint resume and +post-SFT export were not tested. BF16 qualification is tracked separately and +this change exposes only the verified MXFP8 variant. diff --git a/src/megatron/bridge/recipes/qwen/__init__.py b/src/megatron/bridge/recipes/qwen/__init__.py index d7fbb35414..3a3d7d7846 100644 --- a/src/megatron/bridge/recipes/qwen/__init__.py +++ b/src/megatron/bridge/recipes/qwen/__init__.py @@ -21,6 +21,7 @@ from .gb200.qwen35 import ( qwen35_text_9b_pretrain_8gpu_gb200_bf16_config, qwen35_text_35b_a3b_pretrain_8gpu_gb200_bf16_config, + qwen35_text_35b_a3b_sft_long_context_16gpu_gb200_fp8mx_config, ) # Qwen2 models @@ -175,4 +176,5 @@ "qwen35_text_9b_pretrain_8gpu_gb200_bf16_config", "qwen35_text_35b_a3b_pretrain_config", "qwen35_text_35b_a3b_pretrain_8gpu_gb200_bf16_config", + "qwen35_text_35b_a3b_sft_long_context_16gpu_gb200_fp8mx_config", ] diff --git a/src/megatron/bridge/recipes/qwen/gb200/__init__.py b/src/megatron/bridge/recipes/qwen/gb200/__init__.py index a44732396f..24e4359792 100644 --- a/src/megatron/bridge/recipes/qwen/gb200/__init__.py +++ b/src/megatron/bridge/recipes/qwen/gb200/__init__.py @@ -18,6 +18,7 @@ from megatron.bridge.recipes.qwen.gb200.qwen35 import ( qwen35_text_9b_pretrain_8gpu_gb200_bf16_config, qwen35_text_35b_a3b_pretrain_8gpu_gb200_bf16_config, + qwen35_text_35b_a3b_sft_long_context_16gpu_gb200_fp8mx_config, ) @@ -25,4 +26,5 @@ "qwen3_30b_a3b_pretrain_8gpu_gb200_fp8mx_config", "qwen35_text_9b_pretrain_8gpu_gb200_bf16_config", "qwen35_text_35b_a3b_pretrain_8gpu_gb200_bf16_config", + "qwen35_text_35b_a3b_sft_long_context_16gpu_gb200_fp8mx_config", ] diff --git a/src/megatron/bridge/recipes/qwen/gb200/qwen35.py b/src/megatron/bridge/recipes/qwen/gb200/qwen35.py index f151a90b01..c1eb9812a6 100644 --- a/src/megatron/bridge/recipes/qwen/gb200/qwen35.py +++ b/src/megatron/bridge/recipes/qwen/gb200/qwen35.py @@ -12,22 +12,171 @@ # See the License for the specific language governing permissions and # limitations under the License. -"""GB200 text-only pretraining recipes for Qwen3.5 dense and MoE models.""" +"""GB200 text-only pretraining and SFT recipes for Qwen3.5 models.""" from __future__ import annotations +from copy import deepcopy + import torch from transformers import AutoConfig from megatron.bridge import AutoBridge -from megatron.bridge.recipes.common import _pretrain_common +from megatron.bridge.recipes.common import _pretrain_common, _sft_common +from megatron.bridge.recipes.utils.dataset_utils import default_coderforge_config +from megatron.bridge.recipes.utils.environment_utils import COMMON_RECIPE_ENV_VARS +from megatron.bridge.recipes.utils.optimizer_utils import distributed_fused_adam_with_cosine_annealing from megatron.bridge.training.comm_overlap import CommOverlapConfig from megatron.bridge.training.config import ConfigContainer -from megatron.bridge.training.mixed_precision import bf16_mixed +from megatron.bridge.training.mixed_precision import bf16_mixed, bf16_with_mxfp8_mixed _QWEN35_9B_BASE = "Qwen/Qwen3.5-9B-Base" _QWEN35_35B_A3B_BASE = "Qwen/Qwen3.5-35B-A3B-Base" +_QWEN35_35B_A3B_INSTRUCT_REVISION = "59d61f3ce65a6d9863b86d2e96597125219dc754" # pragma: allowlist secret +_CODERFORGE_REVISION = "060fca96cf723b2ebab3181e9e59fafd273df3cb" # pragma: allowlist secret + + +def qwen35_text_35b_a3b_sft_long_context_16gpu_gb200_fp8mx_config( + seq_length: int = 131072, + *, + hf_path: str = "Qwen/Qwen3.5-35B-A3B", + hf_revision: str | None = _QWEN35_35B_A3B_INSTRUCT_REVISION, +) -> ConfigContainer: + """MXFP8 text SFT with packed chat data, TP1/CP8/EP16 and one MTP layer. + + Set a pretrained text checkpoint before training. The default CoderForge + source is normalized and packed by the dataset builder. Model config and + tokenizer share a pinned revision; use ``hf_revision=None`` for an extracted + local causal-LM checkpoint. No vision model is constructed. + + Args: + seq_length: Packed token budget, a positive multiple of 16 for CP8. + hf_path: HF model ID or extracted local text checkpoint. + hf_revision: Matching model/tokenizer revision, or None for local inputs. + + Returns: + Training configuration with MXFP8 parameter gather and CuTeDSL grouped MLP. + """ + if seq_length <= 0 or seq_length % 16: + raise ValueError("seq_length must be positive and divisible by 16 for CP8.") + cfg = _sft_common() + hf_config = AutoConfig.from_pretrained(hf_path, revision=hf_revision) + text_config = deepcopy(getattr(hf_config, "text_config", hf_config)) + if text_config.model_type != "qwen3_5_moe_text": + raise ValueError("Expected a Qwen3.5 MoE text configuration.") + if seq_length > text_config.max_position_embeddings: + raise ValueError("seq_length exceeds the text model's max_position_embeddings.") + if hasattr(hf_config, "text_config"): + text_config.tie_word_embeddings = hf_config.tie_word_embeddings + text_config.architectures = ["Qwen3_5MoeForCausalLM"] + cfg.model = AutoBridge.from_hf_config(text_config).to_megatron_provider(load_weights=False) + cfg.tokenizer.tokenizer_model = hf_path + cfg.tokenizer.hf_tokenizer_kwargs = {"revision": hf_revision} + + cfg.model.seq_length = seq_length + cfg.model.tensor_model_parallel_size = 1 + cfg.model.pipeline_model_parallel_size = 1 + cfg.model.pipeline_dtype = torch.bfloat16 + cfg.model.virtual_pipeline_model_parallel_size = None + cfg.model.context_parallel_size = 8 + cfg.model.expert_model_parallel_size = 16 + cfg.model.expert_tensor_parallel_size = 1 + cfg.model.sequence_parallel = False + cfg.model.mtp_num_layers = 1 + cfg.model.mtp_loss_scaling_factor = 0.1 + cfg.model.transformer_impl = "transformer_engine" + cfg.model.attention_backend = "auto" + cfg.model.cuda_graph_impl = "none" + cfg.model.gradient_accumulation_fusion = True + cfg.model.bias_activation_fusion = True + cfg.model.cross_entropy_loss_fusion = True + cfg.model.cross_entropy_fusion_impl = "te" + cfg.model.calculate_per_token_loss = True + cfg.model.apply_rope_fusion = False + cfg.model.recompute_granularity = "selective" + cfg.model.recompute_modules = ["gdn_norm_out", "moe"] + cfg.model.recompute_method = None + cfg.model.recompute_num_layers = None + cfg.model.moe_token_dispatcher_type = "flex" + cfg.model.moe_flex_dispatcher_backend = "hybridep" + cfg.model.moe_flex_dispatcher_num_sms = 32 + cfg.model.moe_hybridep_num_sms = None + cfg.model.moe_hybridep_pad_uneven_dispatch_inputs = True + cfg.model.moe_router_fusion = True + cfg.model.moe_permute_fusion = True + cfg.model.moe_grouped_gemm = True + cfg.model.moe_use_grouped_tensor = True + cfg.model.use_transformer_engine_op_fuser = True + cfg.model.moe_mlp_glu_interleave_size = 32 + cfg.model.moe_single_grouped_weight = True + cfg.model.moe_single_grouped_bias = False + cfg.model.moe_shared_expert_overlap = False + cfg.model.moe_router_force_load_balancing = False + cfg.model.overlap_moe_expert_parallel_comm = False + cfg.model.delay_wgrad_compute = False + + # Pad each runtime segment to 2*CP, not each conversation to seq_length. + # Physical and logical cu_seqlens remain distinct for attention and MTP. + cfg.dataset = default_coderforge_config( + seq_length=seq_length, + enable_offline_packing=True, + pad_seq_to_mult=16, + ) + cfg.dataset.dataset_kwargs = {"pad_to_max_length": True} + cfg.dataset.hf_dataset.load_kwargs = {"revision": _CODERFORGE_REVISION} + cfg.dataset.do_validation = True + cfg.dataset.hf_validation_proportion = 0.05 + cfg.dataset.num_workers = 2 + # Preserve the original long-context recipe's sampling and worker lifecycle. + cfg.dataset.seed = 1234 + cfg.dataset.persistent_workers = True + cfg.train.train_iters = 500 + cfg.train.global_batch_size = 32 + cfg.train.micro_batch_size = 1 + cfg.train.manual_gc = True + cfg.train.manual_gc_interval = 100 + cfg.train.manual_gc_eval = 100 + cfg.rng.seed = 1234 + cfg.validation.eval_interval = 50 + cfg.validation.eval_iters = 8 + # Preserve the VL baseline's LR schedule rather than silently retuning SFT. + cfg.optimizer, cfg.scheduler = distributed_fused_adam_with_cosine_annealing( + lr_warmup_iters=200, lr_decay_iters=300000, max_lr=2e-5, min_lr=2e-6 + ) + cfg.optimizer.use_precision_aware_optimizer = False + cfg.optimizer.overlap_param_gather = True + cfg.optimizer.overlap_param_gather_with_optimizer_step = False + cfg.mixed_precision = bf16_with_mxfp8_mixed() + cfg.mixed_precision.fp8_param_gather = True + cfg.mixed_precision.reuse_grad_buf_for_mxfp8_param_ag = True + cfg.checkpoint.load_main_params_from_ckpt = True + cfg.checkpoint.load_optim = False + cfg.checkpoint.load_rng = False + cfg.mixed_precision.grad_reduce_in_fp32 = True + cfg.ddp.grad_reduce_in_fp32 = True + cfg.ddp.use_distributed_optimizer = True + cfg.ddp.data_parallel_sharding_strategy = "optim_grads_params" + cfg.ddp.average_in_collective = False + cfg.ddp.overlap_grad_reduce = True + cfg.ddp.overlap_param_gather = True + cfg.comm_overlap = None # Do not overwrite launch-time DDP/optimizer choices. + cfg.rerun_state_machine.check_for_nan_in_loss = True + cfg.env_vars = { + **COMMON_RECIPE_ENV_VARS, + "CUDA_DEVICE_MAX_CONNECTIONS": 1, + "NVTE_BWD_LAYERNORM_SM_MARGIN": 20, + "NVTE_FWD_LAYERNORM_SM_MARGIN": 20, + "NUM_OF_HYBRID_EP_RANKS_PER_NVLINK_DOMAIN": 16, + "NUM_OF_TOKENS_PER_CHUNK_COMBINE_API": 128, + "NVLINK_DOMAIN_SIZE": 72, + "USE_MNNVL": 1, + "NVTE_CUTEDSL_FUSED_GROUPED_MLP": 1, + "CUDNN_FE_GROUPED_GEMM_DYNAMIC_MNKL": 1, + "NVTE_GROUPED_LINEAR_SINGLE_PARAM": 1, + } + cfg.checkpoint.load = None + return cfg def qwen35_text_9b_pretrain_8gpu_gb200_bf16_config() -> ConfigContainer: diff --git a/tests/unit_tests/recipes/test_qwen35_text_long_context.py b/tests/unit_tests/recipes/test_qwen35_text_long_context.py new file mode 100644 index 0000000000..d84e71b260 --- /dev/null +++ b/tests/unit_tests/recipes/test_qwen35_text_long_context.py @@ -0,0 +1,158 @@ +# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. + +import importlib +from copy import deepcopy +from pathlib import Path +from types import SimpleNamespace + +import pytest +import torch +from transformers import AutoConfig + +from megatron.bridge.recipes.qwen.gb200 import qwen35 + + +pytestmark = pytest.mark.unit +_MXFP8 = qwen35.qwen35_text_35b_a3b_sft_long_context_16gpu_gb200_fp8mx_config + + +@pytest.fixture +def hf_config(monkeypatch): + config = AutoConfig.for_model( + "qwen3_5_moe", + text_config={ + "num_hidden_layers": 4, + "hidden_size": 512, + "num_attention_heads": 4, + "num_key_value_heads": 4, + "head_dim": 128, + "max_position_embeddings": 262144, + "num_experts": 32, + "num_experts_per_tok": 2, + "moe_intermediate_size": 128, + "shared_expert_intermediate_size": 128, + "linear_num_key_heads": 8, + "linear_num_value_heads": 8, + "linear_key_head_dim": 128, + "linear_value_head_dim": 128, + "tie_word_embeddings": True, + }, + tie_word_embeddings=False, + ) + calls = [] + + def load(path, **kwargs): + calls.append((path, kwargs)) + return config + + monkeypatch.setattr(qwen35.AutoConfig, "from_pretrained", load) + return config, calls + + +def test_text_long_context_validates_and_preserves_inputs(hf_config, monkeypatch): + config, calls = hf_config + original = deepcopy(config.to_dict()) + training_module = importlib.import_module("megatron.bridge.training.config") + monkeypatch.setattr(training_module, "get_world_size_safe", lambda: 16) + monkeypatch.setattr( + torch.cuda, "get_device_properties", lambda index: SimpleNamespace(major=10, name="NVIDIA GB200") + ) + + cfg = _MXFP8() + cfg.validate() + + assert config.to_dict() == original + assert calls == [("Qwen/Qwen3.5-35B-A3B", {"revision": qwen35._QWEN35_35B_A3B_INSTRUCT_REVISION})] + assert cfg.tokenizer.hf_tokenizer_kwargs["revision"] == calls[0][1]["revision"] + assert cfg.model.share_embeddings_and_output_weights is False + assert cfg.model.mtp_num_layers == 1 + assert cfg.model.mtp_loss_scaling_factor == 0.1 + assert cfg.model.context_parallel_size == 8 + assert cfg.model.expert_model_parallel_size == 16 + assert cfg.model.tensor_model_parallel_size == 1 + assert cfg.model.sequence_parallel is False + assert cfg.model.recompute_modules == ["gdn_norm_out", "moe"] + assert cfg.dataset.hf_dataset.dataset_name == "coderforge" + assert cfg.dataset.hf_dataset.load_kwargs == {"revision": qwen35._CODERFORGE_REVISION} + assert cfg.dataset.dataset_root is None + assert cfg.dataset.hf_validation_proportion == 0.05 + assert cfg.dataset.offline_packing_specs.pad_seq_to_mult == 16 + assert cfg.dataset.offline_packing_specs.packed_sequence_size == 131072 + assert cfg.dataset.dataset_kwargs["pad_to_max_length"] is True + assert cfg.dataset.seed == 1234 + assert cfg.dataset.persistent_workers is True + assert cfg.train.global_batch_size == 32 + assert cfg.train.micro_batch_size == 1 + assert cfg.train.train_iters == 500 + assert cfg.scheduler.lr_warmup_iters == 200 + assert cfg.scheduler.lr_decay_iters == 300000 + assert cfg.ddp.grad_reduce_in_fp32 is True + + +def test_verified_mxfp8_kernel_and_parameter_settings(hf_config): + cfg = _MXFP8() + assert cfg.mixed_precision.fp8_recipe == "mxfp8" + assert cfg.mixed_precision.fp8_param_gather is True + assert cfg.mixed_precision.reuse_grad_buf_for_mxfp8_param_ag is True + assert cfg.checkpoint.load_main_params_from_ckpt is True + assert cfg.checkpoint.load_optim is False + assert cfg.checkpoint.load_rng is False + assert cfg.model.moe_use_grouped_tensor is True + assert cfg.model.use_transformer_engine_op_fuser is True + assert cfg.model.moe_mlp_glu_interleave_size == 32 + assert cfg.model.moe_single_grouped_weight is True + assert cfg.model.moe_single_grouped_bias is False + assert cfg.env_vars["NUM_OF_HYBRID_EP_RANKS_PER_NVLINK_DOMAIN"] == 16 + assert cfg.env_vars["NVTE_CUTEDSL_FUSED_GROUPED_MLP"] == 1 + assert cfg.env_vars["CUDNN_FE_GROUPED_GEMM_DYNAMIC_MNKL"] == 1 + assert cfg.env_vars["NVTE_GROUPED_LINEAR_SINGLE_PARAM"] == 1 + + +@pytest.mark.parametrize("seq_length", [0, -16, 15, 131073]) +def test_invalid_sequence_length_fails_before_hub_access(hf_config, seq_length): + _, calls = hf_config + with pytest.raises(ValueError, match="positive and divisible by 16"): + _MXFP8(seq_length) + assert calls == [] + + +def test_local_text_config_uses_matching_tokenizer(hf_config, monkeypatch): + config, _ = hf_config + monkeypatch.setattr(qwen35.AutoConfig, "from_pretrained", lambda *_args, **_kwargs: config.text_config) + + cfg = _MXFP8(1024, hf_path="local-text-checkpoint", hf_revision=None) + + assert cfg.tokenizer.tokenizer_model == "local-text-checkpoint" + assert cfg.tokenizer.hf_tokenizer_kwargs == {"revision": None} + assert cfg.model.seq_length == 1024 + assert cfg.dataset.offline_packing_specs.packed_sequence_size == 1024 + + +def test_wrong_model_type_is_rejected(hf_config): + config, _ = hf_config + config.text_config.model_type = "qwen3_5_text" + with pytest.raises(ValueError, match="Expected a Qwen3.5 MoE"): + _MXFP8() + + +@pytest.mark.parametrize("seq_length", [16, 32768, 65536, 131072]) +def test_aligned_context_lengths(hf_config, seq_length): + cfg = _MXFP8(seq_length) + assert cfg.model.seq_length == cfg.dataset.seq_length == seq_length + assert cfg.dataset.offline_packing_specs.packed_sequence_size == seq_length + + +def test_context_exceeding_model_limit_is_rejected(hf_config): + config, _ = hf_config + with pytest.raises(ValueError, match="max_position_embeddings"): + _MXFP8(config.text_config.max_position_embeddings + 16) + + +def test_public_runner_finds_unique_text_recipe(hf_config, monkeypatch): + monkeypatch.syspath_prepend(str(Path(__file__).resolve().parents[3] / "scripts/training")) + runner = importlib.import_module("recipe_runner") + assert runner.find_library_recipe(_MXFP8.__name__) is _MXFP8 + assert runner.find_benchmark_recipe(_MXFP8.__name__) is None + cfg = runner.load_recipe(_MXFP8.__name__) + assert cfg.model.mtp_num_layers == 1 + assert cfg.dataset.hf_dataset.dataset_name == "coderforge" From 4052d535d81bafe6fe4598be48c5bbcc64265566 Mon Sep 17 00:00:00 2001 From: Chen Cui Date: Sat, 3 Oct 2026 16:59:33 -0700 Subject: [PATCH 2/5] docs(verification): record Qwen3.5 text MXFP8 run in model card Signed-off-by: Chen Cui --- docs/recipe-usage.md | 7 - .../qwen3.5-35b-a3b-text/card.yaml | 214 ++++++++++++++++++ examples/models/qwen/qwen35_text/README.md | 138 ----------- .../create-model-verification-card/SKILL.md | 8 + .../scripts/validate_card.py | 14 +- .../test_validate_card.py | 43 ++++ 6 files changed, 277 insertions(+), 147 deletions(-) create mode 100644 examples/model_verification_cards/qwen3.5-35b-a3b-text/card.yaml delete mode 100644 examples/models/qwen/qwen35_text/README.md diff --git a/docs/recipe-usage.md b/docs/recipe-usage.md index 01f1e77f0f..3fd192cc45 100644 --- a/docs/recipe-usage.md +++ b/docs/recipe-usage.md @@ -336,10 +336,3 @@ Cluster fabric and orchestration settings do not belong in recipes. Keep credent - [torchrun Documentation](https://docs.pytorch.org/docs/stable/elastic/run.html) - [PyTorch Multinode Training documentation](https://docs.pytorch.org/tutorials/intermediate/ddp_series_multinode.html) - [NeMo-Run documentation](https://docs.nvidia.com/nemo-framework/user-guide/latest/nemorun/index.html#) - -### Qwen3.5 text-only 128K MXFP8 SFT - -The 16-GB200 recipe uses a text-only checkpoint with MTP, CoderForge offline -packs and MXFP8 parameter gather. See the -[checkpoint, data and launch steps](../examples/models/qwen/qwen35_text/README.md) -for runtime prerequisites and the bounded validation results. diff --git a/examples/model_verification_cards/qwen3.5-35b-a3b-text/card.yaml b/examples/model_verification_cards/qwen3.5-35b-a3b-text/card.yaml new file mode 100644 index 0000000000..7b4d156714 --- /dev/null +++ b/examples/model_verification_cards/qwen3.5-35b-a3b-text/card.yaml @@ -0,0 +1,214 @@ +# Agent-readable model verification card. +# Historical observations are not attributed to a later clean source revision. +title: qwen3_5_35b_a3b_text +summary: > + Performance disclaimer: this card does not record a canonical pretrain performance result; reported timing and throughput metrics are functional verification observations, not standalone optimized performance results. + Records the completed 100-update MXFP8 text-only long-context SFT run. + The run used an uncommitted recipe precursor and a release-candidate runtime; + clean-source verification of the published recipe remains unverified. +verification_index: + model_level: + unverified: [hf_to_megatron_cpu, hf_to_megatron_gpu, megatron_to_hf_cpu, megatron_to_hf_gpu, manual_forward_pass, inference] + training: + GB200: + unverified: [pretrain, sft, sft_export_inference, sft_long_context, peft, checkpoint_resume] +model: + hf_id: Qwen/Qwen3.5-35B-A3B + hf_revision: 59d61f3ce65a6d9863b86d2e96597125219dc754 # pragma: allowlist secret + architecture: Qwen3_5MoeForCausalLM + min_transformers_version: "5.12.1" +verification_environment: + # No public image or clean commit identifies the historical runtime exactly. + base_container: null + bridge_commit: null + +items: + hf_to_megatron_cpu: + status: unverified + precision: bf16 + command: null + last_verified: null + expected_result: This workflow has no independently recorded verification in this text-only card. + + hf_to_megatron_gpu: + status: unverified + precision: bf16 + command: null + last_verified: null + expected_result: This workflow has no independently recorded verification in this text-only card. + + megatron_to_hf_cpu: + status: unverified + precision: bf16 + command: null + last_verified: null + expected_result: This workflow has no independently recorded verification in this text-only card. + + megatron_to_hf_gpu: + status: unverified + precision: bf16 + command: null + last_verified: null + expected_result: This workflow has no independently recorded verification in this text-only card. + + manual_forward_pass: + status: unverified + precision: bf16 + command: null + last_verified: null + expected_result: This workflow has no independently recorded verification in this text-only card. + + inference: + status: unverified + precision: bf16 + command: null + last_verified: null + expected_result: This workflow has no independently recorded verification in this text-only card. + + pretrain: + GB200: + status: unverified + precision: bf16 + enabled_features: {} + command: null + last_verified: null + metrics: + initial_loss: null + final_loss: null + last_10_steps_step_time_ms_avg: null + last_10_steps_model_tflops_per_gpu_avg: null + last_10_steps_tokens_per_second_per_gpu_avg: null + expected_result: This workflow has no independently recorded verification in this text-only card. + + sft: + GB200: + status: unverified + precision: bf16 + enabled_features: {} + command: null + last_verified: null + metrics: + initial_loss: null + final_loss: null + last_10_steps_step_time_ms_avg: null + last_10_steps_model_tflops_per_gpu_avg: null + last_10_steps_tokens_per_second_per_gpu_avg: null + expected_result: This workflow has no independently recorded verification in this text-only card. + + sft_export_inference: + GB200: + status: unverified + precision: bf16 + depends_on: sft + commands: null + last_verified: null + expected_result: This workflow has no independently recorded verification in this text-only card. + + sft_long_context: + GB200: + status: unverified + precision: fp8_mx + enabled_features: + sequence_packing: offline + context_parallel_size: 8 + moe_dispatcher: hybridep + command: > + ./scripts/training/train.sh --nodes 4 --gpus-per-node 4 + --recipe qwen35_text_35b_a3b_sft_long_context_16gpu_gb200_fp8mx_config + --pretrained_checkpoint work/model-verification/qwen3.5-35b-a3b-text/text-megatron + --max_steps 100 + dataset.hf_dataset=null dataset.hf_validation_proportion=null + dataset.dataset_root=work/data/coderforge/materialized + dataset.offline_packing_specs.packed_train_data_path=work/data/coderforge/packed-128k/training.parquet + dataset.offline_packing_specs.packed_val_data_path=work/data/coderforge/packed-128k/validation.parquet + dataset.offline_packing_specs.packed_metadata_path=work/data/coderforge/packed-128k/metadata.jsonl + tokenizer.tokenizer_model=work/cache/qwen3.5-35b-a3b-text + '++tokenizer.hf_tokenizer_kwargs.revision=null' + checkpoint.hf_source_path=work/cache/qwen3.5-35b-a3b-text + validation.eval_global_batch_size=32 validation.eval_micro_batch_size=1 + checkpoint.save=null checkpoint.save_interval=null checkpoint.async_save=false + logger.log_interval=1 logger.log_throughput=true + logger.save_config_filepath=work/model-verification/qwen3.5-35b-a3b-text/resolved-config.yaml + last_verified: null + metrics: + initial_loss: null + final_loss: null + last_10_steps_step_time_ms_avg: null + last_10_steps_model_tflops_per_gpu_avg: null + last_10_steps_tokens_per_second_per_gpu_avg: null + expected_result: > + Historical observation, completed 2026-10-01: the text decoder and one + pretrained MTP layer completed exactly 100 optimizer updates on 16 GB200 + GPUs at TP1/PP1/CP8/EP16/ETP1, GBS32/MBS1 and 16 accumulation steps. + LM, MTP, router losses and gradient norms were finite, with zero skipped + or NaN updates and evaluations at steps 50 and 100. Online experiment + logging finished successfully and the post-setup config was retained. + Initial LM loss was 0.2099137008190155; final LM loss was + 0.14850857853889465; final MTP loss was 0.19482117891311646. + Updates 91-100 averaged 28591.89546108246 ms and + 336.1899744993698 model TFLOP/s/GPU, corresponding to + 9168.47224615851 token slots/s/GPU, not supervised-token throughput. + These historical observations are deliberately separate from the null + clean-recipe metrics above. + The run used NeMo 26.10.rc3 and MCore + b5581a9e874fc5be2b91e74cdd3ccdc64a1b5204 from NVIDIA/Megatron-LM PR 7611. + Its Bridge source was base 00ca266c4abc5f8435821823a1dbe05d0c3d6033 + plus an uncommitted recipe precursor, frozen as source bundle SHA256 + 5ea6d3c0c82a47f06f9f031040c2cc78641a4334d961542e25338f9c585ed863. + No clean source commit or public runtime image identifies that run; + the null provenance fields must not be replaced with the later PR head. + The command above describes the published recipe with equivalent + workload settings, checked by construction and full configuration + comparison, but it has not been rerun from a clean checkout. + Inputs were text/MTP weights extracted from the pinned HF model and + togethercomputer/CoderForge-Preview revision + 060fca96cf723b2ebab3181e9e59fafd273df3cb, configuration trajectories, + split SWE_Rebench. Seed 1234 produced a 99:1 split of 77169 rows; + the first 20 percent of training rows (15279) and all 772 validation + rows were packed with first_fit_shuffle, a 131072-token budget and + 16-token alignment. The 6491 training packs contain 205182453 supervised + tokens in the entire prepared corpus; the 333 validation packs contain + 10479031. These counts are corpus totals, not tokens consumed by the run. + The published builder's default five-percent validation split was not + exercised. MXFP8 parameter gather and gradient-buffer reuse, CuTeDSL + grouped MLP, single grouped weights, FP32 gradient reduction, learned + routing and selective gdn_norm_out/moe recomputation were enabled. + Warmup 200 and the 300000-step cosine-decay horizon were retained, so + this run remained in warmup and does not establish full convergence. + Checkpoint writes were disabled; resume and post-SFT export are untested. + + peft: + GB200: + status: unverified + precision: bf16 + enabled_features: {} + command: null + last_verified: null + metrics: + initial_loss: null + final_loss: null + last_10_steps_step_time_ms_avg: null + last_10_steps_model_tflops_per_gpu_avg: null + last_10_steps_tokens_per_second_per_gpu_avg: null + expected_result: This workflow has no independently recorded verification in this text-only card. + + checkpoint_resume: + GB200: + status: unverified + precision: bf16 + command: null + last_verified: null + metrics: + initial_loss: null + final_loss: null + last_10_steps_step_time_ms_avg: null + last_10_steps_model_tflops_per_gpu_avg: null + last_10_steps_tokens_per_second_per_gpu_avg: null + depends_on: pretrain + resume_comparison: + reference_item: pretrain + sentinel_steps: [] + loss_relative_tolerance: 1.0e-2 + loss_absolute_tolerance: 1.0e-6 + sentinels_match: null + expected_result: This workflow has no independently recorded verification in this text-only card. diff --git a/examples/models/qwen/qwen35_text/README.md b/examples/models/qwen/qwen35_text/README.md deleted file mode 100644 index 3af6cba3e5..0000000000 --- a/examples/models/qwen/qwen35_text/README.md +++ /dev/null @@ -1,138 +0,0 @@ -# Qwen3.5 35B-A3B text-only 128K MXFP8 SFT - -`qwen35_text_35b_a3b_sft_long_context_16gpu_gb200_fp8mx_config` trains the -text decoder and one pretrained MTP layer on 16 GB200 GPUs within one NVLink -domain: TP1, PP1, CP8, EP16, ETP1, global batch 32 and microbatch 1. -It enables MXFP8 compute and parameter gather, CuTeDSL grouped MLP, single -expert-weight storage, FP32 gradient reduction, learned expert routing, -HybridEP, and selective `gdn_norm_out`/`moe` recomputation. - -## 1. Prepare the text checkpoint - -Download `Qwen/Qwen3.5-35B-A3B` at revision -`59d61f3ce65a6d9863b86d2e96597125219dc754` into a local HF snapshot. Extract its -text weights, then import them into Megatron format: - -```bash -uv run python scripts/conversion/extract_qwen35_text_checkpoint.py \ - --source "$HF_SNAPSHOT" --output "$TEXT_HF" - -./scripts/conversion/convert.sh import \ - --executor local --device cpu \ - --hf-model "$TEXT_HF" --megatron-path "$TEXT_MEGATRON" \ - --torch-dtype bfloat16 -``` - -The extractor retains decoder, output-head and MTP weights and removes the -vision tower. CPU conversion needs enough host memory for the full text model; -use the conversion launcher's distributed GPU backend when necessary. - -## 2. Prepare the data - -The default builder selects `togethercomputer/CoderForge-Preview`, configuration -`trajectories`, split `SWE_Rebench`, pinned to revision -`060fca96cf723b2ebab3181e9e59fafd273df3cb`. It normalizes messages and tools, -derives a seeded 5% validation split, applies the checkpoint's chat template, -and packs conversations into 131072-token budgets. Conversations are aligned -to 16 tokens for CP8; their attention boundaries remain separate. Assistant -responses are supervised; system/user/tool content and padding are masked. -A 128K pack can contain several shorter conversations. - -Set `NEMO_HOME` to shared storage for materialized and packed data. Do not pass -`--dataset coderforge` when retaining the recipe's pinned source and validation -settings: a dataset preset replaces them. Prebuilt packs must use the same -checkpoint tokenizer and CP8 alignment. Their exact paths can be supplied as -shown in the validation overrides below. - -## 3. Launch training - -The validated runtime is NeMo 26.10.rc3 with MCore revision -`b5581a9e874fc5be2b91e74cdd3ccdc64a1b5204` from the pending -[MTP padding-mask fix #7611](https://github.com/NVIDIA/Megatron-LM/pull/7611). -That fix is required for packed MTP correctness and is **not included** in this -recipe PR or the current MCore main pin. Older TE/cuDNN stacks may lack the -grouped-tensor API or padded THD attention support for 256-wide heads. - -For four GPUs per node, request four nodes within one NVLink domain using your -cluster's topology settings. Export `WANDB_API_KEY` before launching: - -```bash -./scripts/training/train.sh \ - --nodes 4 --gpus-per-node 4 \ - --account "$ACCOUNT" --partition "$PARTITION" \ - --container-image "$CONTAINER_IMAGE" \ - --mount "$WORKSPACE" --env HF_HOME --env NEMO_HOME --env WANDB_API_KEY \ - --mount "$PWD/src:/opt/Megatron-Bridge/src" \ - --mount "$PWD/scripts:/opt/Megatron-Bridge/scripts" \ - --mount "$MCORE_SOURCE:/opt/Megatron-Bridge/3rdparty/Megatron-LM" \ - --recipe qwen35_text_35b_a3b_sft_long_context_16gpu_gb200_fp8mx_config \ - --pretrained_checkpoint "$TEXT_MEGATRON" \ - --save_dir "$WORKSPACE/results/qwen35-text-mxfp8" \ - "logger.wandb_project=$WANDB_PROJECT" \ - "logger.wandb_entity=$WANDB_ENTITY" -``` - -Run from this Bridge checkout and set `MCORE_SOURCE` to a separate MCore -checkout containing the revision above. All mounted paths must be accessible -on the allocated nodes. The recipe sets the CuTeDSL and HybridEP environment flags before imports -through the public runner. It runs 500 updates, with warmup 200 and LR decay -horizon 300000. A 100-update check remains within warmup and demonstrates -short-run stability, not completed-schedule convergence. - -The Python recipe accepts `seq_length`, `hf_path` and `hf_revision`. For an -extracted local HF config use `hf_path=TEXT_HF, hf_revision=None`. Config and -tokenizer share a source. Context lengths must be positive multiples of 16 and -fit the model's position limit. Resizing CP/TP/EP requires revalidating packing -alignment, batch divisibility and dispatcher layout. - -## Validation scope - -The reference launcher and migrated recipe each completed 100 updates on -16 GB200 GPUs. Every update had finite LM, MTP and router losses, zero skipped -or NaN iterations, and evaluations at steps 50 and 100. All four loss histories -matched within `1e-6 + 0.01 * abs(reference)` at every update. - -| Measurement | Reference launcher | Bridge recipe | -| --- | ---: | ---: | -| Final LM loss | 0.1486210 | 0.1485086 | -| Final MTP loss | 0.1949851 | 0.1948212 | -| Last 10 updates, seconds/update | 28.9344 | 28.5919 | -| Last 10 updates, model TFLOP/s/GPU | 332.21 | 336.19 | - -Runs: [reference](https://wandb.ai/nvidia/qwen35-text-sft-mcore/runs/qczs0k0o) -and [Bridge](https://wandb.ai/nvidia/qwen35-text-sft-mcore/runs/9hdu7ypa). -These used Bridge base `00ca266c4abc5f8435821823a1dbe05d0c3d6033` plus this -recipe's precursor, with the execution settings now made recipe defaults. -The published change has separate construction and configuration checks on -its newer main base; the 100-step evidence predates that rebase. - -The comparison reused the reference's prebuilt data: a seeded 99:1 split of -77169 trajectories, first 20% of the training split (15279 trajectories), and -all 772 validation trajectories. `first_fit_shuffle` packing produced 6491 -training packs and 333 validation packs. Seed 1234 and the full 300000-step LR -horizon were preserved. This does not qualify the default builder's 5% split. -For the same bounded workload, append these overrides to the launch command: - -```bash -train.train_iters=100 \ -validation.eval_global_batch_size=32 validation.eval_micro_batch_size=1 \ -dataset.hf_dataset=null dataset.hf_validation_proportion=null \ -"dataset.dataset_root=$MATERIALIZED_DATA" \ -"dataset.offline_packing_specs.packed_train_data_path=$PACKED_DATA/training.parquet" \ -"dataset.offline_packing_specs.packed_val_data_path=$PACKED_DATA/validation.parquet" \ -"dataset.offline_packing_specs.packed_metadata_path=$PACKED_DATA/metadata.jsonl" \ -"tokenizer.tokenizer_model=$TEXT_HF" '++tokenizer.hf_tokenizer_kwargs.revision=null' \ -"checkpoint.hf_source_path=$TEXT_HF" \ -checkpoint.load=null checkpoint.load_optim=false checkpoint.load_rng=false \ -checkpoint.save=null checkpoint.save_interval=null checkpoint.async_save=false \ -logger.log_interval=1 logger.log_throughput=true -``` - -The recipe initializes from pretrained weights with FP32 master-parameter -loading, and disables optimizer/RNG loading. For a full training resume, set -`checkpoint.load_main_params_from_ckpt=false`, `checkpoint.load_optim=true`, -`checkpoint.load_rng=true` and an explicit `checkpoint.load`. - -Checkpoint saving was disabled for both comparison runs; checkpoint resume and -post-SFT export were not tested. BF16 qualification is tracked separately and -this change exposes only the verified MXFP8 variant. diff --git a/skills/create-model-verification-card/SKILL.md b/skills/create-model-verification-card/SKILL.md index 813cc257e8..0e27978d61 100644 --- a/skills/create-model-verification-card/SKILL.md +++ b/skills/create-model-verification-card/SKILL.md @@ -46,6 +46,14 @@ identifier and the exact Bridge commit used for verification. Put them in the container, record the mounted checkout commit. Never substitute a private image path for the public base container identifier. +When no item is verified, either environment field may be null if its exact +public provenance is unavailable. Keep historical observations in the relevant +unverified item's `expected_result`, with the actual source/runtime limitations +stated explicitly and its structured metrics and date null. Do not substitute +a later clean commit or invent a public image for a release-candidate run. +This applies to draft and historical cards for any model. As soon as any item +is verified, both environment fields must satisfy the existing requirements. + The top-level Bridge commit is the default for every item leaf. If one verified item was run from a different clean checkout, put that exact 40-hex commit in the leaf's optional `bridge_commit` field. Omit the field when it would repeat diff --git a/skills/create-model-verification-card/scripts/validate_card.py b/skills/create-model-verification-card/scripts/validate_card.py index 3611bff120..6c19431001 100644 --- a/skills/create-model-verification-card/scripts/validate_card.py +++ b/skills/create-model-verification-card/scripts/validate_card.py @@ -2037,14 +2037,24 @@ def _validate_card(card: Mapping[str, Any], raw: str, deny_terms: tuple[str, ... path=("verification_environment",), errors=errors, ) + # An unverified card may not yet have publishable, exact provenance. + # Never force an invented image/commit, or relax provenance for verified runs. + item_values = card.get("items") + has_verified_item = isinstance(item_values, Mapping) and any( + item.get("status") == "verified" for _, _, item, _ in _iter_item_leaves(item_values) + ) base_container = environment.get("base_container") - if not isinstance(base_container, str) or PUBLIC_BASE_CONTAINER_RE.fullmatch(base_container) is None: + if not (base_container is None and not has_verified_item) and ( + not isinstance(base_container, str) or PUBLIC_BASE_CONTAINER_RE.fullmatch(base_container) is None + ): errors.append( f"{_pointer('verification_environment', 'base_container')}: " "expected a public NVIDIA NeMo or PyTorch container" ) bridge_commit = environment.get("bridge_commit") - if not isinstance(bridge_commit, str) or REVISION_RE.fullmatch(bridge_commit) is None: + if not (bridge_commit is None and not has_verified_item) and ( + not isinstance(bridge_commit, str) or REVISION_RE.fullmatch(bridge_commit) is None + ): errors.append( f"{_pointer('verification_environment', 'bridge_commit')}: expected an immutable 40-hex commit" ) diff --git a/tests/unit_tests/skills/create_model_verification_card/test_validate_card.py b/tests/unit_tests/skills/create_model_verification_card/test_validate_card.py index f83fd9e557..f269f80255 100644 --- a/tests/unit_tests/skills/create_model_verification_card/test_validate_card.py +++ b/tests/unit_tests/skills/create_model_verification_card/test_validate_card.py @@ -796,3 +796,46 @@ def test_resume_reference_only_warm_start(resume_initialization, expected_errors ) assert errors == expected_errors + + +def test_unverified_card_can_record_unknown_provenance_without_fabricated_values(): + validator = _load_validator() + path = REPO_ROOT / "examples/model_verification_cards/qwen3.5-35b-a3b-text/card.yaml" + raw = path.read_text() + card = yaml.safe_load(raw) + + assert card["verification_environment"] == {"base_container": None, "bridge_commit": None} + assert validator._validate_card(card, raw, ()) == [] + + +@pytest.mark.parametrize( + "items", + [ + {"inference": {"status": "verified"}}, + {"sft_long_context": {"GB200": {"status": "verified"}}}, + {"pretrain_fsdp": {"GB200": {"status": "unverified", "variants": {"fp8_mx": {"status": "verified"}}}}}, + {"pretrain_weak_scaling": {"GB300": {"status": "verified"}}}, + ], +) +def test_verified_items_still_require_public_container_and_immutable_commit(items): + validator = _load_validator() + card = {"verification_environment": {"base_container": None, "bridge_commit": None}, "items": items} + + errors = validator._validate_card(card, yaml.safe_dump(card), ()) + + assert "/verification_environment/base_container: expected a public NVIDIA NeMo or PyTorch container" in errors + assert "/verification_environment/bridge_commit: expected an immutable 40-hex commit" in errors + + +@pytest.mark.parametrize("field,value", [("base_container", "not-an-image"), ("bridge_commit", "main")]) +def test_unverified_provenance_rejects_non_null_invalid_values(field, value): + validator = _load_validator() + card = { + "verification_environment": {"base_container": None, "bridge_commit": None}, + "items": {"inference": {"status": "unverified"}}, + } + card["verification_environment"][field] = value + + errors = validator._validate_card(card, yaml.safe_dump(card), ()) + + assert any(error.startswith(f"/verification_environment/{field}:") for error in errors) From c589a81fdfef6f6847e8a9ae16552ed5061770d1 Mon Sep 17 00:00:00 2001 From: Chen Cui Date: Mon, 5 Oct 2026 16:34:06 -0700 Subject: [PATCH 3/5] fix(tests): support Qwen3.5 text recipe and verification catalog Signed-off-by: Chen Cui --- .../model-verification/catalog-v1.json | 261 +++++++++++++ docs/fern/versions/nightly.yml | 2 + .../versions/nightly/pages/models/README.mdx | 8 +- .../nightly/pages/models/qwen/index.mdx | 1 + .../models/qwen/qwen3.5-35b-a3b-text.mdx | 366 +++++++++++++++++ docs/models/README.md | 8 +- docs/models/qwen/index.md | 2 + docs/models/qwen/qwen3.5-35b-a3b-text.md | 367 ++++++++++++++++++ .../generate_model_verification_catalog.py | 2 + tests/unit_tests/recipes/recipe_test_utils.py | 9 +- tests/unit_tests/recipes/test_qwen_recipes.py | 7 +- 11 files changed, 1026 insertions(+), 7 deletions(-) create mode 100644 docs/fern/versions/nightly/pages/models/qwen/qwen3.5-35b-a3b-text.mdx create mode 100644 docs/models/qwen/qwen3.5-35b-a3b-text.md diff --git a/docs/_static/model-verification/catalog-v1.json b/docs/_static/model-verification/catalog-v1.json index d93c56e9ec..9632210936 100644 --- a/docs/_static/model-verification/catalog-v1.json +++ b/docs/_static/model-verification/catalog-v1.json @@ -9671,6 +9671,267 @@ "summary": "Performance disclaimer: this model has not been performance-tuned; reported timing and throughput metrics are sanity checks, not optimized performance results. Qwen3-8B support verification covers conversion, inference, and training.\n", "title": "qwen3_8b" }, + { + "architecture": "Qwen3_5MoeForCausalLM", + "base_container": null, + "bridge_commit": null, + "entries": [ + { + "bridge_commit": null, + "command_topologies": [], + "commands": [], + "dimensions": {}, + "enabled_features": {}, + "entry_id": "qwen3-5-35b-a3b-text-hf-to-megatron-cpu", + "expected_result": "This workflow has no independently recorded verification in this text-only card.", + "hardware": null, + "last_verified": null, + "metrics": {}, + "precision": "bf16", + "source_pointer": "items.hf_to_megatron_cpu", + "status": "unverified", + "variant": null, + "workflow": "hf_to_megatron_cpu" + }, + { + "bridge_commit": null, + "command_topologies": [], + "commands": [], + "dimensions": {}, + "enabled_features": {}, + "entry_id": "qwen3-5-35b-a3b-text-hf-to-megatron-gpu", + "expected_result": "This workflow has no independently recorded verification in this text-only card.", + "hardware": null, + "last_verified": null, + "metrics": {}, + "precision": "bf16", + "source_pointer": "items.hf_to_megatron_gpu", + "status": "unverified", + "variant": null, + "workflow": "hf_to_megatron_gpu" + }, + { + "bridge_commit": null, + "command_topologies": [], + "commands": [], + "dimensions": {}, + "enabled_features": {}, + "entry_id": "qwen3-5-35b-a3b-text-megatron-to-hf-cpu", + "expected_result": "This workflow has no independently recorded verification in this text-only card.", + "hardware": null, + "last_verified": null, + "metrics": {}, + "precision": "bf16", + "source_pointer": "items.megatron_to_hf_cpu", + "status": "unverified", + "variant": null, + "workflow": "megatron_to_hf_cpu" + }, + { + "bridge_commit": null, + "command_topologies": [], + "commands": [], + "dimensions": {}, + "enabled_features": {}, + "entry_id": "qwen3-5-35b-a3b-text-megatron-to-hf-gpu", + "expected_result": "This workflow has no independently recorded verification in this text-only card.", + "hardware": null, + "last_verified": null, + "metrics": {}, + "precision": "bf16", + "source_pointer": "items.megatron_to_hf_gpu", + "status": "unverified", + "variant": null, + "workflow": "megatron_to_hf_gpu" + }, + { + "bridge_commit": null, + "command_topologies": [], + "commands": [], + "dimensions": {}, + "enabled_features": {}, + "entry_id": "qwen3-5-35b-a3b-text-manual-forward-pass", + "expected_result": "This workflow has no independently recorded verification in this text-only card.", + "hardware": null, + "last_verified": null, + "metrics": {}, + "precision": "bf16", + "source_pointer": "items.manual_forward_pass", + "status": "unverified", + "variant": null, + "workflow": "manual_forward_pass" + }, + { + "bridge_commit": null, + "command_topologies": [], + "commands": [], + "dimensions": {}, + "enabled_features": {}, + "entry_id": "qwen3-5-35b-a3b-text-inference", + "expected_result": "This workflow has no independently recorded verification in this text-only card.", + "hardware": null, + "last_verified": null, + "metrics": {}, + "precision": "bf16", + "source_pointer": "items.inference", + "status": "unverified", + "variant": null, + "workflow": "inference" + }, + { + "bridge_commit": null, + "command_topologies": [], + "commands": [], + "dimensions": {}, + "enabled_features": {}, + "entry_id": "qwen3-5-35b-a3b-text-pretrain-gb200", + "expected_result": "This workflow has no independently recorded verification in this text-only card.", + "hardware": "GB200", + "last_verified": null, + "metrics": { + "final_loss": null, + "initial_loss": null, + "last_10_steps_model_tflops_per_gpu_avg": null, + "last_10_steps_step_time_ms_avg": null, + "last_10_steps_tokens_per_second_per_gpu_avg": null + }, + "precision": "bf16", + "source_pointer": "items.pretrain.GB200", + "status": "unverified", + "variant": null, + "workflow": "pretrain" + }, + { + "bridge_commit": null, + "command_topologies": [], + "commands": [], + "dimensions": {}, + "enabled_features": {}, + "entry_id": "qwen3-5-35b-a3b-text-sft-gb200", + "expected_result": "This workflow has no independently recorded verification in this text-only card.", + "hardware": "GB200", + "last_verified": null, + "metrics": { + "final_loss": null, + "initial_loss": null, + "last_10_steps_model_tflops_per_gpu_avg": null, + "last_10_steps_step_time_ms_avg": null, + "last_10_steps_tokens_per_second_per_gpu_avg": null + }, + "precision": "bf16", + "source_pointer": "items.sft.GB200", + "status": "unverified", + "variant": null, + "workflow": "sft" + }, + { + "bridge_commit": null, + "command_topologies": [], + "commands": [], + "dimensions": {}, + "enabled_features": {}, + "entry_id": "qwen3-5-35b-a3b-text-sft-export-inference-gb200", + "expected_result": "This workflow has no independently recorded verification in this text-only card.", + "hardware": "GB200", + "last_verified": null, + "metrics": {}, + "precision": "bf16", + "source_pointer": "items.sft_export_inference.GB200", + "status": "unverified", + "variant": null, + "workflow": "sft_export_inference" + }, + { + "bridge_commit": null, + "command_topologies": [ + { + "gpus_per_node": "4", + "nodes": "4", + "recipe": "qwen35_text_35b_a3b_sft_long_context_16gpu_gb200_fp8mx_config", + "total_gpus": 16 + } + ], + "commands": [ + "./scripts/training/train.sh --nodes 4 --gpus-per-node 4 --recipe qwen35_text_35b_a3b_sft_long_context_16gpu_gb200_fp8mx_config --pretrained_checkpoint work/model-verification/qwen3.5-35b-a3b-text/text-megatron --max_steps 100 dataset.hf_dataset=null dataset.hf_validation_proportion=null dataset.dataset_root=work/data/coderforge/materialized dataset.offline_packing_specs.packed_train_data_path=work/data/coderforge/packed-128k/training.parquet dataset.offline_packing_specs.packed_val_data_path=work/data/coderforge/packed-128k/validation.parquet dataset.offline_packing_specs.packed_metadata_path=work/data/coderforge/packed-128k/metadata.jsonl tokenizer.tokenizer_model=work/cache/qwen3.5-35b-a3b-text '++tokenizer.hf_tokenizer_kwargs.revision=null' checkpoint.hf_source_path=work/cache/qwen3.5-35b-a3b-text validation.eval_global_batch_size=32 validation.eval_micro_batch_size=1 checkpoint.save=null checkpoint.save_interval=null checkpoint.async_save=false logger.log_interval=1 logger.log_throughput=true logger.save_config_filepath=work/model-verification/qwen3.5-35b-a3b-text/resolved-config.yaml" + ], + "dimensions": {}, + "enabled_features": { + "context_parallel_size": 8, + "moe_dispatcher": "hybridep", + "sequence_packing": "offline" + }, + "entry_id": "qwen3-5-35b-a3b-text-sft-long-context-gb200", + "expected_result": "Historical observation, completed 2026-10-01: the text decoder and one pretrained MTP layer completed exactly 100 optimizer updates on 16 GB200 GPUs at TP1/PP1/CP8/EP16/ETP1, GBS32/MBS1 and 16 accumulation steps. LM, MTP, router losses and gradient norms were finite, with zero skipped or NaN updates and evaluations at steps 50 and 100. Online experiment logging finished successfully and the post-setup config was retained. Initial LM loss was 0.2099137008190155; final LM loss was 0.14850857853889465; final MTP loss was 0.19482117891311646. Updates 91-100 averaged 28591.89546108246 ms and 336.1899744993698 model TFLOP/s/GPU, corresponding to 9168.47224615851 token slots/s/GPU, not supervised-token throughput. These historical observations are deliberately separate from the null clean-recipe metrics above. The run used NeMo 26.10.rc3 and MCore b5581a9e874fc5be2b91e74cdd3ccdc64a1b5204 from NVIDIA/Megatron-LM PR 7611. Its Bridge source was base 00ca266c4abc5f8435821823a1dbe05d0c3d6033 plus an uncommitted recipe precursor, frozen as source bundle SHA256 5ea6d3c0c82a47f06f9f031040c2cc78641a4334d961542e25338f9c585ed863. No clean source commit or public runtime image identifies that run; the null provenance fields must not be replaced with the later PR head. The command above describes the published recipe with equivalent workload settings, checked by construction and full configuration comparison, but it has not been rerun from a clean checkout. Inputs were text/MTP weights extracted from the pinned HF model and togethercomputer/CoderForge-Preview revision 060fca96cf723b2ebab3181e9e59fafd273df3cb, configuration trajectories, split SWE_Rebench. Seed 1234 produced a 99:1 split of 77169 rows; the first 20 percent of training rows (15279) and all 772 validation rows were packed with first_fit_shuffle, a 131072-token budget and 16-token alignment. The 6491 training packs contain 205182453 supervised tokens in the entire prepared corpus; the 333 validation packs contain 10479031. These counts are corpus totals, not tokens consumed by the run. The published builder's default five-percent validation split was not exercised. MXFP8 parameter gather and gradient-buffer reuse, CuTeDSL grouped MLP, single grouped weights, FP32 gradient reduction, learned routing and selective gdn_norm_out/moe recomputation were enabled. Warmup 200 and the 300000-step cosine-decay horizon were retained, so this run remained in warmup and does not establish full convergence. Checkpoint writes were disabled; resume and post-SFT export are untested.\n", + "hardware": "GB200", + "last_verified": null, + "metrics": { + "final_loss": null, + "initial_loss": null, + "last_10_steps_model_tflops_per_gpu_avg": null, + "last_10_steps_step_time_ms_avg": null, + "last_10_steps_tokens_per_second_per_gpu_avg": null + }, + "precision": "fp8_mx", + "source_pointer": "items.sft_long_context.GB200", + "status": "unverified", + "variant": null, + "workflow": "sft_long_context" + }, + { + "bridge_commit": null, + "command_topologies": [], + "commands": [], + "dimensions": {}, + "enabled_features": {}, + "entry_id": "qwen3-5-35b-a3b-text-peft-gb200", + "expected_result": "This workflow has no independently recorded verification in this text-only card.", + "hardware": "GB200", + "last_verified": null, + "metrics": { + "final_loss": null, + "initial_loss": null, + "last_10_steps_model_tflops_per_gpu_avg": null, + "last_10_steps_step_time_ms_avg": null, + "last_10_steps_tokens_per_second_per_gpu_avg": null + }, + "precision": "bf16", + "source_pointer": "items.peft.GB200", + "status": "unverified", + "variant": null, + "workflow": "peft" + }, + { + "bridge_commit": null, + "command_topologies": [], + "commands": [], + "dimensions": {}, + "enabled_features": {}, + "entry_id": "qwen3-5-35b-a3b-text-checkpoint-resume-gb200", + "expected_result": "This workflow has no independently recorded verification in this text-only card.", + "hardware": "GB200", + "last_verified": null, + "metrics": { + "final_loss": null, + "initial_loss": null, + "last_10_steps_model_tflops_per_gpu_avg": null, + "last_10_steps_step_time_ms_avg": null, + "last_10_steps_tokens_per_second_per_gpu_avg": null + }, + "precision": "bf16", + "source_pointer": "items.checkpoint_resume.GB200", + "status": "unverified", + "variant": null, + "workflow": "checkpoint_resume" + } + ], + "hf_id": "Qwen/Qwen3.5-35B-A3B", + "hf_revision": "59d61f3ce65a6d9863b86d2e96597125219dc754", + "min_transformers_version": "5.12.1", + "slug": "qwen3.5-35b-a3b-text", + "source_card": "examples/model_verification_cards/qwen3.5-35b-a3b-text/card.yaml", + "summary": "Performance disclaimer: this card does not record a canonical pretrain performance result; reported timing and throughput metrics are functional verification observations, not standalone optimized performance results. Records the completed 100-update MXFP8 text-only long-context SFT run. The run used an uncommitted recipe precursor and a release-candidate runtime; clean-source verification of the published recipe remains unverified.\n", + "title": "qwen3_5_35b_a3b_text" + }, { "architecture": "Qwen3_5MoeForConditionalGeneration", "base_container": "nvcr.io/nvidia/nemo:26.06", diff --git a/docs/fern/versions/nightly.yml b/docs/fern/versions/nightly.yml index d8d46934d2..7b75e53b04 100644 --- a/docs/fern/versions/nightly.yml +++ b/docs/fern/versions/nightly.yml @@ -164,6 +164,8 @@ navigation: path: ./nightly/pages/models/qwen/qwen2.5-vl.mdx - page: Qwen3-VL path: ./nightly/pages/models/qwen/qwen3-vl.mdx + - page: Qwen3.5-35B-A3B Text + path: ./nightly/pages/models/qwen/qwen3.5-35b-a3b-text.mdx - page: Qwen3.6-35B-A3B path: ./nightly/pages/models/qwen/qwen3.6-35b-a3b.mdx - page: Qwen3.8-27B diff --git a/docs/fern/versions/nightly/pages/models/README.mdx b/docs/fern/versions/nightly/pages/models/README.mdx index 20c45c5640..9e0d942825 100644 --- a/docs/fern/versions/nightly/pages/models/README.mdx +++ b/docs/fern/versions/nightly/pages/models/README.mdx @@ -4,7 +4,7 @@ Choose a model to open its interactive import/export and training configurations. Every option comes directly from an authoritative YAML verification card; independent fields are never combined into synthetic commands. -Current inventory: **28 model cards** and **406 concrete configurations**. +Current inventory: **29 model cards** and **418 concrete configurations**. Each model page lets you select import/export, pretraining, benchmarking, SFT, LoRA, or long-context SFT and immediately see the exact command and expected result for the recorded precision and GPU. @@ -170,7 +170,7 @@ Choose a model to inspect its recorded import/export, training, and precision co
Qwen - 5 models + 6 models
@@ -185,6 +185,10 @@ Choose a model to inspect its recorded import/export, training, and precision co Qwen3-8B ✓ Verified + + Qwen3.5-35B-A3B + ○ Unverified + Qwen3.6-35B-A3B ◐ Partial diff --git a/docs/fern/versions/nightly/pages/models/qwen/index.mdx b/docs/fern/versions/nightly/pages/models/qwen/index.mdx index 34e0c12b57..71081e04aa 100644 --- a/docs/fern/versions/nightly/pages/models/qwen/index.mdx +++ b/docs/fern/versions/nightly/pages/models/qwen/index.mdx @@ -11,6 +11,7 @@ Qwen model documentation is organized by model variant. | Qwen3-Next | [qwen3-next.md](qwen3-next.md) | | Qwen2.5-VL | [qwen2.5-vl.md](qwen2.5-vl.md) | | Qwen3-VL | [qwen3-vl.md](qwen3-vl.md) | +| Qwen3.5-35B-A3B Text | [qwen3.5-35b-a3b-text.md](qwen3.5-35b-a3b-text.md) | | Qwen3.6-35B-A3B | [qwen3.6-35b-a3b.md](qwen3.6-35b-a3b.md) | | Qwen3.8-27B | [qwen3.8-27b.md](qwen3.8-27b.md) | | Qwen2-Audio | [qwen2-audio.md](qwen2-audio.md) | diff --git a/docs/fern/versions/nightly/pages/models/qwen/qwen3.5-35b-a3b-text.mdx b/docs/fern/versions/nightly/pages/models/qwen/qwen3.5-35b-a3b-text.mdx new file mode 100644 index 0000000000..bd1f10fd03 --- /dev/null +++ b/docs/fern/versions/nightly/pages/models/qwen/qwen3.5-35b-a3b-text.mdx @@ -0,0 +1,366 @@ +# Qwen3.5-35B-A3B Text + +The text-only Qwen3.5-35B-A3B recipes use the language decoder and MTP weights +from Qwen/Qwen3.5-35B-A3B. They do not construct the vision tower. + +The configurations and validation evidence below are generated from the model +verification card. + +{/* BEGIN GENERATED VERIFIED CONFIGURATIONS */} + +## Verified configurations + +Choose an exact recorded configuration to see its command and expected result. These selectors are generated from the authoritative verification cards and never synthesize combinations. + + +### Run a configuration + +Choose a workflow, precision, and exact recorded combination. The command and expected result update below. + +
+ + +
+
+
+

Import · CPU

+ ○ Unverified +
+
+
Hardware
not specified
+
Precision
BF16
+
Last verified
—
+
+
+
Exact command
+

No runnable command is recorded for this status.

+
+
+
Expected result
+

This workflow has no independently recorded verification in this text-only card.

+
+
+
+
+

Import · GPU

+ ○ Unverified +
+
+
Hardware
not specified
+
Precision
BF16
+
Last verified
—
+
+
+
Exact command
+

No runnable command is recorded for this status.

+
+
+
Expected result
+

This workflow has no independently recorded verification in this text-only card.

+
+
+
+
+

Export · CPU

+ ○ Unverified +
+
+
Hardware
not specified
+
Precision
BF16
+
Last verified
—
+
+
+
Exact command
+

No runnable command is recorded for this status.

+
+
+
Expected result
+

This workflow has no independently recorded verification in this text-only card.

+
+
+
+
+

Export · GPU

+ ○ Unverified +
+
+
Hardware
not specified
+
Precision
BF16
+
Last verified
—
+
+
+
Exact command
+

No runnable command is recorded for this status.

+
+
+
Expected result
+

This workflow has no independently recorded verification in this text-only card.

+
+
+
+
+

Pretrain · GB200

+ ○ Unverified +
+
+
Hardware
GB200
+
Precision
BF16
+
Last verified
—
+
+
+
Recorded metrics
+
+
+
Initial loss
+
None
+
+
+
Final loss
+
None
+
+
+
Step time · last 10 avg
+
None ms
+
+
+
Model throughput · last 10 avg
+
None TFLOP/s/GPU
+
+
+
Token throughput · last 10 avg
+
None tokens/s/GPU
+
+
+
+
+
Exact command
+

No runnable command is recorded for this status.

+
+
+
Expected result
+

This workflow has no independently recorded verification in this text-only card.

+
+
+
+
+

SFT · GB200

+ ○ Unverified +
+
+
Hardware
GB200
+
Precision
BF16
+
Last verified
—
+
+
+
Recorded metrics
+
+
+
Initial loss
+
None
+
+
+
Final loss
+
None
+
+
+
Step time · last 10 avg
+
None ms
+
+
+
Model throughput · last 10 avg
+
None TFLOP/s/GPU
+
+
+
Token throughput · last 10 avg
+
None tokens/s/GPU
+
+
+
+
+
Exact command
+

No runnable command is recorded for this status.

+
+
+
Expected result
+

This workflow has no independently recorded verification in this text-only card.

+
+
+
+
+

Long Context · GB200

+ ○ Unverified +
+
+
Hardware
GB200
+
Precision
FP8 MX
+
Last verified
—
+
+
+
Recorded metrics
+
+
+
Initial loss
+
None
+
+
+
Final loss
+
None
+
+
+
Step time · last 10 avg
+
None ms
+
+
+
Model throughput · last 10 avg
+
None TFLOP/s/GPU
+
+
+
Token throughput · last 10 avg
+
None tokens/s/GPU
+
+
+
+
+
Exact command
+
+
+ Command + +
+
./scripts/training/train.sh --nodes 4 --gpus-per-node 4 --recipe qwen35_text_35b_a3b_sft_long_context_16gpu_gb200_fp8mx_config --pretrained_checkpoint work/model-verification/qwen3.5-35b-a3b-text/text-megatron --max_steps 100 dataset.hf_dataset=null dataset.hf_validation_proportion=null dataset.dataset_root=work/data/coderforge/materialized dataset.offline_packing_specs.packed_train_data_path=work/data/coderforge/packed-128k/training.parquet dataset.offline_packing_specs.packed_val_data_path=work/data/coderforge/packed-128k/validation.parquet dataset.offline_packing_specs.packed_metadata_path=work/data/coderforge/packed-128k/metadata.jsonl tokenizer.tokenizer_model=work/cache/qwen3.5-35b-a3b-text '++tokenizer.hf_tokenizer_kwargs.revision=null' checkpoint.hf_source_path=work/cache/qwen3.5-35b-a3b-text validation.eval_global_batch_size=32 validation.eval_micro_batch_size=1 checkpoint.save=null checkpoint.save_interval=null checkpoint.async_save=false logger.log_interval=1 logger.log_throughput=true logger.save_config_filepath=work/model-verification/qwen3.5-35b-a3b-text/resolved-config.yaml
+
+
+
+
Expected result
+

Historical observation, completed 2026-10-01: the text decoder and one pretrained MTP layer completed exactly 100 optimizer updates on 16 GB200 GPUs at TP1/PP1/CP8/EP16/ETP1, GBS32/MBS1 and 16 accumulation steps. LM, MTP, router losses and gradient norms were finite, with zero skipped or NaN updates and evaluations at steps 50 and 100. Online experiment logging finished successfully and the post-setup config was retained. Initial LM loss was 0.2099137008190155; final LM loss was 0.14850857853889465; final MTP loss was 0.19482117891311646. Updates 91-100 averaged 28591.89546108246 ms and 336.1899744993698 model TFLOP/s/GPU, corresponding to 9168.47224615851 token slots/s/GPU, not supervised-token throughput. These historical observations are deliberately separate from the null clean-recipe metrics above. The run used NeMo 26.10.rc3 and MCore b5581a9e874fc5be2b91e74cdd3ccdc64a1b5204 from NVIDIA/Megatron-LM PR 7611. Its Bridge source was base 00ca266c4abc5f8435821823a1dbe05d0c3d6033 plus an uncommitted recipe precursor, frozen as source bundle SHA256 5ea6d3c0c82a47f06f9f031040c2cc78641a4334d961542e25338f9c585ed863. No clean source commit or public runtime image identifies that run; the null provenance fields must not be replaced with the later PR head. The command above describes the published recipe with equivalent workload settings, checked by construction and full configuration comparison, but it has not been rerun from a clean checkout. Inputs were text/MTP weights extracted from the pinned HF model and togethercomputer/CoderForge-Preview revision 060fca96cf723b2ebab3181e9e59fafd273df3cb, configuration trajectories, split SWE_Rebench. Seed 1234 produced a 99:1 split of 77169 rows; the first 20 percent of training rows (15279) and all 772 validation rows were packed with first_fit_shuffle, a 131072-token budget and 16-token alignment. The 6491 training packs contain 205182453 supervised tokens in the entire prepared corpus; the 333 validation packs contain 10479031. These counts are corpus totals, not tokens consumed by the run. The published builder's default five-percent validation split was not exercised. MXFP8 parameter gather and gradient-buffer reuse, CuTeDSL grouped MLP, single grouped weights, FP32 gradient reduction, learned routing and selective gdn_norm_out/moe recomputation were enabled. Warmup 200 and the 300000-step cosine-decay horizon were retained, so this run remained in warmup and does not establish full convergence. Checkpoint writes were disabled; resume and post-SFT export are untested.

+
+
+
+
+

LoRA · GB200

+ ○ Unverified +
+
+
Hardware
GB200
+
Precision
BF16
+
Last verified
—
+
+
+
Recorded metrics
+
+
+
Initial loss
+
None
+
+
+
Final loss
+
None
+
+
+
Step time · last 10 avg
+
None ms
+
+
+
Model throughput · last 10 avg
+
None TFLOP/s/GPU
+
+
+
Token throughput · last 10 avg
+
None tokens/s/GPU
+
+
+
+
+
Exact command
+

No runnable command is recorded for this status.

+
+
+
Expected result
+

This workflow has no independently recorded verification in this text-only card.

+
+
+
+
+ +{/* END GENERATED VERIFIED CONFIGURATIONS */} diff --git a/docs/models/README.md b/docs/models/README.md index 9c7c2e2d7b..7140653415 100644 --- a/docs/models/README.md +++ b/docs/models/README.md @@ -4,7 +4,7 @@ Choose a model to open its interactive import/export and training configurations. Every option comes directly from an authoritative YAML verification card; independent fields are never combined into synthetic commands. -Current inventory: **28 model cards** and **406 concrete configurations**. +Current inventory: **29 model cards** and **418 concrete configurations**. Each model page lets you select import/export, pretraining, benchmarking, SFT, LoRA, or long-context SFT and immediately see the exact command and expected result for the recorded precision and GPU. @@ -168,7 +168,7 @@ Choose a model to inspect its recorded import/export, training, and precision co
Qwen - 5 models + 6 models
@@ -183,6 +183,10 @@ Choose a model to inspect its recorded import/export, training, and precision co Qwen3-8B ✓ Verified + + Qwen3.5-35B-A3B + ○ Unverified + Qwen3.6-35B-A3B ◐ Partial diff --git a/docs/models/qwen/index.md b/docs/models/qwen/index.md index c688de0c0e..97f21dd5ce 100644 --- a/docs/models/qwen/index.md +++ b/docs/models/qwen/index.md @@ -12,6 +12,7 @@ qwen3-moe.md qwen3-next.md qwen2.5-vl.md qwen3-vl.md +qwen3.5-35b-a3b-text.md qwen3.6-35b-a3b.md qwen3.8-27b.md qwen2-audio.md @@ -29,6 +30,7 @@ qwen3-asr.md | Qwen3-Next | [qwen3-next.md](qwen3-next.md) | | Qwen2.5-VL | [qwen2.5-vl.md](qwen2.5-vl.md) | | Qwen3-VL | [qwen3-vl.md](qwen3-vl.md) | +| Qwen3.5-35B-A3B Text | [qwen3.5-35b-a3b-text.md](qwen3.5-35b-a3b-text.md) | | Qwen3.6-35B-A3B | [qwen3.6-35b-a3b.md](qwen3.6-35b-a3b.md) | | Qwen3.8-27B | [qwen3.8-27b.md](qwen3.8-27b.md) | | Qwen2-Audio | [qwen2-audio.md](qwen2-audio.md) | diff --git a/docs/models/qwen/qwen3.5-35b-a3b-text.md b/docs/models/qwen/qwen3.5-35b-a3b-text.md new file mode 100644 index 0000000000..60936dd6db --- /dev/null +++ b/docs/models/qwen/qwen3.5-35b-a3b-text.md @@ -0,0 +1,367 @@ +# Qwen3.5-35B-A3B Text + +The text-only Qwen3.5-35B-A3B recipes use the language decoder and MTP weights +from Qwen/Qwen3.5-35B-A3B. They do not construct the vision tower. + +The configurations and validation evidence below are generated from the model +verification card. + + + +## Verified configurations + +Choose an exact recorded configuration to see its command and expected result. These selectors are generated from the authoritative verification cards and never synthesize combinations. + + +### Run a configuration + +Choose a workflow, precision, and exact recorded combination. The command and expected result update below. + +
+ + +
+
+
+

Import · CPU

+ ○ Unverified +
+
+
Hardware
not specified
+
Precision
BF16
+
Last verified
—
+
+
+
Exact command
+

No runnable command is recorded for this status.

+
+
+
Expected result
+

This workflow has no independently recorded verification in this text-only card.

+
+
+
+
+

Import · GPU

+ ○ Unverified +
+
+
Hardware
not specified
+
Precision
BF16
+
Last verified
—
+
+
+
Exact command
+

No runnable command is recorded for this status.

+
+
+
Expected result
+

This workflow has no independently recorded verification in this text-only card.

+
+
+
+
+

Export · CPU

+ ○ Unverified +
+
+
Hardware
not specified
+
Precision
BF16
+
Last verified
—
+
+
+
Exact command
+

No runnable command is recorded for this status.

+
+
+
Expected result
+

This workflow has no independently recorded verification in this text-only card.

+
+
+
+
+

Export · GPU

+ ○ Unverified +
+
+
Hardware
not specified
+
Precision
BF16
+
Last verified
—
+
+
+
Exact command
+

No runnable command is recorded for this status.

+
+
+
Expected result
+

This workflow has no independently recorded verification in this text-only card.

+
+
+
+
+

Pretrain · GB200

+ ○ Unverified +
+
+
Hardware
GB200
+
Precision
BF16
+
Last verified
—
+
+
+
Recorded metrics
+
+
+
Initial loss
+
None
+
+
+
Final loss
+
None
+
+
+
Step time · last 10 avg
+
None ms
+
+
+
Model throughput · last 10 avg
+
None TFLOP/s/GPU
+
+
+
Token throughput · last 10 avg
+
None tokens/s/GPU
+
+
+
+
+
Exact command
+

No runnable command is recorded for this status.

+
+
+
Expected result
+

This workflow has no independently recorded verification in this text-only card.

+
+
+
+
+

SFT · GB200

+ ○ Unverified +
+
+
Hardware
GB200
+
Precision
BF16
+
Last verified
—
+
+
+
Recorded metrics
+
+
+
Initial loss
+
None
+
+
+
Final loss
+
None
+
+
+
Step time · last 10 avg
+
None ms
+
+
+
Model throughput · last 10 avg
+
None TFLOP/s/GPU
+
+
+
Token throughput · last 10 avg
+
None tokens/s/GPU
+
+
+
+
+
Exact command
+

No runnable command is recorded for this status.

+
+
+
Expected result
+

This workflow has no independently recorded verification in this text-only card.

+
+
+
+
+

Long Context · GB200

+ ○ Unverified +
+
+
Hardware
GB200
+
Precision
FP8 MX
+
Last verified
—
+
+
+
Recorded metrics
+
+
+
Initial loss
+
None
+
+
+
Final loss
+
None
+
+
+
Step time · last 10 avg
+
None ms
+
+
+
Model throughput · last 10 avg
+
None TFLOP/s/GPU
+
+
+
Token throughput · last 10 avg
+
None tokens/s/GPU
+
+
+
+
+
Exact command
+
+
+ Command + +
+
./scripts/training/train.sh --nodes 4 --gpus-per-node 4 --recipe qwen35_text_35b_a3b_sft_long_context_16gpu_gb200_fp8mx_config --pretrained_checkpoint work/model-verification/qwen3.5-35b-a3b-text/text-megatron --max_steps 100 dataset.hf_dataset=null dataset.hf_validation_proportion=null dataset.dataset_root=work/data/coderforge/materialized dataset.offline_packing_specs.packed_train_data_path=work/data/coderforge/packed-128k/training.parquet dataset.offline_packing_specs.packed_val_data_path=work/data/coderforge/packed-128k/validation.parquet dataset.offline_packing_specs.packed_metadata_path=work/data/coderforge/packed-128k/metadata.jsonl tokenizer.tokenizer_model=work/cache/qwen3.5-35b-a3b-text '++tokenizer.hf_tokenizer_kwargs.revision=null' checkpoint.hf_source_path=work/cache/qwen3.5-35b-a3b-text validation.eval_global_batch_size=32 validation.eval_micro_batch_size=1 checkpoint.save=null checkpoint.save_interval=null checkpoint.async_save=false logger.log_interval=1 logger.log_throughput=true logger.save_config_filepath=work/model-verification/qwen3.5-35b-a3b-text/resolved-config.yaml
+
+
+
+
Expected result
+

Historical observation, completed 2026-10-01: the text decoder and one pretrained MTP layer completed exactly 100 optimizer updates on 16 GB200 GPUs at TP1/PP1/CP8/EP16/ETP1, GBS32/MBS1 and 16 accumulation steps. LM, MTP, router losses and gradient norms were finite, with zero skipped or NaN updates and evaluations at steps 50 and 100. Online experiment logging finished successfully and the post-setup config was retained. Initial LM loss was 0.2099137008190155; final LM loss was 0.14850857853889465; final MTP loss was 0.19482117891311646. Updates 91-100 averaged 28591.89546108246 ms and 336.1899744993698 model TFLOP/s/GPU, corresponding to 9168.47224615851 token slots/s/GPU, not supervised-token throughput. These historical observations are deliberately separate from the null clean-recipe metrics above. The run used NeMo 26.10.rc3 and MCore b5581a9e874fc5be2b91e74cdd3ccdc64a1b5204 from NVIDIA/Megatron-LM PR 7611. Its Bridge source was base 00ca266c4abc5f8435821823a1dbe05d0c3d6033 plus an uncommitted recipe precursor, frozen as source bundle SHA256 5ea6d3c0c82a47f06f9f031040c2cc78641a4334d961542e25338f9c585ed863. No clean source commit or public runtime image identifies that run; the null provenance fields must not be replaced with the later PR head. The command above describes the published recipe with equivalent workload settings, checked by construction and full configuration comparison, but it has not been rerun from a clean checkout. Inputs were text/MTP weights extracted from the pinned HF model and togethercomputer/CoderForge-Preview revision 060fca96cf723b2ebab3181e9e59fafd273df3cb, configuration trajectories, split SWE_Rebench. Seed 1234 produced a 99:1 split of 77169 rows; the first 20 percent of training rows (15279) and all 772 validation rows were packed with first_fit_shuffle, a 131072-token budget and 16-token alignment. The 6491 training packs contain 205182453 supervised tokens in the entire prepared corpus; the 333 validation packs contain 10479031. These counts are corpus totals, not tokens consumed by the run. The published builder's default five-percent validation split was not exercised. MXFP8 parameter gather and gradient-buffer reuse, CuTeDSL grouped MLP, single grouped weights, FP32 gradient reduction, learned routing and selective gdn_norm_out/moe recomputation were enabled. Warmup 200 and the 300000-step cosine-decay horizon were retained, so this run remained in warmup and does not establish full convergence. Checkpoint writes were disabled; resume and post-SFT export are untested. +

+
+
+
+
+

LoRA · GB200

+ ○ Unverified +
+
+
Hardware
GB200
+
Precision
BF16
+
Last verified
—
+
+
+
Recorded metrics
+
+
+
Initial loss
+
None
+
+
+
Final loss
+
None
+
+
+
Step time · last 10 avg
+
None ms
+
+
+
Model throughput · last 10 avg
+
None TFLOP/s/GPU
+
+
+
Token throughput · last 10 avg
+
None tokens/s/GPU
+
+
+
+
+
Exact command
+

No runnable command is recorded for this status.

+
+
+
Expected result
+

This workflow has no independently recorded verification in this text-only card.

+
+
+
+
+ + diff --git a/scripts/docs/generate_model_verification_catalog.py b/scripts/docs/generate_model_verification_catalog.py index 30cfb0836f..ba32a39ae3 100644 --- a/scripts/docs/generate_model_verification_catalog.py +++ b/scripts/docs/generate_model_verification_catalog.py @@ -115,6 +115,7 @@ "qwen3-235b-a22b": "models/qwen/qwen3-235b-a22b.md", "qwen3-30b-a3b": "models/qwen/qwen3-30b-a3b.md", "qwen3-8b": "models/qwen/qwen3-8b.md", + "qwen3.5-35b-a3b-text": "models/qwen/qwen3.5-35b-a3b-text.md", "qwen3.6-35b-a3b": "models/qwen/qwen3.6-35b-a3b.md", "qwen3.8-27b": "models/qwen/qwen3.8-27b.md", "step35-flash": "models/stepfun/step35.md", @@ -130,6 +131,7 @@ "qwen3-235b-a22b": "Qwen3-235B-A22B", "qwen3-30b-a3b": "Qwen3-30B-A3B", "qwen3-8b": "Qwen3-8B", + "qwen3.5-35b-a3b-text": "Qwen3.5-35B-A3B Text", "qwen3.6-35b-a3b": "Qwen3.6-35B-A3B", "qwen3.8-27b": "Qwen3.8-27B", } diff --git a/tests/unit_tests/recipes/recipe_test_utils.py b/tests/unit_tests/recipes/recipe_test_utils.py index dd53ff031c..91cb3b598e 100644 --- a/tests/unit_tests/recipes/recipe_test_utils.py +++ b/tests/unit_tests/recipes/recipe_test_utils.py @@ -261,7 +261,14 @@ def patch_recipe_construction_dependencies(monkeypatch: pytest.MonkeyPatch) -> N def load_offline_auto_config(*args: object, **kwargs: object) -> SimpleNamespace: del args, kwargs - return SimpleNamespace(text_config=SimpleNamespace(architectures=None)) + return SimpleNamespace( + tie_word_embeddings=False, + text_config=SimpleNamespace( + architectures=None, + model_type="qwen3_5_moe_text", + max_position_embeddings=262144, + ), + ) monkeypatch.setattr( AutoConfig, diff --git a/tests/unit_tests/recipes/test_qwen_recipes.py b/tests/unit_tests/recipes/test_qwen_recipes.py index 838d693030..631b8d701a 100644 --- a/tests/unit_tests/recipes/test_qwen_recipes.py +++ b/tests/unit_tests/recipes/test_qwen_recipes.py @@ -97,16 +97,19 @@ def to_megatron_provider(self, load_weights: bool = False): class _FakeTextConfig: architectures = None + model_type = "qwen3_5_moe_text" + max_position_embeddings = 262144 class _FakeRootConfig: text_config = _FakeTextConfig() + tie_word_embeddings = False class _FakeAutoConfig: @staticmethod - def from_pretrained(hf_path: str): - # Ignore hf_path; return a unified config with a nested text config. + def from_pretrained(hf_path: str, *, revision: str | None = None): + # Match the pinned-config API without downloading a model configuration. return _FakeRootConfig() From 061de2c35e7617ef2ceb0bc48534222337f96971 Mon Sep 17 00:00:00 2001 From: Chen Cui Date: Tue, 6 Oct 2026 16:40:55 -0700 Subject: [PATCH 4/5] chore(recipe): remove Qwen3.5 text verification card Signed-off-by: Chen Cui --- .../model-verification/catalog-v1.json | 261 ------------- docs/fern/versions/nightly.yml | 2 - .../versions/nightly/pages/models/README.mdx | 8 +- .../nightly/pages/models/qwen/index.mdx | 1 - .../models/qwen/qwen3.5-35b-a3b-text.mdx | 366 ----------------- docs/models/README.md | 8 +- docs/models/qwen/index.md | 2 - docs/models/qwen/qwen3.5-35b-a3b-text.md | 367 ------------------ .../qwen3.5-35b-a3b-text/card.yaml | 214 ---------- .../generate_model_verification_catalog.py | 2 - .../create-model-verification-card/SKILL.md | 8 - .../scripts/validate_card.py | 14 +- .../test_validate_card.py | 43 -- 13 files changed, 6 insertions(+), 1290 deletions(-) delete mode 100644 docs/fern/versions/nightly/pages/models/qwen/qwen3.5-35b-a3b-text.mdx delete mode 100644 docs/models/qwen/qwen3.5-35b-a3b-text.md delete mode 100644 examples/model_verification_cards/qwen3.5-35b-a3b-text/card.yaml diff --git a/docs/_static/model-verification/catalog-v1.json b/docs/_static/model-verification/catalog-v1.json index 9632210936..d93c56e9ec 100644 --- a/docs/_static/model-verification/catalog-v1.json +++ b/docs/_static/model-verification/catalog-v1.json @@ -9671,267 +9671,6 @@ "summary": "Performance disclaimer: this model has not been performance-tuned; reported timing and throughput metrics are sanity checks, not optimized performance results. Qwen3-8B support verification covers conversion, inference, and training.\n", "title": "qwen3_8b" }, - { - "architecture": "Qwen3_5MoeForCausalLM", - "base_container": null, - "bridge_commit": null, - "entries": [ - { - "bridge_commit": null, - "command_topologies": [], - "commands": [], - "dimensions": {}, - "enabled_features": {}, - "entry_id": "qwen3-5-35b-a3b-text-hf-to-megatron-cpu", - "expected_result": "This workflow has no independently recorded verification in this text-only card.", - "hardware": null, - "last_verified": null, - "metrics": {}, - "precision": "bf16", - "source_pointer": "items.hf_to_megatron_cpu", - "status": "unverified", - "variant": null, - "workflow": "hf_to_megatron_cpu" - }, - { - "bridge_commit": null, - "command_topologies": [], - "commands": [], - "dimensions": {}, - "enabled_features": {}, - "entry_id": "qwen3-5-35b-a3b-text-hf-to-megatron-gpu", - "expected_result": "This workflow has no independently recorded verification in this text-only card.", - "hardware": null, - "last_verified": null, - "metrics": {}, - "precision": "bf16", - "source_pointer": "items.hf_to_megatron_gpu", - "status": "unverified", - "variant": null, - "workflow": "hf_to_megatron_gpu" - }, - { - "bridge_commit": null, - "command_topologies": [], - "commands": [], - "dimensions": {}, - "enabled_features": {}, - "entry_id": "qwen3-5-35b-a3b-text-megatron-to-hf-cpu", - "expected_result": "This workflow has no independently recorded verification in this text-only card.", - "hardware": null, - "last_verified": null, - "metrics": {}, - "precision": "bf16", - "source_pointer": "items.megatron_to_hf_cpu", - "status": "unverified", - "variant": null, - "workflow": "megatron_to_hf_cpu" - }, - { - "bridge_commit": null, - "command_topologies": [], - "commands": [], - "dimensions": {}, - "enabled_features": {}, - "entry_id": "qwen3-5-35b-a3b-text-megatron-to-hf-gpu", - "expected_result": "This workflow has no independently recorded verification in this text-only card.", - "hardware": null, - "last_verified": null, - "metrics": {}, - "precision": "bf16", - "source_pointer": "items.megatron_to_hf_gpu", - "status": "unverified", - "variant": null, - "workflow": "megatron_to_hf_gpu" - }, - { - "bridge_commit": null, - "command_topologies": [], - "commands": [], - "dimensions": {}, - "enabled_features": {}, - "entry_id": "qwen3-5-35b-a3b-text-manual-forward-pass", - "expected_result": "This workflow has no independently recorded verification in this text-only card.", - "hardware": null, - "last_verified": null, - "metrics": {}, - "precision": "bf16", - "source_pointer": "items.manual_forward_pass", - "status": "unverified", - "variant": null, - "workflow": "manual_forward_pass" - }, - { - "bridge_commit": null, - "command_topologies": [], - "commands": [], - "dimensions": {}, - "enabled_features": {}, - "entry_id": "qwen3-5-35b-a3b-text-inference", - "expected_result": "This workflow has no independently recorded verification in this text-only card.", - "hardware": null, - "last_verified": null, - "metrics": {}, - "precision": "bf16", - "source_pointer": "items.inference", - "status": "unverified", - "variant": null, - "workflow": "inference" - }, - { - "bridge_commit": null, - "command_topologies": [], - "commands": [], - "dimensions": {}, - "enabled_features": {}, - "entry_id": "qwen3-5-35b-a3b-text-pretrain-gb200", - "expected_result": "This workflow has no independently recorded verification in this text-only card.", - "hardware": "GB200", - "last_verified": null, - "metrics": { - "final_loss": null, - "initial_loss": null, - "last_10_steps_model_tflops_per_gpu_avg": null, - "last_10_steps_step_time_ms_avg": null, - "last_10_steps_tokens_per_second_per_gpu_avg": null - }, - "precision": "bf16", - "source_pointer": "items.pretrain.GB200", - "status": "unverified", - "variant": null, - "workflow": "pretrain" - }, - { - "bridge_commit": null, - "command_topologies": [], - "commands": [], - "dimensions": {}, - "enabled_features": {}, - "entry_id": "qwen3-5-35b-a3b-text-sft-gb200", - "expected_result": "This workflow has no independently recorded verification in this text-only card.", - "hardware": "GB200", - "last_verified": null, - "metrics": { - "final_loss": null, - "initial_loss": null, - "last_10_steps_model_tflops_per_gpu_avg": null, - "last_10_steps_step_time_ms_avg": null, - "last_10_steps_tokens_per_second_per_gpu_avg": null - }, - "precision": "bf16", - "source_pointer": "items.sft.GB200", - "status": "unverified", - "variant": null, - "workflow": "sft" - }, - { - "bridge_commit": null, - "command_topologies": [], - "commands": [], - "dimensions": {}, - "enabled_features": {}, - "entry_id": "qwen3-5-35b-a3b-text-sft-export-inference-gb200", - "expected_result": "This workflow has no independently recorded verification in this text-only card.", - "hardware": "GB200", - "last_verified": null, - "metrics": {}, - "precision": "bf16", - "source_pointer": "items.sft_export_inference.GB200", - "status": "unverified", - "variant": null, - "workflow": "sft_export_inference" - }, - { - "bridge_commit": null, - "command_topologies": [ - { - "gpus_per_node": "4", - "nodes": "4", - "recipe": "qwen35_text_35b_a3b_sft_long_context_16gpu_gb200_fp8mx_config", - "total_gpus": 16 - } - ], - "commands": [ - "./scripts/training/train.sh --nodes 4 --gpus-per-node 4 --recipe qwen35_text_35b_a3b_sft_long_context_16gpu_gb200_fp8mx_config --pretrained_checkpoint work/model-verification/qwen3.5-35b-a3b-text/text-megatron --max_steps 100 dataset.hf_dataset=null dataset.hf_validation_proportion=null dataset.dataset_root=work/data/coderforge/materialized dataset.offline_packing_specs.packed_train_data_path=work/data/coderforge/packed-128k/training.parquet dataset.offline_packing_specs.packed_val_data_path=work/data/coderforge/packed-128k/validation.parquet dataset.offline_packing_specs.packed_metadata_path=work/data/coderforge/packed-128k/metadata.jsonl tokenizer.tokenizer_model=work/cache/qwen3.5-35b-a3b-text '++tokenizer.hf_tokenizer_kwargs.revision=null' checkpoint.hf_source_path=work/cache/qwen3.5-35b-a3b-text validation.eval_global_batch_size=32 validation.eval_micro_batch_size=1 checkpoint.save=null checkpoint.save_interval=null checkpoint.async_save=false logger.log_interval=1 logger.log_throughput=true logger.save_config_filepath=work/model-verification/qwen3.5-35b-a3b-text/resolved-config.yaml" - ], - "dimensions": {}, - "enabled_features": { - "context_parallel_size": 8, - "moe_dispatcher": "hybridep", - "sequence_packing": "offline" - }, - "entry_id": "qwen3-5-35b-a3b-text-sft-long-context-gb200", - "expected_result": "Historical observation, completed 2026-10-01: the text decoder and one pretrained MTP layer completed exactly 100 optimizer updates on 16 GB200 GPUs at TP1/PP1/CP8/EP16/ETP1, GBS32/MBS1 and 16 accumulation steps. LM, MTP, router losses and gradient norms were finite, with zero skipped or NaN updates and evaluations at steps 50 and 100. Online experiment logging finished successfully and the post-setup config was retained. Initial LM loss was 0.2099137008190155; final LM loss was 0.14850857853889465; final MTP loss was 0.19482117891311646. Updates 91-100 averaged 28591.89546108246 ms and 336.1899744993698 model TFLOP/s/GPU, corresponding to 9168.47224615851 token slots/s/GPU, not supervised-token throughput. These historical observations are deliberately separate from the null clean-recipe metrics above. The run used NeMo 26.10.rc3 and MCore b5581a9e874fc5be2b91e74cdd3ccdc64a1b5204 from NVIDIA/Megatron-LM PR 7611. Its Bridge source was base 00ca266c4abc5f8435821823a1dbe05d0c3d6033 plus an uncommitted recipe precursor, frozen as source bundle SHA256 5ea6d3c0c82a47f06f9f031040c2cc78641a4334d961542e25338f9c585ed863. No clean source commit or public runtime image identifies that run; the null provenance fields must not be replaced with the later PR head. The command above describes the published recipe with equivalent workload settings, checked by construction and full configuration comparison, but it has not been rerun from a clean checkout. Inputs were text/MTP weights extracted from the pinned HF model and togethercomputer/CoderForge-Preview revision 060fca96cf723b2ebab3181e9e59fafd273df3cb, configuration trajectories, split SWE_Rebench. Seed 1234 produced a 99:1 split of 77169 rows; the first 20 percent of training rows (15279) and all 772 validation rows were packed with first_fit_shuffle, a 131072-token budget and 16-token alignment. The 6491 training packs contain 205182453 supervised tokens in the entire prepared corpus; the 333 validation packs contain 10479031. These counts are corpus totals, not tokens consumed by the run. The published builder's default five-percent validation split was not exercised. MXFP8 parameter gather and gradient-buffer reuse, CuTeDSL grouped MLP, single grouped weights, FP32 gradient reduction, learned routing and selective gdn_norm_out/moe recomputation were enabled. Warmup 200 and the 300000-step cosine-decay horizon were retained, so this run remained in warmup and does not establish full convergence. Checkpoint writes were disabled; resume and post-SFT export are untested.\n", - "hardware": "GB200", - "last_verified": null, - "metrics": { - "final_loss": null, - "initial_loss": null, - "last_10_steps_model_tflops_per_gpu_avg": null, - "last_10_steps_step_time_ms_avg": null, - "last_10_steps_tokens_per_second_per_gpu_avg": null - }, - "precision": "fp8_mx", - "source_pointer": "items.sft_long_context.GB200", - "status": "unverified", - "variant": null, - "workflow": "sft_long_context" - }, - { - "bridge_commit": null, - "command_topologies": [], - "commands": [], - "dimensions": {}, - "enabled_features": {}, - "entry_id": "qwen3-5-35b-a3b-text-peft-gb200", - "expected_result": "This workflow has no independently recorded verification in this text-only card.", - "hardware": "GB200", - "last_verified": null, - "metrics": { - "final_loss": null, - "initial_loss": null, - "last_10_steps_model_tflops_per_gpu_avg": null, - "last_10_steps_step_time_ms_avg": null, - "last_10_steps_tokens_per_second_per_gpu_avg": null - }, - "precision": "bf16", - "source_pointer": "items.peft.GB200", - "status": "unverified", - "variant": null, - "workflow": "peft" - }, - { - "bridge_commit": null, - "command_topologies": [], - "commands": [], - "dimensions": {}, - "enabled_features": {}, - "entry_id": "qwen3-5-35b-a3b-text-checkpoint-resume-gb200", - "expected_result": "This workflow has no independently recorded verification in this text-only card.", - "hardware": "GB200", - "last_verified": null, - "metrics": { - "final_loss": null, - "initial_loss": null, - "last_10_steps_model_tflops_per_gpu_avg": null, - "last_10_steps_step_time_ms_avg": null, - "last_10_steps_tokens_per_second_per_gpu_avg": null - }, - "precision": "bf16", - "source_pointer": "items.checkpoint_resume.GB200", - "status": "unverified", - "variant": null, - "workflow": "checkpoint_resume" - } - ], - "hf_id": "Qwen/Qwen3.5-35B-A3B", - "hf_revision": "59d61f3ce65a6d9863b86d2e96597125219dc754", - "min_transformers_version": "5.12.1", - "slug": "qwen3.5-35b-a3b-text", - "source_card": "examples/model_verification_cards/qwen3.5-35b-a3b-text/card.yaml", - "summary": "Performance disclaimer: this card does not record a canonical pretrain performance result; reported timing and throughput metrics are functional verification observations, not standalone optimized performance results. Records the completed 100-update MXFP8 text-only long-context SFT run. The run used an uncommitted recipe precursor and a release-candidate runtime; clean-source verification of the published recipe remains unverified.\n", - "title": "qwen3_5_35b_a3b_text" - }, { "architecture": "Qwen3_5MoeForConditionalGeneration", "base_container": "nvcr.io/nvidia/nemo:26.06", diff --git a/docs/fern/versions/nightly.yml b/docs/fern/versions/nightly.yml index 7b75e53b04..d8d46934d2 100644 --- a/docs/fern/versions/nightly.yml +++ b/docs/fern/versions/nightly.yml @@ -164,8 +164,6 @@ navigation: path: ./nightly/pages/models/qwen/qwen2.5-vl.mdx - page: Qwen3-VL path: ./nightly/pages/models/qwen/qwen3-vl.mdx - - page: Qwen3.5-35B-A3B Text - path: ./nightly/pages/models/qwen/qwen3.5-35b-a3b-text.mdx - page: Qwen3.6-35B-A3B path: ./nightly/pages/models/qwen/qwen3.6-35b-a3b.mdx - page: Qwen3.8-27B diff --git a/docs/fern/versions/nightly/pages/models/README.mdx b/docs/fern/versions/nightly/pages/models/README.mdx index 9e0d942825..20c45c5640 100644 --- a/docs/fern/versions/nightly/pages/models/README.mdx +++ b/docs/fern/versions/nightly/pages/models/README.mdx @@ -4,7 +4,7 @@ Choose a model to open its interactive import/export and training configurations. Every option comes directly from an authoritative YAML verification card; independent fields are never combined into synthetic commands. -Current inventory: **29 model cards** and **418 concrete configurations**. +Current inventory: **28 model cards** and **406 concrete configurations**. Each model page lets you select import/export, pretraining, benchmarking, SFT, LoRA, or long-context SFT and immediately see the exact command and expected result for the recorded precision and GPU. @@ -170,7 +170,7 @@ Choose a model to inspect its recorded import/export, training, and precision co
Qwen - 6 models + 5 models
@@ -185,10 +185,6 @@ Choose a model to inspect its recorded import/export, training, and precision co Qwen3-8B ✓ Verified - - Qwen3.5-35B-A3B - ○ Unverified - Qwen3.6-35B-A3B ◐ Partial diff --git a/docs/fern/versions/nightly/pages/models/qwen/index.mdx b/docs/fern/versions/nightly/pages/models/qwen/index.mdx index 71081e04aa..34e0c12b57 100644 --- a/docs/fern/versions/nightly/pages/models/qwen/index.mdx +++ b/docs/fern/versions/nightly/pages/models/qwen/index.mdx @@ -11,7 +11,6 @@ Qwen model documentation is organized by model variant. | Qwen3-Next | [qwen3-next.md](qwen3-next.md) | | Qwen2.5-VL | [qwen2.5-vl.md](qwen2.5-vl.md) | | Qwen3-VL | [qwen3-vl.md](qwen3-vl.md) | -| Qwen3.5-35B-A3B Text | [qwen3.5-35b-a3b-text.md](qwen3.5-35b-a3b-text.md) | | Qwen3.6-35B-A3B | [qwen3.6-35b-a3b.md](qwen3.6-35b-a3b.md) | | Qwen3.8-27B | [qwen3.8-27b.md](qwen3.8-27b.md) | | Qwen2-Audio | [qwen2-audio.md](qwen2-audio.md) | diff --git a/docs/fern/versions/nightly/pages/models/qwen/qwen3.5-35b-a3b-text.mdx b/docs/fern/versions/nightly/pages/models/qwen/qwen3.5-35b-a3b-text.mdx deleted file mode 100644 index bd1f10fd03..0000000000 --- a/docs/fern/versions/nightly/pages/models/qwen/qwen3.5-35b-a3b-text.mdx +++ /dev/null @@ -1,366 +0,0 @@ -# Qwen3.5-35B-A3B Text - -The text-only Qwen3.5-35B-A3B recipes use the language decoder and MTP weights -from Qwen/Qwen3.5-35B-A3B. They do not construct the vision tower. - -The configurations and validation evidence below are generated from the model -verification card. - -{/* BEGIN GENERATED VERIFIED CONFIGURATIONS */} - -## Verified configurations - -Choose an exact recorded configuration to see its command and expected result. These selectors are generated from the authoritative verification cards and never synthesize combinations. - - -### Run a configuration - -Choose a workflow, precision, and exact recorded combination. The command and expected result update below. - -
- - -
-
-
-

Import · CPU

- ○ Unverified -
-
-
Hardware
not specified
-
Precision
BF16
-
Last verified
—
-
-
-
Exact command
-

No runnable command is recorded for this status.

-
-
-
Expected result
-

This workflow has no independently recorded verification in this text-only card.

-
-
-
-
-

Import · GPU

- ○ Unverified -
-
-
Hardware
not specified
-
Precision
BF16
-
Last verified
—
-
-
-
Exact command
-

No runnable command is recorded for this status.

-
-
-
Expected result
-

This workflow has no independently recorded verification in this text-only card.

-
-
-
-
-

Export · CPU

- ○ Unverified -
-
-
Hardware
not specified
-
Precision
BF16
-
Last verified
—
-
-
-
Exact command
-

No runnable command is recorded for this status.

-
-
-
Expected result
-

This workflow has no independently recorded verification in this text-only card.

-
-
-
-
-

Export · GPU

- ○ Unverified -
-
-
Hardware
not specified
-
Precision
BF16
-
Last verified
—
-
-
-
Exact command
-

No runnable command is recorded for this status.

-
-
-
Expected result
-

This workflow has no independently recorded verification in this text-only card.

-
-
-
-
-

Pretrain · GB200

- ○ Unverified -
-
-
Hardware
GB200
-
Precision
BF16
-
Last verified
—
-
-
-
Recorded metrics
-
-
-
Initial loss
-
None
-
-
-
Final loss
-
None
-
-
-
Step time · last 10 avg
-
None ms
-
-
-
Model throughput · last 10 avg
-
None TFLOP/s/GPU
-
-
-
Token throughput · last 10 avg
-
None tokens/s/GPU
-
-
-
-
-
Exact command
-

No runnable command is recorded for this status.

-
-
-
Expected result
-

This workflow has no independently recorded verification in this text-only card.

-
-
-
-
-

SFT · GB200

- ○ Unverified -
-
-
Hardware
GB200
-
Precision
BF16
-
Last verified
—
-
-
-
Recorded metrics
-
-
-
Initial loss
-
None
-
-
-
Final loss
-
None
-
-
-
Step time · last 10 avg
-
None ms
-
-
-
Model throughput · last 10 avg
-
None TFLOP/s/GPU
-
-
-
Token throughput · last 10 avg
-
None tokens/s/GPU
-
-
-
-
-
Exact command
-

No runnable command is recorded for this status.

-
-
-
Expected result
-

This workflow has no independently recorded verification in this text-only card.

-
-
-
-
-

Long Context · GB200

- ○ Unverified -
-
-
Hardware
GB200
-
Precision
FP8 MX
-
Last verified
—
-
-
-
Recorded metrics
-
-
-
Initial loss
-
None
-
-
-
Final loss
-
None
-
-
-
Step time · last 10 avg
-
None ms
-
-
-
Model throughput · last 10 avg
-
None TFLOP/s/GPU
-
-
-
Token throughput · last 10 avg
-
None tokens/s/GPU
-
-
-
-
-
Exact command
-
-
- Command - -
-
./scripts/training/train.sh --nodes 4 --gpus-per-node 4 --recipe qwen35_text_35b_a3b_sft_long_context_16gpu_gb200_fp8mx_config --pretrained_checkpoint work/model-verification/qwen3.5-35b-a3b-text/text-megatron --max_steps 100 dataset.hf_dataset=null dataset.hf_validation_proportion=null dataset.dataset_root=work/data/coderforge/materialized dataset.offline_packing_specs.packed_train_data_path=work/data/coderforge/packed-128k/training.parquet dataset.offline_packing_specs.packed_val_data_path=work/data/coderforge/packed-128k/validation.parquet dataset.offline_packing_specs.packed_metadata_path=work/data/coderforge/packed-128k/metadata.jsonl tokenizer.tokenizer_model=work/cache/qwen3.5-35b-a3b-text '++tokenizer.hf_tokenizer_kwargs.revision=null' checkpoint.hf_source_path=work/cache/qwen3.5-35b-a3b-text validation.eval_global_batch_size=32 validation.eval_micro_batch_size=1 checkpoint.save=null checkpoint.save_interval=null checkpoint.async_save=false logger.log_interval=1 logger.log_throughput=true logger.save_config_filepath=work/model-verification/qwen3.5-35b-a3b-text/resolved-config.yaml
-
-
-
-
Expected result
-

Historical observation, completed 2026-10-01: the text decoder and one pretrained MTP layer completed exactly 100 optimizer updates on 16 GB200 GPUs at TP1/PP1/CP8/EP16/ETP1, GBS32/MBS1 and 16 accumulation steps. LM, MTP, router losses and gradient norms were finite, with zero skipped or NaN updates and evaluations at steps 50 and 100. Online experiment logging finished successfully and the post-setup config was retained. Initial LM loss was 0.2099137008190155; final LM loss was 0.14850857853889465; final MTP loss was 0.19482117891311646. Updates 91-100 averaged 28591.89546108246 ms and 336.1899744993698 model TFLOP/s/GPU, corresponding to 9168.47224615851 token slots/s/GPU, not supervised-token throughput. These historical observations are deliberately separate from the null clean-recipe metrics above. The run used NeMo 26.10.rc3 and MCore b5581a9e874fc5be2b91e74cdd3ccdc64a1b5204 from NVIDIA/Megatron-LM PR 7611. Its Bridge source was base 00ca266c4abc5f8435821823a1dbe05d0c3d6033 plus an uncommitted recipe precursor, frozen as source bundle SHA256 5ea6d3c0c82a47f06f9f031040c2cc78641a4334d961542e25338f9c585ed863. No clean source commit or public runtime image identifies that run; the null provenance fields must not be replaced with the later PR head. The command above describes the published recipe with equivalent workload settings, checked by construction and full configuration comparison, but it has not been rerun from a clean checkout. Inputs were text/MTP weights extracted from the pinned HF model and togethercomputer/CoderForge-Preview revision 060fca96cf723b2ebab3181e9e59fafd273df3cb, configuration trajectories, split SWE_Rebench. Seed 1234 produced a 99:1 split of 77169 rows; the first 20 percent of training rows (15279) and all 772 validation rows were packed with first_fit_shuffle, a 131072-token budget and 16-token alignment. The 6491 training packs contain 205182453 supervised tokens in the entire prepared corpus; the 333 validation packs contain 10479031. These counts are corpus totals, not tokens consumed by the run. The published builder's default five-percent validation split was not exercised. MXFP8 parameter gather and gradient-buffer reuse, CuTeDSL grouped MLP, single grouped weights, FP32 gradient reduction, learned routing and selective gdn_norm_out/moe recomputation were enabled. Warmup 200 and the 300000-step cosine-decay horizon were retained, so this run remained in warmup and does not establish full convergence. Checkpoint writes were disabled; resume and post-SFT export are untested.

-
-
-
-
-

LoRA · GB200

- ○ Unverified -
-
-
Hardware
GB200
-
Precision
BF16
-
Last verified
—
-
-
-
Recorded metrics
-
-
-
Initial loss
-
None
-
-
-
Final loss
-
None
-
-
-
Step time · last 10 avg
-
None ms
-
-
-
Model throughput · last 10 avg
-
None TFLOP/s/GPU
-
-
-
Token throughput · last 10 avg
-
None tokens/s/GPU
-
-
-
-
-
Exact command
-

No runnable command is recorded for this status.

-
-
-
Expected result
-

This workflow has no independently recorded verification in this text-only card.

-
-
-
-
- -{/* END GENERATED VERIFIED CONFIGURATIONS */} diff --git a/docs/models/README.md b/docs/models/README.md index 7140653415..9c7c2e2d7b 100644 --- a/docs/models/README.md +++ b/docs/models/README.md @@ -4,7 +4,7 @@ Choose a model to open its interactive import/export and training configurations. Every option comes directly from an authoritative YAML verification card; independent fields are never combined into synthetic commands. -Current inventory: **29 model cards** and **418 concrete configurations**. +Current inventory: **28 model cards** and **406 concrete configurations**. Each model page lets you select import/export, pretraining, benchmarking, SFT, LoRA, or long-context SFT and immediately see the exact command and expected result for the recorded precision and GPU. @@ -168,7 +168,7 @@ Choose a model to inspect its recorded import/export, training, and precision co
Qwen - 6 models + 5 models
@@ -183,10 +183,6 @@ Choose a model to inspect its recorded import/export, training, and precision co Qwen3-8B ✓ Verified - - Qwen3.5-35B-A3B - ○ Unverified - Qwen3.6-35B-A3B ◐ Partial diff --git a/docs/models/qwen/index.md b/docs/models/qwen/index.md index 97f21dd5ce..c688de0c0e 100644 --- a/docs/models/qwen/index.md +++ b/docs/models/qwen/index.md @@ -12,7 +12,6 @@ qwen3-moe.md qwen3-next.md qwen2.5-vl.md qwen3-vl.md -qwen3.5-35b-a3b-text.md qwen3.6-35b-a3b.md qwen3.8-27b.md qwen2-audio.md @@ -30,7 +29,6 @@ qwen3-asr.md | Qwen3-Next | [qwen3-next.md](qwen3-next.md) | | Qwen2.5-VL | [qwen2.5-vl.md](qwen2.5-vl.md) | | Qwen3-VL | [qwen3-vl.md](qwen3-vl.md) | -| Qwen3.5-35B-A3B Text | [qwen3.5-35b-a3b-text.md](qwen3.5-35b-a3b-text.md) | | Qwen3.6-35B-A3B | [qwen3.6-35b-a3b.md](qwen3.6-35b-a3b.md) | | Qwen3.8-27B | [qwen3.8-27b.md](qwen3.8-27b.md) | | Qwen2-Audio | [qwen2-audio.md](qwen2-audio.md) | diff --git a/docs/models/qwen/qwen3.5-35b-a3b-text.md b/docs/models/qwen/qwen3.5-35b-a3b-text.md deleted file mode 100644 index 60936dd6db..0000000000 --- a/docs/models/qwen/qwen3.5-35b-a3b-text.md +++ /dev/null @@ -1,367 +0,0 @@ -# Qwen3.5-35B-A3B Text - -The text-only Qwen3.5-35B-A3B recipes use the language decoder and MTP weights -from Qwen/Qwen3.5-35B-A3B. They do not construct the vision tower. - -The configurations and validation evidence below are generated from the model -verification card. - - - -## Verified configurations - -Choose an exact recorded configuration to see its command and expected result. These selectors are generated from the authoritative verification cards and never synthesize combinations. - - -### Run a configuration - -Choose a workflow, precision, and exact recorded combination. The command and expected result update below. - -
- - -
-
-
-

Import · CPU

- ○ Unverified -
-
-
Hardware
not specified
-
Precision
BF16
-
Last verified
—
-
-
-
Exact command
-

No runnable command is recorded for this status.

-
-
-
Expected result
-

This workflow has no independently recorded verification in this text-only card.

-
-
-
-
-

Import · GPU

- ○ Unverified -
-
-
Hardware
not specified
-
Precision
BF16
-
Last verified
—
-
-
-
Exact command
-

No runnable command is recorded for this status.

-
-
-
Expected result
-

This workflow has no independently recorded verification in this text-only card.

-
-
-
-
-

Export · CPU

- ○ Unverified -
-
-
Hardware
not specified
-
Precision
BF16
-
Last verified
—
-
-
-
Exact command
-

No runnable command is recorded for this status.

-
-
-
Expected result
-

This workflow has no independently recorded verification in this text-only card.

-
-
-
-
-

Export · GPU

- ○ Unverified -
-
-
Hardware
not specified
-
Precision
BF16
-
Last verified
—
-
-
-
Exact command
-

No runnable command is recorded for this status.

-
-
-
Expected result
-

This workflow has no independently recorded verification in this text-only card.

-
-
-
-
-

Pretrain · GB200

- ○ Unverified -
-
-
Hardware
GB200
-
Precision
BF16
-
Last verified
—
-
-
-
Recorded metrics
-
-
-
Initial loss
-
None
-
-
-
Final loss
-
None
-
-
-
Step time · last 10 avg
-
None ms
-
-
-
Model throughput · last 10 avg
-
None TFLOP/s/GPU
-
-
-
Token throughput · last 10 avg
-
None tokens/s/GPU
-
-
-
-
-
Exact command
-

No runnable command is recorded for this status.

-
-
-
Expected result
-

This workflow has no independently recorded verification in this text-only card.

-
-
-
-
-

SFT · GB200

- ○ Unverified -
-
-
Hardware
GB200
-
Precision
BF16
-
Last verified
—
-
-
-
Recorded metrics
-
-
-
Initial loss
-
None
-
-
-
Final loss
-
None
-
-
-
Step time · last 10 avg
-
None ms
-
-
-
Model throughput · last 10 avg
-
None TFLOP/s/GPU
-
-
-
Token throughput · last 10 avg
-
None tokens/s/GPU
-
-
-
-
-
Exact command
-

No runnable command is recorded for this status.

-
-
-
Expected result
-

This workflow has no independently recorded verification in this text-only card.

-
-
-
-
-

Long Context · GB200

- ○ Unverified -
-
-
Hardware
GB200
-
Precision
FP8 MX
-
Last verified
—
-
-
-
Recorded metrics
-
-
-
Initial loss
-
None
-
-
-
Final loss
-
None
-
-
-
Step time · last 10 avg
-
None ms
-
-
-
Model throughput · last 10 avg
-
None TFLOP/s/GPU
-
-
-
Token throughput · last 10 avg
-
None tokens/s/GPU
-
-
-
-
-
Exact command
-
-
- Command - -
-
./scripts/training/train.sh --nodes 4 --gpus-per-node 4 --recipe qwen35_text_35b_a3b_sft_long_context_16gpu_gb200_fp8mx_config --pretrained_checkpoint work/model-verification/qwen3.5-35b-a3b-text/text-megatron --max_steps 100 dataset.hf_dataset=null dataset.hf_validation_proportion=null dataset.dataset_root=work/data/coderforge/materialized dataset.offline_packing_specs.packed_train_data_path=work/data/coderforge/packed-128k/training.parquet dataset.offline_packing_specs.packed_val_data_path=work/data/coderforge/packed-128k/validation.parquet dataset.offline_packing_specs.packed_metadata_path=work/data/coderforge/packed-128k/metadata.jsonl tokenizer.tokenizer_model=work/cache/qwen3.5-35b-a3b-text '++tokenizer.hf_tokenizer_kwargs.revision=null' checkpoint.hf_source_path=work/cache/qwen3.5-35b-a3b-text validation.eval_global_batch_size=32 validation.eval_micro_batch_size=1 checkpoint.save=null checkpoint.save_interval=null checkpoint.async_save=false logger.log_interval=1 logger.log_throughput=true logger.save_config_filepath=work/model-verification/qwen3.5-35b-a3b-text/resolved-config.yaml
-
-
-
-
Expected result
-

Historical observation, completed 2026-10-01: the text decoder and one pretrained MTP layer completed exactly 100 optimizer updates on 16 GB200 GPUs at TP1/PP1/CP8/EP16/ETP1, GBS32/MBS1 and 16 accumulation steps. LM, MTP, router losses and gradient norms were finite, with zero skipped or NaN updates and evaluations at steps 50 and 100. Online experiment logging finished successfully and the post-setup config was retained. Initial LM loss was 0.2099137008190155; final LM loss was 0.14850857853889465; final MTP loss was 0.19482117891311646. Updates 91-100 averaged 28591.89546108246 ms and 336.1899744993698 model TFLOP/s/GPU, corresponding to 9168.47224615851 token slots/s/GPU, not supervised-token throughput. These historical observations are deliberately separate from the null clean-recipe metrics above. The run used NeMo 26.10.rc3 and MCore b5581a9e874fc5be2b91e74cdd3ccdc64a1b5204 from NVIDIA/Megatron-LM PR 7611. Its Bridge source was base 00ca266c4abc5f8435821823a1dbe05d0c3d6033 plus an uncommitted recipe precursor, frozen as source bundle SHA256 5ea6d3c0c82a47f06f9f031040c2cc78641a4334d961542e25338f9c585ed863. No clean source commit or public runtime image identifies that run; the null provenance fields must not be replaced with the later PR head. The command above describes the published recipe with equivalent workload settings, checked by construction and full configuration comparison, but it has not been rerun from a clean checkout. Inputs were text/MTP weights extracted from the pinned HF model and togethercomputer/CoderForge-Preview revision 060fca96cf723b2ebab3181e9e59fafd273df3cb, configuration trajectories, split SWE_Rebench. Seed 1234 produced a 99:1 split of 77169 rows; the first 20 percent of training rows (15279) and all 772 validation rows were packed with first_fit_shuffle, a 131072-token budget and 16-token alignment. The 6491 training packs contain 205182453 supervised tokens in the entire prepared corpus; the 333 validation packs contain 10479031. These counts are corpus totals, not tokens consumed by the run. The published builder's default five-percent validation split was not exercised. MXFP8 parameter gather and gradient-buffer reuse, CuTeDSL grouped MLP, single grouped weights, FP32 gradient reduction, learned routing and selective gdn_norm_out/moe recomputation were enabled. Warmup 200 and the 300000-step cosine-decay horizon were retained, so this run remained in warmup and does not establish full convergence. Checkpoint writes were disabled; resume and post-SFT export are untested. -

-
-
-
-
-

LoRA · GB200

- ○ Unverified -
-
-
Hardware
GB200
-
Precision
BF16
-
Last verified
—
-
-
-
Recorded metrics
-
-
-
Initial loss
-
None
-
-
-
Final loss
-
None
-
-
-
Step time · last 10 avg
-
None ms
-
-
-
Model throughput · last 10 avg
-
None TFLOP/s/GPU
-
-
-
Token throughput · last 10 avg
-
None tokens/s/GPU
-
-
-
-
-
Exact command
-

No runnable command is recorded for this status.

-
-
-
Expected result
-

This workflow has no independently recorded verification in this text-only card.

-
-
-
-
- - diff --git a/examples/model_verification_cards/qwen3.5-35b-a3b-text/card.yaml b/examples/model_verification_cards/qwen3.5-35b-a3b-text/card.yaml deleted file mode 100644 index 7b4d156714..0000000000 --- a/examples/model_verification_cards/qwen3.5-35b-a3b-text/card.yaml +++ /dev/null @@ -1,214 +0,0 @@ -# Agent-readable model verification card. -# Historical observations are not attributed to a later clean source revision. -title: qwen3_5_35b_a3b_text -summary: > - Performance disclaimer: this card does not record a canonical pretrain performance result; reported timing and throughput metrics are functional verification observations, not standalone optimized performance results. - Records the completed 100-update MXFP8 text-only long-context SFT run. - The run used an uncommitted recipe precursor and a release-candidate runtime; - clean-source verification of the published recipe remains unverified. -verification_index: - model_level: - unverified: [hf_to_megatron_cpu, hf_to_megatron_gpu, megatron_to_hf_cpu, megatron_to_hf_gpu, manual_forward_pass, inference] - training: - GB200: - unverified: [pretrain, sft, sft_export_inference, sft_long_context, peft, checkpoint_resume] -model: - hf_id: Qwen/Qwen3.5-35B-A3B - hf_revision: 59d61f3ce65a6d9863b86d2e96597125219dc754 # pragma: allowlist secret - architecture: Qwen3_5MoeForCausalLM - min_transformers_version: "5.12.1" -verification_environment: - # No public image or clean commit identifies the historical runtime exactly. - base_container: null - bridge_commit: null - -items: - hf_to_megatron_cpu: - status: unverified - precision: bf16 - command: null - last_verified: null - expected_result: This workflow has no independently recorded verification in this text-only card. - - hf_to_megatron_gpu: - status: unverified - precision: bf16 - command: null - last_verified: null - expected_result: This workflow has no independently recorded verification in this text-only card. - - megatron_to_hf_cpu: - status: unverified - precision: bf16 - command: null - last_verified: null - expected_result: This workflow has no independently recorded verification in this text-only card. - - megatron_to_hf_gpu: - status: unverified - precision: bf16 - command: null - last_verified: null - expected_result: This workflow has no independently recorded verification in this text-only card. - - manual_forward_pass: - status: unverified - precision: bf16 - command: null - last_verified: null - expected_result: This workflow has no independently recorded verification in this text-only card. - - inference: - status: unverified - precision: bf16 - command: null - last_verified: null - expected_result: This workflow has no independently recorded verification in this text-only card. - - pretrain: - GB200: - status: unverified - precision: bf16 - enabled_features: {} - command: null - last_verified: null - metrics: - initial_loss: null - final_loss: null - last_10_steps_step_time_ms_avg: null - last_10_steps_model_tflops_per_gpu_avg: null - last_10_steps_tokens_per_second_per_gpu_avg: null - expected_result: This workflow has no independently recorded verification in this text-only card. - - sft: - GB200: - status: unverified - precision: bf16 - enabled_features: {} - command: null - last_verified: null - metrics: - initial_loss: null - final_loss: null - last_10_steps_step_time_ms_avg: null - last_10_steps_model_tflops_per_gpu_avg: null - last_10_steps_tokens_per_second_per_gpu_avg: null - expected_result: This workflow has no independently recorded verification in this text-only card. - - sft_export_inference: - GB200: - status: unverified - precision: bf16 - depends_on: sft - commands: null - last_verified: null - expected_result: This workflow has no independently recorded verification in this text-only card. - - sft_long_context: - GB200: - status: unverified - precision: fp8_mx - enabled_features: - sequence_packing: offline - context_parallel_size: 8 - moe_dispatcher: hybridep - command: > - ./scripts/training/train.sh --nodes 4 --gpus-per-node 4 - --recipe qwen35_text_35b_a3b_sft_long_context_16gpu_gb200_fp8mx_config - --pretrained_checkpoint work/model-verification/qwen3.5-35b-a3b-text/text-megatron - --max_steps 100 - dataset.hf_dataset=null dataset.hf_validation_proportion=null - dataset.dataset_root=work/data/coderforge/materialized - dataset.offline_packing_specs.packed_train_data_path=work/data/coderforge/packed-128k/training.parquet - dataset.offline_packing_specs.packed_val_data_path=work/data/coderforge/packed-128k/validation.parquet - dataset.offline_packing_specs.packed_metadata_path=work/data/coderforge/packed-128k/metadata.jsonl - tokenizer.tokenizer_model=work/cache/qwen3.5-35b-a3b-text - '++tokenizer.hf_tokenizer_kwargs.revision=null' - checkpoint.hf_source_path=work/cache/qwen3.5-35b-a3b-text - validation.eval_global_batch_size=32 validation.eval_micro_batch_size=1 - checkpoint.save=null checkpoint.save_interval=null checkpoint.async_save=false - logger.log_interval=1 logger.log_throughput=true - logger.save_config_filepath=work/model-verification/qwen3.5-35b-a3b-text/resolved-config.yaml - last_verified: null - metrics: - initial_loss: null - final_loss: null - last_10_steps_step_time_ms_avg: null - last_10_steps_model_tflops_per_gpu_avg: null - last_10_steps_tokens_per_second_per_gpu_avg: null - expected_result: > - Historical observation, completed 2026-10-01: the text decoder and one - pretrained MTP layer completed exactly 100 optimizer updates on 16 GB200 - GPUs at TP1/PP1/CP8/EP16/ETP1, GBS32/MBS1 and 16 accumulation steps. - LM, MTP, router losses and gradient norms were finite, with zero skipped - or NaN updates and evaluations at steps 50 and 100. Online experiment - logging finished successfully and the post-setup config was retained. - Initial LM loss was 0.2099137008190155; final LM loss was - 0.14850857853889465; final MTP loss was 0.19482117891311646. - Updates 91-100 averaged 28591.89546108246 ms and - 336.1899744993698 model TFLOP/s/GPU, corresponding to - 9168.47224615851 token slots/s/GPU, not supervised-token throughput. - These historical observations are deliberately separate from the null - clean-recipe metrics above. - The run used NeMo 26.10.rc3 and MCore - b5581a9e874fc5be2b91e74cdd3ccdc64a1b5204 from NVIDIA/Megatron-LM PR 7611. - Its Bridge source was base 00ca266c4abc5f8435821823a1dbe05d0c3d6033 - plus an uncommitted recipe precursor, frozen as source bundle SHA256 - 5ea6d3c0c82a47f06f9f031040c2cc78641a4334d961542e25338f9c585ed863. - No clean source commit or public runtime image identifies that run; - the null provenance fields must not be replaced with the later PR head. - The command above describes the published recipe with equivalent - workload settings, checked by construction and full configuration - comparison, but it has not been rerun from a clean checkout. - Inputs were text/MTP weights extracted from the pinned HF model and - togethercomputer/CoderForge-Preview revision - 060fca96cf723b2ebab3181e9e59fafd273df3cb, configuration trajectories, - split SWE_Rebench. Seed 1234 produced a 99:1 split of 77169 rows; - the first 20 percent of training rows (15279) and all 772 validation - rows were packed with first_fit_shuffle, a 131072-token budget and - 16-token alignment. The 6491 training packs contain 205182453 supervised - tokens in the entire prepared corpus; the 333 validation packs contain - 10479031. These counts are corpus totals, not tokens consumed by the run. - The published builder's default five-percent validation split was not - exercised. MXFP8 parameter gather and gradient-buffer reuse, CuTeDSL - grouped MLP, single grouped weights, FP32 gradient reduction, learned - routing and selective gdn_norm_out/moe recomputation were enabled. - Warmup 200 and the 300000-step cosine-decay horizon were retained, so - this run remained in warmup and does not establish full convergence. - Checkpoint writes were disabled; resume and post-SFT export are untested. - - peft: - GB200: - status: unverified - precision: bf16 - enabled_features: {} - command: null - last_verified: null - metrics: - initial_loss: null - final_loss: null - last_10_steps_step_time_ms_avg: null - last_10_steps_model_tflops_per_gpu_avg: null - last_10_steps_tokens_per_second_per_gpu_avg: null - expected_result: This workflow has no independently recorded verification in this text-only card. - - checkpoint_resume: - GB200: - status: unverified - precision: bf16 - command: null - last_verified: null - metrics: - initial_loss: null - final_loss: null - last_10_steps_step_time_ms_avg: null - last_10_steps_model_tflops_per_gpu_avg: null - last_10_steps_tokens_per_second_per_gpu_avg: null - depends_on: pretrain - resume_comparison: - reference_item: pretrain - sentinel_steps: [] - loss_relative_tolerance: 1.0e-2 - loss_absolute_tolerance: 1.0e-6 - sentinels_match: null - expected_result: This workflow has no independently recorded verification in this text-only card. diff --git a/scripts/docs/generate_model_verification_catalog.py b/scripts/docs/generate_model_verification_catalog.py index ba32a39ae3..30cfb0836f 100644 --- a/scripts/docs/generate_model_verification_catalog.py +++ b/scripts/docs/generate_model_verification_catalog.py @@ -115,7 +115,6 @@ "qwen3-235b-a22b": "models/qwen/qwen3-235b-a22b.md", "qwen3-30b-a3b": "models/qwen/qwen3-30b-a3b.md", "qwen3-8b": "models/qwen/qwen3-8b.md", - "qwen3.5-35b-a3b-text": "models/qwen/qwen3.5-35b-a3b-text.md", "qwen3.6-35b-a3b": "models/qwen/qwen3.6-35b-a3b.md", "qwen3.8-27b": "models/qwen/qwen3.8-27b.md", "step35-flash": "models/stepfun/step35.md", @@ -131,7 +130,6 @@ "qwen3-235b-a22b": "Qwen3-235B-A22B", "qwen3-30b-a3b": "Qwen3-30B-A3B", "qwen3-8b": "Qwen3-8B", - "qwen3.5-35b-a3b-text": "Qwen3.5-35B-A3B Text", "qwen3.6-35b-a3b": "Qwen3.6-35B-A3B", "qwen3.8-27b": "Qwen3.8-27B", } diff --git a/skills/create-model-verification-card/SKILL.md b/skills/create-model-verification-card/SKILL.md index 0e27978d61..813cc257e8 100644 --- a/skills/create-model-verification-card/SKILL.md +++ b/skills/create-model-verification-card/SKILL.md @@ -46,14 +46,6 @@ identifier and the exact Bridge commit used for verification. Put them in the container, record the mounted checkout commit. Never substitute a private image path for the public base container identifier. -When no item is verified, either environment field may be null if its exact -public provenance is unavailable. Keep historical observations in the relevant -unverified item's `expected_result`, with the actual source/runtime limitations -stated explicitly and its structured metrics and date null. Do not substitute -a later clean commit or invent a public image for a release-candidate run. -This applies to draft and historical cards for any model. As soon as any item -is verified, both environment fields must satisfy the existing requirements. - The top-level Bridge commit is the default for every item leaf. If one verified item was run from a different clean checkout, put that exact 40-hex commit in the leaf's optional `bridge_commit` field. Omit the field when it would repeat diff --git a/skills/create-model-verification-card/scripts/validate_card.py b/skills/create-model-verification-card/scripts/validate_card.py index 6c19431001..3611bff120 100644 --- a/skills/create-model-verification-card/scripts/validate_card.py +++ b/skills/create-model-verification-card/scripts/validate_card.py @@ -2037,24 +2037,14 @@ def _validate_card(card: Mapping[str, Any], raw: str, deny_terms: tuple[str, ... path=("verification_environment",), errors=errors, ) - # An unverified card may not yet have publishable, exact provenance. - # Never force an invented image/commit, or relax provenance for verified runs. - item_values = card.get("items") - has_verified_item = isinstance(item_values, Mapping) and any( - item.get("status") == "verified" for _, _, item, _ in _iter_item_leaves(item_values) - ) base_container = environment.get("base_container") - if not (base_container is None and not has_verified_item) and ( - not isinstance(base_container, str) or PUBLIC_BASE_CONTAINER_RE.fullmatch(base_container) is None - ): + if not isinstance(base_container, str) or PUBLIC_BASE_CONTAINER_RE.fullmatch(base_container) is None: errors.append( f"{_pointer('verification_environment', 'base_container')}: " "expected a public NVIDIA NeMo or PyTorch container" ) bridge_commit = environment.get("bridge_commit") - if not (bridge_commit is None and not has_verified_item) and ( - not isinstance(bridge_commit, str) or REVISION_RE.fullmatch(bridge_commit) is None - ): + if not isinstance(bridge_commit, str) or REVISION_RE.fullmatch(bridge_commit) is None: errors.append( f"{_pointer('verification_environment', 'bridge_commit')}: expected an immutable 40-hex commit" ) diff --git a/tests/unit_tests/skills/create_model_verification_card/test_validate_card.py b/tests/unit_tests/skills/create_model_verification_card/test_validate_card.py index f269f80255..f83fd9e557 100644 --- a/tests/unit_tests/skills/create_model_verification_card/test_validate_card.py +++ b/tests/unit_tests/skills/create_model_verification_card/test_validate_card.py @@ -796,46 +796,3 @@ def test_resume_reference_only_warm_start(resume_initialization, expected_errors ) assert errors == expected_errors - - -def test_unverified_card_can_record_unknown_provenance_without_fabricated_values(): - validator = _load_validator() - path = REPO_ROOT / "examples/model_verification_cards/qwen3.5-35b-a3b-text/card.yaml" - raw = path.read_text() - card = yaml.safe_load(raw) - - assert card["verification_environment"] == {"base_container": None, "bridge_commit": None} - assert validator._validate_card(card, raw, ()) == [] - - -@pytest.mark.parametrize( - "items", - [ - {"inference": {"status": "verified"}}, - {"sft_long_context": {"GB200": {"status": "verified"}}}, - {"pretrain_fsdp": {"GB200": {"status": "unverified", "variants": {"fp8_mx": {"status": "verified"}}}}}, - {"pretrain_weak_scaling": {"GB300": {"status": "verified"}}}, - ], -) -def test_verified_items_still_require_public_container_and_immutable_commit(items): - validator = _load_validator() - card = {"verification_environment": {"base_container": None, "bridge_commit": None}, "items": items} - - errors = validator._validate_card(card, yaml.safe_dump(card), ()) - - assert "/verification_environment/base_container: expected a public NVIDIA NeMo or PyTorch container" in errors - assert "/verification_environment/bridge_commit: expected an immutable 40-hex commit" in errors - - -@pytest.mark.parametrize("field,value", [("base_container", "not-an-image"), ("bridge_commit", "main")]) -def test_unverified_provenance_rejects_non_null_invalid_values(field, value): - validator = _load_validator() - card = { - "verification_environment": {"base_container": None, "bridge_commit": None}, - "items": {"inference": {"status": "unverified"}}, - } - card["verification_environment"][field] = value - - errors = validator._validate_card(card, yaml.safe_dump(card), ()) - - assert any(error.startswith(f"/verification_environment/{field}:") for error in errors) From 9952be170f3afc936e74eeb70ae0116707be5da2 Mon Sep 17 00:00:00 2001 From: Chen Cui Date: Tue, 6 Oct 2026 17:41:28 -0700 Subject: [PATCH 5/5] refactor(recipes): inherit common defaults for Qwen3.5 text SFT Signed-off-by: Chen Cui --- .../bridge/recipes/qwen/gb200/qwen35.py | 32 ------------------- 1 file changed, 32 deletions(-) diff --git a/src/megatron/bridge/recipes/qwen/gb200/qwen35.py b/src/megatron/bridge/recipes/qwen/gb200/qwen35.py index c1eb9812a6..3f288ba76e 100644 --- a/src/megatron/bridge/recipes/qwen/gb200/qwen35.py +++ b/src/megatron/bridge/recipes/qwen/gb200/qwen35.py @@ -75,46 +75,25 @@ def qwen35_text_35b_a3b_sft_long_context_16gpu_gb200_fp8mx_config( cfg.tokenizer.hf_tokenizer_kwargs = {"revision": hf_revision} cfg.model.seq_length = seq_length - cfg.model.tensor_model_parallel_size = 1 - cfg.model.pipeline_model_parallel_size = 1 cfg.model.pipeline_dtype = torch.bfloat16 - cfg.model.virtual_pipeline_model_parallel_size = None cfg.model.context_parallel_size = 8 cfg.model.expert_model_parallel_size = 16 cfg.model.expert_tensor_parallel_size = 1 - cfg.model.sequence_parallel = False cfg.model.mtp_num_layers = 1 - cfg.model.mtp_loss_scaling_factor = 0.1 - cfg.model.transformer_impl = "transformer_engine" - cfg.model.attention_backend = "auto" - cfg.model.cuda_graph_impl = "none" - cfg.model.gradient_accumulation_fusion = True cfg.model.bias_activation_fusion = True - cfg.model.cross_entropy_loss_fusion = True cfg.model.cross_entropy_fusion_impl = "te" cfg.model.calculate_per_token_loss = True - cfg.model.apply_rope_fusion = False cfg.model.recompute_granularity = "selective" cfg.model.recompute_modules = ["gdn_norm_out", "moe"] - cfg.model.recompute_method = None - cfg.model.recompute_num_layers = None cfg.model.moe_token_dispatcher_type = "flex" cfg.model.moe_flex_dispatcher_backend = "hybridep" cfg.model.moe_flex_dispatcher_num_sms = 32 - cfg.model.moe_hybridep_num_sms = None cfg.model.moe_hybridep_pad_uneven_dispatch_inputs = True cfg.model.moe_router_fusion = True - cfg.model.moe_permute_fusion = True - cfg.model.moe_grouped_gemm = True cfg.model.moe_use_grouped_tensor = True cfg.model.use_transformer_engine_op_fuser = True cfg.model.moe_mlp_glu_interleave_size = 32 cfg.model.moe_single_grouped_weight = True - cfg.model.moe_single_grouped_bias = False - cfg.model.moe_shared_expert_overlap = False - cfg.model.moe_router_force_load_balancing = False - cfg.model.overlap_moe_expert_parallel_comm = False - cfg.model.delay_wgrad_compute = False # Pad each runtime segment to 2*CP, not each conversation to seq_length. # Physical and logical cu_seqlens remain distinct for attention and MTP. @@ -127,13 +106,11 @@ def qwen35_text_35b_a3b_sft_long_context_16gpu_gb200_fp8mx_config( cfg.dataset.hf_dataset.load_kwargs = {"revision": _CODERFORGE_REVISION} cfg.dataset.do_validation = True cfg.dataset.hf_validation_proportion = 0.05 - cfg.dataset.num_workers = 2 # Preserve the original long-context recipe's sampling and worker lifecycle. cfg.dataset.seed = 1234 cfg.dataset.persistent_workers = True cfg.train.train_iters = 500 cfg.train.global_batch_size = 32 - cfg.train.micro_batch_size = 1 cfg.train.manual_gc = True cfg.train.manual_gc_interval = 100 cfg.train.manual_gc_eval = 100 @@ -144,24 +121,15 @@ def qwen35_text_35b_a3b_sft_long_context_16gpu_gb200_fp8mx_config( cfg.optimizer, cfg.scheduler = distributed_fused_adam_with_cosine_annealing( lr_warmup_iters=200, lr_decay_iters=300000, max_lr=2e-5, min_lr=2e-6 ) - cfg.optimizer.use_precision_aware_optimizer = False cfg.optimizer.overlap_param_gather = True - cfg.optimizer.overlap_param_gather_with_optimizer_step = False cfg.mixed_precision = bf16_with_mxfp8_mixed() - cfg.mixed_precision.fp8_param_gather = True - cfg.mixed_precision.reuse_grad_buf_for_mxfp8_param_ag = True cfg.checkpoint.load_main_params_from_ckpt = True cfg.checkpoint.load_optim = False cfg.checkpoint.load_rng = False - cfg.mixed_precision.grad_reduce_in_fp32 = True - cfg.ddp.grad_reduce_in_fp32 = True cfg.ddp.use_distributed_optimizer = True cfg.ddp.data_parallel_sharding_strategy = "optim_grads_params" - cfg.ddp.average_in_collective = False cfg.ddp.overlap_grad_reduce = True cfg.ddp.overlap_param_gather = True - cfg.comm_overlap = None # Do not overwrite launch-time DDP/optimizer choices. - cfg.rerun_state_machine.check_for_nan_in_loss = True cfg.env_vars = { **COMMON_RECIPE_ENV_VARS, "CUDA_DEVICE_MAX_CONNECTIONS": 1,