diff --git a/.gitlab/stages/01.build.yml b/.gitlab/stages/01.build.yml index 936f41b6ddc..74495b3270e 100644 --- a/.gitlab/stages/01.build.yml +++ b/.gitlab/stages/01.build.yml @@ -62,12 +62,12 @@ test:pre_build_image: - IMAGE: CI_MCORE_DEV_IMAGE FILE: Dockerfile.ci.dev IMAGE_TYPE: dev - BASE_IMAGE: nvcr.io/nvidia/pytorch:26.08-py3 + BASE_IMAGE: nvcr.io/nvidia/pytorch:26.09-py3 PLATFORM: amd64 - IMAGE: CI_MCORE_DEV_IMAGE FILE: Dockerfile.ci.dev IMAGE_TYPE: dev - BASE_IMAGE: nvcr.io/nvidia/pytorch:26.08-py3 + BASE_IMAGE: nvcr.io/nvidia/pytorch:26.09-py3 PLATFORM: arm64 - IMAGE: UTILITY_IMAGE FILE: Dockerfile.linting diff --git a/docker/.ngc_version.dev b/docker/.ngc_version.dev index c2c1543aa0e..52b683b9c8f 100644 --- a/docker/.ngc_version.dev +++ b/docker/.ngc_version.dev @@ -1 +1 @@ -nvcr.io/nvidia/pytorch:26.08-py3 +nvcr.io/nvidia/pytorch:26.09-py3 diff --git a/docker/Dockerfile.ci.dev b/docker/Dockerfile.ci.dev index a246a5e1aff..a7069b311f8 100644 --- a/docker/Dockerfile.ci.dev +++ b/docker/Dockerfile.ci.dev @@ -17,7 +17,8 @@ ENV UV_LINK_MODE=copy RUN bash -ex <<"EOF" apt-get update - apt-get install -y --no-install-recommends gettext python3-venv psmisc uuid-runtime + # DeepGEMM/DeepJIT requires elfutils/libdwfl.h for its exception support. + apt-get install -y --no-install-recommends gettext python3-venv psmisc uuid-runtime libdw-dev apt-get clean python -m venv /opt/jet ARCH=$(uname -m) @@ -88,6 +89,7 @@ RUN --mount=type=bind,source=docker/common/install_nccl.sh,target=/opt/install_n EOF COPY pyproject.toml uv.lock /workspace/ +COPY docker/patches/mamba-cxx20.patch /workspace/mamba-cxx20.patch COPY megatron/core/__init__.py /workspace/megatron/core/ COPY megatron/core/package_info.py /workspace/megatron/core/ ARG IMAGE_TYPE=dev @@ -131,6 +133,7 @@ RUN --mount=type=cache,id=uv-${TARGETARCH}-${NVTE_CUDA_ARCHS},target=/root/.cach --no-install-package torch \ --no-install-package torchvision \ --no-install-package triton \ + --no-install-package mamba-ssm \ --no-install-package transformer-engine-cu12 \ --no-install-package nvidia-cublas-cu12 \ --no-install-package nvidia-cuda-cupti-cu12 \ @@ -161,6 +164,25 @@ RUN --mount=type=cache,id=uv-${TARGETARCH}-${NVTE_CUDA_ARCHS},target=/root/.cach cp -a "${NCCL_EP_JIT_HEADERS}/../nccl_ep.h" /opt/nccl-ep/include/ test -f /opt/nccl-ep/include/nccl_ep/device/ht_ep.cuh test -f /opt/nccl-ep/include/nccl_ep.h + + # Backport state-spaces/mamba#1000 without changing the pinned runtime sources. + # PyTorch's ATen headers now require C++20 for both host and CUDA compilation. + MAMBA_REV=$(sed -n 's/^mamba-ssm = .*rev = "\([^"]*\)".*/\1/p' /workspace/pyproject.toml) + test -n "${MAMBA_REV}" + MAMBA_BUILD_DIR=$(mktemp -d) + trap 'rm -rf "${MAMBA_BUILD_DIR}"' EXIT + git -C "${MAMBA_BUILD_DIR}" init + git -C "${MAMBA_BUILD_DIR}" fetch --depth 1 https://github.com/state-spaces/mamba.git "${MAMBA_REV}" + git -C "${MAMBA_BUILD_DIR}" checkout --detach "${MAMBA_REV}" + git -C "${MAMBA_BUILD_DIR}" apply /workspace/mamba-cxx20.patch + MAMBA_FORCE_BUILD=TRUE uv pip install --no-deps --no-build-isolation -v "${MAMBA_BUILD_DIR}" + # Keep the source revision and applied patch after deleting the build checkout. + mkdir -p /opt/mamba-build-info + git -C "${MAMBA_BUILD_DIR}" rev-parse HEAD > /opt/mamba-build-info/revision + cp /workspace/mamba-cxx20.patch /opt/mamba-build-info/ + rm -rf "${MAMBA_BUILD_DIR}" + trap - EXIT + EOF # Reuse the wheel without copying a second archive into the final image. diff --git a/docker/common/install.sh b/docker/common/install.sh index 162e879aa86..31a1f4c898c 100644 --- a/docker/common/install.sh +++ b/docker/common/install.sh @@ -86,7 +86,8 @@ main() { # Install tools apt-get update - apt-get install -y wget curl git cmake + # DeepGEMM/DeepJIT requires elfutils/libdwfl.h for its exception support. + apt-get install -y wget curl git cmake libdw-dev # Install CUDA if [[ "$BASE_IMAGE" == "ubuntu" ]]; then diff --git a/docker/patches/mamba-cxx20.patch b/docker/patches/mamba-cxx20.patch new file mode 100644 index 00000000000..915823f7581 --- /dev/null +++ b/docker/patches/mamba-cxx20.patch @@ -0,0 +1,33 @@ +Backport of https://github.com/state-spaces/mamba/commit/653923ce8fb0d47cdd9bcfd5904a0f1d58f91274 +Build the existing pinned Mamba sources with the standard required by PyTorch's ATen headers. + +diff --git a/setup.py b/setup.py +--- a/setup.py ++++ b/setup.py +@@ -210,10 +210,10 @@ def append_nvcc_threads(nvcc_extra_args): + if HIP_BUILD: + + extra_compile_args = { +- "cxx": ["-O3", "-std=c++17"], ++ "cxx": ["-O3", "-std=c++20"], + "nvcc": [ + "-O3", +- "-std=c++17", ++ "-std=c++20", + f"--offload-arch={os.getenv('HIP_ARCHITECTURES', 'native')}", + "-U__CUDA_NO_HALF_OPERATORS__", + "-U__CUDA_NO_HALF_CONVERSIONS__", +@@ -223,11 +223,11 @@ def append_nvcc_threads(nvcc_extra_args): + } + else: + extra_compile_args = { +- "cxx": ["-O3", "-std=c++17"], ++ "cxx": ["-O3", "-std=c++20"], + "nvcc": append_nvcc_threads( + [ + "-O3", +- "-std=c++17", ++ "-std=c++20", + "-U__CUDA_NO_HALF_OPERATORS__", + "-U__CUDA_NO_HALF_CONVERSIONS__", + "-U__CUDA_NO_BFLOAT16_OPERATORS__", diff --git a/megatron/core/fusions/fused_bias_swiglu.py b/megatron/core/fusions/fused_bias_swiglu.py index 8a32d90b871..f8e6ace1fba 100644 --- a/megatron/core/fusions/fused_bias_swiglu.py +++ b/megatron/core/fusions/fused_bias_swiglu.py @@ -3,14 +3,44 @@ # pylint: disable=missing-function-docstring, missing-class-docstring +from functools import cache from typing import Optional import torch import torch.nn.functional as F +from torch._subclasses.fake_tensor import unset_fake_temporarily from megatron.core.jit import jit_fuser from megatron.core.utils import nvtx_decorator + +@cache +def _probe_clamp_boundaries(): + """Read the installed scalar-clamp subgradient using only CPU constants.""" + # PyTorch changed this within the 2.14 development series, so a version check + # cannot distinguish builds. FunctionsManual.cpp at b2c75dd062 uses strict + # inequalities; 4fdf77b940 uses inclusive ones. Probe both scalar-clamp forms + # once on CPU, including calls under no_grad or inference_mode. + # This zero-input probe needs real CPU constants, even during fake tracing. + with unset_fake_temporarily(), torch.inference_mode(False), torch.enable_grad(): + gate = torch.tensor(1.0, dtype=torch.float32, device="cpu", requires_grad=True) + linear = torch.tensor([-1.0, 1.0], dtype=torch.float32, device="cpu", requires_grad=True) + gate_grad, linear_grad = torch.autograd.grad( + gate.clamp(max=1.0) + linear.clamp(min=-1.0, max=1.0).sum(), (gate, linear) + ) + gradients = [gate_grad.item(), *linear_grad.tolist()] + if gradients not in ([0.0, 0.0, 0.0], [1.0, 1.0, 1.0]): + raise RuntimeError(f"Unsupported scalar-clamp boundary gradients: {gradients}") + return gradients[0] == 1.0 + + +@torch.compiler.assume_constant_result +def _clamp_includes_boundaries(): + # Delay autograd initialization until this clamped backward is actually used. + # Keep the cached probe outside tracing and specialize only its Python bool. + return _probe_clamp_boundaries() + + ###### BIAS SWIGLU FUSION/ NO AUTOGRAD ################ @@ -163,14 +193,20 @@ def clamped_swiglu_back(g, y, clamp_value): y_1, y_2 = torch.chunk(y.to(torch.float32), 2, -1) y_1c = y_1.clamp(min=None, max=clamp_value) y_2c = y_2.clamp(min=-clamp_value, max=clamp_value) + if _clamp_includes_boundaries(): + gate_mask = y_1 <= clamp_value + linear_mask = (y_2 >= -clamp_value) & (y_2 <= clamp_value) + else: + gate_mask = y_1 < clamp_value + linear_mask = (y_2 > -clamp_value) & (y_2 < clamp_value) res = torch.cat( ( g * torch.sigmoid(y_1c) * (1 + y_1c * (1 - torch.sigmoid(y_1c))) * y_2c - * (y_1 <= clamp_value).to(g.dtype), - g * F.silu(y_1c) * ((y_2 >= -clamp_value) & (y_2 <= clamp_value)).to(g.dtype), + * gate_mask.to(g.dtype), + g * F.silu(y_1c) * linear_mask.to(g.dtype), ), -1, ) diff --git a/megatron/core/ssm/ssm_inference.py b/megatron/core/ssm/ssm_inference.py index dd266627b95..6c81339d58b 100644 --- a/megatron/core/ssm/ssm_inference.py +++ b/megatron/core/ssm/ssm_inference.py @@ -223,18 +223,15 @@ def ssm_dynamic_inference( if decode_req_count > 0: seq_len = 1 + context.num_speculative_tokens decode_token_count = decode_req_count * seq_len - if context.batch_invariant_mode: - # Batch-invariant execution may include token-only rows to preserve - # model-wide M alignment. Those rows do not represent requests and - # must not be passed to the recurrent decode kernels. - assert decode_token_count <= zxBCdt.shape[0], ( - "Batch-invariant SSM metadata describes more decode tokens " - f"({decode_token_count}) than the input projection contains " - f"({zxBCdt.shape[0]})." - ) - zxBCdt_decode = zxBCdt[:decode_token_count] - else: - zxBCdt_decode = zxBCdt[:decode_token_count] if prefill_req_count > 0 else zxBCdt + # Token-only padding can come from kernel-level batch invariance even + # when the model/context flag is off. Only metadata-backed requests + # belong in the recurrent decode kernels. + assert decode_token_count <= zxBCdt.shape[0], ( + "SSM metadata describes more decode tokens " + f"({decode_token_count}) than the input projection contains " + f"({zxBCdt.shape[0]})." + ) + zxBCdt_decode = zxBCdt[:decode_token_count] # Reshape from [N*S, 1, d] to [N, S, d] for the decode kernels. zxBCdt_decode = zxBCdt_decode.squeeze(1).view(decode_req_count, seq_len, -1) y_decode = self.ssm_decode( @@ -282,16 +279,15 @@ def ssm_dynamic_inference( else: raise RuntimeError("Dynamic inference called with 0 decode and 0 prefill requests") - if context.batch_invariant_mode: - # Restore the projection's token-only padding before the output projection. - # Its row count can be TP-local, unlike the context's global token count. - padding_token_count = zxBCdt.shape[0] - y.shape[0] - assert padding_token_count >= 0, ( - "Batch-invariant SSM produced more token rows " - f"({y.shape[0]}) than the input projection contained ({zxBCdt.shape[0]})." - ) - if padding_token_count > 0: - y = torch.cat((y, y.new_zeros(padding_token_count, *y.shape[1:])), dim=0) + # Restore the projection's token-only padding before the output projection. + # Its row count can be TP-local, unlike the context's global token count. + padding_token_count = zxBCdt.shape[0] - y.shape[0] + assert padding_token_count >= 0, ( + "SSM produced more token rows " + f"({y.shape[0]}) than the input projection contained ({zxBCdt.shape[0]})." + ) + if padding_token_count > 0: + y = torch.cat((y, y.new_zeros(padding_token_count, *y.shape[1:])), dim=0) # Zero padding positions to avoid corrupting quantization amax calculations. if is_using_quantization_scales(self.config): diff --git a/megatron/core/tensor_parallel/random.py b/megatron/core/tensor_parallel/random.py index e6931715c2a..183d67e633f 100644 --- a/megatron/core/tensor_parallel/random.py +++ b/megatron/core/tensor_parallel/random.py @@ -448,16 +448,53 @@ def get_all_rng_states(): return {} +_CUDAGRAPH_NEEDS_GENERATOR_REGISTRATION = None + + +def _probe_cudagraph_needs_generator_registration() -> bool: + """Capture an RNG op on an unregistered generator and report whether capture rejects it.""" + generator = torch.Generator(device="cuda") + graph = torch.cuda.CUDAGraph() + try: + with torch.cuda.graph(graph, capture_error_mode="thread_local"): + torch.rand(1, device="cuda", generator=generator) + except RuntimeError: + # Without lazy registration, capture fails with "RNG op during graph capture but + # generator is not registered with the capturing graph". Treat any capture failure + # as needing registration: on PyTorch with lazy registration, registering anyway + # only costs a deprecation warning, while skipping it on older builds is fatal. + return True + finally: + del graph + return False + + def cudagraph_needs_generator_registration() -> bool: """Whether generators must be registered with a `torch.cuda.CUDAGraph` before capture. - PyTorch >= 2.14 (pytorch/pytorch#176753) lazily registers every generator whose Philox + PyTorch with pytorch/pytorch#176753 lazily registers every generator whose Philox state is consumed during capture, and `CUDAGraph.register_generator_state()` became a deprecated no-op that prints a warning on *every* call. Skip the explicit registration there: it does nothing, and with one call per layer, per graph and per generator it floods stderr (tens of thousands of lines per rank for dynamic inference with CUDA graphs). + + The version string cannot identify that change: 2.14.0a0 nightlies built before it landed + still require explicit registration. Builds older than 2.14.0a0 always require it; for + newer builds, a small capture probe decides, and its result is cached. """ - return not is_torch_min_version("2.14.0a0") + global _CUDAGRAPH_NEEDS_GENERATOR_REGISTRATION + if _CUDAGRAPH_NEEDS_GENERATOR_REGISTRATION is not None: + return _CUDAGRAPH_NEEDS_GENERATOR_REGISTRATION + if not is_torch_min_version("2.14.0a0"): + needs_registration = True + elif torch.cuda.is_current_stream_capturing(): + # The probe cannot capture while another capture is active; register to be safe + # and probe on a later call. + return True + else: + needs_registration = _probe_cudagraph_needs_generator_registration() + _CUDAGRAPH_NEEDS_GENERATOR_REGISTRATION = needs_registration + return needs_registration def prime_cuda_rng_states_for_graph_capture() -> None: diff --git a/megatron/training/checkpointing.py b/megatron/training/checkpointing.py index 498c3de6f3e..dd7e09f495c 100644 --- a/megatron/training/checkpointing.py +++ b/megatron/training/checkpointing.py @@ -3033,9 +3033,9 @@ def load_checkpoint( ) # Same as the torch_dist branch: optimizer load templates select checkpoint-era keys by # version. fsdp_dtensor state has no dtype-keyed FQNs today; keep both paths identical. - optim_sd_kwargs['metadata']['checkpoint_version'] = ( - state_dict.get('checkpoint_version') or 0 - ) + optim_sd_kwargs['metadata']['checkpoint_version'] = (state_dict or {}).get( + 'checkpoint_version' + ) or 0 # Megatron-FSDP materializes optimizer slots with a dummy zero-gradient # step while building a loading state dict. A normal full resume diff --git a/pyproject.toml b/pyproject.toml index 8bdd6d42def..bb881b7ed0b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -255,7 +255,7 @@ requires-dist = ["torch", "packaging", "ninja"] flash_mla = [ { git = "https://github.com/deepseek-ai/FlashMLA", rev = "nv_dev" }, ] -deep_gemm = { git = "https://github.com/deepseek-ai/DeepGEMM.git", rev = "714dd1a4a980f7937a74343d19a8eba4fe321480" } +deep_gemm = { git = "https://github.com/deepseek-ai/DeepGEMM.git", rev = "ea2b7805b97956268bed1adf94591f428ded6cc1" } transformer-engine = { git = "https://github.com/NVIDIA/TransformerEngine.git", rev = "6ea2a74a9e98c99e6d7b164a33775cc457520027" } nemo-run = { git = "https://github.com/NVIDIA-NeMo/Run.git", rev = "e3935393a290aed1822af52139b4b8ee270fed1f" } emerging_optimizers = { git = "https://github.com/NVIDIA-NeMo/Emerging-Optimizers.git", rev = "v0.3.0" } diff --git a/tests/functional_tests/shell_test_utils/_run_training.sh b/tests/functional_tests/shell_test_utils/_run_training.sh index e304eb7795c..adfdfe0e068 100644 --- a/tests/functional_tests/shell_test_utils/_run_training.sh +++ b/tests/functional_tests/shell_test_utils/_run_training.sh @@ -90,6 +90,9 @@ if [[ "$BEFORE_SCRIPT" != null ]]; then eval "$BEFORE_SCRIPT" fi +# Keep functional validation on FA2 while using the unpatched NGC 26.09 stack. +export NVTE_FLASH_ATTN_V4=0 + # Exit earlier to leave time for properly saving checkpoint if [[ "$IS_NEMO_TEST" == "true" ]]; then PARAMS=() @@ -187,6 +190,9 @@ fi # Extract training params PARAMS=("${PARAMS[@]}" "${TRAINING_PARAMS_ARRAY[@]}") +if [[ "$IS_NEMO_TEST" != "true" ]]; then + PARAMS+=("--flash-attention-version" "2") +fi # Set PYTHONPATH export PYTHONPATH="$(pwd):${PYTHONPATH:-}" diff --git a/tests/functional_tests/test_cases/gpt/gpt3_7b_tp1_pp4_memory_speed/golden_values_dev_dgx_h100.json b/tests/functional_tests/test_cases/gpt/gpt3_7b_tp1_pp4_memory_speed/golden_values_dev_dgx_h100.json index 680fdc3042f..10e10642211 100644 --- a/tests/functional_tests/test_cases/gpt/gpt3_7b_tp1_pp4_memory_speed/golden_values_dev_dgx_h100.json +++ b/tests/functional_tests/test_cases/gpt/gpt3_7b_tp1_pp4_memory_speed/golden_values_dev_dgx_h100.json @@ -3,38 +3,40 @@ "start_step": 1, "end_step": 25, "step_interval": 1, + "value_precision": "full", "values": { - "1": 12.58357, - "2": 12.58458, - "3": 12.58367, - "4": 12.57961, - "5": 12.57654, - "6": 12.57618, - "7": 12.55645, - "8": 12.52247, - "9": 12.4945, - "10": 12.4826, - "11": 12.31396, - "12": 12.27433, - "13": 12.20843, - "14": 12.2042, - "15": 11.79508, - "16": 11.77949, - "17": 11.73985, - "18": 11.71381, - "19": 11.5923, - "20": 11.47919, - "21": 11.24284, - "22": 11.35738, - "23": 11.25848, - "24": 11.14118, - "25": 10.97973 + "1": 12.583566665649414, + "2": 12.584580421447754, + "3": 12.58366584777832, + "4": 12.579607009887695, + "5": 12.576536178588867, + "6": 12.576178550720215, + "7": 12.556447982788086, + "8": 12.522472381591797, + "9": 12.49450397491455, + "10": 12.482599258422852, + "11": 12.313955307006836, + "12": 12.274328231811523, + "13": 12.208433151245117, + "14": 12.204201698303223, + "15": 11.795080184936523, + "16": 11.779489517211914, + "17": 11.739847183227539, + "18": 11.713809967041016, + "19": 11.592302322387695, + "20": 11.479185104370117, + "21": 11.24283504486084, + "22": 11.35738468170166, + "23": 11.258482933044434, + "24": 11.141175270080566, + "25": 10.979726791381836 } }, "num-zeros": { "start_step": 1, "end_step": 25, "step_interval": 1, + "value_precision": "full", "values": { "1": 521035744.0, "2": 521663296.0, @@ -67,6 +69,7 @@ "start_step": 1, "end_step": 25, "step_interval": 1, + "value_precision": "full", "values": { "1": 24540168192.0, "2": 24540168192.0, @@ -99,8 +102,9 @@ "start_step": 1, "end_step": 25, "step_interval": 1, + "value_precision": "full", "values": { - "1": 52730679296.0, + "1": 52729630720.0, "2": 60518293504.0, "3": 60518293504.0, "4": 60518293504.0, @@ -131,19 +135,20 @@ "start_step": 2, "end_step": 24, "step_interval": 2, + "value_precision": "full", "values": { - "2": 11.00377, - "4": 1.15004, - "6": 1.14483, - "8": 1.14518, - "10": 1.14392, - "12": 1.1461, - "14": 1.14611, - "16": 1.62175, - "18": 1.14125, - "20": 1.37115, - "22": 1.14417, - "24": 1.14249 + "2": 11.010119438171387, + "4": 1.0651472806930542, + "6": 1.0554732084274292, + "8": 1.0575685501098633, + "10": 1.0632253885269165, + "12": 1.0605608224868774, + "14": 1.0918465852737427, + "16": 1.0609685182571411, + "18": 1.059203028678894, + "20": 1.060057520866394, + "22": 1.0580178499221802, + "24": 1.0608563423156738 } } -} \ No newline at end of file +} diff --git a/tests/functional_tests/test_cases/gpt/gpt3_7b_tp4_pp1_memory_speed/golden_values_dev_dgx_h100.json b/tests/functional_tests/test_cases/gpt/gpt3_7b_tp4_pp1_memory_speed/golden_values_dev_dgx_h100.json index 7152bfbee78..41106e63f17 100644 --- a/tests/functional_tests/test_cases/gpt/gpt3_7b_tp4_pp1_memory_speed/golden_values_dev_dgx_h100.json +++ b/tests/functional_tests/test_cases/gpt/gpt3_7b_tp4_pp1_memory_speed/golden_values_dev_dgx_h100.json @@ -3,38 +3,40 @@ "start_step": 1, "end_step": 25, "step_interval": 1, + "value_precision": "full", "values": { - "1": 12.60113, - "2": 12.59889, - "3": 12.59574, - "4": 12.60007, - "5": 12.5969, - "6": 12.58515, - "7": 12.57631, - "8": 12.53821, - "9": 12.50832, - "10": 12.49349, - "11": 12.33422, - "12": 12.29172, - "13": 12.22869, - "14": 12.22632, - "15": 11.82432, - "16": 11.80748, - "17": 11.77392, - "18": 11.74895, - "19": 11.62163, - "20": 11.51956, - "21": 11.27212, - "22": 11.39455, - "23": 11.29568, - "24": 11.16717, - "25": 11.00276 + "1": 12.601129531860352, + "2": 12.598891258239746, + "3": 12.595743179321289, + "4": 12.600067138671875, + "5": 12.596904754638672, + "6": 12.585152626037598, + "7": 12.576313018798828, + "8": 12.538213729858398, + "9": 12.508321762084961, + "10": 12.493486404418945, + "11": 12.33421802520752, + "12": 12.291722297668457, + "13": 12.228689193725586, + "14": 12.226323127746582, + "15": 11.824323654174805, + "16": 11.80748176574707, + "17": 11.77392292022705, + "18": 11.74894905090332, + "19": 11.621625900268555, + "20": 11.519558906555176, + "21": 11.272119522094727, + "22": 11.394546508789062, + "23": 11.295675277709961, + "24": 11.167167663574219, + "25": 11.002762794494629 } }, "num-zeros": { "start_step": 1, "end_step": 25, "step_interval": 1, + "value_precision": "full", "values": { "1": 523037920.0, "2": 523667072.0, @@ -67,6 +69,7 @@ "start_step": 1, "end_step": 25, "step_interval": 1, + "value_precision": "full", "values": { "1": 20663519232.0, "2": 20663519232.0, @@ -99,6 +102,7 @@ "start_step": 1, "end_step": 25, "step_interval": 1, + "value_precision": "full", "values": { "1": 50289487872.0, "2": 57143791616.0, @@ -131,19 +135,20 @@ "start_step": 2, "end_step": 24, "step_interval": 2, + "value_precision": "full", "values": { - "2": 7.86264, - "4": 1.12286, - "6": 1.09857, - "8": 1.09881, - "10": 1.58549, - "12": 1.10179, - "14": 1.10226, - "16": 1.10141, - "18": 1.10151, - "20": 1.09965, - "22": 1.1005, - "24": 1.10161 + "2": 7.640352725982666, + "4": 1.030343770980835, + "6": 1.0198538303375244, + "8": 1.0180662870407104, + "10": 1.0207350254058838, + "12": 1.0220437049865723, + "14": 1.0201929807662964, + "16": 1.0199676752090454, + "18": 1.0216232538223267, + "20": 1.0194602012634277, + "22": 1.0196055173873901, + "24": 1.0220915079116821 } } } \ No newline at end of file diff --git a/tests/functional_tests/test_cases/gpt/gpt_inference_server_async_sched_tp2_pp2_dp2_qwen3_8b_sse/serve_sse.py b/tests/functional_tests/test_cases/gpt/gpt_inference_server_async_sched_tp2_pp2_dp2_qwen3_8b_sse/serve_sse.py index 17b896bda16..cb257b6f650 100644 --- a/tests/functional_tests/test_cases/gpt/gpt_inference_server_async_sched_tp2_pp2_dp2_qwen3_8b_sse/serve_sse.py +++ b/tests/functional_tests/test_cases/gpt/gpt_inference_server_async_sched_tp2_pp2_dp2_qwen3_8b_sse/serve_sse.py @@ -59,6 +59,8 @@ def _build_server_cmd( "1024", "--attention-backend", "flash", + "--flash-attention-version", + "2", "--micro-batch-size", "1", "--no-load-optim", @@ -156,6 +158,7 @@ def _cleaned_env() -> dict[str, str]: env["CUDA_DEVICE_MAX_CONNECTIONS"] = "1" env["NCCL_ALGO"] = "Ring" env["NVTE_ALLOW_NONDETERMINISTIC_ALGO"] = "0" + env["NVTE_FLASH_ATTN_V4"] = "0" env["CUBLAS_WORKSPACE_CONFIG"] = ":4096:8" return env diff --git a/tests/functional_tests/test_cases/gpt/gpt_inference_server_smoke_tp1_pp1_dp8_583m/serve_smoke.py b/tests/functional_tests/test_cases/gpt/gpt_inference_server_smoke_tp1_pp1_dp8_583m/serve_smoke.py index cf2647d4f4c..fa3ff9ac27e 100644 --- a/tests/functional_tests/test_cases/gpt/gpt_inference_server_smoke_tp1_pp1_dp8_583m/serve_smoke.py +++ b/tests/functional_tests/test_cases/gpt/gpt_inference_server_smoke_tp1_pp1_dp8_583m/serve_smoke.py @@ -65,6 +65,8 @@ def build_server_cmd( "4096", "--attention-backend", "flash", + "--flash-attention-version", + "2", "--use-checkpoint-args", "--micro-batch-size", "1", @@ -129,6 +131,7 @@ def cleaned_env() -> dict: env["CUDA_DEVICE_MAX_CONNECTIONS"] = "1" env["NCCL_ALGO"] = "Ring" env["NVTE_ALLOW_NONDETERMINISTIC_ALGO"] = "0" + env["NVTE_FLASH_ATTN_V4"] = "0" env["CUBLAS_WORKSPACE_CONFIG"] = ":4096:8" return env diff --git a/tests/functional_tests/test_cases/mimo/mimo_hetero_gtp_checkpoint_round_trip/test_checkpoint_round_trip.py b/tests/functional_tests/test_cases/mimo/mimo_hetero_gtp_checkpoint_round_trip/test_checkpoint_round_trip.py index b615f23c1e9..d0e7a9c937d 100644 --- a/tests/functional_tests/test_cases/mimo/mimo_hetero_gtp_checkpoint_round_trip/test_checkpoint_round_trip.py +++ b/tests/functional_tests/test_cases/mimo/mimo_hetero_gtp_checkpoint_round_trip/test_checkpoint_round_trip.py @@ -60,6 +60,7 @@ def _run_launcher( "MIMO_CHECKPOINT_PADDING_MANIFEST_DIR": str(scratch / "padding-manifest"), "MIMO_CHECKPOINT_TEST_PRETRAIN": "1", "MIMO_PRETRAIN_MODULE": _MODULE, + "NVTE_FLASH_ATTN_V4": "0", "TRAIN_ITERS": "2", "TORCHRUN_LOG_DIR": str(scratch / f"torchrun-{name}"), } @@ -75,6 +76,8 @@ def _run_launcher( command = [ "bash", str(_LAUNCHER), + "--flash-attention-version", + "2", "--num-experts", "8", "--no-save-tokenizer-assets", diff --git a/tests/functional_tests/test_cases/moe/deepseek_proxy_mfsdp_v1_ep2_1node/golden_values_dev_dgx_gb200.json b/tests/functional_tests/test_cases/moe/deepseek_proxy_mfsdp_v1_ep2_1node/golden_values_dev_dgx_gb200.json index 3e38bcd20ca..a9d9eb96b05 100644 --- a/tests/functional_tests/test_cases/moe/deepseek_proxy_mfsdp_v1_ep2_1node/golden_values_dev_dgx_gb200.json +++ b/tests/functional_tests/test_cases/moe/deepseek_proxy_mfsdp_v1_ep2_1node/golden_values_dev_dgx_gb200.json @@ -7,54 +7,54 @@ "values": { "1": 10.901354789733887, "2": 10.93243408203125, - "3": 10.647600173950195, - "4": 10.41220474243164, - "5": 10.400508880615234, - "6": 10.36600112915039, - "7": 10.113565444946289, - "8": 9.901735305786133, - "9": 9.964088439941406, - "10": 9.513167381286621, - "11": 10.162917137145996, - "12": 9.86227035522461, - "13": 9.869820594787598, - "14": 9.900150299072266, - "15": 9.506497383117676, - "16": 9.45169448852539, - "17": 9.260071754455566, - "18": 9.302938461303711, - "19": 9.209890365600586, - "20": 8.972551345825195, - "21": 9.005820274353027, - "22": 8.60628604888916, - "23": 9.094013214111328, - "24": 8.685464859008789, - "25": 8.506412506103516, - "26": 8.737756729125977, - "27": 8.826465606689453, - "28": 8.95547866821289, - "29": 8.943782806396484, - "30": 8.493976593017578, - "31": 7.958925724029541, - "32": 8.67808723449707, - "33": 8.746942520141602, - "34": 8.26606273651123, - "35": 8.356012344360352, - "36": 8.2703857421875, - "37": 8.507359504699707, - "38": 8.267014503479004, - "39": 8.632810592651367, - "40": 8.225650787353516, - "41": 8.272510528564453, - "42": 8.42404556274414, - "43": 8.005533218383789, - "44": 8.114350318908691, - "45": 7.996010780334473, - "46": 8.082942962646484, - "47": 8.384140968322754, - "48": 8.09496784210205, - "49": 7.730409622192383, - "50": 8.192166328430176 + "3": 10.332321166992188, + "4": 9.86688232421875, + "5": 9.960663795471191, + "6": 9.902548789978027, + "7": 9.523374557495117, + "8": 9.409852981567383, + "9": 9.356557846069336, + "10": 9.023311614990234, + "11": 9.545365333557129, + "12": 9.33155345916748, + "13": 9.331480026245117, + "14": 9.339519500732422, + "15": 9.000117301940918, + "16": 8.94300651550293, + "17": 8.769431114196777, + "18": 8.849810600280762, + "19": 8.797523498535156, + "20": 8.579107284545898, + "21": 8.634420394897461, + "22": 8.255470275878906, + "23": 8.710733413696289, + "24": 8.332413673400879, + "25": 8.17232608795166, + "26": 8.402215957641602, + "27": 8.497511863708496, + "28": 8.615479469299316, + "29": 8.619725227355957, + "30": 8.147198677062988, + "31": 7.6353278160095215, + "32": 8.37589168548584, + "33": 8.46032428741455, + "34": 7.965855598449707, + "35": 8.054203987121582, + "36": 7.983587265014648, + "37": 8.195871353149414, + "38": 7.951972484588623, + "39": 8.317138671875, + "40": 7.931028366088867, + "41": 7.957067012786865, + "42": 8.140800476074219, + "43": 7.719943046569824, + "44": 7.831162452697754, + "45": 7.713001728057861, + "46": 7.804263114929199, + "47": 8.07406997680664, + "48": 7.802701950073242, + "49": 7.466888427734375, + "50": 7.901459217071533 } }, "num-zeros": { diff --git a/tests/functional_tests/test_cases/moe/deepseek_proxy_mfsdp_v1_ep2_ep_overlap/golden_values_dev_dgx_h100.json b/tests/functional_tests/test_cases/moe/deepseek_proxy_mfsdp_v1_ep2_ep_overlap/golden_values_dev_dgx_h100.json index 0bdaa0b59f8..770accf9a18 100644 --- a/tests/functional_tests/test_cases/moe/deepseek_proxy_mfsdp_v1_ep2_ep_overlap/golden_values_dev_dgx_h100.json +++ b/tests/functional_tests/test_cases/moe/deepseek_proxy_mfsdp_v1_ep2_ep_overlap/golden_values_dev_dgx_h100.json @@ -7,54 +7,54 @@ "values": { "1": 10.910256385803223, "2": 10.938549041748047, - "3": 10.317316055297852, - "4": 9.849357604980469, - "5": 9.950676918029785, - "6": 9.872228622436523, - "7": 9.51567268371582, - "8": 9.397125244140625, - "9": 9.346898078918457, - "10": 9.032954216003418, - "11": 9.533287048339844, - "12": 9.320558547973633, - "13": 9.301362037658691, - "14": 9.329330444335938, - "15": 8.9800443649292, - "16": 8.947829246520996, - "17": 8.755012512207031, - "18": 8.841842651367188, - "19": 8.776716232299805, - "20": 8.554393768310547, - "21": 8.633527755737305, - "22": 8.221697807312012, - "23": 8.692643165588379, - "24": 8.320140838623047, - "25": 8.160679817199707, - "26": 8.387343406677246, - "27": 8.506802558898926, - "28": 8.615043640136719, - "29": 8.58391284942627, - "30": 8.150809288024902, - "31": 7.642322540283203, - "32": 8.36207389831543, - "33": 8.46556568145752, - "34": 7.951313018798828, - "35": 8.0454683303833, - "36": 7.980090618133545, - "37": 8.1902437210083, - "38": 7.954071044921875, - "39": 8.288957595825195, - "40": 7.936356067657471, - "41": 7.957198619842529, - "42": 8.141084671020508, - "43": 7.7138142585754395, - "44": 7.846534252166748, - "45": 7.697491645812988, - "46": 7.778336524963379, - "47": 8.073257446289062, - "48": 7.783638954162598, - "49": 7.468996047973633, - "50": 7.870915412902832 + "3": 10.317255020141602, + "4": 9.849323272705078, + "5": 9.95071029663086, + "6": 9.87224292755127, + "7": 9.5156831741333, + "8": 9.397089004516602, + "9": 9.346919059753418, + "10": 9.033021926879883, + "11": 9.533082008361816, + "12": 9.320527076721191, + "13": 9.30143928527832, + "14": 9.329547882080078, + "15": 8.97988224029541, + "16": 8.948017120361328, + "17": 8.75504207611084, + "18": 8.841780662536621, + "19": 8.776607513427734, + "20": 8.554301261901855, + "21": 8.633434295654297, + "22": 8.221407890319824, + "23": 8.692683219909668, + "24": 8.319025039672852, + "25": 8.159835815429688, + "26": 8.386969566345215, + "27": 8.506211280822754, + "28": 8.613567352294922, + "29": 8.583308219909668, + "30": 8.150187492370605, + "31": 7.642271041870117, + "32": 8.361566543579102, + "33": 8.464994430541992, + "34": 7.950206756591797, + "35": 8.044382095336914, + "36": 7.979213714599609, + "37": 8.189566612243652, + "38": 7.953609466552734, + "39": 8.28823471069336, + "40": 7.935532093048096, + "41": 7.9563188552856445, + "42": 8.140161514282227, + "43": 7.712148666381836, + "44": 7.845901012420654, + "45": 7.696685791015625, + "46": 7.777511119842529, + "47": 8.072707176208496, + "48": 7.7830891609191895, + "49": 7.468073844909668, + "50": 7.87031364440918 } }, "num-zeros": { @@ -288,4 +288,4 @@ "50": 0.644688606262207 } } -} \ No newline at end of file +} diff --git a/tests/functional_tests/test_cases/moe/deepseek_proxy_mfsdp_v2_ep2/golden_values_dev_dgx_h100.json b/tests/functional_tests/test_cases/moe/deepseek_proxy_mfsdp_v2_ep2/golden_values_dev_dgx_h100.json index b1c412c98f3..e8fcf75e777 100644 --- a/tests/functional_tests/test_cases/moe/deepseek_proxy_mfsdp_v2_ep2/golden_values_dev_dgx_h100.json +++ b/tests/functional_tests/test_cases/moe/deepseek_proxy_mfsdp_v2_ep2/golden_values_dev_dgx_h100.json @@ -7,54 +7,54 @@ "values": { "1": 10.994527816772461, "2": 10.974343299865723, - "3": 10.229790687561035, - "4": 9.747961044311523, - "5": 9.598786354064941, - "6": 9.336235046386719, - "7": 9.2498779296875, - "8": 9.119386672973633, - "9": 9.007975578308105, - "10": 8.86236572265625, - "11": 8.900297164916992, - "12": 8.7371826171875, - "13": 8.738578796386719, - "14": 8.664358139038086, - "15": 8.4231595993042, - "16": 8.587013244628906, - "17": 8.246013641357422, - "18": 8.278899192810059, - "19": 8.149882316589355, - "20": 8.167485237121582, - "21": 8.019439697265625, - "22": 8.026062965393066, - "23": 8.044377326965332, - "24": 8.036299705505371, - "25": 8.079054832458496, - "26": 7.781270980834961, - "27": 7.864381790161133, - "28": 7.93831729888916, - "29": 7.985459327697754, - "30": 7.713861465454102, - "31": 7.9197998046875, - "32": 7.8319597244262695, - "33": 7.745386123657227, - "34": 7.7463884353637695, - "35": 7.4665327072143555, - "36": 7.852348327636719, - "37": 7.647491455078125, - "38": 7.449972629547119, - "39": 7.631718158721924, - "40": 7.435605049133301, - "41": 7.625354766845703, - "42": 7.581092357635498, - "43": 7.62886905670166, - "44": 7.463525772094727, - "45": 7.458897113800049, - "46": 7.5108232498168945, - "47": 7.585690975189209, - "48": 7.5075578689575195, - "49": 7.465226173400879, - "50": 7.436612606048584 + "3": 10.22977066040039, + "4": 9.747969627380371, + "5": 9.598889350891113, + "6": 9.336261749267578, + "7": 9.249931335449219, + "8": 9.119340896606445, + "9": 9.008247375488281, + "10": 8.862383842468262, + "11": 8.900238990783691, + "12": 8.737178802490234, + "13": 8.738592147827148, + "14": 8.664324760437012, + "15": 8.423168182373047, + "16": 8.587056159973145, + "17": 8.246075630187988, + "18": 8.278972625732422, + "19": 8.149890899658203, + "20": 8.167665481567383, + "21": 8.019505500793457, + "22": 8.026301383972168, + "23": 8.044529914855957, + "24": 8.03643798828125, + "25": 8.079147338867188, + "26": 7.781403064727783, + "27": 7.864442348480225, + "28": 7.938540458679199, + "29": 7.98561954498291, + "30": 7.713891983032227, + "31": 7.9198503494262695, + "32": 7.832034111022949, + "33": 7.745593547821045, + "34": 7.746420383453369, + "35": 7.466573238372803, + "36": 7.852507591247559, + "37": 7.64756965637207, + "38": 7.44999361038208, + "39": 7.631738662719727, + "40": 7.435657978057861, + "41": 7.625308513641357, + "42": 7.581051826477051, + "43": 7.628737926483154, + "44": 7.463506698608398, + "45": 7.458869934082031, + "46": 7.510734558105469, + "47": 7.585694313049316, + "48": 7.507479190826416, + "49": 7.465153217315674, + "50": 7.436614990234375 } }, "num-zeros": { diff --git a/tests/functional_tests/test_cases/moe/deepseek_proxy_mfsdp_v2_ep2_ep_overlap/golden_values_dev_dgx_h100.json b/tests/functional_tests/test_cases/moe/deepseek_proxy_mfsdp_v2_ep2_ep_overlap/golden_values_dev_dgx_h100.json index a8bf0cde9cf..412330373b1 100644 --- a/tests/functional_tests/test_cases/moe/deepseek_proxy_mfsdp_v2_ep2_ep_overlap/golden_values_dev_dgx_h100.json +++ b/tests/functional_tests/test_cases/moe/deepseek_proxy_mfsdp_v2_ep2_ep_overlap/golden_values_dev_dgx_h100.json @@ -7,54 +7,54 @@ "values": { "1": 10.994527816772461, "2": 10.974343299865723, - "3": 10.229790687561035, - "4": 9.747961044311523, - "5": 9.598786354064941, - "6": 9.336235046386719, - "7": 9.2498779296875, - "8": 9.119386672973633, - "9": 9.007975578308105, - "10": 8.86236572265625, - "11": 8.900297164916992, - "12": 8.7371826171875, - "13": 8.738578796386719, - "14": 8.664358139038086, - "15": 8.4231595993042, - "16": 8.587013244628906, - "17": 8.246013641357422, - "18": 8.278899192810059, - "19": 8.149882316589355, - "20": 8.167485237121582, - "21": 8.019439697265625, - "22": 8.026062965393066, - "23": 8.044377326965332, - "24": 8.036299705505371, - "25": 8.079054832458496, - "26": 7.781270980834961, - "27": 7.864381790161133, - "28": 7.93831729888916, - "29": 7.985459327697754, - "30": 7.713861465454102, - "31": 7.9197998046875, - "32": 7.8319597244262695, - "33": 7.745386123657227, - "34": 7.7463884353637695, - "35": 7.4665327072143555, - "36": 7.852348327636719, - "37": 7.647491455078125, - "38": 7.449972629547119, - "39": 7.631718158721924, - "40": 7.435605049133301, - "41": 7.625354766845703, - "42": 7.581092357635498, - "43": 7.62886905670166, - "44": 7.463525772094727, - "45": 7.458897113800049, - "46": 7.5108232498168945, - "47": 7.585690975189209, - "48": 7.5075578689575195, - "49": 7.465226173400879, - "50": 7.436612606048584 + "3": 10.22977066040039, + "4": 9.747969627380371, + "5": 9.598889350891113, + "6": 9.336261749267578, + "7": 9.249931335449219, + "8": 9.119340896606445, + "9": 9.008247375488281, + "10": 8.862383842468262, + "11": 8.900238990783691, + "12": 8.737178802490234, + "13": 8.738592147827148, + "14": 8.664324760437012, + "15": 8.423168182373047, + "16": 8.587056159973145, + "17": 8.246075630187988, + "18": 8.278972625732422, + "19": 8.149890899658203, + "20": 8.167665481567383, + "21": 8.019505500793457, + "22": 8.026301383972168, + "23": 8.044529914855957, + "24": 8.03643798828125, + "25": 8.079147338867188, + "26": 7.781403064727783, + "27": 7.864442348480225, + "28": 7.938540458679199, + "29": 7.98561954498291, + "30": 7.713891983032227, + "31": 7.9198503494262695, + "32": 7.832034111022949, + "33": 7.745593547821045, + "34": 7.746420383453369, + "35": 7.466573238372803, + "36": 7.852507591247559, + "37": 7.64756965637207, + "38": 7.44999361038208, + "39": 7.631738662719727, + "40": 7.435657978057861, + "41": 7.625308513641357, + "42": 7.581051826477051, + "43": 7.628737926483154, + "44": 7.463506698608398, + "45": 7.458869934082031, + "46": 7.510734558105469, + "47": 7.585694313049316, + "48": 7.507479190826416, + "49": 7.465153217315674, + "50": 7.436614990234375 } }, "num-zeros": { diff --git a/tests/functional_tests/test_cases/moe/deepseek_proxy_mfsdp_v2_ep2_hfsdp_mb2/golden_values_dev_dgx_h100.json b/tests/functional_tests/test_cases/moe/deepseek_proxy_mfsdp_v2_ep2_hfsdp_mb2/golden_values_dev_dgx_h100.json index 2caffdd3c32..e16ad9853f7 100644 --- a/tests/functional_tests/test_cases/moe/deepseek_proxy_mfsdp_v2_ep2_hfsdp_mb2/golden_values_dev_dgx_h100.json +++ b/tests/functional_tests/test_cases/moe/deepseek_proxy_mfsdp_v2_ep2_hfsdp_mb2/golden_values_dev_dgx_h100.json @@ -7,54 +7,54 @@ "values": { "1": 10.984330177307129, "2": 10.991540908813477, - "3": 10.205118179321289, - "4": 9.649384498596191, - "5": 9.466728210449219, - "6": 9.345254898071289, - "7": 9.306957244873047, - "8": 9.136713027954102, - "9": 8.925423622131348, - "10": 8.84184455871582, - "11": 8.705001831054688, - "12": 8.75346565246582, - "13": 8.65507698059082, - "14": 8.623779296875, - "15": 8.572185516357422, - "16": 8.601885795593262, - "17": 8.434591293334961, - "18": 8.319332122802734, - "19": 8.195926666259766, - "20": 8.15245246887207, - "21": 8.176870346069336, - "22": 8.094462394714355, - "23": 7.988713264465332, - "24": 8.012734413146973, - "25": 7.896326065063477, - "26": 7.751435279846191, - "27": 7.7925543785095215, - "28": 7.803656101226807, - "29": 7.761228084564209, - "30": 7.620172023773193, - "31": 7.749785423278809, - "32": 7.624546051025391, - "33": 7.581184387207031, - "34": 7.525897979736328, - "35": 7.564944267272949, - "36": 7.710661888122559, - "37": 7.534018516540527, - "38": 7.476527214050293, - "39": 7.633326053619385, - "40": 7.560534954071045, - "41": 7.489247798919678, - "42": 7.39389705657959, - "43": 7.396523952484131, - "44": 7.409448146820068, - "45": 7.321393966674805, - "46": 7.400388240814209, - "47": 7.385503768920898, - "48": 7.359971046447754, - "49": 7.372410774230957, - "50": 7.431112766265869 + "3": 10.205131530761719, + "4": 9.64937973022461, + "5": 9.466668128967285, + "6": 9.345155715942383, + "7": 9.30705738067627, + "8": 9.136767387390137, + "9": 8.925482749938965, + "10": 8.841987609863281, + "11": 8.705121994018555, + "12": 8.753618240356445, + "13": 8.65505313873291, + "14": 8.623869895935059, + "15": 8.572352409362793, + "16": 8.601971626281738, + "17": 8.4349365234375, + "18": 8.31936264038086, + "19": 8.196059226989746, + "20": 8.152478218078613, + "21": 8.176542282104492, + "22": 8.094558715820312, + "23": 7.988754749298096, + "24": 8.01286506652832, + "25": 7.896248817443848, + "26": 7.751811981201172, + "27": 7.7930169105529785, + "28": 7.80364990234375, + "29": 7.760356903076172, + "30": 7.618964672088623, + "31": 7.750360012054443, + "32": 7.625613689422607, + "33": 7.581803798675537, + "34": 7.526548385620117, + "35": 7.565817356109619, + "36": 7.712406158447266, + "37": 7.535028457641602, + "38": 7.476831436157227, + "39": 7.6341047286987305, + "40": 7.561294078826904, + "41": 7.490050792694092, + "42": 7.39475154876709, + "43": 7.397394180297852, + "44": 7.410472869873047, + "45": 7.3221025466918945, + "46": 7.401252269744873, + "47": 7.386091232299805, + "48": 7.3610148429870605, + "49": 7.373475551605225, + "50": 7.432130813598633 } }, "num-zeros": { diff --git a/tests/functional_tests/test_cases/moe/deepseek_proxy_mfsdp_v2_ep2_hsdp_mb2/golden_values_dev_dgx_h100.json b/tests/functional_tests/test_cases/moe/deepseek_proxy_mfsdp_v2_ep2_hsdp_mb2/golden_values_dev_dgx_h100.json index 8a2cced4596..c7198f75c21 100644 --- a/tests/functional_tests/test_cases/moe/deepseek_proxy_mfsdp_v2_ep2_hsdp_mb2/golden_values_dev_dgx_h100.json +++ b/tests/functional_tests/test_cases/moe/deepseek_proxy_mfsdp_v2_ep2_hsdp_mb2/golden_values_dev_dgx_h100.json @@ -7,54 +7,54 @@ "values": { "1": 10.984330177307129, "2": 10.991540908813477, - "3": 10.205135345458984, - "4": 9.649385452270508, - "5": 9.466609954833984, - "6": 9.345148086547852, - "7": 9.307164192199707, - "8": 9.13676643371582, - "9": 8.925497055053711, - "10": 8.842087745666504, - "11": 8.705370903015137, - "12": 8.75433349609375, - "13": 8.655134201049805, - "14": 8.624410629272461, - "15": 8.573217391967773, - "16": 8.602340698242188, - "17": 8.43627643585205, - "18": 8.319859504699707, - "19": 8.19666862487793, - "20": 8.153854370117188, - "21": 8.177652359008789, - "22": 8.096282958984375, - "23": 7.989590167999268, - "24": 8.014253616333008, - "25": 7.8982086181640625, - "26": 7.753742218017578, - "27": 7.794215679168701, - "28": 7.806078910827637, - "29": 7.764865875244141, - "30": 7.623738765716553, - "31": 7.751580238342285, - "32": 7.625911235809326, - "33": 7.582916259765625, - "34": 7.528222560882568, - "35": 7.566494941711426, - "36": 7.711782455444336, - "37": 7.535035610198975, - "38": 7.478643417358398, - "39": 7.6343584060668945, - "40": 7.561514377593994, - "41": 7.49015998840332, - "42": 7.394611835479736, - "43": 7.397531032562256, - "44": 7.410419464111328, - "45": 7.322272777557373, - "46": 7.401014804840088, - "47": 7.386123180389404, - "48": 7.36057186126709, - "49": 7.37290096282959, - "50": 7.431656837463379 + "3": 10.205157279968262, + "4": 9.649395942687988, + "5": 9.46661376953125, + "6": 9.345149993896484, + "7": 9.307059288024902, + "8": 9.136697769165039, + "9": 8.925455093383789, + "10": 8.841968536376953, + "11": 8.705031394958496, + "12": 8.753540992736816, + "13": 8.655043601989746, + "14": 8.623872756958008, + "15": 8.572341918945312, + "16": 8.602005004882812, + "17": 8.434781074523926, + "18": 8.319397926330566, + "19": 8.196041107177734, + "20": 8.152498245239258, + "21": 8.176743507385254, + "22": 8.094529151916504, + "23": 7.988747596740723, + "24": 8.012996673583984, + "25": 7.896562576293945, + "26": 7.75199031829834, + "27": 7.7929158210754395, + "28": 7.803884029388428, + "29": 7.760400772094727, + "30": 7.618945121765137, + "31": 7.750338554382324, + "32": 7.62603235244751, + "33": 7.582083702087402, + "34": 7.526541709899902, + "35": 7.565951347351074, + "36": 7.712773323059082, + "37": 7.535645484924316, + "38": 7.478216171264648, + "39": 7.634349346160889, + "40": 7.561685085296631, + "41": 7.4901123046875, + "42": 7.394813537597656, + "43": 7.397581577301025, + "44": 7.410630702972412, + "45": 7.322366237640381, + "46": 7.401366710662842, + "47": 7.3863525390625, + "48": 7.36128044128418, + "49": 7.3736491203308105, + "50": 7.432430267333984 } }, "num-zeros": { diff --git a/tests/functional_tests/test_cases/moe/deepseek_proxy_mfsdp_v2_ep2_mb2/golden_values_dev_dgx_h100.json b/tests/functional_tests/test_cases/moe/deepseek_proxy_mfsdp_v2_ep2_mb2/golden_values_dev_dgx_h100.json index 65d05f2ea8c..3c8361b999c 100644 --- a/tests/functional_tests/test_cases/moe/deepseek_proxy_mfsdp_v2_ep2_mb2/golden_values_dev_dgx_h100.json +++ b/tests/functional_tests/test_cases/moe/deepseek_proxy_mfsdp_v2_ep2_mb2/golden_values_dev_dgx_h100.json @@ -7,54 +7,54 @@ "values": { "1": 10.984330177307129, "2": 10.991540908813477, - "3": 10.20511531829834, - "4": 9.649360656738281, - "5": 9.466591835021973, - "6": 9.345145225524902, - "7": 9.30707836151123, - "8": 9.13673210144043, - "9": 8.925445556640625, - "10": 8.842019081115723, - "11": 8.705059051513672, - "12": 8.753610610961914, - "13": 8.655056953430176, - "14": 8.62393569946289, - "15": 8.572381973266602, - "16": 8.602009773254395, - "17": 8.43491268157959, - "18": 8.319398880004883, - "19": 8.19606876373291, - "20": 8.152507781982422, - "21": 8.176657676696777, - "22": 8.094585418701172, - "23": 7.988735198974609, - "24": 8.013263702392578, - "25": 7.896378993988037, - "26": 7.751855373382568, - "27": 7.792813777923584, - "28": 7.803920745849609, - "29": 7.760638236999512, - "30": 7.619563579559326, - "31": 7.75018835067749, - "32": 7.625418663024902, - "33": 7.582130432128906, - "34": 7.526845455169678, - "35": 7.565555095672607, - "36": 7.711994171142578, - "37": 7.534870147705078, - "38": 7.477423191070557, - "39": 7.63400936126709, - "40": 7.561377048492432, - "41": 7.489800453186035, - "42": 7.394506931304932, - "43": 7.397129058837891, - "44": 7.4101786613464355, - "45": 7.321859836578369, - "46": 7.401007652282715, - "47": 7.386066913604736, - "48": 7.360743999481201, - "49": 7.373129367828369, - "50": 7.431926727294922 + "3": 10.205066680908203, + "4": 9.649356842041016, + "5": 9.4666748046875, + "6": 9.345120429992676, + "7": 9.307114601135254, + "8": 9.136754035949707, + "9": 8.925481796264648, + "10": 8.841949462890625, + "11": 8.70529556274414, + "12": 8.754190444946289, + "13": 8.655147552490234, + "14": 8.62423324584961, + "15": 8.573019981384277, + "16": 8.60224723815918, + "17": 8.435911178588867, + "18": 8.319633483886719, + "19": 8.196418762207031, + "20": 8.1533784866333, + "21": 8.177040100097656, + "22": 8.095715522766113, + "23": 7.989101886749268, + "24": 8.01396656036377, + "25": 7.897273063659668, + "26": 7.753263473510742, + "27": 7.793600559234619, + "28": 7.805505275726318, + "29": 7.762482643127441, + "30": 7.621406078338623, + "31": 7.751012802124023, + "32": 7.625071048736572, + "33": 7.581407070159912, + "34": 7.526612281799316, + "35": 7.565842151641846, + "36": 7.71071195602417, + "37": 7.534422874450684, + "38": 7.47728967666626, + "39": 7.633402347564697, + "40": 7.560451507568359, + "41": 7.489355087280273, + "42": 7.393846035003662, + "43": 7.396568775177002, + "44": 7.409462928771973, + "45": 7.32165002822876, + "46": 7.40026330947876, + "47": 7.3854899406433105, + "48": 7.359823703765869, + "49": 7.37227201461792, + "50": 7.431122303009033 } }, "num-zeros": { diff --git a/tests/functional_tests/test_cases/moe/gpt3_moe_mcore_te_ep8_resume_torch_dist_dist_muon/golden_values_dev_dgx_gb200.json b/tests/functional_tests/test_cases/moe/gpt3_moe_mcore_te_ep8_resume_torch_dist_dist_muon/golden_values_dev_dgx_gb200.json index 227f9021394..6f931380e74 100644 --- a/tests/functional_tests/test_cases/moe/gpt3_moe_mcore_te_ep8_resume_torch_dist_dist_muon/golden_values_dev_dgx_gb200.json +++ b/tests/functional_tests/test_cases/moe/gpt3_moe_mcore_te_ep8_resume_torch_dist_dist_muon/golden_values_dev_dgx_gb200.json @@ -3,534 +3,539 @@ "start_step": 1, "end_step": 100, "step_interval": 1, + "value_precision": "full", "values": { - "1": 10.94003, - "2": 10.91043, - "3": 10.92839, - "4": 10.92119, - "5": 10.91516, - "6": 10.92479, - "7": 10.92123, - "8": 10.92326, - "9": 10.93298, - "10": 10.92235, - "11": 10.92041, - "12": 10.93782, - "13": 10.91994, - "14": 10.90821, - "15": 10.91912, - "16": 10.91641, - "17": 10.93505, - "18": 10.9246, - "19": 10.92842, - "20": 10.90096, - "21": 10.92785, - "22": 10.92302, - "23": 10.93391, - "24": 10.90505, - "25": 10.91958, - "26": 10.91426, - "27": 10.93097, - "28": 10.9162, - "29": 10.92589, - "30": 10.91042, - "31": 10.90618, - "32": 10.91541, - "33": 10.91528, - "34": 10.90429, - "35": 10.92802, - "36": 10.90861, - "37": 10.90978, - "38": 10.91578, - "39": 10.91107, - "40": 10.9061, - "41": 10.91108, - "42": 10.91895, - "43": 10.90647, - "44": 10.91641, - "45": 10.90733, - "46": 10.90712, - "47": 10.90561, - "48": 10.89333, - "49": 10.90376, - "50": 10.89536, - "51": 10.89467, - "52": 10.89682, - "53": 10.8995, - "54": 10.88629, - "55": 10.90508, - "56": 10.88803, - "57": 10.89055, - "58": 10.8799, - "59": 10.87662, - "60": 10.86522, - "61": 10.86708, - "62": 10.86791, - "63": 10.87113, - "64": 10.86271, - "65": 10.87066, - "66": 10.866, - "67": 10.85634, - "68": 10.85101, - "69": 10.85365, - "70": 10.85358, - "71": 10.84973, - "72": 10.84574, - "73": 10.84071, - "74": 10.84003, - "75": 10.83937, - "76": 10.83176, - "77": 10.82561, - "78": 10.8128, - "79": 10.83036, - "80": 10.8317, - "81": 10.81377, - "82": 10.8117, - "83": 10.80631, - "84": 10.77977, - "85": 10.78242, - "86": 10.79641, - "87": 10.79633, - "88": 10.80278, - "89": 10.78258, - "90": 10.77858, - "91": 10.78317, - "92": 10.77924, - "93": 10.76094, - "94": 10.76556, - "95": 10.76989, - "96": 10.73857, - "97": 10.73186, - "98": 10.7544, - "99": 10.76734, - "100": 10.73755 + "1": 10.940028190612793, + "2": 10.910425186157227, + "3": 10.928388595581055, + "4": 10.921188354492188, + "5": 10.915172576904297, + "6": 10.924402236938477, + "7": 10.921537399291992, + "8": 10.92303466796875, + "9": 10.933676719665527, + "10": 10.922171592712402, + "11": 10.920392036437988, + "12": 10.937664031982422, + "13": 10.91990852355957, + "14": 10.908479690551758, + "15": 10.919020652770996, + "16": 10.916176795959473, + "17": 10.935304641723633, + "18": 10.924395561218262, + "19": 10.928459167480469, + "20": 10.900659561157227, + "21": 10.928400993347168, + "22": 10.92259407043457, + "23": 10.93402099609375, + "24": 10.905346870422363, + "25": 10.919418334960938, + "26": 10.914515495300293, + "27": 10.930940628051758, + "28": 10.916147232055664, + "29": 10.92598819732666, + "30": 10.910531044006348, + "31": 10.905406951904297, + "32": 10.91554069519043, + "33": 10.91540241241455, + "34": 10.904865264892578, + "35": 10.927868843078613, + "36": 10.908853530883789, + "37": 10.909917831420898, + "38": 10.915998458862305, + "39": 10.911012649536133, + "40": 10.906172752380371, + "41": 10.911545753479004, + "42": 10.91899585723877, + "43": 10.906837463378906, + "44": 10.91661548614502, + "45": 10.907024383544922, + "46": 10.906749725341797, + "47": 10.905570030212402, + "48": 10.893385887145996, + "49": 10.904132843017578, + "50": 10.895672798156738, + "51": 10.895065307617188, + "52": 10.897016525268555, + "53": 10.899434089660645, + "54": 10.886595726013184, + "55": 10.905381202697754, + "56": 10.888248443603516, + "57": 10.89013671875, + "58": 10.879858016967773, + "59": 10.876787185668945, + "60": 10.86523151397705, + "61": 10.868112564086914, + "62": 10.867758750915527, + "63": 10.870795249938965, + "64": 10.862629890441895, + "65": 10.870889663696289, + "66": 10.866762161254883, + "67": 10.85664176940918, + "68": 10.850561141967773, + "69": 10.853750228881836, + "70": 10.853352546691895, + "71": 10.849613189697266, + "72": 10.845426559448242, + "73": 10.840685844421387, + "74": 10.84040641784668, + "75": 10.839654922485352, + "76": 10.83171558380127, + "77": 10.825037002563477, + "78": 10.81246280670166, + "79": 10.830265998840332, + "80": 10.83193588256836, + "81": 10.814144134521484, + "82": 10.811941146850586, + "83": 10.806283950805664, + "84": 10.778970718383789, + "85": 10.782588005065918, + "86": 10.796222686767578, + "87": 10.796788215637207, + "88": 10.802943229675293, + "89": 10.782289505004883, + "90": 10.778926849365234, + "91": 10.78364372253418, + "92": 10.779129028320312, + "93": 10.761343002319336, + "94": 10.764945030212402, + "95": 10.7705717086792, + "96": 10.738081932067871, + "97": 10.732248306274414, + "98": 10.754161834716797, + "99": 10.76768684387207, + "100": 10.737388610839844 } }, "num-zeros": { "start_step": 1, "end_step": 100, "step_interval": 1, + "value_precision": "full", "values": { "1": 1305.0, "2": 1165.0, "3": 1311.0, "4": 1422.0, - "5": 1268.0, - "6": 1214.0, - "7": 1516.0, - "8": 1345.0, - "9": 1328.0, - "10": 1302.0, - "11": 1197.0, - "12": 1216.0, - "13": 1281.0, - "14": 1285.0, - "15": 1234.0, - "16": 1270.0, - "17": 1359.0, - "18": 1188.0, - "19": 1369.0, - "20": 1263.0, - "21": 1472.0, - "22": 1322.0, - "23": 1191.0, - "24": 1533.0, - "25": 1295.0, - "26": 1239.0, - "27": 1424.0, - "28": 1365.0, - "29": 1223.0, + "5": 1205.0, + "6": 1195.0, + "7": 1473.0, + "8": 1314.0, + "9": 1406.0, + "10": 1269.0, + "11": 1195.0, + "12": 1167.0, + "13": 1287.0, + "14": 1282.0, + "15": 1273.0, + "16": 1292.0, + "17": 1352.0, + "18": 1296.0, + "19": 1316.0, + "20": 1296.0, + "21": 1399.0, + "22": 1309.0, + "23": 1275.0, + "24": 1526.0, + "25": 1311.0, + "26": 1229.0, + "27": 1445.0, + "28": 1377.0, + "29": 1276.0, "30": 1210.0, - "31": 1417.0, - "32": 1444.0, - "33": 1356.0, + "31": 1409.0, + "32": 1449.0, + "33": 1341.0, "34": 1382.0, - "35": 1421.0, - "36": 1238.0, - "37": 1358.0, - "38": 1276.0, - "39": 1251.0, - "40": 1400.0, - "41": 1268.0, - "42": 1150.0, - "43": 1345.0, - "44": 1358.0, - "45": 1158.0, - "46": 1296.0, - "47": 1360.0, - "48": 1341.0, - "49": 1344.0, - "50": 1189.0, - "51": 1212.0, - "52": 1346.0, - "53": 1303.0, - "54": 1186.0, - "55": 1152.0, - "56": 1199.0, - "57": 1263.0, - "58": 1404.0, - "59": 1221.0, - "60": 1333.0, - "61": 1235.0, - "62": 1321.0, - "63": 1131.0, - "64": 1268.0, - "65": 1324.0, - "66": 1190.0, - "67": 1376.0, - "68": 1101.0, - "69": 1469.0, - "70": 1163.0, - "71": 1176.0, - "72": 1320.0, - "73": 1233.0, - "74": 1171.0, - "75": 1377.0, - "76": 1349.0, - "77": 1185.0, - "78": 1266.0, - "79": 1288.0, - "80": 1394.0, - "81": 1364.0, - "82": 1137.0, - "83": 1390.0, - "84": 1279.0, - "85": 1129.0, - "86": 1304.0, - "87": 1015.0, - "88": 1228.0, + "35": 1467.0, + "36": 1227.0, + "37": 1387.0, + "38": 1245.0, + "39": 1279.0, + "40": 1413.0, + "41": 1222.0, + "42": 1211.0, + "43": 1343.0, + "44": 1431.0, + "45": 1184.0, + "46": 1275.0, + "47": 1430.0, + "48": 1330.0, + "49": 1347.0, + "50": 1238.0, + "51": 1190.0, + "52": 1266.0, + "53": 1329.0, + "54": 1174.0, + "55": 1269.0, + "56": 1070.0, + "57": 1234.0, + "58": 1410.0, + "59": 1200.0, + "60": 1398.0, + "61": 1291.0, + "62": 1265.0, + "63": 1083.0, + "64": 1285.0, + "65": 1416.0, + "66": 1210.0, + "67": 1414.0, + "68": 1115.0, + "69": 1429.0, + "70": 1234.0, + "71": 1093.0, + "72": 1366.0, + "73": 1210.0, + "74": 1243.0, + "75": 1431.0, + "76": 1325.0, + "77": 1205.0, + "78": 1189.0, + "79": 1308.0, + "80": 1343.0, + "81": 1489.0, + "82": 1214.0, + "83": 1381.0, + "84": 1303.0, + "85": 1174.0, + "86": 1197.0, + "87": 1120.0, + "88": 1273.0, "89": 1296.0, - "90": 1197.0, - "91": 1239.0, - "92": 1174.0, - "93": 1306.0, - "94": 1287.0, - "95": 1221.0, - "96": 1178.0, - "97": 1245.0, - "98": 1209.0, - "99": 1342.0, - "100": 1311.0 + "90": 1256.0, + "91": 1198.0, + "92": 1120.0, + "93": 1204.0, + "94": 1291.0, + "95": 1211.0, + "96": 1172.0, + "97": 1282.0, + "98": 1164.0, + "99": 1319.0, + "100": 1228.0 } }, "mem-allocated-bytes": { "start_step": 1, "end_step": 100, "step_interval": 1, + "value_precision": "full", "values": { - "1": 993736192.0, - "2": 993755136.0, - "3": 993747968.0, - "4": 993717760.0, - "5": 993722880.0, - "6": 993715200.0, - "7": 993746432.0, - "8": 993716736.0, - "9": 993742848.0, - "10": 993731584.0, - "11": 993723392.0, - "12": 993696256.0, - "13": 993779712.0, - "14": 993799680.0, - "15": 993730048.0, - "16": 993718784.0, - "17": 993743360.0, - "18": 993712640.0, - "19": 993774592.0, - "20": 993729536.0, - "21": 993777664.0, - "22": 993731072.0, - "23": 993710080.0, - "24": 993756672.0, - "25": 993680896.0, - "26": 993788928.0, - "27": 993759744.0, - "28": 993737728.0, - "29": 993748992.0, - "30": 993705984.0, - "31": 993734144.0, - "32": 993738240.0, - "33": 993717248.0, - "34": 993778688.0, - "35": 993716224.0, - "36": 993667072.0, - "37": 993728000.0, - "38": 993744384.0, - "39": 993692160.0, - "40": 993733120.0, - "41": 993739264.0, - "42": 993724928.0, - "43": 993736704.0, - "44": 993698816.0, - "45": 993694208.0, - "46": 993744896.0, - "47": 993772544.0, - "48": 993693184.0, - "49": 993721856.0, - "50": 993681920.0, - "51": 993661952.0, - "52": 993697280.0, - "53": 993740288.0, - "54": 993718784.0, - "55": 993700864.0, - "56": 993729024.0, - "57": 993747456.0, - "58": 993792000.0, - "59": 993714688.0, - "60": 993697792.0, - "61": 993669632.0, - "62": 993708544.0, - "63": 993716736.0, - "64": 993713664.0, - "65": 993717760.0, - "66": 993655296.0, - "67": 993721856.0, - "68": 993737728.0, - "69": 993710080.0, - "70": 993684480.0, - "71": 993691648.0, - "72": 993761792.0, - "73": 993692672.0, - "74": 993693696.0, - "75": 993741824.0, - "76": 993707008.0, - "77": 993723392.0, - "78": 993700352.0, - "79": 993714176.0, - "80": 993702400.0, - "81": 993704960.0, - "82": 993678848.0, - "83": 993754112.0, - "84": 993726976.0, - "85": 993730048.0, - "86": 993730560.0, - "87": 993610240.0, - "88": 993665024.0, - "89": 993677824.0, - "90": 993711104.0, - "91": 993729536.0, - "92": 993679872.0, - "93": 993652224.0, - "94": 993689088.0, - "95": 993676800.0, - "96": 993644544.0, - "97": 993658880.0, - "98": 993737728.0, - "99": 993702912.0, - "100": 993714688.0 + "1": 994259456.0, + "2": 994278400.0, + "3": 994271232.0, + "4": 994241024.0, + "5": 994246144.0, + "6": 994239488.0, + "7": 994270208.0, + "8": 994239488.0, + "9": 994266112.0, + "10": 994253824.0, + "11": 994246656.0, + "12": 994221568.0, + "13": 994303488.0, + "14": 994323968.0, + "15": 994252288.0, + "16": 994242048.0, + "17": 994266624.0, + "18": 994237440.0, + "19": 994297856.0, + "20": 994253312.0, + "21": 994301440.0, + "22": 994253824.0, + "23": 994235392.0, + "24": 994278912.0, + "25": 994205184.0, + "26": 994310656.0, + "27": 994283520.0, + "28": 994262016.0, + "29": 994273280.0, + "30": 994228736.0, + "31": 994257920.0, + "32": 994262528.0, + "33": 994241024.0, + "34": 994301440.0, + "35": 994238976.0, + "36": 994190848.0, + "37": 994251264.0, + "38": 994268160.0, + "39": 994215424.0, + "40": 994258944.0, + "41": 994260992.0, + "42": 994247168.0, + "43": 994260480.0, + "44": 994222592.0, + "45": 994217472.0, + "46": 994266624.0, + "47": 994297344.0, + "48": 994215424.0, + "49": 994245120.0, + "50": 994204672.0, + "51": 994185728.0, + "52": 994219520.0, + "53": 994262528.0, + "54": 994243584.0, + "55": 994223104.0, + "56": 994253824.0, + "57": 994270720.0, + "58": 994313728.0, + "59": 994238976.0, + "60": 994221056.0, + "61": 994191872.0, + "62": 994231296.0, + "63": 994237440.0, + "64": 994237952.0, + "65": 994241024.0, + "66": 994178048.0, + "67": 994246144.0, + "68": 994257408.0, + "69": 994232832.0, + "70": 994203648.0, + "71": 994215424.0, + "72": 994285056.0, + "73": 994212352.0, + "74": 994217984.0, + "75": 994267648.0, + "76": 994231296.0, + "77": 994245632.0, + "78": 994224128.0, + "79": 994237440.0, + "80": 994225664.0, + "81": 994228224.0, + "82": 994202624.0, + "83": 994277376.0, + "84": 994250240.0, + "85": 994256384.0, + "86": 994251776.0, + "87": 994134016.0, + "88": 994187264.0, + "89": 994201600.0, + "90": 994234880.0, + "91": 994253312.0, + "92": 994203136.0, + "93": 994174976.0, + "94": 994213888.0, + "95": 994200576.0, + "96": 994168320.0, + "97": 994182656.0, + "98": 994261504.0, + "99": 994228224.0, + "100": 994238976.0 } }, "mem-max-allocated-bytes": { "start_step": 1, "end_step": 100, "step_interval": 1, + "value_precision": "full", "values": { - "1": 3029009408.0, - "2": 3338636800.0, - "3": 3338636800.0, - "4": 3338636800.0, - "5": 3338636800.0, - "6": 3338636800.0, - "7": 3338636800.0, - "8": 3338636800.0, - "9": 3338636800.0, - "10": 3338636800.0, - "11": 3338636800.0, - "12": 3338636800.0, - "13": 3351685632.0, - "14": 3365677056.0, - "15": 3365677056.0, - "16": 3365677056.0, - "17": 3365677056.0, - "18": 3365677056.0, - "19": 3365677056.0, - "20": 3365677056.0, - "21": 3365677056.0, - "22": 3365677056.0, - "23": 3365677056.0, - "24": 3365677056.0, - "25": 3365677056.0, - "26": 3365677056.0, - "27": 3365677056.0, - "28": 3365677056.0, - "29": 3365677056.0, - "30": 3365677056.0, - "31": 3365677056.0, - "32": 3365677056.0, - "33": 3365677056.0, - "34": 3365677056.0, - "35": 3365677056.0, - "36": 3365677056.0, - "37": 3365677056.0, - "38": 3365677056.0, - "39": 3365677056.0, - "40": 3365677056.0, - "41": 3365677056.0, - "42": 3365677056.0, - "43": 3365677056.0, - "44": 3365677056.0, - "45": 3365677056.0, - "46": 3365677056.0, - "47": 3365677056.0, - "48": 3365677056.0, - "49": 3365677056.0, - "50": 3365677056.0, - "51": 3365677056.0, - "52": 3365677056.0, - "53": 3365677056.0, - "54": 3365677056.0, - "55": 3365677056.0, - "56": 3365677056.0, - "57": 3365677056.0, - "58": 3365677056.0, - "59": 3365677056.0, - "60": 3365677056.0, - "61": 3365677056.0, - "62": 3365677056.0, - "63": 3365677056.0, - "64": 3365677056.0, - "65": 3365677056.0, - "66": 3365677056.0, - "67": 3365677056.0, - "68": 3365677056.0, - "69": 3365677056.0, - "70": 3365677056.0, - "71": 3365677056.0, - "72": 3365677056.0, - "73": 3365677056.0, - "74": 3365677056.0, - "75": 3365677056.0, - "76": 3365677056.0, - "77": 3365677056.0, - "78": 3365677056.0, - "79": 3365677056.0, - "80": 3365677056.0, - "81": 3365677056.0, - "82": 3365677056.0, - "83": 3365677056.0, - "84": 3365677056.0, - "85": 3365677056.0, - "86": 3365677056.0, - "87": 3365677056.0, - "88": 3365677056.0, - "89": 3365677056.0, - "90": 3365677056.0, - "91": 3365677056.0, - "92": 3365677056.0, - "93": 3365677056.0, - "94": 3365677056.0, - "95": 3365677056.0, - "96": 3365677056.0, - "97": 3365677056.0, - "98": 3365677056.0, - "99": 3365677056.0, - "100": 3365677056.0 + "1": 3029150720.0, + "2": 3340922368.0, + "3": 3340922368.0, + "4": 3340922368.0, + "5": 3340922368.0, + "6": 3340922368.0, + "7": 3340922368.0, + "8": 3340922368.0, + "9": 3340922368.0, + "10": 3340922368.0, + "11": 3340922368.0, + "12": 3340922368.0, + "13": 3352240128.0, + "14": 3365910528.0, + "15": 3365910528.0, + "16": 3365910528.0, + "17": 3365910528.0, + "18": 3365910528.0, + "19": 3365910528.0, + "20": 3365910528.0, + "21": 3365910528.0, + "22": 3365910528.0, + "23": 3365910528.0, + "24": 3365910528.0, + "25": 3365910528.0, + "26": 3365910528.0, + "27": 3365910528.0, + "28": 3365910528.0, + "29": 3365910528.0, + "30": 3365910528.0, + "31": 3365910528.0, + "32": 3365910528.0, + "33": 3365910528.0, + "34": 3365910528.0, + "35": 3365910528.0, + "36": 3365910528.0, + "37": 3365910528.0, + "38": 3365910528.0, + "39": 3365910528.0, + "40": 3365910528.0, + "41": 3365910528.0, + "42": 3365910528.0, + "43": 3365910528.0, + "44": 3365910528.0, + "45": 3365910528.0, + "46": 3365910528.0, + "47": 3365910528.0, + "48": 3365910528.0, + "49": 3365910528.0, + "50": 3365910528.0, + "51": 3365910528.0, + "52": 3365910528.0, + "53": 3365910528.0, + "54": 3365910528.0, + "55": 3365910528.0, + "56": 3365910528.0, + "57": 3365910528.0, + "58": 3365910528.0, + "59": 3365910528.0, + "60": 3365910528.0, + "61": 3365910528.0, + "62": 3365910528.0, + "63": 3365910528.0, + "64": 3365910528.0, + "65": 3365910528.0, + "66": 3365910528.0, + "67": 3365910528.0, + "68": 3365910528.0, + "69": 3365910528.0, + "70": 3365910528.0, + "71": 3365910528.0, + "72": 3365910528.0, + "73": 3365910528.0, + "74": 3365910528.0, + "75": 3365910528.0, + "76": 3365910528.0, + "77": 3365910528.0, + "78": 3365910528.0, + "79": 3365910528.0, + "80": 3365910528.0, + "81": 3365910528.0, + "82": 3365910528.0, + "83": 3365910528.0, + "84": 3365910528.0, + "85": 3365910528.0, + "86": 3365910528.0, + "87": 3365910528.0, + "88": 3365910528.0, + "89": 3365910528.0, + "90": 3365910528.0, + "91": 3365910528.0, + "92": 3365910528.0, + "93": 3365910528.0, + "94": 3365910528.0, + "95": 3365910528.0, + "96": 3365910528.0, + "97": 3365910528.0, + "98": 3365910528.0, + "99": 3365910528.0, + "100": 3365910528.0 } }, "iteration-time": { "start_step": 2, "end_step": 100, "step_interval": 1, + "value_precision": "full", "values": { - "2": 7.86796, - "3": 0.24945, - "4": 0.20624, - "5": 0.20113, - "6": 0.21009, - "7": 0.20422, - "8": 0.20769, - "9": 0.19944, - "10": 0.20045, - "11": 0.19497, - "12": 0.98395, - "13": 0.19399, - "14": 0.19464, - "15": 0.19588, - "16": 0.19449, - "17": 0.18778, - "18": 0.18537, - "19": 0.18793, - "20": 0.1991, - "21": 0.19206, - "22": 0.19055, - "23": 0.19069, - "24": 0.19233, - "25": 0.19257, - "26": 0.18433, - "27": 0.18813, - "28": 0.1874, - "29": 0.18218, - "30": 0.18571, - "31": 0.19156, - "32": 0.18658, - "33": 0.19426, - "34": 0.19961, - "35": 0.1977, - "36": 0.18969, - "37": 0.18292, - "38": 0.19437, - "39": 0.19284, - "40": 0.1954, - "41": 0.18976, - "42": 0.19663, - "43": 0.18312, - "44": 0.18723, - "45": 0.19078, - "46": 0.19788, - "47": 0.19411, - "48": 0.19557, - "49": 0.1929, - "50": 0.19748, - "51": 0.18722, - "52": 1.35902, - "53": 0.25806, - "54": 0.18669, - "55": 0.18278, - "56": 0.19479, - "57": 0.19021, - "58": 0.18983, - "59": 0.18356, - "60": 0.18936, - "61": 0.18213, - "62": 0.1923, - "63": 0.17935, - "64": 0.18534, - "65": 0.179, - "66": 0.18207, - "67": 0.18974, - "68": 0.19445, - "69": 0.18899, - "70": 0.18671, - "71": 0.1736, - "72": 0.18358, - "73": 0.18734, - "74": 0.18449, - "75": 0.18804, - "76": 0.19105, - "77": 0.17522, - "78": 0.17175, - "79": 0.19403, - "80": 0.19302, - "81": 0.19281, - "82": 0.18878, - "83": 0.18543, - "84": 0.18497, - "85": 0.18438, - "86": 0.19491, - "87": 0.19776, - "88": 0.19143, - "89": 0.18022, - "90": 0.18427, - "91": 0.18472, - "92": 0.1882, - "93": 0.18727, - "94": 0.73002, - "95": 0.19204, - "96": 0.18614, - "97": 0.18888, - "98": 0.18887, - "99": 0.18287, - "100": 0.18481 + "2": 0.2827951908111572, + "3": 0.31684207916259766, + "4": 0.3021562099456787, + "5": 0.23834705352783203, + "6": 0.2212059497833252, + "7": 0.22159504890441895, + "8": 0.2099289894104004, + "9": 0.2396543025970459, + "10": 0.21392560005187988, + "11": 0.22207307815551758, + "12": 0.22284412384033203, + "13": 0.1487598419189453, + "14": 0.14713788032531738, + "15": 0.1490647792816162, + "16": 0.15094709396362305, + "17": 0.14764952659606934, + "18": 0.14497900009155273, + "19": 0.14449381828308105, + "20": 0.14245176315307617, + "21": 0.14626574516296387, + "22": 0.13777661323547363, + "23": 0.1407787799835205, + "24": 0.13884449005126953, + "25": 0.1381092071533203, + "26": 0.1342158317565918, + "27": 0.14044451713562012, + "28": 0.13512682914733887, + "29": 0.13737010955810547, + "30": 0.1382763385772705, + "31": 0.1341719627380371, + "32": 0.13904905319213867, + "33": 0.13717198371887207, + "34": 0.14044570922851562, + "35": 0.1391596794128418, + "36": 0.14247846603393555, + "37": 0.13686633110046387, + "38": 0.13943767547607422, + "39": 0.13613486289978027, + "40": 0.13499975204467773, + "41": 0.132554292678833, + "42": 0.13652729988098145, + "43": 0.13467764854431152, + "44": 0.14021658897399902, + "45": 0.1367502212524414, + "46": 0.13562846183776855, + "47": 0.13490509986877441, + "48": 0.13094472885131836, + "49": 0.13439273834228516, + "50": 0.13861870765686035, + "51": 0.20732498168945312, + "52": 0.15424418449401855, + "53": 0.2551095485687256, + "54": 0.14093899726867676, + "55": 0.1362464427947998, + "56": 0.13698220252990723, + "57": 0.1358497142791748, + "58": 0.13513731956481934, + "59": 0.13623929023742676, + "60": 0.13345050811767578, + "61": 0.1328415870666504, + "62": 0.14179348945617676, + "63": 0.13622784614562988, + "64": 0.13571500778198242, + "65": 0.13056731224060059, + "66": 0.13215899467468262, + "67": 0.1364595890045166, + "68": 0.13535308837890625, + "69": 0.13398075103759766, + "70": 0.13229966163635254, + "71": 0.1406099796295166, + "72": 0.13494539260864258, + "73": 0.13707661628723145, + "74": 0.1415867805480957, + "75": 0.13589167594909668, + "76": 0.136979341506958, + "77": 0.13653111457824707, + "78": 0.13898301124572754, + "79": 0.1347362995147705, + "80": 0.13371539115905762, + "81": 0.13336610794067383, + "82": 0.13306021690368652, + "83": 0.13545727729797363, + "84": 0.13663959503173828, + "85": 0.1423492431640625, + "86": 0.13361358642578125, + "87": 0.13869452476501465, + "88": 0.13547158241271973, + "89": 0.13535213470458984, + "90": 0.1328437328338623, + "91": 0.13509750366210938, + "92": 0.13523459434509277, + "93": 0.13532304763793945, + "94": 0.1358044147491455, + "95": 0.14278125762939453, + "96": 0.136491060256958, + "97": 0.13543438911437988, + "98": 0.1637561321258545, + "99": 0.1713716983795166, + "100": 0.17469286918640137 } } } \ No newline at end of file diff --git a/tests/functional_tests/test_cases/moe/gpt3_moe_mcore_te_ep8_resume_torch_dist_muon/golden_values_dev_dgx_gb200.json b/tests/functional_tests/test_cases/moe/gpt3_moe_mcore_te_ep8_resume_torch_dist_muon/golden_values_dev_dgx_gb200.json index 38ca95b2d6f..e565e9d7648 100644 --- a/tests/functional_tests/test_cases/moe/gpt3_moe_mcore_te_ep8_resume_torch_dist_muon/golden_values_dev_dgx_gb200.json +++ b/tests/functional_tests/test_cases/moe/gpt3_moe_mcore_te_ep8_resume_torch_dist_muon/golden_values_dev_dgx_gb200.json @@ -3,534 +3,539 @@ "start_step": 1, "end_step": 100, "step_interval": 1, + "value_precision": "full", "values": { - "1": 10.94003, - "2": 10.91043, - "3": 10.92834, - "4": 10.92146, - "5": 10.91533, - "6": 10.92467, - "7": 10.92139, - "8": 10.92316, - "9": 10.93345, - "10": 10.92218, - "11": 10.92035, - "12": 10.93759, - "13": 10.92018, - "14": 10.90849, - "15": 10.91888, - "16": 10.91627, - "17": 10.93526, - "18": 10.92451, - "19": 10.92796, - "20": 10.90104, - "21": 10.92813, - "22": 10.92292, - "23": 10.93401, - "24": 10.90544, - "25": 10.9199, - "26": 10.91424, - "27": 10.93153, - "28": 10.91576, - "29": 10.92531, - "30": 10.91037, - "31": 10.90623, - "32": 10.91536, - "33": 10.91507, - "34": 10.90492, - "35": 10.92789, - "36": 10.90883, - "37": 10.90986, - "38": 10.91608, - "39": 10.9115, - "40": 10.90636, - "41": 10.91092, - "42": 10.91884, - "43": 10.90652, - "44": 10.91661, - "45": 10.90695, - "46": 10.90689, - "47": 10.90555, - "48": 10.89359, - "49": 10.90379, - "50": 10.89535, - "51": 10.89502, - "52": 10.89665, - "53": 10.8985, - "54": 10.88656, - "55": 10.90501, - "56": 10.8881, - "57": 10.88945, - "58": 10.87967, - "59": 10.87634, - "60": 10.86493, - "61": 10.86782, - "62": 10.86776, - "63": 10.87163, - "64": 10.86219, - "65": 10.87077, - "66": 10.86657, - "67": 10.85603, - "68": 10.85054, - "69": 10.8543, - "70": 10.85356, - "71": 10.84969, - "72": 10.8461, - "73": 10.84086, - "74": 10.84064, - "75": 10.83991, - "76": 10.83186, - "77": 10.8251, - "78": 10.81274, - "79": 10.83037, - "80": 10.8316, - "81": 10.81424, - "82": 10.81173, - "83": 10.80605, - "84": 10.77967, - "85": 10.7823, - "86": 10.79649, - "87": 10.79622, - "88": 10.80275, - "89": 10.78318, - "90": 10.77903, - "91": 10.78362, - "92": 10.77961, - "93": 10.76103, - "94": 10.76553, - "95": 10.76973, - "96": 10.73844, - "97": 10.73211, - "98": 10.754, - "99": 10.7672, - "100": 10.7375 + "1": 10.940028190612793, + "2": 10.910425186157227, + "3": 10.928335189819336, + "4": 10.921457290649414, + "5": 10.915254592895508, + "6": 10.924782752990723, + "7": 10.92149543762207, + "8": 10.923296928405762, + "9": 10.933662414550781, + "10": 10.92244815826416, + "11": 10.920829772949219, + "12": 10.937786102294922, + "13": 10.919692993164062, + "14": 10.908319473266602, + "15": 10.919576644897461, + "16": 10.916498184204102, + "17": 10.935380935668945, + "18": 10.924417495727539, + "19": 10.928023338317871, + "20": 10.900928497314453, + "21": 10.928129196166992, + "22": 10.922487258911133, + "23": 10.934159278869629, + "24": 10.905404090881348, + "25": 10.920371055603027, + "26": 10.914301872253418, + "27": 10.931076049804688, + "28": 10.915464401245117, + "29": 10.92558765411377, + "30": 10.91069221496582, + "31": 10.905572891235352, + "32": 10.915328979492188, + "33": 10.914925575256348, + "34": 10.904078483581543, + "35": 10.928526878356934, + "36": 10.908609390258789, + "37": 10.910324096679688, + "38": 10.915919303894043, + "39": 10.911232948303223, + "40": 10.905683517456055, + "41": 10.911552429199219, + "42": 10.918734550476074, + "43": 10.90638542175293, + "44": 10.916589736938477, + "45": 10.90713882446289, + "46": 10.90691089630127, + "47": 10.905275344848633, + "48": 10.893331527709961, + "49": 10.904316902160645, + "50": 10.895713806152344, + "51": 10.895317077636719, + "52": 10.896758079528809, + "53": 10.898993492126465, + "54": 10.886012077331543, + "55": 10.905359268188477, + "56": 10.888250350952148, + "57": 10.889857292175293, + "58": 10.879864692687988, + "59": 10.875919342041016, + "60": 10.865079879760742, + "61": 10.86771011352539, + "62": 10.867827415466309, + "63": 10.870774269104004, + "64": 10.862744331359863, + "65": 10.87077808380127, + "66": 10.86647891998291, + "67": 10.856607437133789, + "68": 10.850610733032227, + "69": 10.854192733764648, + "70": 10.853626251220703, + "71": 10.849499702453613, + "72": 10.84597396850586, + "73": 10.840812683105469, + "74": 10.840066909790039, + "75": 10.839336395263672, + "76": 10.831446647644043, + "77": 10.82507610321045, + "78": 10.813162803649902, + "79": 10.830068588256836, + "80": 10.83149242401123, + "81": 10.813776969909668, + "82": 10.812215805053711, + "83": 10.805734634399414, + "84": 10.779777526855469, + "85": 10.782448768615723, + "86": 10.796222686767578, + "87": 10.796459197998047, + "88": 10.802992820739746, + "89": 10.782163619995117, + "90": 10.778440475463867, + "91": 10.783417701721191, + "92": 10.780105590820312, + "93": 10.760335922241211, + "94": 10.7653169631958, + "95": 10.770015716552734, + "96": 10.738885879516602, + "97": 10.732397079467773, + "98": 10.754006385803223, + "99": 10.7677640914917, + "100": 10.736916542053223 } }, "num-zeros": { "start_step": 1, "end_step": 100, "step_interval": 1, + "value_precision": "full", "values": { "1": 1305.0, "2": 1165.0, "3": 1386.0, "4": 1280.0, - "5": 1190.0, - "6": 1134.0, - "7": 1522.0, - "8": 1206.0, - "9": 1319.0, - "10": 1328.0, - "11": 1218.0, - "12": 1154.0, - "13": 1352.0, - "14": 1272.0, - "15": 1192.0, - "16": 1289.0, - "17": 1318.0, - "18": 1271.0, - "19": 1299.0, - "20": 1228.0, - "21": 1392.0, - "22": 1316.0, - "23": 1271.0, - "24": 1520.0, - "25": 1307.0, - "26": 1270.0, - "27": 1398.0, - "28": 1335.0, - "29": 1286.0, - "30": 1212.0, - "31": 1432.0, - "32": 1415.0, - "33": 1373.0, - "34": 1377.0, - "35": 1422.0, - "36": 1215.0, - "37": 1372.0, - "38": 1198.0, - "39": 1232.0, - "40": 1352.0, - "41": 1228.0, - "42": 1182.0, - "43": 1318.0, - "44": 1374.0, - "45": 1183.0, - "46": 1356.0, - "47": 1437.0, - "48": 1354.0, - "49": 1266.0, - "50": 1219.0, - "51": 1198.0, - "52": 1343.0, - "53": 1341.0, - "54": 1114.0, - "55": 1231.0, - "56": 1208.0, - "57": 1309.0, - "58": 1369.0, - "59": 1197.0, - "60": 1350.0, - "61": 1240.0, - "62": 1270.0, - "63": 1112.0, - "64": 1212.0, - "65": 1402.0, - "66": 1158.0, - "67": 1356.0, - "68": 1128.0, - "69": 1533.0, - "70": 1149.0, - "71": 1140.0, - "72": 1344.0, - "73": 1291.0, - "74": 1212.0, - "75": 1458.0, - "76": 1332.0, - "77": 1210.0, - "78": 1248.0, - "79": 1283.0, - "80": 1304.0, - "81": 1340.0, - "82": 1195.0, - "83": 1395.0, - "84": 1279.0, - "85": 1132.0, - "86": 1309.0, - "87": 1042.0, - "88": 1230.0, - "89": 1287.0, - "90": 1258.0, - "91": 1200.0, - "92": 1136.0, - "93": 1266.0, - "94": 1156.0, - "95": 1226.0, - "96": 1183.0, - "97": 1186.0, - "98": 1213.0, - "99": 1367.0, - "100": 1285.0 + "5": 1275.0, + "6": 1172.0, + "7": 1465.0, + "8": 1286.0, + "9": 1416.0, + "10": 1219.0, + "11": 1209.0, + "12": 1204.0, + "13": 1248.0, + "14": 1282.0, + "15": 1260.0, + "16": 1292.0, + "17": 1359.0, + "18": 1304.0, + "19": 1339.0, + "20": 1308.0, + "21": 1517.0, + "22": 1369.0, + "23": 1236.0, + "24": 1553.0, + "25": 1325.0, + "26": 1188.0, + "27": 1388.0, + "28": 1327.0, + "29": 1263.0, + "30": 1255.0, + "31": 1451.0, + "32": 1403.0, + "33": 1376.0, + "34": 1387.0, + "35": 1351.0, + "36": 1245.0, + "37": 1449.0, + "38": 1302.0, + "39": 1238.0, + "40": 1337.0, + "41": 1220.0, + "42": 1236.0, + "43": 1376.0, + "44": 1397.0, + "45": 1198.0, + "46": 1255.0, + "47": 1430.0, + "48": 1315.0, + "49": 1287.0, + "50": 1212.0, + "51": 1278.0, + "52": 1248.0, + "53": 1352.0, + "54": 1177.0, + "55": 1240.0, + "56": 1113.0, + "57": 1223.0, + "58": 1394.0, + "59": 1206.0, + "60": 1396.0, + "61": 1185.0, + "62": 1274.0, + "63": 1186.0, + "64": 1240.0, + "65": 1367.0, + "66": 1222.0, + "67": 1340.0, + "68": 1195.0, + "69": 1499.0, + "70": 1220.0, + "71": 1144.0, + "72": 1322.0, + "73": 1247.0, + "74": 1211.0, + "75": 1424.0, + "76": 1376.0, + "77": 1229.0, + "78": 1228.0, + "79": 1293.0, + "80": 1399.0, + "81": 1329.0, + "82": 1179.0, + "83": 1381.0, + "84": 1249.0, + "85": 1202.0, + "86": 1226.0, + "87": 1105.0, + "88": 1280.0, + "89": 1315.0, + "90": 1145.0, + "91": 1217.0, + "92": 1167.0, + "93": 1183.0, + "94": 1179.0, + "95": 1203.0, + "96": 1153.0, + "97": 1194.0, + "98": 1217.0, + "99": 1280.0, + "100": 1269.0 } }, "mem-allocated-bytes": { "start_step": 1, "end_step": 100, "step_interval": 1, + "value_precision": "full", "values": { - "1": 1097328640.0, - "2": 1097347584.0, - "3": 1097340416.0, - "4": 1097310208.0, - "5": 1097315328.0, - "6": 1097307136.0, - "7": 1097339904.0, - "8": 1097309184.0, - "9": 1097334272.0, - "10": 1097323008.0, - "11": 1097315840.0, - "12": 1097289728.0, - "13": 1097374720.0, - "14": 1097391616.0, - "15": 1097322496.0, - "16": 1097312256.0, - "17": 1097336320.0, - "18": 1097305600.0, - "19": 1097367552.0, - "20": 1097321984.0, - "21": 1097370624.0, - "22": 1097324544.0, - "23": 1097302528.0, - "24": 1097347072.0, - "25": 1097273344.0, - "26": 1097380352.0, - "27": 1097353728.0, - "28": 1097329664.0, - "29": 1097342464.0, - "30": 1097297408.0, - "31": 1097326592.0, - "32": 1097328640.0, - "33": 1097309184.0, - "34": 1097370624.0, - "35": 1097308160.0, - "36": 1097261568.0, - "37": 1097319936.0, - "38": 1097336832.0, - "39": 1097283584.0, - "40": 1097324544.0, - "41": 1097330688.0, - "42": 1097317888.0, - "43": 1097329152.0, - "44": 1097290752.0, - "45": 1097288192.0, - "46": 1097336832.0, - "47": 1097364480.0, - "48": 1097285120.0, - "49": 1097315328.0, - "50": 1097273856.0, - "51": 1097254400.0, - "52": 1097287680.0, - "53": 1097330688.0, - "54": 1097311232.0, - "55": 1097291776.0, - "56": 1097321984.0, - "57": 1097340416.0, - "58": 1097386496.0, - "59": 1097306624.0, - "60": 1097290752.0, - "61": 1097261056.0, - "62": 1097300992.0, - "63": 1097307136.0, - "64": 1097307648.0, - "65": 1097310208.0, - "66": 1097248256.0, - "67": 1097317376.0, - "68": 1097328640.0, - "69": 1097300992.0, - "70": 1097275904.0, - "71": 1097284096.0, - "72": 1097355776.0, - "73": 1097281536.0, - "74": 1097286144.0, - "75": 1097337856.0, - "76": 1097298432.0, - "77": 1097315328.0, - "78": 1097291776.0, - "79": 1097306624.0, - "80": 1097295872.0, - "81": 1097298944.0, - "82": 1097268736.0, - "83": 1097346048.0, - "84": 1097321984.0, - "85": 1097324032.0, - "86": 1097324032.0, - "87": 1097204224.0, - "88": 1097256448.0, - "89": 1097269760.0, - "90": 1097303552.0, - "91": 1097321984.0, - "92": 1097271296.0, - "93": 1097246720.0, - "94": 1097283584.0, - "95": 1097269248.0, - "96": 1097236992.0, - "97": 1097251840.0, - "98": 1097330688.0, - "99": 1097295872.0, - "100": 1097307648.0 + "1": 1096655872.0, + "2": 1096674816.0, + "3": 1096667648.0, + "4": 1096637440.0, + "5": 1096642560.0, + "6": 1096635904.0, + "7": 1096666112.0, + "8": 1096637440.0, + "9": 1096662528.0, + "10": 1096650752.0, + "11": 1096642560.0, + "12": 1096617472.0, + "13": 1096700928.0, + "14": 1096718848.0, + "15": 1096651776.0, + "16": 1096637952.0, + "17": 1096663040.0, + "18": 1096632320.0, + "19": 1096695296.0, + "20": 1096649728.0, + "21": 1096698880.0, + "22": 1096652288.0, + "23": 1096630784.0, + "24": 1096676352.0, + "25": 1096600064.0, + "26": 1096708096.0, + "27": 1096679936.0, + "28": 1096659968.0, + "29": 1096670720.0, + "30": 1096623104.0, + "31": 1096654336.0, + "32": 1096658944.0, + "33": 1096636928.0, + "34": 1096696832.0, + "35": 1096634880.0, + "36": 1096588288.0, + "37": 1096647168.0, + "38": 1096665088.0, + "39": 1096612864.0, + "40": 1096654848.0, + "41": 1096657920.0, + "42": 1096644608.0, + "43": 1096657920.0, + "44": 1096617472.0, + "45": 1096613376.0, + "46": 1096665088.0, + "47": 1096691712.0, + "48": 1096612352.0, + "49": 1096642048.0, + "50": 1096600576.0, + "51": 1096580608.0, + "52": 1096616960.0, + "53": 1096657920.0, + "54": 1096637952.0, + "55": 1096619520.0, + "56": 1096649216.0, + "57": 1096666624.0, + "58": 1096710656.0, + "59": 1096635904.0, + "60": 1096617984.0, + "61": 1096588288.0, + "62": 1096629248.0, + "63": 1096634880.0, + "64": 1096633856.0, + "65": 1096636928.0, + "66": 1096574976.0, + "67": 1096641024.0, + "68": 1096654848.0, + "69": 1096629760.0, + "70": 1096602112.0, + "71": 1096611840.0, + "72": 1096681472.0, + "73": 1096611328.0, + "74": 1096614400.0, + "75": 1096664064.0, + "76": 1096627200.0, + "77": 1096643584.0, + "78": 1096621056.0, + "79": 1096633856.0, + "80": 1096622592.0, + "81": 1096625664.0, + "82": 1096596992.0, + "83": 1096674816.0, + "84": 1096647168.0, + "85": 1096651264.0, + "86": 1096650752.0, + "87": 1096531456.0, + "88": 1096585216.0, + "89": 1096596480.0, + "90": 1096633856.0, + "91": 1096646656.0, + "92": 1096600576.0, + "93": 1096571392.0, + "94": 1096609792.0, + "95": 1096596480.0, + "96": 1096565248.0, + "97": 1096580608.0, + "98": 1096657920.0, + "99": 1096622592.0, + "100": 1096635904.0 } }, "mem-max-allocated-bytes": { "start_step": 1, "end_step": 100, "step_interval": 1, + "value_precision": "full", "values": { - "1": 3080119808.0, - "2": 3437892608.0, - "3": 3438232576.0, - "4": 3438232576.0, - "5": 3438232576.0, - "6": 3438232576.0, - "7": 3438232576.0, - "8": 3438232576.0, - "9": 3438232576.0, - "10": 3438232576.0, - "11": 3438232576.0, - "12": 3438232576.0, - "13": 3457461760.0, - "14": 3471148032.0, - "15": 3471148032.0, - "16": 3471148032.0, - "17": 3471148032.0, - "18": 3471148032.0, - "19": 3471148032.0, - "20": 3471148032.0, - "21": 3471148032.0, - "22": 3471148032.0, - "23": 3471148032.0, - "24": 3471148032.0, - "25": 3471148032.0, - "26": 3471148032.0, - "27": 3471148032.0, - "28": 3471148032.0, - "29": 3471148032.0, - "30": 3471148032.0, - "31": 3471148032.0, - "32": 3471148032.0, - "33": 3471148032.0, - "34": 3471148032.0, - "35": 3471148032.0, - "36": 3471148032.0, - "37": 3471148032.0, - "38": 3471148032.0, - "39": 3471148032.0, - "40": 3471148032.0, - "41": 3471148032.0, - "42": 3471148032.0, - "43": 3471148032.0, - "44": 3471148032.0, - "45": 3471148032.0, - "46": 3471148032.0, - "47": 3471148032.0, - "48": 3471148032.0, - "49": 3471148032.0, - "50": 3471148032.0, - "51": 3471148032.0, - "52": 3471148032.0, - "53": 3471148032.0, - "54": 3471148032.0, - "55": 3471148032.0, - "56": 3471148032.0, - "57": 3471148032.0, - "58": 3471148032.0, - "59": 3471148032.0, - "60": 3471148032.0, - "61": 3471148032.0, - "62": 3471148032.0, - "63": 3471148032.0, - "64": 3471148032.0, - "65": 3471148032.0, - "66": 3471148032.0, - "67": 3471148032.0, - "68": 3471148032.0, - "69": 3471148032.0, - "70": 3471148032.0, - "71": 3471148032.0, - "72": 3471148032.0, - "73": 3471148032.0, - "74": 3471148032.0, - "75": 3471148032.0, - "76": 3471148032.0, - "77": 3471148032.0, - "78": 3471148032.0, - "79": 3471148032.0, - "80": 3471148032.0, - "81": 3471148032.0, - "82": 3471148032.0, - "83": 3471148032.0, - "84": 3471148032.0, - "85": 3471148032.0, - "86": 3471148032.0, - "87": 3471148032.0, - "88": 3471148032.0, - "89": 3471148032.0, - "90": 3471148032.0, - "91": 3471148032.0, - "92": 3471148032.0, - "93": 3471148032.0, - "94": 3471148032.0, - "95": 3471148032.0, - "96": 3471148032.0, - "97": 3471148032.0, - "98": 3471148032.0, - "99": 3471148032.0, - "100": 3471148032.0 + "1": 3081411072.0, + "2": 3439855616.0, + "3": 3439855616.0, + "4": 3439855616.0, + "5": 3439855616.0, + "6": 3439855616.0, + "7": 3439855616.0, + "8": 3439855616.0, + "9": 3439855616.0, + "10": 3439855616.0, + "11": 3439855616.0, + "12": 3439855616.0, + "13": 3456328192.0, + "14": 3467982848.0, + "15": 3467982848.0, + "16": 3467982848.0, + "17": 3467982848.0, + "18": 3467982848.0, + "19": 3467982848.0, + "20": 3467982848.0, + "21": 3467982848.0, + "22": 3467982848.0, + "23": 3467982848.0, + "24": 3467982848.0, + "25": 3467982848.0, + "26": 3467982848.0, + "27": 3467982848.0, + "28": 3467982848.0, + "29": 3467982848.0, + "30": 3467982848.0, + "31": 3467982848.0, + "32": 3467982848.0, + "33": 3467982848.0, + "34": 3467982848.0, + "35": 3467982848.0, + "36": 3467982848.0, + "37": 3467982848.0, + "38": 3467982848.0, + "39": 3467982848.0, + "40": 3467982848.0, + "41": 3467982848.0, + "42": 3467982848.0, + "43": 3467982848.0, + "44": 3467982848.0, + "45": 3467982848.0, + "46": 3467982848.0, + "47": 3467982848.0, + "48": 3467982848.0, + "49": 3467982848.0, + "50": 3467982848.0, + "51": 3467982848.0, + "52": 3467982848.0, + "53": 3467982848.0, + "54": 3467982848.0, + "55": 3467982848.0, + "56": 3467982848.0, + "57": 3467982848.0, + "58": 3467982848.0, + "59": 3467982848.0, + "60": 3467982848.0, + "61": 3467982848.0, + "62": 3467982848.0, + "63": 3467982848.0, + "64": 3467982848.0, + "65": 3467982848.0, + "66": 3467982848.0, + "67": 3467982848.0, + "68": 3467982848.0, + "69": 3467982848.0, + "70": 3467982848.0, + "71": 3467982848.0, + "72": 3467982848.0, + "73": 3467982848.0, + "74": 3467982848.0, + "75": 3467982848.0, + "76": 3467982848.0, + "77": 3467982848.0, + "78": 3467982848.0, + "79": 3467982848.0, + "80": 3467982848.0, + "81": 3467982848.0, + "82": 3467982848.0, + "83": 3467982848.0, + "84": 3467982848.0, + "85": 3467982848.0, + "86": 3467982848.0, + "87": 3467982848.0, + "88": 3467982848.0, + "89": 3467982848.0, + "90": 3467982848.0, + "91": 3467982848.0, + "92": 3467982848.0, + "93": 3467982848.0, + "94": 3467982848.0, + "95": 3467982848.0, + "96": 3467982848.0, + "97": 3467982848.0, + "98": 3467982848.0, + "99": 3467982848.0, + "100": 3467982848.0 } }, "iteration-time": { "start_step": 2, "end_step": 100, "step_interval": 1, + "value_precision": "full", "values": { - "2": 7.93957, - "3": 0.42673, - "4": 0.39745, - "5": 0.39128, - "6": 0.38865, - "7": 0.37305, - "8": 0.37255, - "9": 0.35982, - "10": 0.36334, - "11": 0.36037, - "12": 0.36713, - "13": 0.3567, - "14": 0.39939, - "15": 0.34459, - "16": 0.34451, - "17": 0.34103, - "18": 0.34504, - "19": 0.33985, - "20": 0.34088, - "21": 0.33578, - "22": 0.33617, - "23": 0.33627, - "24": 0.33801, - "25": 0.33133, - "26": 0.33644, - "27": 0.33465, - "28": 0.33122, - "29": 0.33418, - "30": 0.33247, - "31": 0.34477, - "32": 0.33692, - "33": 0.3364, - "34": 0.33481, - "35": 0.37068, - "36": 0.38328, - "37": 0.38407, - "38": 0.37201, - "39": 0.3672, - "40": 0.36341, - "41": 0.36567, - "42": 0.36984, - "43": 0.36041, - "44": 0.36347, - "45": 0.36499, - "46": 0.36949, - "47": 0.36468, - "48": 0.35198, - "49": 0.35306, - "50": 0.35678, - "51": 0.36849, - "52": 0.47152, - "53": 0.36075, - "54": 0.35491, - "55": 0.35308, - "56": 0.35561, - "57": 0.34454, - "58": 0.3399, - "59": 0.33976, - "60": 1.07427, - "61": 0.33067, - "62": 0.34017, - "63": 0.34168, - "64": 0.36047, - "65": 0.35291, - "66": 0.3581, - "67": 0.35905, - "68": 0.35823, - "69": 0.36428, - "70": 0.363, - "71": 0.36518, - "72": 0.36304, - "73": 0.35956, - "74": 0.35983, - "75": 0.35827, - "76": 0.35644, - "77": 0.34467, - "78": 0.35035, - "79": 0.36649, - "80": 0.33991, - "81": 0.34854, - "82": 0.3422, - "83": 0.34454, - "84": 0.86671, - "85": 0.3458, - "86": 0.34758, - "87": 0.34052, - "88": 0.33372, - "89": 0.33629, - "90": 0.33815, - "91": 0.33595, - "92": 0.33742, - "93": 0.33822, - "94": 0.33748, - "95": 0.33325, - "96": 0.33619, - "97": 0.33985, - "98": 0.33752, - "99": 0.33429, - "100": 0.32967 + "2": 0.4688408374786377, + "3": 0.41377687454223633, + "4": 0.35535454750061035, + "5": 0.35517406463623047, + "6": 0.33566737174987793, + "7": 0.3436770439147949, + "8": 0.334136962890625, + "9": 0.3323187828063965, + "10": 0.33437252044677734, + "11": 0.33617544174194336, + "12": 0.33017516136169434, + "13": 0.3223237991333008, + "14": 0.33553242683410645, + "15": 0.31391453742980957, + "16": 0.32451391220092773, + "17": 0.32178592681884766, + "18": 0.3213980197906494, + "19": 0.3239295482635498, + "20": 0.3119161128997803, + "21": 0.3165607452392578, + "22": 0.31688404083251953, + "23": 0.31432509422302246, + "24": 0.3107016086578369, + "25": 0.3101491928100586, + "26": 0.30623364448547363, + "27": 0.30950117111206055, + "28": 0.3059399127960205, + "29": 0.30121612548828125, + "30": 0.30834054946899414, + "31": 0.30414819717407227, + "32": 0.30519771575927734, + "33": 0.3076624870300293, + "34": 0.3053126335144043, + "35": 0.3057873249053955, + "36": 0.311798095703125, + "37": 0.304241418838501, + "38": 0.30770373344421387, + "39": 0.3066427707672119, + "40": 0.2996644973754883, + "41": 0.29881739616394043, + "42": 0.30580973625183105, + "43": 0.3089613914489746, + "44": 0.3083503246307373, + "45": 0.3067476749420166, + "46": 0.3089721202850342, + "47": 0.30374765396118164, + "48": 0.303023099899292, + "49": 0.30832624435424805, + "50": 0.3081533908843994, + "51": 0.3181476593017578, + "52": 0.3192911148071289, + "53": 0.3088192939758301, + "54": 0.4306609630584717, + "55": 0.31147003173828125, + "56": 0.3043210506439209, + "57": 0.30735087394714355, + "58": 0.30358219146728516, + "59": 0.3011476993560791, + "60": 0.29854249954223633, + "61": 0.3031008243560791, + "62": 0.3019988536834717, + "63": 0.30079221725463867, + "64": 0.3070225715637207, + "65": 0.29797840118408203, + "66": 0.2990889549255371, + "67": 0.29992103576660156, + "68": 0.2995917797088623, + "69": 0.30581021308898926, + "70": 0.3042113780975342, + "71": 0.29917407035827637, + "72": 0.3015162944793701, + "73": 0.30152010917663574, + "74": 0.30283331871032715, + "75": 0.2999746799468994, + "76": 0.30205297470092773, + "77": 0.3064913749694824, + "78": 0.3040452003479004, + "79": 0.30129194259643555, + "80": 0.30349302291870117, + "81": 0.3051445484161377, + "82": 0.3018338680267334, + "83": 0.30323052406311035, + "84": 0.368741512298584, + "85": 0.30407214164733887, + "86": 0.30132246017456055, + "87": 0.31278228759765625, + "88": 0.3064706325531006, + "89": 0.3042728900909424, + "90": 0.30127978324890137, + "91": 0.3030283451080322, + "92": 0.30220699310302734, + "93": 0.3051598072052002, + "94": 0.30628347396850586, + "95": 0.30393290519714355, + "96": 0.30298829078674316, + "97": 0.3007979393005371, + "98": 0.30248165130615234, + "99": 0.30201268196105957, + "100": 0.3016819953918457 } } } \ No newline at end of file diff --git a/tests/functional_tests/test_cases/moe/gpt3_moe_mcore_te_ep8_resume_torch_dist_muon/golden_values_dev_dgx_h100.json b/tests/functional_tests/test_cases/moe/gpt3_moe_mcore_te_ep8_resume_torch_dist_muon/golden_values_dev_dgx_h100.json index f3331881838..20b46c8dcb2 100644 --- a/tests/functional_tests/test_cases/moe/gpt3_moe_mcore_te_ep8_resume_torch_dist_muon/golden_values_dev_dgx_h100.json +++ b/tests/functional_tests/test_cases/moe/gpt3_moe_mcore_te_ep8_resume_torch_dist_muon/golden_values_dev_dgx_h100.json @@ -49,62 +49,62 @@ "42": 10.895501136779785, "43": 10.877589225769043, "44": 10.884319305419922, - "45": 10.883659362792969, - "46": 10.879924774169922, - "47": 10.881741523742676, - "48": 10.870139122009277, - "49": 10.87501049041748, - "50": 10.883739471435547, - "51": 10.895332336425781, - "52": 10.872631072998047, - "53": 10.858932495117188, - "54": 10.871038436889648, - "55": 10.870692253112793, - "56": 10.872472763061523, - "57": 10.84573745727539, - "58": 10.861007690429688, - "59": 10.843330383300781, - "60": 10.850812911987305, - "61": 10.861637115478516, - "62": 10.856411933898926, - "63": 10.858017921447754, - "64": 10.836782455444336, - "65": 10.827064514160156, - "66": 10.844486236572266, - "67": 10.825956344604492, - "68": 10.8345947265625, - "69": 10.816028594970703, - "70": 10.831363677978516, - "71": 10.827238082885742, - "72": 10.810770988464355, - "73": 10.809333801269531, - "74": 10.809293746948242, - "75": 10.809850692749023, - "76": 10.807551383972168, - "77": 10.802705764770508, - "78": 10.794357299804688, - "79": 10.801299095153809, - "80": 10.792222023010254, - "81": 10.796609878540039, - "82": 10.796716690063477, - "83": 10.785881996154785, - "84": 10.756775856018066, - "85": 10.759689331054688, - "86": 10.776266098022461, - "87": 10.794424057006836, - "88": 10.77629566192627, - "89": 10.774800300598145, - "90": 10.7692232131958, - "91": 10.741928100585938, - "92": 10.763282775878906, - "93": 10.747415542602539, - "94": 10.735333442687988, - "95": 10.754096984863281, - "96": 10.72393798828125, - "97": 10.714786529541016, - "98": 10.719482421875, - "99": 10.74830150604248, - "100": 10.697206497192383 + "45": 10.883417129516602, + "46": 10.880020141601562, + "47": 10.881783485412598, + "48": 10.870342254638672, + "49": 10.874947547912598, + "50": 10.88391399383545, + "51": 10.895442962646484, + "52": 10.873101234436035, + "53": 10.859453201293945, + "54": 10.871173858642578, + "55": 10.870899200439453, + "56": 10.872713088989258, + "57": 10.844856262207031, + "58": 10.861200332641602, + "59": 10.842803955078125, + "60": 10.850765228271484, + "61": 10.861603736877441, + "62": 10.856385231018066, + "63": 10.858016967773438, + "64": 10.837060928344727, + "65": 10.827271461486816, + "66": 10.844172477722168, + "67": 10.825831413269043, + "68": 10.83493423461914, + "69": 10.816703796386719, + "70": 10.831136703491211, + "71": 10.827393531799316, + "72": 10.81029224395752, + "73": 10.80961799621582, + "74": 10.809812545776367, + "75": 10.809415817260742, + "76": 10.806971549987793, + "77": 10.803040504455566, + "78": 10.794567108154297, + "79": 10.80213451385498, + "80": 10.792370796203613, + "81": 10.796478271484375, + "82": 10.796855926513672, + "83": 10.785656929016113, + "84": 10.756906509399414, + "85": 10.759649276733398, + "86": 10.776212692260742, + "87": 10.79456615447998, + "88": 10.775540351867676, + "89": 10.77484130859375, + "90": 10.769011497497559, + "91": 10.741552352905273, + "92": 10.763520240783691, + "93": 10.747847557067871, + "94": 10.73507308959961, + "95": 10.753804206848145, + "96": 10.724089622497559, + "97": 10.714757919311523, + "98": 10.719000816345215, + "99": 10.748478889465332, + "100": 10.697717666625977 } }, "num-zeros": { @@ -157,62 +157,62 @@ "42": 1122.0, "43": 1174.0, "44": 1224.0, - "45": 1227.0, - "46": 1277.0, - "47": 1344.0, - "48": 1119.0, - "49": 1411.0, - "50": 1207.0, - "51": 1172.0, - "52": 1103.0, - "53": 1272.0, - "54": 1208.0, - "55": 1178.0, - "56": 1240.0, - "57": 1095.0, - "58": 1244.0, - "59": 1167.0, - "60": 1113.0, - "61": 1215.0, - "62": 1154.0, - "63": 1191.0, - "64": 1302.0, - "65": 1272.0, - "66": 1224.0, - "67": 1276.0, - "68": 1357.0, - "69": 1160.0, - "70": 1215.0, - "71": 1333.0, - "72": 1257.0, - "73": 1267.0, - "74": 1267.0, - "75": 1384.0, - "76": 1242.0, - "77": 1278.0, - "78": 1276.0, - "79": 1382.0, - "80": 1200.0, - "81": 1317.0, - "82": 1088.0, - "83": 1302.0, - "84": 1151.0, - "85": 1150.0, - "86": 1359.0, - "87": 1100.0, - "88": 1244.0, - "89": 1246.0, - "90": 1248.0, - "91": 1123.0, - "92": 1233.0, - "93": 1342.0, - "94": 1248.0, - "95": 1185.0, - "96": 1295.0, - "97": 1209.0, - "98": 1261.0, - "99": 1175.0, - "100": 1309.0 + "45": 1180.0, + "46": 1313.0, + "47": 1328.0, + "48": 1141.0, + "49": 1315.0, + "50": 1227.0, + "51": 1186.0, + "52": 1164.0, + "53": 1254.0, + "54": 1231.0, + "55": 1190.0, + "56": 1222.0, + "57": 1189.0, + "58": 1155.0, + "59": 1094.0, + "60": 1099.0, + "61": 1135.0, + "62": 1081.0, + "63": 1099.0, + "64": 1331.0, + "65": 1308.0, + "66": 1242.0, + "67": 1293.0, + "68": 1280.0, + "69": 1195.0, + "70": 1309.0, + "71": 1361.0, + "72": 1215.0, + "73": 1241.0, + "74": 1326.0, + "75": 1340.0, + "76": 1214.0, + "77": 1265.0, + "78": 1289.0, + "79": 1296.0, + "80": 1170.0, + "81": 1267.0, + "82": 1137.0, + "83": 1238.0, + "84": 1191.0, + "85": 1189.0, + "86": 1364.0, + "87": 1221.0, + "88": 1249.0, + "89": 1245.0, + "90": 1318.0, + "91": 1134.0, + "92": 1226.0, + "93": 1326.0, + "94": 1222.0, + "95": 1206.0, + "96": 1289.0, + "97": 1249.0, + "98": 1396.0, + "99": 1185.0, + "100": 1352.0 } }, "mem-allocated-bytes": { diff --git a/tests/functional_tests/test_cases/moe/gpt_grpo_tp8tp4_pp1_ep8ep2_dp8_throughputtest/golden_values_dev_dgx_h100.json b/tests/functional_tests/test_cases/moe/gpt_grpo_tp8tp4_pp1_ep8ep2_dp8_throughputtest/golden_values_dev_dgx_h100.json index 8bcdb8c5b4c..dd2f91b006c 100644 --- a/tests/functional_tests/test_cases/moe/gpt_grpo_tp8tp4_pp1_ep8ep2_dp8_throughputtest/golden_values_dev_dgx_h100.json +++ b/tests/functional_tests/test_cases/moe/gpt_grpo_tp8tp4_pp1_ep8ep2_dp8_throughputtest/golden_values_dev_dgx_h100.json @@ -51,11 +51,12 @@ "start_step": 2, "end_step": 5, "step_interval": 1, + "value_precision": "full", "values": { - "2": 136.37686, - "3": 88.63863, - "4": 88.5902, - "5": 84.06419 + "2": 82.9230728149414, + "3": 75.13343048095703, + "4": 75.17157745361328, + "5": 75.31648254394531 } } -} \ No newline at end of file +} diff --git a/tests/functional_tests/test_cases/t5/t5_mcore_tp1_pp1_vp1/golden_values_dev_dgx_a100.json b/tests/functional_tests/test_cases/t5/t5_mcore_tp1_pp1_vp1/golden_values_dev_dgx_a100.json index 693b9573c0b..fe4bc5e998d 100644 --- a/tests/functional_tests/test_cases/t5/t5_mcore_tp1_pp1_vp1/golden_values_dev_dgx_a100.json +++ b/tests/functional_tests/test_cases/t5/t5_mcore_tp1_pp1_vp1/golden_values_dev_dgx_a100.json @@ -3,534 +3,539 @@ "start_step": 1, "end_step": 100, "step_interval": 1, + "value_precision": "full", "values": { - "1": 10.36194, - "2": 10.37963, - "3": 9.57157, - "4": 9.34779, - "5": 9.24781, - "6": 9.24354, - "7": 9.13399, - "8": 9.01793, - "9": 8.92153, - "10": 8.85612, - "11": 8.74636, - "12": 8.65915, - "13": 8.56447, - "14": 8.49783, - "15": 8.45057, - "16": 8.4692, - "17": 8.24096, - "18": 8.1757, - "19": 8.23258, - "20": 8.11454, - "21": 8.01768, - "22": 7.95726, - "23": 8.0022, - "24": 7.85433, - "25": 7.82433, - "26": 7.7775, - "27": 7.6737, - "28": 7.65052, - "29": 7.53339, - "30": 7.52886, - "31": 7.41641, - "32": 7.47985, - "33": 7.41816, - "34": 7.34661, - "35": 7.30379, - "36": 7.26311, - "37": 7.244, - "38": 7.25481, - "39": 7.23405, - "40": 7.12315, - "41": 7.07014, - "42": 7.12844, - "43": 7.10283, - "44": 7.02411, - "45": 7.03502, - "46": 7.05011, - "47": 7.00399, - "48": 7.01285, - "49": 6.89613, - "50": 6.95483, - "51": 7.0679, - "52": 6.9547, - "53": 6.87937, - "54": 6.86682, - "55": 6.8473, - "56": 6.93259, - "57": 6.89168, - "58": 6.81314, - "59": 6.87193, - "60": 6.72813, - "61": 6.86536, - "62": 6.80113, - "63": 6.8588, - "64": 6.897, - "65": 6.76557, - "66": 6.78696, - "67": 6.6914, - "68": 6.70923, - "69": 6.85508, - "70": 6.65606, - "71": 6.82966, - "72": 6.76407, - "73": 6.79167, - "74": 6.72844, - "75": 6.70885, - "76": 6.80416, - "77": 6.74476, - "78": 6.67336, - "79": 6.78812, - "80": 6.82167, - "81": 6.78562, - "82": 6.71203, - "83": 6.7679, - "84": 6.74299, - "85": 6.72236, - "86": 6.69797, - "87": 6.75281, - "88": 6.75082, - "89": 6.70752, - "90": 6.57338, - "91": 6.69236, - "92": 6.68392, - "93": 6.72683, - "94": 6.7389, - "95": 6.67316, - "96": 6.77326, - "97": 6.78148, - "98": 6.64687, - "99": 6.64064, - "100": 6.7525 + "1": 10.36181926727295, + "2": 10.379711151123047, + "3": 9.571847915649414, + "4": 9.348028182983398, + "5": 9.247902870178223, + "6": 9.243851661682129, + "7": 9.134450912475586, + "8": 9.018454551696777, + "9": 8.922161102294922, + "10": 8.856827735900879, + "11": 8.747197151184082, + "12": 8.660162925720215, + "13": 8.5656099319458, + "14": 8.499092102050781, + "15": 8.452003479003906, + "16": 8.47075080871582, + "17": 8.242928504943848, + "18": 8.17794418334961, + "19": 8.234527587890625, + "20": 8.116927146911621, + "21": 8.020139694213867, + "22": 7.96027946472168, + "23": 8.005057334899902, + "24": 7.856324672698975, + "25": 7.825979709625244, + "26": 7.77891206741333, + "27": 7.675104141235352, + "28": 7.651978969573975, + "29": 7.534967422485352, + "30": 7.530463218688965, + "31": 7.418205261230469, + "32": 7.481710910797119, + "33": 7.42007303237915, + "34": 7.348446846008301, + "35": 7.30583381652832, + "36": 7.265338897705078, + "37": 7.246328830718994, + "38": 7.2570576667785645, + "39": 7.236330032348633, + "40": 7.125462055206299, + "41": 7.072636127471924, + "42": 7.131015777587891, + "43": 7.105363845825195, + "44": 7.026730537414551, + "45": 7.037662506103516, + "46": 7.051845550537109, + "47": 7.003983020782471, + "48": 7.011589050292969, + "49": 6.894206523895264, + "50": 6.953387260437012, + "51": 7.066490650177002, + "52": 6.953217029571533, + "53": 6.878506183624268, + "54": 6.8655266761779785, + "55": 6.84617805480957, + "56": 6.931855201721191, + "57": 6.890820026397705, + "58": 6.812288761138916, + "59": 6.8713531494140625, + "60": 6.727507591247559, + "61": 6.864716053009033, + "62": 6.80047082901001, + "63": 6.858138084411621, + "64": 6.896549224853516, + "65": 6.764918327331543, + "66": 6.786929130554199, + "67": 6.691226482391357, + "68": 6.709066867828369, + "69": 6.855501174926758, + "70": 6.655716896057129, + "71": 6.829084396362305, + "72": 6.76472806930542, + "73": 6.791973114013672, + "74": 6.728721618652344, + "75": 6.7094011306762695, + "76": 6.804281234741211, + "77": 6.744887828826904, + "78": 6.673455715179443, + "79": 6.788580417633057, + "80": 6.822170257568359, + "81": 6.785739421844482, + "82": 6.711966037750244, + "83": 6.768506050109863, + "84": 6.743620872497559, + "85": 6.723179817199707, + "86": 6.698517799377441, + "87": 6.75265645980835, + "88": 6.751332759857178, + "89": 6.708575248718262, + "90": 6.574117660522461, + "91": 6.692963600158691, + "92": 6.684823989868164, + "93": 6.72745943069458, + "94": 6.738877773284912, + "95": 6.674159526824951, + "96": 6.774349212646484, + "97": 6.782050132751465, + "98": 6.647655487060547, + "99": 6.640959739685059, + "100": 6.752841949462891 } }, "num-zeros": { "start_step": 1, "end_step": 100, "step_interval": 1, + "value_precision": "full", "values": { - "1": 41111.0, - "2": 41080.0, - "3": 43307.0, - "4": 47017.0, - "5": 43278.0, - "6": 45510.0, - "7": 47019.0, - "8": 44088.0, - "9": 43997.0, - "10": 45512.0, - "11": 45366.0, - "12": 45399.0, - "13": 44014.0, - "14": 41867.0, - "15": 41881.0, - "16": 44790.0, - "17": 43908.0, - "18": 45398.0, - "19": 44043.0, - "20": 46402.0, - "21": 44051.0, - "22": 45488.0, - "23": 41884.0, - "24": 43221.0, + "1": 41109.0, + "2": 41076.0, + "3": 43311.0, + "4": 47030.0, + "5": 43279.0, + "6": 45519.0, + "7": 46998.0, + "8": 44074.0, + "9": 44014.0, + "10": 45520.0, + "11": 45357.0, + "12": 45411.0, + "13": 44018.0, + "14": 41880.0, + "15": 41864.0, + "16": 44794.0, + "17": 43887.0, + "18": 45412.0, + "19": 44060.0, + "20": 46416.0, + "21": 44087.0, + "22": 45501.0, + "23": 41887.0, + "24": 43224.0, "25": 46967.0, - "26": 43275.0, - "27": 44644.0, - "28": 43492.0, - "29": 44900.0, - "30": 45536.0, - "31": 44650.0, - "32": 46241.0, - "33": 45654.0, - "34": 47069.0, - "35": 43247.0, - "36": 46993.0, - "37": 45586.0, - "38": 42458.0, - "39": 45550.0, - "40": 46931.0, - "41": 43401.0, - "42": 45549.0, - "43": 46263.0, - "44": 43354.0, - "45": 43162.0, - "46": 44667.0, - "47": 43339.0, - "48": 44674.0, - "49": 43885.0, - "50": 45399.0, - "51": 41965.0, - "52": 42024.0, - "53": 47069.0, - "54": 44803.0, - "55": 44011.0, - "56": 44075.0, - "57": 43441.0, - "58": 44683.0, - "59": 42440.0, - "60": 43149.0, - "61": 44799.0, - "62": 43926.0, - "63": 42735.0, - "64": 44043.0, - "65": 45540.0, - "66": 42799.0, - "67": 43995.0, - "68": 44047.0, - "69": 41621.0, - "70": 41708.0, - "71": 45522.0, - "72": 45406.0, - "73": 42724.0, + "26": 43285.0, + "27": 44639.0, + "28": 43493.0, + "29": 44891.0, + "30": 45542.0, + "31": 44639.0, + "32": 46229.0, + "33": 45638.0, + "34": 47081.0, + "35": 43246.0, + "36": 47017.0, + "37": 45597.0, + "38": 42446.0, + "39": 45546.0, + "40": 46926.0, + "41": 43391.0, + "42": 45569.0, + "43": 46261.0, + "44": 43357.0, + "45": 43160.0, + "46": 44659.0, + "47": 43358.0, + "48": 44648.0, + "49": 43889.0, + "50": 45406.0, + "51": 41970.0, + "52": 42000.0, + "53": 47099.0, + "54": 44783.0, + "55": 44005.0, + "56": 44074.0, + "57": 43445.0, + "58": 44701.0, + "59": 42435.0, + "60": 43144.0, + "61": 44811.0, + "62": 43936.0, + "63": 42724.0, + "64": 44028.0, + "65": 45537.0, + "66": 42785.0, + "67": 43992.0, + "68": 44051.0, + "69": 41618.0, + "70": 41715.0, + "71": 45529.0, + "72": 45401.0, + "73": 42715.0, "74": 44791.0, - "75": 43937.0, - "76": 43339.0, - "77": 41680.0, - "78": 43359.0, - "79": 45413.0, - "80": 45496.0, - "81": 42481.0, - "82": 39185.0, - "83": 45709.0, - "84": 43466.0, - "85": 44107.0, - "86": 43893.0, - "87": 41280.0, - "88": 44144.0, - "89": 40454.0, - "90": 39913.0, - "91": 46240.0, - "92": 46277.0, - "93": 43891.0, - "94": 45462.0, - "95": 41787.0, - "96": 41423.0, - "97": 42034.0, - "98": 43407.0, - "99": 39064.0, - "100": 39415.0 + "75": 43934.0, + "76": 43341.0, + "77": 41685.0, + "78": 43356.0, + "79": 45415.0, + "80": 45487.0, + "81": 42491.0, + "82": 39180.0, + "83": 45713.0, + "84": 43477.0, + "85": 44127.0, + "86": 43883.0, + "87": 41313.0, + "88": 44135.0, + "89": 40445.0, + "90": 39898.0, + "91": 46253.0, + "92": 46261.0, + "93": 43871.0, + "94": 45482.0, + "95": 41790.0, + "96": 41418.0, + "97": 42029.0, + "98": 43402.0, + "99": 39066.0, + "100": 39412.0 } }, "mem-allocated-bytes": { "start_step": 1, "end_step": 100, "step_interval": 1, + "value_precision": "full", "values": { - "1": 4159301120.0, - "2": 4159301120.0, - "3": 4159301120.0, - "4": 4159301120.0, - "5": 4159301120.0, - "6": 4159301120.0, - "7": 4159301120.0, - "8": 4159301120.0, - "9": 4159301120.0, - "10": 4159301120.0, - "11": 4159301120.0, - "12": 4159301120.0, - "13": 4159301120.0, - "14": 4159301120.0, - "15": 4159301120.0, - "16": 4159301120.0, - "17": 4159301120.0, - "18": 4159301120.0, - "19": 4159301120.0, - "20": 4159301120.0, - "21": 4159301120.0, - "22": 4159301120.0, - "23": 4159301120.0, - "24": 4159301120.0, - "25": 4159301120.0, - "26": 4159301120.0, - "27": 4159301120.0, - "28": 4159301120.0, - "29": 4159301120.0, - "30": 4159301120.0, - "31": 4159301120.0, - "32": 4159301120.0, - "33": 4159301120.0, - "34": 4159301120.0, - "35": 4159301120.0, - "36": 4159301120.0, - "37": 4159301120.0, - "38": 4159301120.0, - "39": 4159301120.0, - "40": 4159301120.0, - "41": 4159301120.0, - "42": 4159301120.0, - "43": 4159301120.0, - "44": 4159301120.0, - "45": 4159301120.0, - "46": 4159301120.0, - "47": 4159301120.0, - "48": 4159301120.0, - "49": 4159301120.0, - "50": 4159301120.0, - "51": 4159301120.0, - "52": 4159301120.0, - "53": 4159301120.0, - "54": 4159301120.0, - "55": 4159301120.0, - "56": 4159301120.0, - "57": 4159301120.0, - "58": 4159301120.0, - "59": 4159301120.0, - "60": 4159301120.0, - "61": 4159301120.0, - "62": 4159301120.0, - "63": 4159301120.0, - "64": 4159301120.0, - "65": 4159301120.0, - "66": 4159301120.0, - "67": 4159301120.0, - "68": 4159301120.0, - "69": 4159301120.0, - "70": 4159301120.0, - "71": 4159301120.0, - "72": 4159301120.0, - "73": 4159301120.0, - "74": 4159301120.0, - "75": 4159301120.0, - "76": 4159301120.0, - "77": 4159301120.0, - "78": 4159301120.0, - "79": 4159301120.0, - "80": 4159301120.0, - "81": 4159301120.0, - "82": 4159301120.0, - "83": 4159301120.0, - "84": 4159301120.0, - "85": 4159301120.0, - "86": 4159301120.0, - "87": 4159301120.0, - "88": 4159301120.0, - "89": 4159301120.0, - "90": 4159301120.0, - "91": 4159301120.0, - "92": 4159301120.0, - "93": 4159301120.0, - "94": 4159301120.0, - "95": 4159301120.0, - "96": 4159301120.0, - "97": 4159301120.0, - "98": 4159301120.0, - "99": 4159301120.0, - "100": 4159301120.0 + "1": 4167822848.0, + "2": 4167822848.0, + "3": 4167822848.0, + "4": 4167822848.0, + "5": 4167822848.0, + "6": 4167822848.0, + "7": 4167822848.0, + "8": 4167822848.0, + "9": 4167822848.0, + "10": 4167822848.0, + "11": 4167822848.0, + "12": 4167822848.0, + "13": 4167822848.0, + "14": 4167822848.0, + "15": 4167822848.0, + "16": 4167822848.0, + "17": 4167822848.0, + "18": 4167822848.0, + "19": 4167822848.0, + "20": 4167822848.0, + "21": 4167822848.0, + "22": 4167822848.0, + "23": 4167822848.0, + "24": 4167822848.0, + "25": 4167822848.0, + "26": 4167822848.0, + "27": 4167822848.0, + "28": 4167822848.0, + "29": 4167822848.0, + "30": 4167822848.0, + "31": 4167822848.0, + "32": 4167822848.0, + "33": 4167822848.0, + "34": 4167822848.0, + "35": 4167822848.0, + "36": 4167822848.0, + "37": 4167822848.0, + "38": 4167822848.0, + "39": 4167822848.0, + "40": 4167822848.0, + "41": 4167822848.0, + "42": 4167822848.0, + "43": 4167822848.0, + "44": 4167822848.0, + "45": 4167822848.0, + "46": 4167822848.0, + "47": 4167822848.0, + "48": 4167822848.0, + "49": 4167822848.0, + "50": 4167822848.0, + "51": 4167822848.0, + "52": 4167822848.0, + "53": 4167822848.0, + "54": 4167822848.0, + "55": 4167822848.0, + "56": 4167822848.0, + "57": 4167822848.0, + "58": 4167822848.0, + "59": 4167822848.0, + "60": 4167822848.0, + "61": 4167822848.0, + "62": 4167822848.0, + "63": 4167822848.0, + "64": 4167822848.0, + "65": 4167822848.0, + "66": 4167822848.0, + "67": 4167822848.0, + "68": 4167822848.0, + "69": 4167822848.0, + "70": 4167822848.0, + "71": 4167822848.0, + "72": 4167822848.0, + "73": 4167822848.0, + "74": 4167822848.0, + "75": 4167822848.0, + "76": 4167822848.0, + "77": 4167822848.0, + "78": 4167822848.0, + "79": 4167822848.0, + "80": 4167822848.0, + "81": 4167822848.0, + "82": 4167822848.0, + "83": 4167822848.0, + "84": 4167822848.0, + "85": 4167822848.0, + "86": 4167822848.0, + "87": 4167822848.0, + "88": 4167822848.0, + "89": 4167822848.0, + "90": 4167822848.0, + "91": 4167822848.0, + "92": 4167822848.0, + "93": 4167822848.0, + "94": 4167822848.0, + "95": 4167822848.0, + "96": 4167822848.0, + "97": 4167822848.0, + "98": 4167822848.0, + "99": 4167822848.0, + "100": 4167822848.0 } }, "mem-max-allocated-bytes": { "start_step": 1, "end_step": 100, "step_interval": 1, + "value_precision": "full", "values": { - "1": 4369633792.0, - "2": 6189653504.0, - "3": 6189653504.0, - "4": 6189653504.0, - "5": 6189653504.0, - "6": 6189653504.0, - "7": 6189653504.0, - "8": 6189653504.0, - "9": 6189653504.0, - "10": 6189653504.0, - "11": 6189653504.0, - "12": 6189653504.0, - "13": 6189653504.0, - "14": 6189653504.0, - "15": 6189653504.0, - "16": 6189653504.0, - "17": 6189653504.0, - "18": 6189653504.0, - "19": 6189653504.0, - "20": 6189653504.0, - "21": 6189653504.0, - "22": 6189653504.0, - "23": 6189653504.0, - "24": 6189653504.0, - "25": 6189653504.0, - "26": 6189653504.0, - "27": 6189653504.0, - "28": 6189653504.0, - "29": 6189653504.0, - "30": 6189653504.0, - "31": 6189653504.0, - "32": 6189653504.0, - "33": 6189653504.0, - "34": 6189653504.0, - "35": 6189653504.0, - "36": 6189653504.0, - "37": 6189653504.0, - "38": 6189653504.0, - "39": 6189653504.0, - "40": 6189653504.0, - "41": 6189653504.0, - "42": 6189653504.0, - "43": 6189653504.0, - "44": 6189653504.0, - "45": 6189653504.0, - "46": 6189653504.0, - "47": 6189653504.0, - "48": 6189653504.0, - "49": 6189653504.0, - "50": 6189653504.0, - "51": 6189653504.0, - "52": 6189653504.0, - "53": 6189653504.0, - "54": 6189653504.0, - "55": 6189653504.0, - "56": 6189653504.0, - "57": 6189653504.0, - "58": 6189653504.0, - "59": 6189653504.0, - "60": 6189653504.0, - "61": 6189653504.0, - "62": 6189653504.0, - "63": 6189653504.0, - "64": 6189653504.0, - "65": 6189653504.0, - "66": 6189653504.0, - "67": 6189653504.0, - "68": 6189653504.0, - "69": 6189653504.0, - "70": 6189653504.0, - "71": 6189653504.0, - "72": 6189653504.0, - "73": 6189653504.0, - "74": 6189653504.0, - "75": 6189653504.0, - "76": 6189653504.0, - "77": 6189653504.0, - "78": 6189653504.0, - "79": 6189653504.0, - "80": 6189653504.0, - "81": 6189653504.0, - "82": 6189653504.0, - "83": 6189653504.0, - "84": 6189653504.0, - "85": 6189653504.0, - "86": 6189653504.0, - "87": 6189653504.0, - "88": 6189653504.0, - "89": 6189653504.0, - "90": 6189653504.0, - "91": 6189653504.0, - "92": 6189653504.0, - "93": 6189653504.0, - "94": 6189653504.0, - "95": 6189653504.0, - "96": 6189653504.0, - "97": 6189653504.0, - "98": 6189653504.0, - "99": 6189653504.0, - "100": 6189653504.0 + "1": 4379204096.0, + "2": 6197650944.0, + "3": 6197650944.0, + "4": 6197650944.0, + "5": 6197650944.0, + "6": 6197650944.0, + "7": 6197650944.0, + "8": 6197650944.0, + "9": 6197650944.0, + "10": 6197650944.0, + "11": 6197650944.0, + "12": 6197650944.0, + "13": 6197650944.0, + "14": 6197650944.0, + "15": 6197650944.0, + "16": 6197650944.0, + "17": 6197650944.0, + "18": 6197650944.0, + "19": 6197650944.0, + "20": 6197650944.0, + "21": 6197650944.0, + "22": 6197650944.0, + "23": 6197650944.0, + "24": 6197650944.0, + "25": 6197650944.0, + "26": 6197650944.0, + "27": 6197650944.0, + "28": 6197650944.0, + "29": 6197650944.0, + "30": 6197650944.0, + "31": 6197650944.0, + "32": 6197650944.0, + "33": 6197650944.0, + "34": 6197650944.0, + "35": 6197650944.0, + "36": 6197650944.0, + "37": 6197650944.0, + "38": 6197650944.0, + "39": 6197650944.0, + "40": 6197650944.0, + "41": 6197650944.0, + "42": 6197650944.0, + "43": 6197650944.0, + "44": 6197650944.0, + "45": 6197650944.0, + "46": 6197650944.0, + "47": 6197650944.0, + "48": 6197650944.0, + "49": 6197650944.0, + "50": 6197650944.0, + "51": 6197650944.0, + "52": 6197650944.0, + "53": 6197650944.0, + "54": 6197650944.0, + "55": 6197650944.0, + "56": 6197650944.0, + "57": 6197650944.0, + "58": 6197650944.0, + "59": 6197650944.0, + "60": 6197650944.0, + "61": 6197650944.0, + "62": 6197650944.0, + "63": 6197650944.0, + "64": 6197650944.0, + "65": 6197650944.0, + "66": 6197650944.0, + "67": 6197650944.0, + "68": 6197650944.0, + "69": 6197650944.0, + "70": 6197650944.0, + "71": 6197650944.0, + "72": 6197650944.0, + "73": 6197650944.0, + "74": 6197650944.0, + "75": 6197650944.0, + "76": 6197650944.0, + "77": 6197650944.0, + "78": 6197650944.0, + "79": 6197650944.0, + "80": 6197650944.0, + "81": 6197650944.0, + "82": 6197650944.0, + "83": 6197650944.0, + "84": 6197650944.0, + "85": 6197650944.0, + "86": 6197650944.0, + "87": 6197650944.0, + "88": 6197650944.0, + "89": 6197650944.0, + "90": 6197650944.0, + "91": 6197650944.0, + "92": 6197650944.0, + "93": 6197650944.0, + "94": 6197650944.0, + "95": 6197650944.0, + "96": 6197650944.0, + "97": 6197650944.0, + "98": 6197650944.0, + "99": 6197650944.0, + "100": 6197650944.0 } }, "iteration-time": { "start_step": 2, "end_step": 100, "step_interval": 1, + "value_precision": "full", "values": { - "2": 4.86664, - "3": 0.22902, - "4": 0.21131, - "5": 0.2126, - "6": 0.21457, - "7": 0.21959, - "8": 0.21152, - "9": 0.2179, - "10": 0.21182, - "11": 0.20998, - "12": 0.21136, - "13": 0.20911, - "14": 0.21006, - "15": 0.2084, - "16": 0.21247, - "17": 0.20521, - "18": 0.2083, - "19": 0.20935, - "20": 0.20753, - "21": 0.20772, - "22": 0.21059, - "23": 0.21039, - "24": 0.20864, - "25": 0.20938, - "26": 0.20938, - "27": 0.20912, - "28": 0.20967, - "29": 0.21198, - "30": 0.20963, - "31": 0.21352, - "32": 0.20996, - "33": 0.2089, - "34": 0.2134, - "35": 0.20921, - "36": 0.22817, - "37": 0.20779, - "38": 0.21152, - "39": 0.21784, - "40": 0.21019, - "41": 0.2145, - "42": 0.20613, - "43": 0.20716, - "44": 0.20886, - "45": 0.22106, - "46": 0.21104, - "47": 0.20974, - "48": 0.20849, - "49": 0.20945, - "50": 0.20875, - "51": 0.20964, - "52": 0.2188, - "53": 0.20842, - "54": 0.20988, - "55": 0.21024, - "56": 0.20931, - "57": 0.20801, - "58": 0.20954, - "59": 0.22212, - "60": 0.20713, - "61": 0.20909, - "62": 0.79419, - "63": 0.20911, - "64": 0.21414, - "65": 0.20797, - "66": 0.20918, - "67": 0.21433, - "68": 0.20897, - "69": 0.20975, - "70": 0.20894, - "71": 0.20779, - "72": 0.21383, - "73": 1.07789, - "74": 0.21173, - "75": 0.2072, - "76": 0.73249, - "77": 0.79733, - "78": 0.20998, - "79": 0.209, - "80": 0.81438, - "81": 1.60855, - "82": 0.21083, - "83": 0.21098, - "84": 0.20755, - "85": 0.21075, - "86": 0.21374, - "87": 0.2075, - "88": 0.20905, - "89": 0.21411, - "90": 0.20876, - "91": 0.20863, - "92": 0.20877, - "93": 0.21077, - "94": 0.21048, - "95": 0.2106, - "96": 0.20765, - "97": 0.21615, - "98": 0.21014, - "99": 0.20802, - "100": 0.21158 + "2": 0.38613247871398926, + "3": 0.23304009437561035, + "4": 0.21553325653076172, + "5": 0.2142353057861328, + "6": 0.21747708320617676, + "7": 0.2254467010498047, + "8": 0.2115767002105713, + "9": 0.2113022804260254, + "10": 0.22849345207214355, + "11": 0.21188592910766602, + "12": 0.215728759765625, + "13": 0.21613287925720215, + "14": 0.2170240879058838, + "15": 0.21816587448120117, + "16": 0.21091055870056152, + "17": 0.21190690994262695, + "18": 0.21505212783813477, + "19": 0.21269607543945312, + "20": 0.29738450050354004, + "21": 0.21100497245788574, + "22": 0.21803998947143555, + "23": 0.2164146900177002, + "24": 0.211226224899292, + "25": 0.2152726650238037, + "26": 0.21934723854064941, + "27": 0.2299816608428955, + "28": 0.22119379043579102, + "29": 0.21620488166809082, + "30": 0.20897579193115234, + "31": 0.21127796173095703, + "32": 0.2130906581878662, + "33": 0.21341371536254883, + "34": 0.21419167518615723, + "35": 0.20996904373168945, + "36": 0.21297073364257812, + "37": 0.21014165878295898, + "38": 0.2096109390258789, + "39": 0.21566224098205566, + "40": 0.23972725868225098, + "41": 0.21039199829101562, + "42": 0.2088146209716797, + "43": 0.21059489250183105, + "44": 0.2122201919555664, + "45": 0.2117445468902588, + "46": 0.21006369590759277, + "47": 0.2108323574066162, + "48": 0.2099437713623047, + "49": 0.20887279510498047, + "50": 0.20992231369018555, + "51": 0.20900249481201172, + "52": 0.21776056289672852, + "53": 0.21359515190124512, + "54": 0.20955395698547363, + "55": 0.21065735816955566, + "56": 0.21002960205078125, + "57": 0.20943164825439453, + "58": 0.21163630485534668, + "59": 0.21048760414123535, + "60": 0.21137380599975586, + "61": 0.2158064842224121, + "62": 0.25040459632873535, + "63": 0.21120381355285645, + "64": 0.21300411224365234, + "65": 0.21067357063293457, + "66": 0.21258234977722168, + "67": 0.21003222465515137, + "68": 0.7999699115753174, + "69": 0.2137761116027832, + "70": 0.896709680557251, + "71": 1.1601347923278809, + "72": 0.21590542793273926, + "73": 1.120039701461792, + "74": 0.2126932144165039, + "75": 0.21217632293701172, + "76": 0.2104959487915039, + "77": 0.7878854274749756, + "78": 0.21068811416625977, + "79": 0.8267481327056885, + "80": 0.2093491554260254, + "81": 1.156015396118164, + "82": 1.0974938869476318, + "83": 0.2163681983947754, + "84": 0.21170330047607422, + "85": 0.20969796180725098, + "86": 0.21346139907836914, + "87": 0.20920968055725098, + "88": 0.21400904655456543, + "89": 0.21292591094970703, + "90": 0.21260356903076172, + "91": 0.2131180763244629, + "92": 0.21260523796081543, + "93": 0.2128307819366455, + "94": 0.2082347869873047, + "95": 0.2099907398223877, + "96": 0.2117617130279541, + "97": 0.21394968032836914, + "98": 0.21042561531066895, + "99": 0.21189141273498535, + "100": 0.21176791191101074 } } } \ No newline at end of file diff --git a/tests/functional_tests/test_cases/t5/t5_mcore_tp1_pp1_vp1_resume_torch/golden_values_dev_dgx_a100.json b/tests/functional_tests/test_cases/t5/t5_mcore_tp1_pp1_vp1_resume_torch/golden_values_dev_dgx_a100.json index a5fff708d16..1959753671b 100644 --- a/tests/functional_tests/test_cases/t5/t5_mcore_tp1_pp1_vp1_resume_torch/golden_values_dev_dgx_a100.json +++ b/tests/functional_tests/test_cases/t5/t5_mcore_tp1_pp1_vp1_resume_torch/golden_values_dev_dgx_a100.json @@ -3,534 +3,539 @@ "start_step": 1, "end_step": 100, "step_interval": 1, + "value_precision": "full", "values": { - "1": 10.36194, - "2": 10.37963, - "3": 9.57157, - "4": 9.34779, - "5": 9.24781, - "6": 9.24354, - "7": 9.13399, - "8": 9.01793, - "9": 8.92153, - "10": 8.85612, - "11": 8.74636, - "12": 8.65915, - "13": 8.56447, - "14": 8.49783, - "15": 8.45057, - "16": 8.4692, - "17": 8.24096, - "18": 8.1757, - "19": 8.23258, - "20": 8.11454, - "21": 8.01768, - "22": 7.95726, - "23": 8.0022, - "24": 7.85433, - "25": 7.82433, - "26": 7.7775, - "27": 7.6737, - "28": 7.65052, - "29": 7.53339, - "30": 7.52886, - "31": 7.41641, - "32": 7.47985, - "33": 7.41816, - "34": 7.34661, - "35": 7.30379, - "36": 7.26311, - "37": 7.244, - "38": 7.25481, - "39": 7.23405, - "40": 7.12315, - "41": 7.07014, - "42": 7.12844, - "43": 7.10283, - "44": 7.02411, - "45": 7.03502, - "46": 7.05011, - "47": 7.00399, - "48": 7.01285, - "49": 6.89613, - "50": 6.95483, - "51": 7.0679, - "52": 6.9547, - "53": 6.87937, - "54": 6.86682, - "55": 6.8473, - "56": 6.93259, - "57": 6.89168, - "58": 6.81314, - "59": 6.87193, - "60": 6.72813, - "61": 6.86536, - "62": 6.80113, - "63": 6.8588, - "64": 6.897, - "65": 6.76557, - "66": 6.78696, - "67": 6.6914, - "68": 6.70923, - "69": 6.85508, - "70": 6.65606, - "71": 6.82966, - "72": 6.76407, - "73": 6.79167, - "74": 6.72844, - "75": 6.70885, - "76": 6.80416, - "77": 6.74476, - "78": 6.67336, - "79": 6.78812, - "80": 6.82167, - "81": 6.78562, - "82": 6.71203, - "83": 6.7679, - "84": 6.74299, - "85": 6.72236, - "86": 6.69797, - "87": 6.75281, - "88": 6.75082, - "89": 6.70752, - "90": 6.57338, - "91": 6.69236, - "92": 6.68392, - "93": 6.72683, - "94": 6.7389, - "95": 6.67316, - "96": 6.77326, - "97": 6.78148, - "98": 6.64687, - "99": 6.64064, - "100": 6.7525 + "1": 10.36181926727295, + "2": 10.379711151123047, + "3": 9.571847915649414, + "4": 9.348028182983398, + "5": 9.247902870178223, + "6": 9.243851661682129, + "7": 9.134450912475586, + "8": 9.018454551696777, + "9": 8.922161102294922, + "10": 8.856827735900879, + "11": 8.747197151184082, + "12": 8.660162925720215, + "13": 8.5656099319458, + "14": 8.499092102050781, + "15": 8.452003479003906, + "16": 8.47075080871582, + "17": 8.242928504943848, + "18": 8.17794418334961, + "19": 8.234527587890625, + "20": 8.116927146911621, + "21": 8.020139694213867, + "22": 7.96027946472168, + "23": 8.005057334899902, + "24": 7.856324672698975, + "25": 7.825979709625244, + "26": 7.77891206741333, + "27": 7.675104141235352, + "28": 7.651978969573975, + "29": 7.534967422485352, + "30": 7.530463218688965, + "31": 7.418205261230469, + "32": 7.481710910797119, + "33": 7.42007303237915, + "34": 7.348446846008301, + "35": 7.30583381652832, + "36": 7.265338897705078, + "37": 7.246328830718994, + "38": 7.2570576667785645, + "39": 7.236330032348633, + "40": 7.125462055206299, + "41": 7.072636127471924, + "42": 7.131015777587891, + "43": 7.105363845825195, + "44": 7.026730537414551, + "45": 7.037662506103516, + "46": 7.051845550537109, + "47": 7.003983020782471, + "48": 7.011589050292969, + "49": 6.894206523895264, + "50": 6.953387260437012, + "51": 7.066490650177002, + "52": 6.953217029571533, + "53": 6.878506183624268, + "54": 6.8655266761779785, + "55": 6.84617805480957, + "56": 6.931855201721191, + "57": 6.890820026397705, + "58": 6.812288761138916, + "59": 6.8713531494140625, + "60": 6.727507591247559, + "61": 6.864716053009033, + "62": 6.80047082901001, + "63": 6.858138084411621, + "64": 6.896549224853516, + "65": 6.764918327331543, + "66": 6.786929130554199, + "67": 6.691226482391357, + "68": 6.709066867828369, + "69": 6.855501174926758, + "70": 6.655716896057129, + "71": 6.829084396362305, + "72": 6.76472806930542, + "73": 6.791973114013672, + "74": 6.728721618652344, + "75": 6.7094011306762695, + "76": 6.804281234741211, + "77": 6.744887828826904, + "78": 6.673455715179443, + "79": 6.788580417633057, + "80": 6.822170257568359, + "81": 6.785739421844482, + "82": 6.711966037750244, + "83": 6.768506050109863, + "84": 6.743620872497559, + "85": 6.723179817199707, + "86": 6.698517799377441, + "87": 6.75265645980835, + "88": 6.751332759857178, + "89": 6.708575248718262, + "90": 6.574117660522461, + "91": 6.692963600158691, + "92": 6.684823989868164, + "93": 6.72745943069458, + "94": 6.738877773284912, + "95": 6.674159526824951, + "96": 6.774349212646484, + "97": 6.782050132751465, + "98": 6.647655487060547, + "99": 6.640959739685059, + "100": 6.752841949462891 } }, "num-zeros": { "start_step": 1, "end_step": 100, "step_interval": 1, + "value_precision": "full", "values": { - "1": 41111.0, - "2": 41080.0, - "3": 43307.0, - "4": 47017.0, - "5": 43278.0, - "6": 45510.0, - "7": 47019.0, - "8": 44088.0, - "9": 43997.0, - "10": 45512.0, - "11": 45366.0, - "12": 45399.0, - "13": 44014.0, - "14": 41867.0, - "15": 41881.0, - "16": 44790.0, - "17": 43908.0, - "18": 45398.0, - "19": 44043.0, - "20": 46402.0, - "21": 44051.0, - "22": 45488.0, - "23": 41884.0, - "24": 43221.0, + "1": 41109.0, + "2": 41076.0, + "3": 43311.0, + "4": 47030.0, + "5": 43279.0, + "6": 45519.0, + "7": 46998.0, + "8": 44074.0, + "9": 44014.0, + "10": 45520.0, + "11": 45357.0, + "12": 45411.0, + "13": 44018.0, + "14": 41880.0, + "15": 41864.0, + "16": 44794.0, + "17": 43887.0, + "18": 45412.0, + "19": 44060.0, + "20": 46416.0, + "21": 44087.0, + "22": 45501.0, + "23": 41887.0, + "24": 43224.0, "25": 46967.0, - "26": 43275.0, - "27": 44644.0, - "28": 43492.0, - "29": 44900.0, - "30": 45536.0, - "31": 44650.0, - "32": 46241.0, - "33": 45654.0, - "34": 47069.0, - "35": 43247.0, - "36": 46993.0, - "37": 45586.0, - "38": 42458.0, - "39": 45550.0, - "40": 46931.0, - "41": 43401.0, - "42": 45549.0, - "43": 46263.0, - "44": 43354.0, - "45": 43162.0, - "46": 44667.0, - "47": 43339.0, - "48": 44674.0, - "49": 43885.0, - "50": 45399.0, - "51": 41965.0, - "52": 42024.0, - "53": 47069.0, - "54": 44803.0, - "55": 44011.0, - "56": 44075.0, - "57": 43441.0, - "58": 44683.0, - "59": 42440.0, - "60": 43149.0, - "61": 44799.0, - "62": 43926.0, - "63": 42735.0, - "64": 44043.0, - "65": 45540.0, - "66": 42799.0, - "67": 43995.0, - "68": 44047.0, - "69": 41621.0, - "70": 41708.0, - "71": 45522.0, - "72": 45406.0, - "73": 42724.0, + "26": 43285.0, + "27": 44639.0, + "28": 43493.0, + "29": 44891.0, + "30": 45542.0, + "31": 44639.0, + "32": 46229.0, + "33": 45638.0, + "34": 47081.0, + "35": 43246.0, + "36": 47017.0, + "37": 45597.0, + "38": 42446.0, + "39": 45546.0, + "40": 46926.0, + "41": 43391.0, + "42": 45569.0, + "43": 46261.0, + "44": 43357.0, + "45": 43160.0, + "46": 44659.0, + "47": 43358.0, + "48": 44648.0, + "49": 43889.0, + "50": 45406.0, + "51": 41970.0, + "52": 42000.0, + "53": 47099.0, + "54": 44783.0, + "55": 44005.0, + "56": 44074.0, + "57": 43445.0, + "58": 44701.0, + "59": 42435.0, + "60": 43144.0, + "61": 44811.0, + "62": 43936.0, + "63": 42724.0, + "64": 44028.0, + "65": 45537.0, + "66": 42785.0, + "67": 43992.0, + "68": 44051.0, + "69": 41618.0, + "70": 41715.0, + "71": 45529.0, + "72": 45401.0, + "73": 42715.0, "74": 44791.0, - "75": 43937.0, - "76": 43339.0, - "77": 41680.0, - "78": 43359.0, - "79": 45413.0, - "80": 45496.0, - "81": 42481.0, - "82": 39185.0, - "83": 45709.0, - "84": 43466.0, - "85": 44107.0, - "86": 43893.0, - "87": 41280.0, - "88": 44144.0, - "89": 40454.0, - "90": 39913.0, - "91": 46240.0, - "92": 46277.0, - "93": 43891.0, - "94": 45462.0, - "95": 41787.0, - "96": 41423.0, - "97": 42034.0, - "98": 43407.0, - "99": 39064.0, - "100": 39415.0 + "75": 43934.0, + "76": 43341.0, + "77": 41685.0, + "78": 43356.0, + "79": 45415.0, + "80": 45487.0, + "81": 42491.0, + "82": 39180.0, + "83": 45713.0, + "84": 43477.0, + "85": 44127.0, + "86": 43883.0, + "87": 41313.0, + "88": 44135.0, + "89": 40445.0, + "90": 39898.0, + "91": 46253.0, + "92": 46261.0, + "93": 43871.0, + "94": 45482.0, + "95": 41790.0, + "96": 41418.0, + "97": 42029.0, + "98": 43402.0, + "99": 39066.0, + "100": 39412.0 } }, "mem-allocated-bytes": { "start_step": 1, "end_step": 100, "step_interval": 1, + "value_precision": "full", "values": { - "1": 4159301120.0, - "2": 4159301120.0, - "3": 4159301120.0, - "4": 4159301120.0, - "5": 4159301120.0, - "6": 4159301120.0, - "7": 4159301120.0, - "8": 4159301120.0, - "9": 4159301120.0, - "10": 4159301120.0, - "11": 4159301120.0, - "12": 4159301120.0, - "13": 4159301120.0, - "14": 4159301120.0, - "15": 4159301120.0, - "16": 4159301120.0, - "17": 4159301120.0, - "18": 4159301120.0, - "19": 4159301120.0, - "20": 4159301120.0, - "21": 4159301120.0, - "22": 4159301120.0, - "23": 4159301120.0, - "24": 4159301120.0, - "25": 4159301120.0, - "26": 4159301120.0, - "27": 4159301120.0, - "28": 4159301120.0, - "29": 4159301120.0, - "30": 4159301120.0, - "31": 4159301120.0, - "32": 4159301120.0, - "33": 4159301120.0, - "34": 4159301120.0, - "35": 4159301120.0, - "36": 4159301120.0, - "37": 4159301120.0, - "38": 4159301120.0, - "39": 4159301120.0, - "40": 4159301120.0, - "41": 4159301120.0, - "42": 4159301120.0, - "43": 4159301120.0, - "44": 4159301120.0, - "45": 4159301120.0, - "46": 4159301120.0, - "47": 4159301120.0, - "48": 4159301120.0, - "49": 4159301120.0, - "50": 4159301120.0, - "51": 4159301120.0, - "52": 4159301120.0, - "53": 4159301120.0, - "54": 4159301120.0, - "55": 4159301120.0, - "56": 4159301120.0, - "57": 4159301120.0, - "58": 4159301120.0, - "59": 4159301120.0, - "60": 4159301120.0, - "61": 4159301120.0, - "62": 4159301120.0, - "63": 4159301120.0, - "64": 4159301120.0, - "65": 4159301120.0, - "66": 4159301120.0, - "67": 4159301120.0, - "68": 4159301120.0, - "69": 4159301120.0, - "70": 4159301120.0, - "71": 4159301120.0, - "72": 4159301120.0, - "73": 4159301120.0, - "74": 4159301120.0, - "75": 4159301120.0, - "76": 4159301120.0, - "77": 4159301120.0, - "78": 4159301120.0, - "79": 4159301120.0, - "80": 4159301120.0, - "81": 4159301120.0, - "82": 4159301120.0, - "83": 4159301120.0, - "84": 4159301120.0, - "85": 4159301120.0, - "86": 4159301120.0, - "87": 4159301120.0, - "88": 4159301120.0, - "89": 4159301120.0, - "90": 4159301120.0, - "91": 4159301120.0, - "92": 4159301120.0, - "93": 4159301120.0, - "94": 4159301120.0, - "95": 4159301120.0, - "96": 4159301120.0, - "97": 4159301120.0, - "98": 4159301120.0, - "99": 4159301120.0, - "100": 4159301120.0 + "1": 4167822848.0, + "2": 4167822848.0, + "3": 4167822848.0, + "4": 4167822848.0, + "5": 4167822848.0, + "6": 4167822848.0, + "7": 4167822848.0, + "8": 4167822848.0, + "9": 4167822848.0, + "10": 4167822848.0, + "11": 4167822848.0, + "12": 4167822848.0, + "13": 4167822848.0, + "14": 4167822848.0, + "15": 4167822848.0, + "16": 4167822848.0, + "17": 4167822848.0, + "18": 4167822848.0, + "19": 4167822848.0, + "20": 4167822848.0, + "21": 4167822848.0, + "22": 4167822848.0, + "23": 4167822848.0, + "24": 4167822848.0, + "25": 4167822848.0, + "26": 4167822848.0, + "27": 4167822848.0, + "28": 4167822848.0, + "29": 4167822848.0, + "30": 4167822848.0, + "31": 4167822848.0, + "32": 4167822848.0, + "33": 4167822848.0, + "34": 4167822848.0, + "35": 4167822848.0, + "36": 4167822848.0, + "37": 4167822848.0, + "38": 4167822848.0, + "39": 4167822848.0, + "40": 4167822848.0, + "41": 4167822848.0, + "42": 4167822848.0, + "43": 4167822848.0, + "44": 4167822848.0, + "45": 4167822848.0, + "46": 4167822848.0, + "47": 4167822848.0, + "48": 4167822848.0, + "49": 4167822848.0, + "50": 4167822848.0, + "51": 4167822848.0, + "52": 4167822848.0, + "53": 4167822848.0, + "54": 4167822848.0, + "55": 4167822848.0, + "56": 4167822848.0, + "57": 4167822848.0, + "58": 4167822848.0, + "59": 4167822848.0, + "60": 4167822848.0, + "61": 4167822848.0, + "62": 4167822848.0, + "63": 4167822848.0, + "64": 4167822848.0, + "65": 4167822848.0, + "66": 4167822848.0, + "67": 4167822848.0, + "68": 4167822848.0, + "69": 4167822848.0, + "70": 4167822848.0, + "71": 4167822848.0, + "72": 4167822848.0, + "73": 4167822848.0, + "74": 4167822848.0, + "75": 4167822848.0, + "76": 4167822848.0, + "77": 4167822848.0, + "78": 4167822848.0, + "79": 4167822848.0, + "80": 4167822848.0, + "81": 4167822848.0, + "82": 4167822848.0, + "83": 4167822848.0, + "84": 4167822848.0, + "85": 4167822848.0, + "86": 4167822848.0, + "87": 4167822848.0, + "88": 4167822848.0, + "89": 4167822848.0, + "90": 4167822848.0, + "91": 4167822848.0, + "92": 4167822848.0, + "93": 4167822848.0, + "94": 4167822848.0, + "95": 4167822848.0, + "96": 4167822848.0, + "97": 4167822848.0, + "98": 4167822848.0, + "99": 4167822848.0, + "100": 4167822848.0 } }, "mem-max-allocated-bytes": { "start_step": 1, "end_step": 100, "step_interval": 1, + "value_precision": "full", "values": { - "1": 4369633792.0, - "2": 6189653504.0, - "3": 6189653504.0, - "4": 6189653504.0, - "5": 6189653504.0, - "6": 6189653504.0, - "7": 6189653504.0, - "8": 6189653504.0, - "9": 6189653504.0, - "10": 6189653504.0, - "11": 6189653504.0, - "12": 6189653504.0, - "13": 6189653504.0, - "14": 6189653504.0, - "15": 6189653504.0, - "16": 6189653504.0, - "17": 6189653504.0, - "18": 6189653504.0, - "19": 6189653504.0, - "20": 6189653504.0, - "21": 6189653504.0, - "22": 6189653504.0, - "23": 6189653504.0, - "24": 6189653504.0, - "25": 6189653504.0, - "26": 6189653504.0, - "27": 6189653504.0, - "28": 6189653504.0, - "29": 6189653504.0, - "30": 6189653504.0, - "31": 6189653504.0, - "32": 6189653504.0, - "33": 6189653504.0, - "34": 6189653504.0, - "35": 6189653504.0, - "36": 6189653504.0, - "37": 6189653504.0, - "38": 6189653504.0, - "39": 6189653504.0, - "40": 6189653504.0, - "41": 6189653504.0, - "42": 6189653504.0, - "43": 6189653504.0, - "44": 6189653504.0, - "45": 6189653504.0, - "46": 6189653504.0, - "47": 6189653504.0, - "48": 6189653504.0, - "49": 6189653504.0, - "50": 6189653504.0, - "51": 6189653504.0, - "52": 6189653504.0, - "53": 6189653504.0, - "54": 6189653504.0, - "55": 6189653504.0, - "56": 6189653504.0, - "57": 6189653504.0, - "58": 6189653504.0, - "59": 6189653504.0, - "60": 6189653504.0, - "61": 6189653504.0, - "62": 6189653504.0, - "63": 6189653504.0, - "64": 6189653504.0, - "65": 6189653504.0, - "66": 6189653504.0, - "67": 6189653504.0, - "68": 6189653504.0, - "69": 6189653504.0, - "70": 6189653504.0, - "71": 6189653504.0, - "72": 6189653504.0, - "73": 6189653504.0, - "74": 6189653504.0, - "75": 6189653504.0, - "76": 6189653504.0, - "77": 6189653504.0, - "78": 6189653504.0, - "79": 6189653504.0, - "80": 6189653504.0, - "81": 6189653504.0, - "82": 6189653504.0, - "83": 6189653504.0, - "84": 6189653504.0, - "85": 6189653504.0, - "86": 6189653504.0, - "87": 6189653504.0, - "88": 6189653504.0, - "89": 6189653504.0, - "90": 6189653504.0, - "91": 6189653504.0, - "92": 6189653504.0, - "93": 6189653504.0, - "94": 6189653504.0, - "95": 6189653504.0, - "96": 6189653504.0, - "97": 6189653504.0, - "98": 6189653504.0, - "99": 6189653504.0, - "100": 6189653504.0 + "1": 4379204096.0, + "2": 6197650944.0, + "3": 6197650944.0, + "4": 6197650944.0, + "5": 6197650944.0, + "6": 6197650944.0, + "7": 6197650944.0, + "8": 6197650944.0, + "9": 6197650944.0, + "10": 6197650944.0, + "11": 6197650944.0, + "12": 6197650944.0, + "13": 6197650944.0, + "14": 6197650944.0, + "15": 6197650944.0, + "16": 6197650944.0, + "17": 6197650944.0, + "18": 6197650944.0, + "19": 6197650944.0, + "20": 6197650944.0, + "21": 6197650944.0, + "22": 6197650944.0, + "23": 6197650944.0, + "24": 6197650944.0, + "25": 6197650944.0, + "26": 6197650944.0, + "27": 6197650944.0, + "28": 6197650944.0, + "29": 6197650944.0, + "30": 6197650944.0, + "31": 6197650944.0, + "32": 6197650944.0, + "33": 6197650944.0, + "34": 6197650944.0, + "35": 6197650944.0, + "36": 6197650944.0, + "37": 6197650944.0, + "38": 6197650944.0, + "39": 6197650944.0, + "40": 6197650944.0, + "41": 6197650944.0, + "42": 6197650944.0, + "43": 6197650944.0, + "44": 6197650944.0, + "45": 6197650944.0, + "46": 6197650944.0, + "47": 6197650944.0, + "48": 6197650944.0, + "49": 6197650944.0, + "50": 6197650944.0, + "51": 6197650944.0, + "52": 6197650944.0, + "53": 6197650944.0, + "54": 6197650944.0, + "55": 6197650944.0, + "56": 6197650944.0, + "57": 6197650944.0, + "58": 6197650944.0, + "59": 6197650944.0, + "60": 6197650944.0, + "61": 6197650944.0, + "62": 6197650944.0, + "63": 6197650944.0, + "64": 6197650944.0, + "65": 6197650944.0, + "66": 6197650944.0, + "67": 6197650944.0, + "68": 6197650944.0, + "69": 6197650944.0, + "70": 6197650944.0, + "71": 6197650944.0, + "72": 6197650944.0, + "73": 6197650944.0, + "74": 6197650944.0, + "75": 6197650944.0, + "76": 6197650944.0, + "77": 6197650944.0, + "78": 6197650944.0, + "79": 6197650944.0, + "80": 6197650944.0, + "81": 6197650944.0, + "82": 6197650944.0, + "83": 6197650944.0, + "84": 6197650944.0, + "85": 6197650944.0, + "86": 6197650944.0, + "87": 6197650944.0, + "88": 6197650944.0, + "89": 6197650944.0, + "90": 6197650944.0, + "91": 6197650944.0, + "92": 6197650944.0, + "93": 6197650944.0, + "94": 6197650944.0, + "95": 6197650944.0, + "96": 6197650944.0, + "97": 6197650944.0, + "98": 6197650944.0, + "99": 6197650944.0, + "100": 6197650944.0 } }, "iteration-time": { "start_step": 2, "end_step": 100, "step_interval": 1, + "value_precision": "full", "values": { - "2": 5.79961, - "3": 0.22278, - "4": 0.2152, - "5": 0.20721, - "6": 0.20667, - "7": 0.21386, - "8": 0.20357, - "9": 0.20613, - "10": 0.20875, - "11": 0.20828, - "12": 0.20694, - "13": 0.20829, - "14": 0.20863, - "15": 0.20751, - "16": 0.21061, - "17": 0.20482, - "18": 0.20736, - "19": 0.2088, - "20": 0.2069, - "21": 0.20658, - "22": 0.20978, - "23": 0.208, - "24": 0.20424, - "25": 0.2076, - "26": 0.20965, - "27": 0.20882, - "28": 0.20869, - "29": 0.21017, - "30": 0.20785, - "31": 0.2064, - "32": 0.20818, - "33": 0.20788, - "34": 0.20721, - "35": 0.20431, - "36": 0.20733, - "37": 0.20715, - "38": 0.2092, - "39": 0.21239, - "40": 0.20992, - "41": 0.20788, - "42": 0.20336, - "43": 0.20704, - "44": 0.20915, - "45": 0.20527, - "46": 1.10348, - "47": 0.20715, - "48": 0.20639, - "49": 0.20722, - "50": 0.2069, - "51": 0.20906, - "52": 0.2179, - "53": 0.209, - "54": 0.20662, - "55": 0.20652, - "56": 0.20664, - "57": 0.20592, - "58": 0.20882, - "59": 0.20565, - "60": 0.20652, - "61": 0.20822, - "62": 0.20731, - "63": 0.70673, - "64": 0.20823, - "65": 0.20693, - "66": 0.20711, - "67": 0.20954, - "68": 0.20868, - "69": 0.20816, - "70": 0.20906, - "71": 0.20647, - "72": 0.20914, - "73": 0.20441, - "74": 1.08828, - "75": 0.72345, - "76": 1.06526, - "77": 0.20749, - "78": 0.2064, - "79": 1.06642, - "80": 0.21052, - "81": 0.20587, - "82": 0.20628, - "83": 0.20854, - "84": 0.20656, - "85": 0.20449, - "86": 0.20804, - "87": 0.76655, - "88": 0.20935, - "89": 0.20996, - "90": 0.20806, - "91": 0.75071, - "92": 0.20851, - "93": 0.20907, - "94": 0.20425, - "95": 0.20568, - "96": 0.20651, - "97": 0.21055, - "98": 0.20777, - "99": 0.20731, - "100": 0.20918 + "2": 0.4551560878753662, + "3": 0.21785330772399902, + "4": 0.19936752319335938, + "5": 0.19991803169250488, + "6": 0.20128941535949707, + "7": 0.202744722366333, + "8": 0.2004106044769287, + "9": 0.19887685775756836, + "10": 0.2008192539215088, + "11": 0.2000904083251953, + "12": 0.2001323699951172, + "13": 0.20117974281311035, + "14": 0.20229887962341309, + "15": 0.20003628730773926, + "16": 0.2016892433166504, + "17": 0.19829249382019043, + "18": 0.20043134689331055, + "19": 0.19935846328735352, + "20": 0.19895505905151367, + "21": 0.20000982284545898, + "22": 0.20049190521240234, + "23": 0.2020888328552246, + "24": 0.20193099975585938, + "25": 0.2002556324005127, + "26": 0.2018575668334961, + "27": 0.20357775688171387, + "28": 0.20117521286010742, + "29": 0.2025446891784668, + "30": 0.20000481605529785, + "31": 0.20397710800170898, + "32": 0.2013840675354004, + "33": 0.2010178565979004, + "34": 0.20246386528015137, + "35": 0.20330190658569336, + "36": 0.2010667324066162, + "37": 0.19982314109802246, + "38": 0.20348095893859863, + "39": 0.2035069465637207, + "40": 0.20201802253723145, + "41": 0.20458292961120605, + "42": 0.20049405097961426, + "43": 0.201629638671875, + "44": 0.2002239227294922, + "45": 0.2006683349609375, + "46": 0.2007310390472412, + "47": 0.20369362831115723, + "48": 0.20049500465393066, + "49": 0.20035886764526367, + "50": 0.19788527488708496, + "51": 0.20467448234558105, + "52": 0.1998133659362793, + "53": 0.2005925178527832, + "54": 0.19953417778015137, + "55": 0.19899582862854004, + "56": 0.2005023956298828, + "57": 0.19908905029296875, + "58": 0.19912314414978027, + "59": 0.1992032527923584, + "60": 0.2010364532470703, + "61": 0.20086073875427246, + "62": 0.19945025444030762, + "63": 0.2012789249420166, + "64": 0.20190668106079102, + "65": 0.20104193687438965, + "66": 0.20068979263305664, + "67": 0.20225834846496582, + "68": 0.7520992755889893, + "69": 0.8523368835449219, + "70": 0.20131635665893555, + "71": 1.1325016021728516, + "72": 0.20259428024291992, + "73": 1.1291320323944092, + "74": 0.20203566551208496, + "75": 0.20015311241149902, + "76": 0.19977855682373047, + "77": 1.029984951019287, + "78": 0.2014625072479248, + "79": 0.8005738258361816, + "80": 0.19962453842163086, + "81": 1.154477596282959, + "82": 0.8194148540496826, + "83": 0.19890379905700684, + "84": 0.20033740997314453, + "85": 0.20167303085327148, + "86": 0.20156097412109375, + "87": 0.2007768154144287, + "88": 0.20098519325256348, + "89": 0.20073795318603516, + "90": 0.19956231117248535, + "91": 0.2010326385498047, + "92": 0.20005202293395996, + "93": 0.20020079612731934, + "94": 0.20137500762939453, + "95": 0.19857573509216309, + "96": 0.1999368667602539, + "97": 0.20207953453063965, + "98": 0.20024371147155762, + "99": 0.19941449165344238, + "100": 0.19959521293640137 } } -} \ No newline at end of file +} diff --git a/tests/functional_tests/test_cases/t5/t5_mcore_tp1_pp1_vp1_resume_torch/golden_values_dev_dgx_h100.json b/tests/functional_tests/test_cases/t5/t5_mcore_tp1_pp1_vp1_resume_torch/golden_values_dev_dgx_h100.json index 74ba9f3dc43..d97f5039ed7 100644 --- a/tests/functional_tests/test_cases/t5/t5_mcore_tp1_pp1_vp1_resume_torch/golden_values_dev_dgx_h100.json +++ b/tests/functional_tests/test_cases/t5/t5_mcore_tp1_pp1_vp1_resume_torch/golden_values_dev_dgx_h100.json @@ -3,534 +3,539 @@ "start_step": 1, "end_step": 100, "step_interval": 1, + "value_precision": "full", "values": { - "1": 10.34378, - "2": 10.3586, - "3": 9.62116, - "4": 9.39354, - "5": 9.26693, - "6": 9.27611, - "7": 9.16087, - "8": 9.03863, - "9": 8.95116, - "10": 8.88834, - "11": 8.76106, - "12": 8.67192, - "13": 8.6323, - "14": 8.5566, - "15": 8.49075, - "16": 8.51355, - "17": 8.27446, - "18": 8.23386, - "19": 8.27349, - "20": 8.1698, - "21": 8.07637, - "22": 8.01588, - "23": 8.04836, - "24": 7.9153, - "25": 7.8742, - "26": 7.84663, - "27": 7.72267, - "28": 7.70205, - "29": 7.5891, - "30": 7.58691, - "31": 7.47531, - "32": 7.5306, - "33": 7.45886, - "34": 7.38984, - "35": 7.33856, - "36": 7.30891, - "37": 7.27927, - "38": 7.29089, - "39": 7.26655, - "40": 7.16271, - "41": 7.10891, - "42": 7.15542, - "43": 7.14532, - "44": 7.05009, - "45": 7.06544, - "46": 7.0794, - "47": 7.03116, - "48": 7.03476, - "49": 6.92494, - "50": 6.99432, - "51": 7.09863, - "52": 6.98193, - "53": 6.90478, - "54": 6.88825, - "55": 6.85965, - "56": 6.95936, - "57": 6.91082, - "58": 6.82047, - "59": 6.88999, - "60": 6.73957, - "61": 6.87621, - "62": 6.82104, - "63": 6.87031, - "64": 6.90824, - "65": 6.76869, - "66": 6.80207, - "67": 6.70941, - "68": 6.70949, - "69": 6.86832, - "70": 6.65878, - "71": 6.84947, - "72": 6.77654, - "73": 6.80183, - "74": 6.7289, - "75": 6.71273, - "76": 6.81776, - "77": 6.76144, - "78": 6.68621, - "79": 6.80909, - "80": 6.82905, - "81": 6.79215, - "82": 6.71251, - "83": 6.76871, - "84": 6.75362, - "85": 6.74205, - "86": 6.71639, - "87": 6.76062, - "88": 6.75309, - "89": 6.72448, - "90": 6.58857, - "91": 6.69949, - "92": 6.69998, - "93": 6.75081, - "94": 6.75574, - "95": 6.67827, - "96": 6.76655, - "97": 6.78818, - "98": 6.65471, - "99": 6.65788, - "100": 6.75977 + "1": 10.343809127807617, + "2": 10.35859203338623, + "3": 9.621138572692871, + "4": 9.393589973449707, + "5": 9.267155647277832, + "6": 9.276350975036621, + "7": 9.161163330078125, + "8": 9.038921356201172, + "9": 8.951240539550781, + "10": 8.889043807983398, + "11": 8.760700225830078, + "12": 8.672532081604004, + "13": 8.632795333862305, + "14": 8.557355880737305, + "15": 8.491463661193848, + "16": 8.514768600463867, + "17": 8.275775909423828, + "18": 8.235566139221191, + "19": 8.274949073791504, + "20": 8.171558380126953, + "21": 8.078373908996582, + "22": 8.018048286437988, + "23": 8.050060272216797, + "24": 7.916262626647949, + "25": 7.874800205230713, + "26": 7.8470845222473145, + "27": 7.723288536071777, + "28": 7.7025041580200195, + "29": 7.589680194854736, + "30": 7.587080001831055, + "31": 7.476034641265869, + "32": 7.532017230987549, + "33": 7.460168838500977, + "34": 7.391164779663086, + "35": 7.340322971343994, + "36": 7.310537338256836, + "37": 7.280948638916016, + "38": 7.2927093505859375, + "39": 7.267828464508057, + "40": 7.164637565612793, + "41": 7.110754489898682, + "42": 7.156435489654541, + "43": 7.147419452667236, + "44": 7.0518107414245605, + "45": 7.0677947998046875, + "46": 7.080720901489258, + "47": 7.030524730682373, + "48": 7.031854629516602, + "49": 6.9221673011779785, + "50": 6.990994453430176, + "51": 7.095733165740967, + "52": 6.978329181671143, + "53": 6.901208400726318, + "54": 6.884012222290039, + "55": 6.855714797973633, + "56": 6.956377029418945, + "57": 6.906090259552002, + "58": 6.8175530433654785, + "59": 6.888942241668701, + "60": 6.736663341522217, + "61": 6.8750996589660645, + "62": 6.821629047393799, + "63": 6.865473747253418, + "64": 6.907495498657227, + "65": 6.768472671508789, + "66": 6.802971839904785, + "67": 6.71140718460083, + "68": 6.706887722015381, + "69": 6.877745151519775, + "70": 6.666089057922363, + "71": 6.84719705581665, + "72": 6.783570766448975, + "73": 6.801311492919922, + "74": 6.731091499328613, + "75": 6.70947265625, + "76": 6.820851802825928, + "77": 6.765815734863281, + "78": 6.679129600524902, + "79": 6.81079626083374, + "80": 6.8366265296936035, + "81": 6.796154499053955, + "82": 6.7111287117004395, + "83": 6.773612022399902, + "84": 6.752481460571289, + "85": 6.744526386260986, + "86": 6.7186598777771, + "87": 6.768910884857178, + "88": 6.756497859954834, + "89": 6.7277302742004395, + "90": 6.5948920249938965, + "91": 6.704666614532471, + "92": 6.6968913078308105, + "93": 6.746057510375977, + "94": 6.757419109344482, + "95": 6.67865514755249, + "96": 6.770571708679199, + "97": 6.7866668701171875, + "98": 6.657751560211182, + "99": 6.6576361656188965, + "100": 6.76534366607666 } }, "num-zeros": { "start_step": 1, "end_step": 100, "step_interval": 1, + "value_precision": "full", "values": { - "1": 41090.0, - "2": 41079.0, - "3": 43332.0, - "4": 47046.0, - "5": 43268.0, - "6": 45489.0, + "1": 41094.0, + "2": 41078.0, + "3": 43330.0, + "4": 47015.0, + "5": 43279.0, + "6": 45491.0, "7": 47021.0, - "8": 44083.0, - "9": 44016.0, - "10": 45529.0, - "11": 45364.0, - "12": 45397.0, - "13": 43976.0, - "14": 41870.0, - "15": 41880.0, - "16": 44771.0, - "17": 43875.0, - "18": 45402.0, - "19": 44024.0, - "20": 46407.0, - "21": 44105.0, - "22": 45467.0, - "23": 41950.0, - "24": 43221.0, - "25": 46945.0, - "26": 43272.0, - "27": 44651.0, - "28": 43474.0, - "29": 44879.0, - "30": 45524.0, - "31": 44641.0, - "32": 46222.0, - "33": 45622.0, - "34": 47093.0, - "35": 43233.0, - "36": 47062.0, - "37": 45569.0, - "38": 42446.0, - "39": 45545.0, - "40": 46947.0, - "41": 43428.0, - "42": 45559.0, - "43": 46237.0, - "44": 43325.0, - "45": 43207.0, - "46": 44642.0, - "47": 43315.0, - "48": 44637.0, - "49": 43901.0, - "50": 45414.0, - "51": 42031.0, - "52": 42051.0, - "53": 47130.0, - "54": 44773.0, - "55": 44013.0, - "56": 44068.0, - "57": 43442.0, - "58": 44696.0, - "59": 42429.0, - "60": 43175.0, - "61": 44779.0, - "62": 43915.0, - "63": 42707.0, - "64": 44054.0, - "65": 45550.0, - "66": 42797.0, - "67": 43985.0, - "68": 44083.0, - "69": 41631.0, - "70": 41700.0, - "71": 45489.0, - "72": 45418.0, - "73": 42749.0, - "74": 44776.0, - "75": 43924.0, - "76": 43346.0, - "77": 41667.0, - "78": 43334.0, - "79": 45405.0, - "80": 45484.0, - "81": 42474.0, - "82": 39200.0, - "83": 45705.0, - "84": 43462.0, - "85": 44104.0, - "86": 43841.0, - "87": 41289.0, - "88": 44165.0, - "89": 40422.0, - "90": 39866.0, - "91": 46224.0, - "92": 46233.0, - "93": 43900.0, - "94": 45498.0, - "95": 41846.0, - "96": 41474.0, - "97": 42025.0, - "98": 43440.0, - "99": 39044.0, - "100": 39468.0 + "8": 44100.0, + "9": 44030.0, + "10": 45531.0, + "11": 45378.0, + "12": 45394.0, + "13": 43968.0, + "14": 41850.0, + "15": 41883.0, + "16": 44760.0, + "17": 43872.0, + "18": 45404.0, + "19": 44026.0, + "20": 46415.0, + "21": 44107.0, + "22": 45465.0, + "23": 41937.0, + "24": 43237.0, + "25": 46954.0, + "26": 43268.0, + "27": 44655.0, + "28": 43469.0, + "29": 44865.0, + "30": 45533.0, + "31": 44637.0, + "32": 46231.0, + "33": 45616.0, + "34": 47098.0, + "35": 43228.0, + "36": 47036.0, + "37": 45580.0, + "38": 42433.0, + "39": 45559.0, + "40": 46938.0, + "41": 43393.0, + "42": 45547.0, + "43": 46238.0, + "44": 43345.0, + "45": 43196.0, + "46": 44656.0, + "47": 43313.0, + "48": 44648.0, + "49": 43868.0, + "50": 45416.0, + "51": 42051.0, + "52": 42057.0, + "53": 47136.0, + "54": 44780.0, + "55": 44015.0, + "56": 44073.0, + "57": 43447.0, + "58": 44721.0, + "59": 42396.0, + "60": 43162.0, + "61": 44778.0, + "62": 43911.0, + "63": 42726.0, + "64": 44041.0, + "65": 45552.0, + "66": 42795.0, + "67": 43981.0, + "68": 44100.0, + "69": 41619.0, + "70": 41681.0, + "71": 45513.0, + "72": 45386.0, + "73": 42754.0, + "74": 44803.0, + "75": 43950.0, + "76": 43358.0, + "77": 41661.0, + "78": 43338.0, + "79": 45432.0, + "80": 45483.0, + "81": 42465.0, + "82": 39223.0, + "83": 45700.0, + "84": 43506.0, + "85": 44088.0, + "86": 43855.0, + "87": 41256.0, + "88": 44183.0, + "89": 40449.0, + "90": 39826.0, + "91": 46222.0, + "92": 46293.0, + "93": 43897.0, + "94": 45506.0, + "95": 41867.0, + "96": 41430.0, + "97": 42039.0, + "98": 43416.0, + "99": 39088.0, + "100": 39457.0 } }, "mem-allocated-bytes": { "start_step": 1, "end_step": 100, "step_interval": 1, + "value_precision": "full", "values": { - "1": 4161267200.0, - "2": 4161267200.0, - "3": 4161267200.0, - "4": 4161267200.0, - "5": 4161267200.0, - "6": 4161267200.0, - "7": 4161267200.0, - "8": 4161267200.0, - "9": 4161267200.0, - "10": 4161267200.0, - "11": 4161267200.0, - "12": 4161267200.0, - "13": 4161267200.0, - "14": 4161267200.0, - "15": 4161267200.0, - "16": 4161267200.0, - "17": 4161267200.0, - "18": 4161267200.0, - "19": 4161267200.0, - "20": 4161267200.0, - "21": 4161267200.0, - "22": 4161267200.0, - "23": 4161267200.0, - "24": 4161267200.0, - "25": 4161267200.0, - "26": 4161267200.0, - "27": 4161267200.0, - "28": 4161267200.0, - "29": 4161267200.0, - "30": 4161267200.0, - "31": 4161267200.0, - "32": 4161267200.0, - "33": 4161267200.0, - "34": 4161267200.0, - "35": 4161267200.0, - "36": 4161267200.0, - "37": 4161267200.0, - "38": 4161267200.0, - "39": 4161267200.0, - "40": 4161267200.0, - "41": 4161267200.0, - "42": 4161267200.0, - "43": 4161267200.0, - "44": 4161267200.0, - "45": 4161267200.0, - "46": 4161267200.0, - "47": 4161267200.0, - "48": 4161267200.0, - "49": 4161267200.0, - "50": 4161267200.0, - "51": 4161267200.0, - "52": 4161267200.0, - "53": 4161267200.0, - "54": 4161267200.0, - "55": 4161267200.0, - "56": 4161267200.0, - "57": 4161267200.0, - "58": 4161267200.0, - "59": 4161267200.0, - "60": 4161267200.0, - "61": 4161267200.0, - "62": 4161267200.0, - "63": 4161267200.0, - "64": 4161267200.0, - "65": 4161267200.0, - "66": 4161267200.0, - "67": 4161267200.0, - "68": 4161267200.0, - "69": 4161267200.0, - "70": 4161267200.0, - "71": 4161267200.0, - "72": 4161267200.0, - "73": 4161267200.0, - "74": 4161267200.0, - "75": 4161267200.0, - "76": 4161267200.0, - "77": 4161267200.0, - "78": 4161267200.0, - "79": 4161267200.0, - "80": 4161267200.0, - "81": 4161267200.0, - "82": 4161267200.0, - "83": 4161267200.0, - "84": 4161267200.0, - "85": 4161267200.0, - "86": 4161267200.0, - "87": 4161267200.0, - "88": 4161267200.0, - "89": 4161267200.0, - "90": 4161267200.0, - "91": 4161267200.0, - "92": 4161267200.0, - "93": 4161267200.0, - "94": 4161267200.0, - "95": 4161267200.0, - "96": 4161267200.0, - "97": 4161267200.0, - "98": 4161267200.0, - "99": 4161267200.0, - "100": 4161267200.0 + "1": 4169264640.0, + "2": 4169264640.0, + "3": 4169264640.0, + "4": 4169264640.0, + "5": 4169264640.0, + "6": 4169264640.0, + "7": 4169264640.0, + "8": 4169264640.0, + "9": 4169264640.0, + "10": 4169264640.0, + "11": 4169264640.0, + "12": 4169264640.0, + "13": 4169264640.0, + "14": 4169264640.0, + "15": 4169264640.0, + "16": 4169264640.0, + "17": 4169264640.0, + "18": 4169264640.0, + "19": 4169264640.0, + "20": 4169264640.0, + "21": 4169264640.0, + "22": 4169264640.0, + "23": 4169264640.0, + "24": 4169264640.0, + "25": 4169264640.0, + "26": 4169264640.0, + "27": 4169264640.0, + "28": 4169264640.0, + "29": 4169264640.0, + "30": 4169264640.0, + "31": 4169264640.0, + "32": 4169264640.0, + "33": 4169264640.0, + "34": 4169264640.0, + "35": 4169264640.0, + "36": 4169264640.0, + "37": 4169264640.0, + "38": 4169264640.0, + "39": 4169264640.0, + "40": 4169264640.0, + "41": 4169264640.0, + "42": 4169264640.0, + "43": 4169264640.0, + "44": 4169264640.0, + "45": 4169264640.0, + "46": 4169264640.0, + "47": 4169264640.0, + "48": 4169264640.0, + "49": 4169264640.0, + "50": 4169264640.0, + "51": 4169264640.0, + "52": 4169264640.0, + "53": 4169264640.0, + "54": 4169264640.0, + "55": 4169264640.0, + "56": 4169264640.0, + "57": 4169264640.0, + "58": 4169264640.0, + "59": 4169264640.0, + "60": 4169264640.0, + "61": 4169264640.0, + "62": 4169264640.0, + "63": 4169264640.0, + "64": 4169264640.0, + "65": 4169264640.0, + "66": 4169264640.0, + "67": 4169264640.0, + "68": 4169264640.0, + "69": 4169264640.0, + "70": 4169264640.0, + "71": 4169264640.0, + "72": 4169264640.0, + "73": 4169264640.0, + "74": 4169264640.0, + "75": 4169264640.0, + "76": 4169264640.0, + "77": 4169264640.0, + "78": 4169264640.0, + "79": 4169264640.0, + "80": 4169264640.0, + "81": 4169264640.0, + "82": 4169264640.0, + "83": 4169264640.0, + "84": 4169264640.0, + "85": 4169264640.0, + "86": 4169264640.0, + "87": 4169264640.0, + "88": 4169264640.0, + "89": 4169264640.0, + "90": 4169264640.0, + "91": 4169264640.0, + "92": 4169264640.0, + "93": 4169264640.0, + "94": 4169264640.0, + "95": 4169264640.0, + "96": 4169264640.0, + "97": 4169264640.0, + "98": 4169264640.0, + "99": 4169264640.0, + "100": 4169264640.0 } }, "mem-max-allocated-bytes": { "start_step": 1, "end_step": 100, "step_interval": 1, + "value_precision": "full", "values": { - "1": 4378808832.0, - "2": 6186638848.0, - "3": 6186638848.0, - "4": 6186638848.0, - "5": 6186638848.0, - "6": 6186638848.0, - "7": 6186638848.0, - "8": 6186638848.0, - "9": 6186638848.0, - "10": 6186638848.0, - "11": 6186638848.0, - "12": 6186638848.0, - "13": 6186638848.0, - "14": 6186638848.0, - "15": 6186638848.0, - "16": 6186638848.0, - "17": 6186638848.0, - "18": 6186638848.0, - "19": 6186638848.0, - "20": 6186638848.0, - "21": 6186638848.0, - "22": 6186638848.0, - "23": 6186638848.0, - "24": 6186638848.0, - "25": 6186638848.0, - "26": 6186638848.0, - "27": 6186638848.0, - "28": 6186638848.0, - "29": 6186638848.0, - "30": 6186638848.0, - "31": 6186638848.0, - "32": 6186638848.0, - "33": 6186638848.0, - "34": 6186638848.0, - "35": 6186638848.0, - "36": 6186638848.0, - "37": 6186638848.0, - "38": 6186638848.0, - "39": 6186638848.0, - "40": 6186638848.0, - "41": 6186638848.0, - "42": 6186638848.0, - "43": 6186638848.0, - "44": 6186638848.0, - "45": 6186638848.0, - "46": 6186638848.0, - "47": 6186638848.0, - "48": 6186638848.0, - "49": 6186638848.0, - "50": 6186638848.0, - "51": 6186638848.0, - "52": 6186638848.0, - "53": 6186638848.0, - "54": 6186638848.0, - "55": 6186638848.0, - "56": 6186638848.0, - "57": 6186638848.0, - "58": 6186638848.0, - "59": 6186638848.0, - "60": 6186638848.0, - "61": 6186638848.0, - "62": 6186638848.0, - "63": 6186638848.0, - "64": 6186638848.0, - "65": 6186638848.0, - "66": 6186638848.0, - "67": 6186638848.0, - "68": 6186638848.0, - "69": 6186638848.0, - "70": 6186638848.0, - "71": 6186638848.0, - "72": 6186638848.0, - "73": 6186638848.0, - "74": 6186638848.0, - "75": 6186638848.0, - "76": 6186638848.0, - "77": 6186638848.0, - "78": 6186638848.0, - "79": 6186638848.0, - "80": 6186638848.0, - "81": 6186638848.0, - "82": 6186638848.0, - "83": 6186638848.0, - "84": 6186638848.0, - "85": 6186638848.0, - "86": 6186638848.0, - "87": 6186638848.0, - "88": 6186638848.0, - "89": 6186638848.0, - "90": 6186638848.0, - "91": 6186638848.0, - "92": 6186638848.0, - "93": 6186638848.0, - "94": 6186638848.0, - "95": 6186638848.0, - "96": 6186638848.0, - "97": 6186638848.0, - "98": 6186638848.0, - "99": 6186638848.0, - "100": 6186638848.0 + "1": 4388903424.0, + "2": 6197519872.0, + "3": 6199879168.0, + "4": 6199879168.0, + "5": 6199879168.0, + "6": 6199879168.0, + "7": 6199879168.0, + "8": 6199879168.0, + "9": 6199879168.0, + "10": 6199879168.0, + "11": 6199879168.0, + "12": 6199879168.0, + "13": 6199879168.0, + "14": 6199879168.0, + "15": 6199879168.0, + "16": 6199879168.0, + "17": 6199879168.0, + "18": 6199879168.0, + "19": 6199879168.0, + "20": 6199879168.0, + "21": 6199879168.0, + "22": 6199879168.0, + "23": 6199879168.0, + "24": 6199879168.0, + "25": 6199879168.0, + "26": 6199879168.0, + "27": 6199879168.0, + "28": 6199879168.0, + "29": 6199879168.0, + "30": 6199879168.0, + "31": 6199879168.0, + "32": 6199879168.0, + "33": 6199879168.0, + "34": 6199879168.0, + "35": 6199879168.0, + "36": 6199879168.0, + "37": 6199879168.0, + "38": 6199879168.0, + "39": 6199879168.0, + "40": 6199879168.0, + "41": 6199879168.0, + "42": 6199879168.0, + "43": 6199879168.0, + "44": 6199879168.0, + "45": 6199879168.0, + "46": 6199879168.0, + "47": 6199879168.0, + "48": 6199879168.0, + "49": 6199879168.0, + "50": 6199879168.0, + "51": 6199879168.0, + "52": 6199879168.0, + "53": 6199879168.0, + "54": 6199879168.0, + "55": 6199879168.0, + "56": 6199879168.0, + "57": 6199879168.0, + "58": 6199879168.0, + "59": 6199879168.0, + "60": 6199879168.0, + "61": 6199879168.0, + "62": 6199879168.0, + "63": 6199879168.0, + "64": 6199879168.0, + "65": 6199879168.0, + "66": 6199879168.0, + "67": 6199879168.0, + "68": 6199879168.0, + "69": 6199879168.0, + "70": 6199879168.0, + "71": 6199879168.0, + "72": 6199879168.0, + "73": 6199879168.0, + "74": 6199879168.0, + "75": 6199879168.0, + "76": 6199879168.0, + "77": 6199879168.0, + "78": 6199879168.0, + "79": 6199879168.0, + "80": 6199879168.0, + "81": 6199879168.0, + "82": 6199879168.0, + "83": 6199879168.0, + "84": 6199879168.0, + "85": 6199879168.0, + "86": 6199879168.0, + "87": 6199879168.0, + "88": 6199879168.0, + "89": 6199879168.0, + "90": 6199879168.0, + "91": 6199879168.0, + "92": 6199879168.0, + "93": 6199879168.0, + "94": 6199879168.0, + "95": 6199879168.0, + "96": 6199879168.0, + "97": 6199879168.0, + "98": 6199879168.0, + "99": 6199879168.0, + "100": 6199879168.0 } }, "iteration-time": { "start_step": 2, "end_step": 100, "step_interval": 1, + "value_precision": "full", "values": { - "2": 5.76133, - "3": 0.16718, - "4": 0.13762, - "5": 0.13883, - "6": 0.13654, - "7": 0.14019, - "8": 0.13442, - "9": 0.13581, - "10": 0.13796, - "11": 0.13683, - "12": 0.13647, - "13": 0.1354, - "14": 0.13595, - "15": 0.13472, - "16": 0.13897, - "17": 0.13439, - "18": 0.13558, - "19": 0.13444, - "20": 0.13397, - "21": 0.13499, - "22": 0.13574, - "23": 0.13477, - "24": 0.13334, - "25": 0.1347, - "26": 0.1397, - "27": 0.13938, - "28": 0.13991, - "29": 0.14002, - "30": 0.14249, - "31": 0.13688, - "32": 0.13793, - "33": 0.1385, - "34": 0.13814, - "35": 0.13793, - "36": 0.13941, - "37": 0.14029, - "38": 0.13828, - "39": 0.14316, - "40": 0.13662, - "41": 0.13548, - "42": 0.13658, - "43": 0.13831, - "44": 0.13676, - "45": 0.14037, - "46": 0.13834, - "47": 0.13955, - "48": 0.14072, - "49": 0.13745, - "50": 0.13632, - "51": 0.14833, - "52": 0.14038, - "53": 0.13786, - "54": 0.13779, - "55": 0.13657, - "56": 0.13617, - "57": 0.13797, - "58": 0.13965, - "59": 0.13579, - "60": 0.1404, - "61": 0.1399, - "62": 0.1354, - "63": 0.13592, - "64": 0.13614, - "65": 0.13779, - "66": 0.1358, - "67": 0.13991, - "68": 0.13586, - "69": 0.183, - "70": 0.14148, - "71": 0.13692, - "72": 0.136, - "73": 0.68205, - "74": 0.13911, - "75": 0.13345, - "76": 0.13774, - "77": 0.13683, - "78": 0.13501, - "79": 0.1349, - "80": 0.98406, - "81": 0.70159, - "82": 0.1407, - "83": 0.13605, - "84": 0.13524, - "85": 0.67329, - "86": 1.06497, - "87": 0.14848, - "88": 1.02754, - "89": 0.14271, - "90": 0.68711, - "91": 0.13852, - "92": 0.13575, - "93": 0.13923, - "94": 0.13496, - "95": 0.13579, - "96": 0.13696, - "97": 0.13577, - "98": 0.13696, - "99": 0.13554, - "100": 0.13374 + "2": 0.28190016746520996, + "3": 0.1499638557434082, + "4": 0.13162970542907715, + "5": 0.12887358665466309, + "6": 0.13396072387695312, + "7": 0.13086175918579102, + "8": 0.13727569580078125, + "9": 0.13064074516296387, + "10": 0.1359422206878662, + "11": 0.12894868850708008, + "12": 0.12816190719604492, + "13": 0.12774062156677246, + "14": 0.1259148120880127, + "15": 0.1254420280456543, + "16": 0.12766289710998535, + "17": 0.12457132339477539, + "18": 0.14759540557861328, + "19": 0.13588690757751465, + "20": 0.12578487396240234, + "21": 0.12756013870239258, + "22": 0.1262342929840088, + "23": 0.1298389434814453, + "24": 0.12788963317871094, + "25": 0.1252443790435791, + "26": 0.128692626953125, + "27": 0.13181376457214355, + "28": 0.1263284683227539, + "29": 0.13057732582092285, + "30": 0.12687993049621582, + "31": 0.13794326782226562, + "32": 0.12939214706420898, + "33": 0.13011837005615234, + "34": 0.1304006576538086, + "35": 0.12736892700195312, + "36": 0.13009023666381836, + "37": 0.1262819766998291, + "38": 0.12838292121887207, + "39": 0.1300973892211914, + "40": 0.13031435012817383, + "41": 0.13364601135253906, + "42": 0.12507128715515137, + "43": 0.1269979476928711, + "44": 0.1264188289642334, + "45": 0.12658977508544922, + "46": 0.12674808502197266, + "47": 0.12940144538879395, + "48": 0.1280076503753662, + "49": 0.12723398208618164, + "50": 0.1317002773284912, + "51": 0.14043831825256348, + "52": 0.12904715538024902, + "53": 0.1258068084716797, + "54": 0.12544631958007812, + "55": 0.12542200088500977, + "56": 0.12571167945861816, + "57": 0.12841439247131348, + "58": 0.12613129615783691, + "59": 0.1256730556488037, + "60": 0.1260683536529541, + "61": 0.1255331039428711, + "62": 0.12629270553588867, + "63": 0.12530827522277832, + "64": 0.1288142204284668, + "65": 0.12718749046325684, + "66": 0.12654328346252441, + "67": 0.12682223320007324, + "68": 0.7315971851348877, + "69": 0.6559479236602783, + "70": 0.13131928443908691, + "71": 0.7681601047515869, + "72": 0.1317133903503418, + "73": 1.0243499279022217, + "74": 0.12707185745239258, + "75": 0.12672924995422363, + "76": 0.13074350357055664, + "77": 1.0533807277679443, + "78": 0.1326589584350586, + "79": 0.7080132961273193, + "80": 0.1302330493927002, + "81": 0.7102844715118408, + "82": 0.760246992111206, + "83": 0.12903904914855957, + "84": 0.12622785568237305, + "85": 0.12671136856079102, + "86": 0.1299288272857666, + "87": 0.1276395320892334, + "88": 0.12688517570495605, + "89": 0.13068056106567383, + "90": 0.12440347671508789, + "91": 0.12612080574035645, + "92": 0.12558984756469727, + "93": 0.12508082389831543, + "94": 0.1278982162475586, + "95": 0.13044166564941406, + "96": 0.1281144618988037, + "97": 0.1318352222442627, + "98": 0.12740826606750488, + "99": 0.1272449493408203, + "100": 0.12534666061401367 } } } \ No newline at end of file diff --git a/tests/functional_tests/test_cases/t5/t5_mcore_tp2_pp1_vp1/golden_values_dev_dgx_a100.json b/tests/functional_tests/test_cases/t5/t5_mcore_tp2_pp1_vp1/golden_values_dev_dgx_a100.json index 0193e5f5017..ce1e0398ffb 100644 --- a/tests/functional_tests/test_cases/t5/t5_mcore_tp2_pp1_vp1/golden_values_dev_dgx_a100.json +++ b/tests/functional_tests/test_cases/t5/t5_mcore_tp2_pp1_vp1/golden_values_dev_dgx_a100.json @@ -3,220 +3,223 @@ "start_step": 1, "end_step": 100, "step_interval": 1, + "value_precision": "full", "values": { - "1": 10.39821, - "2": 10.4007, - "3": 9.62023, - "4": 9.38329, - "5": 9.25187, - "6": 9.24515, - "7": 9.13845, - "8": 9.03555, - "9": 8.9593, - "10": 8.92431, - "11": 8.80715, - "12": 8.72512, - "13": 8.62189, - "14": 8.52303, - "15": 8.47667, - "16": 8.45787, - "17": 8.25346, - "18": 8.19319, - "19": 8.24459, - "20": 8.11599, - "21": 8.04218, - "22": 7.97872, - "23": 8.01002, - "24": 7.86818, - "25": 7.84139, - "26": 7.81516, - "27": 7.69178, - "28": 7.66198, - "29": 7.55989, - "30": 7.56019, - "31": 7.44474, - "32": 7.51611, - "33": 7.43834, - "34": 7.37038, - "35": 7.3192, - "36": 7.27111, - "37": 7.25763, - "38": 7.27615, - "39": 7.24405, - "40": 7.13327, - "41": 7.08122, - "42": 7.1359, - "43": 7.12211, - "44": 7.03239, - "45": 7.05393, - "46": 7.07116, - "47": 7.01986, - "48": 7.02615, - "49": 6.90939, - "50": 6.98061, - "51": 7.08211, - "52": 6.97094, - "53": 6.89404, - "54": 6.90167, - "55": 6.85704, - "56": 6.94704, - "57": 6.91528, - "58": 6.83069, - "59": 6.89691, - "60": 6.74074, - "61": 6.89519, - "62": 6.82835, - "63": 6.90521, - "64": 6.92089, - "65": 6.77632, - "66": 6.80486, - "67": 6.70909, - "68": 6.7379, - "69": 6.88062, - "70": 6.68075, - "71": 6.86735, - "72": 6.78579, - "73": 6.80272, - "74": 6.74518, - "75": 6.73978, - "76": 6.83802, - "77": 6.77412, - "78": 6.70174, - "79": 6.82316, - "80": 6.85462, - "81": 6.81102, - "82": 6.73846, - "83": 6.7895, - "84": 6.78403, - "85": 6.76082, - "86": 6.73676, - "87": 6.78809, - "88": 6.77484, - "89": 6.75017, - "90": 6.61477, - "91": 6.72651, - "92": 6.71923, - "93": 6.76297, - "94": 6.77698, - "95": 6.70734, - "96": 6.79912, - "97": 6.82167, - "98": 6.68967, - "99": 6.68397, - "100": 6.78751 + "1": 10.39828109741211, + "2": 10.400813102722168, + "3": 9.620298385620117, + "4": 9.383430480957031, + "5": 9.25208854675293, + "6": 9.245514869689941, + "7": 9.138854026794434, + "8": 9.036055564880371, + "9": 8.959883689880371, + "10": 8.92492961883545, + "11": 8.807955741882324, + "12": 8.726144790649414, + "13": 8.62308120727539, + "14": 8.524236679077148, + "15": 8.478184700012207, + "16": 8.459382057189941, + "17": 8.254984855651855, + "18": 8.195185661315918, + "19": 8.246625900268555, + "20": 8.118219375610352, + "21": 8.04456901550293, + "22": 7.981522560119629, + "23": 8.012811660766602, + "24": 7.870227336883545, + "25": 7.843097686767578, + "26": 7.816550254821777, + "27": 7.693161964416504, + "28": 7.6633405685424805, + "29": 7.561553955078125, + "30": 7.561794281005859, + "31": 7.446415424346924, + "32": 7.517987251281738, + "33": 7.440102577209473, + "34": 7.372261047363281, + "35": 7.321381568908691, + "36": 7.273429870605469, + "37": 7.259757041931152, + "38": 7.278515815734863, + "39": 7.246405601501465, + "40": 7.135606288909912, + "41": 7.083959579467773, + "42": 7.138038158416748, + "43": 7.124552249908447, + "44": 7.034744739532471, + "45": 7.056392192840576, + "46": 7.0733642578125, + "47": 7.0199761390686035, + "48": 7.024813652038574, + "49": 6.907512664794922, + "50": 6.979202747344971, + "51": 7.080755710601807, + "52": 6.9693922996521, + "53": 6.89262056350708, + "54": 6.900609970092773, + "55": 6.856237888336182, + "56": 6.94619607925415, + "57": 6.914494514465332, + "58": 6.829756736755371, + "59": 6.8964338302612305, + "60": 6.739909648895264, + "61": 6.894719123840332, + "62": 6.827923774719238, + "63": 6.904747486114502, + "64": 6.920497894287109, + "65": 6.77598762512207, + "66": 6.804821014404297, + "67": 6.708977699279785, + "68": 6.7375006675720215, + "69": 6.880411148071289, + "70": 6.680320739746094, + "71": 6.867317199707031, + "72": 6.785577774047852, + "73": 6.802548885345459, + "74": 6.745162487030029, + "75": 6.739590644836426, + "76": 6.837775230407715, + "77": 6.773850917816162, + "78": 6.701603412628174, + "79": 6.822885036468506, + "80": 6.854265213012695, + "81": 6.811123847961426, + "82": 6.7382917404174805, + "83": 6.789459705352783, + "84": 6.783761978149414, + "85": 6.760376930236816, + "86": 6.736575603485107, + "87": 6.787918567657471, + "88": 6.7744460105896, + "89": 6.749938011169434, + "90": 6.614755630493164, + "91": 6.72636604309082, + "92": 6.719092845916748, + "93": 6.762794494628906, + "94": 6.776913166046143, + "95": 6.70740270614624, + "96": 6.799248218536377, + "97": 6.821681022644043, + "98": 6.689704895019531, + "99": 6.6839280128479, + "100": 6.787258625030518 } }, "num-zeros": { "start_step": 1, "end_step": 100, "step_interval": 1, + "value_precision": "full", "values": { - "1": 41053.0, - "2": 40901.0, - "3": 43285.0, - "4": 46978.0, + "1": 41061.0, + "2": 40902.0, + "3": 43277.0, + "4": 46976.0, "5": 43091.0, - "6": 45480.0, - "7": 46960.0, - "8": 44055.0, - "9": 43996.0, - "10": 45547.0, - "11": 45370.0, - "12": 45378.0, - "13": 43956.0, - "14": 41754.0, + "6": 45488.0, + "7": 46966.0, + "8": 44053.0, + "9": 43983.0, + "10": 45555.0, + "11": 45372.0, + "12": 45374.0, + "13": 43948.0, + "14": 41750.0, "15": 41798.0, - "16": 44736.0, - "17": 43837.0, - "18": 45361.0, - "19": 44005.0, - "20": 46291.0, - "21": 43977.0, - "22": 45432.0, - "23": 41869.0, - "24": 43186.0, - "25": 46915.0, - "26": 43231.0, - "27": 44595.0, - "28": 43462.0, - "29": 44838.0, - "30": 45503.0, - "31": 44590.0, - "32": 46178.0, - "33": 45607.0, - "34": 47077.0, - "35": 43210.0, - "36": 46982.0, - "37": 45578.0, - "38": 42374.0, + "16": 44745.0, + "17": 43834.0, + "18": 45359.0, + "19": 43987.0, + "20": 46281.0, + "21": 43988.0, + "22": 45434.0, + "23": 41867.0, + "24": 43208.0, + "25": 46928.0, + "26": 43209.0, + "27": 44593.0, + "28": 43471.0, + "29": 44842.0, + "30": 45487.0, + "31": 44588.0, + "32": 46174.0, + "33": 45609.0, + "34": 47069.0, + "35": 43217.0, + "36": 46969.0, + "37": 45581.0, + "38": 42390.0, "39": 45539.0, - "40": 46912.0, - "41": 43376.0, - "42": 45449.0, - "43": 46232.0, - "44": 43314.0, - "45": 43162.0, - "46": 44637.0, - "47": 43227.0, - "48": 44609.0, - "49": 43921.0, - "50": 45360.0, - "51": 41997.0, - "52": 41970.0, - "53": 47105.0, - "54": 44757.0, - "55": 44031.0, + "40": 46919.0, + "41": 43372.0, + "42": 45445.0, + "43": 46226.0, + "44": 43302.0, + "45": 43147.0, + "46": 44644.0, + "47": 43239.0, + "48": 44611.0, + "49": 43925.0, + "50": 45364.0, + "51": 42019.0, + "52": 41974.0, + "53": 47116.0, + "54": 44766.0, + "55": 44007.0, "56": 44149.0, - "57": 43361.0, - "58": 44679.0, - "59": 42411.0, - "60": 43130.0, - "61": 44589.0, - "62": 43938.0, - "63": 42369.0, - "64": 43999.0, - "65": 45514.0, - "66": 42733.0, - "67": 43886.0, - "68": 44052.0, - "69": 41605.0, - "70": 41665.0, - "71": 45432.0, - "72": 45386.0, - "73": 42708.0, - "74": 44710.0, - "75": 43883.0, - "76": 43287.0, - "77": 41648.0, - "78": 43197.0, - "79": 45340.0, - "80": 45462.0, - "81": 42489.0, - "82": 39177.0, - "83": 45697.0, - "84": 43504.0, - "85": 44117.0, - "86": 43840.0, - "87": 41212.0, - "88": 44119.0, - "89": 40400.0, - "90": 39818.0, - "91": 46226.0, - "92": 46228.0, - "93": 43878.0, - "94": 45433.0, - "95": 41784.0, - "96": 41410.0, - "97": 41959.0, - "98": 43378.0, - "99": 39090.0, - "100": 39427.0 + "57": 43375.0, + "58": 44683.0, + "59": 42406.0, + "60": 43133.0, + "61": 44583.0, + "62": 43927.0, + "63": 42379.0, + "64": 44009.0, + "65": 45499.0, + "66": 42735.0, + "67": 43885.0, + "68": 44045.0, + "69": 41616.0, + "70": 41671.0, + "71": 45433.0, + "72": 45384.0, + "73": 42703.0, + "74": 44734.0, + "75": 43885.0, + "76": 43295.0, + "77": 41647.0, + "78": 43187.0, + "79": 45354.0, + "80": 45456.0, + "81": 42514.0, + "82": 39199.0, + "83": 45686.0, + "84": 43511.0, + "85": 44105.0, + "86": 43848.0, + "87": 41235.0, + "88": 44118.0, + "89": 40419.0, + "90": 39829.0, + "91": 46219.0, + "92": 46222.0, + "93": 43869.0, + "94": 45455.0, + "95": 41768.0, + "96": 41429.0, + "97": 41955.0, + "98": 43380.0, + "99": 39089.0, + "100": 39418.0 } }, "mem-allocated-bytes": { "start_step": 1, "end_step": 100, "step_interval": 1, + "value_precision": "full", "values": { "1": 2171549696.0, "2": 2171549696.0, @@ -324,213 +327,215 @@ "start_step": 1, "end_step": 100, "step_interval": 1, + "value_precision": "full", "values": { - "1": 2404233728.0, - "2": 3337112064.0, - "3": 3337112064.0, - "4": 3337112064.0, - "5": 3337112064.0, - "6": 3337112064.0, - "7": 3337112064.0, - "8": 3337112064.0, - "9": 3337112064.0, - "10": 3337112064.0, - "11": 3337112064.0, - "12": 3337112064.0, - "13": 3337112064.0, - "14": 3337112064.0, - "15": 3337112064.0, - "16": 3337112064.0, - "17": 3337112064.0, - "18": 3337112064.0, - "19": 3337112064.0, - "20": 3337112064.0, - "21": 3337112064.0, - "22": 3337112064.0, - "23": 3337112064.0, - "24": 3337112064.0, - "25": 3337112064.0, - "26": 3337112064.0, - "27": 3337112064.0, - "28": 3337112064.0, - "29": 3337112064.0, - "30": 3337112064.0, - "31": 3337112064.0, - "32": 3337112064.0, - "33": 3337112064.0, - "34": 3337112064.0, - "35": 3337112064.0, - "36": 3337112064.0, - "37": 3337112064.0, - "38": 3337112064.0, - "39": 3337112064.0, - "40": 3337112064.0, - "41": 3337112064.0, - "42": 3337112064.0, - "43": 3337112064.0, - "44": 3337112064.0, - "45": 3337112064.0, - "46": 3337112064.0, - "47": 3337112064.0, - "48": 3337112064.0, - "49": 3337112064.0, - "50": 3337112064.0, - "51": 3337112064.0, - "52": 3337112064.0, - "53": 3337112064.0, - "54": 3337112064.0, - "55": 3337112064.0, - "56": 3337112064.0, - "57": 3337112064.0, - "58": 3337112064.0, - "59": 3337112064.0, - "60": 3337112064.0, - "61": 3337112064.0, - "62": 3337112064.0, - "63": 3337112064.0, - "64": 3337112064.0, - "65": 3337112064.0, - "66": 3337112064.0, - "67": 3337112064.0, - "68": 3337112064.0, - "69": 3337112064.0, - "70": 3337112064.0, - "71": 3337112064.0, - "72": 3337112064.0, - "73": 3337112064.0, - "74": 3337112064.0, - "75": 3337112064.0, - "76": 3337112064.0, - "77": 3337112064.0, - "78": 3337112064.0, - "79": 3337112064.0, - "80": 3337112064.0, - "81": 3337112064.0, - "82": 3337112064.0, - "83": 3337112064.0, - "84": 3337112064.0, - "85": 3337112064.0, - "86": 3337112064.0, - "87": 3337112064.0, - "88": 3337112064.0, - "89": 3337112064.0, - "90": 3337112064.0, - "91": 3337112064.0, - "92": 3337112064.0, - "93": 3337112064.0, - "94": 3337112064.0, - "95": 3337112064.0, - "96": 3337112064.0, - "97": 3337112064.0, - "98": 3337112064.0, - "99": 3337112064.0, - "100": 3337112064.0 + "1": 2407641600.0, + "2": 3337506304.0, + "3": 3337506304.0, + "4": 3337506304.0, + "5": 3337506304.0, + "6": 3337506304.0, + "7": 3337506304.0, + "8": 3337506304.0, + "9": 3337506304.0, + "10": 3337506304.0, + "11": 3337506304.0, + "12": 3337506304.0, + "13": 3337506304.0, + "14": 3337506304.0, + "15": 3337506304.0, + "16": 3337506304.0, + "17": 3337506304.0, + "18": 3337506304.0, + "19": 3337506304.0, + "20": 3337506304.0, + "21": 3337506304.0, + "22": 3337506304.0, + "23": 3337506304.0, + "24": 3337506304.0, + "25": 3337506304.0, + "26": 3337506304.0, + "27": 3337506304.0, + "28": 3337506304.0, + "29": 3337506304.0, + "30": 3337506304.0, + "31": 3337506304.0, + "32": 3337506304.0, + "33": 3337506304.0, + "34": 3337506304.0, + "35": 3337506304.0, + "36": 3337506304.0, + "37": 3337506304.0, + "38": 3337506304.0, + "39": 3337506304.0, + "40": 3337506304.0, + "41": 3337506304.0, + "42": 3337506304.0, + "43": 3337506304.0, + "44": 3337506304.0, + "45": 3337506304.0, + "46": 3337506304.0, + "47": 3337506304.0, + "48": 3337506304.0, + "49": 3337506304.0, + "50": 3337506304.0, + "51": 3337506304.0, + "52": 3337506304.0, + "53": 3337506304.0, + "54": 3337506304.0, + "55": 3337506304.0, + "56": 3337506304.0, + "57": 3337506304.0, + "58": 3337506304.0, + "59": 3337506304.0, + "60": 3337506304.0, + "61": 3337506304.0, + "62": 3337506304.0, + "63": 3337506304.0, + "64": 3337506304.0, + "65": 3337506304.0, + "66": 3337506304.0, + "67": 3337506304.0, + "68": 3337506304.0, + "69": 3337506304.0, + "70": 3337506304.0, + "71": 3337506304.0, + "72": 3337506304.0, + "73": 3337506304.0, + "74": 3337506304.0, + "75": 3337506304.0, + "76": 3337506304.0, + "77": 3337506304.0, + "78": 3337506304.0, + "79": 3337506304.0, + "80": 3337506304.0, + "81": 3337506304.0, + "82": 3337506304.0, + "83": 3337506304.0, + "84": 3337506304.0, + "85": 3337506304.0, + "86": 3337506304.0, + "87": 3337506304.0, + "88": 3337506304.0, + "89": 3337506304.0, + "90": 3337506304.0, + "91": 3337506304.0, + "92": 3337506304.0, + "93": 3337506304.0, + "94": 3337506304.0, + "95": 3337506304.0, + "96": 3337506304.0, + "97": 3337506304.0, + "98": 3337506304.0, + "99": 3337506304.0, + "100": 3337506304.0 } }, "iteration-time": { "start_step": 2, "end_step": 100, "step_interval": 1, + "value_precision": "full", "values": { - "2": 5.53338, - "3": 0.35853, - "4": 0.34408, - "5": 1.0216, - "6": 0.34361, - "7": 0.3428, - "8": 0.34273, - "9": 0.34293, - "10": 0.34287, - "11": 0.34142, - "12": 0.37927, - "13": 0.34321, - "14": 0.34248, - "15": 0.348, - "16": 0.34989, - "17": 0.35584, - "18": 0.3517, - "19": 0.35716, - "20": 0.35642, - "21": 0.34468, - "22": 0.3429, - "23": 0.34298, - "24": 0.34279, - "25": 0.34265, - "26": 0.34273, - "27": 0.34348, - "28": 0.34371, - "29": 0.34502, - "30": 0.3432, - "31": 0.34368, - "32": 0.34353, - "33": 0.34216, - "34": 0.34348, - "35": 0.3442, - "36": 0.34396, - "37": 0.34764, - "38": 0.34295, - "39": 0.34282, - "40": 0.34289, - "41": 0.34325, - "42": 0.34284, - "43": 0.34315, - "44": 0.34259, - "45": 0.34288, - "46": 0.34262, - "47": 0.34309, - "48": 0.341, - "49": 0.34341, - "50": 0.34228, - "51": 0.34295, - "52": 0.34231, - "53": 0.34215, - "54": 0.34278, - "55": 0.34418, - "56": 0.34144, - "57": 0.3425, - "58": 0.3435, - "59": 0.34123, - "60": 0.34257, - "61": 0.34261, - "62": 0.34287, - "63": 0.34283, - "64": 0.3426, - "65": 0.34245, - "66": 0.34222, - "67": 0.3419, - "68": 0.34186, - "69": 0.34244, - "70": 0.34338, - "71": 0.34125, - "72": 0.34211, - "73": 0.34182, - "74": 0.34284, - "75": 0.34403, - "76": 0.34308, - "77": 0.34151, - "78": 0.3431, - "79": 0.34266, - "80": 0.34298, - "81": 0.34214, - "82": 0.34222, - "83": 0.3416, - "84": 0.34233, - "85": 0.34266, - "86": 0.3415, - "87": 0.34416, - "88": 0.34559, - "89": 0.34386, - "90": 0.34438, - "91": 0.34423, - "92": 0.35233, - "93": 0.35184, - "94": 0.3523, - "95": 0.35279, - "96": 0.35039, - "97": 0.34788, - "98": 0.34283, - "99": 0.35252, - "100": 0.35117 + "2": 0.436309814453125, + "3": 0.3448631763458252, + "4": 0.3337080478668213, + "5": 0.3316769599914551, + "6": 0.3306541442871094, + "7": 0.3346736431121826, + "8": 0.33601999282836914, + "9": 0.33530497550964355, + "10": 0.3345458507537842, + "11": 0.3351614475250244, + "12": 0.3367798328399658, + "13": 0.3353898525238037, + "14": 0.33373117446899414, + "15": 0.3354179859161377, + "16": 0.335507869720459, + "17": 0.3345303535461426, + "18": 0.33513808250427246, + "19": 0.3390519618988037, + "20": 0.3370475769042969, + "21": 0.3345344066619873, + "22": 0.3348982334136963, + "23": 0.3364138603210449, + "24": 0.3340120315551758, + "25": 0.33262085914611816, + "26": 0.3310062885284424, + "27": 0.3295912742614746, + "28": 0.3298354148864746, + "29": 0.33065152168273926, + "30": 0.33067822456359863, + "31": 0.3308854103088379, + "32": 0.3308134078979492, + "33": 0.3304300308227539, + "34": 0.32956624031066895, + "35": 0.3328359127044678, + "36": 0.3304882049560547, + "37": 0.3308706283569336, + "38": 0.33001255989074707, + "39": 0.33097219467163086, + "40": 0.33072328567504883, + "41": 0.3314495086669922, + "42": 0.33388209342956543, + "43": 0.3364219665527344, + "44": 0.3342761993408203, + "45": 0.334392786026001, + "46": 0.3358314037322998, + "47": 0.3355107307434082, + "48": 0.3344252109527588, + "49": 0.3352363109588623, + "50": 0.3414280414581299, + "51": 0.3411865234375, + "52": 0.34078502655029297, + "53": 0.34107255935668945, + "54": 0.3406393527984619, + "55": 0.34049081802368164, + "56": 0.33986592292785645, + "57": 0.33872008323669434, + "58": 0.34061288833618164, + "59": 0.34032344818115234, + "60": 0.34059739112854004, + "61": 0.34279799461364746, + "62": 0.34277963638305664, + "63": 0.3397941589355469, + "64": 0.34185791015625, + "65": 0.33959412574768066, + "66": 0.3402070999145508, + "67": 0.34306788444519043, + "68": 0.3404500484466553, + "69": 0.33957362174987793, + "70": 0.3408980369567871, + "71": 0.33985185623168945, + "72": 0.34020185470581055, + "73": 0.3412191867828369, + "74": 0.33903980255126953, + "75": 0.3391299247741699, + "76": 0.3404207229614258, + "77": 0.34073877334594727, + "78": 0.3390920162200928, + "79": 0.3400719165802002, + "80": 0.3399057388305664, + "81": 0.3406093120574951, + "82": 0.3399839401245117, + "83": 0.33957839012145996, + "84": 0.34017252922058105, + "85": 0.34169745445251465, + "86": 0.340576171875, + "87": 0.3406555652618408, + "88": 0.34012651443481445, + "89": 0.39943432807922363, + "90": 0.3405919075012207, + "91": 0.3427445888519287, + "92": 0.3659491539001465, + "93": 0.4031693935394287, + "94": 0.3412740230560303, + "95": 0.34098172187805176, + "96": 0.35372424125671387, + "97": 0.3472867012023926, + "98": 0.39914703369140625, + "99": 0.3451380729675293, + "100": 0.34844493865966797 } } -} \ No newline at end of file +} diff --git a/tests/functional_tests/test_cases/t5/t5_mcore_tp2_pp1_vp1/golden_values_dev_dgx_h100.json b/tests/functional_tests/test_cases/t5/t5_mcore_tp2_pp1_vp1/golden_values_dev_dgx_h100.json index 1aff3b8a761..b29c8ef9c3c 100644 --- a/tests/functional_tests/test_cases/t5/t5_mcore_tp2_pp1_vp1/golden_values_dev_dgx_h100.json +++ b/tests/functional_tests/test_cases/t5/t5_mcore_tp2_pp1_vp1/golden_values_dev_dgx_h100.json @@ -3,534 +3,539 @@ "start_step": 1, "end_step": 100, "step_interval": 1, + "value_precision": "full", "values": { - "1": 10.39951, - "2": 10.38571, - "3": 9.69428, - "4": 9.3944, - "5": 9.26539, - "6": 9.23937, - "7": 9.12962, - "8": 9.04425, - "9": 8.9724, - "10": 8.91963, - "11": 8.77348, - "12": 8.65859, - "13": 8.61186, - "14": 8.52349, - "15": 8.46189, - "16": 8.46607, - "17": 8.24355, - "18": 8.19882, - "19": 8.2498, - "20": 8.13486, - "21": 8.0384, - "22": 7.97326, - "23": 8.03247, - "24": 7.89817, - "25": 7.84733, - "26": 7.83428, - "27": 7.69687, - "28": 7.67701, - "29": 7.57165, - "30": 7.5669, - "31": 7.45146, - "32": 7.5171, - "33": 7.43238, - "34": 7.37826, - "35": 7.31989, - "36": 7.29316, - "37": 7.26709, - "38": 7.27949, - "39": 7.24309, - "40": 7.14486, - "41": 7.09381, - "42": 7.15405, - "43": 7.12312, - "44": 7.04948, - "45": 7.05693, - "46": 7.06614, - "47": 7.02152, - "48": 7.02577, - "49": 6.91681, - "50": 6.98388, - "51": 7.0916, - "52": 6.97395, - "53": 6.90599, - "54": 6.88942, - "55": 6.86568, - "56": 6.95854, - "57": 6.91829, - "58": 6.83089, - "59": 6.8893, - "60": 6.75392, - "61": 6.88649, - "62": 6.81532, - "63": 6.87794, - "64": 6.91937, - "65": 6.77826, - "66": 6.80054, - "67": 6.70994, - "68": 6.72518, - "69": 6.89208, - "70": 6.67257, - "71": 6.85799, - "72": 6.78771, - "73": 6.80511, - "74": 6.73899, - "75": 6.72846, - "76": 6.82739, - "77": 6.76865, - "78": 6.6901, - "79": 6.8056, - "80": 6.84205, - "81": 6.80129, - "82": 6.71932, - "83": 6.78489, - "84": 6.76583, - "85": 6.7547, - "86": 6.71617, - "87": 6.77246, - "88": 6.76657, - "89": 6.73319, - "90": 6.60431, - "91": 6.70489, - "92": 6.70896, - "93": 6.75381, - "94": 6.75606, - "95": 6.70767, - "96": 6.784, - "97": 6.79729, - "98": 6.66536, - "99": 6.66996, - "100": 6.7722 + "1": 10.399426460266113, + "2": 10.385791778564453, + "3": 9.69452953338623, + "4": 9.394609451293945, + "5": 9.26562786102295, + "6": 9.239529609680176, + "7": 9.129944801330566, + "8": 9.044675827026367, + "9": 8.972999572753906, + "10": 8.920392990112305, + "11": 8.774179458618164, + "12": 8.659585952758789, + "13": 8.6128568649292, + "14": 8.524750709533691, + "15": 8.463022232055664, + "16": 8.467516899108887, + "17": 8.245426177978516, + "18": 8.200642585754395, + "19": 8.25161361694336, + "20": 8.137120246887207, + "21": 8.040753364562988, + "22": 7.975871562957764, + "23": 8.034978866577148, + "24": 7.900232315063477, + "25": 7.848944187164307, + "26": 7.835908889770508, + "27": 7.698367595672607, + "28": 7.678611755371094, + "29": 7.573217868804932, + "30": 7.568641662597656, + "31": 7.453379154205322, + "32": 7.5189385414123535, + "33": 7.434437274932861, + "34": 7.380302906036377, + "35": 7.322277545928955, + "36": 7.295465469360352, + "37": 7.269250869750977, + "38": 7.2817606925964355, + "39": 7.245542526245117, + "40": 7.1475419998168945, + "41": 7.096142768859863, + "42": 7.156813621520996, + "43": 7.125374794006348, + "44": 7.051925182342529, + "45": 7.059228897094727, + "46": 7.067972183227539, + "47": 7.021419048309326, + "48": 7.0242695808410645, + "49": 6.9145894050598145, + "50": 6.982226371765137, + "51": 7.090176582336426, + "52": 6.97264289855957, + "53": 6.904593467712402, + "54": 6.88820219039917, + "55": 6.8647379875183105, + "56": 6.957503795623779, + "57": 6.917046070098877, + "58": 6.829699516296387, + "59": 6.888305187225342, + "60": 6.753302574157715, + "61": 6.885923385620117, + "62": 6.814296722412109, + "63": 6.877450466156006, + "64": 6.918863773345947, + "65": 6.777997016906738, + "66": 6.799970626831055, + "67": 6.709354400634766, + "68": 6.724421501159668, + "69": 6.891300678253174, + "70": 6.671708106994629, + "71": 6.857809066772461, + "72": 6.786740303039551, + "73": 6.804319858551025, + "74": 6.738752365112305, + "75": 6.7280192375183105, + "76": 6.826900005340576, + "77": 6.768486022949219, + "78": 6.689863681793213, + "79": 6.805016994476318, + "80": 6.841881275177002, + "81": 6.80092191696167, + "82": 6.719220161437988, + "83": 6.784390449523926, + "84": 6.765786170959473, + "85": 6.753852367401123, + "86": 6.715795040130615, + "87": 6.77221155166626, + "88": 6.765851974487305, + "89": 6.733570098876953, + "90": 6.6042985916137695, + "91": 6.704691410064697, + "92": 6.7092719078063965, + "93": 6.754078388214111, + "94": 6.756124973297119, + "95": 6.708515644073486, + "96": 6.785633563995361, + "97": 6.798409938812256, + "98": 6.665285587310791, + "99": 6.669540882110596, + "100": 6.772034645080566 } }, "num-zeros": { "start_step": 1, "end_step": 100, "step_interval": 1, + "value_precision": "full", "values": { - "1": 41058.0, - "2": 40905.0, - "3": 43263.0, - "4": 46976.0, - "5": 43072.0, - "6": 45465.0, - "7": 46985.0, - "8": 44064.0, - "9": 43998.0, - "10": 45528.0, - "11": 45341.0, - "12": 45369.0, - "13": 43947.0, - "14": 41729.0, - "15": 41858.0, - "16": 44758.0, - "17": 43850.0, - "18": 45365.0, - "19": 43995.0, - "20": 46270.0, - "21": 44045.0, - "22": 45479.0, - "23": 41870.0, - "24": 43206.0, - "25": 46947.0, - "26": 43261.0, - "27": 44604.0, - "28": 43469.0, - "29": 44852.0, - "30": 45485.0, - "31": 44611.0, + "1": 41060.0, + "2": 40914.0, + "3": 43268.0, + "4": 46994.0, + "5": 43085.0, + "6": 45482.0, + "7": 46970.0, + "8": 44060.0, + "9": 44003.0, + "10": 45524.0, + "11": 45344.0, + "12": 45368.0, + "13": 43949.0, + "14": 41737.0, + "15": 41842.0, + "16": 44762.0, + "17": 43836.0, + "18": 45362.0, + "19": 43986.0, + "20": 46265.0, + "21": 44046.0, + "22": 45480.0, + "23": 41872.0, + "24": 43193.0, + "25": 46938.0, + "26": 43253.0, + "27": 44611.0, + "28": 43462.0, + "29": 44851.0, + "30": 45481.0, + "31": 44607.0, "32": 46173.0, - "33": 45591.0, - "34": 47071.0, + "33": 45587.0, + "34": 47075.0, "35": 43249.0, "36": 46971.0, - "37": 45580.0, - "38": 42401.0, - "39": 45547.0, - "40": 46912.0, + "37": 45567.0, + "38": 42397.0, + "39": 45529.0, + "40": 46899.0, "41": 43378.0, - "42": 45463.0, - "43": 46251.0, - "44": 43290.0, - "45": 43166.0, - "46": 44615.0, - "47": 43207.0, - "48": 44596.0, - "49": 43886.0, - "50": 45407.0, - "51": 42007.0, - "52": 42015.0, - "53": 47086.0, - "54": 44763.0, - "55": 43991.0, - "56": 44063.0, - "57": 43391.0, - "58": 44711.0, - "59": 42434.0, - "60": 43134.0, - "61": 44607.0, - "62": 43936.0, - "63": 42369.0, - "64": 43980.0, - "65": 45546.0, - "66": 42759.0, - "67": 43920.0, - "68": 44046.0, - "69": 41596.0, - "70": 41664.0, - "71": 45458.0, - "72": 45347.0, - "73": 42728.0, - "74": 44723.0, - "75": 43921.0, - "76": 43294.0, - "77": 41632.0, - "78": 43176.0, - "79": 45377.0, - "80": 45478.0, - "81": 42490.0, - "82": 39181.0, - "83": 45682.0, - "84": 43470.0, - "85": 44097.0, - "86": 43935.0, - "87": 41214.0, - "88": 44131.0, - "89": 40417.0, - "90": 39852.0, - "91": 46185.0, - "92": 46215.0, - "93": 43863.0, - "94": 45450.0, - "95": 41757.0, - "96": 41347.0, - "97": 42007.0, - "98": 43468.0, - "99": 39039.0, - "100": 39402.0 + "42": 45459.0, + "43": 46263.0, + "44": 43286.0, + "45": 43175.0, + "46": 44630.0, + "47": 43215.0, + "48": 44606.0, + "49": 43921.0, + "50": 45404.0, + "51": 42030.0, + "52": 42023.0, + "53": 47075.0, + "54": 44772.0, + "55": 44014.0, + "56": 44068.0, + "57": 43386.0, + "58": 44702.0, + "59": 42423.0, + "60": 43135.0, + "61": 44593.0, + "62": 43943.0, + "63": 42381.0, + "64": 43983.0, + "65": 45534.0, + "66": 42753.0, + "67": 43933.0, + "68": 44054.0, + "69": 41593.0, + "70": 41668.0, + "71": 45454.0, + "72": 45344.0, + "73": 42724.0, + "74": 44718.0, + "75": 43925.0, + "76": 43293.0, + "77": 41625.0, + "78": 43190.0, + "79": 45373.0, + "80": 45477.0, + "81": 42493.0, + "82": 39172.0, + "83": 45683.0, + "84": 43468.0, + "85": 44122.0, + "86": 43906.0, + "87": 41210.0, + "88": 44136.0, + "89": 40418.0, + "90": 39877.0, + "91": 46165.0, + "92": 46224.0, + "93": 43856.0, + "94": 45447.0, + "95": 41747.0, + "96": 41355.0, + "97": 42006.0, + "98": 43463.0, + "99": 39025.0, + "100": 39414.0 } }, "mem-allocated-bytes": { "start_step": 1, "end_step": 100, "step_interval": 1, + "value_precision": "full", "values": { - "1": 2171287552.0, - "2": 2171287552.0, - "3": 2171287552.0, - "4": 2171287552.0, - "5": 2171287552.0, - "6": 2171287552.0, - "7": 2171287552.0, - "8": 2171287552.0, - "9": 2171287552.0, - "10": 2171287552.0, - "11": 2171287552.0, - "12": 2171287552.0, - "13": 2171287552.0, - "14": 2171287552.0, - "15": 2171287552.0, - "16": 2171287552.0, - "17": 2171287552.0, - "18": 2171287552.0, - "19": 2171287552.0, - "20": 2171287552.0, - "21": 2171287552.0, - "22": 2171287552.0, - "23": 2171287552.0, - "24": 2171287552.0, - "25": 2171287552.0, - "26": 2171287552.0, - "27": 2171287552.0, - "28": 2171287552.0, - "29": 2171287552.0, - "30": 2171287552.0, - "31": 2171287552.0, - "32": 2171287552.0, - "33": 2171287552.0, - "34": 2171287552.0, - "35": 2171287552.0, - "36": 2171287552.0, - "37": 2171287552.0, - "38": 2171287552.0, - "39": 2171287552.0, - "40": 2171287552.0, - "41": 2171287552.0, - "42": 2171287552.0, - "43": 2171287552.0, - "44": 2171287552.0, - "45": 2171287552.0, - "46": 2171287552.0, - "47": 2171287552.0, - "48": 2171287552.0, - "49": 2171287552.0, - "50": 2171287552.0, - "51": 2171287552.0, - "52": 2171287552.0, - "53": 2171287552.0, - "54": 2171287552.0, - "55": 2171287552.0, - "56": 2171287552.0, - "57": 2171287552.0, - "58": 2171287552.0, - "59": 2171287552.0, - "60": 2171287552.0, - "61": 2171287552.0, - "62": 2171287552.0, - "63": 2171287552.0, - "64": 2171287552.0, - "65": 2171287552.0, - "66": 2171287552.0, - "67": 2171287552.0, - "68": 2171287552.0, - "69": 2171287552.0, - "70": 2171287552.0, - "71": 2171287552.0, - "72": 2171287552.0, - "73": 2171287552.0, - "74": 2171287552.0, - "75": 2171287552.0, - "76": 2171287552.0, - "77": 2171287552.0, - "78": 2171287552.0, - "79": 2171287552.0, - "80": 2171287552.0, - "81": 2171287552.0, - "82": 2171287552.0, - "83": 2171287552.0, - "84": 2171287552.0, - "85": 2171287552.0, - "86": 2171287552.0, - "87": 2171287552.0, - "88": 2171287552.0, - "89": 2171287552.0, - "90": 2171287552.0, - "91": 2171287552.0, - "92": 2171287552.0, - "93": 2171287552.0, - "94": 2171287552.0, - "95": 2171287552.0, - "96": 2171287552.0, - "97": 2171287552.0, - "98": 2171287552.0, - "99": 2171287552.0, - "100": 2171287552.0 + "1": 2172073984.0, + "2": 2172073984.0, + "3": 2172073984.0, + "4": 2172073984.0, + "5": 2172073984.0, + "6": 2172073984.0, + "7": 2172073984.0, + "8": 2172073984.0, + "9": 2172073984.0, + "10": 2172073984.0, + "11": 2172073984.0, + "12": 2172073984.0, + "13": 2172073984.0, + "14": 2172073984.0, + "15": 2172073984.0, + "16": 2172073984.0, + "17": 2172073984.0, + "18": 2172073984.0, + "19": 2172073984.0, + "20": 2172073984.0, + "21": 2172073984.0, + "22": 2172073984.0, + "23": 2172073984.0, + "24": 2172073984.0, + "25": 2172073984.0, + "26": 2172073984.0, + "27": 2172073984.0, + "28": 2172073984.0, + "29": 2172073984.0, + "30": 2172073984.0, + "31": 2172073984.0, + "32": 2172073984.0, + "33": 2172073984.0, + "34": 2172073984.0, + "35": 2172073984.0, + "36": 2172073984.0, + "37": 2172073984.0, + "38": 2172073984.0, + "39": 2172073984.0, + "40": 2172073984.0, + "41": 2172073984.0, + "42": 2172073984.0, + "43": 2172073984.0, + "44": 2172073984.0, + "45": 2172073984.0, + "46": 2172073984.0, + "47": 2172073984.0, + "48": 2172073984.0, + "49": 2172073984.0, + "50": 2172073984.0, + "51": 2172073984.0, + "52": 2172073984.0, + "53": 2172073984.0, + "54": 2172073984.0, + "55": 2172073984.0, + "56": 2172073984.0, + "57": 2172073984.0, + "58": 2172073984.0, + "59": 2172073984.0, + "60": 2172073984.0, + "61": 2172073984.0, + "62": 2172073984.0, + "63": 2172073984.0, + "64": 2172073984.0, + "65": 2172073984.0, + "66": 2172073984.0, + "67": 2172073984.0, + "68": 2172073984.0, + "69": 2172073984.0, + "70": 2172073984.0, + "71": 2172073984.0, + "72": 2172073984.0, + "73": 2172073984.0, + "74": 2172073984.0, + "75": 2172073984.0, + "76": 2172073984.0, + "77": 2172073984.0, + "78": 2172073984.0, + "79": 2172073984.0, + "80": 2172073984.0, + "81": 2172073984.0, + "82": 2172073984.0, + "83": 2172073984.0, + "84": 2172073984.0, + "85": 2172073984.0, + "86": 2172073984.0, + "87": 2172073984.0, + "88": 2172073984.0, + "89": 2172073984.0, + "90": 2172073984.0, + "91": 2172073984.0, + "92": 2172073984.0, + "93": 2172073984.0, + "94": 2172073984.0, + "95": 2172073984.0, + "96": 2172073984.0, + "97": 2172073984.0, + "98": 2172073984.0, + "99": 2172073984.0, + "100": 2172073984.0 } }, "mem-max-allocated-bytes": { "start_step": 1, "end_step": 100, "step_interval": 1, + "value_precision": "full", "values": { - "1": 2402923008.0, - "2": 3332787712.0, - "3": 3332787712.0, - "4": 3332787712.0, - "5": 3332787712.0, - "6": 3332787712.0, - "7": 3332787712.0, - "8": 3332787712.0, - "9": 3332787712.0, - "10": 3332787712.0, - "11": 3332787712.0, - "12": 3332787712.0, - "13": 3332787712.0, - "14": 3332787712.0, - "15": 3332787712.0, - "16": 3332787712.0, - "17": 3332787712.0, - "18": 3332787712.0, - "19": 3332787712.0, - "20": 3332787712.0, - "21": 3332787712.0, - "22": 3332787712.0, - "23": 3332787712.0, - "24": 3332787712.0, - "25": 3332787712.0, - "26": 3332787712.0, - "27": 3332787712.0, - "28": 3332787712.0, - "29": 3332787712.0, - "30": 3332787712.0, - "31": 3332787712.0, - "32": 3332787712.0, - "33": 3332787712.0, - "34": 3332787712.0, - "35": 3332787712.0, - "36": 3332787712.0, - "37": 3332787712.0, - "38": 3332787712.0, - "39": 3332787712.0, - "40": 3332787712.0, - "41": 3332787712.0, - "42": 3332787712.0, - "43": 3332787712.0, - "44": 3332787712.0, - "45": 3332787712.0, - "46": 3332787712.0, - "47": 3332787712.0, - "48": 3332787712.0, - "49": 3332787712.0, - "50": 3332787712.0, - "51": 3332787712.0, - "52": 3332787712.0, - "53": 3332787712.0, - "54": 3332787712.0, - "55": 3332787712.0, - "56": 3332787712.0, - "57": 3332787712.0, - "58": 3332787712.0, - "59": 3332787712.0, - "60": 3332787712.0, - "61": 3332787712.0, - "62": 3332787712.0, - "63": 3332787712.0, - "64": 3332787712.0, - "65": 3332787712.0, - "66": 3332787712.0, - "67": 3332787712.0, - "68": 3332787712.0, - "69": 3332787712.0, - "70": 3332787712.0, - "71": 3332787712.0, - "72": 3332787712.0, - "73": 3332787712.0, - "74": 3332787712.0, - "75": 3332787712.0, - "76": 3332787712.0, - "77": 3332787712.0, - "78": 3332787712.0, - "79": 3332787712.0, - "80": 3332787712.0, - "81": 3332787712.0, - "82": 3332787712.0, - "83": 3332787712.0, - "84": 3332787712.0, - "85": 3332787712.0, - "86": 3332787712.0, - "87": 3332787712.0, - "88": 3332787712.0, - "89": 3332787712.0, - "90": 3332787712.0, - "91": 3332787712.0, - "92": 3332787712.0, - "93": 3332787712.0, - "94": 3332787712.0, - "95": 3332787712.0, - "96": 3332787712.0, - "97": 3332787712.0, - "98": 3332787712.0, - "99": 3332787712.0, - "100": 3332787712.0 + "1": 2406724096.0, + "2": 3337113088.0, + "3": 3337113088.0, + "4": 3337113088.0, + "5": 3337113088.0, + "6": 3337113088.0, + "7": 3337113088.0, + "8": 3337113088.0, + "9": 3337113088.0, + "10": 3337113088.0, + "11": 3337113088.0, + "12": 3337113088.0, + "13": 3337113088.0, + "14": 3337113088.0, + "15": 3337113088.0, + "16": 3337113088.0, + "17": 3337113088.0, + "18": 3337113088.0, + "19": 3337113088.0, + "20": 3337113088.0, + "21": 3337113088.0, + "22": 3337113088.0, + "23": 3337113088.0, + "24": 3337113088.0, + "25": 3337113088.0, + "26": 3337113088.0, + "27": 3337113088.0, + "28": 3337113088.0, + "29": 3337113088.0, + "30": 3337113088.0, + "31": 3337113088.0, + "32": 3337113088.0, + "33": 3337113088.0, + "34": 3337113088.0, + "35": 3337113088.0, + "36": 3337113088.0, + "37": 3337113088.0, + "38": 3337113088.0, + "39": 3337113088.0, + "40": 3337113088.0, + "41": 3337113088.0, + "42": 3337113088.0, + "43": 3337113088.0, + "44": 3337113088.0, + "45": 3337113088.0, + "46": 3337113088.0, + "47": 3337113088.0, + "48": 3337113088.0, + "49": 3337113088.0, + "50": 3337113088.0, + "51": 3337113088.0, + "52": 3337113088.0, + "53": 3337113088.0, + "54": 3337113088.0, + "55": 3337113088.0, + "56": 3337113088.0, + "57": 3337113088.0, + "58": 3337113088.0, + "59": 3337113088.0, + "60": 3337113088.0, + "61": 3337113088.0, + "62": 3337113088.0, + "63": 3337113088.0, + "64": 3337113088.0, + "65": 3337113088.0, + "66": 3337113088.0, + "67": 3337113088.0, + "68": 3337113088.0, + "69": 3337113088.0, + "70": 3337113088.0, + "71": 3337113088.0, + "72": 3337113088.0, + "73": 3337113088.0, + "74": 3337113088.0, + "75": 3337113088.0, + "76": 3337113088.0, + "77": 3337113088.0, + "78": 3337113088.0, + "79": 3337113088.0, + "80": 3337113088.0, + "81": 3337113088.0, + "82": 3337113088.0, + "83": 3337113088.0, + "84": 3337113088.0, + "85": 3337113088.0, + "86": 3337113088.0, + "87": 3337113088.0, + "88": 3337113088.0, + "89": 3337113088.0, + "90": 3337113088.0, + "91": 3337113088.0, + "92": 3337113088.0, + "93": 3337113088.0, + "94": 3337113088.0, + "95": 3337113088.0, + "96": 3337113088.0, + "97": 3337113088.0, + "98": 3337113088.0, + "99": 3337113088.0, + "100": 3337113088.0 } }, "iteration-time": { "start_step": 2, "end_step": 100, "step_interval": 1, + "value_precision": "full", "values": { - "2": 6.70584, - "3": 0.26196, - "4": 0.48042, - "5": 0.24385, - "6": 0.25249, - "7": 0.23905, - "8": 0.36052, - "9": 0.24012, - "10": 0.32335, - "11": 0.35108, - "12": 0.23558, - "13": 0.23532, - "14": 0.23327, - "15": 0.23315, - "16": 0.23887, - "17": 0.23812, - "18": 0.2347, - "19": 0.23621, - "20": 0.24252, - "21": 0.23182, - "22": 0.23055, - "23": 0.23048, - "24": 0.2335, - "25": 0.22964, - "26": 0.23028, - "27": 0.23105, - "28": 0.23107, - "29": 1.05837, - "30": 0.23439, - "31": 0.23136, - "32": 0.23099, - "33": 0.23078, - "34": 0.23072, - "35": 0.23039, - "36": 0.23988, - "37": 0.23286, - "38": 0.23315, - "39": 0.2323, - "40": 0.23123, - "41": 0.23058, - "42": 0.23348, - "43": 0.23675, - "44": 0.23107, - "45": 0.23124, - "46": 0.23123, - "47": 0.23023, - "48": 0.22963, - "49": 0.22984, - "50": 0.23333, - "51": 0.22995, - "52": 0.23153, - "53": 0.22949, - "54": 0.22941, - "55": 0.23036, - "56": 0.23048, - "57": 0.23223, - "58": 0.23119, - "59": 0.22933, - "60": 0.23197, - "61": 0.23202, - "62": 0.23068, - "63": 0.23236, - "64": 0.23089, - "65": 0.23629, - "66": 0.23103, - "67": 0.2334, - "68": 0.22989, - "69": 0.23132, - "70": 0.23022, - "71": 0.2293, - "72": 0.2298, - "73": 0.23296, - "74": 0.2289, - "75": 0.22925, - "76": 0.22919, - "77": 0.22927, - "78": 0.22965, - "79": 0.22866, - "80": 0.22996, - "81": 0.22811, - "82": 0.23274, - "83": 0.22915, - "84": 0.23159, - "85": 0.22969, - "86": 0.23009, - "87": 0.22959, - "88": 0.22852, - "89": 0.22732, - "90": 0.23206, - "91": 0.37377, - "92": 0.23344, - "93": 0.23282, - "94": 0.23302, - "95": 0.23417, - "96": 0.23546, - "97": 0.23772, - "98": 0.35804, - "99": 0.27596, - "100": 0.23675 + "2": 0.35861802101135254, + "3": 0.2634587287902832, + "4": 0.27805495262145996, + "5": 0.2311718463897705, + "6": 0.2755892276763916, + "7": 0.30517578125, + "8": 0.2745015621185303, + "9": 0.2841975688934326, + "10": 0.3011777400970459, + "11": 0.2723205089569092, + "12": 0.2418208122253418, + "13": 0.24179697036743164, + "14": 0.24392294883728027, + "15": 0.2418208122253418, + "16": 0.2426905632019043, + "17": 0.2329714298248291, + "18": 0.23312687873840332, + "19": 0.24397969245910645, + "20": 0.2367696762084961, + "21": 0.23555898666381836, + "22": 0.2516493797302246, + "23": 0.23411178588867188, + "24": 0.23275136947631836, + "25": 0.24016666412353516, + "26": 0.22649455070495605, + "27": 0.22089147567749023, + "28": 0.2199993133544922, + "29": 0.2230377197265625, + "30": 0.22208833694458008, + "31": 0.22444963455200195, + "32": 0.2202150821685791, + "33": 0.21844077110290527, + "34": 0.2209758758544922, + "35": 0.22067975997924805, + "36": 0.22008132934570312, + "37": 0.21879863739013672, + "38": 0.21829748153686523, + "39": 0.21923112869262695, + "40": 0.22410917282104492, + "41": 0.22061586380004883, + "42": 0.22414779663085938, + "43": 0.22202754020690918, + "44": 0.22270560264587402, + "45": 0.22231054306030273, + "46": 0.22220301628112793, + "47": 0.22289347648620605, + "48": 0.22387027740478516, + "49": 0.22531795501708984, + "50": 0.2252027988433838, + "51": 0.22182035446166992, + "52": 0.22213315963745117, + "53": 0.2223820686340332, + "54": 0.22008752822875977, + "55": 0.2197742462158203, + "56": 0.2197263240814209, + "57": 0.21973776817321777, + "58": 0.22022461891174316, + "59": 0.2198505401611328, + "60": 0.2173619270324707, + "61": 0.21826648712158203, + "62": 0.218841552734375, + "63": 0.2180776596069336, + "64": 0.21796584129333496, + "65": 0.21694135665893555, + "66": 0.22083115577697754, + "67": 0.22264313697814941, + "68": 0.2222602367401123, + "69": 0.22091317176818848, + "70": 0.2229595184326172, + "71": 0.22348761558532715, + "72": 0.2264392375946045, + "73": 0.22520184516906738, + "74": 0.2293410301208496, + "75": 0.22809767723083496, + "76": 0.22878670692443848, + "77": 0.22696375846862793, + "78": 0.2252969741821289, + "79": 0.22527027130126953, + "80": 0.22532868385314941, + "81": 0.22556066513061523, + "82": 0.22553014755249023, + "83": 0.22348928451538086, + "84": 0.22576332092285156, + "85": 0.22304844856262207, + "86": 0.21872806549072266, + "87": 0.263749361038208, + "88": 0.2577354907989502, + "89": 0.48883676528930664, + "90": 0.21892023086547852, + "91": 0.22089481353759766, + "92": 0.5032768249511719, + "93": 0.22866415977478027, + "94": 0.2188413143157959, + "95": 0.5306363105773926, + "96": 0.22100234031677246, + "97": 0.2224564552307129, + "98": 0.5105063915252686, + "99": 0.2188866138458252, + "100": 0.26142072677612305 } } -} \ No newline at end of file +} diff --git a/tests/functional_tests/test_cases/t5/t5_mcore_tp4_pp1/golden_values_dev_dgx_h100.json b/tests/functional_tests/test_cases/t5/t5_mcore_tp4_pp1/golden_values_dev_dgx_h100.json index 04387996c42..50eb89b9df3 100644 --- a/tests/functional_tests/test_cases/t5/t5_mcore_tp4_pp1/golden_values_dev_dgx_h100.json +++ b/tests/functional_tests/test_cases/t5/t5_mcore_tp4_pp1/golden_values_dev_dgx_h100.json @@ -3,534 +3,539 @@ "start_step": 1, "end_step": 100, "step_interval": 1, + "value_precision": "full", "values": { - "1": 10.36299, - "2": 10.35648, - "3": 9.6273, - "4": 9.36316, - "5": 9.23355, - "6": 9.24064, - "7": 9.10948, - "8": 8.9972, - "9": 8.89871, - "10": 8.8578, - "11": 8.71304, - "12": 8.6491, - "13": 8.59442, - "14": 8.5273, - "15": 8.46723, - "16": 8.48138, - "17": 8.25532, - "18": 8.19955, - "19": 8.24447, - "20": 8.14935, - "21": 8.04111, - "22": 7.97705, - "23": 8.03616, - "24": 7.89733, - "25": 7.85265, - "26": 7.83076, - "27": 7.7087, - "28": 7.6812, - "29": 7.56568, - "30": 7.56875, - "31": 7.45741, - "32": 7.52639, - "33": 7.43899, - "34": 7.37987, - "35": 7.33105, - "36": 7.29388, - "37": 7.26307, - "38": 7.28482, - "39": 7.25488, - "40": 7.15453, - "41": 7.09625, - "42": 7.15802, - "43": 7.13206, - "44": 7.04964, - "45": 7.06371, - "46": 7.07436, - "47": 7.02344, - "48": 7.03619, - "49": 6.92987, - "50": 6.99378, - "51": 7.10387, - "52": 6.99412, - "53": 6.91395, - "54": 6.90416, - "55": 6.87224, - "56": 6.96987, - "57": 6.92462, - "58": 6.83845, - "59": 6.90284, - "60": 6.75424, - "61": 6.89414, - "62": 6.83224, - "63": 6.89201, - "64": 6.92617, - "65": 6.78253, - "66": 6.81281, - "67": 6.7195, - "68": 6.73283, - "69": 6.88373, - "70": 6.67708, - "71": 6.86122, - "72": 6.78638, - "73": 6.80829, - "74": 6.73557, - "75": 6.74053, - "76": 6.8321, - "77": 6.77202, - "78": 6.69852, - "79": 6.81094, - "80": 6.85116, - "81": 6.80195, - "82": 6.72409, - "83": 6.78649, - "84": 6.76935, - "85": 6.75765, - "86": 6.71833, - "87": 6.76916, - "88": 6.76835, - "89": 6.74162, - "90": 6.60343, - "91": 6.71499, - "92": 6.7073, - "93": 6.75676, - "94": 6.76516, - "95": 6.69778, - "96": 6.78163, - "97": 6.7916, - "98": 6.665, - "99": 6.66887, - "100": 6.76763 + "1": 10.362956047058105, + "2": 10.356405258178711, + "3": 9.627273559570312, + "4": 9.363333702087402, + "5": 9.233860969543457, + "6": 9.240845680236816, + "7": 9.109770774841309, + "8": 8.997429847717285, + "9": 8.898931503295898, + "10": 8.8584623336792, + "11": 8.71377944946289, + "12": 8.650016784667969, + "13": 8.595564842224121, + "14": 8.528546333312988, + "15": 8.468491554260254, + "16": 8.48269271850586, + "17": 8.257022857666016, + "18": 8.2014741897583, + "19": 8.2464599609375, + "20": 8.151448249816895, + "21": 8.043624877929688, + "22": 7.979751110076904, + "23": 8.038796424865723, + "24": 7.899237632751465, + "25": 7.854281902313232, + "26": 7.832311630249023, + "27": 7.710287570953369, + "28": 7.68275785446167, + "29": 7.5674262046813965, + "30": 7.570520401000977, + "31": 7.459196090698242, + "32": 7.528255462646484, + "33": 7.441061019897461, + "34": 7.381735324859619, + "35": 7.333186626434326, + "36": 7.295981407165527, + "37": 7.265478134155273, + "38": 7.287073135375977, + "39": 7.257126808166504, + "40": 7.156900405883789, + "41": 7.098905563354492, + "42": 7.16079044342041, + "43": 7.134678363800049, + "44": 7.052248001098633, + "45": 7.066233158111572, + "46": 7.076325416564941, + "47": 7.023309707641602, + "48": 7.035020351409912, + "49": 6.927795886993408, + "50": 6.992145538330078, + "51": 7.102492809295654, + "52": 6.992738246917725, + "53": 6.912599086761475, + "54": 6.903322219848633, + "55": 6.871036052703857, + "56": 6.96876859664917, + "57": 6.924088478088379, + "58": 6.837710380554199, + "59": 6.901991844177246, + "60": 6.7534589767456055, + "61": 6.893715858459473, + "62": 6.831646919250488, + "63": 6.891280651092529, + "64": 6.92561674118042, + "65": 6.78196382522583, + "66": 6.812472343444824, + "67": 6.719153881072998, + "68": 6.732641696929932, + "69": 6.883572101593018, + "70": 6.677075386047363, + "71": 6.860844135284424, + "72": 6.786078453063965, + "73": 6.80823278427124, + "74": 6.735328197479248, + "75": 6.740242958068848, + "76": 6.831969738006592, + "77": 6.771693229675293, + "78": 6.6976728439331055, + "79": 6.8105268478393555, + "80": 6.850856781005859, + "81": 6.801382064819336, + "82": 6.724090576171875, + "83": 6.785991668701172, + "84": 6.7691426277160645, + "85": 6.757512092590332, + "86": 6.717645645141602, + "87": 6.769176006317139, + "88": 6.767646312713623, + "89": 6.7422380447387695, + "90": 6.603097915649414, + "91": 6.714541912078857, + "92": 6.707432270050049, + "93": 6.756669521331787, + "94": 6.765012264251709, + "95": 6.697880744934082, + "96": 6.781554698944092, + "97": 6.791520118713379, + "98": 6.665158748626709, + "99": 6.668912887573242, + "100": 6.767621040344238 } }, "num-zeros": { "start_step": 1, "end_step": 100, "step_interval": 1, + "value_precision": "full", "values": { - "1": 41056.0, - "2": 40904.0, - "3": 43306.0, - "4": 46989.0, - "5": 43080.0, - "6": 45460.0, - "7": 46971.0, - "8": 44058.0, - "9": 44010.0, - "10": 45502.0, - "11": 45362.0, - "12": 45377.0, - "13": 43980.0, - "14": 41758.0, - "15": 41884.0, - "16": 44781.0, - "17": 43870.0, - "18": 45372.0, - "19": 44025.0, - "20": 46307.0, - "21": 44052.0, - "22": 45472.0, + "1": 41050.0, + "2": 40916.0, + "3": 43300.0, + "4": 46985.0, + "5": 43101.0, + "6": 45458.0, + "7": 46969.0, + "8": 44055.0, + "9": 43993.0, + "10": 45500.0, + "11": 45372.0, + "12": 45375.0, + "13": 43976.0, + "14": 41749.0, + "15": 41896.0, + "16": 44767.0, + "17": 43878.0, + "18": 45371.0, + "19": 44044.0, + "20": 46296.0, + "21": 44055.0, + "22": 45467.0, "23": 41869.0, - "24": 43213.0, - "25": 46922.0, - "26": 43257.0, - "27": 44625.0, - "28": 43463.0, - "29": 44843.0, - "30": 45509.0, - "31": 44608.0, - "32": 46162.0, - "33": 45558.0, - "34": 47039.0, - "35": 43219.0, - "36": 47008.0, - "37": 45542.0, - "38": 42385.0, - "39": 45535.0, - "40": 46904.0, - "41": 43403.0, - "42": 45368.0, - "43": 46185.0, - "44": 43337.0, - "45": 43181.0, - "46": 44600.0, - "47": 43258.0, - "48": 44599.0, - "49": 43849.0, - "50": 45392.0, - "51": 41929.0, - "52": 41997.0, + "24": 43208.0, + "25": 46944.0, + "26": 43256.0, + "27": 44616.0, + "28": 43481.0, + "29": 44854.0, + "30": 45505.0, + "31": 44604.0, + "32": 46140.0, + "33": 45544.0, + "34": 47047.0, + "35": 43217.0, + "36": 47017.0, + "37": 45539.0, + "38": 42376.0, + "39": 45546.0, + "40": 46888.0, + "41": 43417.0, + "42": 45371.0, + "43": 46199.0, + "44": 43318.0, + "45": 43173.0, + "46": 44607.0, + "47": 43269.0, + "48": 44605.0, + "49": 43848.0, + "50": 45408.0, + "51": 41919.0, + "52": 42017.0, "53": 47089.0, - "54": 44590.0, - "55": 44000.0, - "56": 44054.0, - "57": 43449.0, - "58": 44697.0, - "59": 42394.0, - "60": 43182.0, - "61": 44623.0, - "62": 43887.0, - "63": 42379.0, - "64": 43988.0, - "65": 45487.0, - "66": 42729.0, - "67": 43924.0, - "68": 43962.0, - "69": 41641.0, - "70": 41661.0, - "71": 45458.0, - "72": 45397.0, - "73": 42717.0, - "74": 44760.0, - "75": 43882.0, - "76": 43333.0, - "77": 41645.0, - "78": 43192.0, - "79": 45393.0, - "80": 45452.0, - "81": 42459.0, - "82": 39226.0, - "83": 45659.0, - "84": 43449.0, - "85": 43995.0, + "54": 44599.0, + "55": 44016.0, + "56": 44051.0, + "57": 43416.0, + "58": 44688.0, + "59": 42376.0, + "60": 43179.0, + "61": 44640.0, + "62": 43898.0, + "63": 42391.0, + "64": 43996.0, + "65": 45480.0, + "66": 42736.0, + "67": 43934.0, + "68": 43933.0, + "69": 41633.0, + "70": 41665.0, + "71": 45464.0, + "72": 45400.0, + "73": 42748.0, + "74": 44757.0, + "75": 43889.0, + "76": 43343.0, + "77": 41658.0, + "78": 43190.0, + "79": 45402.0, + "80": 45432.0, + "81": 42470.0, + "82": 39209.0, + "83": 45661.0, + "84": 43438.0, + "85": 44000.0, "86": 43856.0, - "87": 41265.0, - "88": 43916.0, - "89": 40436.0, - "90": 39794.0, - "91": 46225.0, - "92": 46227.0, - "93": 43878.0, - "94": 45486.0, - "95": 41760.0, - "96": 41453.0, - "97": 42078.0, - "98": 43357.0, - "99": 39075.0, - "100": 39466.0 + "87": 41272.0, + "88": 43922.0, + "89": 40417.0, + "90": 39796.0, + "91": 46261.0, + "92": 46216.0, + "93": 43882.0, + "94": 45499.0, + "95": 41767.0, + "96": 41468.0, + "97": 42057.0, + "98": 43360.0, + "99": 39079.0, + "100": 39437.0 } }, "mem-allocated-bytes": { "start_step": 1, "end_step": 100, "step_interval": 1, + "value_precision": "full", "values": { - "1": 1106624512.0, - "2": 1106624512.0, - "3": 1106624512.0, - "4": 1106624512.0, - "5": 1106624512.0, - "6": 1106624512.0, - "7": 1106624512.0, - "8": 1106624512.0, - "9": 1106624512.0, - "10": 1106624512.0, - "11": 1106624512.0, - "12": 1106624512.0, - "13": 1106624512.0, - "14": 1106624512.0, - "15": 1106624512.0, - "16": 1106624512.0, - "17": 1106624512.0, - "18": 1106624512.0, - "19": 1106624512.0, - "20": 1106624512.0, - "21": 1106624512.0, - "22": 1106624512.0, - "23": 1106624512.0, - "24": 1106624512.0, - "25": 1106624512.0, - "26": 1106624512.0, - "27": 1106624512.0, - "28": 1106624512.0, - "29": 1106624512.0, - "30": 1106624512.0, - "31": 1106624512.0, - "32": 1106624512.0, - "33": 1106624512.0, - "34": 1106624512.0, - "35": 1106624512.0, - "36": 1106624512.0, - "37": 1106624512.0, - "38": 1106624512.0, - "39": 1106624512.0, - "40": 1106624512.0, - "41": 1106624512.0, - "42": 1106624512.0, - "43": 1106624512.0, - "44": 1106624512.0, - "45": 1106624512.0, - "46": 1106624512.0, - "47": 1106624512.0, - "48": 1106624512.0, - "49": 1106624512.0, - "50": 1106624512.0, - "51": 1106624512.0, - "52": 1106624512.0, - "53": 1106624512.0, - "54": 1106624512.0, - "55": 1106624512.0, - "56": 1106624512.0, - "57": 1106624512.0, - "58": 1106624512.0, - "59": 1106624512.0, - "60": 1106624512.0, - "61": 1106624512.0, - "62": 1106624512.0, - "63": 1106624512.0, - "64": 1106624512.0, - "65": 1106624512.0, - "66": 1106624512.0, - "67": 1106624512.0, - "68": 1106624512.0, - "69": 1106624512.0, - "70": 1106624512.0, - "71": 1106624512.0, - "72": 1106624512.0, - "73": 1106624512.0, - "74": 1106624512.0, - "75": 1106624512.0, - "76": 1106624512.0, - "77": 1106624512.0, - "78": 1106624512.0, - "79": 1106624512.0, - "80": 1106624512.0, - "81": 1106624512.0, - "82": 1106624512.0, - "83": 1106624512.0, - "84": 1106624512.0, - "85": 1106624512.0, - "86": 1106624512.0, - "87": 1106624512.0, - "88": 1106624512.0, - "89": 1106624512.0, - "90": 1106624512.0, - "91": 1106624512.0, - "92": 1106624512.0, - "93": 1106624512.0, - "94": 1106624512.0, - "95": 1106624512.0, - "96": 1106624512.0, - "97": 1106624512.0, - "98": 1106624512.0, - "99": 1106624512.0, - "100": 1106624512.0 + "1": 1105414656.0, + "2": 1105414656.0, + "3": 1105414656.0, + "4": 1105414656.0, + "5": 1105414656.0, + "6": 1105414656.0, + "7": 1105414656.0, + "8": 1105414656.0, + "9": 1105414656.0, + "10": 1105414656.0, + "11": 1105414656.0, + "12": 1105414656.0, + "13": 1105414656.0, + "14": 1105414656.0, + "15": 1105414656.0, + "16": 1105414656.0, + "17": 1105414656.0, + "18": 1105414656.0, + "19": 1105414656.0, + "20": 1105414656.0, + "21": 1105414656.0, + "22": 1105414656.0, + "23": 1105414656.0, + "24": 1105414656.0, + "25": 1105414656.0, + "26": 1105414656.0, + "27": 1105414656.0, + "28": 1105414656.0, + "29": 1105414656.0, + "30": 1105414656.0, + "31": 1105414656.0, + "32": 1105414656.0, + "33": 1105414656.0, + "34": 1105414656.0, + "35": 1105414656.0, + "36": 1105414656.0, + "37": 1105414656.0, + "38": 1105414656.0, + "39": 1105414656.0, + "40": 1105414656.0, + "41": 1105414656.0, + "42": 1105414656.0, + "43": 1105414656.0, + "44": 1105414656.0, + "45": 1105414656.0, + "46": 1105414656.0, + "47": 1105414656.0, + "48": 1105414656.0, + "49": 1105414656.0, + "50": 1105414656.0, + "51": 1105414656.0, + "52": 1105414656.0, + "53": 1105414656.0, + "54": 1105414656.0, + "55": 1105414656.0, + "56": 1105414656.0, + "57": 1105414656.0, + "58": 1105414656.0, + "59": 1105414656.0, + "60": 1105414656.0, + "61": 1105414656.0, + "62": 1105414656.0, + "63": 1105414656.0, + "64": 1105414656.0, + "65": 1105414656.0, + "66": 1105414656.0, + "67": 1105414656.0, + "68": 1105414656.0, + "69": 1105414656.0, + "70": 1105414656.0, + "71": 1105414656.0, + "72": 1105414656.0, + "73": 1105414656.0, + "74": 1105414656.0, + "75": 1105414656.0, + "76": 1105414656.0, + "77": 1105414656.0, + "78": 1105414656.0, + "79": 1105414656.0, + "80": 1105414656.0, + "81": 1105414656.0, + "82": 1105414656.0, + "83": 1105414656.0, + "84": 1105414656.0, + "85": 1105414656.0, + "86": 1105414656.0, + "87": 1105414656.0, + "88": 1105414656.0, + "89": 1105414656.0, + "90": 1105414656.0, + "91": 1105414656.0, + "92": 1105414656.0, + "93": 1105414656.0, + "94": 1105414656.0, + "95": 1105414656.0, + "96": 1105414656.0, + "97": 1105414656.0, + "98": 1105414656.0, + "99": 1105414656.0, + "100": 1105414656.0 } }, "mem-max-allocated-bytes": { "start_step": 1, "end_step": 100, "step_interval": 1, + "value_precision": "full", "values": { - "1": 1394868736.0, - "2": 1831391744.0, - "3": 1831391744.0, - "4": 1831391744.0, - "5": 1831391744.0, - "6": 1831391744.0, - "7": 1831391744.0, - "8": 1831391744.0, - "9": 1831391744.0, - "10": 1831391744.0, - "11": 1831391744.0, - "12": 1831391744.0, - "13": 1831391744.0, - "14": 1831391744.0, - "15": 1831391744.0, - "16": 1831391744.0, - "17": 1831391744.0, - "18": 1831391744.0, - "19": 1831391744.0, - "20": 1831391744.0, - "21": 1831391744.0, - "22": 1831391744.0, - "23": 1831391744.0, - "24": 1831391744.0, - "25": 1831391744.0, - "26": 1831391744.0, - "27": 1831391744.0, - "28": 1831391744.0, - "29": 1831391744.0, - "30": 1831391744.0, - "31": 1831391744.0, - "32": 1831391744.0, - "33": 1831391744.0, - "34": 1831391744.0, - "35": 1831391744.0, - "36": 1831391744.0, - "37": 1831391744.0, - "38": 1831391744.0, - "39": 1831391744.0, - "40": 1831391744.0, - "41": 1831391744.0, - "42": 1831391744.0, - "43": 1831391744.0, - "44": 1831391744.0, - "45": 1831391744.0, - "46": 1831391744.0, - "47": 1831391744.0, - "48": 1831391744.0, - "49": 1831391744.0, - "50": 1831391744.0, - "51": 1831391744.0, - "52": 1831391744.0, - "53": 1831391744.0, - "54": 1831391744.0, - "55": 1831391744.0, - "56": 1831391744.0, - "57": 1831391744.0, - "58": 1831391744.0, - "59": 1831391744.0, - "60": 1831391744.0, - "61": 1831391744.0, - "62": 1831391744.0, - "63": 1831391744.0, - "64": 1831391744.0, - "65": 1831391744.0, - "66": 1831391744.0, - "67": 1831391744.0, - "68": 1831391744.0, - "69": 1831391744.0, - "70": 1831391744.0, - "71": 1831391744.0, - "72": 1831391744.0, - "73": 1831391744.0, - "74": 1831391744.0, - "75": 1831391744.0, - "76": 1831391744.0, - "77": 1831391744.0, - "78": 1831391744.0, - "79": 1831391744.0, - "80": 1831391744.0, - "81": 1831391744.0, - "82": 1831391744.0, - "83": 1831391744.0, - "84": 1831391744.0, - "85": 1831391744.0, - "86": 1831391744.0, - "87": 1831391744.0, - "88": 1831391744.0, - "89": 1831391744.0, - "90": 1831391744.0, - "91": 1831391744.0, - "92": 1831391744.0, - "93": 1831391744.0, - "94": 1831391744.0, - "95": 1831391744.0, - "96": 1831391744.0, - "97": 1831391744.0, - "98": 1831391744.0, - "99": 1831391744.0, - "100": 1831391744.0 + "1": 1394052096.0, + "2": 1834640384.0, + "3": 1834640384.0, + "4": 1834640384.0, + "5": 1834640384.0, + "6": 1834640384.0, + "7": 1834640384.0, + "8": 1834640384.0, + "9": 1834640384.0, + "10": 1834640384.0, + "11": 1834640384.0, + "12": 1834640384.0, + "13": 1834640384.0, + "14": 1834640384.0, + "15": 1834640384.0, + "16": 1834640384.0, + "17": 1834640384.0, + "18": 1834640384.0, + "19": 1834640384.0, + "20": 1834640384.0, + "21": 1834640384.0, + "22": 1834640384.0, + "23": 1834640384.0, + "24": 1834640384.0, + "25": 1834640384.0, + "26": 1834640384.0, + "27": 1834640384.0, + "28": 1834640384.0, + "29": 1834640384.0, + "30": 1834640384.0, + "31": 1834640384.0, + "32": 1834640384.0, + "33": 1834640384.0, + "34": 1834640384.0, + "35": 1834640384.0, + "36": 1834640384.0, + "37": 1834640384.0, + "38": 1834640384.0, + "39": 1834640384.0, + "40": 1834640384.0, + "41": 1834640384.0, + "42": 1834640384.0, + "43": 1834640384.0, + "44": 1834640384.0, + "45": 1834640384.0, + "46": 1834640384.0, + "47": 1834640384.0, + "48": 1834640384.0, + "49": 1834640384.0, + "50": 1834640384.0, + "51": 1834640384.0, + "52": 1834640384.0, + "53": 1834640384.0, + "54": 1834640384.0, + "55": 1834640384.0, + "56": 1834640384.0, + "57": 1834640384.0, + "58": 1834640384.0, + "59": 1834640384.0, + "60": 1834640384.0, + "61": 1834640384.0, + "62": 1834640384.0, + "63": 1834640384.0, + "64": 1834640384.0, + "65": 1834640384.0, + "66": 1834640384.0, + "67": 1834640384.0, + "68": 1834640384.0, + "69": 1834640384.0, + "70": 1834640384.0, + "71": 1834640384.0, + "72": 1834640384.0, + "73": 1834640384.0, + "74": 1834640384.0, + "75": 1834640384.0, + "76": 1834640384.0, + "77": 1834640384.0, + "78": 1834640384.0, + "79": 1834640384.0, + "80": 1834640384.0, + "81": 1834640384.0, + "82": 1834640384.0, + "83": 1834640384.0, + "84": 1834640384.0, + "85": 1834640384.0, + "86": 1834640384.0, + "87": 1834640384.0, + "88": 1834640384.0, + "89": 1834640384.0, + "90": 1834640384.0, + "91": 1834640384.0, + "92": 1834640384.0, + "93": 1834640384.0, + "94": 1834640384.0, + "95": 1834640384.0, + "96": 1834640384.0, + "97": 1834640384.0, + "98": 1834640384.0, + "99": 1834640384.0, + "100": 1834640384.0 } }, "iteration-time": { "start_step": 2, "end_step": 100, "step_interval": 1, + "value_precision": "full", "values": { - "2": 7.47927, - "3": 0.6158, - "4": 0.51752, - "5": 0.66619, - "6": 0.47018, - "7": 0.65775, - "8": 0.45465, - "9": 0.44477, - "10": 0.4464, - "11": 0.44072, - "12": 0.44006, - "13": 0.43793, - "14": 0.43992, - "15": 0.43884, - "16": 0.43748, - "17": 0.44095, - "18": 0.44144, - "19": 0.43749, - "20": 0.44031, - "21": 0.43839, - "22": 0.43692, - "23": 0.43908, - "24": 0.44201, - "25": 0.43908, - "26": 0.43862, - "27": 0.43698, - "28": 0.43584, - "29": 0.43811, - "30": 0.44059, - "31": 0.44142, - "32": 0.44007, - "33": 0.43972, - "34": 0.43897, - "35": 0.44586, - "36": 0.44807, - "37": 0.44166, - "38": 0.44196, - "39": 0.44186, - "40": 0.43986, - "41": 0.43872, - "42": 0.43927, - "43": 0.43954, - "44": 0.43951, - "45": 0.44476, - "46": 0.44193, - "47": 0.45554, - "48": 0.50764, - "49": 0.50389, - "50": 0.66749, - "51": 0.44167, - "52": 0.44428, - "53": 0.44131, - "54": 0.44202, - "55": 0.44153, - "56": 0.44641, - "57": 0.43942, - "58": 0.44089, - "59": 0.43782, - "60": 0.44108, - "61": 0.43889, - "62": 0.43937, - "63": 0.43873, - "64": 0.43878, - "65": 0.4389, - "66": 0.44243, - "67": 0.47511, - "68": 0.44455, - "69": 0.44159, - "70": 0.52057, - "71": 0.45036, - "72": 0.43708, - "73": 0.43696, - "74": 0.43705, - "75": 0.43703, - "76": 0.44058, - "77": 0.44343, - "78": 0.4396, - "79": 0.43851, - "80": 0.4372, - "81": 0.43648, - "82": 0.43715, - "83": 0.43881, - "84": 0.43527, - "85": 0.4367, - "86": 0.4374, - "87": 0.43682, - "88": 0.4419, - "89": 0.44224, - "90": 0.44399, - "91": 0.44932, - "92": 0.44674, - "93": 0.44418, - "94": 0.49819, - "95": 0.58506, - "96": 0.44746, - "97": 0.59525, - "98": 0.56804, - "99": 0.54124, - "100": 0.48434 + "2": 0.5458650588989258, + "3": 0.4481790065765381, + "4": 0.4268147945404053, + "5": 0.42113351821899414, + "6": 0.41686582565307617, + "7": 0.4218320846557617, + "8": 0.42291855812072754, + "9": 0.41841840744018555, + "10": 0.4236176013946533, + "11": 0.4186692237854004, + "12": 0.41692376136779785, + "13": 0.4231102466583252, + "14": 0.4194953441619873, + "15": 0.4224097728729248, + "16": 0.42180895805358887, + "17": 0.4349813461303711, + "18": 0.42179298400878906, + "19": 0.41827392578125, + "20": 0.4198575019836426, + "21": 0.4137735366821289, + "22": 0.4122042655944824, + "23": 0.41033172607421875, + "24": 0.4125678539276123, + "25": 0.41161108016967773, + "26": 0.41041088104248047, + "27": 0.4087710380554199, + "28": 0.4168734550476074, + "29": 0.4154644012451172, + "30": 0.41448163986206055, + "31": 0.4391467571258545, + "32": 0.42397379875183105, + "33": 0.42021870613098145, + "34": 0.41657352447509766, + "35": 0.4195826053619385, + "36": 0.41510844230651855, + "37": 0.4145050048828125, + "38": 0.4142284393310547, + "39": 0.4110257625579834, + "40": 0.4121410846710205, + "41": 0.415477991104126, + "42": 0.41384053230285645, + "43": 0.4119839668273926, + "44": 0.629683256149292, + "45": 0.4125998020172119, + "46": 0.41138243675231934, + "47": 0.4084606170654297, + "48": 0.411740779876709, + "49": 0.6922652721405029, + "50": 0.4211845397949219, + "51": 0.66581130027771, + "52": 0.4189608097076416, + "53": 0.653613805770874, + "54": 0.42131853103637695, + "55": 0.4141976833343506, + "56": 0.420093297958374, + "57": 0.41852259635925293, + "58": 0.41338276863098145, + "59": 0.41910219192504883, + "60": 0.42175841331481934, + "61": 0.4124424457550049, + "62": 0.4091823101043701, + "63": 0.40962767601013184, + "64": 0.4083867073059082, + "65": 0.40871644020080566, + "66": 0.4088132381439209, + "67": 0.4077143669128418, + "68": 0.40738821029663086, + "69": 0.40727972984313965, + "70": 0.4086778163909912, + "71": 0.40961360931396484, + "72": 0.4081563949584961, + "73": 0.4156606197357178, + "74": 0.4195129871368408, + "75": 0.4165382385253906, + "76": 0.4174938201904297, + "77": 0.41691088676452637, + "78": 0.41205334663391113, + "79": 0.4099156856536865, + "80": 0.40990686416625977, + "81": 0.41157031059265137, + "82": 0.407820463180542, + "83": 0.41163182258605957, + "84": 0.4080057144165039, + "85": 0.40927672386169434, + "86": 0.411602258682251, + "87": 0.410083532333374, + "88": 0.41027379035949707, + "89": 0.41033506393432617, + "90": 0.4064218997955322, + "91": 0.40647220611572266, + "92": 0.40799713134765625, + "93": 0.40494346618652344, + "94": 0.40859222412109375, + "95": 0.4111142158508301, + "96": 0.46715664863586426, + "97": 0.6449761390686035, + "98": 0.4107213020324707, + "99": 0.41503214836120605, + "100": 0.5943217277526855 } } -} \ No newline at end of file +} diff --git a/tests/functional_tests/test_cases/t5/t5_mcore_tp4_pp1_resume_torch_dist/golden_values_dev_dgx_h100.json b/tests/functional_tests/test_cases/t5/t5_mcore_tp4_pp1_resume_torch_dist/golden_values_dev_dgx_h100.json index f8b61e6142f..8706dbbe3cc 100644 --- a/tests/functional_tests/test_cases/t5/t5_mcore_tp4_pp1_resume_torch_dist/golden_values_dev_dgx_h100.json +++ b/tests/functional_tests/test_cases/t5/t5_mcore_tp4_pp1_resume_torch_dist/golden_values_dev_dgx_h100.json @@ -3,534 +3,539 @@ "start_step": 1, "end_step": 100, "step_interval": 1, + "value_precision": "full", "values": { - "1": 10.36299, - "2": 10.35648, - "3": 9.6273, - "4": 9.36316, - "5": 9.23355, - "6": 9.24064, - "7": 9.10948, - "8": 8.9972, - "9": 8.89871, - "10": 8.8578, - "11": 8.71304, - "12": 8.6491, - "13": 8.59442, - "14": 8.5273, - "15": 8.46723, - "16": 8.48138, - "17": 8.25532, - "18": 8.19955, - "19": 8.24447, - "20": 8.14935, - "21": 8.04111, - "22": 7.97705, - "23": 8.03616, - "24": 7.89733, - "25": 7.85265, - "26": 7.83076, - "27": 7.7087, - "28": 7.6812, - "29": 7.56568, - "30": 7.56875, - "31": 7.45741, - "32": 7.52639, - "33": 7.43899, - "34": 7.37987, - "35": 7.33105, - "36": 7.29388, - "37": 7.26307, - "38": 7.28482, - "39": 7.25488, - "40": 7.15453, - "41": 7.09625, - "42": 7.15802, - "43": 7.13206, - "44": 7.04964, - "45": 7.06371, - "46": 7.07436, - "47": 7.02344, - "48": 7.03619, - "49": 6.92987, - "50": 6.99378, - "51": 7.10387, - "52": 6.99412, - "53": 6.91395, - "54": 6.90416, - "55": 6.87224, - "56": 6.96987, - "57": 6.92462, - "58": 6.83845, - "59": 6.90284, - "60": 6.75424, - "61": 6.89414, - "62": 6.83224, - "63": 6.89201, - "64": 6.92617, - "65": 6.78253, - "66": 6.81281, - "67": 6.7195, - "68": 6.73283, - "69": 6.88373, - "70": 6.67708, - "71": 6.86122, - "72": 6.78638, - "73": 6.80829, - "74": 6.73557, - "75": 6.74053, - "76": 6.8321, - "77": 6.77202, - "78": 6.69852, - "79": 6.81094, - "80": 6.85116, - "81": 6.80195, - "82": 6.72409, - "83": 6.78649, - "84": 6.76935, - "85": 6.75765, - "86": 6.71833, - "87": 6.76916, - "88": 6.76835, - "89": 6.74162, - "90": 6.60343, - "91": 6.71499, - "92": 6.7073, - "93": 6.75676, - "94": 6.76516, - "95": 6.69778, - "96": 6.78163, - "97": 6.7916, - "98": 6.665, - "99": 6.66887, - "100": 6.76763 + "1": 10.362956047058105, + "2": 10.356405258178711, + "3": 9.627273559570312, + "4": 9.363333702087402, + "5": 9.233860969543457, + "6": 9.240845680236816, + "7": 9.109770774841309, + "8": 8.997429847717285, + "9": 8.898931503295898, + "10": 8.8584623336792, + "11": 8.71377944946289, + "12": 8.650016784667969, + "13": 8.595564842224121, + "14": 8.528546333312988, + "15": 8.468491554260254, + "16": 8.48269271850586, + "17": 8.257022857666016, + "18": 8.2014741897583, + "19": 8.2464599609375, + "20": 8.151448249816895, + "21": 8.043624877929688, + "22": 7.979751110076904, + "23": 8.038796424865723, + "24": 7.899237632751465, + "25": 7.854281902313232, + "26": 7.832311630249023, + "27": 7.710287570953369, + "28": 7.68275785446167, + "29": 7.5674262046813965, + "30": 7.570520401000977, + "31": 7.459196090698242, + "32": 7.528255462646484, + "33": 7.441061019897461, + "34": 7.381735324859619, + "35": 7.333186626434326, + "36": 7.295981407165527, + "37": 7.265478134155273, + "38": 7.287073135375977, + "39": 7.257126808166504, + "40": 7.156900405883789, + "41": 7.098905563354492, + "42": 7.16079044342041, + "43": 7.134678363800049, + "44": 7.052248001098633, + "45": 7.066233158111572, + "46": 7.076325416564941, + "47": 7.023309707641602, + "48": 7.035020351409912, + "49": 6.927795886993408, + "50": 6.992145538330078, + "51": 7.102492809295654, + "52": 6.992738246917725, + "53": 6.912599086761475, + "54": 6.903322219848633, + "55": 6.871036052703857, + "56": 6.96876859664917, + "57": 6.924088478088379, + "58": 6.837710380554199, + "59": 6.901991844177246, + "60": 6.7534589767456055, + "61": 6.893715858459473, + "62": 6.831646919250488, + "63": 6.891280651092529, + "64": 6.92561674118042, + "65": 6.78196382522583, + "66": 6.812472343444824, + "67": 6.719153881072998, + "68": 6.732641696929932, + "69": 6.883572101593018, + "70": 6.677075386047363, + "71": 6.860844135284424, + "72": 6.786078453063965, + "73": 6.80823278427124, + "74": 6.735328197479248, + "75": 6.740242958068848, + "76": 6.831969738006592, + "77": 6.771693229675293, + "78": 6.6976728439331055, + "79": 6.8105268478393555, + "80": 6.850856781005859, + "81": 6.801382064819336, + "82": 6.724090576171875, + "83": 6.785991668701172, + "84": 6.7691426277160645, + "85": 6.757512092590332, + "86": 6.717645645141602, + "87": 6.769176006317139, + "88": 6.767646312713623, + "89": 6.7422380447387695, + "90": 6.603097915649414, + "91": 6.714541912078857, + "92": 6.707432270050049, + "93": 6.756669521331787, + "94": 6.765012264251709, + "95": 6.697880744934082, + "96": 6.781554698944092, + "97": 6.791520118713379, + "98": 6.665158748626709, + "99": 6.668912887573242, + "100": 6.767621040344238 } }, "num-zeros": { "start_step": 1, "end_step": 100, "step_interval": 1, + "value_precision": "full", "values": { - "1": 41056.0, - "2": 40904.0, - "3": 43306.0, - "4": 46989.0, - "5": 43080.0, - "6": 45460.0, - "7": 46971.0, - "8": 44058.0, - "9": 44010.0, - "10": 45502.0, - "11": 45362.0, - "12": 45377.0, - "13": 43980.0, - "14": 41758.0, - "15": 41884.0, - "16": 44781.0, - "17": 43870.0, - "18": 45372.0, - "19": 44025.0, - "20": 46307.0, - "21": 44052.0, - "22": 45472.0, + "1": 41050.0, + "2": 40916.0, + "3": 43300.0, + "4": 46985.0, + "5": 43101.0, + "6": 45458.0, + "7": 46969.0, + "8": 44055.0, + "9": 43993.0, + "10": 45500.0, + "11": 45372.0, + "12": 45375.0, + "13": 43976.0, + "14": 41749.0, + "15": 41896.0, + "16": 44767.0, + "17": 43878.0, + "18": 45371.0, + "19": 44044.0, + "20": 46296.0, + "21": 44055.0, + "22": 45467.0, "23": 41869.0, - "24": 43213.0, - "25": 46922.0, - "26": 43257.0, - "27": 44625.0, - "28": 43463.0, - "29": 44843.0, - "30": 45509.0, - "31": 44608.0, - "32": 46162.0, - "33": 45558.0, - "34": 47039.0, - "35": 43219.0, - "36": 47008.0, - "37": 45542.0, - "38": 42385.0, - "39": 45535.0, - "40": 46904.0, - "41": 43403.0, - "42": 45368.0, - "43": 46185.0, - "44": 43337.0, - "45": 43181.0, - "46": 44600.0, - "47": 43258.0, - "48": 44599.0, - "49": 43849.0, - "50": 45392.0, - "51": 41929.0, - "52": 41997.0, + "24": 43208.0, + "25": 46944.0, + "26": 43256.0, + "27": 44616.0, + "28": 43481.0, + "29": 44854.0, + "30": 45505.0, + "31": 44604.0, + "32": 46140.0, + "33": 45544.0, + "34": 47047.0, + "35": 43217.0, + "36": 47017.0, + "37": 45539.0, + "38": 42376.0, + "39": 45546.0, + "40": 46888.0, + "41": 43417.0, + "42": 45371.0, + "43": 46199.0, + "44": 43318.0, + "45": 43173.0, + "46": 44607.0, + "47": 43269.0, + "48": 44605.0, + "49": 43848.0, + "50": 45408.0, + "51": 41919.0, + "52": 42017.0, "53": 47089.0, - "54": 44590.0, - "55": 44000.0, - "56": 44054.0, - "57": 43449.0, - "58": 44697.0, - "59": 42394.0, - "60": 43182.0, - "61": 44623.0, - "62": 43887.0, - "63": 42379.0, - "64": 43988.0, - "65": 45487.0, - "66": 42729.0, - "67": 43924.0, - "68": 43962.0, - "69": 41641.0, - "70": 41661.0, - "71": 45458.0, - "72": 45397.0, - "73": 42717.0, - "74": 44760.0, - "75": 43882.0, - "76": 43333.0, - "77": 41645.0, - "78": 43192.0, - "79": 45393.0, - "80": 45452.0, - "81": 42459.0, - "82": 39226.0, - "83": 45659.0, - "84": 43449.0, - "85": 43995.0, + "54": 44599.0, + "55": 44016.0, + "56": 44051.0, + "57": 43416.0, + "58": 44688.0, + "59": 42376.0, + "60": 43179.0, + "61": 44640.0, + "62": 43898.0, + "63": 42391.0, + "64": 43996.0, + "65": 45480.0, + "66": 42736.0, + "67": 43934.0, + "68": 43933.0, + "69": 41633.0, + "70": 41665.0, + "71": 45464.0, + "72": 45400.0, + "73": 42748.0, + "74": 44757.0, + "75": 43889.0, + "76": 43343.0, + "77": 41658.0, + "78": 43190.0, + "79": 45402.0, + "80": 45432.0, + "81": 42470.0, + "82": 39209.0, + "83": 45661.0, + "84": 43438.0, + "85": 44000.0, "86": 43856.0, - "87": 41265.0, - "88": 43916.0, - "89": 40436.0, - "90": 39794.0, - "91": 46225.0, - "92": 46227.0, - "93": 43878.0, - "94": 45486.0, - "95": 41760.0, - "96": 41453.0, - "97": 42078.0, - "98": 43357.0, - "99": 39075.0, - "100": 39466.0 + "87": 41272.0, + "88": 43922.0, + "89": 40417.0, + "90": 39796.0, + "91": 46261.0, + "92": 46216.0, + "93": 43882.0, + "94": 45499.0, + "95": 41767.0, + "96": 41468.0, + "97": 42057.0, + "98": 43360.0, + "99": 39079.0, + "100": 39437.0 } }, "mem-allocated-bytes": { "start_step": 1, "end_step": 100, "step_interval": 1, + "value_precision": "full", "values": { - "1": 1106624512.0, - "2": 1106624512.0, - "3": 1106624512.0, - "4": 1106624512.0, - "5": 1106624512.0, - "6": 1106624512.0, - "7": 1106624512.0, - "8": 1106624512.0, - "9": 1106624512.0, - "10": 1106624512.0, - "11": 1106624512.0, - "12": 1106624512.0, - "13": 1106624512.0, - "14": 1106624512.0, - "15": 1106624512.0, - "16": 1106624512.0, - "17": 1106624512.0, - "18": 1106624512.0, - "19": 1106624512.0, - "20": 1106624512.0, - "21": 1106624512.0, - "22": 1106624512.0, - "23": 1106624512.0, - "24": 1106624512.0, - "25": 1106624512.0, - "26": 1106624512.0, - "27": 1106624512.0, - "28": 1106624512.0, - "29": 1106624512.0, - "30": 1106624512.0, - "31": 1106624512.0, - "32": 1106624512.0, - "33": 1106624512.0, - "34": 1106624512.0, - "35": 1106624512.0, - "36": 1106624512.0, - "37": 1106624512.0, - "38": 1106624512.0, - "39": 1106624512.0, - "40": 1106624512.0, - "41": 1106624512.0, - "42": 1106624512.0, - "43": 1106624512.0, - "44": 1106624512.0, - "45": 1106624512.0, - "46": 1106624512.0, - "47": 1106624512.0, - "48": 1106624512.0, - "49": 1106624512.0, - "50": 1106624512.0, - "51": 1106624512.0, - "52": 1106624512.0, - "53": 1106624512.0, - "54": 1106624512.0, - "55": 1106624512.0, - "56": 1106624512.0, - "57": 1106624512.0, - "58": 1106624512.0, - "59": 1106624512.0, - "60": 1106624512.0, - "61": 1106624512.0, - "62": 1106624512.0, - "63": 1106624512.0, - "64": 1106624512.0, - "65": 1106624512.0, - "66": 1106624512.0, - "67": 1106624512.0, - "68": 1106624512.0, - "69": 1106624512.0, - "70": 1106624512.0, - "71": 1106624512.0, - "72": 1106624512.0, - "73": 1106624512.0, - "74": 1106624512.0, - "75": 1106624512.0, - "76": 1106624512.0, - "77": 1106624512.0, - "78": 1106624512.0, - "79": 1106624512.0, - "80": 1106624512.0, - "81": 1106624512.0, - "82": 1106624512.0, - "83": 1106624512.0, - "84": 1106624512.0, - "85": 1106624512.0, - "86": 1106624512.0, - "87": 1106624512.0, - "88": 1106624512.0, - "89": 1106624512.0, - "90": 1106624512.0, - "91": 1106624512.0, - "92": 1106624512.0, - "93": 1106624512.0, - "94": 1106624512.0, - "95": 1106624512.0, - "96": 1106624512.0, - "97": 1106624512.0, - "98": 1106624512.0, - "99": 1106624512.0, - "100": 1106624512.0 + "1": 1105414656.0, + "2": 1105414656.0, + "3": 1105414656.0, + "4": 1105414656.0, + "5": 1105414656.0, + "6": 1105414656.0, + "7": 1105414656.0, + "8": 1105414656.0, + "9": 1105414656.0, + "10": 1105414656.0, + "11": 1105414656.0, + "12": 1105414656.0, + "13": 1105414656.0, + "14": 1105414656.0, + "15": 1105414656.0, + "16": 1105414656.0, + "17": 1105414656.0, + "18": 1105414656.0, + "19": 1105414656.0, + "20": 1105414656.0, + "21": 1105414656.0, + "22": 1105414656.0, + "23": 1105414656.0, + "24": 1105414656.0, + "25": 1105414656.0, + "26": 1105414656.0, + "27": 1105414656.0, + "28": 1105414656.0, + "29": 1105414656.0, + "30": 1105414656.0, + "31": 1105414656.0, + "32": 1105414656.0, + "33": 1105414656.0, + "34": 1105414656.0, + "35": 1105414656.0, + "36": 1105414656.0, + "37": 1105414656.0, + "38": 1105414656.0, + "39": 1105414656.0, + "40": 1105414656.0, + "41": 1105414656.0, + "42": 1105414656.0, + "43": 1105414656.0, + "44": 1105414656.0, + "45": 1105414656.0, + "46": 1105414656.0, + "47": 1105414656.0, + "48": 1105414656.0, + "49": 1105414656.0, + "50": 1105414656.0, + "51": 1105414656.0, + "52": 1105414656.0, + "53": 1105414656.0, + "54": 1105414656.0, + "55": 1105414656.0, + "56": 1105414656.0, + "57": 1105414656.0, + "58": 1105414656.0, + "59": 1105414656.0, + "60": 1105414656.0, + "61": 1105414656.0, + "62": 1105414656.0, + "63": 1105414656.0, + "64": 1105414656.0, + "65": 1105414656.0, + "66": 1105414656.0, + "67": 1105414656.0, + "68": 1105414656.0, + "69": 1105414656.0, + "70": 1105414656.0, + "71": 1105414656.0, + "72": 1105414656.0, + "73": 1105414656.0, + "74": 1105414656.0, + "75": 1105414656.0, + "76": 1105414656.0, + "77": 1105414656.0, + "78": 1105414656.0, + "79": 1105414656.0, + "80": 1105414656.0, + "81": 1105414656.0, + "82": 1105414656.0, + "83": 1105414656.0, + "84": 1105414656.0, + "85": 1105414656.0, + "86": 1105414656.0, + "87": 1105414656.0, + "88": 1105414656.0, + "89": 1105414656.0, + "90": 1105414656.0, + "91": 1105414656.0, + "92": 1105414656.0, + "93": 1105414656.0, + "94": 1105414656.0, + "95": 1105414656.0, + "96": 1105414656.0, + "97": 1105414656.0, + "98": 1105414656.0, + "99": 1105414656.0, + "100": 1105414656.0 } }, "mem-max-allocated-bytes": { "start_step": 1, "end_step": 100, "step_interval": 1, + "value_precision": "full", "values": { - "1": 1394868736.0, - "2": 1831391744.0, - "3": 1831391744.0, - "4": 1831391744.0, - "5": 1831391744.0, - "6": 1831391744.0, - "7": 1831391744.0, - "8": 1831391744.0, - "9": 1831391744.0, - "10": 1831391744.0, - "11": 1831391744.0, - "12": 1831391744.0, - "13": 1831391744.0, - "14": 1831391744.0, - "15": 1831391744.0, - "16": 1831391744.0, - "17": 1831391744.0, - "18": 1831391744.0, - "19": 1831391744.0, - "20": 1831391744.0, - "21": 1831391744.0, - "22": 1831391744.0, - "23": 1831391744.0, - "24": 1831391744.0, - "25": 1831391744.0, - "26": 1831391744.0, - "27": 1831391744.0, - "28": 1831391744.0, - "29": 1831391744.0, - "30": 1831391744.0, - "31": 1831391744.0, - "32": 1831391744.0, - "33": 1831391744.0, - "34": 1831391744.0, - "35": 1831391744.0, - "36": 1831391744.0, - "37": 1831391744.0, - "38": 1831391744.0, - "39": 1831391744.0, - "40": 1831391744.0, - "41": 1831391744.0, - "42": 1831391744.0, - "43": 1831391744.0, - "44": 1831391744.0, - "45": 1831391744.0, - "46": 1831391744.0, - "47": 1831391744.0, - "48": 1831391744.0, - "49": 1831391744.0, - "50": 1831391744.0, - "51": 1831391744.0, - "52": 1831391744.0, - "53": 1831391744.0, - "54": 1831391744.0, - "55": 1831391744.0, - "56": 1831391744.0, - "57": 1831391744.0, - "58": 1831391744.0, - "59": 1831391744.0, - "60": 1831391744.0, - "61": 1831391744.0, - "62": 1831391744.0, - "63": 1831391744.0, - "64": 1831391744.0, - "65": 1831391744.0, - "66": 1831391744.0, - "67": 1831391744.0, - "68": 1831391744.0, - "69": 1831391744.0, - "70": 1831391744.0, - "71": 1831391744.0, - "72": 1831391744.0, - "73": 1831391744.0, - "74": 1831391744.0, - "75": 1831391744.0, - "76": 1831391744.0, - "77": 1831391744.0, - "78": 1831391744.0, - "79": 1831391744.0, - "80": 1831391744.0, - "81": 1831391744.0, - "82": 1831391744.0, - "83": 1831391744.0, - "84": 1831391744.0, - "85": 1831391744.0, - "86": 1831391744.0, - "87": 1831391744.0, - "88": 1831391744.0, - "89": 1831391744.0, - "90": 1831391744.0, - "91": 1831391744.0, - "92": 1831391744.0, - "93": 1831391744.0, - "94": 1831391744.0, - "95": 1831391744.0, - "96": 1831391744.0, - "97": 1831391744.0, - "98": 1831391744.0, - "99": 1831391744.0, - "100": 1831391744.0 + "1": 1394052096.0, + "2": 1834640384.0, + "3": 1834640384.0, + "4": 1834640384.0, + "5": 1834640384.0, + "6": 1834640384.0, + "7": 1834640384.0, + "8": 1834640384.0, + "9": 1834640384.0, + "10": 1834640384.0, + "11": 1834640384.0, + "12": 1834640384.0, + "13": 1834640384.0, + "14": 1834640384.0, + "15": 1834640384.0, + "16": 1834640384.0, + "17": 1834640384.0, + "18": 1834640384.0, + "19": 1834640384.0, + "20": 1834640384.0, + "21": 1834640384.0, + "22": 1834640384.0, + "23": 1834640384.0, + "24": 1834640384.0, + "25": 1834640384.0, + "26": 1834640384.0, + "27": 1834640384.0, + "28": 1834640384.0, + "29": 1834640384.0, + "30": 1834640384.0, + "31": 1834640384.0, + "32": 1834640384.0, + "33": 1834640384.0, + "34": 1834640384.0, + "35": 1834640384.0, + "36": 1834640384.0, + "37": 1834640384.0, + "38": 1834640384.0, + "39": 1834640384.0, + "40": 1834640384.0, + "41": 1834640384.0, + "42": 1834640384.0, + "43": 1834640384.0, + "44": 1834640384.0, + "45": 1834640384.0, + "46": 1834640384.0, + "47": 1834640384.0, + "48": 1834640384.0, + "49": 1834640384.0, + "50": 1834640384.0, + "51": 1834640384.0, + "52": 1834640384.0, + "53": 1834640384.0, + "54": 1834640384.0, + "55": 1834640384.0, + "56": 1834640384.0, + "57": 1834640384.0, + "58": 1834640384.0, + "59": 1834640384.0, + "60": 1834640384.0, + "61": 1834640384.0, + "62": 1834640384.0, + "63": 1834640384.0, + "64": 1834640384.0, + "65": 1834640384.0, + "66": 1834640384.0, + "67": 1834640384.0, + "68": 1834640384.0, + "69": 1834640384.0, + "70": 1834640384.0, + "71": 1834640384.0, + "72": 1834640384.0, + "73": 1834640384.0, + "74": 1834640384.0, + "75": 1834640384.0, + "76": 1834640384.0, + "77": 1834640384.0, + "78": 1834640384.0, + "79": 1834640384.0, + "80": 1834640384.0, + "81": 1834640384.0, + "82": 1834640384.0, + "83": 1834640384.0, + "84": 1834640384.0, + "85": 1834640384.0, + "86": 1834640384.0, + "87": 1834640384.0, + "88": 1834640384.0, + "89": 1834640384.0, + "90": 1834640384.0, + "91": 1834640384.0, + "92": 1834640384.0, + "93": 1834640384.0, + "94": 1834640384.0, + "95": 1834640384.0, + "96": 1834640384.0, + "97": 1834640384.0, + "98": 1834640384.0, + "99": 1834640384.0, + "100": 1834640384.0 } }, "iteration-time": { "start_step": 2, "end_step": 100, "step_interval": 1, + "value_precision": "full", "values": { - "2": 7.24142, - "3": 0.45423, - "4": 0.43898, - "5": 0.43119, - "6": 0.42991, - "7": 0.6455, - "8": 0.43363, - "9": 0.43566, - "10": 0.43997, - "11": 0.44272, - "12": 0.43586, - "13": 0.43815, - "14": 0.44426, - "15": 0.43817, - "16": 0.43062, - "17": 0.43338, - "18": 0.42676, - "19": 0.42546, - "20": 0.42588, - "21": 0.43422, - "22": 0.43291, - "23": 0.4317, - "24": 0.43265, - "25": 0.42964, - "26": 0.43405, - "27": 0.43265, - "28": 0.43033, - "29": 0.43805, - "30": 0.44204, - "31": 0.43215, - "32": 0.43307, - "33": 0.42937, - "34": 0.4314, - "35": 0.43168, - "36": 0.42758, - "37": 0.43306, - "38": 0.43715, - "39": 0.43433, - "40": 0.43692, - "41": 0.43279, - "42": 0.43444, - "43": 0.43375, - "44": 0.43762, - "45": 0.55134, - "46": 0.43612, - "47": 0.43389, - "48": 0.43916, - "49": 0.44043, - "50": 0.438, - "51": 0.44146, - "52": 0.43375, - "53": 0.49329, - "54": 0.43643, - "55": 0.53033, - "56": 0.43354, - "57": 0.43891, - "58": 0.42854, - "59": 0.42847, - "60": 0.42688, - "61": 0.42623, - "62": 0.42589, - "63": 0.42655, - "64": 0.43019, - "65": 0.42845, - "66": 0.42789, - "67": 0.43406, - "68": 0.43002, - "69": 0.4295, - "70": 0.42561, - "71": 0.42811, - "72": 0.4274, - "73": 0.42558, - "74": 0.42493, - "75": 0.42486, - "76": 0.42648, - "77": 0.42617, - "78": 0.42452, - "79": 0.42776, - "80": 0.42747, - "81": 0.4361, - "82": 0.432, - "83": 0.43222, - "84": 0.43151, - "85": 0.42825, - "86": 0.42927, - "87": 0.42749, - "88": 0.42767, - "89": 0.43423, - "90": 0.51774, - "91": 0.42874, - "92": 0.42887, - "93": 0.52416, - "94": 0.42699, - "95": 0.4254, - "96": 0.42674, - "97": 0.42645, - "98": 0.42623, - "99": 0.57506, - "100": 0.43051 + "2": 0.551311731338501, + "3": 0.4560813903808594, + "4": 0.43741822242736816, + "5": 0.49352455139160156, + "6": 0.43998289108276367, + "7": 0.4246954917907715, + "8": 0.4212660789489746, + "9": 0.41823244094848633, + "10": 0.4175906181335449, + "11": 0.41715264320373535, + "12": 0.4253396987915039, + "13": 0.42140936851501465, + "14": 0.42632007598876953, + "15": 0.42626476287841797, + "16": 0.4332437515258789, + "17": 0.42589569091796875, + "18": 0.4260549545288086, + "19": 0.42212581634521484, + "20": 0.42111778259277344, + "21": 0.41991567611694336, + "22": 0.4184110164642334, + "23": 0.42252039909362793, + "24": 0.42522311210632324, + "25": 0.41715216636657715, + "26": 0.4205658435821533, + "27": 0.41690635681152344, + "28": 0.4201843738555908, + "29": 0.418825626373291, + "30": 0.4172377586364746, + "31": 0.4181370735168457, + "32": 0.4265627861022949, + "33": 0.41939234733581543, + "34": 0.4229557514190674, + "35": 0.4165961742401123, + "36": 0.4203755855560303, + "37": 0.41544222831726074, + "38": 0.41791725158691406, + "39": 0.42171335220336914, + "40": 0.4179818630218506, + "41": 0.4200413227081299, + "42": 0.4175863265991211, + "43": 0.417147159576416, + "44": 0.7705390453338623, + "45": 0.4149465560913086, + "46": 0.413837194442749, + "47": 0.41562604904174805, + "48": 0.4177279472351074, + "49": 0.7471063137054443, + "50": 0.467510461807251, + "51": 0.4211115837097168, + "52": 0.41280603408813477, + "53": 0.7100205421447754, + "54": 0.41422295570373535, + "55": 0.4138209819793701, + "56": 0.41307806968688965, + "57": 0.41864919662475586, + "58": 0.4319617748260498, + "59": 0.4481384754180908, + "60": 0.4114100933074951, + "61": 0.41447925567626953, + "62": 0.419114351272583, + "63": 0.41585659980773926, + "64": 0.4154789447784424, + "65": 0.4107046127319336, + "66": 0.41082000732421875, + "67": 0.41113734245300293, + "68": 0.40947508811950684, + "69": 0.411668062210083, + "70": 0.40984034538269043, + "71": 0.40926027297973633, + "72": 0.4111483097076416, + "73": 0.4095783233642578, + "74": 1.5529837608337402, + "75": 0.41099095344543457, + "76": 0.41005778312683105, + "77": 0.4089188575744629, + "78": 0.4124011993408203, + "79": 0.41387939453125, + "80": 0.41254758834838867, + "81": 0.4141559600830078, + "82": 0.4126288890838623, + "83": 0.415372371673584, + "84": 0.4131789207458496, + "85": 0.41266822814941406, + "86": 0.41209864616394043, + "87": 0.41001009941101074, + "88": 0.4139084815979004, + "89": 0.40880894660949707, + "90": 0.40965962409973145, + "91": 0.41248083114624023, + "92": 0.41150403022766113, + "93": 0.4156808853149414, + "94": 0.409961462020874, + "95": 0.4093923568725586, + "96": 0.41687798500061035, + "97": 0.9734253883361816, + "98": 0.4135251045227051, + "99": 0.41591811180114746, + "100": 0.702125072479248 } } -} \ No newline at end of file +} diff --git a/tests/performance_tests/shell_test_utils/determinism/perf_breakdown.sh b/tests/performance_tests/shell_test_utils/determinism/perf_breakdown.sh index 2ff1d2ea4e8..c109b566ccc 100755 --- a/tests/performance_tests/shell_test_utils/determinism/perf_breakdown.sh +++ b/tests/performance_tests/shell_test_utils/determinism/perf_breakdown.sh @@ -9,6 +9,7 @@ LOG_DIR="${2:?usage: $0 LEADERBOARD_DIR LOG_DIR}" SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" export CUDA_DEVICE_MAX_CONNECTIONS=1 +export NVTE_FLASH_ATTN_V4=0 export LOG_DIR # Clear stale per-rank logs from prior runs (torchrun never overwrites). @@ -45,6 +46,7 @@ bash "$SCRIPT_DIR/run_nsys_breakdown.sh" "$OUT" -- \ --mock-data \ --split 1,0,0 \ --transformer-impl transformer_engine \ + --flash-attention-version 2 \ --use-mcore-models \ --no-gradient-accumulation-fusion \ --bf16 \ diff --git a/tests/performance_tests/shell_test_utils/run_perf_test.sh b/tests/performance_tests/shell_test_utils/run_perf_test.sh index d1e0e28bfa7..67523348178 100755 --- a/tests/performance_tests/shell_test_utils/run_perf_test.sh +++ b/tests/performance_tests/shell_test_utils/run_perf_test.sh @@ -33,6 +33,8 @@ for ARG in "$@"; do export "$KEY"="$VAL" done +export NVTE_FLASH_ATTN_V4=0 + : "${CONFIG_PATH:?CONFIG_PATH (path to model_config.yaml) is required}" : "${CHECKPOINT_LOAD_PATH:?CHECKPOINT_LOAD_PATH is required}" : "${RESULTS_ROOT:?RESULTS_ROOT is required}" @@ -135,6 +137,7 @@ MODEL_ARGS+=( --tensor-model-parallel-size "$TP" --pipeline-model-parallel-size "$PP" --expert-model-parallel-size "$EP" + --flash-attention-version 2 ) # ── Make image-bundled extras (mamba-ssm) visible to the cog venv ───────────── diff --git a/tests/test_utils/python_scripts/launch_nemo_run_workload.py b/tests/test_utils/python_scripts/launch_nemo_run_workload.py index 20bc52188df..93e391ba196 100644 --- a/tests/test_utils/python_scripts/launch_nemo_run_workload.py +++ b/tests/test_utils/python_scripts/launch_nemo_run_workload.py @@ -247,6 +247,24 @@ def main( volumes=artifacts, ) + if ( + scope == "unit-tests" + and test_case == "tests/unit_tests/transformer/**/*.py" + and environment == "dev" + and tag == "latest" + and platform == "dgx_h100" + ): + # NCCL 2.31 finalization waits for peers. Nonblocking communicators let + # PyTorch start every communicator's finalization before waiting, avoiding + # cyclic waits between the CP group's P2P and collective communicators (#7744). + executor.env_vars["TORCH_NCCL_USE_COMM_NONBLOCKING"] = "1" + executor.env_vars["PYTEST_ADDOPTS"] = "-o faulthandler_timeout=600" + executor.env_vars["TORCH_CPP_LOG_LEVEL"] = "INFO" + executor.env_vars["NCCL_DEBUG_SUBSYS"] = "INIT,BOOTSTRAP,ENV,DESTROY" + executor.env_vars["NCCL_DEBUG_FILE"] = ( + "/opt/megatron-lm/assets_dir/logs/nccl_debug.%h.%p.log" + ) + n_attempts = 0 while n_attempts < 3: tee_buffer = _ThreadSafeBuffer() diff --git a/tests/test_utils/recipes/gb200/unit-tests.yaml b/tests/test_utils/recipes/gb200/unit-tests.yaml index ac9557fc698..df848f119d4 100644 --- a/tests/test_utils/recipes/gb200/unit-tests.yaml +++ b/tests/test_utils/recipes/gb200/unit-tests.yaml @@ -44,6 +44,7 @@ spec: ENVIRONMENT={environment} BUCKET="{test_case}" UNIT_TEST_REPEAT={n_repeat} + export NVTE_FLASH_ATTN_V4=0 export GPUS_PER_NODE={gpus} if [[ "$TAG" == "latest" ]]; then diff --git a/tests/test_utils/recipes/h100/mimo.yaml b/tests/test_utils/recipes/h100/mimo.yaml index ca1f90867f9..b3aacffced6 100644 --- a/tests/test_utils/recipes/h100/mimo.yaml +++ b/tests/test_utils/recipes/h100/mimo.yaml @@ -96,6 +96,8 @@ products: set -euo pipefail cd /opt/megatron-lm export PYTHONPATH="$(pwd):${{PYTHONPATH:-}}" + # Nested launchers must retain the native packages already built in CI. + export UV_NO_SYNC=1 uv run --no-sync pytest -s \ tests/functional_tests/test_cases/mimo/{test_case}/test_checkpoint_round_trip.py diff --git a/tests/test_utils/recipes/h100/module_performance.yaml b/tests/test_utils/recipes/h100/module_performance.yaml index 6af791ff3e9..979796ee886 100644 --- a/tests/test_utils/recipes/h100/module_performance.yaml +++ b/tests/test_utils/recipes/h100/module_performance.yaml @@ -45,6 +45,7 @@ spec: cd /opt/megatron-lm + export NVTE_FLASH_ATTN_V4=0 uv run --no-sync python -m torch.distributed.run \ --log-dir {assets_dir}/logs/1 \ --tee "0:3,7:3" \ @@ -56,5 +57,6 @@ products: - test_case: [moe_perf] products: - environment: [dev] - scope: [mr, mr-github] + # https://github.com/NVIDIA/Megatron-LM/issues/7739 + scope: [mr, mr-github-broken] platforms: [dgx_h100] diff --git a/tests/test_utils/recipes/h100/moe-dynamic-inference.yaml b/tests/test_utils/recipes/h100/moe-dynamic-inference.yaml index 12e2dcadf60..23405685ec2 100644 --- a/tests/test_utils/recipes/h100/moe-dynamic-inference.yaml +++ b/tests/test_utils/recipes/h100/moe-dynamic-inference.yaml @@ -91,7 +91,8 @@ products: - test_case: [gpt_dynamic_inference_tp4_pp1_ep4_16B_inference_optimized_swiglu_logitsmatch] products: - environment: [dev] - scope: [mr, mr-github] + # https://github.com/NVIDIA/Megatron-LM/issues/7700 + scope: [mr, mr-github-broken] platforms: [dgx_h100] # Regression guard for the MTP + sequence-parallel all-gather fixes in # https://github.com/NVIDIA/Megatron-LM/pull/5748 (under block-scope CUDA diff --git a/tests/test_utils/recipes/h100/moe.yaml b/tests/test_utils/recipes/h100/moe.yaml index 72147055404..36b7e86c7ca 100644 --- a/tests/test_utils/recipes/h100/moe.yaml +++ b/tests/test_utils/recipes/h100/moe.yaml @@ -187,11 +187,13 @@ products: scope: [mr, mr-github] platforms: [dgx_h100] - environment: [lts] - scope: [nightly] + # A100 exact numerical trajectories differ across allocations: #7801. + scope: [nightly-broken] + cadence: [nightly] - test_case: [gpt3_moe_mcore_te_ep8_resume_torch_dist_dist_optimizer] products: - environment: [dev] - scope: [mr, mr-github] + scope: [mr-broken, mr-github-broken] # https://github.com/NVIDIA/Megatron-LM/issues/7201 platforms: [dgx_h100] - test_case: [gpt3_moe_mcore_te_tp4_ep2_etp2_pp2_scoped_cudagraph] products: @@ -201,12 +203,14 @@ products: - test_case: [gpt3_moe_mcore_te_ep8_resume_torch_dist_dist_muon] products: - environment: [dev] - scope: [mr, mr-github, mr-slim] + # https://github.com/NVIDIA/Megatron-LM/issues/7734 + scope: [mr-broken, mr-github-broken, mr-slim-broken] platforms: [dgx_h100] - test_case: [gpt3_moe_mcore_te_ep8_resume_torch_dist_muon] products: - environment: [dev] - scope: [mr, mr-github, mr-slim] + # https://github.com/NVIDIA/Megatron-LM/issues/7734 + scope: [mr, mr-github-broken, mr-slim] platforms: [dgx_h100] - test_case: [gpt3_moe_mcore_te_tp2_pp2_ep4_etp1_no_mtp_no_a2a_ovlp_fine_grained_offloading] products: diff --git a/tests/test_utils/recipes/h100/unit-tests.yaml b/tests/test_utils/recipes/h100/unit-tests.yaml index 0d5b4eccc5c..996587dda06 100644 --- a/tests/test_utils/recipes/h100/unit-tests.yaml +++ b/tests/test_utils/recipes/h100/unit-tests.yaml @@ -44,6 +44,7 @@ spec: ENVIRONMENT={environment} BUCKET="{test_case}" UNIT_TEST_REPEAT={n_repeat} + export NVTE_FLASH_ATTN_V4=0 # GTP scales to any world size; the test fixtures don't — most GTP cases # pytest.skip unless world_size==4 exactly (a few also run at 8, see diff --git a/tests/unit_tests/determinism/kernels/test_fused_activations.py b/tests/unit_tests/determinism/kernels/test_fused_activations.py index ceccf04600e..33145d8d633 100644 --- a/tests/unit_tests/determinism/kernels/test_fused_activations.py +++ b/tests/unit_tests/determinism/kernels/test_fused_activations.py @@ -110,6 +110,33 @@ def test_mlp_activation_fusions_replay_bit_exactly(case): assert_replays_bit_exact(fn, inputs, replays=3, what=case) +@pytest.mark.parametrize("weighted", [False, True]) +def test_clamped_swiglu_boundaries_replay_bit_exactly(weighted): + seeded() + # Include both endpoints and their adjacent BF16 values in each clamp branch. + bounds = torch.tensor( + [-10.0625, -10.0, -9.9375, 9.9375, 10.0, 10.0625], device="cuda", dtype=DTYPE + ) + x = torch.cat((bounds, bounds)).repeat(32, 1).requires_grad_(True) + if weighted: + fn = lambda x, w: weighted_bias_swiglu_impl(x, None, w, clamp_value=10.0) + inputs = (x, _weights(32) + 1.0) + else: + fn = lambda x: bias_swiglu_impl(x, None, clamp_value=10.0) + inputs = (x,) + _, grads = assert_replays_bit_exact(fn, inputs, replays=3, what="clamped SwiGLU boundaries") + + # Replay alone would also accept a consistently wrong endpoint derivative. + # Compare its zero mask with the installed PyTorch scalar-clamp contract. + reference_x = x.detach().clone().requires_grad_(True) + gate, linear = reference_x.float().chunk(2, dim=-1) + reference = (F.silu(gate.clamp(max=10.0)) * linear.clamp(-10.0, 10.0)).to(DTYPE) + if weighted: + reference = (reference * inputs[1]).to(DTYPE) + reference_grad = torch.autograd.grad(reference.sum(), reference_x)[0] + assert torch.equal(grads["in[0]"] == 0, reference_grad == 0) + + # --- plain compiled activations ----------------------------------------------------------- ACTIVATION_CASES = { diff --git a/tests/unit_tests/inference/engines/test_dynamic_engine.py b/tests/unit_tests/inference/engines/test_dynamic_engine.py index 63ce0f02395..ad8e6fef05d 100644 --- a/tests/unit_tests/inference/engines/test_dynamic_engine.py +++ b/tests/unit_tests/inference/engines/test_dynamic_engine.py @@ -920,6 +920,7 @@ class DynamicEngineTestConfig: fp8: bool = False hidden_size: Optional[int] = None model_provider: str = "gpt" + flash_attention_version: Optional[int] = 2 # Which linear-attention mixer a hybrid stack uses ("mamba", "gdp", or # "gdn"). Ignored unless model_provider == "hybrid": all three build a # HybridModel, and only the layer pattern and stack spec (and so the @@ -1191,6 +1192,7 @@ def _build_test_env(cls, test_config): # Transformer config. transformer_config = TransformerConfig( params_dtype=torch.bfloat16, + flash_attention_version=test_config.flash_attention_version, num_layers=4, mtp_num_layers=test_config.num_speculative_tokens, mtp_use_repeated_layer=test_config.mtp_use_repeated_layer, @@ -1300,6 +1302,7 @@ def _build_test_env(cls, test_config): # Transformer config. transformer_config = TransformerConfig( params_dtype=torch.bfloat16, + flash_attention_version=test_config.flash_attention_version, num_layers=( 3 if pp_size == 1 else 6 ), # 1 Mamba layer, 1 attention layer, 1 MLP layer diff --git a/tests/unit_tests/inference/engines/test_dynamic_engine_parallel_and_chunked_prefill.py b/tests/unit_tests/inference/engines/test_dynamic_engine_parallel_and_chunked_prefill.py index b836bc094c2..40878320102 100644 --- a/tests/unit_tests/inference/engines/test_dynamic_engine_parallel_and_chunked_prefill.py +++ b/tests/unit_tests/inference/engines/test_dynamic_engine_parallel_and_chunked_prefill.py @@ -1404,14 +1404,15 @@ def _create_model( ssm_mixer="mamba", hidden_size=32, num_query_groups=None, - flash_attention_version=None, + flash_attention_version=2, init_method_std=0.02, ): """Create a GPT or hybrid model with optional CUDA graph support. `ssm_mixer` selects the hybrid stack's linear-attention mixer ("mamba" or "gdp"); it is ignored for GPT. `hidden_size`, `num_query_groups`, - `flash_attention_version` and `init_method_std` apply to GPT only. + and `init_method_std` apply to GPT only. `flash_attention_version` + applies to both model types. """ cuda_graph_impl = "local" if num_cuda_graphs else "none" @@ -1444,6 +1445,7 @@ def _create_model( elif model_provider == "hybrid": config = TransformerConfig( params_dtype=torch.bfloat16, + flash_attention_version=flash_attention_version, num_layers=3, hidden_size=256, **hybrid_mixer_kwargs(ssm_mixer), diff --git a/tests/unit_tests/inference/engines/test_hybrid_prefix_caching_e2e.py b/tests/unit_tests/inference/engines/test_hybrid_prefix_caching_e2e.py index 9bd43c1a256..3aed41e0100 100644 --- a/tests/unit_tests/inference/engines/test_hybrid_prefix_caching_e2e.py +++ b/tests/unit_tests/inference/engines/test_hybrid_prefix_caching_e2e.py @@ -168,6 +168,7 @@ def ssm_mixer(self, request): def _create_model(self, num_cuda_graphs=None): transformer_config = TransformerConfig( + flash_attention_version=2, params_dtype=torch.bfloat16, num_layers=3, hidden_size=256, diff --git a/tests/unit_tests/inference/engines/test_prefix_caching_cuda_graphs.py b/tests/unit_tests/inference/engines/test_prefix_caching_cuda_graphs.py index 11fe6f60d0e..0f1da4215a9 100644 --- a/tests/unit_tests/inference/engines/test_prefix_caching_cuda_graphs.py +++ b/tests/unit_tests/inference/engines/test_prefix_caching_cuda_graphs.py @@ -349,6 +349,7 @@ def _create_hybrid_model(self, num_cuda_graphs=None, ssm_mixer="mamba"): """Create a hybrid (SSM + attention) model with the requested mixer.""" cuda_graph_impl = "local" if num_cuda_graphs else "none" config = TransformerConfig( + flash_attention_version=2, params_dtype=torch.bfloat16, num_layers=3, hidden_size=256, diff --git a/tests/unit_tests/inference/engines/test_static_engine.py b/tests/unit_tests/inference/engines/test_static_engine.py index 025c197fff7..557a77291db 100644 --- a/tests/unit_tests/inference/engines/test_static_engine.py +++ b/tests/unit_tests/inference/engines/test_static_engine.py @@ -50,6 +50,7 @@ def setup_engine( self.vocab_size = vocab_size self.sequence_length = 64 transformer_config = TransformerConfig( + flash_attention_version=2, num_layers=4, hidden_size=self.hidden_size, num_attention_heads=4, diff --git a/tests/unit_tests/inference/test_hybrid_moe.py b/tests/unit_tests/inference/test_hybrid_moe.py index 9587cc2d285..6852b8de527 100644 --- a/tests/unit_tests/inference/test_hybrid_moe.py +++ b/tests/unit_tests/inference/test_hybrid_moe.py @@ -273,6 +273,7 @@ def _assert_cuda_graphs_were_replayed(expect_replayed, rank, label): class TestDynamicInferenceNVLS(_TestDynamicInferenceBase): """NVLS dispatcher: combinatorial sweep of EP request states.""" + @pytest.mark.flaky_in_dev # Issue #7958: batch-invariant prefill/full-forward parity. @requires_te_batch_invariant_attention @torch.inference_mode() def test_batch_invariant_prefill_matches_full_forward(self): diff --git a/tests/unit_tests/inference/test_mtp_cuda_graph_inference.py b/tests/unit_tests/inference/test_mtp_cuda_graph_inference.py index 618faaafd17..d7039b79a4e 100644 --- a/tests/unit_tests/inference/test_mtp_cuda_graph_inference.py +++ b/tests/unit_tests/inference/test_mtp_cuda_graph_inference.py @@ -163,6 +163,7 @@ def _build_model( """Build a GPT or Hybrid model with MTP layers and local CUDA graph support.""" model_parallel_cuda_manual_seed(123, inference_rng_tracker=True, force_reset_rng=True) config = TransformerConfig( + flash_attention_version=2, num_layers=self.NUM_LAYERS, hidden_size=self.HIDDEN_SIZE, num_attention_heads=self.NUM_ATTN_HEADS, @@ -979,6 +980,7 @@ def _build_model(self, inference_moe_token_dispatcher_type='nccl', model_type='g model_parallel_cuda_manual_seed(123, inference_rng_tracker=True, force_reset_rng=True) mtp_num_layers = 2 config = TransformerConfig( + flash_attention_version=2, num_layers=self.NUM_LAYERS, hidden_size=self.HIDDEN_SIZE, num_attention_heads=self.NUM_ATTN_HEADS, @@ -1317,6 +1319,7 @@ def _build_model(self, model_type='gpt'): model_parallel_cuda_manual_seed(123, inference_rng_tracker=True, force_reset_rng=True) mtp_num_layers = 1 config = TransformerConfig( + flash_attention_version=2, num_layers=self.NUM_LAYERS, hidden_size=self.HIDDEN_SIZE, num_attention_heads=self.NUM_ATTN_HEADS, @@ -1561,6 +1564,7 @@ def _build_model(self, *, inference_cuda_graph_scope='block', model_type='hybrid """Build a GPT or Hybrid model with MTP and local CUDA graph support.""" model_parallel_cuda_manual_seed(123, inference_rng_tracker=True, force_reset_rng=True) config = TransformerConfig( + flash_attention_version=2, num_layers=self.NUM_LAYERS, hidden_size=self.HIDDEN_SIZE, num_attention_heads=self.NUM_ATTN_HEADS, diff --git a/tests/unit_tests/inference/text_generation_controllers/test_text_generation_controller.py b/tests/unit_tests/inference/text_generation_controllers/test_text_generation_controller.py index 1832f574906..818ec8bbe09 100644 --- a/tests/unit_tests/inference/text_generation_controllers/test_text_generation_controller.py +++ b/tests/unit_tests/inference/text_generation_controllers/test_text_generation_controller.py @@ -123,6 +123,7 @@ def setup_model( num_attention_heads=4, use_cpu_initialization=True, attention_backend=AttnBackend.auto if inference_optimized else AttnBackend.local, + flash_attention_version=2, params_dtype=dtype, symmetric_ar_type=symmetric_ar_type, fp8="hybrid" if fp8 else None, diff --git a/tests/unit_tests/models/test_gpt_model_batch_invariant.py b/tests/unit_tests/models/test_gpt_model_batch_invariant.py index a45b77877c6..a997f29d4c6 100644 --- a/tests/unit_tests/models/test_gpt_model_batch_invariant.py +++ b/tests/unit_tests/models/test_gpt_model_batch_invariant.py @@ -204,6 +204,7 @@ def test_forward_batch_invariant(self): assert torch.equal(logits_full, logits_chunked) + @pytest.mark.flaky_in_dev def test_dynamic_engine_matches_batched_forward_rl(self): _configure_flash_attention_env() seq_len = 48 @@ -277,6 +278,7 @@ def test_dynamic_engine_matches_batched_forward_rl(self): inference_log_probs, baseline_log_probs ), "Log probabilities from dynamic engine did not match batched forward." + @pytest.mark.flaky_in_dev def test_dynamic_engine_is_batch_invariant(self): """Check that the dynamic engine itself is batch invariant: changing the order in which requests are added does not change per-request outputs.""" @@ -352,6 +354,7 @@ def _run_engine_with_order(order): assert r1.prompt_log_probs == r2.prompt_log_probs assert r1.generated_log_probs == r2.generated_log_probs + @pytest.mark.flaky_in_dev def test_dynamic_engine_is_batch_invariant_under_kv_cache_saturation(self): """Saturating the KV cache must not change what requests generate. diff --git a/tests/unit_tests/tensor_parallel/test_random.py b/tests/unit_tests/tensor_parallel/test_random.py index e603ffd4abd..7f90f08989d 100644 --- a/tests/unit_tests/tensor_parallel/test_random.py +++ b/tests/unit_tests/tensor_parallel/test_random.py @@ -9,6 +9,7 @@ CudaRNGStatesTracker, checkpoint, convert_cuda_rng_state, + cudagraph_needs_generator_registration, get_cuda_rng_tracker, model_parallel_cuda_manual_seed, ) @@ -87,6 +88,29 @@ def test_double_fork_cuda_rng_states_tracker(use_cudagraphable_rng): assert torch.equal(double_fork_state2, single_fork_state2) +def test_cudagraph_generator_registration_matches_capture_behavior(): + """Capturing a tracker fork must work when registration follows the detected requirement. + + Regression test for PyTorch 2.14.0a0 builds without lazy generator registration, which + the version check alone treated as not needing registration. + """ + rng_tracker = CudaRNGStatesTracker(use_cudagraphable_rng=True) + rng_tracker.add("state1", 1234) + needs_registration = cudagraph_needs_generator_registration() + assert cudagraph_needs_generator_registration() is needs_registration + + graph = torch.cuda.CUDAGraph() + if needs_registration: + for state in rng_tracker.get_states().values(): + graph.register_generator_state(state) + with torch.cuda.graph(graph): + with rng_tracker.fork("state1"): + out = torch.rand(16, device="cuda") + graph.replay() + torch.cuda.synchronize() + assert torch.all((out >= 0) & (out < 1)) + + def test_convert_cuda_rng_state(): ## Get the default rng state torch.cuda.manual_seed(999) diff --git a/tests/unit_tests/transformer/test_submodule_callables.py b/tests/unit_tests/transformer/test_submodule_callables.py index 029fcb88865..b240dc50953 100644 --- a/tests/unit_tests/transformer/test_submodule_callables.py +++ b/tests/unit_tests/transformer/test_submodule_callables.py @@ -195,8 +195,23 @@ def teardown_method(self, method): reset_hybrid_ep_buffer() @pytest.mark.skipif(not is_te_min_version("1.9.0.dev0"), reason="Requires TE >= 1.9.0.dev0") - @pytest.mark.parametrize("dispatcher_type", get_valid_token_dispatcher_types()) - @pytest.mark.parametrize("grouped_gemm", [True, False]) + @pytest.mark.parametrize( + "grouped_gemm, dispatcher_type", + [ + pytest.param( + grouped_gemm, + dispatcher_type, + # NCCL 2.31 nonblocking all-to-all crashes in dev (#7744). + marks=( + pytest.mark.flaky_in_dev + if not grouped_gemm and dispatcher_type == "alltoall" + else () + ), + ) + for grouped_gemm in [True, False] + for dispatcher_type in get_valid_token_dispatcher_types() + ], + ) @pytest.mark.parametrize("permute_fusion", [True, False]) def test_1f1b_overlap(self, dispatcher_type, grouped_gemm, permute_fusion): """ diff --git a/uv.lock b/uv.lock index c59799f00fe..7949e75a581 100644 --- a/uv.lock +++ b/uv.lock @@ -1124,8 +1124,8 @@ wheels = [ [[package]] name = "deep-gemm" -version = "2.5.0+local" -source = { git = "https://github.com/deepseek-ai/DeepGEMM.git?rev=714dd1a4a980f7937a74343d19a8eba4fe321480#714dd1a4a980f7937a74343d19a8eba4fe321480" } +version = "2.8.0+local" +source = { git = "https://github.com/deepseek-ai/DeepGEMM.git?rev=ea2b7805b97956268bed1adf94591f428ded6cc1#ea2b7805b97956268bed1adf94591f428ded6cc1" } [[package]] name = "defusedxml" @@ -2528,7 +2528,7 @@ requires-dist = [ provides-extras = ["training", "mlm", "dev", "inference", "otel", "lts", "te", "ssm"] [package.metadata.requires-dev] -batch-invariant = [{ name = "deep-gemm", git = "https://github.com/deepseek-ai/DeepGEMM.git?rev=714dd1a4a980f7937a74343d19a8eba4fe321480" }] +batch-invariant = [{ name = "deep-gemm", git = "https://github.com/deepseek-ai/DeepGEMM.git?rev=ea2b7805b97956268bed1adf94591f428ded6cc1" }] build = [ { name = "cython", specifier = ">=3.0.0" }, { name = "hatchling" }, @@ -2560,7 +2560,7 @@ linting = [ { name = "ruff", specifier = "~=0.9.0" }, ] no-pypi-wheels = [ - { name = "deep-gemm", git = "https://github.com/deepseek-ai/DeepGEMM.git?rev=714dd1a4a980f7937a74343d19a8eba4fe321480" }, + { name = "deep-gemm", git = "https://github.com/deepseek-ai/DeepGEMM.git?rev=ea2b7805b97956268bed1adf94591f428ded6cc1" }, { name = "emerging-optimizers", git = "https://github.com/NVIDIA-NeMo/Emerging-Optimizers.git?rev=v0.3.0" }, { name = "flash-mla", git = "https://github.com/deepseek-ai/FlashMLA?rev=nv_dev" }, ]