Skip to content
Merged
10 changes: 3 additions & 7 deletions src/mcore_bridge/model/modules/kernels/ple_kernels.py
Original file line number Diff line number Diff line change
Expand Up @@ -51,7 +51,7 @@ def _gather_ple_rows_from_pinned(
def gather_ple_rows(host_table, ids, row_start, row_end, out=None):
"""Gather n-gram rows from the CPU-pinned table with a triton kernel.

Returns ``None`` when the fast path is not usable (no triton, CPU-resident
Returns ``None`` when the fast path is not usable (no triton, non-CUDA
ids, or the table is not bf16), so the caller can fall back to the torch path.

Args:
Expand All @@ -60,12 +60,8 @@ def gather_ple_rows(host_table, ids, row_start, row_end, out=None):
row_start / row_end: this rank's global row range.
out: optional preallocated ``[*ids.shape, embedding_dim]`` bf16 device tensor.
"""
# Gate on "triton exists + ids live on an accelerator", not on is_cuda:
# triton ships per-vendor backends (ROCm in-tree; NPU/XPU via vendor forks
# such as triton-ascend), so a platform that cannot run this kernel fails
# at launch instead of being silently excluded here. The device-side read
# of the pinned host table is only verified on CUDA so far.
if not HAVE_TRITON or ids.device.type == 'cpu':
# Pinned-host fast path is CUDA-only; use the torch fallback elsewhere.
if not HAVE_TRITON or ids.device.type != 'cuda':
return None
if host_table.dtype != torch.bfloat16:
return None
Expand Down
57 changes: 33 additions & 24 deletions src/mcore_bridge/tuners/lora.py
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@
TERowParallelGroupedLinear, TERowParallelLinear)
from megatron.core.models.common.embeddings.language_model_embedding import LanguageModelEmbedding
from megatron.core.parallel_state import get_expert_tensor_parallel_world_size, get_tensor_model_parallel_world_size
from megatron.core.tensor_parallel.layers import ColumnParallelLinear, RowParallelLinear
from megatron.core.tensor_parallel.random import get_cuda_rng_tracker, get_expert_parallel_rng_tracker_name
from megatron.core.transformer.mlp import apply_swiglu_sharded_factory
from megatron.core.transformer.module import MegatronModule
Expand Down Expand Up @@ -117,7 +118,8 @@ def __init__(
if use_dora:
raise ValueError(f'{self.__class__.__name__} does not support DoRA yet, please set it to False')

self.is_parallel_a = isinstance(base_layer, (TERowParallelLinear, TERowParallelGroupedLinear))
self.is_parallel_a = isinstance(base_layer,
(TERowParallelLinear, TERowParallelGroupedLinear, RowParallelLinear))
self.is_grouped = isinstance(base_layer, TEGroupedLinear)
self.fan_in_fan_out = fan_in_fan_out
self._active_adapter = adapter_name
Expand Down Expand Up @@ -165,6 +167,9 @@ def update_layer(self, adapter_name, r, *, lora_alpha, **kwargs):

self.lora_dropout[adapter_name] = lora_dropout_layer

replicated_base = (
is_torch_npu_available() and isinstance(self.base_layer, TELinear)
and getattr(self.base_layer, 'parallel_mode', None) == 'duplicated')
# lora needs to be forced to upgrade to 32-bit precision, otherwise it will overflow
kwargs = {
'skip_bias_add': False,
Expand Down Expand Up @@ -215,15 +220,18 @@ def update_layer(self, adapter_name, r, *, lora_alpha, **kwargs):
**kwargs,
)
else:
lora_a = TERowParallelLinear(
input_size=in_features,
native_row = isinstance(self.base_layer, RowParallelLinear)
row_input_size = self.in_features if native_row or is_torch_npu_available() else in_features
row_linear_cls = RowParallelLinear if native_row else TERowParallelLinear
lora_a = row_linear_cls(
input_size=row_input_size,
output_size=r,
bias=False,
input_is_parallel=True,
input_is_parallel=getattr(self.base_layer, 'input_is_parallel', True),
**kwargs,
)
lora_b = _build_local_te_linear(r, self.out_features, lora_bias, **kwargs)
lora_a.parallel_mode = self.base_layer.parallel_mode # fix moe_shared_expert_overlap
lora_a.parallel_mode = getattr(self.base_layer, 'parallel_mode', None) # fix moe_shared_expert_overlap
else:
if is_torch_npu_available():
out_features = self.out_features
Expand Down Expand Up @@ -262,14 +270,19 @@ def update_layer(self, adapter_name, r, *, lora_alpha, **kwargs):
)
else:
lora_a = _build_local_te_linear(self.in_features, r, lora_bias, **kwargs)
lora_b = TEColumnParallelLinear(
input_size=r,
output_size=out_features,
bias=lora_bias,
gather_output=False,
**kwargs,
)
lora_b.parallel_mode = self.base_layer.parallel_mode # fix moe_shared_expert_overlap
if replicated_base:
# Match the base layer's replicated output.
lora_b = _build_local_te_linear(r, self.out_features, lora_bias, **kwargs)
else:
lora_b = TEColumnParallelLinear(
input_size=r,
output_size=out_features,
bias=lora_bias,
gather_output=False,
**kwargs,
)
lora_b.parallel_mode = getattr(self.base_layer, 'parallel_mode',
None) # fix moe_shared_expert_overlap
for lora in [lora_a, lora_b]:
# When parallel_mode is set to None by moe_shared_expert_overlap,
# disable UB comm overlap; the corresponding collectives are driven
Expand All @@ -280,18 +293,14 @@ def update_layer(self, adapter_name, r, *, lora_alpha, **kwargs):
lora.ub_overlap_ag_fprop = False
lora.ub_overlap_rs_dgrad = False

# With sequence parallelism the replicated (non-sharded) LoRA factor only sees this TP
# rank's sequence shard: for RowParallel targets lora_A reduce-scatters its output before
# lora_B, and for ColumnParallel targets lora_A consumes the sequence-sharded input. Its
# gradient must therefore be summed over the TP group. Megatron does this in
# finalize_model_grads for parameters flagged `sequence_parallel` (same as layernorm
# weights); without the flag each TP rank trains a different copy and export_weights
# saves rank 0 only (observed: last layer linear_proj.lora_B saved as all zeros).
# Sequence-parallel inputs need TP reduction for replicated LoRA weights.
# Both factors are replicated when the base layer is duplicated.
if (self.tp_size > 1 and not isinstance(self.base_layer, TopKRouter)
and (getattr(self.config, 'sequence_parallel', False) or self.sequence_parallel)):
replicated = lora_b if self.is_parallel_a else lora_a
for p in replicated.parameters():
p.sequence_parallel = True
replicated_factors = (lora_a, lora_b) if replicated_base else (lora_b if self.is_parallel_a else lora_a, )
for factor in replicated_factors:
for p in factor.parameters():
p.sequence_parallel = True
self.lora_A[adapter_name] = lora_a
self.lora_B[adapter_name] = lora_b
if hasattr(self, 'lora_bias'):
Expand Down Expand Up @@ -425,7 +434,7 @@ def forward(self, x: torch.Tensor, *args: Any, **kwargs: Any):
f'Got base_layer type: {type(self.base_layer)}. ')
else:
(result, x), bias = self.base_layer(x, *args, **kwargs)
elif isinstance(self.base_layer, (TELinear, TEGroupedLinear)):
elif isinstance(self.base_layer, (TELinear, TEGroupedLinear, ColumnParallelLinear, RowParallelLinear)):
result, bias = self.base_layer(x, *args, **kwargs)
elif isinstance(self.base_layer, TopKRouter):
with self._patch_router_gating():
Expand Down
5 changes: 5 additions & 0 deletions src/mcore_bridge/tuners/patcher.py
Original file line number Diff line number Diff line change
@@ -1,12 +1,14 @@
# Copyright (c) ModelScope Contributors. All rights reserved.
from megatron.core.extensions.transformer_engine import TEGroupedLinear, TELayerNormColumnParallelLinear, TELinear
from megatron.core.tensor_parallel.layers import ColumnParallelLinear, RowParallelLinear
from megatron.core.transformer.module import MegatronModule
from megatron.core.transformer.moe.router import TopKRouter
from peft import LoraModel
from peft.tuners.lora import Linear as LoraLinear
from peft.tuners.lora import model
from peft.tuners.tuners_utils import BaseTunerLayer
from torch import nn
from transformers.utils import is_torch_npu_available
from typing import Optional

from mcore_bridge.utils import patch_deepcopy
Expand All @@ -28,6 +30,9 @@ def dispatch_megatron(
target_base_layer = target

linear_cls = (TELayerNormColumnParallelLinear, TELinear, TEGroupedLinear, TopKRouter)
if is_torch_npu_available():
# MindSpeed uses native mcore classes for TE parallel linears.
linear_cls += (ColumnParallelLinear, RowParallelLinear)
if isinstance(target_base_layer, linear_cls):
new_module = LoraParallelLinear(base_layer=target, adapter_name=adapter_name, **kwargs)

Expand Down
Loading