Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
82 changes: 55 additions & 27 deletions src/mcore_bridge/model/modules/kernels/ple_kernels.py
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,28 @@
except Exception: # pragma: no cover - triton absent
HAVE_TRITON = False

# Bound the three FP32 gate-gradient partials; norm-weight reduction needs two
# such tiles. Required input/output gradients are not included in this budget.
_PLE_BACKWARD_WORKSPACE_BYTES = 64 * 1024 * 1024


def _ple_backward_chunk_rows(width):
return max(1, _PLE_BACKWARD_WORKSPACE_BYTES // max(1, 3 * width * 4))


def _ple_norm_weight_gradient(gated, dnormed, rstd, groups, dtype):
"""Accumulate the norm weight gradient without a token-sized FP32 product."""
total, width = gated.shape
channels = width // groups
gradient = torch.zeros(width, dtype=torch.float32, device=gated.device)
chunk = _ple_backward_chunk_rows(width)
for start in range(0, total, chunk):
end = min(start + chunk, total)
normalized = gated[start:end].reshape(-1, groups, channels) * rstd[start:end].unsqueeze(-1)
gradient.add_((dnormed[start:end] * normalized.flatten(-2)).sum(dim=0))
return gradient.to(dtype)


if HAVE_TRITON:

@triton.jit
Expand Down Expand Up @@ -481,46 +503,52 @@ def backward(ctx, dout):
# The convolution backward has consumed the recomputed norm output.
del normed

# norm_conv backward: dwc on host, dx via kernel (fp32).
x_hat = (gated.view(T, n, C) * rstdc.unsqueeze(-1)).view(T, W)
dwc = (dnormed * x_hat).sum(dim=0).to(wc.dtype)
del x_hat
dgated_norm = torch.empty(T, W, dtype=torch.float32, device=dev)
dwc = _ple_norm_weight_gradient(gated, dnormed, rstdc, n, wc.dtype)
if T > 0:
_ple_norm_bwd_kernel[(T * n, )](gated, wc, rstdc, dnormed, dgated_norm, T, N=n, C=C, BLOCK_C=block_c)
dgated += dgated_norm
# Release token-sized FP32 temporaries before gate gradient buffers.
del gated, dnormed, dgated_norm
# Each program owns one token/group and reads its whole input
# row before writing, so the norm input gradient can reuse it.
_ple_norm_bwd_kernel[(T * n, )](gated, wc, rstdc, dnormed, dnormed, T, N=n, C=C, BLOCK_C=block_c)
dgated += dnormed
del gated, dnormed

dkey = torch.empty_like(key)
dquery = torch.empty_like(hc_state)
dvalue_pern = torch.empty(T, n, C, dtype=torch.float32, device=dev)
dwk_part = torch.empty(T, W, dtype=torch.float32, device=dev)
dwq_part = torch.empty(T, W, dtype=torch.float32, device=dev)
if T > 0:
_ple_gate_bwd_kernel[(T * n, )](
dgated,
key,
hc_state,
value,
dvalue = torch.empty_like(value)
dwk = torch.zeros_like(wk, dtype=torch.float32)
dwq = torch.zeros_like(wq, dtype=torch.float32)
chunk = _ple_backward_chunk_rows(W)
for start in range(0, T, chunk):
end = min(start + chunk, T)
rows = end - start
dvalue_pern = torch.empty(rows, n, C, dtype=torch.float32, device=dev)
dwk_part = torch.empty(rows, W, dtype=torch.float32, device=dev)
dwq_part = torch.empty(rows, W, dtype=torch.float32, device=dev)
_ple_gate_bwd_kernel[(rows * n, )](
dgated[start:end],
key[start:end],
hc_state[start:end],
value[start:end],
wk,
wq,
gate,
rstdk,
rstdq,
dkey,
dquery,
gate[start:end],
rstdk[start:end],
rstdq[start:end],
dkey[start:end],
dquery[start:end],
dvalue_pern,
dwk_part,
dwq_part,
T,
rows,
N=n,
C=C,
SQRTC=_math.sqrt(C),
BLOCK_C=block_c)
dvalue = dvalue_pern.sum(dim=1).to(value.dtype)
dwk = dwk_part.sum(dim=0).to(wk.dtype)
dwq = dwq_part.sum(dim=0).to(wq.dtype)
dvalue[start:end] = dvalue_pern.sum(dim=1).to(value.dtype)
dwk.add_(dwk_part.sum(dim=0))
dwq.add_(dwq_part.sum(dim=0))
del dvalue_pern, dwk_part, dwq_part
dwk = dwk.to(wk.dtype)
dwq = dwq.to(wq.dtype)
dconv_w = dconvw.view(convw2d.shape).view(W, 1, Kk).to(conv_w_dtype)

return (dquery, dkey, dvalue, dwk, dwq, dwc, dconv_w, None, None, None, None)
Expand Down
76 changes: 76 additions & 0 deletions tests/test_ple_backward_workspace.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,76 @@
# Copyright (c) ModelScope Contributors. All rights reserved.
import importlib.util
import pytest
import torch
from pathlib import Path

_PATH = Path(__file__).parents[1] / 'src/mcore_bridge/model/modules/kernels/ple_kernels.py'
_SPEC = importlib.util.spec_from_file_location('ple_workspace_under_test', _PATH)
kernels = importlib.util.module_from_spec(_SPEC)
_SPEC.loader.exec_module(kernels)


@pytest.mark.parametrize('total', [0, 1, 19, 257])
@pytest.mark.parametrize('groups', [2, 4])
@pytest.mark.parametrize('budget', [128, 1 << 30])
def test_norm_weight_gradient_matches_full_product(monkeypatch, total, groups, budget):
torch.manual_seed(43)
width = groups * 16
gated = torch.randn(total, width)
dnormed = torch.randn_like(gated)
rstd = torch.rand(total, groups)
expected = (dnormed * (gated.reshape(total, groups, 16) * rstd[..., None]).reshape(total, width)).sum(0)
monkeypatch.setattr(kernels, '_PLE_BACKWARD_WORKSPACE_BYTES', budget)
actual = kernels._ple_norm_weight_gradient(gated, dnormed, rstd, groups, torch.float32)
torch.testing.assert_close(actual, expected, atol=1e-5, rtol=2e-5)


def test_gate_partial_workspace_is_bounded():
width = 10240
rows = kernels._ple_backward_chunk_rows(width)
assert 3 * rows * width * 4 <= kernels._PLE_BACKWARD_WORKSPACE_BYTES
assert rows < 262144


@pytest.mark.skipif(not torch.cuda.is_available() or not kernels.HAVE_TRITON, reason='CUDA and Triton required')
@pytest.mark.parametrize('rows,seq_len', [(1, 37), (2, 257)])
def test_chunked_gate_backward_matches_single_chunk(monkeypatch, rows, seq_len):
torch.manual_seed(47)
groups, channels = 4, 16
total, width = rows * seq_len, groups * channels
shapes = [(total, width), (total, width), (total, channels), (width, ), (width, ), (width, ), (width, 1, 4)]
values = [torch.randn(shape, device='cuda') * 0.2 for shape in shapes]
single = [value.clone().requires_grad_() for value in values]
chunked = [value.clone().requires_grad_() for value in values]
gradient = torch.randn(total, width, device='cuda')
monkeypatch.setattr(kernels, '_PLE_BACKWARD_WORKSPACE_BYTES', 1 << 30)
expected = kernels.ple_gate_conv_triton(*single, groups, 1e-6, 3, seq_len)
expected_grads = torch.autograd.grad(expected, single, gradient)
monkeypatch.setattr(kernels, '_PLE_BACKWARD_WORKSPACE_BYTES', 4096)
actual = kernels.ple_gate_conv_triton(*chunked, groups, 1e-6, 3, seq_len)
actual_grads = torch.autograd.grad(actual, chunked, gradient)
torch.testing.assert_close(actual, expected, rtol=0, atol=0)
for result, reference in zip(actual_grads, expected_grads):
torch.testing.assert_close(result, reference, atol=2e-4, rtol=2e-3)


@pytest.mark.skipif(not torch.cuda.is_available(), reason='CUDA peak-memory measurement required')
def test_norm_weight_gradient_peak_memory_is_bounded(monkeypatch):
torch.manual_seed(53)
total, groups, channels = 4096, 4, 128
width = groups * channels
gated = torch.randn(total, width, device='cuda')
dnormed = torch.randn_like(gated)
rstd = torch.rand(total, groups, device='cuda')
peaks, results = [], []
for budget in [1 << 30, 128 * 1024]:
monkeypatch.setattr(kernels, '_PLE_BACKWARD_WORKSPACE_BYTES', budget)
torch.cuda.synchronize()
baseline = torch.cuda.memory_allocated()
torch.cuda.reset_peak_memory_stats()
result = kernels._ple_norm_weight_gradient(gated, dnormed, rstd, groups, torch.float32)
torch.cuda.synchronize()
peaks.append(torch.cuda.max_memory_allocated() - baseline)
results.append(result)
torch.testing.assert_close(results[1], results[0], atol=1e-4, rtol=2e-4)
assert peaks[1] < peaks[0] / 2, peaks