diff --git a/hyper_parallel/core/shard/_op_dispatch.py b/hyper_parallel/core/shard/_op_dispatch.py index 7d2205e96be1c3d6840476c953acbda73f6a6a00..0e4c9823d9c6d9bfd4548951bde69bcdc7b11512 100644 --- a/hyper_parallel/core/shard/_op_dispatch.py +++ b/hyper_parallel/core/shard/_op_dispatch.py @@ -185,20 +185,16 @@ class OpDispatcher: else: raise - def _with_layout_infer(self, func: callable, *args, **kwargs) -> Tensor: - """_with_layout_infer""" - func_name = platform.get_op_name(func) - packed_call = None - # Ops in unpack_ops use packed fallback args (e.g. ScatterUpdate: (prim_obj, op_name: str, (input_x, indices, updates))). - if(func_name in self.unpack_ops and len(args) == 3 and - isinstance(args[1], str) and isinstance(args[2],(tuple,list))): - packed_call = (args[0], args[1]) - args = tuple(args[2]) - - cache_key = LayoutCacheKey([]) + def _process_args_and_kwargs( + self, args, kwargs, cache_key: "LayoutCacheKey" + ) -> tuple[list, list, list, dict]: + """_process_args_and_kwargs""" + # input_layouts contain prarmeters which have layout, extra_args contain other parameters input_layouts = [] extra_args = [] + # input_args are position prarmeters, input_kwargs are keyword parameters input_args = [] + input_kwargs = kwargs.copy() # Normal ops pass real inputs directly (e.g. SumExt: args = (dtensor, axis: list, keep_dims: bool, dtype: None)). for arg in args: @@ -225,6 +221,41 @@ class OpDispatcher: else: input_args.append(arg) + for k, val in kwargs.items(): + if val is None: + input_layouts.append(None) + continue + if not hasattr(val, "layout"): + id_str = "scalar" + if not isinstance(val, Tensor): + id_str = str(val) + cache_key.layout_ids.append(id_str) + extra_args.append(val) + input_layouts.append(None) + else: + layout = val.layout + layout_id = layout.compact_str + cache_key.layout_ids.append(str(layout_id)) + input_layouts.append(layout) + if isinstance(val, DTensor): + input_kwargs[k] = val.to_local() + + return input_layouts, extra_args, input_args, input_kwargs + + def _with_layout_infer(self, func: callable, *args, **kwargs) -> Tensor: + """_with_layout_infer""" + func_name = platform.get_op_name(func) + packed_call = None + # Ops in unpack_ops use packed fallback args (e.g. ScatterUpdate: (prim_obj, op_name: str, (input_x, indices, updates))). + if(func_name in self.unpack_ops and len(args) == 3 and + isinstance(args[1], str) and isinstance(args[2],(tuple,list))): + packed_call = (args[0], args[1]) + args = tuple(args[2]) + + cache_key = LayoutCacheKey([]) + input_layouts, extra_args, input_args, input_kwargs = self._process_args_and_kwargs( + args, kwargs, cache_key + ) cache_manager = LayoutCacheManager.get_instance() layout_cache = cache_manager.get_layout_cache() if func_name not in layout_cache: @@ -245,9 +276,9 @@ class OpDispatcher: op_impl = func if packed_call is not None: - py_output = op_impl(packed_call[0], packed_call[1], tuple(input_args), **kwargs) + py_output = op_impl(packed_call[0], packed_call[1], tuple(input_args), **input_kwargs) else: - py_output = op_impl(*input_args, **kwargs) + py_output = op_impl(*input_args, **input_kwargs) if isinstance(py_output, (tuple, list)): output = () @@ -383,23 +414,15 @@ class OpDispatcher: return DTensor.from_local(py_output, infer_output_tuple[0]) - def _with_layout_infer_with_shape(self, func: callable, *args, **kwargs) -> Tensor: - """_with_layout_infer_with_shape""" - func_name = platform.get_op_name(func) - packed_call = None - # Packed fallback args for some ops (e.g. Mod: (prim_obj, "Mod", (x, y))). - if (func_name in self.unpack_ops and len(args) == 3 and - isinstance(args[1], str) and isinstance(args[2], (tuple, list))): - packed_call = (args[0], args[1]) - args = tuple(args[2]) - - cache_key = LayoutCacheKey([]) + def _process_args_and_kwargs_with_shape( + self, args, kwargs, cache_key: "LayoutCacheKey" + ) -> tuple[list, list, list, list, dict]: + """_process_args_and_kwargs_with_shape""" input_layouts = [] extra_args = [] input_shapes = [] input_args = [] - - # Normal ops pass real inputs directly (e.g. GreaterEqual: (dtensor_x, dtensor_y)). + input_kwargs = kwargs.copy() for arg in args: if arg is None: input_layouts.append(None) @@ -432,6 +455,48 @@ class OpDispatcher: input_shapes.append(input_shape) cache_key.layout_ids.append(str(input_shape)) + for k, val in kwargs.items(): + if val is None: + input_layouts.append(None) + continue + if not hasattr(val, "layout"): + id_str = "scalar" + if not isinstance(val, Tensor): + id_str = str(val) + cache_key.layout_ids.append(id_str) + extra_args.append(val) + input_layouts.append(None) + else: + layout = val.layout + layout_id = layout.compact_str + cache_key.layout_ids.append(str(layout_id)) + input_layouts.append(layout) + if isinstance(val, DTensor): + input_kwargs[k] = val.to_local() + + if not hasattr(val, "shape"): + input_shapes.append(None) + else: + input_shape = val.shape + input_shapes.append(input_shape) + cache_key.layout_ids.append(str(input_shape)) + + return input_layouts, input_shapes, extra_args, input_args, input_kwargs + + def _with_layout_infer_with_shape(self, func: callable, *args, **kwargs) -> Tensor: + """_with_layout_infer_with_shape""" + func_name = platform.get_op_name(func) + packed_call = None + # Packed fallback args for some ops (e.g. Mod: (prim_obj, "Mod", (x, y))). + if (func_name in self.unpack_ops and len(args) == 3 and + isinstance(args[1], str) and isinstance(args[2], (tuple, list))): + packed_call = (args[0], args[1]) + args = tuple(args[2]) + + cache_key = LayoutCacheKey([]) + input_layouts, input_shapes, extra_args, input_args, input_kwargs = \ + self._process_args_and_kwargs_with_shape(args, kwargs, cache_key) + cache_manager = LayoutCacheManager.get_instance() layout_cache = cache_manager.get_layout_cache() if func_name not in layout_cache: @@ -453,9 +518,9 @@ class OpDispatcher: op_impl = func if packed_call is not None: - py_output = op_impl(packed_call[0], packed_call[1], tuple(input_args), **kwargs) + py_output = op_impl(packed_call[0], packed_call[1], tuple(input_args), **input_kwargs) else: - py_output = op_impl(*input_args, **kwargs) + py_output = op_impl(*input_args, **input_kwargs) # 设置输出布局 if isinstance(py_output, (tuple, list)): diff --git a/hyper_parallel/core/shard/ops/parallel_norm.py b/hyper_parallel/core/shard/ops/parallel_norm.py index acb8161d85dd57bf4a4d35a88fb2f096beac5d5a..a068bc5fec51c845bf6f594be8d1c0bf54630371 100644 --- a/hyper_parallel/core/shard/ops/parallel_norm.py +++ b/hyper_parallel/core/shard/ops/parallel_norm.py @@ -88,3 +88,78 @@ class NormDistributedOp(DistributedOp): output_map = x_layout.alias_tensor_map[:begin_norm_axis] + ("None",) * len(gamma_tensor_map) out_layout = output_layout(*output_map) return x_layout, out_layout + +class LayerNormDistributedOp(DistributedOp): + """Distributed implementation for torch.nn.functional.layer_norm.""" + + def infer_layout(self, layouts, extra_args=None): + """ + Infer output layout for layer_norm. + + PyTorch rules: + - normalized_shape specifies the last N dimensions to normalize over. + - All dimensions in normalized_shape MUST be unsharded for correctness. + - Output layout is identical to input layout (shape unchanged). + + Args: + layouts (tuple): Layouts of inputs. Expected: + layouts[0] (Layout): Input tensor layout (required). + extra_args (tuple): Should contain 'normalized_shape'. Expected: + extra_args[0] (int | list | tuple): Normalized shape to be unsharded. + + Returns: + Layout object representing output tensor layout (same as input if valid). + """ + if not layouts or layouts[0] is None: + raise ValueError("layer_norm requires a valid input tensor layout.") + input_layout = layouts[0] + in_tensor_map = input_layout.tensor_map # e.g., (-1, 0, -1) for 3D tensor + + if not extra_args or extra_args[0] is None: + raise ValueError("layer_norm requires normalized_shape in extra_args.") + normalized_shape = extra_args[0] + + if isinstance(normalized_shape, int): + normalized_shape = (normalized_shape,) + elif isinstance(normalized_shape, (list, tuple)): + normalized_shape = tuple(normalized_shape) + else: + raise ValueError(f"normalized_shape must be int, list, or tuple, got {type(normalized_shape)}") + + input_ndim = len(in_tensor_map) + norm_ndim = len(normalized_shape) + + if norm_ndim > input_ndim: + raise ValueError( + f"normalized_shape {normalized_shape} (dims={norm_ndim}) is larger than input ndim={input_ndim}." + ) + + # The last `norm_ndim` dimensions are going to be normalized + dims_to_normalize = list(range(input_ndim - norm_ndim, input_ndim)) + + # All normalized dims must be unsharded + for dim in dims_to_normalize: + if in_tensor_map[dim] != -1: + raise ValueError( + f"Operation {self.op_name}: Cannot perform sharding on normalized dimension {dim}, " + f"but found sharding assignment: {in_tensor_map[dim]}" + ) + + mesh_shape = input_layout.mesh_shape + alias_name = input_layout.alias_name + rank_list = input_layout.rank_list + + # Create output layout + def idx_to_alias(idx, aliases): + if idx == -1: + return "None" + return aliases[len(aliases) - idx - 1] + output_map = tuple(idx_to_alias(idx, alias_name) for idx in in_tensor_map) + + output_layout = Layout( + mesh_shape=mesh_shape, + alias_name=alias_name, + rank_list=rank_list + ) + output_layout = output_layout(*output_map) + return output_layout diff --git a/hyper_parallel/core/shard/ops/parallel_split.py b/hyper_parallel/core/shard/ops/parallel_split.py index a5a755f389d6c74e038582790fc9cef097cab233..165c271e678caede72f9bff3da84e445730aef62 100644 --- a/hyper_parallel/core/shard/ops/parallel_split.py +++ b/hyper_parallel/core/shard/ops/parallel_split.py @@ -16,6 +16,7 @@ Distributed implementation for TopK operator. """ +import math from .parallel_ops import DistributedOp @@ -90,23 +91,51 @@ class SplitDistributedOp(DistributedOp): Rules: 1. Shared axis can not be split. + 2. Default: dim = 0 if not specified. Args: layouts (Layout): Layout of input tensor - extra_args (list): split size or sections, axis, input shape + extra_args (list): split size or sections, axis, input shape. Expected: + extra_args[0]: split_size (required) + extra_args[1]: axis (optional) + extra_args[2][0]: input_shape Returns: tuple: Layouts for output tensors """ + if not layouts or layouts[0] is None: + raise ValueError("split requires a valid input tensor layout.") input_layout = layouts[0] - axis = extra_args[0] - # Check shared axis can not be split. + + if len(extra_args) == 2: + split_size = extra_args[0] + axis = 0 # default + input_shape = extra_args[1][0] + elif len(extra_args) == 3: + split_size = extra_args[0] + axis = extra_args[1] + input_shape = extra_args[2][0] + else: + raise ValueError("Split ops extra_args requires 'axis' and contains 'output_num' optionally.") + tensor_map = input_layout.tensor_map + input_dim = len(tensor_map) + if axis < 0: + axis = input_dim + axis + if not 0 <= axis < input_dim: + raise ValueError(f"Dimension out of range (expected [0, {input_dim}), got {axis}).") + + # Check shared axis can not be split. if tensor_map[axis] != -1: raise ValueError(f"Can not split tensor at sharded axis[{axis}], layout: {input_layout}") - output_num = extra_args[1] + output_num = 1 + if isinstance(split_size, int): + output_num = math.ceil(input_shape[axis] / split_size) + elif isinstance(split_size, (list, tuple)): + output_num = len(split_size) + output_layouts = (input_layout,) * output_num return output_layouts diff --git a/hyper_parallel/core/shard/ops/parallel_topk.py b/hyper_parallel/core/shard/ops/parallel_topk.py index 36683d250b01dfd69fa9e970b79f80417e7adde3..da7f1be4611eb14d744d6a49c943abbd5c25a413 100644 --- a/hyper_parallel/core/shard/ops/parallel_topk.py +++ b/hyper_parallel/core/shard/ops/parallel_topk.py @@ -25,21 +25,42 @@ class TopKDistributedOp(DistributedOp): """ Infer output layouts for TopK operator. - TopK: values, indices = topk(input, k) + TopK: values, indices = topk(input, k, dim) Rules: - 1. Output layouts must match input layout exactly - 2. Both values and indices have same layout as input + 1. dim = -1 if not specified. + 2. The dimension `dim` MUST be unsharded to ensure global top-k correctness. + 3. Both values and indices have same layout as input Args: - layouts (Layout): Layout of input tensor - k (int, optional): The k in topk, not affecting layout + layouts (tuple): Layouts of inputs. Expected: + layouts[0] (Layout): Input tensor layout (required). + extra_args (tuple, optional): Requires k and optionally contains dim. Expected: + extra_args[0] (int, required): K value. + extra_args[1] (int, optional): Dimension to compute topk. Defaults to -1. Returns: tuple: Layouts for values and indices tensors """ + if not layouts or layouts[0] is None: + raise ValueError("topk requires a valid input tensor layout.") - if not layouts[0]: - raise ValueError(f"layouts is empty: {layouts[0]}") + input_layout = layouts[0] + in_tensor_map = input_layout.tensor_map - return layouts[0], layouts[0] + dim = -1 # If dim is not given, the last dimension of the input is chosen. + if len(extra_args) >= 2 and extra_args[1] is not None: + dim = extra_args[1] + input_dim = len(in_tensor_map) + if dim < 0: + dim = input_dim + dim # -1 represents the last dimension + if not 0 <= dim < input_dim: + raise ValueError(f"Dimension out of range (expected to be in [0, {input_dim}), but got {dim}).") + + # The chosen dim must NOT be sharded + if in_tensor_map[dim] != -1: + raise ValueError( + f"Operation {self.op_name}: Cannot perform sharding on params along the chosen dim" + ) + + return input_layout, input_layout diff --git a/hyper_parallel/core/shard/ops/yaml/norm_ops.yaml b/hyper_parallel/core/shard/ops/yaml/norm_ops.yaml index c3f857b6d190a3c73c4a00f1cb9bb6c30084fecb..e7692ba8c54aa26be8eec48140d15a76b2aeca90 100644 --- a/hyper_parallel/core/shard/ops/yaml/norm_ops.yaml +++ b/hyper_parallel/core/shard/ops/yaml/norm_ops.yaml @@ -2,3 +2,8 @@ RmsNorm: dist_op_name: _rms_norm_dist_op distributed_op_class: NormDistributedOp distributed_op_file: parallel_norm + +layer_norm: + dist_op_name: _torch_layer_norm_dist_op + distributed_op_class: LayerNormDistributedOp + distributed_op_file: parallel_norm diff --git a/hyper_parallel/core/shard/ops/yaml/split_ops.yaml b/hyper_parallel/core/shard/ops/yaml/split_ops.yaml index b18f0774be007c6d1077d28eeb00d3fc1ad6e6b4..4521e4a25b69c11cbd1b27782a69f119e406841b 100644 --- a/hyper_parallel/core/shard/ops/yaml/split_ops.yaml +++ b/hyper_parallel/core/shard/ops/yaml/split_ops.yaml @@ -2,6 +2,7 @@ Split: dist_op_name: _split_dist_op distributed_op_class: SplitDistributedOp distributed_op_file: parallel_split + infer_layout_suffix: WithShape SplitTensor: dist_op_name: _split_dist_op @@ -24,3 +25,9 @@ SplitWithSizeView: dist_op_name: _split_dist_op distributed_op_class: SplitWithSizeViewDistributedOp distributed_op_file: parallel_split + +split: + dist_op_name: _torch_split_dist_op + distributed_op_class: SplitDistributedOp + distributed_op_file: parallel_split + infer_layout_suffix: WithShape diff --git a/hyper_parallel/core/shard/ops/yaml/topk_ops.yaml b/hyper_parallel/core/shard/ops/yaml/topk_ops.yaml index 58bae5d9dd012628657aa12a81073c69cf39afd4..52dd825e713e61d3dcc47b24820f85fd487ff7b2 100644 --- a/hyper_parallel/core/shard/ops/yaml/topk_ops.yaml +++ b/hyper_parallel/core/shard/ops/yaml/topk_ops.yaml @@ -2,3 +2,8 @@ TopkExt: dist_op_name: _topk_ext_dist_op distributed_op_class: TopKDistributedOp distributed_op_file: parallel_topk + +topk: + dist_op_name: _torch_topk_ext_dist_op + distributed_op_class: TopKDistributedOp + distributed_op_file: parallel_topk diff --git a/tests/mindspore/ut/parallel_ops_infer/test_parallel_norm.py b/tests/mindspore/ut/parallel_ops_infer/test_parallel_norm.py index d3864b74ed3d176740df2215802b04069ca56b7e..8585402d18faec31a4c5b324d9357a0023dbc9fd 100644 --- a/tests/mindspore/ut/parallel_ops_infer/test_parallel_norm.py +++ b/tests/mindspore/ut/parallel_ops_infer/test_parallel_norm.py @@ -14,10 +14,13 @@ # ============================================================================ """parallel_norm test""" +import pytest from hyper_parallel import Layout from hyper_parallel.core.shard.ops.parallel_norm import NormDistributedOp +from hyper_parallel.core.shard.ops.parallel_norm import LayerNormDistributedOp op = NormDistributedOp("RmsNorm") +torch_op = LayerNormDistributedOp("layer_naorm") def test_rmsnorm_layout_data_parallel(): @@ -149,3 +152,71 @@ def test_rmsnorm_invalid_layout(): assert False, "Expected ValueError for invalid splitting on normalization axis" except ValueError as e: assert "RmsNorm is disabled to support the splitting after" in str(e) + + +def test_torch_layernorm_layout_data_parallel(): + """ + Feature: LayerNorm data parallel + Description: Data parallel scenario with no splitting on normalization axis + Expectation: Success + """ + base_mesh_shape = (8,) + base_alias_name = ("dp",) + base_rank_list = list(range(8)) + x_layout = Layout(base_mesh_shape, base_alias_name, base_rank_list) + x_layout = x_layout("dp", "None") + + out_layout = torch_op.infer_layout((x_layout,), extra_args=((64,),)) + + expected_map = (0, -1) + assert out_layout.to_dict()["tensor_map"] == expected_map + + +def test_torch_layernorm_layout_tensor_parallel(): + """ + Feature: LayerNorm tensor parallel + Description: Tensor parallel scenario with splitting on non-normalization axes + Expectation: Success + """ + base_mesh_shape = (2, 4) + base_alias_name = ("dp", "mp") + base_rank_list = list(range(8)) + x_layout = Layout(base_mesh_shape, base_alias_name, base_rank_list) + x_layout = x_layout("dp", "mp", "None") + + # Normalize last dim + out_layout = torch_op.infer_layout((x_layout,), extra_args=((128,),)) + + expected_map = (1, 0, -1) + assert out_layout.to_dict()["tensor_map"] == expected_map + + +def test_torch_layernorm_layout_normalized_dim_sharded(): + """ + Feature: Test error when normalized dimension is sharded + Description: Test error when normalized dimension is sharded + Expectation: Success + """ + base_mesh_shape = (2, 4) + base_alias_name = ("dp", "mp") + base_rank_list = list(range(8)) + x_layout = Layout(base_mesh_shape, base_alias_name, base_rank_list) + x_layout = x_layout("None", "mp") # sharded on 'mp' + + with pytest.raises(ValueError, match="Cannot perform sharding on normalized dimension 1"): + torch_op.infer_layout((x_layout,), extra_args=((64,),)) + + +def test_torch_layernorm_layout_normalized_shape_too_large(): + """ + Feature: Test normalized_shape larger than input ndim + Description: Test normalized_shape larger than input ndim + Expectation: Success + """ + base_mesh_shape = (2, 4) + base_alias_name = ("dp", "mp") + base_rank_list = list(range(8)) + x_layout = Layout(base_mesh_shape, base_alias_name, base_rank_list) + x_layout = x_layout("dp", "mp") + with pytest.raises(ValueError, match="larger than input ndim"): + torch_op.infer_layout((x_layout,), extra_args=((128, 64, 56),)) diff --git a/tests/mindspore/ut/parallel_ops_infer/test_parallel_split.py b/tests/mindspore/ut/parallel_ops_infer/test_parallel_split.py index fa586bd8be84e3d61bc48641088ce9e5aa8cf09e..01130f517385c37a8f6722e52fbb4659faa98e0a 100644 --- a/tests/mindspore/ut/parallel_ops_infer/test_parallel_split.py +++ b/tests/mindspore/ut/parallel_ops_infer/test_parallel_split.py @@ -21,6 +21,7 @@ from hyper_parallel import Layout # 初始化一个SplitDistributedOp实例 split_op = SplitDistributedOp("split") +torch_split_op = SplitDistributedOp("split") # 定义一个辅助函数来创建Layout对象 @@ -42,11 +43,12 @@ def test_infer_layout_normal(): """ input_layout = create_layout([1, -1, 0]) axis = 1 - output_num = 2 - extra_args = [axis, output_num] + split_size = 2 + input_shape = ((12, 16, 20), None) + extra_args = [split_size, axis, input_shape] output_layouts = split_op.infer_layout([input_layout], extra_args) - assert len(output_layouts) == output_num + assert len(output_layouts) == 8 assert all(layout.tensor_map == input_layout.tensor_map for layout in output_layouts) @@ -58,10 +60,40 @@ def test_infer_layout_invalid_axis(): """ input_layout = create_layout([1, 0, -1]) axis = 0 - output_num = 2 - extra_args = [axis, output_num] + split_size = 2 + input_shape = ((12, 16, 20), None) + extra_args = [split_size, axis, input_shape] - with pytest.raises(ValueError): + with pytest.raises(ValueError, match="Can not split tensor at sharded axis"): + split_op.infer_layout([input_layout], extra_args) + + +def test_infer_layout_axis_out_of_range(): + """ + Feature: Test axis out of range + Description: Test axis out of range + Expectation: Success + """ + input_layout = create_layout([-1, -1]) + axis = 5 # invalid for 2D tensor + split_size = 2 + input_shape = ((12, 16, 20), None) + extra_args = [split_size, axis, input_shape] + + with pytest.raises(ValueError, match="Dimension out of range"): + split_op.infer_layout([input_layout], extra_args) + + +def test_infer_layout_insufficient_extra_args(): + """ + Feature: Test missing extra_arg + Description: Test missing extra_arg + Expectation: Success + """ + input_layout = create_layout([-1, -1]) + extra_args = [0] # missing output_num + + with pytest.raises(ValueError, match="Split ops extra_args requires 'axis' and contains 'output_num' optionally."): split_op.infer_layout([input_layout], extra_args) diff --git a/tests/mindspore/ut/parallel_ops_infer/test_parallel_topk.py b/tests/mindspore/ut/parallel_ops_infer/test_parallel_topk.py index ce715262c9b4d5772fe43cb490a6217f821f5259..b6aa3d88c621be961852c52eb7129f7500d563ea 100644 --- a/tests/mindspore/ut/parallel_ops_infer/test_parallel_topk.py +++ b/tests/mindspore/ut/parallel_ops_infer/test_parallel_topk.py @@ -13,11 +13,12 @@ # limitations under the License. """parallel_topk test""" +import pytest from hyper_parallel import Layout from hyper_parallel.core.shard.ops.parallel_topk import TopKDistributedOp op = TopKDistributedOp("TopK") - +torch_op = TopKDistributedOp("topk") def test_topk_layout_data_parallel(): """ @@ -33,7 +34,7 @@ def test_topk_layout_data_parallel(): x_layout = Layout(base_mesh_shape, base_alias_name, base_rank_list) x_layout = x_layout("dp", "None") - values_layout, indices_layout = op.infer_layout((x_layout,), True) + values_layout, indices_layout = op.infer_layout((x_layout,), ()) expected_map = (1, -1) # Expected output tensor map assert values_layout.to_dict()["tensor_map"] == indices_layout.to_dict()["tensor_map"] == expected_map, \ f"Data Parallel with transpose_a test failed. Expected {expected_map}," \ @@ -54,7 +55,7 @@ def test_topk_layout_tensor_parallel(): x_layout = Layout(base_mesh_shape, base_alias_name, base_rank_list) x_layout = x_layout("mp", "None") - values_layout, indices_layout = op.infer_layout((x_layout,), True) + values_layout, indices_layout = op.infer_layout((x_layout,), ()) expected_map = (0, -1) # Expected output tensor map assert values_layout.to_dict()["tensor_map"] == indices_layout.to_dict()["tensor_map"] == expected_map, \ f"Data Parallel with transpose_a test failed. Expected {expected_map}," \ @@ -67,16 +68,91 @@ def test_topk_layout_tensor_and_data_parallel(): Description: Tensor parallel scenario Expectation: Success """ + base_mesh_shape = (2, 4, 3) + base_alias_name = ("dp", "mp", "tp") + base_rank_list = list(range(24)) + + # Data and Tensor Parallel + x_layout = Layout(base_mesh_shape, base_alias_name, base_rank_list) + x_layout = x_layout("dp", "mp", "None") + + values_layout, indices_layout = op.infer_layout((x_layout,), ()) + expected_map = (2, 1, -1) # Expected output tensor map + assert values_layout.to_dict()["tensor_map"] == indices_layout.to_dict()["tensor_map"] == expected_map, \ + f"Data Parallel with transpose_a test failed. Expected {expected_map}," \ + f" got {values_layout.to_dict()['tensor_map']}" + + +def test_torch_topk_layout_data_parallel(): + """ + Feature: TopK data parallel + Description: Data parallel scenario (shard on first dim, topk on last unsharded dim) + Expectation: Success + """ base_mesh_shape = (2, 4) base_alias_name = ("dp", "mp") base_rank_list = list(range(8)) - # Data and Tensor Parallel x_layout = Layout(base_mesh_shape, base_alias_name, base_rank_list) - x_layout = x_layout("dp", "mp") + x_layout = x_layout("dp", "None") # This sets _tensor_map = (1, -1) - values_layout, indices_layout = op.infer_layout((x_layout,), True) - expected_map = (1, 0) # Expected output tensor map + values_layout, indices_layout = torch_op.infer_layout((x_layout,), ()) + expected_map = (1, -1) assert values_layout.to_dict()["tensor_map"] == indices_layout.to_dict()["tensor_map"] == expected_map, \ - f"Data Parallel with transpose_a test failed. Expected {expected_map}," \ + f"Data Parallel with torch topk test failed. Expected {expected_map}," \ + f" got {values_layout.to_dict()['tensor_map']}" + + +def test_torch_topk_layout_tensor_parallel(): + """ + Feature: TopK tensor parallel + Description: Tensor parallel scenario (shard on last dim is NOT allowed, so shard on first dim with 'mp') + Expectation: Success + """ + base_mesh_shape = (2, 4) + base_alias_name = ("dp", "mp") + base_rank_list = list(range(8)) + + x_layout = Layout(base_mesh_shape, base_alias_name, base_rank_list) + x_layout = x_layout("mp", "None") # → tensor_map = (0, -1) + + values_layout, indices_layout = torch_op.infer_layout((x_layout,), ()) + expected_map = (0, -1) + assert values_layout.to_dict()["tensor_map"] == indices_layout.to_dict()["tensor_map"] == expected_map, \ + f"Data Parallel with torch topk test failed. Expected {expected_map}," \ f" got {values_layout.to_dict()['tensor_map']}" + + +def test_torch_topk_layout_mixed_parallel_invalid(): + """ + Feature: Test topk on a sharded dimension + Description: Test topk on a sharded dimension + Expectation: Success + """ + base_mesh_shape = (2, 4) + base_alias_name = ("dp", "mp") + base_rank_list = list(range(8)) + + x_layout = Layout(base_mesh_shape, base_alias_name, base_rank_list) + x_layout = x_layout("dp", "mp") + + # topk on last dim (dim=-1), which is sharded → should fail + with pytest.raises(ValueError, match="Cannot perform sharding on params along the chosen dim"): + torch_op.infer_layout((x_layout,), ()) + + +def test_torch_topk_layout_error_dim_out_of_range(): + """ + Feature: Test indicating a invalid dim + Description: Test indicating a invalid dim + Expectation: Success + """ + base_mesh_shape = (2, 4) + base_alias_name = ("dp", "mp") + base_rank_list = list(range(8)) + + x_layout = Layout(base_mesh_shape, base_alias_name, base_rank_list) + x_layout = x_layout("None", "None") + + with pytest.raises(ValueError, match="Dimension out of range"): + torch_op.infer_layout((x_layout,), extra_args=(2, 5)) # dim=5 invalid diff --git a/tests/torch/shard/ops/parallel_op_layernorm.py b/tests/torch/shard/ops/parallel_op_layernorm.py new file mode 100644 index 0000000000000000000000000000000000000000..6bbe93ac0b8cb0028542cbb81eb6243fc7b3402b --- /dev/null +++ b/tests/torch/shard/ops/parallel_op_layernorm.py @@ -0,0 +1,94 @@ +# Copyright 2026 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""test torch dtensor with distributed layer_norm""" + +import numpy as np +import torch +from hyper_parallel import Layout +from tests.torch.utils import init_dist +from tests.torch.shard.utils import local_to_global, global_to_local + +# Generate input data using numpy at file header +np.random.seed(42) +standalone_input_np = np.random.randn(8, 16).astype(np.float32) + +def test_distributed_layer_norm_layout_inference(): + """ + Feature: dtensor + torch.nn.functional.layer_norm layout inference + Description: + - Test layout inference for layer_norm in distributed setting. + - Ensure that: + a) Output layout matches input layout when normalized dims are unsharded. + b) Results are consistent between standalone and distributed. + Expectation: Success. + """ + init_dist() + normalized_shape = (16,) # normalize over last dimension + + # Standalone (single-device) + standalone_input = torch.from_numpy(standalone_input_np).npu() + weight = torch.ones(normalized_shape).npu() + bias = torch.zeros(normalized_shape).npu() + standalone_output = torch.nn.functional.layer_norm( + standalone_input, normalized_shape, weight, bias + ) + + # Distributed setup + layout = Layout((2, 4), ("dp", "tp")) + x_layout = layout("dp", "None") + + dist_input = global_to_local(standalone_input, x_layout) + # Weight and bias must be replicated (same on all devices) + dist_weight = global_to_local(weight, layout("None",)) + dist_bias = global_to_local(bias, layout("None",)) + + dist_output = torch.nn.functional.layer_norm( + dist_input, normalized_shape, dist_weight, dist_bias + ) + + assert dist_output.layout == x_layout, "LayerNorm: output layout mismatch input" + + gathered_output = local_to_global(dist_output) + assert torch.allclose( + standalone_output, gathered_output, atol=1e-5 + ), "LayerNorm output mismatch between standalone and distributed" + + +def test_distributed_layer_norm_sharded_normalized_dim_error(): + """ + Feature: dtensor + layer_norm error on sharded normalized dimension + Description: + - Attempt to run layer_norm where normalized_shape includes a sharded dimension. + - Should raise ValueError during layout inference. + Expectation: Raise ValueError. + """ + init_dist() + normalized_shape = (16,) + + layout = Layout((2, 4), ("dp", "tp")) + # WRONG: shard on dim=1, which is part of normalized_shape → invalid + x_layout = layout("dp", "tp") + + standalone_input = torch.from_numpy(standalone_input_np).npu() + dist_input = global_to_local(standalone_input, x_layout) + dist_weight = global_to_local(torch.ones(normalized_shape).npu(), layout("None",)) + dist_bias = global_to_local(torch.zeros(normalized_shape).npu(), layout("None",)) + + try: + torch.nn.functional.layer_norm(dist_input, normalized_shape, dist_weight, dist_bias) + assert False, "Expected ValueError when normalized dimension is sharded" + except ValueError as e: + assert "Cannot perform sharding on normalized dimension" in str(e) + assert "dimension 1" in str(e) or "dimension -1" in str(e) diff --git a/tests/torch/shard/ops/parallel_op_split.py b/tests/torch/shard/ops/parallel_op_split.py new file mode 100644 index 0000000000000000000000000000000000000000..7d2075566c1e495fdcc00b92118f1b5484922c8e --- /dev/null +++ b/tests/torch/shard/ops/parallel_op_split.py @@ -0,0 +1,168 @@ +# Copyright 2026 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""test torch dtensor with distributed split""" +import numpy as np +import torch +from hyper_parallel import DTensor, Layout +from tests.torch.utils import init_dist +from tests.torch.shard.utils import global_to_local, local_to_global + +# Generate input data using numpy at file header +np.random.seed(42) +standalone_input_np = np.random.randn(16, 20).astype(np.float32) + +def test_distributed_split_layout_inference_default_dim(): + """ + Feature: dtensor + torch.split layout inference + Description: + - Test layout inference for torch.split in distributed setting. + - Ensure that: + a) Output layouts match input layout when splitting on unsharded dim. + b) Results are consistent with standalone execution. + Expectation: Success. + """ + init_dist() + split_size = 4 + + # Standalone + standalone_input = torch.from_numpy(standalone_input_np).npu() + standalone_outputs = torch.split(standalone_input, split_size) + assert len(standalone_outputs) == 4 # 16 / 4 = 4 chunks + + # Distributed setup + layout = Layout((4, 2), ("dp", "tp")) + # Shard only on dim=0 + x_layout = layout("None", "dp") + + dist_input = global_to_local(standalone_input, x_layout) + + # Perform split + dist_outputs = torch.split(dist_input, split_size) + + assert isinstance(dist_outputs, tuple) + assert len(dist_outputs) == 4 + for out in dist_outputs: + assert isinstance(out, DTensor) + assert out.layout == x_layout, f"Output layout {out.layout} != input layout {x_layout}" + + for i, (ref, dist_out) in enumerate(zip(standalone_outputs, dist_outputs)): + gathered = local_to_global(dist_out) + assert torch.allclose(ref, gathered, atol=1e-5), f"Chunk {i} mismatch" + + +def test_distributed_split_layout_inference(): + """ + Feature: dtensor + torch.split layout inference + Description: + - Test layout inference for torch.split in distributed setting. + - Ensure that: + a) Output layouts match input layout when splitting on unsharded dim. + b) Results are consistent with standalone execution. + Expectation: Success. + """ + init_dist() + split_size = 4 + axis = 1 # split along axis 1 + + # Standalone + standalone_input = torch.from_numpy(standalone_input_np).npu() + standalone_outputs = torch.split(standalone_input, split_size, dim=axis) + assert len(standalone_outputs) == 5 # 20 / 4 = 5 chunks + + # Distributed setup + layout = Layout((4, 2), ("dp", "tp")) + # Shard only on dim=0 + x_layout = layout("dp", "None") + + dist_input = global_to_local(standalone_input, x_layout) + + # Perform split + dist_outputs = torch.split(dist_input, split_size, dim=axis) + + assert isinstance(dist_outputs, tuple) + assert len(dist_outputs) == 5 + for out in dist_outputs: + assert isinstance(out, DTensor) + assert out.layout == x_layout, f"Output layout {out.layout} != input layout {x_layout}" + + for i, (ref, dist_out) in enumerate(zip(standalone_outputs, dist_outputs)): + gathered = local_to_global(dist_out) + assert torch.allclose(ref, gathered, atol=1e-5), f"Chunk {i} mismatch" + + +def test_distributed_split_layout_inference_split_list(): + """ + Feature: dtensor + torch.split layout inference + Description: + - Test layout inference for torch.split in distributed setting. + - Ensure that: + a) Output layouts match input layout when splitting on unsharded dim. + b) Results are consistent with standalone execution. + Expectation: Success. + """ + init_dist() + split_size = (8, 12) + axis = 1 # split along axis 1 + + # Standalone + standalone_input = torch.from_numpy(standalone_input_np).npu() + standalone_outputs = torch.split(standalone_input, split_size, dim=axis) + assert len(standalone_outputs) == 2 + + # Distributed setup + layout = Layout((4, 2), ("dp", "tp")) + # Shard only on dim=0 + x_layout = layout("dp", "None") + + dist_input = global_to_local(standalone_input, x_layout) + + # Perform split + dist_outputs = torch.split(dist_input, split_size, dim=axis) + + assert isinstance(dist_outputs, tuple) + assert len(dist_outputs) == 2 + for out in dist_outputs: + assert isinstance(out, DTensor) + assert out.layout == x_layout, f"Output layout {out.layout} != input layout {x_layout}" + + for i, (ref, dist_out) in enumerate(zip(standalone_outputs, dist_outputs)): + gathered = local_to_global(dist_out) + assert torch.allclose(ref, gathered, atol=1e-5), f"Chunk {i} mismatch" + + +def test_distributed_split_on_sharded_dim_error(): + """ + Feature: dtensor + torch.split error on sharded axis + Description: + - Attempt to split along a sharded dimension. + - Should raise ValueError during layout inference. + Expectation: Raise ValueError with expected message. + """ + init_dist() + split_size = 4 + axis = 0 # this dim is sharded + + layout = Layout((4, 2), ("dp", "tp")) + x_layout = layout("dp", "tp") + + local_tensor = torch.randn(4, 12).npu() + dist_input = DTensor.from_local(local_tensor, x_layout) + + try: + torch.split(dist_input, split_size, dim=axis) + assert False, "Expected ValueError when splitting along sharded dimension" + except ValueError as e: + assert "Can not split tensor at sharded axis" in str(e) + assert "layout:" in str(e) diff --git a/tests/torch/shard/ops/parallel_op_topk.py b/tests/torch/shard/ops/parallel_op_topk.py new file mode 100644 index 0000000000000000000000000000000000000000..1165155a144fe3c3d4a4da4c02433b9da80f56a0 --- /dev/null +++ b/tests/torch/shard/ops/parallel_op_topk.py @@ -0,0 +1,91 @@ +# Copyright 2026 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""test torch dtensor with distributed topk""" + +import numpy as np +import torch +from hyper_parallel import Layout +from tests.torch.utils import init_dist +from tests.torch.shard.utils import local_to_global, global_to_local + +# Generate input data using numpy at file header +np.random.seed(42) +standalone_input_np = np.random.randn(8, 16).astype(np.float32) + +def test_distributed_topk_layout_inference(): + """ + Feature: dtensor + torch.topk layout inference + Description: + - Test layout inference for torch.topk in distributed setting. + - Ensure that: + a) The output layouts (values, indices) match input layout. + b) Results are consistent between standalone and distributed. + Expectation: Success. + """ + init_dist() + k = 5 + dim = -1 # last dimension + + # Standalone (single-device) + standalone_input = torch.from_numpy(standalone_input_np).npu() + standalone_values, standalone_indices = torch.topk(standalone_input, k, dim=dim) + + # Distributed setup + layout = Layout((2, 4), ("dp", "tp")) + x_layout = layout("dp", "None") # shard on dim=0 (dp), keep dim=1 unsharded + + dist_input = global_to_local(standalone_input, x_layout) + dist_values, dist_indices = torch.topk(dist_input, k, dim=dim) + + # Layout correctness check + assert dist_values.layout == x_layout, "Torch TopK: output values layout mismatch input" + assert dist_indices.layout == x_layout, "Torch TopK: output indices layout mismatch input" + + # Gather distributed results back to global view + gathered_values = local_to_global(dist_values) + gathered_indices = local_to_global(dist_indices) + + assert torch.allclose( + standalone_values, gathered_values, atol=1e-5 + ), "Topk values mismatch between standalone and distributed" + + assert torch.equal( + standalone_indices, gathered_indices + ), "Topk indices mismatch between standalone and distributed" + + +def test_distributed_topk_sharded_dim_error(): + """ + Feature: dtensor + torch.topk error on sharded dim + Description: + - Attempt to perform topk along a sharded dimension. + - Should raise ValueError during layout inference. + Expectation: Raise ValueError. + """ + init_dist() + k = 3 + dim = 1 # this dim will be sharded + + layout = Layout((2, 4), ("dp", "tp")) + x_layout = layout("dp", "tp") + + standalone_input = torch.from_numpy(standalone_input_np).npu() + dist_input = global_to_local(standalone_input, x_layout) + + try: + torch.topk(dist_input, k, dim=dim) + assert False, "Expected ValueError when topk along sharded dimension" + except ValueError as e: + assert "Cannot perform sharding on params along the chosen dim" in str(e) diff --git a/tests/torch/shard/ops/test_parallel_op_layernorm.py b/tests/torch/shard/ops/test_parallel_op_layernorm.py new file mode 100644 index 0000000000000000000000000000000000000000..f0db24899294275faab508988212bc2ca4ce2e31 --- /dev/null +++ b/tests/torch/shard/ops/test_parallel_op_layernorm.py @@ -0,0 +1,43 @@ +# Copyright 2026 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""test base dtensor""" +from tests.torch.utils import torchrun_case +from tests.common.mark_utils import arg_mark + + +@arg_mark(plat_marks=["platform_ascend910b"], level_mark="level0", card_mark="allcards", essential_mark="essential") +def test_distributed_layernorm(): + ''' + Feature: test parallel op linear. + Description: test parallel op linear. + Expectation: Run success. + ''' + master_port = 10890 + file_name = "parallel_op_layernorm.py" + case_name = "test_distributed_layer_norm_layout_inference" + torchrun_case(file_name, case_name, master_port) + + +@arg_mark(plat_marks=["platform_ascend910b"], level_mark="level0", card_mark="allcards", essential_mark="essential") +def test_distributed_layernorm_dim_error(): + ''' + Feature: test parallel op linear. + Description: test parallel op linear. + Expectation: Run success. + ''' + master_port = 10890 + file_name = "parallel_op_layernorm.py" + case_name = "test_distributed_layer_norm_sharded_normalized_dim_error" + torchrun_case(file_name, case_name, master_port) diff --git a/tests/torch/shard/ops/test_parallel_op_split.py b/tests/torch/shard/ops/test_parallel_op_split.py new file mode 100644 index 0000000000000000000000000000000000000000..8f40bda64c2093a4786bce788ee668f55f71d66e --- /dev/null +++ b/tests/torch/shard/ops/test_parallel_op_split.py @@ -0,0 +1,69 @@ +# Copyright 2026 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""test base dtensor""" +from tests.torch.utils import torchrun_case +from tests.common.mark_utils import arg_mark + + +@arg_mark(plat_marks=["platform_ascend910b"], level_mark="level0", card_mark="allcards", essential_mark="essential") +def test_distributed_split_default_dim(): + ''' + Feature: test parallel op linear. + Description: test parallel op linear. + Expectation: Run success. + ''' + master_port = 10889 + file_name = "parallel_op_split.py" + case_name = "test_distributed_split_layout_inference_default_dim" + torchrun_case(file_name, case_name, master_port) + + +@arg_mark(plat_marks=["platform_ascend910b"], level_mark="level0", card_mark="allcards", essential_mark="essential") +def test_distributed_split(): + ''' + Feature: test parallel op linear. + Description: test parallel op linear. + Expectation: Run success. + ''' + master_port = 10889 + file_name = "parallel_op_split.py" + case_name = "test_distributed_split_layout_inference" + torchrun_case(file_name, case_name, master_port) + + +@arg_mark(plat_marks=["platform_ascend910b"], level_mark="level0", card_mark="allcards", essential_mark="essential") +def test_distributed_split_split_list(): + ''' + Feature: test parallel op linear. + Description: test parallel op linear. + Expectation: Run success. + ''' + master_port = 10889 + file_name = "parallel_op_split.py" + case_name = "test_distributed_split_layout_inference_split_list" + torchrun_case(file_name, case_name, master_port) + + +@arg_mark(plat_marks=["platform_ascend910b"], level_mark="level0", card_mark="allcards", essential_mark="essential") +def test_distributed_split_on_sharded_dim(): + ''' + Feature: test parallel op linear. + Description: test parallel op linear. + Expectation: Run success. + ''' + master_port = 10889 + file_name = "parallel_op_split.py" + case_name = "test_distributed_split_on_sharded_dim_error" + torchrun_case(file_name, case_name, master_port) diff --git a/tests/torch/shard/ops/test_parallel_op_topk.py b/tests/torch/shard/ops/test_parallel_op_topk.py new file mode 100644 index 0000000000000000000000000000000000000000..8240aa63ef20fa7b6bda752df57d7b6cec0d71b3 --- /dev/null +++ b/tests/torch/shard/ops/test_parallel_op_topk.py @@ -0,0 +1,43 @@ +# Copyright 2026 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""test base dtensor""" +from tests.torch.utils import torchrun_case +from tests.common.mark_utils import arg_mark + + +@arg_mark(plat_marks=["platform_ascend910b"], level_mark="level0", card_mark="allcards", essential_mark="essential") +def test_distributed_topk(): + ''' + Feature: test parallel op linear. + Description: test parallel op linear. + Expectation: Run success. + ''' + master_port = 10888 + file_name = "parallel_op_topk.py" + case_name = "test_distributed_topk_layout_inference" + torchrun_case(file_name, case_name, master_port) + + +@arg_mark(plat_marks=["platform_ascend910b"], level_mark="level0", card_mark="allcards", essential_mark="essential") +def test_distributed_topk_dim_error(): + ''' + Feature: test parallel op linear. + Description: test parallel op linear. + Expectation: Run success. + ''' + master_port = 10888 + file_name = "parallel_op_topk.py" + case_name = "test_distributed_topk_sharded_dim_error" + torchrun_case(file_name, case_name, master_port)