飞桨框架 3.0 核心升级：动静统一自动并行，轻松开发大模型_AI阅读总结 — 包阅AI

• ProcessMesh是指用于大模型训练或推理的多个硬件设备拓扑结构。我们将一个设备（比如一块 GPU 卡）映射为一个进程，将多个设备映射为多个进程组成的一维或多维数组，下图展示了由4个设备构成的2*2 ProcessMesh 抽象表示。
• Placements是指表示张量在不同设备上的切分状态，分为 Replicate、Shard 和 Partial 这3种切分状态。如下图所示，Replicate 表示张量在不同设备上会以复制的形式存在；Shard 表示按照特定的维度在不同设备上进行切分；Partial 表示设备上的张量不完整，需要进行 Reduce Sum 或者 Reduce Mean 等不同方式的操作后，才能得到完整的状态。

import paddleimport paddle.distributed as distmesh = dist.ProcessMesh([[2, 4, 5], [0, 1, 3]], dim_names=['x', 'y'])dense_tensor = paddle.to_tensor([[1,2,3],                                 [4,5,6],                                 [7,8,9],                                 [10,11,12]])placements = [dist.Shard(0), dist.Shard(1)]dist_tensor = dist.shard_tensor(dense_tensor, mesh, placements)

import paddleimport paddle.distributed as distmesh0 = dist.ProcessMesh([0, 1], dim_names=['x'])mesh1 = dist.ProcessMesh([2, 3], dim_names=['x'])dense_tensor = paddle.to_tensor([[1,2,3],                                 [4,5,6]])placements0 = [dist.Replicate()]placements1 = [dist.Shard(0)]dist_tensor = dist.shard_tensor(dense_tensor, mesh0, placements0)dist_tensor_after_reshard = dist.reshard(dist_tensor, mesh1, placements1)

import paddleimport paddle.distributed as dist
mesh = dist.ProcessMesh([0, 1], dim_names=['x'])dense_tensorA = paddle.to_tensor([[1,2,], [3,4]])dense_tensorB = paddle.to_tensor([[5,6], [7,8]])placementsA = [dist.Replicate()]placementsB = [dist.Shard(0)]
dist_tensorA = dist.shard_tensor(dense_tensorA, mesh, placementsA)dist_tensorB = dist.shard_tensor(dense_tensorB, mesh, placementsB)dist_tensorC = Matmul(dist_tensorA, dist_tensorB)dist_tensorD = relu(dist_tensorC)

... dist_model = dist.to_static(    model, dataloader, paddle.mean, opt)dist_model.train()for step, inputs in enumerate(dataloader()):    data = inputs    loss = dist_model(data)    print(step, loss)

import paddleimport paddle.distributed as distfrom paddle.io import BatchSampler, DataLoader, Datasetimport numpy as npmesh0 = dist.ProcessMesh([[0, 1], [2, 3]], dim_names=['x', 'y']) mesh1 = dist.ProcessMesh([[4, 5], [6, 7]], dim_names=['x', 'y']) class RandomDataset(Dataset):    def __init__(self, seq_len, hidden, num_samples=100):        super().__init__()        self.seq_len = seq_len        self.hidden = hidden        self.num_samples = num_samples    def __getitem__(self, index):        input = np.random.uniform(size=[self.seq_len, self.hidden]).astype("float32")        label = np.random.uniform(size=[self.seq_len, self.hidden]).astype("float32")        return input, label    def __len__(self):        return self.num_samplesclass MlpModel(paddle.nn.Layer):    def __init__(self):        super(MlpModel, self).__init__()        self.w0 = dist.shard_tensor(                    self.create_parameter(shape=[1024, 4096]),                    mesh0, [dist.Replicate(), dist.Shard(1)])          self.w1 = dist.shard_tensor(                    self.create_parameter(shape=[4096, 1024]),                    mesh1, [dist.Replicate(), dist.Shard(0)])      def forward(self, x):        y = paddle.matmul(x, self.w0)        y = dist.reshard(y, mesh1, [dist.Shard(0), dist.Shard(2)])          z = paddle.matmul(y, self.w1)        return zmodel = MlpModel()dataset = RandomDataset(128, 1024)sampler = BatchSampler(    dataset,    batch_size=2,)dataloader = DataLoader(    dataset,    batch_sampler=sampler,)dataloader = dist.shard_dataloader(dataloader, meshes=[mesh0, mesh1], shard_dims='x')opt = paddle.optimizer.AdamW(learning_rate=0.001, parameters=model.parameters())opt = dist.shard_optimizer(opt)for step, inputs in enumerate(dataloader()):    data = inputs[0]    logits = model(data)    loss = paddle.mean(logits)    loss.backward()    opt.step()    opt.clear_grad()

import paddleimport paddle.distributed as diststrategy = dist.Strategy()
strategy.fused_passes.enable = Truestrategy.fused_passes.gemm_epilogue = True

from paddle.distributed as diststrategy = dist.Strategy()pipeline = strategy.pipelinepipeline.enable = Truepipeline.schedule_mode = "1F1B"  pipeline.vpp_degree = 2