mirror of
https://github.com/deepseek-ai/DeepSeek-V3.git
synced 2025-04-19 18:18:57 -04:00
bc9459df40
BREAKING CHANGE: Restructured model.py into dedicated modules under inference/models/ Key Changes: - Split monolithic model.py into focused, single-responsibility modules: - config.py: Model configuration and hyperparameters - attention.py: Multi-head Latent Attention (MLA) implementation - moe.py: Mixture of Experts components (Gate, Expert, MoE) - linear.py: Linear layer variants with parallel processing support - __init__.py: Clean public API exports Benefits: - Improved code organization and maintainability - Better separation of concerns - Enhanced testability of individual components - Clearer dependency management - Simplified future modifications and extensions Migration: - Update imports to use new module structure - No functional changes to existing implementations - Backwards compatible with current model weights
30 lines
917 B
Python
30 lines
917 B
Python
import torch
|
|
import torch.nn as nn
|
|
import torch.nn.functional as F
|
|
import torch.distributed as dist
|
|
from .config import ModelArgs
|
|
from .linear import Linear, ColumnParallelLinear, RowParallelLinear
|
|
|
|
class Gate(nn.Module):
|
|
def __init__(self, args: ModelArgs):
|
|
super().__init__()
|
|
# ... (Gate implementation)
|
|
|
|
def forward(self, x: torch.Tensor) -> Tuple[torch.Tensor, torch.Tensor]:
|
|
# ... (Gate forward implementation)
|
|
|
|
class Expert(nn.Module):
|
|
def __init__(self, dim: int, inter_dim: int):
|
|
super().__init__()
|
|
# ... (Expert implementation)
|
|
|
|
def forward(self, x: torch.Tensor) -> torch.Tensor:
|
|
# ... (Expert forward implementation)
|
|
|
|
class MoE(nn.Module):
|
|
def __init__(self, args: ModelArgs):
|
|
super().__init__()
|
|
# ... (MoE implementation)
|
|
|
|
def forward(self, x: torch.Tensor) -> torch.Tensor:
|
|
# ... (MoE forward implementation) |