AI生成对抗网络（GAN）完全教程
1. GAN原理与训练机制
生成对抗网络（Generative Adversarial Network）由Ian Goodfellow于2014年提出，其核心思想源自博弈论中的二人零和博弈——生成器（Generator）和判别器（Discriminator）在对抗中共同进化。
生成器G：从随机噪声z中学习数据分布，生成逼真样本。 判别器D：区分真实数据与生成数据。
两者的目标函数构成极小极大博弈：
min_G max_D V(D,G) = E[log D(x)] + E[log(1 - D(G(z)))]
训练流程（交替优化）：
- 固定G，训练D：让D更好地分辨真假
- 固定D，训练G：让G生成更逼真的样本欺骗D
- 重复直到纳什均衡
import torch
import torch.nn as nn
import torch.optim as optim
from torch.utils.data import DataLoader
from torchvision import datasets, transforms
from torchvision.utils import save_image
class Generator(nn.Module):
"""基础生成器：噪声向量 → 图像"""
def __init__(self, latent_dim=100, img_channels=1, img_size=28):
super().__init__()
self.img_size = img_size
self.net = nn.Sequential(
nn.Linear(latent_dim, 256),
nn.BatchNorm1d(256),
nn.LeakyReLU(0.2),
nn.Linear(256, 512),
nn.BatchNorm1d(512),
nn.LeakyReLU(0.2),
nn.Linear(512, 1024),
nn.BatchNorm1d(1024),
nn.LeakyReLU(0.2),
nn.Linear(1024, img_channels * img_size * img_size),
nn.Tanh()
)
def forward(self, z):
return self.net(z).view(-1, 1, self.img_size, self.img_size)
class Discriminator(nn.Module):
"""基础判别器：图像 → 真假概率"""
def __init__(self, img_channels=1, img_size=28):
super().__init__()
self.net = nn.Sequential(
nn.Linear(img_channels * img_size * img_size, 512),
nn.LeakyReLU(0.2),
nn.Dropout(0.3),
nn.Linear(512, 256),
nn.LeakyReLU(0.2),
nn.Dropout(0.3),
nn.Linear(256, 1),
nn.Sigmoid()
)
def forward(self, img):
return self.net(img.view(img.size(0), -1))
def train_gan(epochs=50, batch_size=64, latent_dim=100, lr=0.0002):
"""标准GAN训练循环"""
device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
# 数据加载（以MNIST为例）
transform = transforms.Compose([
transforms.ToTensor(),
transforms.Normalize([0.5], [0.5])
])
dataset = datasets.MNIST('./data', train=True, download=True, transform=transform)
dataloader = DataLoader(dataset, batch_size=batch_size, shuffle=True)
G = Generator(latent_dim).to(device)
D = Discriminator().to(device)
criterion = nn.BCELoss()
opt_G = optim.Adam(G.parameters(), lr=lr, betas=(0.5, 0.999))
opt_D = optim.Adam(D.parameters(), lr=lr, betas=(0.5, 0.999))
for epoch in range(epochs):
for i, (real_imgs, _) in enumerate(dataloader):
real_imgs = real_imgs.to(device)
bs = real_imgs.size(0)
real_label = torch.ones(bs, 1, device=device)
fake_label = torch.zeros(bs, 1, device=device)
# ===== 训练判别器 =====
z = torch.randn(bs, latent_dim, device=device)
fake_imgs = G(z)
d_real = D(real_imgs)
d_fake = D(fake_imgs.detach())
d_loss = criterion(d_real, real_label) + criterion(d_fake, fake_label)
opt_D.zero_grad()
d_loss.backward()
opt_D.step()
# ===== 训练生成器 =====
d_fake = D(fake_imgs)
g_loss = criterion(d_fake, real_label) # G希望D认为fake是real
opt_G.zero_grad()
g_loss.backward()
opt_G.step()
print(f"Epoch [{epoch+1}/{epochs}] D_loss: {d_loss.item():.4f} "
f"G_loss: {g_loss.item():.4f}")
# 每个epoch保存样本
save_image(fake_imgs[:25], f'generated_epoch_{epoch+1:03d}.png',
nrow=5, normalize=True)
2. 经典架构演进
2.1 DCGAN（Deep Convolutional GAN）
DCGAN确立了卷积GAN的设计准则：用转置卷积替代全连接层，用BatchNorm稳定训练。
class DCDiscriminator(nn.Module):
"""DCGAN判别器"""
def __init__(self, img_channels=3, ndf=64):
super().__init__()
self.net = nn.Sequential(
# 输入: (C, 64, 64)
nn.Conv2d(img_channels, ndf, 4, 2, 1, bias=False),
nn.LeakyReLU(0.2, inplace=True),
# (ndf, 32, 32)
nn.Conv2d(ndf, ndf * 2, 4, 2, 1, bias=False),
nn.BatchNorm2d(ndf * 2),
nn.LeakyReLU(0.2, inplace=True),
# (ndf*2, 16, 16)
nn.Conv2d(ndf * 2, ndf * 4, 4, 2, 1, bias=False),
nn.BatchNorm2d(ndf * 4),
nn.LeakyReLU(0.2, inplace=True),
# (ndf*4, 8, 8)
nn.Conv2d(ndf * 4, ndf * 8, 4, 2, 1, bias=False),
nn.BatchNorm2d(ndf * 8),
nn.LeakyReLU(0.2, inplace=True),
# (ndf*8, 4, 4)
nn.Conv2d(ndf * 8, 1, 4, 1, 0, bias=False),
nn.Sigmoid()
)
def forward(self, x):
return self.net(x).view(-1, 1)
class DCGenerator(nn.Module):
"""DCGAN生成器"""
def __init__(self, latent_dim=100, img_channels=3, ngf=64):
super().__init__()
self.net = nn.Sequential(
# 输入: (latent_dim, 1, 1)
nn.ConvTranspose2d(latent_dim, ngf * 8, 4, 1, 0, bias=False),
nn.BatchNorm2d(ngf * 8),
nn.ReLU(True),
# (ngf*8, 4, 4)
nn.ConvTranspose2d(ngf * 8, ngf * 4, 4, 2, 1, bias=False),
nn.BatchNorm2d(ngf * 4),
nn.ReLU(True),
# (ngf*4, 8, 8)
nn.ConvTranspose2d(ngf * 4, ngf * 2, 4, 2, 1, bias=False),
nn.BatchNorm2d(ngf * 2),
nn.ReLU(True),
# (ngf*2, 16, 16)
nn.ConvTranspose2d(ngf * 2, ngf, 4, 2, 1, bias=False),
nn.BatchNorm2d(ngf),
nn.ReLU(True),
# (ngf, 32, 32)
nn.ConvTranspose2d(ngf, img_channels, 4, 2, 1, bias=False),
nn.Tanh()
# (C, 64, 64)
)
def forward(self, z):
return self.net(z.view(z.size(0), -1, 1, 1))
2.2 WGAN（Wasserstein GAN）
WGAN用Wasserstein距离替代JS散度，解决了原始GAN训练不稳定和模式崩溃问题。
class WGANGenerator(nn.Module):
"""WGAN生成器（与DCGAN结构类似，无Sigmoid输出）"""
def __init__(self, latent_dim=100, img_channels=3, ngf=64):
super().__init__()
self.net = nn.Sequential(
nn.ConvTranspose2d(latent_dim, ngf * 8, 4, 1, 0, bias=False),
nn.BatchNorm2d(ngf * 8), nn.ReLU(True),
nn.ConvTranspose2d(ngf * 8, ngf * 4, 4, 2, 1, bias=False),
nn.BatchNorm2d(ngf * 4), nn.ReLU(True),
nn.ConvTranspose2d(ngf * 4, ngf * 2, 4, 2, 1, bias=False),
nn.BatchNorm2d(ngf * 2), nn.ReLU(True),
nn.ConvTranspose2d(ngf * 2, ngf, 4, 2, 1, bias=False),
nn.BatchNorm2d(ngf), nn.ReLU(True),
nn.ConvTranspose2d(ngf, img_channels, 4, 2, 1, bias=False),
nn.Tanh()
)
def forward(self, z):
return self.net(z.view(z.size(0), -1, 1, 1))
class WGANCritic(nn.Module):
"""WGAN判别器（Critic）：无Sigmoid，输出无界实数"""
def __init__(self, img_channels=3, ndf=64):
super().__init__()
self.net = nn.Sequential(
nn.Conv2d(img_channels, ndf, 4, 2, 1),
nn.LeakyReLU(0.2, inplace=True),
nn.Conv2d(ndf, ndf * 2, 4, 2, 1, bias=False),
nn.InstanceNorm2d(ndf * 2, affine=True), # WGAN-GP用InstanceNorm
nn.LeakyReLU(0.2, inplace=True),
nn.Conv2d(ndf * 2, ndf * 4, 4, 2, 1, bias=False),
nn.InstanceNorm2d(ndf * 4, affine=True),
nn.LeakyReLU(0.2, inplace=True),
nn.Conv2d(ndf * 4, ndf * 8, 4, 2, 1, bias=False),
nn.InstanceNorm2d(ndf * 8, affine=True),
nn.LeakyReLU(0.2, inplace=True),
nn.Conv2d(ndf * 8, 1, 4, 1, 0),
# 无Sigmoid — WGAN输出无界
)
def forward(self, x):
return self.net(x).view(-1, 1)
def compute_gradient_penalty(critic, real_imgs, fake_imgs, device, lambda_gp=10):
"""WGAN-GP梯度惩罚"""
alpha = torch.rand(real_imgs.size(0), 1, 1, 1, device=device)
interpolated = (alpha * real_imgs + (1 - alpha) * fake_imgs).requires_grad_(True)
d_interpolated = critic(interpolated)
gradients = torch.autograd.grad(
outputs=d_interpolated,
inputs=interpolated,
grad_outputs=torch.ones_like(d_interpolated),
create_graph=True,
retain_graph=True
)[0]
gradients = gradients.view(gradients.size(0), -1)
penalty = ((gradients.norm(2, dim=1) - 1) ** 2).mean()
return lambda_gp * penalty
def train_wgan_gp(epochs=100, n_critic=5, lambda_gp=10):
"""WGAN-GP训练循环"""
device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
G = WGANGenerator().to(device)
C = WGANCritic().to(device)
opt_G = optim.Adam(G.parameters(), lr=1e-4, betas=(0.0, 0.9))
opt_C = optim.Adam(C.parameters(), lr=1e-4, betas=(0.0, 0.9))
for epoch in range(epochs):
for i, (real_imgs, _) in enumerate(dataloader):
real_imgs = real_imgs.to(device)
bs = real_imgs.size(0)
# ===== 训练Critic（多次） =====
z = torch.randn(bs, 100, device=device)
fake_imgs = G(z).detach()
c_real = C(real_imgs).mean()
c_fake = C(fake_imgs).mean()
gp = compute_gradient_penalty(C, real_imgs, fake_imgs, device, lambda_gp)
c_loss = c_fake - c_real + gp # Wasserstein距离 + 梯度惩罚
opt_C.zero_grad()
c_loss.backward()
opt_C.step()
# 每n_critic次更新一次生成器
if i % n_critic == 0:
z = torch.randn(bs, 100, device=device)
fake_imgs = G(z)
g_loss = -C(fake_imgs).mean()
opt_G.zero_grad()
g_loss.backward()
opt_G.step()
print(f"Epoch [{epoch+1}/{epochs}] C_loss: {c_loss.item():.4f} "
f"G_loss: {g_loss.item():.4f}")
2.3 StyleGAN
StyleGAN引入风格映射网络和自适应实例归一化（AdaIN），实现分层风格控制。
class MappingNetwork(nn.Module):
"""StyleGAN映射网络：z → w（中间潜在空间）"""
def __init__(self, latent_dim=512, w_dim=512, num_layers=8):
super().__init__()
layers = []
for i in range(num_layers):
layers.extend([
nn.Linear(latent_dim if i == 0 else w_dim, w_dim),
nn.LeakyReLU(0.2)
])
self.net = nn.Sequential(*layers)
def forward(self, z):
# z先做像素归一化
z = z / (z.norm(dim=1, keepdim=True) + 1e-8)
return self.net(z)
class AdaIN(nn.Module):
"""自适应实例归一化"""
def __init__(self, channels, w_dim=512):
super().__init__()
self.norm = nn.InstanceNorm2d(channels, affine=False)
self.style_scale = nn.Linear(w_dim, channels)
self.style_bias = nn.Linear(w_dim, channels)
def forward(self, x, w):
normalized = self.norm(x)
scale = self.style_scale(w).view(x.size(0), -1, 1, 1)
bias = self.style_bias(w).view(x.size(0), -1, 1, 1)
return normalized * (1 + scale) + bias
class StyleBlock(nn.Module):
"""StyleGAN生成器的一个块"""
def __init__(self, in_channels, out_channels, w_dim=512):
super().__init__()
self.conv = nn.Conv2d(in_channels, out_channels, 3, 1, 1)
self.adain = AdaIN(out_channels, w_dim)
self.noise_scale = nn.Parameter(torch.zeros(1))
self.act = nn.LeakyReLU(0.2)
def forward(self, x, w, noise=None):
x = self.conv(x)
if noise is not None:
x = x + self.noise_scale * noise
x = self.adain(x, w)
return self.act(x)
3. 条件生成与可控生成
3.1 条件GAN（cGAN）
class ConditionalGenerator(nn.Module):
"""条件生成器：同时接受噪声和类别标签"""
def __init__(self, latent_dim=100, num_classes=10, embed_dim=50):
super().__init__()
self.label_embed = nn.Embedding(num_classes, embed_dim)
self.net = nn.Sequential(
nn.Linear(latent_dim + embed_dim, 256),
nn.BatchNorm1d(256),
nn.ReLU(),
nn.Linear(256, 512),
nn.BatchNorm1d(512),
nn.ReLU(),
nn.Linear(512, 784),
nn.Tanh()
)
def forward(self, z, label):
label_vec = self.label_embed(label)
input_vec = torch.cat([z, label_vec], dim=1)
return self.net(input_vec).view(-1, 1, 28, 28)
class ConditionalDiscriminator(nn.Module):
"""条件判别器：同时接受图像和类别标签"""
def __init__(self, num_classes=10, embed_dim=50):
super().__init__()
self.label_embed = nn.Embedding(num_classes, embed_dim)
self.net = nn.Sequential(
nn.Linear(784 + embed_dim, 512),
nn.LeakyReLU(0.2),
nn.Dropout(0.3),
nn.Linear(512, 256),
nn.LeakyReLU(0.2),
nn.Dropout(0.3),
nn.Linear(256, 1),
nn.Sigmoid()
)
def forward(self, img, label):
label_vec = self.label_embed(label)
img_flat = img.view(img.size(0), -1)
input_vec = torch.cat([img_flat, label_vec], dim=1)
return self.net(input_vec)
3.2 InfoGAN — 无监督可控生成
InfoGAN通过最大化潜在编码与生成结果的互信息，实现无监督的属性解耦：
class InfoGANDiscriminator(nn.Module):
"""InfoGAN判别器 + Q网络（推断潜在编码）"""
def __init__(self, img_channels=1, num_disc_codes=10, num_cont_codes=2):
super().__init__()
self.features = nn.Sequential(
nn.Conv2d(img_channels, 64, 4, 2, 1),
nn.LeakyReLU(0.2),
nn.Conv2d(64, 128, 4, 2, 1, bias=False),
nn.BatchNorm2d(128),
nn.LeakyReLU(0.2),
)
# D head: 真假判断
self.d_head = nn.Sequential(
nn.Linear(128 * 7 * 7, 1),
nn.Sigmoid()
)
# Q head: 推断潜在编码（用于InfoGAN的互信息最大化）
self.q_head = nn.Sequential(
nn.Linear(128 * 7 * 7, 128),
nn.LeakyReLU(0.2),
)
self.q_disc = nn.Linear(128, num_disc_codes) # 离散编码
self.q_cont_mu = nn.Linear(128, num_cont_codes) # 连续编码均值
self.q_cont_var = nn.Linear(128, num_cont_codes) # 连续编码方差
def forward(self, x):
feat = self.features(x).view(x.size(0), -1)
d_out = self.d_head(feat)
q_feat = self.q_head(feat)
q_disc = self.q_disc(q_feat)
q_mu = self.q_cont_mu(q_feat)
q_var = torch.exp(self.q_cont_var(q_feat))
return d_out, q_disc, q_mu, q_var
4. 图像超分辨率GAN
4.1 SRGAN
class SRResidualBlock(nn.Module):
"""SRGAN残差块"""
def __init__(self, channels=64):
super().__init__()
self.block = nn.Sequential(
nn.Conv2d(channels, channels, 3, 1, 1, bias=False),
nn.BatchNorm2d(channels),
nn.PReLU(),
nn.Conv2d(channels, channels, 3, 1, 1, bias=False),
nn.BatchNorm2d(channels),
)
def forward(self, x):
return x + self.block(x)
class SRGenerator(nn.Module):
"""SRGAN生成器：低分辨率 → 高分辨率（4x放大）"""
def __init__(self, scale_factor=4, num_blocks=16):
super().__init__()
self.pre_layer = nn.Sequential(
nn.Conv2d(3, 64, 9, 1, 4),
nn.PReLU()
)
blocks = [SRResidualBlock(64) for _ in range(num_blocks)]
blocks.extend([
nn.Conv2d(64, 64, 3, 1, 1, bias=False),
nn.BatchNorm2d(64),
])
self.residuals = nn.Sequential(*blocks)
# 上采样层
upsample = []
for _ in range(scale_factor // 2):
upsample.extend([
nn.Conv2d(64, 256, 3, 1, 1),
nn.PixelShuffle(2), # 亚像素卷积上采样
nn.PReLU(),
])
self.upsample = nn.Sequential(*upsample)
self.final = nn.Conv2d(64, 3, 9, 1, 4)
def forward(self, x):
pre = self.pre_layer(x)
res = self.residuals(pre)
up = self.upsample(pre + res)
return torch.tanh(self.final(up))
class SRDiscriminator(nn.Module):
"""SRGAN判别器"""
def __init__(self):
super().__init__()
def block(in_c, out_c, stride):
return nn.Sequential(
nn.Conv2d(in_c, out_c, 3, stride, 1, bias=False),
nn.BatchNorm2d(out_c),
nn.LeakyReLU(0.2, inplace=True)
)
self.net = nn.Sequential(
nn.Conv2d(3, 64, 3, 1, 1),
nn.LeakyReLU(0.2),
block(64, 64, 2),
block(64, 128, 1),
block(128, 128, 2),
block(128, 256, 1),
block(256, 256, 2),
block(256, 512, 1),
block(512, 512, 2),
nn.AdaptiveAvgPool2d(1),
nn.Flatten(),
nn.Linear(512, 1024),
nn.LeakyReLU(0.2),
nn.Linear(1024, 1),
nn.Sigmoid()
)
def forward(self, x):
return self.net(x)
class PerceptualLoss(nn.Module):
"""感知损失：基于预训练VGG的特征匹配"""
def __init__(self):
super().__init__()
vgg = models.vgg19(pretrained=True).features[:36]
self.vgg = nn.Sequential(*list(vgg.children())).eval()
for param in self.vgg.parameters():
param.requires_grad = False
self.mse = nn.MSELoss()
def forward(self, sr_img, hr_img):
sr_features = self.vgg(sr_img)
hr_features = self.vgg(hr_img)
return self.mse(sr_features, hr_features)
5. 图像翻译
5.1 Pix2Pix（配对图像翻译）
class UNetDown(nn.Module):
def __init__(self, in_c, out_c, normalize=True):
super().__init__()
layers = [nn.Conv2d(in_c, out_c, 4, 2, 1, bias=False)]
if normalize:
layers.append(nn.BatchNorm2d(out_c))
layers.append(nn.LeakyReLU(0.2))
self.model = nn.Sequential(*layers)
def forward(self, x):
return self.model(x)
class UNetUp(nn.Module):
def __init__(self, in_c, out_c, dropout=False):
super().__init__()
layers = [
nn.ConvTranspose2d(in_c, out_c, 4, 2, 1, bias=False),
nn.BatchNorm2d(out_c),
nn.ReLU()
]
if dropout:
layers.append(nn.Dropout(0.5))
self.model = nn.Sequential(*layers)
def forward(self, x, skip):
x = self.model(x)
return torch.cat([x, skip], dim=1)
class Pix2PixGenerator(nn.Module):
"""U-Net结构的Pix2Pix生成器"""
def __init__(self, in_channels=3, out_channels=3):
super().__init__()
# Encoder
self.down1 = UNetDown(in_channels, 64, normalize=False)
self.down2 = UNetDown(64, 128)
self.down3 = UNetDown(128, 256)
self.down4 = UNetDown(256, 512)
self.down5 = UNetDown(512, 512)
self.down6 = UNetDown(512, 512)
self.down7 = UNetDown(512, 512)
self.down8 = UNetDown(512, 512, normalize=False)
# Decoder（带skip connection）
self.up1 = UNetUp(512, 512, dropout=True)
self.up2 = UNetUp(1024, 512, dropout=True)
self.up3 = UNetUp(1024, 512, dropout=True)
self.up4 = UNetUp(1024, 512)
self.up5 = UNetUp(1024, 256)
self.up6 = UNetUp(512, 128)
self.up7 = UNetUp(256, 64)
self.final = nn.Sequential(
nn.ConvTranspose2d(128, out_channels, 4, 2, 1),
nn.Tanh()
)
def forward(self, x):
d1 = self.down1(x)
d2 = self.down2(d1)
d3 = self.down3(d2)
d4 = self.down4(d3)
d5 = self.down5(d4)
d6 = self.down6(d5)
d7 = self.down7(d6)
d8 = self.down8(d7)
u1 = self.up1(d8, d7)
u2 = self.up2(u1, d6)
u3 = self.up3(u2, d5)
u4 = self.up4(u3, d4)
u5 = self.up5(u4, d3)
u6 = self.up6(u5, d2)
u7 = self.up7(u6, d1)
return self.final(u7)
class PatchDiscriminator(nn.Module):
"""PatchGAN判别器：对图像局部区域判别"""
def __init__(self, in_channels=6):
super().__init__()
self.net = nn.Sequential(
nn.Conv2d(in_channels, 64, 4, 2, 1),
nn.LeakyReLU(0.2),
nn.Conv2d(64, 128, 4, 2, 1, bias=False),
nn.BatchNorm2d(128),
nn.LeakyReLU(0.2),
nn.Conv2d(128, 256, 4, 2, 1, bias=False),
nn.BatchNorm2d(256),
nn.LeakyReLU(0.2),
nn.Conv2d(256, 512, 4, 1, 1, bias=False),
nn.BatchNorm2d(512),
nn.LeakyReLU(0.2),
nn.Conv2d(512, 1, 4, 1, 1),
)
def forward(self, x):
return self.net(x)
5.2 CycleGAN（无配对图像翻译）
CycleGAN引入循环一致性损失，无需配对数据即可实现域间翻译：
class CycleGAN:
"""CycleGAN训练框架"""
def __init__(self, G_A2B, G_B2A, D_A, D_B, lambda_cycle=10):
self.G_A2B = G_A2B # A域 → B域
self.G_B2A = G_B2A # B域 → A域
self.D_A = D_A # A域判别器
self.D_B = D_B # B域判别器
self.lambda_cycle = lambda_cycle
def compute_cycle_loss(self, real_a, real_b):
"""循环一致性损失：A → B → A ≈ A"""
fake_b = self.G_A2B(real_a)
recovered_a = self.G_B2A(fake_b)
fake_a = self.G_B2A(real_b)
recovered_b = self.G_A2B(fake_a)
cycle_loss_a = nn.functional.l1_loss(recovered_a, real_a)
cycle_loss_b = nn.functional.l1_loss(recovered_b, real_b)
return cycle_loss_a + cycle_loss_b, fake_a, fake_b
def compute_identity_loss(self, real_a, real_b):
"""身份损失：保持域内样本不变"""
same_b = self.G_A2B(real_b)
same_a = self.G_B2A(real_a)
identity_loss = (
nn.functional.l1_loss(same_b, real_b) +
nn.functional.l1_loss(same_a, real_a)
)
return identity_loss
6. 视频生成GAN
视频生成GAN在图像生成基础上增加时序一致性约束。
class TemporalDiscriminator(nn.Module):
"""时序判别器：检测帧间时序连贯性"""
def __init__(self, in_channels=3, num_frames=16):
super().__init__()
self.num_frames = num_frames
# 3D卷积处理时空特征
self.net = nn.Sequential(
nn.Conv3d(in_channels, 64, (3, 4, 4), (1, 2, 2), (1, 1, 1)),
nn.LeakyReLU(0.2),
nn.Conv3d(64, 128, (3, 4, 4), (1, 2, 2), (1, 1, 1), bias=False),
nn.BatchNorm3d(128),
nn.LeakyReLU(0.2),
nn.Conv3d(128, 256, (3, 4, 4), (1, 2, 2), (1, 1, 1), bias=False),
nn.BatchNorm3d(256),
nn.LeakyReLU(0.2),
nn.Conv3d(256, 1, (3, 4, 4), (1, 2, 2), (0, 0, 0)),
)
def forward(self, video):
"""video: (B, C, T, H, W)"""
return self.net(video).mean()
class VideoGenerator(nn.Module):
"""视频生成器：噪声 + 随机帧 → 视频序列"""
def __init__(self, latent_dim=512, num_frames=16, img_size=64):
super().__init__()
self.num_frames = num_frames
# 空间生成（每帧）
self.spatial_gen = DCGenerator(latent_dim, 3, 64)
# 时序运动网络
self.motion_net = nn.Sequential(
nn.Linear(latent_dim, 256),
nn.ReLU(),
nn.Linear(256, num_frames * 64), # 每帧的运动编码
)
# 光流预测
self.flow_predictor = nn.Sequential(
nn.Conv2d(64 + 3, 64, 3, 1, 1),
nn.ReLU(),
nn.Conv2d(64, 2, 3, 1, 1), # 2通道：x,y方向光流
)
def forward(self, z):
B = z.size(0)
# 生成关键帧
key_frame = self.spatial_gen(z)
# 生成运动序列
motion_codes = self.motion_net(z).view(B, self.num_frames, 64)
frames = [key_frame]
for t in range(1, self.num_frames):
# 基于运动编码预测光流
motion = motion_codes[:, t].view(B, 64, 1, 1).expand(-1, -1, 64, 64)
flow_input = torch.cat([key_frame, motion], dim=1)
flow = self.flow_predictor(flow_input)
# 光流变形生成新帧
warped = self._apply_flow(frames[-1], flow)
frames.append(warped)
return torch.stack(frames, dim=2) # (B, C, T, H, W)
def _apply_flow(self, img, flow):
"""用光流对图像进行变形"""
B, C, H, W = img.shape
grid_y, grid_x = torch.meshgrid(
torch.linspace(-1, 1, H, device=img.device),
torch.linspace(-1, 1, W, device=img.device),
indexing='ij'
)
grid = torch.stack([grid_x, grid_y], dim=0).unsqueeze(0).expand(B, -1, -1, -1)
grid = grid + flow * 0.1 # 缩放光流
return nn.functional.grid_sample(img, grid.permute(0, 2, 3, 1),
align_corners=True)
7. 3D-aware GAN
3D-aware GAN（如pi-GAN、EG3D）从2D图像学习3D结构，实现多视角一致的生成。
class NeuralRadianceField(nn.Module):
"""简化的NeRF渲染器 — 用于3D-aware GAN"""
def __init__(self, latent_dim=512, hidden_dim=256):
super().__init__()
# 位置编码
self.pos_encoding_dim = 63 # L=10 for position
self.dir_encoding_dim = 27 # L=4 for direction
# MLP网络
input_dim = self.pos_encoding_dim + latent_dim
self.sigma_net = nn.Sequential(
nn.Linear(input_dim, hidden_dim),
nn.ReLU(),
nn.Linear(hidden_dim, hidden_dim),
nn.ReLU(),
nn.Linear(hidden_dim, hidden_dim),
nn.ReLU(),
nn.Linear(hidden_dim, hidden_dim + 1), # hidden + sigma
)
self.color_net = nn.Sequential(
nn.Linear(hidden_dim + self.dir_encoding_dim + latent_dim, 128),
nn.ReLU(),
nn.Linear(128, 3),
nn.Sigmoid()
)
def positional_encoding(self, x, L):
"""位置编码：将低维坐标映射到高维"""
encodings = [x]
for i in range(L):
encodings.append(torch.sin(2 ** i * x))
encodings.append(torch.cos(2 ** i * x))
return torch.cat(encodings, dim=-1)
def forward(self, positions, directions, w):
"""
positions: (B, N, 3) 采样点3D坐标
directions: (B, N, 3) 观察方向
w: (B, latent_dim) StyleGAN的w向量
"""
pos_enc = self.positional_encoding(positions, 10)
dir_enc = self.positional_encoding(directions, 4)
B, N, _ = positions.shape
w_expand = w.unsqueeze(1).expand(-1, N, -1)
# 密度预测
h_sigma = torch.cat([pos_enc, w_expand], dim=-1)
h = self.sigma_net(h_sigma)
sigma = torch.relu(h[..., -1:])
feature = h[..., :-1]
# 颜色预测
h_color = torch.cat([feature, dir_enc, w_expand], dim=-1)
color = self.color_net(h_color)
return color, sigma
8. GAN训练技巧与稳定性
class GANTrainer:
"""GAN训练实用技巧集合"""
@staticmethod
def label_smoothing(real_label=0.9, fake_label=0.1):
"""标签平滑：防止判别器过于自信"""
return real_label, fake_label
@staticmethod
def instance_noise(images, epoch, max_epochs, noise_std=0.1):
"""实例噪声：训练初期添加噪声帮助D学习，逐步衰减"""
decay = 1 - epoch / max_epochs
noise = torch.randn_like(images) * noise_std * decay
return images + noise
@staticmethod
def spectral_norm_layers(model):
"""谱归一化：稳定判别器训练"""
for name, module in model.named_modules():
if isinstance(module, (nn.Conv2d, nn.Linear)):
nn.utils.spectral_norm(module)
return model
@staticmethod
def two_timescale_update_rule(G, D, g_lr=1e-4, d_lr=4e-4):
"""双时间尺度更新规则：D学习率高于G"""
opt_G = optim.Adam(G.parameters(), lr=g_lr, betas=(0.0, 0.999))
opt_D = optim.Adam(D.parameters(), lr=d_lr, betas=(0.0, 0.999))
return opt_G, opt_D
@staticmethod
def exponential_moving_average(model, ema_model, decay=0.999):
"""EMA：平滑生成器参数，提升生成质量"""
with torch.no_grad():
for param, ema_param in zip(model.parameters(),
ema_model.parameters()):
ema_param.data.mul_(decay).add_(param.data, alpha=1 - decay)
@staticmethod
def gradient_penalty_check(d_loss, threshold=1000):
"""梯度监控：检测训练崩溃"""
if d_loss > threshold:
print("[WARNING] 判别器损失异常高，可能存在训练不稳定")
return True
return False
常见训练问题与对策：
| 问题 | 现象 | 解决方案 |
|---|---|---|
| 模式崩溃 | G只生成少数几种样本 | 使用WGAN-GP、增加多样性正则 |
| 训练震荡 | D_loss和G_loss剧烈波动 | 降低学习率、使用TTUR |
| 梯度消失 | G_loss下降后D_loss趋近0 | 使用Wasserstein距离、谱归一化 |
| 生成质量差 | 图像模糊或有伪影 | 增加模型容量、使用渐进式训练 |
9. GAN vs Diffusion对比
| 维度 | GAN | Diffusion Model |
|---|---|---|
| 生成速度 | 快（单次前向传播） | 慢（需多步去噪） |
| 训练稳定性 | 较差（对抗训练难收敛） | 较好（简单的去噪损失） |
| 模式多样性 | 易模式崩溃 | 天然多样性好 |
| 图像质量 | 高分辨率已成熟 | 当前SOTA质量 |
| 可控性 | 通过latent space控制 | 通过条件引导控制 |
| 理论基础 | 博弈论（纳什均衡） | 扩散过程（随机微分方程） |
| 典型应用 | 实时生成、视频、编辑 | 文生图、高保真合成 |
# GAN生成 — 一次前向传播
with torch.no_grad():
z = torch.randn(1, 512, device=device)
image = gan_generator(z) # ~10ms
# Diffusion生成 — 多步迭代去噪
x = torch.randn(1, 3, 256, 256, device=device)
for t in range(1000, 0, -1): # 1000步去噪
t_tensor = torch.tensor([t], device=device)
noise_pred = unet(x, t_tensor)
x = scheduler.step(noise_pred, t_tensor, x) # ~10s
GAN在需要实时生成的场景（游戏、视频流）仍有不可替代的优势；Diffusion在质量和多样性上更胜一筹。两者融合（如GAN蒸馏加速Diffusion）是当前趋势。
10. 实战案例：人脸生成与编辑
import torch
from torchvision import transforms
class FaceGANPipeline:
"""完整的人脸生成与编辑流水线"""
def __init__(self, stylegan_weights, device='cuda'):
self.device = torch.device(device)
self.G = self._load_stylegan(stylegan_weights)
self.G.eval()
# 平均w向量（用于风格混合）
self.w_mean = self._compute_w_mean(num_samples=10000)
def _load_stylegan(self, weights_path):
"""加载预训练的StyleGAN"""
G = StyleGANGenerator(z_dim=512, w_dim=512, img_resolution=1024)
G.load_state_dict(torch.load(weights_path, map_location=self.device))
return G.to(self.device).eval()
def _compute_w_mean(self, num_samples=10000):
"""计算W空间的平均向量（用于编辑基准）"""
w_list = []
with torch.no_grad():
for _ in range(num_samples // 64):
z = torch.randn(64, 512, device=self.device)
w = self.G.mapping(z)
w_list.append(w[:, 0, :]) # 取第一层的w
return torch.cat(w_list, dim=0).mean(dim=0, keepdim=True)
@torch.no_grad()
def generate_random(self, num_images=1, truncation_psi=0.7):
"""随机人脸生成（truncation控制质量-多样性权衡）"""
z = torch.randn(num_images, 512, device=self.device)
w = self.G.mapping(z)
# 截断技巧：向均值靠拢，提升质量但降低多样性
w_truncated = self.w_mean + truncation_psi * (w - self.w_mean)
return self.G.synthesis(w_truncated)
@torch.no_grad()
def edit_attribute(self, source_z, direction, alpha=1.0):
"""属性编辑：沿语义方向移动"""
w = self.G.mapping(source_z)
# direction: 预训练的属性方向向量（如"微笑"、"年龄"）
# 通常通过在latent space中训练线性分类器获得
w_edited = w + alpha * direction.view(1, 1, -1)
return self.G.synthesis(w_edited)
@torch.no_grad()
def style_mix(self, z1, z2, crossover_point=7):
"""风格混合：前半部分用z1的风格，后半部分用z2"""
w1 = self.G.mapping(z1)
w2 = self.G.mapping(z2)
# StyleGAN有18层，前8层控制粗糙特征（姿态、脸型）
# 后10层控制细节（肤色、纹理）
w_mixed = torch.cat([
w1[:, :crossover_point, :],
w2[:, crossover_point:, :]
], dim=1)
return self.G.synthesis(w_mixed)
@torch.no_grad()
def image_inversion(self, target_image, num_steps=1000, lr=0.01):
"""图像反演：找到真实图像对应的潜在编码"""
z = torch.randn(1, 512, device=self.device, requires_grad=True)
optimizer = torch.optim.Adam([z], lr=lr)
target = transforms.Resize((1024, 1024))(target_image).unsqueeze(0)
target = target.to(self.device)
for step in range(num_steps):
w = self.G.mapping(z)
generated = self.G.synthesis(w)
# 多尺度重建损失
loss = nn.functional.mse_loss(generated, target)
loss += nn.functional.mse_loss(
nn.functional.avg_pool2d(generated, 2),
nn.functional.avg_pool2d(target, 2)
)
optimizer.zero_grad()
loss.backward()
optimizer.step()
if step % 100 == 0:
print(f"Step {step}, Loss: {loss.item():.6f}")
return self.G.mapping(z.detach())
# === 使用示例 ===
# pipeline = FaceGANPipeline('stylegan2_ffhq_1024.pt')
#
# 生成随机人脸
# face = pipeline.generate_random(num_images=1, truncation_psi=0.7)
#
# 添加微笑（假设已有微笑方向向量）
# smile_face = pipeline.edit_attribute(z, smile_direction, alpha=2.0)
#
# 风格混合
# mixed = pipeline.style_mix(z1, z2, crossover_point=7)
11. 评估指标
11.1 FID（Fréchet Inception Distance）
FID衡量生成图像分布与真实图像分布的距离，值越低越好。
import numpy as np
from scipy import linalg
from torchvision.models import inception_v3
import torch
def calculate_fid(real_features, gen_features):
"""计算Fréchet Inception Distance"""
# 假设特征已通过InceptionV3提取
mu1, sigma1 = real_features.mean(0), np.cov(real_features, rowvar=False)
mu2, sigma2 = gen_features.mean(0), np.cov(gen_features, rowvar=False)
diff = mu1 - mu2
# 计算矩阵平方根
covmean, _ = linalg.sqrtm(sigma1 @ sigma2, disp=False)
if np.iscomplexobj(covmean):
covmean = covmean.real
fid = diff @ diff + np.trace(sigma1 + sigma2 - 2 * covmean)
return float(fid)
def extract_inception_features(images, batch_size=64, device='cuda'):
"""使用InceptionV3提取特征"""
inception = inception_v3(pretrained=True, transform_input=False)
inception.fc = nn.Identity() # 移除分类头，输出2048维特征
inception = inception.to(device).eval()
features = []
with torch.no_grad():
for i in range(0, len(images), batch_size):
batch = images[i:i+batch_size].to(device)
# InceptionV3需要299x299输入
batch = nn.functional.interpolate(batch, size=(299, 299),
mode='bilinear')
feat = inception(batch)
features.append(feat.cpu().numpy())
return np.concatenate(features, axis=0)
# FID评估流程
# real_features = extract_inception_features(real_images)
# gen_features = extract_inception_features(generated_images)
# fid_score = calculate_fid(real_features, gen_features)
# print(f"FID Score: {fid_score:.2f}") # 越低越好，通常 < 10 为优秀
11.2 IS（Inception Score）
def calculate_inception_score(generated_images, num_splits=10,
batch_size=64, device='cuda'):
"""计算Inception Score"""
inception = inception_v3(pretrained=True).to(device).eval()
preds = []
with torch.no_grad():
for i in range(0, len(generated_images), batch_size):
batch = generated_images[i:i+batch_size].to(device)
batch = nn.functional.interpolate(batch, size=(299, 299),
mode='bilinear')
pred = torch.softmax(inception(batch), dim=1)
preds.append(pred.cpu().numpy())
preds = np.concatenate(preds, axis=0)
# IS = exp(E[KL(p(y|x) || p(y))])
scores = []
for i in range(num_splits):
part = preds[i * len(preds) // num_splits:
(i + 1) * len(preds) // num_splits]
py = part.mean(axis=0) # p(y) 边际分布
kl = part * (np.log(part + 1e-10) - np.log(py + 1e-10))
kl_mean = kl.sum(axis=1).mean()
scores.append(np.exp(kl_mean))
return np.mean(scores), np.std(scores)
# IS越高越好，表示生成图像质量高且多样
# IS的局限：只关注类别分布，不与真实数据比较
11.3 LPIPS（Learned Perceptual Image Patch Similarity）
class LPIPS(nn.Module):
"""简化版LPIPS — 基于VGG特征的感知距离"""
def __init__(self):
super().__init__()
vgg = models.vgg16(pretrained=True).features
# 提取多个层的特征
self.slice1 = nn.Sequential(*list(vgg[:4])) # conv1_2
self.slice2 = nn.Sequential(*list(vgg[4:9])) # conv2_2
self.slice3 = nn.Sequential(*list(vgg[9:16])) # conv3_3
self.slice4 = nn.Sequential(*list(vgg[16:23])) # conv4_3
# 各层权重（可学习）
self.weights = nn.Parameter(torch.tensor([1.0, 1.0, 1.0, 1.0]))
for param in self.parameters():
param.requires_grad = False
def forward(self, x, y):
"""计算两幅图像间的感知距离"""
x_features = self._extract_features(x)
y_features = self._extract_features(y)
distance = 0
for i, (xf, yf) in enumerate(zip(x_features, y_features)):
# L2归一化后计算距离
xf = xf / (xf.norm(dim=1, keepdim=True) + 1e-10)
yf = yf / (yf.norm(dim=1, keepdim=True) + 1e-10)
distance += self.weights[i] * ((xf - yf) ** 2).mean(dim=(2, 3))
return distance
def _extract_features(self, x):
h1 = self.slice1(x)
h2 = self.slice2(h1)
h3 = self.slice3(h2)
h4 = self.slice4(h3)
return [h1, h2, h3, h4]
# 使用
# lpips = LPIPS()
# perceptual_distance = lpips(image_a, image_b)
# 值越小表示两幅图像在感知上越相似
指标选择指南
| 指标 | 衡量什么 | 适用场景 | 局限性 |
|---|---|---|---|
| FID | 分布相似度 | 整体质量评估 | 需要大量样本 |
| IS | 质量+多样性 | 类条件生成 | 不与真实数据对比 |
| LPIPS | 感知相似度 | 图像翻译/重建评估 | 不衡量生成质量 |
| PSNR/SSIM | 像素级相似度 | 超分辨率评估 | 与人类感知不完全一致 |
总结
从2014年的原始GAN到如今的3D-aware GAN和视频生成，GAN技术经历了爆发式发展。虽然Diffusion Model在文生图领域取得了突破，GAN凭借其高速生成能力和成熟的架构设计，在实时应用、视频处理和图像编辑等领域依然不可替代。
核心要点：
- 理解对抗训练的博弈本质是掌握GAN的基础
- WGAN-GP解决了训练稳定性问题，是实践中的可靠选择
- StyleGAN系列在人脸生成领域达到了照片级真实感
- CycleGAN证明了无配对数据也能实现高质量域间翻译
- 实际项目中，FID是最常用的评估指标，配合LPIPS做感知质量评估
- GAN与Diffusion的融合是未来方向，取长补短方为上策
