预训练阶段：
  大规模无标注语料 → 自监督学习 → 通用语言模型

微调阶段：
  通用语言模型 + 小规模标注数据 → 任务特定模型

# MLM 示例
# 原始句子: "I love natural language processing"
# 遮盖后:   "I love [MASK] language [MASK]"
# 目标:     预测 "natural" 和 "processing"

from transformers import BertForSequenceClassification, BertTokenizer
import torch

# 加载预训练模型（用于分类任务）
model = BertForSequenceClassification.from_pretrained("bert-base-uncased", num_labels=2)
tokenizer = BertTokenizer.from_pretrained("bert-base-uncased")

# 准备输入
text = "This movie is absolutely wonderful!"
inputs = tokenizer(text, return_tensors="pt", padding=True, truncation=True, max_length=128)

# 前向传播
outputs = model(**inputs)
logits = outputs.logits
prediction = torch.argmax(logits, dim=-1)
print(f"Prediction: {prediction.item()}")  # 0 或 1

# GPT-3 风格的少样本提示
prompt = """
Classify the sentiment as positive or negative.

Text: I love this product!
Sentiment: positive

Text: This is terrible.
Sentiment: negative

Text: The quality is amazing.
Sentiment:"""

# GPT-3 会生成 "positive"

from transformers import TrainingArguments, Trainer

training_args = TrainingArguments(
    output_dir="./results",
    num_train_epochs=3,
    per_device_train_batch_size=16,
    learning_rate=2e-5,
    weight_decay=0.01,
)

trainer = Trainer(
    model=model,
    args=training_args,
    train_dataset=train_dataset,
    eval_dataset=eval_dataset,
)

trainer.train()

from peft import LoraConfig, get_peft_model

# 配置 LoRA
lora_config = LoraConfig(
    r=8,           # 低秩维度
    lora_alpha=32, # 缩放因子
    target_modules=["query", "value"],  # 应用 LoRA 的模块
    lora_dropout=0.1,
    task_type="SEQ_CLS",
)

# 获取 LoRA 模型
model = get_peft_model(base_model, lora_config)
model.print_trainable_parameters()
# 可训练参数: 0.1% of total

from transformers import BertTokenizer, BertForSequenceClassification
from transformers import Trainer, TrainingArguments
from torch.utils.data import Dataset
import torch

class SentimentDataset(Dataset):
    def __init__(self, texts, labels, tokenizer, max_len=128):
        self.texts = texts
        self.labels = labels
        self.tokenizer = tokenizer
        self.max_len = max_len

    def __len__(self):
        return len(self.texts)

    def __getitem__(self, idx):
        encoding = self.tokenizer(
            self.texts[idx],
            max_length=self.max_len,
            padding="max_length",
            truncation=True,
            return_tensors="pt",
        )
        return {
            "input_ids": encoding["input_ids"].squeeze(),
            "attention_mask": encoding["attention_mask"].squeeze(),
            "labels": torch.tensor(self.labels[idx], dtype=torch.long),
        }

# 数据
texts = ["这部电影太好看了", "非常失望，浪费钱", "剧情精彩，推荐", "无聊至极"]
labels = [1, 0, 1, 0]

# 模型
tokenizer = BertTokenizer.from_pretrained("bert-base-chinese")
model = BertForSequenceClassification.from_pretrained("bert-base-chinese", num_labels=2)

# 训练
dataset = SentimentDataset(texts, labels, tokenizer)
training_args = TrainingArguments(
    output_dir="./results",
    num_train_epochs=3,
    per_device_train_batch_size=2,
    learning_rate=2e-5,
)
trainer = Trainer(model=model, args=training_args, train_dataset=dataset)
trainer.train()

# 推理
test_text = "这个产品真的很棒"
inputs = tokenizer(test_text, return_tensors="pt", max_length=128, truncation=True, padding="max_length")
with torch.no_grad():
    outputs = model(**inputs)
    pred = torch.argmax(outputs.logits, dim=-1)
    print(f"'{test_text}' → {'正面' if pred.item() == 1 else '负面'}")

from transformers import GPT2LMHeadModel, GPT2Tokenizer

tokenizer = GPT2Tokenizer.from_pretrained("gpt2")
model = GPT2LMHeadModel.from_pretrained("gpt2")

prompt = "Natural language processing is"
inputs = tokenizer(prompt, return_tensors="pt")

# 生成文本
outputs = model.generate(
    **inputs,
    max_new_tokens=50,
    temperature=0.7,
    top_k=50,
    top_p=0.95,
    do_sample=True,
    no_repeat_ngram_size=2,
)

generated_text = tokenizer.decode(outputs[0], skip_special_tokens=True)
print(generated_text)

from transformers import AutoModelForSequenceClassification, AutoTokenizer
from peft import LoraConfig, get_peft_model, TaskType

# 基础模型
model_name = "bert-base-uncased"
tokenizer = AutoTokenizer.from_pretrained(model_name)
base_model = AutoModelForSequenceClassification.from_pretrained(model_name, num_labels=2)

# LoRA 配置
lora_config = LoraConfig(
    task_type=TaskType.SEQ_CLS,
    r=16,
    lora_alpha=32,
    target_modules=["query", "value"],
    lora_dropout=0.1,
)

# 获取 LoRA 模型
model = get_peft_model(base_model, lora_config)
print(f"Trainable parameters: {model.num_parameters(only_trainable=True) / 1e6:.2f}M")
print(f"Total parameters: {model.num_parameters() / 1e6:.2f}M")