第10章：自然语言处理（Natural Language Processing）
一、导读
1.1 本章学习目标
第10章介绍自然语言处理（NLP）——让计算机理解和生成人类语言的技术。语言是人类交流的核心工具，让机器理解语言是AI长期以来的挑战。近年来，深度学习和大语言模型彻底改变了NLP的面貌。
本章的核心学习目标包括：
- 理解NLP的任务层次：从分词、词性标注到句法分析、语义理解。
- 掌握语言模型：N-gram、神经网络语言模型、预训练语言模型。
- 理解文本表示：词袋模型、TF-IDF、词向量（Word2Vec、GloVe）。
- 掌握主要NLP任务：文本分类、命名实体识别、机器翻译、问答系统。
- 了解大语言模型：GPT、BERT 等预训练模型的原理和应用。
- 理解NLP的挑战：歧义、上下文依赖、常识推理。
1.2 为什么第十章如此重要
自然语言处理是AI最成功的应用领域之一。从搜索引擎到智能助手，从机器翻译到文本生成，NLP技术已经深入到日常生活的方方面面。特别是 ChatGPT、GPT-4 等大语言模型的出现，展示了AI在语言理解和生成方面的惊人能力。
理解NLP的基本原理，不仅有助于使用这些工具，也有助于理解当前AI技术的能力边界。本章将从基础的文本处理开始，逐步深入到现代的深度学习方法。
1.3 核心问题引导
在阅读本章时，请带着以下问题思考：
语言的歧义性：为什么"我看见那个人用望远镜"有歧义？计算机如何处理这种歧义？
词向量的魔力：为什么 Word2Vec 能捕捉词义关系？"国王-男人+女人=女王"是怎么做到的？
注意力机制的革命：Transformer 为什么比 RNN 更适合处理语言？
预训练模型的威力：为什么 BERT 和 GPT 能在这么多任务上表现优异？
大语言模型的局限：ChatGPT 会"幻觉"、会犯错，它的根本局限是什么？
二、核心概念详解
2.1 文本预处理
2.1.1 分词与清洗
python
import re
from collections import Counter
class TextPreprocessor:
def __init__(self, language='en'):
self.language = language
def tokenize(self, text):
if self.language == 'en':
text = text.lower()
tokens = re.findall(r'\b\w+\b', text)
return tokens
else:
return list(text)
def remove_stopwords(self, tokens, stopwords=None):
if stopwords is None:
stopwords = {'the', 'a', 'an', 'is', 'are', 'was', 'were', 'in', 'on', 'at', 'to', 'for'}
return [t for t in tokens if t not in stopwords]
def stem(self, word):
suffixes = ['ing', 'ed', 'ly', 'tion', 's', 'es']
for suffix in suffixes:
if word.endswith(suffix) and len(word) - len(suffix) > 3:
return word[:-len(suffix)]
return word
def preprocess(self, text):
tokens = self.tokenize(text)
tokens = self.remove_stopwords(tokens)
tokens = [self.stem(t) for t in tokens]
return tokens
preprocessor = TextPreprocessor()
text = "The quick brown foxes are jumping over the lazy dogs"
tokens = preprocessor.preprocess(text)
print(f"预处理结果: {tokens}")2.1.2 文本表示
词袋模型（Bag of Words）：
python
class BagOfWords:
def __init__(self):
self.vocabulary = {}
def fit(self, documents):
words = set()
for doc in documents:
words.update(doc)
self.vocabulary = {word: i for i, word in enumerate(sorted(words))}
def transform(self, documents):
import numpy as np
result = np.zeros((len(documents), len(self.vocabulary)))
for i, doc in enumerate(documents):
for word in doc:
if word in self.vocabulary:
result[i, self.vocabulary[word]] += 1
return result
def fit_transform(self, documents):
self.fit(documents)
return self.transform(documents)
docs = [
['hello', 'world'],
['hello', 'ai'],
['world', 'peace'],
]
bow = BagOfWords()
matrix = bow.fit_transform(docs)
print(f"词袋矩阵:\n{matrix}")
print(f"词汇表: {bow.vocabulary}")TF-IDF：
python
import numpy as np
class TFIDF:
def __init__(self):
self.vocabulary = {}
self.idf = {}
def fit(self, documents):
words = set()
for doc in documents:
words.update(doc)
self.vocabulary = {word: i for i, word in enumerate(sorted(words))}
n_docs = len(documents)
for word in self.vocabulary:
doc_count = sum(1 for doc in documents if word in doc)
self.idf[word] = np.log(n_docs / (1 + doc_count)) + 1
def transform(self, documents):
result = np.zeros((len(documents), len(self.vocabulary)))
for i, doc in enumerate(documents):
word_counts = Counter(doc)
total = len(doc)
for word, count in word_counts.items():
if word in self.vocabulary:
tf = count / total
result[i, self.vocabulary[word]] = tf * self.idf[word]
return result
tfidf = TFIDF()
matrix = tfidf.fit_transform(docs)
print(f"TF-IDF 矩阵:\n{matrix}")2.2 语言模型
2.2.1 N-gram 语言模型
python
class NGramLM:
def __init__(self, n=2):
self.n = n
self.ngram_counts = {}
self.context_counts = {}
def fit(self, text):
tokens = text if isinstance(text, list) else text.split()
for i in range(len(tokens) - self.n + 1):
ngram = tuple(tokens[i:i+self.n])
context = ngram[:-1]
word = ngram[-1]
self.ngram_counts[ngram] = self.ngram_counts.get(ngram, 0) + 1
self.context_counts[context] = self.context_counts.get(context, 0) + 1
def probability(self, word, context):
context = tuple(context)
ngram = context + (word,)
if context not in self.context_counts or self.context_counts[context] == 0:
return 1e-6
return self.ngram_counts.get(ngram, 0) / self.context_counts[context]
def generate(self, length=10, start=None):
if start is None:
start = list(self.context_counts.keys())[0]
else:
start = tuple(start)
result = list(start)
for _ in range(length):
context = tuple(result[-self.n+1:])
candidates = {k[-1]: v for k, v in self.ngram_counts.items()
if k[:-1] == context}
if not candidates:
break
total = sum(candidates.values())
probs = {k: v/total for k, v in candidates.items()}
next_word = max(probs, key=probs.get)
result.append(next_word)
return result
text = "the cat sat on the mat the cat liked the mat"
lm = NGramLM(n=2)
lm.fit(text)
generated = lm.generate(length=8, start=('the',))
print(f"生成文本: {' '.join(generated)}")2.3 词向量
2.3.1 Word2Vec 原理
python
class SimpleWord2Vec:
def __init__(self, embedding_dim=10, window=2):
self.embedding_dim = embedding_dim
self.window = window
self.vocabulary = {}
self.embeddings = {}
def build_vocab(self, sentences):
words = set()
for sent in sentences:
words.update(sent)
self.vocabulary = {word: i for i, word in enumerate(sorted(words))}
np.random.seed(42)
for word in self.vocabulary:
self.embeddings[word] = np.random.randn(self.embedding_dim) * 0.01
def train(self, sentences, epochs=100, lr=0.01):
for epoch in range(epochs):
for sent in sentences:
for i, word in enumerate(sent):
context = []
for j in range(max(0, i-self.window), min(len(sent), i+self.window+1)):
if i != j:
context.append(sent[j])
for ctx_word in context:
self._update(word, ctx_word, lr)
def _update(self, target, context, lr):
target_vec = self.embeddings[target]
context_vec = self.embeddings[context]
score = np.dot(target_vec, context_vec)
pred = 1 / (1 + np.exp(-score))
error = 1 - pred
grad = error * context_vec
self.embeddings[target] += lr * grad
self.embeddings[context] += lr * error * target_vec
def similarity(self, word1, word2):
v1 = self.embeddings[word1]
v2 = self.embeddings[word2]
return np.dot(v1, v2) / (np.linalg.norm(v1) * np.linalg.norm(v2))
sentences = [
['king', 'queen', 'royal', 'palace'],
['man', 'woman', 'person', 'human'],
['king', 'man', 'crown', 'throne'],
['queen', 'woman', 'crown', 'throne'],
]
w2v = SimpleWord2Vec(embedding_dim=5)
w2v.build_vocab(sentences)
w2v.train(sentences, epochs=500)
print(f"king-queen 相似度: {w2v.similarity('king', 'queen'):.4f}")2.4 文本分类
2.4.1 朴素贝叶斯文本分类
python
class NaiveBayesTextClassifier:
def __init__(self):
self.class_priors = {}
self.word_probs = {}
self.vocabulary = set()
def fit(self, texts, labels):
from collections import Counter, defaultdict
class_counts = Counter(labels)
total = len(labels)
word_counts = defaultdict(lambda: Counter())
for text, label in zip(texts, labels):
self.vocabulary.update(text)
word_counts[label].update(text)
for label in class_counts:
self.class_priors[label] = class_counts[label] / total
self.word_probs[label] = {}
total_words = sum(word_counts[label].values())
vocab_size = len(self.vocabulary)
for word in self.vocabulary:
count = word_counts[label].get(word, 0)
self.word_probs[label][word] = (count + 1) / (total_words + vocab_size)
def predict(self, text):
scores = {}
for label in self.class_priors:
score = np.log(self.class_priors[label])
for word in text:
prob = self.word_probs[label].get(word, 1e-6)
score += np.log(prob)
scores[label] = score
return max(scores, key=scores.get)
# 情感分类
texts = [
['good', 'great', 'love', 'excellent'],
['bad', 'terrible', 'hate', 'awful'],
['good', 'nice', 'wonderful'],
['bad', 'poor', 'horrible'],
['excellent', 'amazing', 'love'],
['terrible', 'awful', 'hate'],
]
labels = ['positive', 'negative', 'positive', 'negative', 'positive', 'negative']
classifier = NaiveBayesTextClassifier()
classifier.fit(texts, labels)
print(f"分类 'good love': {classifier.predict(['good', 'love'])}")
print(f"分类 'bad terrible': {classifier.predict(['bad', 'terrible'])}")2.5 序列标注
2.5.1 命名实体识别
python
class SimpleNER:
def __init__(self):
self.word_tag_probs = {}
self.tag_transitions = {}
def fit(self, sentences):
from collections import defaultdict, Counter
word_tags = defaultdict(Counter)
tag_trans = defaultdict(Counter)
for tokens, tags in sentences:
prev_tag = 'START'
for token, tag in zip(tokens, tags):
word_tags[token][tag] += 1
tag_trans[prev_tag][tag] += 1
prev_tag = tag
for word, tags in word_tags.items():
total = sum(tags.values())
self.word_tag_probs[word] = {t: c/total for t, c in tags.items()}
for prev_tag, tags in tag_trans.items():
total = sum(tags.values())
self.tag_transitions[prev_tag] = {t: c/total for t, c in tags.items()}
def predict(self, tokens):
tags = []
prev_tag = 'START'
for token in tokens:
if token in self.word_tag_probs:
best_tag = max(self.word_tag_probs[token],
key=self.word_tag_probs[token].get)
else:
best_tag = 'O'
tags.append(best_tag)
prev_tag = best_tag
return tags
# 简单的 NER 训练数据
train_data = [
(['John', 'lives', 'in', 'New', 'York'], ['B-PER', 'O', 'O', 'B-LOC', 'I-LOC']),
(['Mary', 'works', 'at', 'Google'], ['B-PER', 'O', 'O', 'B-ORG']),
(['Beijing', 'is', 'the', 'capital', 'of', 'China'], ['B-LOC', 'O', 'O', 'O', 'O', 'B-LOC']),
]
ner = SimpleNER()
ner.fit(train_data)
result = ner.predict(['John', 'works', 'at', 'Google', 'in', 'Beijing'])
print(f"NER 结果: {result}")2.6 大语言模型
2.6.1 Transformer 架构
python
class TransformerBlock:
def __init__(self, d_model, n_heads, d_ff):
self.attention = SelfAttention(d_model, n_heads)
self.norm1 = lambda x: x
self.norm2 = lambda x: x
self.ff = FFN(d_model, d_ff)
def forward(self, x):
attn_out = self.attention.forward(x)
x = self.norm1(x + attn_out)
ff_out = self.ff.forward(x)
x = self.norm2(x + ff_out)
return x
class FFN:
def __init__(self, d_model, d_ff):
self.W1 = np.random.randn(d_model, d_ff) * 0.01
self.W2 = np.random.randn(d_ff, d_model) * 0.01
def forward(self, x):
h = np.maximum(0, x @ self.W1)
return h @ self.W2
# Transformer 示例
block = TransformerBlock(d_model=8, n_heads=2, d_ff=16)
X = np.random.randn(4, 8)
output = block.forward(X)
print(f"Transformer 输出形状: {output.shape}")三、实战案例
3.1 案例一：简单的聊天机器人
python
class SimpleChatbot:
def __init__(self):
self.patterns = {
'greeting': (['hello', 'hi', 'hey'], 'Hello! How can I help you?'),
'farewell': (['bye', 'goodbye', 'see you'], 'Goodbye! Have a nice day!'),
'thanks': (['thank', 'thanks'], 'You are welcome!'),
'help': (['help', 'assist'], 'I can answer simple questions.'),
}
def respond(self, text):
text = text.lower()
tokens = text.split()
for intent, (keywords, response) in self.patterns.items():
if any(kw in tokens for kw in keywords):
return response
return "I'm not sure I understand. Can you rephrase?"
bot = SimpleChatbot()
print(bot.respond("Hello there"))
print(bot.respond("Can you help me"))
print(bot.respond("Thank you"))
print(bot.respond("What is AI"))四、本章小结
本章介绍了自然语言处理的核心概念和方法。从基础的文本预处理和表示，到语言模型和词向量，再到文本分类和序列标注，我们逐步构建了NLP的知识体系。最后介绍了Transformer和大语言模型的基本原理。
核心知识点回顾：
- NLP任务层次：分词→词性标注→句法分析→语义理解
- 文本表示：词袋、TF-IDF、词向量
- N-gram语言模型是最简单的统计语言模型
- Word2Vec 通过上下文学习词向量
- 朴素贝叶斯是经典的文本分类方法
- Transformer 的自注意力机制是现代NLP的基础
- 大语言模型通过预训练+微调实现多任务学习
五、思考与练习
思考题
词向量的局限：Word2Vec 为每个词生成固定的向量，这有什么局限？上下文相关的词表示（如 BERT）如何解决这个问题？
大语言模型的能力边界：ChatGPT 能写代码、做翻译、回答问题，但它会"幻觉"、会犯错。它的根本局限是什么？
编程练习
实现 TF-IDF 文本分类：用 TF-IDF 和逻辑回归实现新闻分类。
实现简单的机器翻译：用 Seq2Seq 模型实现英法翻译。
微调预训练模型：使用 HuggingFace Transformers 微调 BERT 进行情感分析。