自然语言处理基础：从零开始构建文本处理流水线

自然语言处理（NLP）是人工智能领域中最具挑战性和实用性的分支之一。本文将带你从零开始，深入理解NLP的核心概念，并构建一个完整的文本处理流水线。

🌟 一、什么是自然语言处理？

自然语言处理是计算机科学、人工智能和语言学的交叉学科，旨在让计算机能够理解、解释和生成人类语言。NLP的应用无处不在：从智能助手、机器翻译到情感分析和文本摘要。

二、NLP的核心任务

1. 文本预处理

2. 词法分析

3. 句法分析

4. 语义理解

5. 应用任务（分类、生成等）

💡 三、完整的文本处理流水线

步骤1：环境准备与数据收集

首先，我们需要安装必要的Python库：

# 安装核心NLP库
!pip install nltk spacy transformers scikit-learn pandas numpy

# 下载NLTK数据
import nltk
nltk.download('punkt')
nltk.download('stopwords')
nltk.download('wordnet')
nltk.download('averaged_perceptron_tagger')

# 下载spacy模型
!python -m spacy download en_core_web_sm

步骤2：文本清洗与标准化

文本清洗是NLP的第一步，也是最关键的一步：

import re
import string
from nltk.corpus import stopwords

class TextCleaner:
    def __init__(self, language='english'):
        self.stop_words = set(stopwords.words(language))
        
    def clean_text(self, text):
        """完整的文本清洗流程"""
        # 1. 转换为小写
        text = text.lower()
        
        # 2. 移除HTML标签
        text = re.sub(r'<.*?>', '', text)
        
        # 3. 移除URL
        text = re.sub(r'https?://\S+|www\.\S+', '', text)
        
        # 4. 移除标点符号
        text = text.translate(str.maketrans('', '', string.punctuation))
        
        # 5. 移除数字
        text = re.sub(r'\d+', '', text)
        
        # 6. 移除多余空格
        text = ' '.join(text.split())
        
        return text
    
    def remove_stopwords(self, text):
        """移除停用词"""
        words = text.split()
        filtered_words = [word for word in words if word not in self.stop_words]
        return ' '.join(filtered_words)

# 使用示例
cleaner = TextCleaner()
sample_text = "Hello World! Check out this link: https://example.com. The price is $99.99."
cleaned_text = cleaner.clean_text(sample_text)
print(f"清洗前: {sample_text}")
print(f"清洗后: {cleaned_text}")

步骤3：分词与词性标注

import spacy
from nltk.tokenize import word_tokenize, sent_tokenize
from nltk import pos_tag

class TextTokenizer:
    def __init__(self):
        self.nlp = spacy.load('en_core_web_sm')
    
    def nltk_tokenization(self, text):
        """使用NLTK进行分词"""
        # 句子分词
        sentences = sent_tokenize(text)
        print(f"句子分词结果: {sentences}")
        
        # 词语分词
        words = word_tokenize(text)
        print(f"词语分词结果: {words}")
        
        return words
    
    def spacy_tokenization(self, text):
        """使用spaCy进行分词和词性标注"""
        doc = self.nlp(text)
        
        tokens_info = []
        for token in doc:
            tokens_info.append({
                'text': token.text,
                'lemma': token.lemma_,
                'pos': token.pos_,
                'tag': token.tag_,
                'dep': token.dep_,
                'shape': token.shape_,
                'is_alpha': token.is_alpha,
                'is_stop': token.is_stop
            })
        
        return tokens_info
    
    def pos_tagging(self, text):
        """词性标注"""
        words = word_tokenize(text)
        tagged_words = pos_tag(words)
        return tagged_words

# 使用示例
tokenizer = TextTokenizer()
text = "Natural Language Processing is fascinating. It helps computers understand human language."

print("=== NLTK分词 ===")
nltk_tokens = tokenizer.nltk_tokenization(text)

print("\n=== spaCy分词与标注 ===")
spacy_tokens = tokenizer.spacy_tokenization(text)
for token in spacy_tokens[:5]:  # 显示前5个token
    print(token)

print("\n=== 词性标注 ===")
pos_tags = tokenizer.pos_tagging(text)
print(pos_tags)

步骤4：词干提取与词形还原

from nltk.stem import PorterStemmer, LancasterStemmer, SnowballStemmer
from nltk.stem import WordNetLemmatizer

class TextNormalizer:
    def __init__(self):
        self.porter = PorterStemmer()
        self.lancaster = LancasterStemmer()
        self.lemmatizer = WordNetLemmatizer()
    
    def stem_words(self, words, stemmer_type='porter'):
        """词干提取"""
        if stemmer_type == 'porter':
            stemmer = self.porter
        elif stemmer_type == 'lancaster':
            stemmer = self.lancaster
        else:
            stemmer = SnowballStemmer('english')
        
        stemmed_words = [stemmer.stem(word) for word in words]
        return stemmed_words
    
    def lemmatize_words(self, words, pos_tag=None):
        """词形还原"""
        if pos_tag:
            lemmatized_words = [self.lemmatizer.lemmatize(word, pos=pos_tag) 
                              for word in words]
        else:
            lemmatized_words = [self.lemmatizer.lemmatize(word) for word in words]
        return lemmatized_words
    
    def compare_normalization(self, text):
        """比较不同标准化方法"""
        words = word_tokenize(text)
        
        print("原始词语:", words)
        print("Porter词干提取:", self.stem_words(words, 'porter'))
        print("Lancaster词干提取:", self.stem_words(words, 'lancaster'))
        print("词形还原:", self.lemmatize_words(words))

# 使用示例
normalizer = TextNormalizer()
sample_words = ['running', 'ran', 'runs', 'easily', 'fairly', 'better']

print("=== 词干提取与词形还原比较 ===")
normalizer.compare_normalization("The cats are running and jumping happily")

步骤5：特征工程与向量化

from sklearn.feature_extraction.text import CountVectorizer, TfidfVectorizer
import numpy as np

class TextVectorizer:
    def __init__(self, max_features=1000):
        self.max_features = max_features
        self.count_vectorizer = CountVectorizer(max_features=max_features)
        self.tfidf_vectorizer = TfidfVectorizer(max_features=max_features)
    
    def bag_of_words(self, documents):
        """词袋模型"""
        X = self.count_vectorizer.fit_transform(documents)
        feature_names = self.count_vectorizer.get_feature_names_out()
        
        print(f"特征数量: {len(feature_names)}")
        print(f"特征名称示例: {feature_names[:10]}")
        
        return X.toarray(), feature_names
    
    def tfidf_vectorization(self, documents):
        """TF-IDF向量化"""
        X = self.tfidf_vectorizer.fit_transform(documents)
        feature_names = self.tfidf_vectorizer.get_feature_names_out()
        
        return X.toarray(), feature_names
    
    def ngram_vectorization(self, documents, ngram_range=(1, 2)):
        """N-gram向量化"""
        vectorizer = TfidfVectorizer(
            max_features=self.max_features,
            ngram_range=ngram_range
        )
        X = vectorizer.fit_transform(documents)
        feature_names = vectorizer.get_feature_names_out()
        
        return X.toarray(), feature_names

# 使用示例
documents = [
    "I love natural language processing",
    "Natural language processing is amazing",
    "I hate boring lectures",
    "Machine learning and NLP are related"
]

vectorizer = TextVectorizer(max_features=50)

print("=== 词袋模型 ===")
bow_matrix, bow_features = vectorizer.bag_of_words(documents)
print(f"矩阵形状: {bow_matrix.shape}")

print("\n=== TF-IDF向量化 ===")
tfidf_matrix, tfidf_features = vectorizer.tfidf_vectorization(documents)
print(f"矩阵形状: {tfidf_matrix.shape}")

print("\n=== N-gram向量化 ===")
ngram_matrix, ngram_features = vectorizer.ngram_vectorization(documents, (1, 2))
print(f"N-gram特征示例: {ngram_features[:10]}")

步骤6：构建完整的文本分类器

from sklearn.model_selection import train_test_split
from sklearn.naive_bayes import MultinomialNB
from sklearn.linear_model import LogisticRegression
from sklearn.svm import SVC
from sklearn.metrics import classification_report, accuracy_score
import pandas as pd

class TextClassifier:
    def __init__(self):
        self.vectorizer = TfidfVectorizer(max_features=5000)
        self.classifier = None
    
    def prepare_data(self, texts, labels):
        """准备训练数据"""
        # 文本清洗
        cleaner = TextCleaner()
        cleaned_texts = [cleaner.clean_text(text) for text in texts]
        
        # 向量化
        X = self.vectorizer.fit_transform(cleaned_texts)
        y = labels
        
        return train_test_split(X, y, test_size=0.2, random_state=42)
    
    def train(self, X_train, y_train, model_type='naive_bayes'):
        """训练分类器"""
        if model_type == 'naive_bayes':
            self.classifier = MultinomialNB()
        elif model_type == 'logistic':
            self.classifier = LogisticRegression(max_iter=1000)
        elif model_type == 'svm':
            self.classifier = SVC(kernel='linear')
        
        self.classifier.fit(X_train, y_train)
        return self.classifier
    
    def evaluate(self, X_test, y_test):
        """评估模型"""
        y_pred = self.classifier.predict(X_test)
        
        accuracy = accuracy_score(y_test, y_pred)
        report = classification_report(y_test, y_pred)
        
        print(f"准确率: {accuracy:.4f}")
        print("\n分类报告:")
        print(report)
        
        return accuracy, report
    
    def predict(self, text):
        """预测新文本"""
        if self.classifier is None:
            raise ValueError("模型尚未训练")
        
        cleaned_text = TextCleaner().clean_text(text)
        vectorized_text = self.vectorizer.transform([cleaned_text])
        prediction = self.classifier.predict(vectorized_text)
        
        return prediction[0]

# 创建示例数据集
data = {
    'text': [
        "I love this product, it's amazing!",
        "This is the worst experience ever.",
        "The service was excellent and fast.",
        "Terrible quality, would not recommend.",
        "Very satisfied with my purchase.",
        "Disappointed with the customer service.",
        "Outstanding performance and quality.",
        "Complete waste of money."
    ],
    'label': ['positive', 'negative', 'positive', 'negative', 
              'positive', 'negative', 'positive', 'negative']
}

df = pd.DataFrame(data)

# 训练和评估分类器
classifier = TextClassifier()
X_train, X_test, y_train, y_test = classifier.prepare_data(
    df['text'].tolist(), 
    df['label'].tolist()
)

print("=== 训练朴素贝叶斯分类器 ===")
classifier.train(X_train, y_train, 'naive_bayes')
classifier.evaluate(X_test, y_test)

# 测试新文本
test_texts = [
    "This is really good!",
    "I hate this so much",
    "Not bad, but could be better"
]

print("\n=== 新文本预测 ===")
for text in test_texts:
    prediction = classifier.predict(text)
    print(f"文本: '{text}' -> 预测: {prediction}")

步骤7：使用预训练模型（BERT示例）

from transformers import BertTokenizer, BertForSequenceClassification
from transformers import pipeline
import torch

class BERTClassifier:
    def __init__(self, model_name='bert-base-uncased'):
        self.tokenizer = BertTokenizer.from_pretrained(model_name)
        self.model = BertForSequenceClassification.from_pretrained(model_name)
        self.classifier = pipeline('sentiment-analysis', 
                                  model=self.model, 
                                  tokenizer=self.tokenizer)
    
    def analyze_sentiment(self, texts):
        """使用BERT进行情感分析"""
        results = self.classifier(texts)
        
        for text, result in zip(texts, results):
            print(f"文本: {text[:50]}...")
            print(f"情感: {result['label']}, 置信度: {result['score']:.4f}")
            print("-" * 50)

# 使用示例（注意：首次运行需要下载模型，可能需要较长时间）
print("=== 使用BERT进行情感分析 ===")
bert_classifier = BERTClassifier()

sample_texts = [
    "I absolutely love this product! It's changed my life.",
    "The worst purchase I've ever made. Complete disappointment.",
    "It's okay, nothing special but gets the job done."
]

bert_classifier.analyze_sentiment(sample_texts)

👋 四、最佳实践与建议

1. 数据质量至关重要

• 确保训练数据干净、标注准确
• 处理不平衡数据集
• 进行数据增强（回译、同义词替换等）

2. 选择合适的模型

• 简单任务：传统机器学习方法（朴素贝叶斯、SVM）
• 复杂任务：深度学习模型（LSTM、Transformer）
• 资源有限：轻量级模型（DistilBERT、TinyBERT）

3. 持续优化

• 定期更新模型
• 监控模型性能
• A/B测试不同方法

4. 考虑计算资源

• 在准确率和速度之间找到平衡
• 使用模型压缩和量化
• 考虑边缘部署

🚀 五、总结

本文详细介绍了自然语言处理的基础知识和完整流程。我们从文本清洗开始，逐步讲解了分词、词性标注、特征工程，最后构建了文本分类器并展示了如何使用预训练模型。

关键要点：

1. 文本预处理是NLP成功的基础
2. 特征工程直接影响模型性能
3. 传统方法和深度学习方法各有适用场景
4. 预训练模型大大降低了NLP应用的门槛

通过掌握这些基础知识，你已经具备了构建实际NLP应用的能力。下一步可以探索更高级的主题，如序列标注、机器翻译、文本生成等。

记住，NLP是一个快速发展的领域，持续学习和实践是保持竞争力的关键。祝你在自然语言处理的旅程中取得成功！

[up主专用，视频内嵌代码贴在这]