【NLP】学习笔记，进行中。。。。

nltk.download('brown')  
# 上面可能网络原因失败，可以手动下载：
# https://github.com/nltk/nltk_data/tree/gh-pages
# 然后把 packages 文件夹下的所有内容复制到指定目录即可
# 为了使用其它功能，还需要：
# 解压缩 /nltk_data/tokenizers/punkt.zip

from nltk.corpus import brown

brown.categories()  # 语料库类别
brown.fileids()  # 语料库文件
brown.categories(fileids)  # 根据文件返回类别
brown.fileids(categories)  # 根据类别返回文件



brown.raw()  #  返回语料, str 类型
brown.words()  # 语料中的单词
brown.sents()  # 语料的句子
# raw，words，sents 可以指定 fileids=[f1, f2, f3]，categories=[c1, c2]。也可以用 index 来索引，例如：
brown.words(categories='news')[10:30]   



brown.abspath(fileid='ca01')  # 文件路径

brown.readme()  # readme

import nltk

text = "How long does it take to get a Ph.D. degree? I don't know."
nltk.tokenize.sent_tokenize(text)
# 另一个可训练的分句工具
tokenizer = nltk.tokenize.punkt.PunktSentenceTokenizer()

import nltk

text = 'Hello! Everyone!! Good morning!'
print(nltk.tokenize.word_tokenize(text=text))  # 最通用的方法
print(text.split())  # 朴素的方法
print(nltk.tokenize.regexp_tokenize(text=text, pattern='\w+'))  # 允许用户自定义
print(nltk.tokenize.wordpunct_tokenize(text=text))
print(nltk.tokenize.blankline_tokenize(text=text))

import nltk

pst = nltk.stem.PorterStemmer()
lst = nltk.stem.LancasterStemmer()
sbt = nltk.stem.SnowballStemmer(language='english')  # 可以支持多种语言

pst.stem('eating')
lst.stem('shopping')
sbt.stem('asked')

from nltk.corpus import stopwords

stop_words = stopwords.words('english')  # 支持几十种语言
stop_words = set(["hi", "bye"])  #  也可以自定义停用词

# 然后就可以移除了
tokens = [word for word in text.split() if word not in stopwords]

freq_dist = nltk.FreqDist(tokens)  # 类似 Counter 的词频统计
freq_dist1 = sorted(freq_dist.items(), key=lambda x: x[1], reverse=True)
rare_words = [i[0] for i in freq_dist1[-10:]]
# 然后就可以移除了
tokens = [word for word in text.split() if word not in rare_words]

import re
from collections import Counter

# WORDS 一般可以从语料中获取，这里为了展示代码临时写成这样
WORDS = Counter(['apple', 'correction', 'statement', 'tutors'])


def P(word, N=sum(WORDS.values())):
    "Probability of `word`."
    return WORDS[word] / N


def correction(word):
    "Most probable spelling correction for word."
    return max(candidates(word), key=P)


def candidates(word):
    "Generate possible spelling corrections for word."
    return (known([word]) or known(edits1(word)) or known(edits2(word)) or [word])


def known(words):
    "The subset of `words` that appear in the dictionary of WORDS."
    return set(w for w in words if w in WORDS)


def edits1(word):
    "All edits that are one edit away from `word`."
    letters = 'abcdefghijklmnopqrstuvwxyz'
    splits = [(word[:i], word[i:]) for i in range(len(word) + 1)]
    deletes = [L + R[1:] for L, R in splits if R]
    transposes = [L + R[1] + R[0] + R[2:] for L, R in splits if len(R) > 1]
    replaces = [L + c + R[1:] for L, R in splits if R for c in letters]
    inserts = [L + c + R for L, R in splits for c in letters]
    return set(deletes + transposes + replaces + inserts)


def edits2(word):
    "All edits that are two edits away from `word`."
    return (e2 for e1 in edits1(word) for e2 in edits1(e1))


if __name__ == "__main__":
    print(correction('aple'))
    print(correction('correcton'))
    print(correction('statament'))
    print(correction('tutpore'))

from nltk.metrics import edit_distance
edit_distance('aple', 'apple')

text = 'While in France, Christine Lagarde discussed short-term ' \
       'stimulus efforts in a recent interview at 5:00 P.M with the Wall Street Journal.'
nltk.pos_tag(nltk.word_tokenize(text=text))

# 下载模型：https://nlp.stanford.edu/software/tagger.shtml ，我是下载好后放到 Dowdloads 下面了

from nltk.tag import StanfordPOSTagger
from nltk import word_tokenize

jar = '/Users/guofei/Downloads/stanford-postagger-full-2020-11-17/stanford-postagger.jar'
model = '/Users/guofei/Downloads/stanford-postagger-full-2020-11-17/models/english-left3words-distsim.tagger'

pos_tagger = StanfordPOSTagger(model, jar, encoding='utf8')


text = 'While in France, Christine Lagarde discussed short-term ' \
       'stimulus efforts in a recent interview at 5:00 P.M with the Wall Street Journal.'

tokenized_text=word_tokenize(text)
classified_text = pos_tagger.tag(tokenized_text)

print(classified_text)

from nltk.tag import StanfordPOSTagger
import jieba
jar = '/Users/guofei/Downloads/stanford-postagger-full-2020-11-17/stanford-postagger.jar'
model = '/Users/guofei/Downloads/stanford-postagger-full-2020-11-17/models/chinese-nodistsim.tagger'
model = '/Users/guofei/Downloads/stanford-postagger-full-2020-11-17/models/chinese-distsim.tagger'
pos_tagger = StanfordPOSTagger(model, jar, encoding='utf8')
text='张伟在北京上班，李雷在南京逛街'
tokenized_text = list(jieba.cut(text))
classified_text = pos_tagger.tag(tokenized_text)

from nltk.corpus import brown
import nltk

brown_tagged_sents = brown.tagged_sents()
train_data = brown_tagged_sents[:int(0.9 * len(brown_tagged_sents))]
test_data = brown_tagged_sents[int(0.9 * len(brown_tagged_sents)):]

default_tagger = nltk.DefaultTagger('NN')  # NN 最多，默认设为NN，准确率就可达到 13%
default_tagger.evaluate(brown_tagged_sents)

unigram_tagger = nltk.tag.UnigramTagger(train=train_data, backoff=default_tagger)
bigram_tagger = nltk.tag.BigramTagger(train=train_data, backoff=unigram_tagger)
trigram_tagger = nltk.tag.TrigramTagger(train=train_data, backoff=bigram_tagger)


print(unigram_tagger.evaluate(test_data))  # 89%
print(bigram_tagger.evaluate(test_data))  # 91%
print(trigram_tagger.evaluate(test_data))  # 91%

import nltk

text = 'London is a big city in the United Kingdom.'

nltk.ne_chunk(nltk.pos_tag(nltk.word_tokenize(text=text)), binary=False)
#