使用TensorFlow进行文本分类¶

import collections
import pathlib

import tensorflow as tf
from tensorflow.keras import layers
from tensorflow.keras import losses
from tensorflow.keras import utils
from tensorflow.keras.layers import TextVectorization

import tensorflow_datasets as tfds
import tensorflow_text as tf_text

import plotly.express as px
import matplotlib.pyplot as plt


# Training set.
train_ds = tfds.load(
    'imdb_reviews',
    split='train[:80%]',
    shuffle_files=True,
    as_supervised=True)

# Validation set - a tf.data.Dataset object
val_ds = tfds.load(
    'imdb_reviews',
    split='train[80%:]',
    shuffle_files=True,
    as_supervised=True)


# Check the count of records
print(train_ds.cardinality().numpy())
print(val_ds.cardinality().numpy())

20000
5000

for data, label in  train_ds.take(1):
  print(type(data))
  print('Text:', data.numpy())
  print('Label:', label.numpy())

<class 'tensorflow.python.framework.ops.EagerTensor'>
Text: b"This was an absolutely terrible movie. Don't be lured in by Christopher Walken or Michael Ironside. Both are great actors, but this must simply be their worst role in history. Even their great acting could not redeem this movie's ridiculous storyline. This movie is an early nineties US propaganda piece. The most pathetic scenes were those when the Columbian rebels were making their cases for revolutions. Maria Conchita Alonso appeared phony, and her pseudo-love affair with Walken was nothing but a pathetic emotional plug in a movie that was devoid of any real meaning. I am disappointed that there are movies like this, ruining actor's like Christopher Walken's good name. I could barely sit through it."
Label: 0

train_ds = train_ds.map(lambda text, label: (tf_text.case_fold_utf8(text), label))
val_ds = val_ds.map(lambda text, label: (tf_text.case_fold_utf8(text), label))

str_regex_pattern = [("[^A-Za-z0-9(),!?\'\`]", " "),("\'s", " \'s",) ,("\'ve", " \'ve"),("n\'t", " n\'t"),("\'re", " \'re"),("\'d", " \'d")
,("\'ll", " \'ll"),(",", " , "),("!", " ! "),("\(", " \( "),("\)", " \) "),("\?", " \? "),("\s{2,}", " ")]

for pattern, rewrite in str_regex_pattern:
  train_ds = train_ds.map(lambda text, label: (tf.strings.regex_replace(text, pattern=pattern, rewrite=rewrite), label))
  val_ds = val_ds.map(lambda text, label: (tf.strings.regex_replace(text, pattern=pattern, rewrite=rewrite), label))

# Do not use validation set as that will lead to data leak
train_text = train_ds.map(lambda text, label: text)

tokenizer = tf_text.WhitespaceTokenizer()

unique_tokens = collections.defaultdict(lambda: 0)
sentence_length =  []
for text in train_text.as_numpy_iterator():
  tokens = tokenizer.tokenize(text).numpy()
  sentence_length.append(len(tokens))
  for token in tokens:
    unique_tokens[token] += 1

# check out the average sentence length -> ~250 tokens
print(sum(sentence_length)/len(sentence_length))

# print 10 most used tokens - token, frequency
d_view = [ (v,k) for k,v in unique_tokens.items()]
d_view.sort(reverse=True)  
for v,k in d_view[:10]:
    print("%s: %d" % (k,v))

b'the': 269406
b',': 221098
b'and': 131502
b'a': 130309
b'of': 116695
b'to': 108605
b'is': 88351
b'br': 81558
b'it': 77094
b'in': 75177

fig = px.scatter(x=range(len(d_view)), y=[cnt for cnt, word in d_view])
fig.show()

keys = [token for cnt, token in d_view][:vocab_size]
values = range(2, len(keys) + 2)  # Reserve `0` for padding, `1` for OOV tokens.

num_oov_buckets =1

# Note: must assign the key_dtype and value_dtype when the keys and values are Python arrays
init = tf.lookup.KeyValueTensorInitializer(
    keys= keys,
    values= values,
    key_dtype=tf.string, value_dtype=tf.int64)

table = tf.lookup.StaticVocabularyTable(
   init,
   num_oov_buckets=num_oov_buckets)

# Test the look up table with sample input
input_tensor = tf.constant(["emerson", "lake", "palmer", "king"])
print(table[input_tensor].numpy())

array([20000,  2065, 14207,   618])

def text_index_lookup(text, label):
  tokenized = tokenizer.tokenize(text)
  vectorized = table.lookup(tokenized)
  return vectorized, label

train_ds = train_ds.map(text_index_lookup)
val_ds = val_ds.map(text_index_lookup)

AUTOTUNE = tf.data.AUTOTUNE

def configure_dataset(dataset):
  return dataset.cache().prefetch(buffer_size=AUTOTUNE)

train_ds = configure_dataset(train_ds)
val_ds = configure_dataset(val_ds)

BATCH_SIZE = 32
train_ds = train_ds.padded_batch(BATCH_SIZE  )
val_ds = val_ds.padded_batch(BATCH_SIZE  )

# Test set.
test_ds = tfds.load(
    'imdb_reviews',
    split='test',
    # batch_size=BATCH_SIZE,
    shuffle_files=True,
    as_supervised=True)

test_ds = test_ds.map(lambda text, label: (tf_text.case_fold_utf8(text), label))

for pattern, rewrite in str_regex_pattern:
  test_ds = test_ds.map(lambda text, label: (tf.strings.regex_replace(text, pattern=pattern, rewrite=rewrite), label))

test_ds = test_ds.map(text_index_lookup)
test_ds = configure_dataset(test_ds)
test_ds = test_ds.padded_batch(BATCH_SIZE  )

vocab_size += 2 # 0 for padding and 1 for oov token

def create_model(vocab_size, num_labels, dropout_rate):
  model = tf.keras.Sequential([
      tf.keras.layers.Embedding(vocab_size, 128, mask_zero=True),

      tf.keras.layers.Conv1D(32, 3, padding="valid", activation="relu", strides=1),
      tf.keras.layers.MaxPooling1D(pool_size=2),

      tf.keras.layers.Conv1D(64, 4, padding="valid", activation="relu", strides=1),
      tf.keras.layers.MaxPooling1D(pool_size=2),

      tf.keras.layers.Conv1D(128, 5, padding="valid", activation="relu", strides=1),
      tf.keras.layers.GlobalMaxPooling1D( ),

      tf.keras.layers.Dropout(dropout_rate),

      tf.keras.layers.Dense(num_labels)
  ])
  return model

tf.keras.backend.clear_session()
model = create_model(vocab_size=vocab_size, num_labels=2, dropout_rate=0.5)

# 在SGD中使用momentum将显著提高收敛速度
loss = losses.SparseCategoricalCrossentropy(from_logits=True)
optimizer = tf.keras.optimizers.SGD(learning_rate=0.01, momentum=0.9)

model.compile(loss=loss, optimizer=optimizer, metrics='accuracy')

print(model.summary())

Model: "sequential"
_________________________________________________________________
 Layer (type)                Output Shape              Param #   
=================================================================
 embedding (Embedding)       (None, None, 128)         2560256   

 conv1d (Conv1D)             (None, None, 32)          12320     

 max_pooling1d (MaxPooling1D  (None, None, 32)         0         
 )                                                               

 conv1d_1 (Conv1D)           (None, None, 64)          8256      

 max_pooling1d_1 (MaxPooling  (None, None, 64)         0         
 1D)                                                             

 conv1d_2 (Conv1D)           (None, None, 128)         41088     

 global_max_pooling1d (Globa  (None, 128)              0         
 lMaxPooling1D)                                                  

 dropout (Dropout)           (None, 128)               0         

 dense (Dense)               (None, 2)                 258       

=================================================================
Total params: 2,622,178
Trainable params: 2,622,178
Non-trainable params: 0
_________________________________________________________________

# early stopping reduces the risk of overfitting
early_stopping = tf.keras.callbacks.EarlyStopping(patience=10)
epochs = 100
history = model.fit(x=train_ds, validation_data=val_ds,epochs=epochs, callbacks=[early_stopping])

loss, accuracy = model.evaluate(test_ds)

print("Loss: ", loss)
print("Accuracy: {:2.2%}".format(accuracy))

782/782 [==============================] - 57s 72ms/step - loss: 0.4583 - accuracy: 0.8678
Loss:  0.45827823877334595
Accuracy: 86.78%

input = tf.keras.layers.Input([None] )
x = tf.keras.layers.Embedding(
        input_dim=vocab_size,
        output_dim=128,
        # Use masking to handle the variable sequence lengths
        mask_zero=True)(input)

x = tf.keras.layers.Bidirectional(tf.keras.layers.LSTM(64))(x)
x = tf.keras.layers.Dense(64, activation='relu')(x)
x = tf.keras.layers.Dropout(dropout_rate)(x)
output = tf.keras.layers.Dense(num_labels)(x)

lstm_model = tf.keras.Model(inputs=input, outputs=output, name="text_lstm_model")

loss = losses.SparseCategoricalCrossentropy(from_logits=True)
optimizer = tf.keras.optimizers.SGD(learning_rate=0.01, momentum=0.9)

lstm_model.compile(loss=loss, optimizer=optimizer, metrics='accuracy')

lstm_model.summary()

Model: "text_lstm_model"
_________________________________________________________________
 Layer (type)                Output Shape              Param #   
=================================================================
 input_5 (InputLayer)        [(None, None)]            0         

 embedding_5 (Embedding)     (None, None, 128)         2560256   

 bidirectional_4 (Bidirectio  (None, 128)              98816     
 nal)                                                            

 dense_4 (Dense)             (None, 64)                8256      

 dropout_2 (Dropout)         (None, 64)                0         

 dense_5 (Dense)             (None, 2)                 130       

=================================================================
Total params: 2,667,458
Trainable params: 2,667,458
Non-trainable params: 0
_________________________________________________________________

history_2 = lstm_model.fit(x=train_ds, validation_data=val_ds, epochs=epochs, callbacks=[early_stopping])

loss, accuracy = lstm_model.evaluate(test_ds)

print("Loss: ", loss)
print("Accuracy: {:2.2%}".format(accuracy))

782/782 [==============================] - 84s 106ms/step - loss: 0.4105 - accuracy: 0.8160
Loss:  0.4105057716369629
Accuracy: 81.60%