In Part 1, we built the core mathematical engine of an Artificial Neural Network (ANN) using pure NumPy to solve numerical classification. However, human communication does not happen in raw floating-point numbers—it happens in words, phrases, and natural language.
To make an AI understand and generate text, we must convert unstructured words into mathematical vectors through Tokenization and Vocabulary Indexing, then train a Multi-Layer Neural Network to predict the next word in a sequence. In this guide, we will build a complete, zero-dependency Natural Language Processing (NLP) text-prediction AI in Python from scratch.
1. How AI Understands Natural Language
Computers cannot process string characters directly. Language modeling relies on a distinct conversion pipeline:
- Text Normalization & Tokenization: Cleaning punctuation, converting to lowercase, and splitting raw sentences into discrete tokens (words).
- Vocabulary Mapping: Creating lookup dictionaries to assign every unique word a unique numerical ID, and vice versa.
- One-Hot Vector Encoding: Converting each token ID into a binary vector where the target index is 1 and all other positions are 0.
- Sequence Modeling: Feeding context words into the network to predict the probability distribution across the entire vocabulary for the next word.
2. Softmax & Cross-Entropy
For multi-class text prediction, we replace the binary Sigmoid function with Softmax on the output layer. Softmax converts raw output logits into normalized probabilities that sum strictly to 1.0.
To measure the distance between the predicted probability distribution and the true one-hot target vector, we compute categorical Cross-Entropy Loss.
3. Complete Python Script: Self-Learning NLP Engine
The complete script below builds a custom vocabulary, encodes word sequences, and trains a neural network from scratch using only NumPy to generate text continuation:
import numpy as np
import re
class ScratchNLPEngine:
def __init__(self, learning_rate=0.05):
self.lr = learning_rate
self.word_to_idx = {}
self.idx_to_word = {}
self.vocab_size = 0
def build_vocab(self, corpus):
clean_text = re.sub(r'[^\w\s]', '', corpus.lower())
tokens = clean_text.split()
unique_words = sorted(list(set(tokens)))
self.vocab_size = len(unique_words)
for idx, word in enumerate(unique_words):
self.word_to_idx[word] = idx
self.idx_to_word[idx] = word
return tokens
def one_hot_encode(self, index):
vector = np.zeros((1, self.vocab_size))
vector[0, index] = 1.0
return vector
def relu(self, z):
return np.maximum(0, z)
def relu_derivative(self, z):
return (z > 0).astype(float)
def softmax(self, z):
exp_z = np.exp(z - np.max(z, axis=1, keepdims=True))
return exp_z / np.sum(exp_z, axis=1, keepdims=True)
def initialize_network(self, hidden_dim=32):
np.random.seed(42)
self.W1 = np.random.randn(self.vocab_size, hidden_dim) * np.sqrt(2.0 / self.vocab_size)
self.b1 = np.zeros((1, hidden_dim))
self.W2 = np.random.randn(hidden_dim, self.vocab_size) * np.sqrt(2.0 / hidden_dim)
self.b2 = np.zeros((1, self.vocab_size))
def forward(self, x):
self.z1 = np.dot(x, self.W1) + self.b1
self.a1 = self.relu(self.z1)
self.z2 = np.dot(self.a1, self.W2) + self.b2
self.probs = self.softmax(self.z2)
return self.probs
def backward(self, x, target_idx):
dz2 = self.probs.copy()
dz2[0, target_idx] -= 1.0
dW2 = np.dot(self.a1.T, dz2)
db2 = np.sum(dz2, axis=0, keepdims=True)
da1 = np.dot(dz2, self.W2.T)
dz1 = da1 * self.relu_derivative(self.z1)
dW1 = np.dot(x.T, dz1)
db1 = np.sum(dz1, axis=0, keepdims=True)
self.W2 -= self.lr * dW2
self.b2 -= self.lr * db2
self.W1 -= self.lr * dW1
self.b1 -= self.lr * db1
def train(self, tokens, epochs=500):
training_pairs = []
for i in range(len(tokens) - 1):
training_pairs.append((self.word_to_idx[tokens[i]], self.word_to_idx[tokens[i+1]]))
for epoch in range(1, epochs + 1):
total_loss = 0.0
for input_idx, target_idx in training_pairs:
x = self.one_hot_encode(input_idx)
probs = self.forward(x)
loss = -np.log(probs[0, target_idx] + 1e-9)
total_loss += loss
self.backward(x, target_idx)
if epoch % 100 == 0 or epoch == 1:
print(f"Epoch {epoch:4d}/{epochs} | Average Loss: {total_loss / len(training_pairs):.5f}")
def generate_text(self, start_word, length=6):
current_word = start_word.lower()
if current_word not in self.word_to_idx:
return "Error: Word not found."
output_sentence = [current_word]
for _ in range(length):
x = self.one_hot_encode(self.word_to_idx[current_word])
probs = self.forward(x)
predicted_idx = np.argmax(probs)
predicted_word = self.idx_to_word[predicted_idx]
output_sentence.append(predicted_word)
current_word = predicted_word
return " ".join(output_sentence)
if __name__ == "__main__":
corpus = "artificial intelligence learns patterns from data. neural network models optimize weights using gradient descent. technology builds automated systems for future applications."
engine = ScratchNLPEngine(learning_rate=0.08)
tokens = engine.build_vocab(corpus)
engine.initialize_network(hidden_dim=24)
print("--- Training NLP Model ---")
engine.train(tokens, epochs=400)
print("\n--- Generating Text ---")
for word in ["artificial", "neural", "technology"]:
print(f"Prompt: '{word}' -> Generated: '{engine.generate_text(word, length=4)}'")
4. Architectural Breakdown
- One-Hot Layer: Represents discrete input words as clean orthogonal vectors.
- ReLU Hidden Layer: Introduces non-linearity to map relationships between words across multidimensional latent space.
- Softmax Output Layer: Computes categorical distribution across all possible tokens, ensuring the network outputs valid probabilities.