compression.py

import fasttext
import numpy as np
from sklearn.decomposition import TruncatedSVD

# Step 1: Load the FastText model
model_path = "model/fasttext_model2.bin"
model = fasttext.load_model(model_path)

# Step 2: Extract word vectors and vocabulary
words = model.get_words()
d_original = model.get_dimension()  # Original embedding dimension (e.g., 300)
vectors = np.array([model.get_word_vector(word) for word in words], dtype=np.float32)

# Step 3: Reduce dimensions using Truncated SVD (PCA alternative)
d_reduced = 100  # New dimension size (adjust based on needs)
svd = TruncatedSVD(n_components=d_reduced)
compressed_vectors = svd.fit_transform(vectors)

# Step 4: Save compressed embeddings and vocabulary
np.savez_compressed("compressed_fasttext_embeddings.npz", vectors=compressed_vectors, words=words)

print(f"Compression completed! Original dim: {d_original}, Reduced dim: {d_reduced}")
print("Saved as 'compressed_fasttext_embeddings.npz'")