Building Custom Ad Embeddings with FAISS

import os, glob, json
import numpy as np
import faiss
import matplotlib.pyplot as plt
from sentence_transformers import SentenceTransformer
from sklearn.decomposition import PCA

# ── The only 3 constants you ever need to change ──
FOLDER  = "out_going"          # folder holding your ad files
INDEX_F = "ad_bot.index"       # persisted FAISS binary on disk
META_F  = "ad_metadata.json"   # state tracker — dict of processed filenames

os.makedirs(FOLDER, exist_ok=True)

def create_dummy_ads():
    categories = ["Tech", "Fashion", "Food", "Cars", "Travel"]
    templates  = [
        "Buy our {cat} product! It is {adj}.",
        "Discover the best {cat} deals. {adj} offer today.",
        "Top {cat} service. Very {adj}.",
    ]
    adjectives = ["affordable", "reliable", "fast", "stylish"]
    for i in range(1000):
        cat  = categories[i % len(categories)]
        adj  = adjectives[i  % len(adjectives)]
        body = templates[i   % len(templates)].format(cat=cat, adj=adj)
        ext  = ".md" if i % 2 == 0 else ".txt"
        with open(f"{FOLDER}/ad_{i:04d}{ext}", "w") as f:
            f.write(f"# Ad {i}\nCategory: {cat}\n{body}")

# 1. Load a lightweight, CPU-optimised embedding model
model = SentenceTransformer('all-MiniLM-L6-v2')
dim   = model.get_sentence_embedding_dimension()  # returns 384

# 2. Load existing index + metadata, or start fresh
if os.path.exists(INDEX_F) and os.path.exists(META_F):
    print("Loading existing index...")
    index = faiss.read_index(INDEX_F)
    with open(META_F) as f:
        processed = json.load(f)
else:
    print("No index found. Creating IndexFlatL2...")
    index     = faiss.IndexFlatL2(dim)
    processed = {}

print(f"Index size: {index.ntotal} vectors | Metadata: {len(processed)} entries")

# 1. Collect every .txt and .md currently in the folder
all_files = (
    glob.glob(os.path.join(FOLDER, "*.txt")) +
    glob.glob(os.path.join(FOLDER, "*.md"))
)

# 2. Keep only files NOT already in our metadata dict  ← THE incremental step
new_files = [
    f for f in all_files
    if os.path.basename(f) not in processed
]

print(f"Total: {len(all_files)} | Processed: {len(processed)} | New: {len(new_files)}")

# 3. Early-exit if nothing to do
if not new_files:
    print("Nothing new. Index is up to date.")
    exit(0)   # exit code 0 = success for Windows Task Scheduler

# 1. Read all new files into memory
new_texts, new_names = [], []
for fp in new_files:
    with open(fp, "r", encoding="utf-8") as f:
        new_texts.append(f.read())
    new_names.append(os.path.basename(fp))

# 2. Batch encode — ALWAYS pass a list, not a single string
#    Returns numpy array shape (N, 384), dtype float32
vectors = model.encode(
    new_texts,
    batch_size=32,               # process 32 texts at a time
    show_progress_bar=True,      # tqdm progress bar in terminal
    convert_to_numpy=True        # FAISS requires numpy float32
)

# 3. Append vectors to the FAISS index (does not touch existing vectors)
index.add(vectors)              # index.ntotal increases by len(new_texts)

# 4. Record each new file in metadata with its FAISS position (ID)
start_id = len(processed)
for i, name in enumerate(new_names):
    processed[name] = {
        "faiss_id": start_id + i,  # integer position in the index
        "status":   "indexed"
    }

# 5. Persist both files to disk
faiss.write_index(index, INDEX_F)
with open(META_F, "w") as f:
    json.dump(processed, f, indent=4)
print(f"Saved. Index now holds {index.ntotal} vectors.")

def visualise_clusters(model):
    all_paths = (glob.glob(f"{FOLDER}/*.txt") +
                 glob.glob(f"{FOLDER}/*.md"))
    texts = []
    for fp in all_paths:
        with open(fp) as f:
            texts.append(f.read())

    embeddings = model.encode(texts, batch_size=64, show_progress_bar=True)

    # Reduce 384 dims to 2 dims for the scatter plot
    pca     = PCA(n_components=2)
    reduced = pca.fit_transform(embeddings)  # shape: (1000, 2)

    plt.figure(figsize=(10, 6))
    plt.scatter(reduced[:, 0], reduced[:, 1],
                alpha=0.5, c='#5B21B6', s=12)
    plt.title("Ad Clusters — PCA of 384-dim Embeddings")
    plt.xlabel("PC 1")
    plt.ylabel("PC 2")
    plt.grid(True, alpha=0.3)
    plt.tight_layout()
    plt.savefig("ad_clusters.png", dpi=150)
    print("Saved: ad_clusters.png")