Skip to content

Add News dataset simulator generation notebook #17

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
Gitanaskhan26 wants to merge 1 commit into
base: main
Choose a base branch
from
Draft

Add News dataset simulator generation notebook #17

Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
311 changes: 311 additions & 0 deletions news/News_Dataset_Generation.ipynb
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,311 @@
{
"cells": [
{
"cell_type": "markdown",
"metadata": {},
"source": [
"# News Dataset Generation\n",
"\n",
"This notebook outlines the data preparation process for the `news_nytimes_lda.npz` file, which is used by the `NewsDataset` simulator in `pgmpy`. \n",
"\n",
"The dataset simulates a media consumer's reading experience of news articles under different viewing devices (desktop vs. mobile), implementing the benchmark introduced by Johansson et al. (2016).\n",
"\n",
"The process includes:\n",
"1. Downloading the NY Times bag-of-words corpus from the UCI Machine Learning Repository (300,000 articles).\n",
"2. Parsing the data into memory-efficient sparse matrices.\n",
"3. Fitting Latent Dirichlet Allocation (LDA) models for $K=50$ and $K=200$ topics.\n",
"4. Extracting the vocabulary and visualizing the learned topics.\n",
"5. Saving the required arrays into an `.npz` format for hosting.\n",
"\n",
"*Note: Generating this dataset requires fitting LDA on 300,000 documents. Running this notebook sequentially requires approximately 4-8 GB of RAM and may take 30-60 minutes depending on hardware.*\n"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"import gzip\n",
"import logging\n",
"import os\n",
"import time\n",
"import urllib.error\n",
"import urllib.request\n",
"from pathlib import Path\n",
"\n",
"import numpy as np\n",
"from scipy import sparse\n",
"from sklearn.decomposition import LatentDirichletAllocation\n",
"\n",
"logging.basicConfig(level=logging.INFO, format=\"%(asctime)s [%(levelname)s] %(message)s\")\n",
"log = logging.getLogger(__name__)\n",
"\n",
"# Constants\n",
"UCI_BASE = \"https://archive.ics.uci.edu/ml/machine-learning-databases/bag-of-words\"\n",
"DOCWORD_URL = f\"{UCI_BASE}/docword.nytimes.txt.gz\"\n",
"VOCAB_URL = f\"{UCI_BASE}/vocab.nytimes.txt\"\n",
"CACHE_DIR = Path(\"_cache_news_prep\")\n",
"\n",
"SEED = 42\n",
"OUTPUT_FILE = \"news_nytimes_lda.npz\" \n"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## 1. Download Raw Data\n",
"Fetch the raw gzipped word counts and vocabulary list from the UCI Machine Learning Repository.\n"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"def download_file(url: str, dest: Path) -> Path:\n",
" if dest.exists():\n",
" log.info(f\"Using cached: {dest}\")\n",
" return dest\n",
" dest.parent.mkdir(parents=True, exist_ok=True)\n",
" log.info(f\"Downloading {url} ...\")\n",
" try:\n",
" urllib.request.urlretrieve(url, dest)\n",
" except urllib.error.URLError as e:\n",
" if \"CERTIFICATE_VERIFY_FAILED\" in str(e):\n",
" import ssl\n",
" log.warning(\"SSL verification failed, retrying with unverified context ...\")\n",
" ctx = ssl.create_default_context()\n",
" ctx.check_hostname = False\n",
" ctx.verify_mode = ssl.CERT_NONE\n",
" opener = urllib.request.build_opener(urllib.request.HTTPSHandler(context=ctx))\n",
" with opener.open(url) as response, open(dest, \"wb\") as out_file:\n",
" import shutil\n",
" shutil.copyfileobj(response, out_file)\n",
" else:\n",
" raise\n",
" log.info(f\"Saved to {dest} ({dest.stat().st_size / 1e6:.1f} MB)\")\n",
" return dest\n",
"\n",
"docword_path = download_file(DOCWORD_URL, CACHE_DIR / \"docword.nytimes.txt.gz\")\n",
"vocab_path = download_file(VOCAB_URL, CACHE_DIR / \"vocab.nytimes.txt\")\n"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## 2. Parsing the Dataset\n",
"The UCI docword format is parsed directly into a `scipy.sparse.csr_matrix` for memory efficiency.\n"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"def load_vocabulary(vocab_path: Path) -> np.ndarray:\n",
" log.info(\"Loading vocabulary ...\")\n",
" with open(vocab_path) as f:\n",
" words = [line.strip() for line in f if line.strip()]\n",
" log.info(f\"Vocabulary size: {len(words)}\")\n",
" return np.array(words)\n",
"\n",
"def load_docword_sparse(docword_path: Path) -> sparse.csr_matrix:\n",
" log.info(f\"Parsing sparse matrix from {docword_path} ...\")\n",
" t0 = time.time()\n",
" open_fn = gzip.open if str(docword_path).endswith(\".gz\") else open\n",
" \n",
" with open_fn(docword_path, \"rt\") as f:\n",
" D = int(f.readline().strip())\n",
" W = int(f.readline().strip())\n",
" NNZ = int(f.readline().strip())\n",
" log.info(f\" D={D:,}, W={W:,}, NNZ={NNZ:,}\")\n",
" \n",
" rows = np.empty(NNZ, dtype=np.int32)\n",
" cols = np.empty(NNZ, dtype=np.int32)\n",
" vals = np.empty(NNZ, dtype=np.int32)\n",
" \n",
" for i, line in enumerate(f):\n",
" parts = line.split()\n",
" rows[i] = int(parts[0]) - 1\n",
" cols[i] = int(parts[1]) - 1\n",
" vals[i] = int(parts[2])\n",
" if (i + 1) % 20_000_000 == 0:\n",
" log.info(f\" ... parsed {i + 1:,}/{NNZ:,} entries\")\n",
" \n",
" mat = sparse.coo_matrix((vals, (rows, cols)), shape=(D, W)).tocsr()\n",
" elapsed = time.time() - t0\n",
" log.info(f\" Sparse matrix shape: {mat.shape}, nnz: {mat.nnz:,}, parsed in {elapsed:.1f}s\")\n",
" return mat\n",
"\n",
"full_vocab = load_vocabulary(vocab_path)\n",
"word_counts_full = load_docword_sparse(docword_path)\n"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## 3. Topic Modeling (LDA)\n",
"Latent Dirichlet Allocation (LDA) models are fit to the dataset.\n",
"* **$K=50$**: The standard hyperparameter used in the original benchmarking paper.\n",
"* **$K=200$**: Pre-computed to allow `pgmpy` to use agglomerative clustering for dynamically merging topics on the fly (supporting $K \\le 200$).\n"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"def fit_lda(word_counts: sparse.csr_matrix, n_components: int, seed: int) -> tuple:\n",
" log.info(f\"Fitting LDA with K={n_components} (batch mode, random_state={seed}) ...\")\n",
" t0 = time.time()\n",
" lda = LatentDirichletAllocation(\n",
" n_components=n_components,\n",
" learning_method=\"batch\",\n",
" random_state=seed,\n",
" n_jobs=1,\n",
" verbose=10,\n",
" )\n",
" topic_dists = lda.fit_transform(word_counts)\n",
" elapsed = time.time() - t0\n",
" log.info(f\" LDA K={n_components} fit in {elapsed:.1f}s ({elapsed / 60:.1f} min)\")\n",
" return topic_dists, lda.components_\n",
"\n",
"topic_dists_k50, lda_components_k50 = fit_lda(word_counts_full, n_components=50, seed=SEED)\n",
"topic_dists_k200, lda_components_k200 = fit_lda(word_counts_full, n_components=200, seed=SEED)\n"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## 4. Vocabulary Selection\n",
"The covariate vocabulary is constructed by taking the union of the top 100 words from each of the 50 baseline topics.\n"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"def select_vocabulary(lda_components: np.ndarray, top_n_words: int = 100) -> np.ndarray:\n",
" top_words_per_topic = np.argsort(lda_components, axis=1)[:, -top_n_words:]\n",
" vocab_indices = np.unique(top_words_per_topic.flatten())\n",
" log.info(\n",
" f\" Selected {len(vocab_indices)} unique words from top-{top_n_words} across {lda_components.shape[0]} topics\"\n",
" )\n",
" return vocab_indices\n",
"\n",
"vocab_indices = select_vocabulary(lda_components_k50, top_n_words=100)\n",
"vocab_3477 = full_vocab[vocab_indices]\n",
"word_counts_3477 = word_counts_full[:, vocab_indices]\n",
"\n",
"log.info(f\"Final Vocabulary shape: {vocab_3477.shape}\")\n",
"log.info(f\"Subset word counts shape: {word_counts_3477.shape}\")\n"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## 5. Visualizing the Learned Topics\n",
"Display the top words for the first 10 topics to verify semantic coherence.\n"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"def display_topics(components, feature_names, no_top_words, n_topics=10):\n",
" for topic_idx, topic in enumerate(components[:n_topics]):\n",
" print(f\"Topic {topic_idx + 1}:\")\n",
" print(\" \".join([feature_names[i] for i in topic.argsort()[:-no_top_words - 1:-1]]))\n",
" print()\n",
"\n",
"display_topics(lda_components_k50, full_vocab, 15, n_topics=10)\n"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## 6. Saving the NPZ File\n",
"The data is packaged into a compressed `.npz` file. Since `np.savez_compressed` does not natively support `scipy.sparse` matrices, they are deconstructed into their underlying CSR arrays (`data`, `indices`, `indptr`).\n"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"# Convert to float32 for storage efficiency\n",
"topic_dists_k50 = topic_dists_k50.astype(np.float32)\n",
"topic_dists_k200 = topic_dists_k200.astype(np.float32)\n",
"lda_components_k50 = lda_components_k50.astype(np.float32)\n",
"lda_components_k200 = lda_components_k200.astype(np.float32)\n",
"\n",
"log.info(f\"Saving to {OUTPUT_FILE} ...\")\n",
"t0 = time.time()\n",
"\n",
"np.savez_compressed(\n",
" OUTPUT_FILE,\n",
" # Precomputed topic distributions\n",
" topic_distributions_k50=topic_dists_k50,\n",
" topic_distributions_k200=topic_dists_k200,\n",
" # LDA components\n",
" lda_components_k50=lda_components_k50,\n",
" lda_components_k200=lda_components_k200,\n",
" # Precomputed vocabulary\n",
" vocab_3477=vocab_3477,\n",
" # Word counts at defaults\n",
" word_counts_3477_data=word_counts_3477.data if sparse.issparse(word_counts_3477) else word_counts_3477,\n",
" word_counts_3477_indices=word_counts_3477.indices if sparse.issparse(word_counts_3477) else np.array([]),\n",
" word_counts_3477_indptr=word_counts_3477.indptr if sparse.issparse(word_counts_3477) else np.array([]),\n",
" word_counts_3477_shape=np.array(word_counts_3477.shape),\n",
" # Full word counts\n",
" word_counts_full_data=word_counts_full.data,\n",
" word_counts_full_indices=word_counts_full.indices,\n",
" word_counts_full_indptr=word_counts_full.indptr,\n",
" word_counts_full_shape=np.array(word_counts_full.shape),\n",
" # Full vocabulary\n",
" full_vocab=full_vocab,\n",
")\n",
"\n",
"file_size = os.path.getsize(OUTPUT_FILE)\n",
"elapsed = time.time() - t0\n",
"log.info(f\"Saved {OUTPUT_FILE} ({file_size / 1e6:.1f} MB) in {elapsed:.1f}s\")\n"
]
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.8.0"
}
},
"nbformat": 4,
"nbformat_minor": 4
}