{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "provenance": []
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "source": [
        "## Part I: Data Pre-processing"
      ],
      "metadata": {
        "id": "p6yklm2xWn9f"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "import pandas as pd"
      ],
      "metadata": {
        "id": "YycoIJomXwqH"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# Download the Google Analogy dataset\n",
        "!wget http://download.tensorflow.org/data/questions-words.txt"
      ],
      "metadata": {
        "id": "KKAXuhZIUxD8"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# Preprocess the dataset\n",
        "file_name = \"questions-words\"\n",
        "with open(f\"{file_name}.txt\", \"r\") as f:\n",
        "    data = f.read().splitlines()"
      ],
      "metadata": {
        "id": "xue5pVFLVNQQ"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# check data from the first 10 entries\n",
        "for entry in data[:10]:\n",
        "    print(entry)"
      ],
      "metadata": {
        "id": "h7dOdJOsZzAF"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# TODO1: Write your code here for processing data to pd.DataFrame\n",
        "# Please note that the first five mentions of \": \" indicate `semantic`,\n",
        "# and the remaining nine belong to the `syntatic` category."
      ],
      "metadata": {
        "id": "wmYQ0IWZZxf3"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# Create the dataframe\n",
        "df = pd.DataFrame(\n",
        "    {\n",
        "        \"Question\": questions,\n",
        "        \"Category\": categories,\n",
        "        \"SubCategory\": sub_categories,\n",
        "    }\n",
        ")"
      ],
      "metadata": {
        "id": "_bKA05rVZb_i"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "df.head()"
      ],
      "metadata": {
        "id": "UN2FBcicZmpV"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "df.to_csv(f\"{file_name}.csv\", index=False)"
      ],
      "metadata": {
        "id": "nMGvoDeiZhbp"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Part II: Use pre-trained word embeddings\n",
        "- After finish Part I, you can run Part II code blocks only."
      ],
      "metadata": {
        "id": "Zi2SNNuHWiZO"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "import pandas as pd\n",
        "import numpy as np\n",
        "import gensim.downloader\n",
        "from tqdm import tqdm\n",
        "import matplotlib.pyplot as plt\n",
        "from sklearn.manifold import TSNE"
      ],
      "metadata": {
        "id": "yB4rpJymXiSN"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "data = pd.read_csv(\"questions-words.csv\")"
      ],
      "metadata": {
        "id": "-pGLoyKSHXuQ"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "MODEL_NAME = \"glove-wiki-gigaword-100\"\n",
        "# You can try other models.\n",
        "# https://radimrehurek.com/gensim/models/word2vec.html#pretrained-models\n",
        "\n",
        "# Load the pre-trained model (using GloVe vectors here)\n",
        "model = gensim.downloader.load(MODEL_NAME)\n",
        "print(\"The Gensim model loaded successfully!\")"
      ],
      "metadata": {
        "id": "YWa_1hF3aZHO"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# Do predictions and preserve the gold answers (word_D)\n",
        "preds = []\n",
        "golds = []\n",
        "\n",
        "for analogy in tqdm(data[\"Question\"]):\n",
        "      # TODO2: Write your code here to use pre-trained word embeddings for getting predictions of the analogy task.\n",
        "      # You should also preserve the gold answers during iterations for evaluations later.\n",
        "      \"\"\" Hints\n",
        "      # Unpack the analogy (e.g., \"man\", \"woman\", \"king\", \"queen\")\n",
        "      # Perform vector arithmetic: word_b + word_c - word_a should be close to word_d\n",
        "      # Source: https://github.com/piskvorky/gensim/blob/develop/gensim/models/keyedvectors.py#L776\n",
        "      # Mikolov et al., 2013: big - biggest and small - smallest\n",
        "      # Mikolov et al., 2013: X = vector(”biggest”) − vector(”big”) + vector(”small”).\n",
        "      \"\"\""
      ],
      "metadata": {
        "id": "YTsqJcP1WSTH"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# Perform evaluations. You do not need to modify this block!!\n",
        "\n",
        "def calculate_accuracy(gold: np.ndarray, pred: np.ndarray) -> float:\n",
        "    return np.mean(gold == pred)\n",
        "\n",
        "golds_np, preds_np = np.array(golds), np.array(preds)\n",
        "data = pd.read_csv(\"questions-words.csv\")\n",
        "\n",
        "# Evaluation: categories\n",
        "for category in data[\"Category\"].unique():\n",
        "    mask = data[\"Category\"] == category\n",
        "    golds_cat, preds_cat = golds_np[mask], preds_np[mask]\n",
        "    acc_cat = calculate_accuracy(golds_cat, preds_cat)\n",
        "    print(f\"Category: {category}, Accuracy: {acc_cat * 100}%\")\n",
        "\n",
        "# Evaluation: sub-categories\n",
        "for sub_category in data[\"SubCategory\"].unique():\n",
        "    mask = data[\"SubCategory\"] == sub_category\n",
        "    golds_subcat, preds_subcat = golds_np[mask], preds_np[mask]\n",
        "    acc_subcat = calculate_accuracy(golds_subcat, preds_subcat)\n",
        "    print(f\"Sub-Category{sub_category}, Accuracy: {acc_subcat * 100}%\")"
      ],
      "metadata": {
        "id": "xG7vcPXAW6uT"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# Collect words from Google Analogy dataset\n",
        "SUB_CATEGORY = \": family\"\n",
        "\n",
        "# TODO3: Plot t-SNE for the words in the SUB_CATEGORY `: family`\n",
        "\n",
        "\n",
        "plt.title(\"Word Relationships from Google Analogy Task\")\n",
        "plt.show()\n",
        "plt.savefig(\"word_relationships.png\", bbox_inches=\"tight\")"
      ],
      "metadata": {
        "id": "7_z6CybBXKZu"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "### Part III: Train your own word embeddings"
      ],
      "metadata": {
        "id": "DKRPJxgKXH4j"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "### Get the latest English Wikipedia articles and do sampling.\n",
        "- Usually, we start from Wikipedia dump (https://dumps.wikimedia.org/enwiki/latest/enwiki-latest-pages-articles.xml.bz2). However, the downloading step will take very long. Also, the cleaning step for the Wikipedia corpus ([`gensim.corpora.wikicorpus.WikiCorpus`](https://radimrehurek.com/gensim/corpora/wikicorpus.html#gensim.corpora.wikicorpus.WikiCorpus)) will take much time. Therefore, we provide cleaned files for you."
      ],
      "metadata": {
        "id": "VC_0fE1UzL8T"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# Download the split Wikipedia files\n",
        "# Each file contain 562365 lines (articles).\n",
        "!gdown --id 1jiu9E1NalT2Y8EIuWNa1xf2Tw1f1XuGd -O wiki_texts_part_0.txt.gz\n",
        "!gdown --id 1ABblLRd9HXdXvaNv8H9fFq984bhnowoG -O wiki_texts_part_1.txt.gz\n",
        "!gdown --id 1z2VFNhpPvCejTP5zyejzKj5YjI_Bn42M -O wiki_texts_part_2.txt.gz\n",
        "!gdown --id 1VKjded9BxADRhIoCzXy_W8uzVOTWIf0g -O wiki_texts_part_3.txt.gz\n",
        "!gdown --id 16mBeG26m9LzHXdPe8UrijUIc6sHxhknz -O wiki_texts_part_4.txt.gz"
      ],
      "metadata": {
        "id": "FkubArwCCYxR"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# Download the split Wikipedia files\n",
        "# Each file contain 562365 lines (articles), except the last file.\n",
        "!gdown --id 17JFvxOH-kc-VmvGkhG7p3iSZSpsWdgJI -O wiki_texts_part_5.txt.gz\n",
        "!gdown --id 19IvB2vOJRGlrYulnTXlZECR8zT5v550P -O wiki_texts_part_6.txt.gz\n",
        "!gdown --id 1sjwO8A2SDOKruv6-8NEq7pEIuQ50ygVV -O wiki_texts_part_7.txt.gz\n",
        "!gdown --id 1s7xKWJmyk98Jbq6Fi1scrHy7fr_ellUX -O wiki_texts_part_8.txt.gz\n",
        "!gdown --id 17eQXcrvY1cfpKelLbP2BhQKrljnFNykr -O wiki_texts_part_9.txt.gz\n",
        "!gdown --id 1J5TAN6bNBiSgTIYiPwzmABvGhAF58h62 -O wiki_texts_part_10.txt.gz"
      ],
      "metadata": {
        "id": "8S3ibNT3C8Xk"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# Extract the downloaded wiki_texts_parts files.\n",
        "!gunzip -k wiki_texts_part_*.gz"
      ],
      "metadata": {
        "id": "DUg_c79BC7OL"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# Combine the extracted wiki_texts_parts files.\n",
        "!cat wiki_texts_part_*.txt > wiki_texts_combined.txt"
      ],
      "metadata": {
        "id": "7duk2RbYDB02"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# Check the first ten lines of the combined file\n",
        "!head -n 10 wiki_texts_combined.txt"
      ],
      "metadata": {
        "id": "givLH7NrDs6X"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "Please note that we used the default parameters of [`gensim.corpora.wikicorpus.WikiCorpus`](https://radimrehurek.com/gensim/corpora/wikicorpus.html#gensim.corpora.wikicorpus.WikiCorpus) for cleaning the Wiki raw file. Thus, words with one character were discarded."
      ],
      "metadata": {
        "id": "Hfwx92QCEhrq"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# Now you need to do sampling because the corpus is too big.\n",
        "# You can further perform analysis with a greater sampling ratio.\n",
        "\n",
        "import random\n",
        "\n",
        "wiki_txt_path = \"wiki_texts_combined.txt\"\n",
        "# wiki_texts_combined.txt is a text file separated by linebreaks (\\n).\n",
        "# Each row in wiki_texts_combined.txt indicates a Wikipedia article.\n",
        "\n",
        "with open(wiki_txt_path, \"r\", encoding=\"utf-8\") as f:\n",
        "    with open(output_path, \"w\", encoding=\"utf-8\") as output_file:\n",
        "    # TODO4: Sample `20%` Wikipedia articles\n",
        "    # Write your code here"
      ],
      "metadata": {
        "id": "vUAzButoP03w"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# TODO5: Train your own word embeddings with the sampled articles\n",
        "# https://radimrehurek.com/gensim/models/word2vec.html#gensim.models.word2vec.Word2Vec\n",
        "# Hint: You should perform some pre-processing before training."
      ],
      "metadata": {
        "id": "G7q1Xzunxkdc"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "data = pd.read_csv(\"questions-words.csv\")"
      ],
      "metadata": {
        "id": "qWiQF70izxP7"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# Do predictions and preserve the gold answers (word_D)\n",
        "preds = []\n",
        "golds = []\n",
        "\n",
        "for analogy in tqdm(data[\"Question\"]):\n",
        "      # TODO6: Write your code here to use your trained word embeddings for getting predictions of the analogy task.\n",
        "      # You should also preserve the gold answers during iterations for evaluations later.\n",
        "      \"\"\" Hints\n",
        "      # Unpack the analogy (e.g., \"man\", \"woman\", \"king\", \"queen\")\n",
        "      # Perform vector arithmetic: word_b + word_c - word_a should be close to word_d\n",
        "      # Source: https://github.com/piskvorky/gensim/blob/develop/gensim/models/keyedvectors.py#L776\n",
        "      # Mikolov et al., 2013: big - biggest and small - smallest\n",
        "      # Mikolov et al., 2013: X = vector(”biggest”) − vector(”big”) + vector(”small”).\n",
        "      \"\"\""
      ],
      "metadata": {
        "id": "q6xpqgdIy5x1"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# Collect words from Google Analogy dataset\n",
        "SUB_CATEGORY = \": family\"\n",
        "\n",
        "# TODO7: Plot t-SNE for the words in the SUB_CATEGORY `: family`\n",
        "\n",
        "\n",
        "plt.title(\"Word Relationships from Google Analogy Task\")\n",
        "plt.show()\n",
        "plt.savefig(\"word_relationships.png\", bbox_inches=\"tight\")"
      ],
      "metadata": {
        "id": "AjZ14dQL0mhf"
      },
      "execution_count": null,
      "outputs": []
    }
  ]
}