"
]
},
"metadata": {},
"output_type": "display_data"
}
],
"source": [
"# [Generative AI] 本格程式由 Claude（Anthropic）產生，作為示範；不是要繳交的版本。\n",
"# Collect words from Google Analogy dataset\n",
"SUB_CATEGORY = \": family\"\n",
"\n",
"# TODO3: Plot t-SNE for the words in the SUB_CATEGORY `: family`\n",
"family_words = sorted({w for q in data[data[\"SubCategory\"] == SUB_CATEGORY][\"Question\"]\n",
" for w in q.lower().split()})\n",
"family_words = [w for w in family_words if w in model.key_to_index]\n",
"vectors = np.array([model[w] for w in family_words])\n",
"# t-SNE：把 100 維壓成 2 維。字只有幾十個，所以 perplexity 要調小（必須小於樣本數）。\n",
"points = TSNE(n_components=2, perplexity=15, random_state=42, init=\"pca\").fit_transform(vectors)\n",
"plt.figure(figsize=(12, 9))\n",
"plt.scatter(points[:, 0], points[:, 1], s=15)\n",
"for (x, y), w in zip(points, family_words):\n",
" plt.annotate(w, (x, y), fontsize=9)\n",
"\n",
"\n",
"plt.title(\"Word Relationships from Google Analogy Task\")\n",
"plt.show()\n",
"plt.savefig(\"word_relationships.png\", bbox_inches=\"tight\")\n"
]
},
{
"cell_type": "markdown",
"id": "6319fe46",
"metadata": {
"id": "DKRPJxgKXH4j"
},
"source": [
"### Part III: Train your own word embeddings"
]
},
{
"cell_type": "markdown",
"id": "c5e66556",
"metadata": {
"id": "VC_0fE1UzL8T"
},
"source": [
"### Get the latest English Wikipedia articles and do sampling.\n",
"- Usually, we start from Wikipedia dump (https://dumps.wikimedia.org/enwiki/latest/enwiki-latest-pages-articles.xml.bz2). However, the downloading step will take very long. Also, the cleaning step for the Wikipedia corpus ([`gensim.corpora.wikicorpus.WikiCorpus`](https://radimrehurek.com/gensim/corpora/wikicorpus.html#gensim.corpora.wikicorpus.WikiCorpus)) will take much time. Therefore, we provide cleaned files for you."
]
},
{
"cell_type": "code",
"execution_count": 15,
"id": "c6100ec9",
"metadata": {
"execution": {
"iopub.execute_input": "2026-10-04T07:39:13.189745Z",
"iopub.status.busy": "2026-10-04T07:39:13.189745Z",
"iopub.status.idle": "2026-10-04T07:39:13.263643Z",
"shell.execute_reply": "2026-10-04T07:39:13.263643Z"
},
"id": "FkubArwCCYxR"
},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"data\\wiki_texts_part_0.txt.gz 1.51 GB\n",
"data\\wiki_texts_part_1.txt.gz 0.84 GB\n",
"data\\wiki_texts_part_2.txt.gz 0.67 GB\n",
"data\\wiki_texts_part_3.txt.gz 0.60 GB\n",
"data\\wiki_texts_part_4.txt.gz 0.57 GB\n",
"data\\wiki_texts_part_5.txt.gz 0.58 GB\n",
"data\\wiki_texts_part_6.txt.gz 0.56 GB\n",
"data\\wiki_texts_part_7.txt.gz 0.53 GB\n",
"data\\wiki_texts_part_8.txt.gz 0.55 GB\n",
"data\\wiki_texts_part_9.txt.gz 0.55 GB\n",
"data\\wiki_texts_part_10.txt.gz 0.00 GB\n"
]
}
],
"source": [
"# [Data loading changed] 規定允許修改資料載入部分。\n",
"# 原本是 11 行 `!gdown --id ...`。改成 Python 迴圈：檔案已經存在就跳過，斷線後可以重跑。\n",
"# Each file contain 562365 lines (articles), except the last file.\n",
"import os, gdown\n",
"DATA_DIR = \"data\"\n",
"os.makedirs(DATA_DIR, exist_ok=True)\n",
"WIKI_IDS = [\n",
" \"1jiu9E1NalT2Y8EIuWNa1xf2Tw1f1XuGd\", \"1ABblLRd9HXdXvaNv8H9fFq984bhnowoG\",\n",
" \"1z2VFNhpPvCejTP5zyejzKj5YjI_Bn42M\", \"1VKjded9BxADRhIoCzXy_W8uzVOTWIf0g\",\n",
" \"16mBeG26m9LzHXdPe8UrijUIc6sHxhknz\", \"17JFvxOH-kc-VmvGkhG7p3iSZSpsWdgJI\",\n",
" \"19IvB2vOJRGlrYulnTXlZECR8zT5v550P\", \"1sjwO8A2SDOKruv6-8NEq7pEIuQ50ygVV\",\n",
" \"1s7xKWJmyk98Jbq6Fi1scrHy7fr_ellUX\", \"17eQXcrvY1cfpKelLbP2BhQKrljnFNykr\",\n",
" \"1J5TAN6bNBiSgTIYiPwzmABvGhAF58h62\",\n",
"]\n",
"for i, fid in enumerate(WIKI_IDS):\n",
" path = os.path.join(DATA_DIR, f\"wiki_texts_part_{i}.txt.gz\")\n",
" if not os.path.exists(path):\n",
" gdown.download(id=fid, output=path, quiet=True)\n",
" print(path, f\"{os.path.getsize(path)/1e9:.2f} GB\")\n"
]
},
{
"cell_type": "code",
"execution_count": 16,
"id": "7f45cce8",
"metadata": {
"execution": {
"iopub.execute_input": "2026-10-04T07:39:13.326732Z",
"iopub.status.busy": "2026-10-04T07:39:13.326732Z",
"iopub.status.idle": "2026-10-04T07:39:13.329568Z",
"shell.execute_reply": "2026-10-04T07:39:13.329568Z"
},
"id": "8S3ibNT3C8Xk"
},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"all parts downloaded\n"
]
}
],
"source": [
"# [Data loading changed] 規定允許修改資料載入部分。\n",
"# 上一格已經把 11 個檔都下載完（模板原本把下載拆成兩格）。\n",
"print(\"all parts downloaded\")\n"
]
},
{
"cell_type": "code",
"execution_count": 17,
"id": "5ebb816e",
"metadata": {
"execution": {
"iopub.execute_input": "2026-10-04T07:39:13.388893Z",
"iopub.status.busy": "2026-10-04T07:39:13.388893Z",
"iopub.status.idle": "2026-10-04T07:39:13.393896Z",
"shell.execute_reply": "2026-10-04T07:39:13.393896Z"
},
"id": "DUg_c79BC7OL"
},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"11 parts: ['wiki_texts_part_0.txt.gz', 'wiki_texts_part_1.txt.gz', 'wiki_texts_part_2.txt.gz', 'wiki_texts_part_3.txt.gz', 'wiki_texts_part_4.txt.gz', 'wiki_texts_part_5.txt.gz', 'wiki_texts_part_6.txt.gz', 'wiki_texts_part_7.txt.gz', 'wiki_texts_part_8.txt.gz', 'wiki_texts_part_9.txt.gz', 'wiki_texts_part_10.txt.gz']\n"
]
}
],
"source": [
"# [Data loading changed] 規定允許修改資料載入部分。\n",
"# 原本是 `!gunzip -k`（解壓縮並保留壓縮檔）＋ 下一格 `!cat`（合併成一個大檔）。\n",
"# 這樣同一份資料會在硬碟上存三份（壓縮檔＋解壓檔＋合併檔），本機硬碟吃不消。\n",
"# 改成：不解壓、不合併，下一步直接用 gzip 一行一行讀壓縮檔。結果完全一樣。\n",
"import glob, gzip, re\n",
"wiki_parts = sorted(glob.glob(os.path.join(DATA_DIR, \"wiki_texts_part_*.txt.gz\")),\n",
" key=lambda p: int(re.search(r\"part_(\\d+)\", p).group(1)))\n",
"print(len(wiki_parts), \"parts:\", [os.path.basename(p) for p in wiki_parts])\n"
]
},
{
"cell_type": "code",
"execution_count": 18,
"id": "d57639d8",
"metadata": {
"execution": {
"iopub.execute_input": "2026-10-04T07:39:13.447833Z",
"iopub.status.busy": "2026-10-04T07:39:13.447833Z",
"iopub.status.idle": "2026-10-04T07:39:13.451827Z",
"shell.execute_reply": "2026-10-04T07:39:13.451827Z"
},
"id": "7duk2RbYDB02"
},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"total compressed size: 6.93 GB\n"
]
}
],
"source": [
"# [Data loading changed] 規定允許修改資料載入部分。\n",
"# 不需要合併：wiki_parts 這個清單按 0 到 10 的順序排好，依序讀就等於讀合併檔。\n",
"print(\"total compressed size:\", round(sum(os.path.getsize(p) for p in wiki_parts) / 1e9, 2), \"GB\")\n"
]
},
{
"cell_type": "code",
"execution_count": 19,
"id": "cbc79ab7",
"metadata": {
"execution": {
"iopub.execute_input": "2026-10-04T07:39:13.495890Z",
"iopub.status.busy": "2026-10-04T07:39:13.495890Z",
"iopub.status.idle": "2026-10-04T07:39:13.503264Z",
"shell.execute_reply": "2026-10-04T07:39:13.503264Z"
},
"id": "givLH7NrDs6X"
},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"0 anarchism is political philosophy and movement that is against all forms of authority and seeks to abolish the institutions it claims maintain unnecessary coercion and hierarchy typically including th ...\n",
"1 albedo change in greenland the map shows the difference between the amount of sunlight greenland reflected in the summer of versus the average percent it reflected between and some areas reflect close ...\n",
"2 or is the first letter and the first vowel letter of the latin alphabet used in the modern english alphabet and others worldwide its name in english is pronounced plural aes it is similar in shape to ...\n",
"3 alabama is state in the southeastern region of the united states it borders tennessee to the north georgia to the east florida and the gulf of mexico to the south and mississippi to the west alabama i ...\n",
"4 in greek mythology achilles or achilleus was hero of the trojan war who was known as being the greatest of all the greek warriors the central character in homer iliad he was the son of the nereid thet ...\n",
"5 abraham lincoln february april was an american lawyer politician and statesman who served as the th president of the united states from until his assassination in he led the united states through the ...\n",
"6 aristotle aristotélēs bc was an ancient greek philosopher and polymath his writings cover broad range of subjects spanning the natural sciences philosophy linguistics economics politics psychology and ...\n",
"7 an american in paris is jazz influenced symphonic poem or tone poem for orchestra by american composer george gershwin first performed in it was inspired by the time that gershwin had spent in paris a ...\n",
"8 the academy award for best production design recognizes achievement for art direction in film the category original name was best art direction but was changed to its current name in for the th academ ...\n",
"9 the academy awards of merit commonly known as the oscars or academy awards are awards for artistic and technical merit given for excellence within the american and international film industry they are ...\n"
]
}
],
"source": [
"# [Data loading changed] 規定允許修改資料載入部分。\n",
"# 原本是 `!head -n 10`。每一行是一整篇文章，很長，所以每行只印前 200 個字元。\n",
"with gzip.open(wiki_parts[0], \"rt\", encoding=\"utf-8\") as f:\n",
" for i, line in enumerate(f):\n",
" print(i, line[:200].strip(), \"...\")\n",
" if i == 9:\n",
" break\n"
]
},
{
"cell_type": "markdown",
"id": "9aa171fa",
"metadata": {
"id": "Hfwx92QCEhrq"
},
"source": [
"Please note that we used the default parameters of [`gensim.corpora.wikicorpus.WikiCorpus`](https://radimrehurek.com/gensim/corpora/wikicorpus.html#gensim.corpora.wikicorpus.WikiCorpus) for cleaning the Wiki raw file. Thus, words with one character were discarded."
]
},
{
"cell_type": "code",
"execution_count": 20,
"id": "5e594fbf",
"metadata": {
"execution": {
"iopub.execute_input": "2026-10-04T07:39:13.538270Z",
"iopub.status.busy": "2026-10-04T07:39:13.538270Z",
"iopub.status.idle": "2026-10-04T07:41:07.217591Z",
"shell.execute_reply": "2026-10-04T07:41:07.217591Z"
},
"id": "vUAzButoP03w"
},
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
"\r",
" 0%| | 0/11 [00:00 big 會讓比較級、最高級、複數、過去式的答案消失\n",
"TOKEN_RE = re.compile(r\"^[a-z]+$\")\n",
"\n",
"def preprocess_file(src, dst):\n",
" n_tokens = 0\n",
" with open(src, encoding=\"utf-8\") as fin, open(dst, \"w\", encoding=\"utf-8\", newline=\"\\n\") as fout:\n",
" for line in fin:\n",
" tokens = [w for w in line.split() if TOKEN_RE.match(w)]\n",
" n_tokens += len(tokens)\n",
" fout.write(\" \".join(tokens) + \"\\n\")\n",
" return n_tokens\n",
"\n",
"# 超參數：向量長度跟 GloVe 一樣是 100，比較才公平；skip-gram 是老師投影片的重點。\n",
"# max_final_vocab：只留最常見的 30 萬個字（老師建議的「只保留高頻字」）。\n",
"W2V_PARAMS = dict(vector_size=100, window=5, sg=1, negative=5, sample=1e-3,\n",
" min_count=5, max_final_vocab=300_000, epochs=5, workers=15, seed=42)\n",
"\n",
"def train_w2v(clean_path):\n",
" t = time.time()\n",
" m = Word2Vec(corpus_file=clean_path, **W2V_PARAMS)\n",
" print(f\"trained on {clean_path}: {(time.time() - t) / 60:.1f} min, vocab {len(m.wv.key_to_index):,}\")\n",
" return m\n",
"\n",
"clean_path = \"wiki_sampled_20_clean.txt\"\n",
"n_tokens = preprocess_file(output_path, clean_path)\n",
"print(f\"20% wiki tokens after preprocessing: {n_tokens:,}\")\n",
"my_model = train_w2v(clean_path)\n",
"my_model.save(\"w2v_wiki20.model\")\n"
]
},
{
"cell_type": "code",
"execution_count": 22,
"id": "9d75dfdf",
"metadata": {
"execution": {
"iopub.execute_input": "2026-10-04T08:20:39.878053Z",
"iopub.status.busy": "2026-10-04T08:20:39.878053Z",
"iopub.status.idle": "2026-10-04T08:20:39.903527Z",
"shell.execute_reply": "2026-10-04T08:20:39.903527Z"
},
"id": "qWiQF70izxP7"
},
"outputs": [],
"source": [
"data = pd.read_csv(\"questions-words.csv\")"
]
},
{
"cell_type": "code",
"execution_count": 23,
"id": "52ef1ae9",
"metadata": {
"execution": {
"iopub.execute_input": "2026-10-04T08:20:39.941151Z",
"iopub.status.busy": "2026-10-04T08:20:39.941151Z",
"iopub.status.idle": "2026-10-04T08:22:23.671423Z",
"shell.execute_reply": "2026-10-04T08:22:23.671423Z"
},
"id": "q6xpqgdIy5x1"
},
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
"\r",
" 0%| | 0/19544 [00:00"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"data": {
"text/plain": [
"
"
]
},
"metadata": {},
"output_type": "display_data"
}
],
"source": [
"# [Generative AI] 本格程式由 Claude（Anthropic）產生，作為示範；不是要繳交的版本。\n",
"# Collect words from Google Analogy dataset\n",
"SUB_CATEGORY = \": family\"\n",
"\n",
"# TODO7: Plot t-SNE for the words in the SUB_CATEGORY `: family`\n",
"family_words = sorted({w for q in data[data[\"SubCategory\"] == SUB_CATEGORY][\"Question\"]\n",
" for w in q.lower().split()})\n",
"family_words = [w for w in family_words if w in my_wv.key_to_index]\n",
"vectors = np.array([my_wv[w] for w in family_words])\n",
"# t-SNE：把 100 維壓成 2 維。字只有幾十個，所以 perplexity 要調小（必須小於樣本數）。\n",
"points = TSNE(n_components=2, perplexity=15, random_state=42, init=\"pca\").fit_transform(vectors)\n",
"plt.figure(figsize=(12, 9))\n",
"plt.scatter(points[:, 0], points[:, 1], s=15)\n",
"for (x, y), w in zip(points, family_words):\n",
" plt.annotate(w, (x, y), fontsize=9)\n",
"\n",
"\n",
"plt.title(\"Word Relationships from Google Analogy Task\")\n",
"plt.show()\n",
"plt.savefig(\"word_relationships.png\", bbox_inches=\"tight\")\n"
]
},
{
"cell_type": "markdown",
"id": "1bab848c",
"metadata": {},
"source": [
"Running environment:\n",
"\n",
"Python version:\n",
"\n",
"##### 1. Which embedding model do you use? What are the pre-processing steps? What are the hyperparameter settings? (5%)\n",
"\n",
"Answer: \n",
"\n",
"##### 2. What will the performance be like if you sample 5%, 10% and 20% of wiki text in TODO4? (10%, 3% for each)\n",
"\n",
"Answer: \n",
"\n",
"##### 3. What is the performance for different categories or sub-categories when trained on different corpora? (15%)\n",
"\n",
"3.1. Present your results. (5%)\n",
"\n",
"Answer: \n",
"\n",
"3.2. Introduce the corpus you selected and explain what are the differences between the Wikipedia corpus and your corpus. (including data size, topic difference, structural difference … ) (5%)\n",
"\n",
"Answer: \n",
"\n",
"3.3. Explain why the performance increases or decreases. (5%)\n",
"\n",
"Answer: \n",
"\n",
"##### 4. Select a few words and use their embeddings to retrieve the five most similar words. What do you observe? (10%)\n",
"\n",
"Answer: \n",
"\n",
"##### 5. … Anything that can strengthen your report. (5%)\n",
"\n",
"Answer: \n",
"\n",
"##### 6. Generative AI Usage\n",
"\n",
"Answer:\n"
]
},
{
"cell_type": "markdown",
"id": "ed47849f",
"metadata": {},
"source": [
"## Code For 3. What is the performance for different categories or sub-categories when trained on different corpora? (15%)"
]
},
{
"cell_type": "code",
"execution_count": 25,
"id": "92abd4e0",
"metadata": {
"execution": {
"iopub.execute_input": "2026-10-04T08:22:24.167327Z",
"iopub.status.busy": "2026-10-04T08:22:24.166327Z",
"iopub.status.idle": "2026-10-04T08:26:21.729203Z",
"shell.execute_reply": "2026-10-04T08:26:21.729203Z"
}
},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"GloVe (pre-trained): overall 63.11% (OOV questions 0)\n"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"Wiki 20%: overall 48.21% (OOV questions 0)\n"
]
}
],
"source": [
"# [Generative AI] 本格程式由 Claude（Anthropic）產生，作為示範；不是要繳交的版本。\n",
"# 共用工具：用任何一份詞向量考一次試。\n",
"# 每題一次拿前 10 名候選字，記下正確答案排第幾名（之後算 top-k、做錯題分析都用得到）。\n",
"results, details, models = {}, {}, {}\n",
"\n",
"def evaluate(wv, name, method=\"add\", max_k=10):\n",
" sim = wv.most_similar if method == \"add\" else wv.most_similar_cosmul\n",
" ranks, preds, oov = [], [], 0\n",
" for q in data[\"Question\"]:\n",
" a, b, c, d = q.lower().split()\n",
" if all(w in wv.key_to_index for w in (a, b, c)):\n",
" cands = [w for w, _ in sim(positive=[b, c], negative=[a], topn=max_k)]\n",
" preds.append(cands[0])\n",
" ranks.append(cands.index(d) + 1 if d in cands else None)\n",
" else:\n",
" preds.append(None)\n",
" ranks.append(None)\n",
" oov += 1\n",
" hits = np.array([r == 1 for r in ranks])\n",
" res = {\"Overall\": hits.mean() * 100}\n",
" for col in [\"Category\", \"SubCategory\"]:\n",
" for v in data[col].unique():\n",
" res[v] = hits[(data[col] == v).values].mean() * 100\n",
" res[\"OOV questions\"] = oov\n",
" results[name], details[name] = res, {\"ranks\": ranks, \"preds\": preds}\n",
" print(f\"{name}: overall {res['Overall']:.2f}% (OOV questions {oov})\")\n",
" return res\n",
"\n",
"def answer_coverage(wv):\n",
" # 每個小類裡，四個字全部都在字典裡的題目佔幾 %（答案不在字典裡，再聰明也答不對）\n",
" ok = data[\"Question\"].map(lambda q: all(w in wv.key_to_index for w in q.lower().split()))\n",
" return ok.groupby(data[\"SubCategory\"], sort=False).mean().mul(100).round(1)\n",
"\n",
"models[\"GloVe (pre-trained)\"], models[\"Wiki 20%\"] = model, my_wv\n",
"_ = evaluate(model, \"GloVe (pre-trained)\")\n",
"_ = evaluate(my_wv, \"Wiki 20%\")\n"
]
},
{
"cell_type": "code",
"execution_count": 26,
"id": "f446eac9",
"metadata": {
"execution": {
"iopub.execute_input": "2026-10-04T08:26:21.789832Z",
"iopub.status.busy": "2026-10-04T08:26:21.788826Z",
"iopub.status.idle": "2026-10-04T08:26:22.255209Z",
"shell.execute_reply": "2026-10-04T08:26:22.255209Z"
}
},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"keys: ['RelComments', 'RelQuestion', 'THREAD_SEQUENCE'] | question keys: ['RELQ_DATE', 'RELQ_ID', 'RelQSubject', 'RELQ_USERNAME', 'RELQ_USERID', 'RelQBody', 'RELQ_CATEGORY']\n",
"subject: Thailand:IT Minsitry blocks CNN; Facebook;\n",
"first comment: have they blocked porn??? ![](\"http://www.qatarliving.com/files/images/Da.gif\")
\n"
]
}
],
"source": [
"# [Generative AI] 本格程式由 Claude（Anthropic）產生，作為示範；不是要繳交的版本。\n",
"# 第 3 題：換一份「不是維基百科」的語料。\n",
"# 選的是 SemEval 2016/2017 Task 3 的論壇問答：卡達生活論壇（Qatar Living）網友的發問與留言，gensim-data 可以直接下載。\n",
"# 注意：gensim 幫這份資料附的讀檔程式用了舊版 smart_open 的寫法，新版會 ImportError。\n",
"# 所以只用 gensim 下載（return_path=True 只回傳檔案位置），讀檔自己來：每一行是一個討論串的 JSON。\n",
"import gensim.downloader, itertools, json\n",
"from gensim.utils import simple_preprocess\n",
"\n",
"FORUM_PATH = gensim.downloader.load(\"semeval-2016-2017-task3-subtaskA-unannotated\", return_path=True)\n",
"\n",
"def iter_threads():\n",
" with gzip.open(FORUM_PATH, \"rt\", encoding=\"utf-8\") as f:\n",
" for line in f:\n",
" yield json.loads(line)\n",
"\n",
"def thread_texts(thread):\n",
" # 只拿真正的文字：發問標題、發問內容、每一則留言。使用者名稱、日期、編號都不要。\n",
" q = thread.get(\"RelQuestion\", {})\n",
" yield q.get(\"RelQSubject\") or \"\"\n",
" yield q.get(\"RelQBody\") or \"\"\n",
" for c in thread.get(\"RelComments\", []):\n",
" yield c.get(\"RelCText\") or \"\"\n",
"\n",
"first = next(iter_threads())\n",
"print(\"keys:\", list(first.keys()), \"| question keys:\", list(first[\"RelQuestion\"].keys()))\n",
"print(\"subject:\", first[\"RelQuestion\"].get(\"RelQSubject\"))\n",
"print(\"first comment:\", first[\"RelComments\"][0][\"RelCText\"][:200])\n"
]
},
{
"cell_type": "code",
"execution_count": 27,
"id": "c4ab3eea",
"metadata": {
"execution": {
"iopub.execute_input": "2026-10-04T08:26:22.313147Z",
"iopub.status.busy": "2026-10-04T08:26:22.312140Z",
"iopub.status.idle": "2026-10-04T08:28:36.359131Z",
"shell.execute_reply": "2026-10-04T08:28:36.359131Z"
}
},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"forum: 189,941 threads, 2,118,254 posts, 72,919,300 tokens\n"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"5% wiki tokens: 166,194,401\n"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"wiki control: 72,920,923 tokens\n"
]
}
],
"source": [
"# [Generative AI] 本格程式由 Claude（Anthropic）產生，作為示範；不是要繳交的版本。\n",
"# 用跟維基百科一樣的規則前處理：先拿掉網頁語法和網址，再小寫、只留 2～15 個字母的英文字（WikiCorpus 的預設）。\n",
"HTML_URL_RE = re.compile(r\"<[^>]+>|https?://\\S+|www\\.\\S+\")\n",
"forum_tokens = forum_lines = n_threads = 0\n",
"with open(\"forum_clean.txt\", \"w\", encoding=\"utf-8\", newline=\"\\n\") as fout:\n",
" for thread in iter_threads():\n",
" n_threads += 1\n",
" for text in thread_texts(thread):\n",
" text = HTML_URL_RE.sub(\" \", text)\n",
" toks = [w for w in simple_preprocess(text, min_len=2, max_len=15) if TOKEN_RE.match(w)]\n",
" if len(toks) >= 3:\n",
" fout.write(\" \".join(toks) + \"\\n\")\n",
" forum_tokens += len(toks)\n",
" forum_lines += 1\n",
"print(f\"forum: {n_threads:,} threads, {forum_lines:,} posts, {forum_tokens:,} tokens\")\n",
"\n",
"# 對照組：從維基 5% 裡依序拿文章，拿到「字數跟論壇一樣多」就停。\n",
"# 這樣兩份資料一樣大，分數的差別就只剩「內容種類」造成的。\n",
"n5 = preprocess_file(\"wiki_sampled_5.txt\", \"wiki_sampled_5_clean.txt\")\n",
"print(f\"5% wiki tokens: {n5:,}\")\n",
"ctrl_tokens = 0\n",
"with open(\"wiki_sampled_5_clean.txt\", encoding=\"utf-8\") as fin, open(\"wiki_control_clean.txt\", \"w\", encoding=\"utf-8\", newline=\"\\n\") as fout:\n",
" for line in fin:\n",
" if ctrl_tokens >= forum_tokens:\n",
" break\n",
" fout.write(line)\n",
" ctrl_tokens += len(line.split())\n",
"print(f\"wiki control: {ctrl_tokens:,} tokens\")\n"
]
},
{
"cell_type": "code",
"execution_count": 28,
"id": "e2d8d5be",
"metadata": {
"execution": {
"iopub.execute_input": "2026-10-04T08:28:36.422957Z",
"iopub.status.busy": "2026-10-04T08:28:36.422957Z",
"iopub.status.idle": "2026-10-04T08:37:43.458264Z",
"shell.execute_reply": "2026-10-04T08:37:43.457258Z"
}
},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"trained on forum_clean.txt: 3.6 min, vocab 100,310\n"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"trained on wiki_control_clean.txt: 4.0 min, vocab 224,050\n"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"Forum (same size): overall 26.01% (OOV questions 3510)\n"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"Wiki control (same size): overall 41.98% (OOV questions 169)\n"
]
},
{
"data": {
"text/html": [
"
\n"
]
}
],
"source": [
"# [Generative AI] 本格程式由 Claude（Anthropic）產生，作為示範；不是要繳交的版本。\n",
"# 第 3 題：換一份「不是維基百科」的語料。\n",
"# 選的是 SemEval 2016/2017 Task 3 的論壇問答：卡達生活論壇（Qatar Living）網友的發問與留言，gensim-data 可以直接下載。\n",
"# 注意：gensim 幫這份資料附的讀檔程式用了舊版 smart_open 的寫法，新版會 ImportError。\n",
"# 所以只用 gensim 下載（return_path=True 只回傳檔案位置），讀檔自己來：每一行是一個討論串的 JSON。\n",
"import gensim.downloader, itertools, json\n",
"from gensim.utils import simple_preprocess\n",
"\n",
"FORUM_PATH = gensim.downloader.load(\"semeval-2016-2017-task3-subtaskA-unannotated\", return_path=True)\n",
"\n",
"def iter_threads():\n",
" with gzip.open(FORUM_PATH, \"rt\", encoding=\"utf-8\") as f:\n",
" for line in f:\n",
" yield json.loads(line)\n",
"\n",
"def thread_texts(thread):\n",
" # 只拿真正的文字：發問標題、發問內容、每一則留言。使用者名稱、日期、編號都不要。\n",
" q = thread.get(\"RelQuestion\", {})\n",
" yield q.get(\"RelQSubject\") or \"\"\n",
" yield q.get(\"RelQBody\") or \"\"\n",
" for c in thread.get(\"RelComments\", []):\n",
" yield c.get(\"RelCText\") or \"\"\n",
"\n",
"first = next(iter_threads())\n",
"print(\"keys:\", list(first.keys()), \"| question keys:\", list(first[\"RelQuestion\"].keys()))\n",
"print(\"subject:\", first[\"RelQuestion\"].get(\"RelQSubject\"))\n",
"print(\"first comment:\", first[\"RelComments\"][0][\"RelCText\"][:200])\n"
]
},
{
"cell_type": "code",
"execution_count": 27,
"id": "c4ab3eea",
"metadata": {
"execution": {
"iopub.execute_input": "2026-10-04T08:26:22.313147Z",
"iopub.status.busy": "2026-10-04T08:26:22.312140Z",
"iopub.status.idle": "2026-10-04T08:28:36.359131Z",
"shell.execute_reply": "2026-10-04T08:28:36.359131Z"
}
},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"forum: 189,941 threads, 2,118,254 posts, 72,919,300 tokens\n"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"5% wiki tokens: 166,194,401\n"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"wiki control: 72,920,923 tokens\n"
]
}
],
"source": [
"# [Generative AI] 本格程式由 Claude（Anthropic）產生，作為示範；不是要繳交的版本。\n",
"# 用跟維基百科一樣的規則前處理：先拿掉網頁語法和網址，再小寫、只留 2～15 個字母的英文字（WikiCorpus 的預設）。\n",
"HTML_URL_RE = re.compile(r\"<[^>]+>|https?://\\S+|www\\.\\S+\")\n",
"forum_tokens = forum_lines = n_threads = 0\n",
"with open(\"forum_clean.txt\", \"w\", encoding=\"utf-8\", newline=\"\\n\") as fout:\n",
" for thread in iter_threads():\n",
" n_threads += 1\n",
" for text in thread_texts(thread):\n",
" text = HTML_URL_RE.sub(\" \", text)\n",
" toks = [w for w in simple_preprocess(text, min_len=2, max_len=15) if TOKEN_RE.match(w)]\n",
" if len(toks) >= 3:\n",
" fout.write(\" \".join(toks) + \"\\n\")\n",
" forum_tokens += len(toks)\n",
" forum_lines += 1\n",
"print(f\"forum: {n_threads:,} threads, {forum_lines:,} posts, {forum_tokens:,} tokens\")\n",
"\n",
"# 對照組：從維基 5% 裡依序拿文章，拿到「字數跟論壇一樣多」就停。\n",
"# 這樣兩份資料一樣大，分數的差別就只剩「內容種類」造成的。\n",
"n5 = preprocess_file(\"wiki_sampled_5.txt\", \"wiki_sampled_5_clean.txt\")\n",
"print(f\"5% wiki tokens: {n5:,}\")\n",
"ctrl_tokens = 0\n",
"with open(\"wiki_sampled_5_clean.txt\", encoding=\"utf-8\") as fin, open(\"wiki_control_clean.txt\", \"w\", encoding=\"utf-8\", newline=\"\\n\") as fout:\n",
" for line in fin:\n",
" if ctrl_tokens >= forum_tokens:\n",
" break\n",
" fout.write(line)\n",
" ctrl_tokens += len(line.split())\n",
"print(f\"wiki control: {ctrl_tokens:,} tokens\")\n"
]
},
{
"cell_type": "code",
"execution_count": 28,
"id": "e2d8d5be",
"metadata": {
"execution": {
"iopub.execute_input": "2026-10-04T08:28:36.422957Z",
"iopub.status.busy": "2026-10-04T08:28:36.422957Z",
"iopub.status.idle": "2026-10-04T08:37:43.458264Z",
"shell.execute_reply": "2026-10-04T08:37:43.457258Z"
}
},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"trained on forum_clean.txt: 3.6 min, vocab 100,310\n"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"trained on wiki_control_clean.txt: 4.0 min, vocab 224,050\n"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"Forum (same size): overall 26.01% (OOV questions 3510)\n"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"Wiki control (same size): overall 41.98% (OOV questions 169)\n"
]
},
{
"data": {
"text/html": [
"\n",
"\n",
"\n",
" \n",
"
\n",
"
"
],
"text/plain": [
" Wiki control (same size) Forum (same size) \\\n",
"Overall 41.98 26.01 \n",
"Semantic 46.95 12.88 \n",
"Syntactic 37.85 36.92 \n",
": capital-common-countries 79.05 34.19 \n",
": capital-world 55.02 12.82 \n",
": currency 6.35 1.27 \n",
": city-in-state 36.12 2.59 \n",
": family 65.02 62.06 \n",
": gram1-adjective-to-adverb 9.38 12.30 \n",
": gram2-opposite 9.61 13.05 \n",
": gram3-comparative 52.70 66.82 \n",
": gram4-superlative 17.47 41.71 \n",
": gram5-present-participle 31.82 51.99 \n",
": gram6-nationality-adjective 76.42 20.95 \n",
": gram7-past-tense 41.67 34.81 \n",
": gram8-plural 35.51 40.62 \n",
": gram9-plural-verbs 33.45 44.48 \n",
"OOV questions 169.00 3510.00 \n",
"\n",
" Wiki 20% \n",
"Overall 48.21 \n",
"Semantic 56.09 \n",
"Syntactic 41.66 \n",
": capital-common-countries 77.47 \n",
": capital-world 72.57 \n",
": currency 9.24 \n",
": city-in-state 34.25 \n",
": family 74.11 \n",
": gram1-adjective-to-adverb 16.03 \n",
": gram2-opposite 16.87 \n",
": gram3-comparative 45.95 \n",
": gram4-superlative 26.20 \n",
": gram5-present-participle 32.39 \n",
": gram6-nationality-adjective 80.61 \n",
": gram7-past-tense 43.85 \n",
": gram8-plural 42.49 \n",
": gram9-plural-verbs 41.84 \n",
"OOV questions 0.00 "
]
},
"metadata": {},
"output_type": "display_data"
},
{
"data": {
"text/html": [
"| \n", " | Wiki control (same size) | \n", "Forum (same size) | \n", "Wiki 20% | \n", "
|---|---|---|---|
| Overall | \n", "41.98 | \n", "26.01 | \n", "48.21 | \n", "
| Semantic | \n", "46.95 | \n", "12.88 | \n", "56.09 | \n", "
| Syntactic | \n", "37.85 | \n", "36.92 | \n", "41.66 | \n", "
| : capital-common-countries | \n", "79.05 | \n", "34.19 | \n", "77.47 | \n", "
| : capital-world | \n", "55.02 | \n", "12.82 | \n", "72.57 | \n", "
| : currency | \n", "6.35 | \n", "1.27 | \n", "9.24 | \n", "
| : city-in-state | \n", "36.12 | \n", "2.59 | \n", "34.25 | \n", "
| : family | \n", "65.02 | \n", "62.06 | \n", "74.11 | \n", "
| : gram1-adjective-to-adverb | \n", "9.38 | \n", "12.30 | \n", "16.03 | \n", "
| : gram2-opposite | \n", "9.61 | \n", "13.05 | \n", "16.87 | \n", "
| : gram3-comparative | \n", "52.70 | \n", "66.82 | \n", "45.95 | \n", "
| : gram4-superlative | \n", "17.47 | \n", "41.71 | \n", "26.20 | \n", "
| : gram5-present-participle | \n", "31.82 | \n", "51.99 | \n", "32.39 | \n", "
| : gram6-nationality-adjective | \n", "76.42 | \n", "20.95 | \n", "80.61 | \n", "
| : gram7-past-tense | \n", "41.67 | \n", "34.81 | \n", "43.85 | \n", "
| : gram8-plural | \n", "35.51 | \n", "40.62 | \n", "42.49 | \n", "
| : gram9-plural-verbs | \n", "33.45 | \n", "44.48 | \n", "41.84 | \n", "
| OOV questions | \n", "169.00 | \n", "3510.00 | \n", "0.00 | \n", "
\n",
"\n",
"\n",
" \n",
"
\n",
"
"
],
"text/plain": [
" coverage: wiki control coverage: forum\n",
"SubCategory \n",
": capital-common-countries 100.0 100.0\n",
": capital-world 98.3 48.6\n",
": currency 86.8 39.0\n",
": city-in-state 100.0 68.3\n",
": family 100.0 91.3\n",
": gram1-adjective-to-adverb 100.0 100.0\n",
": gram2-opposite 100.0 100.0\n",
": gram3-comparative 100.0 100.0\n",
": gram4-superlative 94.1 100.0\n",
": gram5-present-participle 100.0 100.0\n",
": gram6-nationality-adjective 100.0 95.1\n",
": gram7-past-tense 100.0 100.0\n",
": gram8-plural 100.0 100.0\n",
": gram9-plural-verbs 100.0 100.0"
]
},
"metadata": {},
"output_type": "display_data"
}
],
"source": [
"# [Generative AI] 本格程式由 Claude（Anthropic）產生，作為示範；不是要繳交的版本。\n",
"forum_model = train_w2v(\"forum_clean.txt\")\n",
"ctrl_model = train_w2v(\"wiki_control_clean.txt\")\n",
"models[\"Forum (same size)\"], models[\"Wiki control (same size)\"] = forum_model.wv, ctrl_model.wv\n",
"evaluate(forum_model.wv, \"Forum (same size)\")\n",
"evaluate(ctrl_model.wv, \"Wiki control (same size)\")\n",
"\n",
"cols = [\"Wiki control (same size)\", \"Forum (same size)\", \"Wiki 20%\"]\n",
"display(pd.DataFrame({k: results[k] for k in cols}).round(2))\n",
"# 答案覆蓋率：題目四個字都在字典裡的比例。覆蓋率低 = 很多題連答的機會都沒有。\n",
"display(pd.DataFrame({\"coverage: wiki control\": answer_coverage(ctrl_model.wv),\n",
" \"coverage: forum\": answer_coverage(forum_model.wv)}))\n"
]
},
{
"cell_type": "code",
"execution_count": 29,
"id": "9f2c3908",
"metadata": {
"execution": {
"iopub.execute_input": "2026-10-04T08:37:43.546280Z",
"iopub.status.busy": "2026-10-04T08:37:43.546280Z",
"iopub.status.idle": "2026-10-04T08:37:43.590040Z",
"shell.execute_reply": "2026-10-04T08:37:43.590040Z"
}
},
"outputs": [
{
"data": {
"text/html": [
"| \n", " | coverage: wiki control | \n", "coverage: forum | \n", "
|---|---|---|
| SubCategory | \n", "\n", " | \n", " |
| : capital-common-countries | \n", "100.0 | \n", "100.0 | \n", "
| : capital-world | \n", "98.3 | \n", "48.6 | \n", "
| : currency | \n", "86.8 | \n", "39.0 | \n", "
| : city-in-state | \n", "100.0 | \n", "68.3 | \n", "
| : family | \n", "100.0 | \n", "91.3 | \n", "
| : gram1-adjective-to-adverb | \n", "100.0 | \n", "100.0 | \n", "
| : gram2-opposite | \n", "100.0 | \n", "100.0 | \n", "
| : gram3-comparative | \n", "100.0 | \n", "100.0 | \n", "
| : gram4-superlative | \n", "94.1 | \n", "100.0 | \n", "
| : gram5-present-participle | \n", "100.0 | \n", "100.0 | \n", "
| : gram6-nationality-adjective | \n", "100.0 | \n", "95.1 | \n", "
| : gram7-past-tense | \n", "100.0 | \n", "100.0 | \n", "
| : gram8-plural | \n", "100.0 | \n", "100.0 | \n", "
| : gram9-plural-verbs | \n", "100.0 | \n", "100.0 | \n", "
\n",
"\n",
"\n",
" \n",
"
\n",
"
"
],
"text/plain": [
" word corpus \\\n",
"0 visa Wiki control (same size) \n",
"1 visa Forum (same size) \n",
"2 salary Wiki control (same size) \n",
"3 salary Forum (same size) \n",
"4 doha Wiki control (same size) \n",
"5 doha Forum (same size) \n",
"6 family Wiki control (same size) \n",
"7 family Forum (same size) \n",
"8 king Wiki control (same size) \n",
"9 king Forum (same size) \n",
"10 paris Wiki control (same size) \n",
"11 paris Forum (same size) \n",
"12 computer Wiki control (same size) \n",
"13 computer Forum (same size) \n",
"\n",
" top 5 \n",
"0 visas, passport, passports, bdtc, retransfer \n",
"1 rp, viza, iqama, vissa, residence \n",
"2 salaries, refund, repayment, remuneration, paid \n",
"3 slary, salry, sallary, salery, salaray \n",
"4 manama, qatar, muharraq, abbasiyyin, dhabi \n",
"5 qatar, doah, qata, alkhore, masaieed \n",
"6 relatives, father, parents, grandparents, mother \n",
"7 familly, fmaily, rescidence, familiy, famliy \n",
"8 queen, prince, pretender, harefoot, harthacnut \n",
"9 edshel, ypsilanti, bhumibol, edsel, boorjer \n",
"10 ferrand, marseille, brussels, france, lausanne \n",
"11 amsterdam, france, bangkok, toronto, aires \n",
"12 computers, software, mainframe, hardware, comp... \n",
"13 desktop, soundcard, computers, hardware, pc "
]
},
"metadata": {},
"output_type": "display_data"
}
],
"source": [
"# [Generative AI] 本格程式由 Claude（Anthropic）產生，作為示範；不是要繳交的版本。\n",
"# 同一個字，在論壇和維基（一樣大小）學到的鄰居差在哪\n",
"rows = []\n",
"for w in [\"visa\", \"salary\", \"doha\", \"family\", \"king\", \"paris\", \"computer\"]:\n",
" for name in [\"Wiki control (same size)\", \"Forum (same size)\"]:\n",
" wv = models[name]\n",
" rows.append({\"word\": w, \"corpus\": name,\n",
" \"top 5\": \", \".join(x for x, _ in wv.most_similar(w, topn=5)) if w in wv.key_to_index else \"(not in vocab)\"})\n",
"display(pd.DataFrame(rows))\n"
]
},
{
"cell_type": "markdown",
"id": "fa05e9fc",
"metadata": {},
"source": [
"## Extra code for report question 2 (5% / 10% / 20%)"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "6f1971ad",
"metadata": {},
"outputs": [],
"source": [
"# [Generative AI] 本格程式由 Claude（Anthropic）產生，作為示範；不是要繳交的版本。\n",
"# 第 2 題：5%、10%、20%（助教說這題不用附程式，這裡附上是為了示範）\n",
"n10 = preprocess_file(\"wiki_sampled_10.txt\", \"wiki_sampled_10_clean.txt\")\n",
"print(f\"10% wiki tokens: {n10:,}\")\n",
"for pct in [5, 10]:\n",
" m = train_w2v(f\"wiki_sampled_{pct}_clean.txt\")\n",
" models[f\"Wiki {pct}%\"] = m.wv\n",
" evaluate(m.wv, f\"Wiki {pct}%\")\n",
"display(pd.DataFrame({k: results[k] for k in [\"Wiki 5%\", \"Wiki 10%\", \"Wiki 20%\"]}).round(2))\n"
]
},
{
"cell_type": "markdown",
"id": "53d369c6",
"metadata": {},
"source": [
"## Extra code for report question 4 (most similar words)"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "93753136",
"metadata": {},
"outputs": [],
"source": [
"# [Generative AI] 本格程式由 Claude（Anthropic）產生，作為示範；不是要繳交的版本。\n",
"# 第 4 題：挑字，各找 5 個最像的字。同時看現成的 GloVe 和自己訓練的 Wiki 20%。\n",
"# 挑字的原則：有一個字兩種意思（apple、bank）、反義詞（good）、專有名詞（taiwan）、一般名詞（cat、computer）、考卷裡的字（king）\n",
"probe = [\"king\", \"cat\", \"apple\", \"bank\", \"good\", \"taiwan\", \"computer\"]\n",
"rows = []\n",
"for w in probe:\n",
" for name in [\"GloVe (pre-trained)\", \"Wiki 20%\"]:\n",
" wv = models[name]\n",
" if w in wv.key_to_index:\n",
" rows.append({\"word\": w, \"model\": name,\n",
" \"top 5 (similarity)\": \", \".join(f\"{x} ({s:.2f})\" for x, s in wv.most_similar(w, topn=5))})\n",
"pd.set_option(\"display.max_colwidth\", 200)\n",
"display(pd.DataFrame(rows))\n"
]
},
{
"cell_type": "markdown",
"id": "488efd44",
"metadata": {},
"source": [
"## Extra code for report question 5 (anything that strengthens the report)"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "dcb95a2a",
"metadata": {},
"outputs": [],
"source": [
"# [Generative AI] 本格程式由 Claude（Anthropic）產生，作為示範；不是要繳交的版本。\n",
"# 第 5 題 (a)：前 k 名有猜中就算對，正確率會變多少？（老師說 queen 可能不是第一名，而是第二、三名）\n",
"rows = {}\n",
"for name in [\"GloVe (pre-trained)\", \"Wiki 20%\"]:\n",
" ranks = details[name][\"ranks\"]\n",
" rows[name] = {f\"top-{k}\": np.mean([r is not None and r <= k for r in ranks]) * 100 for k in [1, 3, 5, 10]}\n",
"display(pd.DataFrame(rows).T.round(2))\n"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "59ce5e75",
"metadata": {},
"outputs": [],
"source": [
"# [Generative AI] 本格程式由 Claude（Anthropic）產生，作為示範；不是要繳交的版本。\n",
"# 第 5 題 (b)：錯題分析——電腦答錯時，它到底猜了什麼？\n",
"d20 = details[\"Wiki 20%\"]\n",
"err = data.assign(pred=d20[\"preds\"], rank=d20[\"ranks\"],\n",
" gold=data[\"Question\"].str.lower().str.split().str[3])\n",
"wrong = err[err[\"pred\"] != err[\"gold\"]]\n",
"print(\"wrong answers:\", len(wrong), \"of\", len(err))\n",
"print(\" OOV (question word not in vocab):\", int(wrong[\"pred\"].isna().sum()))\n",
"print(\" gold answer was 2nd-10th:\", int(wrong[\"rank\"].notna().sum()))\n",
"for sub in [\": capital-world\", \": currency\", \": family\", \": gram3-comparative\", \": gram6-nationality-adjective\", \": gram8-plural\"]:\n",
" print(\"\\n\" + sub)\n",
" display(wrong[(wrong[\"SubCategory\"] == sub) & wrong[\"pred\"].notna()][[\"Question\", \"gold\", \"pred\", \"rank\"]].head(6))\n"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "de4453ee",
"metadata": {},
"outputs": [],
"source": [
"# [Generative AI] 本格程式由 Claude（Anthropic）產生，作為示範；不是要繳交的版本。\n",
"# 第 5 題 (c)：為什麼不移除停用詞？先看考卷，再真的做實驗比較。\n",
"import nltk\n",
"nltk.download(\"stopwords\", quiet=True)\n",
"from nltk.corpus import stopwords\n",
"sw = set(stopwords.words(\"english\"))\n",
"has_sw = data[\"Question\"].map(lambda q: any(w in sw for w in q.lower().split()))\n",
"print(\"questions containing a stop word:\", int(has_sw.sum()))\n",
"print(data[has_sw][\"SubCategory\"].value_counts())\n",
"print(data[has_sw][\"Question\"].head(5).tolist())\n",
"\n",
"# 實驗：同樣是 5% 維基、同樣的參數，只差在「有沒有移除停用詞」\n",
"with open(\"wiki_sampled_5_clean.txt\", encoding=\"utf-8\") as fin, open(\"wiki_sampled_5_nostop.txt\", \"w\", encoding=\"utf-8\", newline=\"\\n\") as fout:\n",
" for line in fin:\n",
" fout.write(\" \".join(w for w in line.split() if w not in sw) + \"\\n\")\n",
"m = train_w2v(\"wiki_sampled_5_nostop.txt\")\n",
"models[\"Wiki 5% (stop words removed)\"] = m.wv\n",
"evaluate(m.wv, \"Wiki 5% (stop words removed)\")\n",
"cols = [\"Wiki 5%\", \"Wiki 5% (stop words removed)\"]\n",
"display(pd.DataFrame({k: results[k] for k in cols}).round(2))\n"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "a993ee22",
"metadata": {},
"outputs": [],
"source": [
"# [Generative AI] 本格程式由 Claude（Anthropic）產生，作為示範；不是要繳交的版本。\n",
"# 第 5 題 (d)：超參數比較（都用 5% 維基，一次只改一個設定，其他跟主模型一樣）\n",
"variants = {\n",
" \"Wiki 5% CBOW (sg=0)\": dict(sg=0),\n",
" \"Wiki 5% window=10\": dict(window=10),\n",
" \"Wiki 5% vector_size=300\": dict(vector_size=300),\n",
"}\n",
"for name, change in variants.items():\n",
" t = time.time()\n",
" m = Word2Vec(corpus_file=\"wiki_sampled_5_clean.txt\", **{**W2V_PARAMS, **change})\n",
" print(f\"{name}: trained in {(time.time() - t) / 60:.1f} min\")\n",
" models[name] = m.wv\n",
" evaluate(m.wv, name)\n",
"cols = [\"Wiki 5%\"] + list(variants)\n",
"display(pd.DataFrame({k: results[k] for k in cols}).round(2))\n"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "83e568cc",
"metadata": {},
"outputs": [],
"source": [
"# [Generative AI] 本格程式由 Claude（Anthropic）產生，作為示範；不是要繳交的版本。\n",
"# 第 5 題 (e)：換一種「算答案」的公式。3CosAdd 是 b - a + c；3CosMul（Levy & Goldberg 2014）用乘除，\n",
"# 比較不會被「某一個字特別像」帶走。同一份 Wiki 20% 詞向量，只換公式。\n",
"evaluate(my_wv, \"Wiki 20% (3CosMul)\", method=\"cosmul\")\n",
"display(pd.DataFrame({k: results[k] for k in [\"Wiki 20%\", \"Wiki 20% (3CosMul)\"]}).round(2))\n"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "e1bf7318",
"metadata": {},
"outputs": [],
"source": [
"# [Generative AI] 本格程式由 Claude（Anthropic）產生，作為示範；不是要繳交的版本。\n",
"# 第 5 題 (f)：同一批 family 字，用 PCA 和 t-SNE 兩種方法壓成 2 維，圖長得不一樣嗎？\n",
"from sklearn.decomposition import PCA\n",
"words = sorted({w for q in data[data[\"SubCategory\"] == \": family\"][\"Question\"] for w in q.lower().split()})\n",
"words = [w for w in words if w in my_wv.key_to_index]\n",
"vecs = np.array([my_wv[w] for w in words])\n",
"fig, axes = plt.subplots(1, 2, figsize=(18, 8))\n",
"for ax, (title, pts) in zip(axes, [\n",
" (\"PCA\", PCA(n_components=2, random_state=42).fit_transform(vecs)),\n",
" (\"t-SNE\", TSNE(n_components=2, perplexity=15, random_state=42, init=\"pca\").fit_transform(vecs))]):\n",
" ax.scatter(pts[:, 0], pts[:, 1], s=12)\n",
" for (x, y), w in zip(pts, words):\n",
" ax.annotate(w, (x, y), fontsize=8)\n",
" ax.set_title(f\"{title} (Wiki 20%, family)\")\n",
"plt.show()\n"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "330e6073",
"metadata": {},
"outputs": [],
"source": [
"# [Generative AI] 本格程式由 Claude（Anthropic）產生，作為示範；不是要繳交的版本。\n",
"# 第 5 題 (g)：各小類「四個字都在字典裡」的比例：GloVe vs Wiki 20%\n",
"display(pd.DataFrame({\"GloVe\": answer_coverage(model), \"Wiki 20%\": answer_coverage(my_wv)}))\n"
]
},
{
"cell_type": "markdown",
"id": "72afc8b6",
"metadata": {},
"source": [
"## Summary of all results"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "f8b2f228",
"metadata": {},
"outputs": [],
"source": [
"# [Generative AI] 本格程式由 Claude（Anthropic）產生，作為示範；不是要繳交的版本。\n",
"# 全部結果總表（報告引用的數字都來自這裡）\n",
"pd.set_option(\"display.max_columns\", 30)\n",
"display(pd.DataFrame(results).round(2))\n"
]
},
{
"cell_type": "markdown",
"id": "e8414fa8",
"metadata": {},
"source": [
"## Running environment"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "e9b3b212",
"metadata": {},
"outputs": [],
"source": [
"# [Generative AI] 本格程式由 Claude（Anthropic）產生，作為示範；不是要繳交的版本。\n",
"import sys, platform, gensim, sklearn\n",
"print(\"Python\", sys.version)\n",
"print(platform.platform(), \"|\", platform.processor())\n",
"print(\"gensim\", gensim.__version__, \"| numpy\", np.__version__, \"| pandas\", pd.__version__, \"| scikit-learn\", sklearn.__version__)\n"
]
}
],
"metadata": {
"colab": {
"provenance": []
},
"kernelspec": {
"display_name": "Python 3",
"language": "python",
"name": "python3"
},
"language_info": {
"name": "python"
}
},
"nbformat": 4,
"nbformat_minor": 5
}
| \n", " | word | \n", "corpus | \n", "top 5 | \n", "
|---|---|---|---|
| 0 | \n", "visa | \n", "Wiki control (same size) | \n", "visas, passport, passports, bdtc, retransfer | \n", "
| 1 | \n", "visa | \n", "Forum (same size) | \n", "rp, viza, iqama, vissa, residence | \n", "
| 2 | \n", "salary | \n", "Wiki control (same size) | \n", "salaries, refund, repayment, remuneration, paid | \n", "
| 3 | \n", "salary | \n", "Forum (same size) | \n", "slary, salry, sallary, salery, salaray | \n", "
| 4 | \n", "doha | \n", "Wiki control (same size) | \n", "manama, qatar, muharraq, abbasiyyin, dhabi | \n", "
| 5 | \n", "doha | \n", "Forum (same size) | \n", "qatar, doah, qata, alkhore, masaieed | \n", "
| 6 | \n", "family | \n", "Wiki control (same size) | \n", "relatives, father, parents, grandparents, mother | \n", "
| 7 | \n", "family | \n", "Forum (same size) | \n", "familly, fmaily, rescidence, familiy, famliy | \n", "
| 8 | \n", "king | \n", "Wiki control (same size) | \n", "queen, prince, pretender, harefoot, harthacnut | \n", "
| 9 | \n", "king | \n", "Forum (same size) | \n", "edshel, ypsilanti, bhumibol, edsel, boorjer | \n", "
| 10 | \n", "paris | \n", "Wiki control (same size) | \n", "ferrand, marseille, brussels, france, lausanne | \n", "
| 11 | \n", "paris | \n", "Forum (same size) | \n", "amsterdam, france, bangkok, toronto, aires | \n", "
| 12 | \n", "computer | \n", "Wiki control (same size) | \n", "computers, software, mainframe, hardware, comp... | \n", "
| 13 | \n", "computer | \n", "Forum (same size) | \n", "desktop, soundcard, computers, hardware, pc | \n", "