"
]
},
"metadata": {},
"output_type": "display_data"
}
],
"source": [
"# [Generative AI] The code in this cell was generated by Claude (Anthropic).\n",
"# Collect words from Google Analogy dataset\n",
"SUB_CATEGORY = \": family\"\n",
"\n",
"# TODO3: Plot t-SNE for the words in the SUB_CATEGORY `: family`\n",
"family_words = sorted({w for q in data[data[\"SubCategory\"] == SUB_CATEGORY][\"Question\"]\n",
" for w in q.lower().split()})\n",
"family_words = [w for w in family_words if w in model.key_to_index]\n",
"vectors = np.array([model[w] for w in family_words])\n",
"# t-SNE：把 100 維壓成 2 維。字只有幾十個，所以 perplexity 要調小（必須小於樣本數）。\n",
"points = TSNE(n_components=2, perplexity=15, random_state=42, init=\"pca\").fit_transform(vectors)\n",
"plt.figure(figsize=(12, 9))\n",
"plt.scatter(points[:, 0], points[:, 1], s=15)\n",
"for (x, y), w in zip(points, family_words):\n",
" plt.annotate(w, (x, y), fontsize=9)\n",
"\n",
"\n",
"plt.title(\"Word Relationships from Google Analogy Task\")\n",
"plt.show()\n",
"plt.savefig(\"word_relationships.png\", bbox_inches=\"tight\")\n"
]
},
{
"cell_type": "markdown",
"id": "880b0f6f",
"metadata": {
"id": "DKRPJxgKXH4j"
},
"source": [
"### Part III: Train your own word embeddings"
]
},
{
"cell_type": "markdown",
"id": "674980eb",
"metadata": {
"id": "VC_0fE1UzL8T"
},
"source": [
"### Get the latest English Wikipedia articles and do sampling.\n",
"- Usually, we start from Wikipedia dump (https://dumps.wikimedia.org/enwiki/latest/enwiki-latest-pages-articles.xml.bz2). However, the downloading step will take very long. Also, the cleaning step for the Wikipedia corpus ([`gensim.corpora.wikicorpus.WikiCorpus`](https://radimrehurek.com/gensim/corpora/wikicorpus.html#gensim.corpora.wikicorpus.WikiCorpus)) will take much time. Therefore, we provide cleaned files for you."
]
},
{
"cell_type": "code",
"execution_count": 15,
"id": "3652c905",
"metadata": {
"execution": {
"iopub.execute_input": "2026-10-04T13:07:55.551472Z",
"iopub.status.busy": "2026-10-04T13:07:55.551472Z",
"iopub.status.idle": "2026-10-04T13:07:55.611776Z",
"shell.execute_reply": "2026-10-04T13:07:55.611776Z"
},
"id": "FkubArwCCYxR"
},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"data\\wiki_texts_part_0.txt.gz 1.51 GB\n",
"data\\wiki_texts_part_1.txt.gz 0.84 GB\n",
"data\\wiki_texts_part_2.txt.gz 0.67 GB\n",
"data\\wiki_texts_part_3.txt.gz 0.60 GB\n",
"data\\wiki_texts_part_4.txt.gz 0.57 GB\n",
"data\\wiki_texts_part_5.txt.gz 0.58 GB\n",
"data\\wiki_texts_part_6.txt.gz 0.56 GB\n",
"data\\wiki_texts_part_7.txt.gz 0.53 GB\n",
"data\\wiki_texts_part_8.txt.gz 0.55 GB\n",
"data\\wiki_texts_part_9.txt.gz 0.55 GB\n",
"data\\wiki_texts_part_10.txt.gz 0.00 GB\n"
]
}
],
"source": [
"# [Generative AI] The code in this cell was generated by Claude (Anthropic).\n",
"# [Data loading changed] Only the data-loading part is modified (allowed by the assignment rules).\n",
"# 原本是 11 行 `!gdown --id ...`。改成 Python 迴圈：檔案已經存在就跳過，斷線後可以重跑。\n",
"# Each file contain 562365 lines (articles), except the last file.\n",
"import os, gdown\n",
"DATA_DIR = \"data\"\n",
"os.makedirs(DATA_DIR, exist_ok=True)\n",
"WIKI_IDS = [\n",
" \"1jiu9E1NalT2Y8EIuWNa1xf2Tw1f1XuGd\", \"1ABblLRd9HXdXvaNv8H9fFq984bhnowoG\",\n",
" \"1z2VFNhpPvCejTP5zyejzKj5YjI_Bn42M\", \"1VKjded9BxADRhIoCzXy_W8uzVOTWIf0g\",\n",
" \"16mBeG26m9LzHXdPe8UrijUIc6sHxhknz\", \"17JFvxOH-kc-VmvGkhG7p3iSZSpsWdgJI\",\n",
" \"19IvB2vOJRGlrYulnTXlZECR8zT5v550P\", \"1sjwO8A2SDOKruv6-8NEq7pEIuQ50ygVV\",\n",
" \"1s7xKWJmyk98Jbq6Fi1scrHy7fr_ellUX\", \"17eQXcrvY1cfpKelLbP2BhQKrljnFNykr\",\n",
" \"1J5TAN6bNBiSgTIYiPwzmABvGhAF58h62\",\n",
"]\n",
"for i, fid in enumerate(WIKI_IDS):\n",
" path = os.path.join(DATA_DIR, f\"wiki_texts_part_{i}.txt.gz\")\n",
" if not os.path.exists(path):\n",
" gdown.download(id=fid, output=path, quiet=True)\n",
" print(path, f\"{os.path.getsize(path)/1e9:.2f} GB\")\n"
]
},
{
"cell_type": "code",
"execution_count": 16,
"id": "f1b6f4b5",
"metadata": {
"execution": {
"iopub.execute_input": "2026-10-04T13:07:55.650064Z",
"iopub.status.busy": "2026-10-04T13:07:55.650064Z",
"iopub.status.idle": "2026-10-04T13:07:55.653185Z",
"shell.execute_reply": "2026-10-04T13:07:55.653185Z"
},
"id": "8S3ibNT3C8Xk"
},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"all parts downloaded\n"
]
}
],
"source": [
"# [Generative AI] The code in this cell was generated by Claude (Anthropic).\n",
"# [Data loading changed] Only the data-loading part is modified (allowed by the assignment rules).\n",
"# 上一格已經把 11 個檔都下載完（模板原本把下載拆成兩格）。\n",
"print(\"all parts downloaded\")\n"
]
},
{
"cell_type": "code",
"execution_count": 17,
"id": "76e0aa58",
"metadata": {
"execution": {
"iopub.execute_input": "2026-10-04T13:07:55.691569Z",
"iopub.status.busy": "2026-10-04T13:07:55.691569Z",
"iopub.status.idle": "2026-10-04T13:07:55.695701Z",
"shell.execute_reply": "2026-10-04T13:07:55.695701Z"
},
"id": "DUg_c79BC7OL"
},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"11 parts: ['wiki_texts_part_0.txt.gz', 'wiki_texts_part_1.txt.gz', 'wiki_texts_part_2.txt.gz', 'wiki_texts_part_3.txt.gz', 'wiki_texts_part_4.txt.gz', 'wiki_texts_part_5.txt.gz', 'wiki_texts_part_6.txt.gz', 'wiki_texts_part_7.txt.gz', 'wiki_texts_part_8.txt.gz', 'wiki_texts_part_9.txt.gz', 'wiki_texts_part_10.txt.gz']\n"
]
}
],
"source": [
"# [Generative AI] The code in this cell was generated by Claude (Anthropic).\n",
"# [Data loading changed] Only the data-loading part is modified (allowed by the assignment rules).\n",
"# 原本是 `!gunzip -k`（解壓縮並保留壓縮檔）＋ 下一格 `!cat`（合併成一個大檔）。\n",
"# 這樣同一份資料會在硬碟上存三份（壓縮檔＋解壓檔＋合併檔），本機硬碟吃不消。\n",
"# 改成：不解壓、不合併，下一步直接用 gzip 一行一行讀壓縮檔。結果完全一樣。\n",
"import glob, gzip, re\n",
"wiki_parts = sorted(glob.glob(os.path.join(DATA_DIR, \"wiki_texts_part_*.txt.gz\")),\n",
" key=lambda p: int(re.search(r\"part_(\\d+)\", p).group(1)))\n",
"print(len(wiki_parts), \"parts:\", [os.path.basename(p) for p in wiki_parts])\n"
]
},
{
"cell_type": "code",
"execution_count": 18,
"id": "a94efac7",
"metadata": {
"execution": {
"iopub.execute_input": "2026-10-04T13:07:55.736641Z",
"iopub.status.busy": "2026-10-04T13:07:55.735642Z",
"iopub.status.idle": "2026-10-04T13:07:55.740506Z",
"shell.execute_reply": "2026-10-04T13:07:55.739502Z"
},
"id": "7duk2RbYDB02"
},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"total compressed size: 6.93 GB\n"
]
}
],
"source": [
"# [Generative AI] The code in this cell was generated by Claude (Anthropic).\n",
"# [Data loading changed] Only the data-loading part is modified (allowed by the assignment rules).\n",
"# 不需要合併：wiki_parts 這個清單按 0 到 10 的順序排好，依序讀就等於讀合併檔。\n",
"print(\"total compressed size:\", round(sum(os.path.getsize(p) for p in wiki_parts) / 1e9, 2), \"GB\")\n"
]
},
{
"cell_type": "code",
"execution_count": 19,
"id": "5a22c489",
"metadata": {
"execution": {
"iopub.execute_input": "2026-10-04T13:07:55.778145Z",
"iopub.status.busy": "2026-10-04T13:07:55.777084Z",
"iopub.status.idle": "2026-10-04T13:07:55.784055Z",
"shell.execute_reply": "2026-10-04T13:07:55.784055Z"
},
"id": "givLH7NrDs6X"
},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"0 anarchism is political philosophy and movement that is against all forms of authority and seeks to abolish the institutions it claims maintain unnecessary coercion and hierarchy typically including th ...\n",
"1 albedo change in greenland the map shows the difference between the amount of sunlight greenland reflected in the summer of versus the average percent it reflected between and some areas reflect close ...\n",
"2 or is the first letter and the first vowel letter of the latin alphabet used in the modern english alphabet and others worldwide its name in english is pronounced plural aes it is similar in shape to ...\n",
"3 alabama is state in the southeastern region of the united states it borders tennessee to the north georgia to the east florida and the gulf of mexico to the south and mississippi to the west alabama i ...\n",
"4 in greek mythology achilles or achilleus was hero of the trojan war who was known as being the greatest of all the greek warriors the central character in homer iliad he was the son of the nereid thet ...\n",
"5 abraham lincoln february april was an american lawyer politician and statesman who served as the th president of the united states from until his assassination in he led the united states through the ...\n",
"6 aristotle aristotélēs bc was an ancient greek philosopher and polymath his writings cover broad range of subjects spanning the natural sciences philosophy linguistics economics politics psychology and ...\n",
"7 an american in paris is jazz influenced symphonic poem or tone poem for orchestra by american composer george gershwin first performed in it was inspired by the time that gershwin had spent in paris a ...\n",
"8 the academy award for best production design recognizes achievement for art direction in film the category original name was best art direction but was changed to its current name in for the th academ ...\n",
"9 the academy awards of merit commonly known as the oscars or academy awards are awards for artistic and technical merit given for excellence within the american and international film industry they are ...\n"
]
}
],
"source": [
"# [Generative AI] The code in this cell was generated by Claude (Anthropic).\n",
"# [Data loading changed] Only the data-loading part is modified (allowed by the assignment rules).\n",
"# 原本是 `!head -n 10`。每一行是一整篇文章，很長，所以每行只印前 200 個字元。\n",
"with gzip.open(wiki_parts[0], \"rt\", encoding=\"utf-8\") as f:\n",
" for i, line in enumerate(f):\n",
" print(i, line[:200].strip(), \"...\")\n",
" if i == 9:\n",
" break\n"
]
},
{
"cell_type": "markdown",
"id": "0e4f3e63",
"metadata": {
"id": "Hfwx92QCEhrq"
},
"source": [
"Please note that we used the default parameters of [`gensim.corpora.wikicorpus.WikiCorpus`](https://radimrehurek.com/gensim/corpora/wikicorpus.html#gensim.corpora.wikicorpus.WikiCorpus) for cleaning the Wiki raw file. Thus, words with one character were discarded."
]
},
{
"cell_type": "code",
"execution_count": 20,
"id": "b8be0a12",
"metadata": {
"execution": {
"iopub.execute_input": "2026-10-04T13:07:55.822175Z",
"iopub.status.busy": "2026-10-04T13:07:55.822175Z",
"iopub.status.idle": "2026-10-04T13:10:49.095084Z",
"shell.execute_reply": "2026-10-04T13:10:49.094581Z"
},
"id": "vUAzButoP03w"
},
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
"\r",
" 0%| | 0/11 [00:00 big 會讓比較級、最高級、複數、過去式的答案消失\n",
"TOKEN_RE = re.compile(r\"^[a-z]+$\")\n",
"\n",
"def preprocess_file(src, dst):\n",
" n_in = n_tokens = 0\n",
" with open(src, encoding=\"utf-8\") as fin, open(dst, \"w\", encoding=\"utf-8\", newline=\"\\n\") as fout:\n",
" for line in fin:\n",
" raw = line.split()\n",
" tokens = [w for w in raw if TOKEN_RE.match(w)]\n",
" n_in += len(raw)\n",
" n_tokens += len(tokens)\n",
" fout.write(\" \".join(tokens) + \"\\n\")\n",
" print(f\"{src}: removed non-[a-z] tokens {n_in - n_tokens:,} of {n_in:,} ({(n_in - n_tokens) / n_in:.2%})\")\n",
" return n_tokens\n",
"\n",
"# 超參數：向量長度跟 GloVe 一樣是 100，比較才公平；skip-gram 是老師投影片的重點。\n",
"# max_final_vocab：只留最常見的 30 萬個字（老師建議的「只保留高頻字」）。\n",
"# 注意：設了 max_final_vocab 之後，gensim 會自動把 min_count 往上調到剛好留下 30 萬字（實際值印在下面）。\n",
"# seed：固定初始值；但 workers>1 時多執行緒的計算順序不固定，訓練結果仍會有小幅差異（報告第 5 題 (a) 有量）。\n",
"W2V_PARAMS = dict(vector_size=100, window=5, sg=1, negative=5, sample=1e-3,\n",
" min_count=5, max_final_vocab=300_000, epochs=5, workers=15, seed=42)\n",
"\n",
"def train_w2v(clean_path):\n",
" t = time.time()\n",
" m = Word2Vec(corpus_file=clean_path, **W2V_PARAMS)\n",
" print(f\"trained on {clean_path}: {(time.time() - t) / 60:.1f} min, vocab {len(m.wv.key_to_index):,}, \"\n",
" f\"effective min_count {m.effective_min_count}\")\n",
" return m\n",
"\n",
"clean_path = \"wiki_sampled_20_clean.txt\"\n",
"n_tokens = preprocess_file(output_path, clean_path)\n",
"print(f\"20% wiki tokens after preprocessing: {n_tokens:,}\")\n",
"my_model = train_w2v(clean_path)\n",
"my_model.save(\"w2v_wiki20.model\")\n"
]
},
{
"cell_type": "code",
"execution_count": 22,
"id": "b47e8df0",
"metadata": {
"execution": {
"iopub.execute_input": "2026-10-04T15:00:02.173670Z",
"iopub.status.busy": "2026-10-04T15:00:02.169643Z",
"iopub.status.idle": "2026-10-04T15:00:02.421648Z",
"shell.execute_reply": "2026-10-04T15:00:02.420641Z"
},
"id": "qWiQF70izxP7"
},
"outputs": [],
"source": [
"data = pd.read_csv(\"questions-words.csv\")"
]
},
{
"cell_type": "code",
"execution_count": 23,
"id": "a69122a1",
"metadata": {
"execution": {
"iopub.execute_input": "2026-10-04T15:00:02.558875Z",
"iopub.status.busy": "2026-10-04T15:00:02.557875Z",
"iopub.status.idle": "2026-10-04T15:04:10.818354Z",
"shell.execute_reply": "2026-10-04T15:04:10.817350Z"
},
"id": "q6xpqgdIy5x1"
},
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
"\r",
" 0%| | 0/19544 [00:00"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"data": {
"text/plain": [
"
"
]
},
"metadata": {},
"output_type": "display_data"
}
],
"source": [
"# [Generative AI] The code in this cell was generated by Claude (Anthropic).\n",
"# Collect words from Google Analogy dataset\n",
"SUB_CATEGORY = \": family\"\n",
"\n",
"# TODO7: Plot t-SNE for the words in the SUB_CATEGORY `: family`\n",
"family_words = sorted({w for q in data[data[\"SubCategory\"] == SUB_CATEGORY][\"Question\"]\n",
" for w in q.lower().split()})\n",
"family_words = [w for w in family_words if w in my_wv.key_to_index]\n",
"vectors = np.array([my_wv[w] for w in family_words])\n",
"# t-SNE：把 100 維壓成 2 維。字只有幾十個，所以 perplexity 要調小（必須小於樣本數）。\n",
"points = TSNE(n_components=2, perplexity=15, random_state=42, init=\"pca\").fit_transform(vectors)\n",
"plt.figure(figsize=(12, 9))\n",
"plt.scatter(points[:, 0], points[:, 1], s=15)\n",
"for (x, y), w in zip(points, family_words):\n",
" plt.annotate(w, (x, y), fontsize=9)\n",
"\n",
"\n",
"plt.title(\"Word Relationships from Google Analogy Task\")\n",
"plt.show()\n",
"plt.savefig(\"word_relationships.png\", bbox_inches=\"tight\")\n"
]
},
{
"cell_type": "markdown",
"id": "bfac31b3",
"metadata": {},
"source": [
"Running environment:\n",
"\n",
"Python version:\n",
"\n",
"##### 1. Which embedding model do you use? What are the pre-processing steps? What are the hyperparameter settings? (5%)\n",
"\n",
"Answer: \n",
"\n",
"##### 2. What will the performance be like if you sample 5%, 10% and 20% of wiki text in TODO4? (10%, 3% for each)\n",
"\n",
"Answer: \n",
"\n",
"##### 3. What is the performance for different categories or sub-categories when trained on different corpora? (15%)\n",
"\n",
"3.1. Present your results. (5%)\n",
"\n",
"Answer: \n",
"\n",
"3.2. Introduce the corpus you selected and explain what are the differences between the Wikipedia corpus and your corpus. (including data size, topic difference, structural difference … ) (5%)\n",
"\n",
"Answer: \n",
"\n",
"3.3. Explain why the performance increases or decreases. (5%)\n",
"\n",
"Answer: \n",
"\n",
"##### 4. Select a few words and use their embeddings to retrieve the five most similar words. What do you observe? (10%)\n",
"\n",
"Answer: \n",
"\n",
"##### 5. … Anything that can strengthen your report. (5%)\n",
"\n",
"Answer: \n",
"\n",
"##### 6. Generative AI Usage\n",
"\n",
"Answer:\n"
]
},
{
"cell_type": "markdown",
"id": "f78d8e41",
"metadata": {},
"source": [
"## Code For 3. What is the performance for different categories or sub-categories when trained on different corpora? (15%)"
]
},
{
"cell_type": "code",
"execution_count": 25,
"id": "331d35da",
"metadata": {
"execution": {
"iopub.execute_input": "2026-10-04T15:04:20.079258Z",
"iopub.status.busy": "2026-10-04T15:04:20.078259Z",
"iopub.status.idle": "2026-10-04T15:11:51.322498Z",
"shell.execute_reply": "2026-10-04T15:11:51.321493Z"
}
},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"GloVe (pre-trained): overall 63.11% (OOV questions 0)\n"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"Wiki 20%: overall 47.52% (OOV questions 0)\n"
]
}
],
"source": [
"# Write your code here\n",
"# [Generative AI] The code in this cell was generated by Claude (Anthropic).\n",
"# 共用工具：用任何一份詞向量考一次試。\n",
"# 每題一次拿前 10 名候選字，記下正確答案排第幾名（之後算 top-k、做錯題分析都用得到）。\n",
"results, details, models = {}, {}, {}\n",
"\n",
"def evaluate(wv, name, method=\"add\", max_k=10):\n",
" sim = wv.most_similar if method == \"add\" else wv.most_similar_cosmul\n",
" ranks, preds, oov = [], [], 0\n",
" for q in data[\"Question\"]:\n",
" a, b, c, d = q.lower().split()\n",
" if all(w in wv.key_to_index for w in (a, b, c)):\n",
" cands = [w for w, _ in sim(positive=[b, c], negative=[a], topn=max_k)]\n",
" preds.append(cands[0])\n",
" ranks.append(cands.index(d) + 1 if d in cands else None)\n",
" else:\n",
" preds.append(None)\n",
" ranks.append(None)\n",
" oov += 1\n",
" hits = np.array([r == 1 for r in ranks])\n",
" res = {\"Overall\": hits.mean() * 100}\n",
" for col in [\"Category\", \"SubCategory\"]:\n",
" for v in data[col].unique():\n",
" res[v] = hits[(data[col] == v).values].mean() * 100\n",
" res[\"OOV questions\"] = oov\n",
" results[name], details[name] = res, {\"ranks\": ranks, \"preds\": preds}\n",
" print(f\"{name}: overall {res['Overall']:.2f}% (OOV questions {oov})\")\n",
" return res\n",
"\n",
"def answer_coverage(wv):\n",
" # 每個小類裡，四個字全部都在字典裡的題目佔幾 %（答案不在字典裡，再聰明也答不對）\n",
" ok = data[\"Question\"].map(lambda q: all(w in wv.key_to_index for w in q.lower().split()))\n",
" return ok.groupby(data[\"SubCategory\"], sort=False).mean().mul(100).round(1)\n",
"\n",
"models[\"GloVe (pre-trained)\"], models[\"Wiki 20%\"] = model, my_wv\n",
"_ = evaluate(model, \"GloVe (pre-trained)\")\n",
"_ = evaluate(my_wv, \"Wiki 20%\")\n"
]
},
{
"cell_type": "code",
"execution_count": 26,
"id": "84130529",
"metadata": {
"execution": {
"iopub.execute_input": "2026-10-04T15:11:51.569870Z",
"iopub.status.busy": "2026-10-04T15:11:51.569365Z",
"iopub.status.idle": "2026-10-04T15:11:54.255622Z",
"shell.execute_reply": "2026-10-04T15:11:54.254613Z"
}
},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"keys: ['RelComments', 'RelQuestion', 'THREAD_SEQUENCE'] | question keys: ['RELQ_DATE', 'RELQ_ID', 'RelQSubject', 'RELQ_USERNAME', 'RELQ_USERID', 'RelQBody', 'RELQ_CATEGORY']\n",
"subject: Thailand:IT Minsitry blocks CNN; Facebook;\n",
"first comment: have they blocked porn??? ![](\"http://www.qatarliving.com/files/images/Da.gif\")
\n"
]
}
],
"source": [
"# [Generative AI] The code in this cell was generated by Claude (Anthropic).\n",
"# 第 3 題：換一份「不是維基百科」的語料。\n",
"# 選的是 SemEval 2016/2017 Task 3 的論壇問答：卡達生活論壇（Qatar Living）網友的發問與留言。\n",
"# 出處：gensim-data https://github.com/RaRe-Technologies/gensim-data （資料集 semeval-2016-2017-task3-subtaskA-unannotated）\n",
"# SemEval-2016 Task 3: Community Question Answering, https://alt.qcri.org/semeval2016/task3/\n",
"# 注意：gensim 幫這份資料附的讀檔程式用了舊版 smart_open 的寫法，新版會 ImportError。\n",
"# 所以只用 gensim 下載（return_path=True 只回傳檔案位置），讀檔自己來：每一行是一個討論串的 JSON。\n",
"import gensim.downloader, itertools, json\n",
"from gensim.utils import simple_preprocess\n",
"\n",
"FORUM_PATH = gensim.downloader.load(\"semeval-2016-2017-task3-subtaskA-unannotated\", return_path=True)\n",
"\n",
"def iter_threads():\n",
" with gzip.open(FORUM_PATH, \"rt\", encoding=\"utf-8\") as f:\n",
" for line in f:\n",
" yield json.loads(line)\n",
"\n",
"def thread_texts(thread):\n",
" # 只拿真正的文字：發問標題、發問內容、每一則留言。使用者名稱、日期、編號都不要。\n",
" q = thread.get(\"RelQuestion\", {})\n",
" yield q.get(\"RelQSubject\") or \"\"\n",
" yield q.get(\"RelQBody\") or \"\"\n",
" for c in thread.get(\"RelComments\", []):\n",
" yield c.get(\"RelCText\") or \"\"\n",
"\n",
"first = next(iter_threads())\n",
"print(\"keys:\", list(first.keys()), \"| question keys:\", list(first[\"RelQuestion\"].keys()))\n",
"print(\"subject:\", first[\"RelQuestion\"].get(\"RelQSubject\"))\n",
"print(\"first comment:\", first[\"RelComments\"][0][\"RelCText\"][:200])\n"
]
},
{
"cell_type": "code",
"execution_count": 27,
"id": "0e5d418c",
"metadata": {
"execution": {
"iopub.execute_input": "2026-10-04T15:11:54.508928Z",
"iopub.status.busy": "2026-10-04T15:11:54.507923Z",
"iopub.status.idle": "2026-10-04T15:18:38.167557Z",
"shell.execute_reply": "2026-10-04T15:18:38.166553Z"
}
},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"forum: 189,941 threads, 2,118,254 posts, 72,919,300 tokens\n"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"wiki_sampled_5.txt: removed non-[a-z] tokens 1,211,749 of 167,406,150 (0.72%)\n",
"5% wiki tokens: 166,194,401\n"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"wiki control: 72,920,923 tokens\n"
]
}
],
"source": [
"# [Generative AI] The code in this cell was generated by Claude (Anthropic).\n",
"# 用跟維基百科一樣的規則前處理：先拿掉網頁語法和網址，再小寫、只留 2～15 個字母的英文字（WikiCorpus 的預設）。\n",
"HTML_URL_RE = re.compile(r\"<[^>]+>|https?://\\S+|www\\.\\S+\")\n",
"forum_tokens = forum_lines = n_threads = 0\n",
"with open(\"forum_clean.txt\", \"w\", encoding=\"utf-8\", newline=\"\\n\") as fout:\n",
" for thread in iter_threads():\n",
" n_threads += 1\n",
" for text in thread_texts(thread):\n",
" text = HTML_URL_RE.sub(\" \", text)\n",
" toks = [w for w in simple_preprocess(text, min_len=2, max_len=15) if TOKEN_RE.match(w)]\n",
" if len(toks) >= 3:\n",
" fout.write(\" \".join(toks) + \"\\n\")\n",
" forum_tokens += len(toks)\n",
" forum_lines += 1\n",
"print(f\"forum: {n_threads:,} threads, {forum_lines:,} posts, {forum_tokens:,} tokens\")\n",
"\n",
"# 對照組：從維基 5% 裡依序拿文章，拿到「字數跟論壇一樣多」就停。\n",
"# 這樣兩份資料一樣大，分數的差別就只剩「內容種類」造成的。\n",
"n5 = preprocess_file(\"wiki_sampled_5.txt\", \"wiki_sampled_5_clean.txt\")\n",
"print(f\"5% wiki tokens: {n5:,}\")\n",
"ctrl_tokens = 0\n",
"with open(\"wiki_sampled_5_clean.txt\", encoding=\"utf-8\") as fin, open(\"wiki_control_clean.txt\", \"w\", encoding=\"utf-8\", newline=\"\\n\") as fout:\n",
" for line in fin:\n",
" if ctrl_tokens >= forum_tokens:\n",
" break\n",
" fout.write(line)\n",
" ctrl_tokens += len(line.split())\n",
"print(f\"wiki control: {ctrl_tokens:,} tokens\")\n"
]
},
{
"cell_type": "code",
"execution_count": 28,
"id": "077a22fb",
"metadata": {
"execution": {
"iopub.execute_input": "2026-10-04T15:18:38.440427Z",
"iopub.status.busy": "2026-10-04T15:18:38.440427Z",
"iopub.status.idle": "2026-10-04T15:36:46.302414Z",
"shell.execute_reply": "2026-10-04T15:36:46.301400Z"
}
},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"trained on forum_clean.txt: 8.0 min, vocab 100,310, effective min_count 5\n"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"trained on wiki_control_clean.txt: 8.6 min, vocab 224,050, effective min_count 5\n"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"Forum (same size): overall 25.95% (OOV questions 3510)\n"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"Wiki control (same size): overall 41.83% (OOV questions 169)\n"
]
},
{
"data": {
"text/html": [
"
\n"
]
}
],
"source": [
"# [Generative AI] The code in this cell was generated by Claude (Anthropic).\n",
"# 第 3 題：換一份「不是維基百科」的語料。\n",
"# 選的是 SemEval 2016/2017 Task 3 的論壇問答：卡達生活論壇（Qatar Living）網友的發問與留言。\n",
"# 出處：gensim-data https://github.com/RaRe-Technologies/gensim-data （資料集 semeval-2016-2017-task3-subtaskA-unannotated）\n",
"# SemEval-2016 Task 3: Community Question Answering, https://alt.qcri.org/semeval2016/task3/\n",
"# 注意：gensim 幫這份資料附的讀檔程式用了舊版 smart_open 的寫法，新版會 ImportError。\n",
"# 所以只用 gensim 下載（return_path=True 只回傳檔案位置），讀檔自己來：每一行是一個討論串的 JSON。\n",
"import gensim.downloader, itertools, json\n",
"from gensim.utils import simple_preprocess\n",
"\n",
"FORUM_PATH = gensim.downloader.load(\"semeval-2016-2017-task3-subtaskA-unannotated\", return_path=True)\n",
"\n",
"def iter_threads():\n",
" with gzip.open(FORUM_PATH, \"rt\", encoding=\"utf-8\") as f:\n",
" for line in f:\n",
" yield json.loads(line)\n",
"\n",
"def thread_texts(thread):\n",
" # 只拿真正的文字：發問標題、發問內容、每一則留言。使用者名稱、日期、編號都不要。\n",
" q = thread.get(\"RelQuestion\", {})\n",
" yield q.get(\"RelQSubject\") or \"\"\n",
" yield q.get(\"RelQBody\") or \"\"\n",
" for c in thread.get(\"RelComments\", []):\n",
" yield c.get(\"RelCText\") or \"\"\n",
"\n",
"first = next(iter_threads())\n",
"print(\"keys:\", list(first.keys()), \"| question keys:\", list(first[\"RelQuestion\"].keys()))\n",
"print(\"subject:\", first[\"RelQuestion\"].get(\"RelQSubject\"))\n",
"print(\"first comment:\", first[\"RelComments\"][0][\"RelCText\"][:200])\n"
]
},
{
"cell_type": "code",
"execution_count": 27,
"id": "0e5d418c",
"metadata": {
"execution": {
"iopub.execute_input": "2026-10-04T15:11:54.508928Z",
"iopub.status.busy": "2026-10-04T15:11:54.507923Z",
"iopub.status.idle": "2026-10-04T15:18:38.167557Z",
"shell.execute_reply": "2026-10-04T15:18:38.166553Z"
}
},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"forum: 189,941 threads, 2,118,254 posts, 72,919,300 tokens\n"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"wiki_sampled_5.txt: removed non-[a-z] tokens 1,211,749 of 167,406,150 (0.72%)\n",
"5% wiki tokens: 166,194,401\n"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"wiki control: 72,920,923 tokens\n"
]
}
],
"source": [
"# [Generative AI] The code in this cell was generated by Claude (Anthropic).\n",
"# 用跟維基百科一樣的規則前處理：先拿掉網頁語法和網址，再小寫、只留 2～15 個字母的英文字（WikiCorpus 的預設）。\n",
"HTML_URL_RE = re.compile(r\"<[^>]+>|https?://\\S+|www\\.\\S+\")\n",
"forum_tokens = forum_lines = n_threads = 0\n",
"with open(\"forum_clean.txt\", \"w\", encoding=\"utf-8\", newline=\"\\n\") as fout:\n",
" for thread in iter_threads():\n",
" n_threads += 1\n",
" for text in thread_texts(thread):\n",
" text = HTML_URL_RE.sub(\" \", text)\n",
" toks = [w for w in simple_preprocess(text, min_len=2, max_len=15) if TOKEN_RE.match(w)]\n",
" if len(toks) >= 3:\n",
" fout.write(\" \".join(toks) + \"\\n\")\n",
" forum_tokens += len(toks)\n",
" forum_lines += 1\n",
"print(f\"forum: {n_threads:,} threads, {forum_lines:,} posts, {forum_tokens:,} tokens\")\n",
"\n",
"# 對照組：從維基 5% 裡依序拿文章，拿到「字數跟論壇一樣多」就停。\n",
"# 這樣兩份資料一樣大，分數的差別就只剩「內容種類」造成的。\n",
"n5 = preprocess_file(\"wiki_sampled_5.txt\", \"wiki_sampled_5_clean.txt\")\n",
"print(f\"5% wiki tokens: {n5:,}\")\n",
"ctrl_tokens = 0\n",
"with open(\"wiki_sampled_5_clean.txt\", encoding=\"utf-8\") as fin, open(\"wiki_control_clean.txt\", \"w\", encoding=\"utf-8\", newline=\"\\n\") as fout:\n",
" for line in fin:\n",
" if ctrl_tokens >= forum_tokens:\n",
" break\n",
" fout.write(line)\n",
" ctrl_tokens += len(line.split())\n",
"print(f\"wiki control: {ctrl_tokens:,} tokens\")\n"
]
},
{
"cell_type": "code",
"execution_count": 28,
"id": "077a22fb",
"metadata": {
"execution": {
"iopub.execute_input": "2026-10-04T15:18:38.440427Z",
"iopub.status.busy": "2026-10-04T15:18:38.440427Z",
"iopub.status.idle": "2026-10-04T15:36:46.302414Z",
"shell.execute_reply": "2026-10-04T15:36:46.301400Z"
}
},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"trained on forum_clean.txt: 8.0 min, vocab 100,310, effective min_count 5\n"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"trained on wiki_control_clean.txt: 8.6 min, vocab 224,050, effective min_count 5\n"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"Forum (same size): overall 25.95% (OOV questions 3510)\n"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"Wiki control (same size): overall 41.83% (OOV questions 169)\n"
]
},
{
"data": {
"text/html": [
"\n",
"\n",
"\n",
" \n",
"
\n",
"
"
],
"text/plain": [
" Wiki control (same size) Forum (same size) \\\n",
"Overall 41.83 25.95 \n",
"Semantic 46.19 12.49 \n",
"Syntactic 38.21 37.12 \n",
": capital-common-countries 73.32 32.41 \n",
": capital-world 53.93 12.27 \n",
": currency 6.47 1.15 \n",
": city-in-state 36.52 2.59 \n",
": family 65.02 62.25 \n",
": gram1-adjective-to-adverb 10.69 12.10 \n",
": gram2-opposite 10.34 12.19 \n",
": gram3-comparative 52.85 65.84 \n",
": gram4-superlative 17.91 44.03 \n",
": gram5-present-participle 30.11 53.22 \n",
": gram6-nationality-adjective 75.11 19.07 \n",
": gram7-past-tense 42.63 33.97 \n",
": gram8-plural 36.64 44.22 \n",
": gram9-plural-verbs 35.86 44.48 \n",
"OOV questions 169.00 3510.00 \n",
"\n",
" Wiki 20% \n",
"Overall 47.52 \n",
"Semantic 53.10 \n",
"Syntactic 42.89 \n",
": capital-common-countries 74.31 \n",
": capital-world 69.94 \n",
": currency 10.39 \n",
": city-in-state 27.77 \n",
": family 77.87 \n",
": gram1-adjective-to-adverb 17.84 \n",
": gram2-opposite 15.02 \n",
": gram3-comparative 48.80 \n",
": gram4-superlative 29.23 \n",
": gram5-present-participle 32.95 \n",
": gram6-nationality-adjective 82.93 \n",
": gram7-past-tense 41.79 \n",
": gram8-plural 44.67 \n",
": gram9-plural-verbs 43.68 \n",
"OOV questions 0.00 "
]
},
"metadata": {},
"output_type": "display_data"
},
{
"data": {
"text/html": [
"| \n", " | Wiki control (same size) | \n", "Forum (same size) | \n", "Wiki 20% | \n", "
|---|---|---|---|
| Overall | \n", "41.83 | \n", "25.95 | \n", "47.52 | \n", "
| Semantic | \n", "46.19 | \n", "12.49 | \n", "53.10 | \n", "
| Syntactic | \n", "38.21 | \n", "37.12 | \n", "42.89 | \n", "
| : capital-common-countries | \n", "73.32 | \n", "32.41 | \n", "74.31 | \n", "
| : capital-world | \n", "53.93 | \n", "12.27 | \n", "69.94 | \n", "
| : currency | \n", "6.47 | \n", "1.15 | \n", "10.39 | \n", "
| : city-in-state | \n", "36.52 | \n", "2.59 | \n", "27.77 | \n", "
| : family | \n", "65.02 | \n", "62.25 | \n", "77.87 | \n", "
| : gram1-adjective-to-adverb | \n", "10.69 | \n", "12.10 | \n", "17.84 | \n", "
| : gram2-opposite | \n", "10.34 | \n", "12.19 | \n", "15.02 | \n", "
| : gram3-comparative | \n", "52.85 | \n", "65.84 | \n", "48.80 | \n", "
| : gram4-superlative | \n", "17.91 | \n", "44.03 | \n", "29.23 | \n", "
| : gram5-present-participle | \n", "30.11 | \n", "53.22 | \n", "32.95 | \n", "
| : gram6-nationality-adjective | \n", "75.11 | \n", "19.07 | \n", "82.93 | \n", "
| : gram7-past-tense | \n", "42.63 | \n", "33.97 | \n", "41.79 | \n", "
| : gram8-plural | \n", "36.64 | \n", "44.22 | \n", "44.67 | \n", "
| : gram9-plural-verbs | \n", "35.86 | \n", "44.48 | \n", "43.68 | \n", "
| OOV questions | \n", "169.00 | \n", "3510.00 | \n", "0.00 | \n", "
\n",
"\n",
"\n",
" \n",
"
\n",
"
"
],
"text/plain": [
" coverage: wiki control coverage: forum\n",
"SubCategory \n",
": capital-common-countries 100.0 100.0\n",
": capital-world 98.3 48.6\n",
": currency 86.8 39.0\n",
": city-in-state 100.0 68.3\n",
": family 100.0 91.3\n",
": gram1-adjective-to-adverb 100.0 100.0\n",
": gram2-opposite 100.0 100.0\n",
": gram3-comparative 100.0 100.0\n",
": gram4-superlative 94.1 100.0\n",
": gram5-present-participle 100.0 100.0\n",
": gram6-nationality-adjective 100.0 95.1\n",
": gram7-past-tense 100.0 100.0\n",
": gram8-plural 100.0 100.0\n",
": gram9-plural-verbs 100.0 100.0"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"Accuracy on questions whose 4 words are in BOTH vocabularies:\n"
]
},
{
"data": {
"text/html": [
"| \n", " | coverage: wiki control | \n", "coverage: forum | \n", "
|---|---|---|
| SubCategory | \n", "\n", " | \n", " |
| : capital-common-countries | \n", "100.0 | \n", "100.0 | \n", "
| : capital-world | \n", "98.3 | \n", "48.6 | \n", "
| : currency | \n", "86.8 | \n", "39.0 | \n", "
| : city-in-state | \n", "100.0 | \n", "68.3 | \n", "
| : family | \n", "100.0 | \n", "91.3 | \n", "
| : gram1-adjective-to-adverb | \n", "100.0 | \n", "100.0 | \n", "
| : gram2-opposite | \n", "100.0 | \n", "100.0 | \n", "
| : gram3-comparative | \n", "100.0 | \n", "100.0 | \n", "
| : gram4-superlative | \n", "94.1 | \n", "100.0 | \n", "
| : gram5-present-participle | \n", "100.0 | \n", "100.0 | \n", "
| : gram6-nationality-adjective | \n", "100.0 | \n", "95.1 | \n", "
| : gram7-past-tense | \n", "100.0 | \n", "100.0 | \n", "
| : gram8-plural | \n", "100.0 | \n", "100.0 | \n", "
| : gram9-plural-verbs | \n", "100.0 | \n", "100.0 | \n", "
\n",
"\n",
"\n",
" \n",
"
\n",
"
"
],
"text/plain": [
" Wiki control (same size) Forum (same size)\n",
"questions used 15684.00 15684.00\n",
"Overall 43.26 32.32\n",
"Semantic 53.33 21.50\n",
"Syntactic 38.33 37.61\n",
": capital-common-countries 73.32 32.41\n",
": capital-world 62.16 25.24\n",
": currency 9.27 3.31\n",
": city-in-state 39.43 3.80\n",
": family 68.83 68.18\n",
": gram1-adjective-to-adverb 10.69 12.10\n",
": gram2-opposite 10.34 12.19\n",
": gram3-comparative 52.85 65.84\n",
": gram4-superlative 19.03 46.59\n",
": gram5-present-participle 30.11 53.22\n",
": gram6-nationality-adjective 76.20 20.05\n",
": gram7-past-tense 42.63 33.97\n",
": gram8-plural 36.64 44.22\n",
": gram9-plural-verbs 35.86 44.48"
]
},
"metadata": {},
"output_type": "display_data"
}
],
"source": [
"# [Generative AI] The code in this cell was generated by Claude (Anthropic).\n",
"forum_model = train_w2v(\"forum_clean.txt\")\n",
"ctrl_model = train_w2v(\"wiki_control_clean.txt\")\n",
"models[\"Forum (same size)\"], models[\"Wiki control (same size)\"] = forum_model.wv, ctrl_model.wv\n",
"_ = evaluate(forum_model.wv, \"Forum (same size)\")\n",
"_ = evaluate(ctrl_model.wv, \"Wiki control (same size)\")\n",
"\n",
"cols = [\"Wiki control (same size)\", \"Forum (same size)\", \"Wiki 20%\"]\n",
"display(pd.DataFrame({k: results[k] for k in cols}).round(2))\n",
"# 答案覆蓋率：題目四個字都在字典裡的比例。覆蓋率低 = 很多題連答的機會都沒有。\n",
"display(pd.DataFrame({\"coverage: wiki control\": answer_coverage(ctrl_model.wv),\n",
" \"coverage: forum\": answer_coverage(forum_model.wv)}))\n",
"\n",
"# 公平一點：只算「四個字在兩本字典裡都查得到」的題目，排除「字典裡沒有這個字」的影響。\n",
"both = data[\"Question\"].map(lambda q: all(w in forum_model.wv.key_to_index and w in ctrl_model.wv.key_to_index\n",
" for w in q.lower().split())).values\n",
"fair = {}\n",
"for name in [\"Wiki control (same size)\", \"Forum (same size)\"]:\n",
" hits = np.array([r == 1 for r in details[name][\"ranks\"]])\n",
" row = {\"questions used\": int(both.sum()), \"Overall\": hits[both].mean() * 100}\n",
" for col in [\"Category\", \"SubCategory\"]:\n",
" for v in data[col].unique():\n",
" m = (data[col] == v).values & both\n",
" row[v] = hits[m].mean() * 100 if m.sum() else float(\"nan\")\n",
" fair[name] = row\n",
"print(\"Accuracy on questions whose 4 words are in BOTH vocabularies:\")\n",
"display(pd.DataFrame(fair).round(2))\n"
]
},
{
"cell_type": "code",
"execution_count": 29,
"id": "bbccc4a3",
"metadata": {
"execution": {
"iopub.execute_input": "2026-10-04T15:36:46.456880Z",
"iopub.status.busy": "2026-10-04T15:36:46.455881Z",
"iopub.status.idle": "2026-10-04T15:36:46.502512Z",
"shell.execute_reply": "2026-10-04T15:36:46.501410Z"
}
},
"outputs": [
{
"data": {
"text/html": [
"| \n", " | Wiki control (same size) | \n", "Forum (same size) | \n", "
|---|---|---|
| questions used | \n", "15684.00 | \n", "15684.00 | \n", "
| Overall | \n", "43.26 | \n", "32.32 | \n", "
| Semantic | \n", "53.33 | \n", "21.50 | \n", "
| Syntactic | \n", "38.33 | \n", "37.61 | \n", "
| : capital-common-countries | \n", "73.32 | \n", "32.41 | \n", "
| : capital-world | \n", "62.16 | \n", "25.24 | \n", "
| : currency | \n", "9.27 | \n", "3.31 | \n", "
| : city-in-state | \n", "39.43 | \n", "3.80 | \n", "
| : family | \n", "68.83 | \n", "68.18 | \n", "
| : gram1-adjective-to-adverb | \n", "10.69 | \n", "12.10 | \n", "
| : gram2-opposite | \n", "10.34 | \n", "12.19 | \n", "
| : gram3-comparative | \n", "52.85 | \n", "65.84 | \n", "
| : gram4-superlative | \n", "19.03 | \n", "46.59 | \n", "
| : gram5-present-participle | \n", "30.11 | \n", "53.22 | \n", "
| : gram6-nationality-adjective | \n", "76.20 | \n", "20.05 | \n", "
| : gram7-past-tense | \n", "42.63 | \n", "33.97 | \n", "
| : gram8-plural | \n", "36.64 | \n", "44.22 | \n", "
| : gram9-plural-verbs | \n", "35.86 | \n", "44.48 | \n", "
\n",
"\n",
"\n",
" \n",
"
\n",
"
"
],
"text/plain": [
" word corpus \\\n",
"0 visa Wiki control (same size) \n",
"1 visa Forum (same size) \n",
"2 salary Wiki control (same size) \n",
"3 salary Forum (same size) \n",
"4 doha Wiki control (same size) \n",
"5 doha Forum (same size) \n",
"6 family Wiki control (same size) \n",
"7 family Forum (same size) \n",
"8 king Wiki control (same size) \n",
"9 king Forum (same size) \n",
"10 paris Wiki control (same size) \n",
"11 paris Forum (same size) \n",
"12 computer Wiki control (same size) \n",
"13 computer Forum (same size) \n",
"\n",
" top 5 \n",
"0 visas, passport, passports, bdtc, retransfer \n",
"1 viza, vissa, rp, iqama, residance \n",
"2 salaries, wages, repayment, pension, paid \n",
"3 slary, salry, sallary, salery, salaray \n",
"4 qatar, manama, abbasiyyin, dubai, dhabi \n",
"5 qatar, doah, qata, qatat, london \n",
"6 families, relatives, phyllanthaceae, galte, paternal \n",
"7 familly, fmaily, familiy, wive, rescidence \n",
"8 queen, monarch, prince, vajiravudh, regent \n",
"9 edshel, edsel, buger, queen, katt \n",
"10 universelle, fontainebleau, marseille, peinture, bercy \n",
"11 amsterdam, cancun, tokyo, aires, london \n",
"12 computers, software, computing, imsai, mainframe \n",
"13 soundcard, desktop, pc, computers, laptop "
]
},
"metadata": {},
"output_type": "display_data"
}
],
"source": [
"# [Generative AI] The code in this cell was generated by Claude (Anthropic).\n",
"# 同一個字，在論壇和維基（一樣大小）學到的鄰居差在哪\n",
"rows = []\n",
"for w in [\"visa\", \"salary\", \"doha\", \"family\", \"king\", \"paris\", \"computer\"]:\n",
" for name in [\"Wiki control (same size)\", \"Forum (same size)\"]:\n",
" wv = models[name]\n",
" rows.append({\"word\": w, \"corpus\": name,\n",
" \"top 5\": \", \".join(x for x, _ in wv.most_similar(w, topn=5)) if w in wv.key_to_index else \"(not in vocab)\"})\n",
"pd.set_option(\"display.max_colwidth\", 200)\n",
"display(pd.DataFrame(rows))\n"
]
},
{
"cell_type": "code",
"execution_count": 30,
"id": "6e7f7017",
"metadata": {
"execution": {
"iopub.execute_input": "2026-10-04T15:36:46.747191Z",
"iopub.status.busy": "2026-10-04T15:36:46.747191Z",
"iopub.status.idle": "2026-10-04T15:37:02.339082Z",
"shell.execute_reply": "2026-10-04T15:37:02.339082Z"
}
},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Wiki control (per 1M): 72,920,923 tokens\n"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"Forum (per 1M): 72,919,300 tokens\n"
]
},
{
"data": {
"text/html": [
"| \n", " | word | \n", "corpus | \n", "top 5 | \n", "
|---|---|---|---|
| 0 | \n", "visa | \n", "Wiki control (same size) | \n", "visas, passport, passports, bdtc, retransfer | \n", "
| 1 | \n", "visa | \n", "Forum (same size) | \n", "viza, vissa, rp, iqama, residance | \n", "
| 2 | \n", "salary | \n", "Wiki control (same size) | \n", "salaries, wages, repayment, pension, paid | \n", "
| 3 | \n", "salary | \n", "Forum (same size) | \n", "slary, salry, sallary, salery, salaray | \n", "
| 4 | \n", "doha | \n", "Wiki control (same size) | \n", "qatar, manama, abbasiyyin, dubai, dhabi | \n", "
| 5 | \n", "doha | \n", "Forum (same size) | \n", "qatar, doah, qata, qatat, london | \n", "
| 6 | \n", "family | \n", "Wiki control (same size) | \n", "families, relatives, phyllanthaceae, galte, paternal | \n", "
| 7 | \n", "family | \n", "Forum (same size) | \n", "familly, fmaily, familiy, wive, rescidence | \n", "
| 8 | \n", "king | \n", "Wiki control (same size) | \n", "queen, monarch, prince, vajiravudh, regent | \n", "
| 9 | \n", "king | \n", "Forum (same size) | \n", "edshel, edsel, buger, queen, katt | \n", "
| 10 | \n", "paris | \n", "Wiki control (same size) | \n", "universelle, fontainebleau, marseille, peinture, bercy | \n", "
| 11 | \n", "paris | \n", "Forum (same size) | \n", "amsterdam, cancun, tokyo, aires, london | \n", "
| 12 | \n", "computer | \n", "Wiki control (same size) | \n", "computers, software, computing, imsai, mainframe | \n", "
| 13 | \n", "computer | \n", "Forum (same size) | \n", "soundcard, desktop, pc, computers, laptop | \n", "
\n",
"\n",
"\n",
" \n",
"
\n",
"
"
],
"text/plain": [
" Wiki control (per 1M) Forum (per 1M) forum / wiki\n",
"better 116.98 990.40 8.47\n",
"best 553.01 988.65 1.79\n",
"cheaper 7.78 80.27 10.32\n",
"cheapest 0.96 27.50 28.64\n",
"bigger 13.12 53.58 4.08\n",
"biggest 37.26 59.97 1.61\n",
"looking 56.31 680.74 12.09\n",
"going 104.22 914.05 8.77\n",
"working 187.26 583.48 3.12\n",
"costs 43.99 77.13 1.75\n",
"goes 66.47 235.33 3.54\n",
"wife 212.38 605.50 2.85\n",
"husband 104.85 418.87 3.99\n",
"he 5122.78 3669.63 0.72\n",
"she 1266.80 1885.25 1.49\n",
"capital 160.38 43.23 0.27\n",
"illinois 90.02 1.10 0.01\n",
"albanian 13.37 0.75 0.06\n",
"brazilian 42.43 33.46 0.79\n",
"kwanza 0.19 0.01 0.07"
]
},
"metadata": {},
"output_type": "display_data"
}
],
"source": [
"# [Generative AI] The code in this cell was generated by Claude (Anthropic).\n",
"# 第 3 題補充證據：同樣的字，在兩份一樣大的教材裡各出現幾次（每 100 萬字）。\n",
"from collections import Counter\n",
"\n",
"probe_words = [\"better\", \"best\", \"cheaper\", \"cheapest\", \"bigger\", \"biggest\", \"looking\", \"going\",\n",
" \"working\", \"costs\", \"goes\", \"wife\", \"husband\", \"he\", \"she\",\n",
" \"capital\", \"illinois\", \"albanian\", \"brazilian\", \"kwanza\"]\n",
"wanted = set(probe_words)\n",
"freq = {}\n",
"for name, path in [(\"Wiki control (per 1M)\", \"wiki_control_clean.txt\"), (\"Forum (per 1M)\", \"forum_clean.txt\")]:\n",
" counts, total = Counter(), 0\n",
" with open(path, encoding=\"utf-8\") as f:\n",
" for line in f:\n",
" toks = line.split()\n",
" total += len(toks)\n",
" counts.update(t for t in toks if t in wanted)\n",
" print(f\"{name}: {total:,} tokens\")\n",
" freq[name] = {w: counts[w] / total * 1e6 for w in probe_words}\n",
"table = pd.DataFrame(freq)\n",
"table[\"forum / wiki\"] = table[\"Forum (per 1M)\"] / table[\"Wiki control (per 1M)\"]\n",
"display(table.round(2))\n"
]
},
{
"cell_type": "markdown",
"id": "c0fc5893",
"metadata": {},
"source": [
"## Extra code for report question 2 (5% / 10% / 20%)"
]
},
{
"cell_type": "code",
"execution_count": 31,
"id": "378fabb8",
"metadata": {
"execution": {
"iopub.execute_input": "2026-10-04T15:37:02.486269Z",
"iopub.status.busy": "2026-10-04T15:37:02.486269Z",
"iopub.status.idle": "2026-10-04T16:16:07.368340Z",
"shell.execute_reply": "2026-10-04T16:16:07.367336Z"
}
},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"wiki_sampled_10.txt: removed non-[a-z] tokens 2,417,608 of 335,470,319 (0.72%)\n",
"10% wiki tokens: 333,052,711\n"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"trained on wiki_sampled_5_clean.txt: 11.3 min, vocab 278,954, effective min_count 9\n"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"Wiki 5%: overall 46.31% (OOV questions 107)\n"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"trained on wiki_sampled_10_clean.txt: 22.4 min, vocab 294,612, effective min_count 16\n"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"Wiki 10%: overall 47.11% (OOV questions 0)\n"
]
},
{
"data": {
"text/html": [
"| \n", " | Wiki control (per 1M) | \n", "Forum (per 1M) | \n", "forum / wiki | \n", "
|---|---|---|---|
| better | \n", "116.98 | \n", "990.40 | \n", "8.47 | \n", "
| best | \n", "553.01 | \n", "988.65 | \n", "1.79 | \n", "
| cheaper | \n", "7.78 | \n", "80.27 | \n", "10.32 | \n", "
| cheapest | \n", "0.96 | \n", "27.50 | \n", "28.64 | \n", "
| bigger | \n", "13.12 | \n", "53.58 | \n", "4.08 | \n", "
| biggest | \n", "37.26 | \n", "59.97 | \n", "1.61 | \n", "
| looking | \n", "56.31 | \n", "680.74 | \n", "12.09 | \n", "
| going | \n", "104.22 | \n", "914.05 | \n", "8.77 | \n", "
| working | \n", "187.26 | \n", "583.48 | \n", "3.12 | \n", "
| costs | \n", "43.99 | \n", "77.13 | \n", "1.75 | \n", "
| goes | \n", "66.47 | \n", "235.33 | \n", "3.54 | \n", "
| wife | \n", "212.38 | \n", "605.50 | \n", "2.85 | \n", "
| husband | \n", "104.85 | \n", "418.87 | \n", "3.99 | \n", "
| he | \n", "5122.78 | \n", "3669.63 | \n", "0.72 | \n", "
| she | \n", "1266.80 | \n", "1885.25 | \n", "1.49 | \n", "
| capital | \n", "160.38 | \n", "43.23 | \n", "0.27 | \n", "
| illinois | \n", "90.02 | \n", "1.10 | \n", "0.01 | \n", "
| albanian | \n", "13.37 | \n", "0.75 | \n", "0.06 | \n", "
| brazilian | \n", "42.43 | \n", "33.46 | \n", "0.79 | \n", "
| kwanza | \n", "0.19 | \n", "0.01 | \n", "0.07 | \n", "
\n",
"\n",
"\n",
" \n",
"
\n",
"
"
],
"text/plain": [
" Wiki 5% Wiki 10% Wiki 20%\n",
"Overall 46.31 47.11 47.52\n",
"Semantic 55.65 54.36 53.10\n",
"Syntactic 38.55 41.10 42.89\n",
": capital-common-countries 86.36 73.52 74.31\n",
": capital-world 69.81 69.98 69.94\n",
": currency 6.81 7.85 10.39\n",
": city-in-state 37.82 34.50 27.77\n",
": family 68.97 71.94 77.87\n",
": gram1-adjective-to-adverb 13.21 11.29 17.84\n",
": gram2-opposite 12.07 14.04 15.02\n",
": gram3-comparative 45.42 49.10 48.80\n",
": gram4-superlative 18.18 22.82 29.23\n",
": gram5-present-participle 29.07 34.75 32.95\n",
": gram6-nationality-adjective 76.55 76.74 82.93\n",
": gram7-past-tense 43.53 44.81 41.79\n",
": gram8-plural 39.11 43.47 44.67\n",
": gram9-plural-verbs 39.77 43.56 43.68\n",
"OOV questions 107.00 0.00 0.00"
]
},
"metadata": {},
"output_type": "display_data"
}
],
"source": [
"# [Generative AI] The code in this cell was generated by Claude (Anthropic).\n",
"# 第 2 題：5%、10%、20%（助教說這題不用附程式，這裡附上供參考）\n",
"n10 = preprocess_file(\"wiki_sampled_10.txt\", \"wiki_sampled_10_clean.txt\")\n",
"print(f\"10% wiki tokens: {n10:,}\")\n",
"for pct in [5, 10]:\n",
" m = train_w2v(f\"wiki_sampled_{pct}_clean.txt\")\n",
" models[f\"Wiki {pct}%\"] = m.wv\n",
" _ = evaluate(m.wv, f\"Wiki {pct}%\")\n",
"display(pd.DataFrame({k: results[k] for k in [\"Wiki 5%\", \"Wiki 10%\", \"Wiki 20%\"]}).round(2))\n"
]
},
{
"cell_type": "markdown",
"id": "25af9f04",
"metadata": {},
"source": [
"## Extra code for report question 4 (most similar words)"
]
},
{
"cell_type": "code",
"execution_count": 32,
"id": "721331b4",
"metadata": {
"execution": {
"iopub.execute_input": "2026-10-04T16:16:07.469292Z",
"iopub.status.busy": "2026-10-04T16:16:07.469292Z",
"iopub.status.idle": "2026-10-04T16:16:07.606960Z",
"shell.execute_reply": "2026-10-04T16:16:07.606960Z"
}
},
"outputs": [
{
"data": {
"text/html": [
"| \n", " | Wiki 5% | \n", "Wiki 10% | \n", "Wiki 20% | \n", "
|---|---|---|---|
| Overall | \n", "46.31 | \n", "47.11 | \n", "47.52 | \n", "
| Semantic | \n", "55.65 | \n", "54.36 | \n", "53.10 | \n", "
| Syntactic | \n", "38.55 | \n", "41.10 | \n", "42.89 | \n", "
| : capital-common-countries | \n", "86.36 | \n", "73.52 | \n", "74.31 | \n", "
| : capital-world | \n", "69.81 | \n", "69.98 | \n", "69.94 | \n", "
| : currency | \n", "6.81 | \n", "7.85 | \n", "10.39 | \n", "
| : city-in-state | \n", "37.82 | \n", "34.50 | \n", "27.77 | \n", "
| : family | \n", "68.97 | \n", "71.94 | \n", "77.87 | \n", "
| : gram1-adjective-to-adverb | \n", "13.21 | \n", "11.29 | \n", "17.84 | \n", "
| : gram2-opposite | \n", "12.07 | \n", "14.04 | \n", "15.02 | \n", "
| : gram3-comparative | \n", "45.42 | \n", "49.10 | \n", "48.80 | \n", "
| : gram4-superlative | \n", "18.18 | \n", "22.82 | \n", "29.23 | \n", "
| : gram5-present-participle | \n", "29.07 | \n", "34.75 | \n", "32.95 | \n", "
| : gram6-nationality-adjective | \n", "76.55 | \n", "76.74 | \n", "82.93 | \n", "
| : gram7-past-tense | \n", "43.53 | \n", "44.81 | \n", "41.79 | \n", "
| : gram8-plural | \n", "39.11 | \n", "43.47 | \n", "44.67 | \n", "
| : gram9-plural-verbs | \n", "39.77 | \n", "43.56 | \n", "43.68 | \n", "
| OOV questions | \n", "107.00 | \n", "0.00 | \n", "0.00 | \n", "
\n",
"\n",
"\n",
" \n",
"
\n",
"
"
],
"text/plain": [
" word model \\\n",
"0 king GloVe (pre-trained) \n",
"1 king Wiki 20% \n",
"2 cat GloVe (pre-trained) \n",
"3 cat Wiki 20% \n",
"4 apple GloVe (pre-trained) \n",
"5 apple Wiki 20% \n",
"6 bank GloVe (pre-trained) \n",
"7 bank Wiki 20% \n",
"8 good GloVe (pre-trained) \n",
"9 good Wiki 20% \n",
"10 taiwan GloVe (pre-trained) \n",
"11 taiwan Wiki 20% \n",
"12 computer GloVe (pre-trained) \n",
"13 computer Wiki 20% \n",
"\n",
" top 5 (similarity) \n",
"0 prince (0.77), queen (0.75), son (0.70), brother (0.70), monarch (0.70) \n",
"1 pretender (0.73), queen (0.71), prince (0.71), nangklao (0.69), uncrowned (0.68) \n",
"2 dog (0.88), rabbit (0.74), cats (0.73), monkey (0.73), pet (0.72) \n",
"3 rabbit (0.81), mouse (0.79), dog (0.79), sourpuss (0.77), kitten (0.76) \n",
"4 microsoft (0.74), ibm (0.68), intel (0.68), software (0.68), dell (0.67) \n",
"5 blackberry (0.81), tvos (0.73), magix (0.71), iphone (0.71), webos (0.71) \n",
"6 banks (0.81), banking (0.75), credit (0.70), investment (0.69), financial (0.68) \n",
"7 bayernlb (0.76), citibank (0.75), disconto (0.75), comdirect (0.74), banking (0.74) \n",
"8 better (0.89), sure (0.83), really (0.83), kind (0.83), very (0.83) \n",
"9 natured (0.74), luck (0.73), thankful (0.72), funny (0.72), bad (0.72) \n",
"10 mainland (0.86), china (0.83), taiwanese (0.79), taipei (0.79), hong (0.77) \n",
"11 taipei (0.84), guangdong (0.82), hainan (0.82), china (0.80), fujian (0.78) \n",
"12 computers (0.88), software (0.84), technology (0.76), pc (0.74), hardware (0.73) \n",
"13 computing (0.80), computers (0.80), software (0.75), mainframe (0.75), multimedia (0.74) "
]
},
"metadata": {},
"output_type": "display_data"
}
],
"source": [
"# [Generative AI] The code in this cell was generated by Claude (Anthropic).\n",
"# 第 4 題：挑字，各找 5 個最像的字。同時看現成的 GloVe 和自己訓練的 Wiki 20%。\n",
"# 挑字的原則：一字多義（apple、bank）、反義詞（good）、專有名詞（taiwan）、一般名詞（cat、computer）、考卷裡的字（king）\n",
"# 注意：兩個模型的 cosine 分數尺度不同，只能比「同一個模型內的排名」，不能拿分數互相比大小。\n",
"probe = [\"king\", \"cat\", \"apple\", \"bank\", \"good\", \"taiwan\", \"computer\"]\n",
"rows = []\n",
"for w in probe:\n",
" for name in [\"GloVe (pre-trained)\", \"Wiki 20%\"]:\n",
" wv = models[name]\n",
" if w in wv.key_to_index:\n",
" rows.append({\"word\": w, \"model\": name,\n",
" \"top 5 (similarity)\": \", \".join(f\"{x} ({s:.2f})\" for x, s in wv.most_similar(w, topn=5))})\n",
"pd.set_option(\"display.max_colwidth\", 200)\n",
"display(pd.DataFrame(rows))\n"
]
},
{
"cell_type": "code",
"execution_count": 33,
"id": "c78e6bfe",
"metadata": {
"execution": {
"iopub.execute_input": "2026-10-04T16:16:07.722769Z",
"iopub.status.busy": "2026-10-04T16:16:07.722769Z",
"iopub.status.idle": "2026-10-04T16:16:08.654361Z",
"shell.execute_reply": "2026-10-04T16:16:08.654361Z"
}
},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"king -> [('pretender', np.int64(1237)), ('queen', np.int64(92609)), ('prince', np.int64(91842)), ('nangklao', np.int64(34)), ('uncrowned', np.int64(90))]\n",
"cat -> [('rabbit', np.int64(7844)), ('mouse', np.int64(13524)), ('dog', np.int64(36766)), ('sourpuss', np.int64(64)), ('kitten', np.int64(1159))]\n",
"apple -> [('blackberry', np.int64(1285)), ('tvos', np.int64(102)), ('magix', np.int64(36)), ('iphone', np.int64(2800)), ('webos', np.int64(60))]\n",
"bank -> [('bayernlb', np.int64(44)), ('citibank', np.int64(501)), ('disconto', np.int64(48)), ('comdirect', np.int64(31)), ('banking', np.int64(14185))]\n"
]
},
{
"data": {
"text/html": [
"| \n", " | word | \n", "model | \n", "top 5 (similarity) | \n", "
|---|---|---|---|
| 0 | \n", "king | \n", "GloVe (pre-trained) | \n", "prince (0.77), queen (0.75), son (0.70), brother (0.70), monarch (0.70) | \n", "
| 1 | \n", "king | \n", "Wiki 20% | \n", "pretender (0.73), queen (0.71), prince (0.71), nangklao (0.69), uncrowned (0.68) | \n", "
| 2 | \n", "cat | \n", "GloVe (pre-trained) | \n", "dog (0.88), rabbit (0.74), cats (0.73), monkey (0.73), pet (0.72) | \n", "
| 3 | \n", "cat | \n", "Wiki 20% | \n", "rabbit (0.81), mouse (0.79), dog (0.79), sourpuss (0.77), kitten (0.76) | \n", "
| 4 | \n", "apple | \n", "GloVe (pre-trained) | \n", "microsoft (0.74), ibm (0.68), intel (0.68), software (0.68), dell (0.67) | \n", "
| 5 | \n", "apple | \n", "Wiki 20% | \n", "blackberry (0.81), tvos (0.73), magix (0.71), iphone (0.71), webos (0.71) | \n", "
| 6 | \n", "bank | \n", "GloVe (pre-trained) | \n", "banks (0.81), banking (0.75), credit (0.70), investment (0.69), financial (0.68) | \n", "
| 7 | \n", "bank | \n", "Wiki 20% | \n", "bayernlb (0.76), citibank (0.75), disconto (0.75), comdirect (0.74), banking (0.74) | \n", "
| 8 | \n", "good | \n", "GloVe (pre-trained) | \n", "better (0.89), sure (0.83), really (0.83), kind (0.83), very (0.83) | \n", "
| 9 | \n", "good | \n", "Wiki 20% | \n", "natured (0.74), luck (0.73), thankful (0.72), funny (0.72), bad (0.72) | \n", "
| 10 | \n", "taiwan | \n", "GloVe (pre-trained) | \n", "mainland (0.86), china (0.83), taiwanese (0.79), taipei (0.79), hong (0.77) | \n", "
| 11 | \n", "taiwan | \n", "Wiki 20% | \n", "taipei (0.84), guangdong (0.82), hainan (0.82), china (0.80), fujian (0.78) | \n", "
| 12 | \n", "computer | \n", "GloVe (pre-trained) | \n", "computers (0.88), software (0.84), technology (0.76), pc (0.74), hardware (0.73) | \n", "
| 13 | \n", "computer | \n", "Wiki 20% | \n", "computing (0.80), computers (0.80), software (0.75), mainframe (0.75), multimedia (0.74) | \n", "
\n",
"\n",
"\n",
" \n",
"
\n",
"
"
],
"text/plain": [
" word top 5 among the 50k most frequent words\n",
"0 king pretender, queen, prince, throne, usurper\n",
"1 cat rabbit, mouse, dog, kitten, hound\n",
"2 apple blackberry, iphone, raspberry, microsoft, ipad\n",
"3 bank citibank, banking, citigroup, savings, banks\n",
"4 good natured, luck, thankful, funny, bad\n",
"5 taiwan taipei, guangdong, hainan, china, fujian\n",
"6 computer computing, computers, software, mainframe, multimedia"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"apple top 15: ['blackberry', 'tvos', 'magix', 'iphone', 'webos', 'raspberry', 'kitkat', 'slackware', 'iigs', 'meego', 'viewsonic', 'cdrom', 'lineageos', 'froyo', 'adafruit']\n",
"similarity(apple, banana) = 0.55\n",
"similarity(apple, fruit) = 0.42\n",
"similarity(apple, microsoft) = 0.67\n",
"similarity(bank, river) = 0.46\n",
"similarity(bank, money) = 0.51\n",
"similarity(good, bad) = 0.72\n"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"... of guard dog bushy doozy best friend and hero to tom tom ms lulu the pet psychic sourpuss owner sourpuss owner and friend of millie animals bip and bop two nutty squirrel ...\n",
"... toon characters including fanny zilch mighty mouse heckle and jeckle gandy goose sourpuss dinky duck little roquefort the terry bears dimwit and luno terry pre existing c ...\n",
"... k puck in hockey almost certainly from irish poc according to the oed puss as in sourpuss comes from irish pus pouting mouth rapparee an irish highwayman from ropaire sta ...\n",
"... n year title role notes hangin in episode the princess and the pea today special sourpuss sal episode smiles murielle episode three monkeys of bah roghar part episode thr ...\n",
"... ack flag discharge and flipper in few days before her th birthday her first band sourpuss played set at australia summersault festival where she met tim armstrong frontma ...\n"
]
}
],
"source": [
"# [Generative AI] The code in this cell was generated by Claude (Anthropic).\n",
"# 第 4 題補充證據\n",
"# (1) Wiki 20% 的鄰居裡常有冷門字：印出每個鄰居在 20% 維基裡出現幾次（字典門檻見 TODO5 的 effective min_count）\n",
"for w in [\"king\", \"cat\", \"apple\", \"bank\"]:\n",
" print(w, \"->\", [(x, my_wv.get_vecattr(x, \"count\")) for x, _ in my_wv.most_similar(w, topn=5)])\n",
"\n",
"# (2) 只在最常見的 5 萬個字裡找鄰居（restrict_vocab），冷門字就被排除\n",
"rows = [{\"word\": w, \"top 5 among the 50k most frequent words\":\n",
" \", \".join(x for x, _ in my_wv.most_similar(w, topn=5, restrict_vocab=50000))} for w in probe]\n",
"display(pd.DataFrame(rows))\n",
"\n",
"# (3) 一字多義：apple 的前 15 名鄰居，以及幾組相似度\n",
"print(\"apple top 15:\", [x for x, _ in my_wv.most_similar(\"apple\", topn=15)])\n",
"for a, b in [(\"apple\", \"banana\"), (\"apple\", \"fruit\"), (\"apple\", \"microsoft\"),\n",
" (\"bank\", \"river\"), (\"bank\", \"money\"), (\"good\", \"bad\")]:\n",
" print(f\"similarity({a}, {b}) = {my_wv.similarity(a, b):.2f}\")\n",
"\n",
"# (4) 回到教材查 cat 的鄰居 sourpuss 是什麼：印出前 5 處上下文\n",
"found = []\n",
"with open(\"wiki_sampled_20_clean.txt\", encoding=\"utf-8\") as f:\n",
" for line in f:\n",
" i = line.find(\" sourpuss \")\n",
" if i >= 0:\n",
" found.append(line[max(0, i - 80): i + 90].strip())\n",
" if len(found) == 5:\n",
" break\n",
"for s in found:\n",
" print(\"...\", s, \"...\")\n"
]
},
{
"cell_type": "markdown",
"id": "7909bc7b",
"metadata": {},
"source": [
"## Extra code for report question 5 (anything that strengthens the report)"
]
},
{
"cell_type": "code",
"execution_count": 34,
"id": "3b841e6a",
"metadata": {
"execution": {
"iopub.execute_input": "2026-10-04T16:16:08.769786Z",
"iopub.status.busy": "2026-10-04T16:16:08.769786Z",
"iopub.status.idle": "2026-10-04T16:50:50.583040Z",
"shell.execute_reply": "2026-10-04T16:50:50.569633Z"
}
},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Wiki 5% seed=1: overall 45.53% (OOV questions 107)\n"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"Wiki 5% seed=2: overall 46.12% (OOV questions 107)\n"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"Wiki 5% seed=3: overall 45.85% (OOV questions 107)\n"
]
},
{
"data": {
"text/html": [
"| \n", " | word | \n", "top 5 among the 50k most frequent words | \n", "
|---|---|---|
| 0 | \n", "king | \n", "pretender, queen, prince, throne, usurper | \n", "
| 1 | \n", "cat | \n", "rabbit, mouse, dog, kitten, hound | \n", "
| 2 | \n", "apple | \n", "blackberry, iphone, raspberry, microsoft, ipad | \n", "
| 3 | \n", "bank | \n", "citibank, banking, citigroup, savings, banks | \n", "
| 4 | \n", "good | \n", "natured, luck, thankful, funny, bad | \n", "
| 5 | \n", "taiwan | \n", "taipei, guangdong, hainan, china, fujian | \n", "
| 6 | \n", "computer | \n", "computing, computers, software, mainframe, multimedia | \n", "
\n",
"\n",
"\n",
" \n",
"
\n",
"
"
],
"text/plain": [
" seed 42 seed 1 seed 2 seed 3 max - min\n",
"Overall 46.31 45.53 46.12 45.85 0.78\n",
"Semantic 55.65 53.43 54.64 54.29 2.22\n",
"Syntactic 38.55 38.96 39.04 38.84 0.49\n",
": capital-common-countries 86.36 81.23 81.62 82.61 5.14\n",
": capital-world 69.81 66.89 67.29 67.13 2.92\n",
": currency 6.81 7.62 8.20 8.43 1.62\n",
": city-in-state 37.82 35.95 39.48 36.85 3.53\n",
": family 68.97 68.97 67.98 74.70 6.72\n",
": gram1-adjective-to-adverb 13.21 12.40 12.50 11.79 1.41\n",
": gram2-opposite 12.07 10.22 9.98 12.19 2.22\n",
": gram3-comparative 45.42 49.17 44.29 49.55 5.26\n",
": gram4-superlative 18.18 20.77 15.60 19.07 5.17\n",
": gram5-present-participle 29.07 28.03 33.24 30.02 5.21\n",
": gram6-nationality-adjective 76.55 78.42 80.55 79.36 4.00\n",
": gram7-past-tense 43.53 44.55 46.73 40.38 6.35\n",
": gram8-plural 39.11 36.71 36.79 38.51 2.40\n",
": gram9-plural-verbs 39.77 38.05 38.97 37.59 2.18"
]
},
"metadata": {},
"output_type": "display_data"
}
],
"source": [
"# [Generative AI] The code in this cell was generated by Claude (Anthropic).\n",
"# 第 5 題 (a)：同樣設定、只換亂數種子，結果會晃多少？——先量雜訊，後面的比較才知道哪些差距算數。\n",
"# 用 5% 維基（後面的比較實驗都用 5%）：原本的 seed=42 再加 seed=1、2、3，共 4 次。\n",
"noise_runs = {\"seed 42\": results[\"Wiki 5%\"]}\n",
"for s in [1, 2, 3]:\n",
" m = Word2Vec(corpus_file=\"wiki_sampled_5_clean.txt\", **{**W2V_PARAMS, \"seed\": s})\n",
" _ = evaluate(m.wv, f\"Wiki 5% seed={s}\")\n",
" noise_runs[f\"seed {s}\"] = results[f\"Wiki 5% seed={s}\"]\n",
"noise = pd.DataFrame(noise_runs).drop(index=\"OOV questions\")\n",
"noise[\"max - min\"] = noise.max(axis=1) - noise.min(axis=1)\n",
"display(noise.round(2))\n",
"NOISE = noise[\"max - min\"]\n",
"\n",
"def compare(base, other):\n",
" # 跟 base 比較；差距的絕對值大於「4 次重跑的最大落差」才標 True\n",
" t = pd.DataFrame({base: results[base], other: results[other]}).drop(index=\"OOV questions\")\n",
" t[\"diff\"] = t[other] - t[base]\n",
" t[\"noise (max-min of 4 seeds)\"] = NOISE\n",
" t[\"beyond noise?\"] = t[\"diff\"].abs() > t[\"noise (max-min of 4 seeds)\"]\n",
" return t.round(2)\n"
]
},
{
"cell_type": "code",
"execution_count": 35,
"id": "3cede143",
"metadata": {
"execution": {
"iopub.execute_input": "2026-10-04T16:50:50.747434Z",
"iopub.status.busy": "2026-10-04T16:50:50.745920Z",
"iopub.status.idle": "2026-10-04T17:01:03.900669Z",
"shell.execute_reply": "2026-10-04T17:01:03.900669Z"
}
},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"questions containing a stop word: 206\n",
"SubCategory\n",
": family 86\n",
": gram1-adjective-to-adverb 62\n",
": currency 58\n",
"Name: count, dtype: int64\n",
"['Algeria dinar Korea won', 'Angola kwanza Korea won', 'Argentina peso Korea won', 'Armenia dram Korea won', 'Brazil real Korea won']\n"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"trained on wiki_sampled_5_nostop.txt: 8.2 min, vocab 278,810, effective min_count 9\n"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"Wiki 5% (stop words removed): overall 44.73% (OOV questions 284)\n"
]
},
{
"data": {
"text/html": [
"| \n", " | seed 42 | \n", "seed 1 | \n", "seed 2 | \n", "seed 3 | \n", "max - min | \n", "
|---|---|---|---|---|---|
| Overall | \n", "46.31 | \n", "45.53 | \n", "46.12 | \n", "45.85 | \n", "0.78 | \n", "
| Semantic | \n", "55.65 | \n", "53.43 | \n", "54.64 | \n", "54.29 | \n", "2.22 | \n", "
| Syntactic | \n", "38.55 | \n", "38.96 | \n", "39.04 | \n", "38.84 | \n", "0.49 | \n", "
| : capital-common-countries | \n", "86.36 | \n", "81.23 | \n", "81.62 | \n", "82.61 | \n", "5.14 | \n", "
| : capital-world | \n", "69.81 | \n", "66.89 | \n", "67.29 | \n", "67.13 | \n", "2.92 | \n", "
| : currency | \n", "6.81 | \n", "7.62 | \n", "8.20 | \n", "8.43 | \n", "1.62 | \n", "
| : city-in-state | \n", "37.82 | \n", "35.95 | \n", "39.48 | \n", "36.85 | \n", "3.53 | \n", "
| : family | \n", "68.97 | \n", "68.97 | \n", "67.98 | \n", "74.70 | \n", "6.72 | \n", "
| : gram1-adjective-to-adverb | \n", "13.21 | \n", "12.40 | \n", "12.50 | \n", "11.79 | \n", "1.41 | \n", "
| : gram2-opposite | \n", "12.07 | \n", "10.22 | \n", "9.98 | \n", "12.19 | \n", "2.22 | \n", "
| : gram3-comparative | \n", "45.42 | \n", "49.17 | \n", "44.29 | \n", "49.55 | \n", "5.26 | \n", "
| : gram4-superlative | \n", "18.18 | \n", "20.77 | \n", "15.60 | \n", "19.07 | \n", "5.17 | \n", "
| : gram5-present-participle | \n", "29.07 | \n", "28.03 | \n", "33.24 | \n", "30.02 | \n", "5.21 | \n", "
| : gram6-nationality-adjective | \n", "76.55 | \n", "78.42 | \n", "80.55 | \n", "79.36 | \n", "4.00 | \n", "
| : gram7-past-tense | \n", "43.53 | \n", "44.55 | \n", "46.73 | \n", "40.38 | \n", "6.35 | \n", "
| : gram8-plural | \n", "39.11 | \n", "36.71 | \n", "36.79 | \n", "38.51 | \n", "2.40 | \n", "
| : gram9-plural-verbs | \n", "39.77 | \n", "38.05 | \n", "38.97 | \n", "37.59 | \n", "2.18 | \n", "
\n",
"\n",
"\n",
" \n",
"
\n",
"
"
],
"text/plain": [
" Wiki 5% Wiki 5% (stop words removed) diff \\\n",
"Overall 46.31 44.73 -1.58 \n",
"Semantic 55.65 53.74 -1.92 \n",
"Syntactic 38.55 37.25 -1.30 \n",
": capital-common-countries 86.36 81.42 -4.94 \n",
": capital-world 69.81 70.05 0.24 \n",
": currency 6.81 7.74 0.92 \n",
": city-in-state 37.82 34.41 -3.40 \n",
": family 68.97 53.16 -15.81 \n",
": gram1-adjective-to-adverb 13.21 15.22 2.02 \n",
": gram2-opposite 12.07 8.99 -3.08 \n",
": gram3-comparative 45.42 37.16 -8.26 \n",
": gram4-superlative 18.18 15.33 -2.85 \n",
": gram5-present-participle 29.07 33.52 4.45 \n",
": gram6-nationality-adjective 76.55 79.80 3.25 \n",
": gram7-past-tense 43.53 41.79 -1.73 \n",
": gram8-plural 39.11 42.57 3.45 \n",
": gram9-plural-verbs 39.77 27.13 -12.64 \n",
"\n",
" noise (max-min of 4 seeds) beyond noise? \n",
"Overall 0.78 True \n",
"Semantic 2.22 False \n",
"Syntactic 0.49 True \n",
": capital-common-countries 5.14 False \n",
": capital-world 2.92 False \n",
": currency 1.62 False \n",
": city-in-state 3.53 False \n",
": family 6.72 True \n",
": gram1-adjective-to-adverb 1.41 True \n",
": gram2-opposite 2.22 True \n",
": gram3-comparative 5.26 True \n",
": gram4-superlative 5.17 False \n",
": gram5-present-participle 5.21 False \n",
": gram6-nationality-adjective 4.00 False \n",
": gram7-past-tense 6.35 False \n",
": gram8-plural 2.40 True \n",
": gram9-plural-verbs 2.18 True "
]
},
"metadata": {},
"output_type": "display_data"
}
],
"source": [
"# [Generative AI] The code in this cell was generated by Claude (Anthropic).\n",
"# 第 5 題 (b)：為什麼不移除停用詞？先看考卷，再真的做實驗比較。\n",
"# 停用詞表：NLTK English stopwords（https://www.nltk.org/）\n",
"import nltk\n",
"nltk.download(\"stopwords\", quiet=True)\n",
"from nltk.corpus import stopwords\n",
"sw = set(stopwords.words(\"english\"))\n",
"has_sw = data[\"Question\"].map(lambda q: any(w in sw for w in q.lower().split()))\n",
"print(\"questions containing a stop word:\", int(has_sw.sum()))\n",
"print(data[has_sw][\"SubCategory\"].value_counts())\n",
"print(data[has_sw][\"Question\"].head(5).tolist())\n",
"\n",
"# 實驗：同樣是 5% 維基、同樣的參數，只差在「有沒有移除停用詞」\n",
"with open(\"wiki_sampled_5_clean.txt\", encoding=\"utf-8\") as fin, open(\"wiki_sampled_5_nostop.txt\", \"w\", encoding=\"utf-8\", newline=\"\\n\") as fout:\n",
" for line in fin:\n",
" fout.write(\" \".join(w for w in line.split() if w not in sw) + \"\\n\")\n",
"m = train_w2v(\"wiki_sampled_5_nostop.txt\")\n",
"models[\"Wiki 5% (stop words removed)\"] = m.wv\n",
"_ = evaluate(m.wv, \"Wiki 5% (stop words removed)\")\n",
"display(compare(\"Wiki 5%\", \"Wiki 5% (stop words removed)\"))\n"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "7170fc4c",
"metadata": {},
"outputs": [],
"source": [
"# [Generative AI] The code in this cell was generated by Claude (Anthropic).\n",
"# 第 5 題 (c)：超參數比較（都用 5% 維基，一次只改一個設定，其他跟主模型一樣）\n",
"variants = {\n",
" \"Wiki 5% CBOW (sg=0)\": dict(sg=0),\n",
" \"Wiki 5% window=10\": dict(window=10),\n",
" \"Wiki 5% vector_size=300\": dict(vector_size=300),\n",
"}\n",
"for name, change in variants.items():\n",
" t = time.time()\n",
" m = Word2Vec(corpus_file=\"wiki_sampled_5_clean.txt\", **{**W2V_PARAMS, **change})\n",
" print(f\"{name}: trained in {(time.time() - t) / 60:.1f} min\")\n",
" models[name] = m.wv\n",
" _ = evaluate(m.wv, name)\n",
"for name in variants:\n",
" display(compare(\"Wiki 5%\", name))\n"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "19d1540a",
"metadata": {},
"outputs": [],
"source": [
"# [Generative AI] The code in this cell was generated by Claude (Anthropic).\n",
"# 第 5 題 (d)：前 k 名有猜中就算對，正確率會變多少？（老師說 queen 可能不是第一名，而是第二、三名）\n",
"rows = {}\n",
"for name in [\"GloVe (pre-trained)\", \"Wiki 20%\"]:\n",
" ranks = details[name][\"ranks\"]\n",
" rows[name] = {f\"top-{k}\": np.mean([r is not None and r <= k for r in ranks]) * 100 for k in [1, 3, 5, 10]}\n",
"display(pd.DataFrame(rows).T.round(2))\n"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "0f3fe91a",
"metadata": {},
"outputs": [],
"source": [
"# [Generative AI] The code in this cell was generated by Claude (Anthropic).\n",
"# 第 5 題 (e)：錯題分析——電腦答錯時，它到底猜了什麼？\n",
"d20 = details[\"Wiki 20%\"]\n",
"err = data.assign(pred=d20[\"preds\"], rank=d20[\"ranks\"],\n",
" gold=data[\"Question\"].str.lower().str.split().str[3])\n",
"wrong = err[err[\"pred\"] != err[\"gold\"]]\n",
"print(\"wrong answers:\", len(wrong), \"of\", len(err))\n",
"print(\" OOV (question word not in vocab):\", int(wrong[\"pred\"].isna().sum()))\n",
"print(\" gold answer was 2nd-10th:\", int(wrong[\"rank\"].notna().sum()))\n",
"for sub in [\": capital-world\", \": currency\", \": family\", \": gram3-comparative\", \": gram6-nationality-adjective\", \": gram8-plural\"]:\n",
" print(\"\\n\" + sub)\n",
" display(wrong[(wrong[\"SubCategory\"] == sub) & wrong[\"pred\"].notna()][[\"Question\", \"gold\", \"pred\", \"rank\"]].head(6))\n"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "fa29c7ef",
"metadata": {},
"outputs": [],
"source": [
"# [Generative AI] The code in this cell was generated by Claude (Anthropic).\n",
"# 第 5 題 (f)：換一種「算答案」的公式。3CosAdd 是 b - a + c；3CosMul 用乘除。\n",
"# 出處：Levy & Goldberg (2014), Linguistic Regularities in Sparse and Explicit Word Representations, CoNLL. https://aclanthology.org/W14-1618/\n",
"# 同一份 Wiki 20% 詞向量，只換公式。\n",
"_ = evaluate(my_wv, \"Wiki 20% (3CosMul)\", method=\"cosmul\")\n",
"display(pd.DataFrame({k: results[k] for k in [\"Wiki 20%\", \"Wiki 20% (3CosMul)\"]}).round(2))\n"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "dec8ef24",
"metadata": {},
"outputs": [],
"source": [
"# [Generative AI] The code in this cell was generated by Claude (Anthropic).\n",
"# 第 5 題 (g)：同一批 family 字，用 PCA 和 t-SNE 兩種方法壓成 2 維，圖長得不一樣嗎？\n",
"from sklearn.decomposition import PCA\n",
"words = sorted({w for q in data[data[\"SubCategory\"] == \": family\"][\"Question\"] for w in q.lower().split()})\n",
"words = [w for w in words if w in my_wv.key_to_index]\n",
"vecs = np.array([my_wv[w] for w in words])\n",
"fig, axes = plt.subplots(1, 2, figsize=(18, 8))\n",
"for ax, (title, pts) in zip(axes, [\n",
" (\"PCA\", PCA(n_components=2, random_state=42).fit_transform(vecs)),\n",
" (\"t-SNE\", TSNE(n_components=2, perplexity=15, random_state=42, init=\"pca\").fit_transform(vecs))]):\n",
" ax.scatter(pts[:, 0], pts[:, 1], s=12)\n",
" for (x, y), w in zip(pts, words):\n",
" ax.annotate(w, (x, y), fontsize=8)\n",
" ax.set_title(f\"{title} (Wiki 20%, family)\")\n",
"plt.show()\n"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "17f0e5a4",
"metadata": {},
"outputs": [],
"source": [
"# [Generative AI] The code in this cell was generated by Claude (Anthropic).\n",
"# 第 5 題 (h)：各小類「四個字都在字典裡」的比例：GloVe vs Wiki 20%\n",
"display(pd.DataFrame({\"GloVe\": answer_coverage(model), \"Wiki 20%\": answer_coverage(my_wv)}))\n"
]
},
{
"cell_type": "markdown",
"id": "e399f54b",
"metadata": {},
"source": [
"## Summary of all results"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "8f5ce88f",
"metadata": {},
"outputs": [],
"source": [
"# [Generative AI] The code in this cell was generated by Claude (Anthropic).\n",
"# 全部結果總表（報告引用的數字都來自這裡）\n",
"pd.set_option(\"display.max_columns\", 40)\n",
"display(pd.DataFrame(results).round(2))\n"
]
},
{
"cell_type": "markdown",
"id": "8df6a919",
"metadata": {},
"source": [
"## Running environment"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "06394f3b",
"metadata": {},
"outputs": [],
"source": [
"# [Generative AI] The code in this cell was generated by Claude (Anthropic).\n",
"import sys, platform, gensim, sklearn\n",
"print(\"Python\", sys.version)\n",
"print(platform.platform(), \"|\", platform.processor())\n",
"print(\"gensim\", gensim.__version__, \"| numpy\", np.__version__, \"| pandas\", pd.__version__, \"| scikit-learn\", sklearn.__version__)\n"
]
}
],
"metadata": {
"colab": {
"provenance": []
},
"kernelspec": {
"display_name": "Python 3",
"language": "python",
"name": "python3"
},
"language_info": {
"name": "python"
}
},
"nbformat": 4,
"nbformat_minor": 5
}
| \n", " | Wiki 5% | \n", "Wiki 5% (stop words removed) | \n", "diff | \n", "noise (max-min of 4 seeds) | \n", "beyond noise? | \n", "
|---|---|---|---|---|---|
| Overall | \n", "46.31 | \n", "44.73 | \n", "-1.58 | \n", "0.78 | \n", "True | \n", "
| Semantic | \n", "55.65 | \n", "53.74 | \n", "-1.92 | \n", "2.22 | \n", "False | \n", "
| Syntactic | \n", "38.55 | \n", "37.25 | \n", "-1.30 | \n", "0.49 | \n", "True | \n", "
| : capital-common-countries | \n", "86.36 | \n", "81.42 | \n", "-4.94 | \n", "5.14 | \n", "False | \n", "
| : capital-world | \n", "69.81 | \n", "70.05 | \n", "0.24 | \n", "2.92 | \n", "False | \n", "
| : currency | \n", "6.81 | \n", "7.74 | \n", "0.92 | \n", "1.62 | \n", "False | \n", "
| : city-in-state | \n", "37.82 | \n", "34.41 | \n", "-3.40 | \n", "3.53 | \n", "False | \n", "
| : family | \n", "68.97 | \n", "53.16 | \n", "-15.81 | \n", "6.72 | \n", "True | \n", "
| : gram1-adjective-to-adverb | \n", "13.21 | \n", "15.22 | \n", "2.02 | \n", "1.41 | \n", "True | \n", "
| : gram2-opposite | \n", "12.07 | \n", "8.99 | \n", "-3.08 | \n", "2.22 | \n", "True | \n", "
| : gram3-comparative | \n", "45.42 | \n", "37.16 | \n", "-8.26 | \n", "5.26 | \n", "True | \n", "
| : gram4-superlative | \n", "18.18 | \n", "15.33 | \n", "-2.85 | \n", "5.17 | \n", "False | \n", "
| : gram5-present-participle | \n", "29.07 | \n", "33.52 | \n", "4.45 | \n", "5.21 | \n", "False | \n", "
| : gram6-nationality-adjective | \n", "76.55 | \n", "79.80 | \n", "3.25 | \n", "4.00 | \n", "False | \n", "
| : gram7-past-tense | \n", "43.53 | \n", "41.79 | \n", "-1.73 | \n", "6.35 | \n", "False | \n", "
| : gram8-plural | \n", "39.11 | \n", "42.57 | \n", "3.45 | \n", "2.40 | \n", "True | \n", "
| : gram9-plural-verbs | \n", "39.77 | \n", "27.13 | \n", "-12.64 | \n", "2.18 | \n", "True | \n", "