Skip to content

Commit b39e6ef

Browse files
rasbtrasbt
committed
add exercise solutions
1 parent ca803b5 commit b39e6ef

File tree

8 files changed

+320
-9
lines changed

8 files changed

+320
-9
lines changed

‎02_data/02.ipynb

+1-1
Original file line numberDiff line numberDiff line change
@@ -584,7 +584,7 @@
584584
"name": "python",
585585
"nbconvert_exporter": "python",
586586
"pygments_lexer": "ipython3",
587-
"version": "3.10.6"
587+
"version": "3.10.10"
588588
}
589589
},
590590
"nbformat": 4,

‎03_architecture/03.ipynb

+78-1
Original file line numberDiff line numberDiff line change
@@ -399,6 +399,83 @@
399399
"source": [
400400
"model.eval(); # disable dropout"
401401
]
402+
},
403+
{
404+
"cell_type": "markdown",
405+
"id": "02fa7ae0-f30d-454c-a92a-a75894ea68d2",
406+
"metadata": {},
407+
"source": [
408+
"<br>\n",
409+
"<br>\n",
410+
"<br>\n",
411+
"<br>\n",
412+
"\n",
413+
"\n",
414+
"\n",
415+
"# Solution"
416+
]
417+
},
418+
{
419+
"cell_type": "code",
420+
"execution_count": null,
421+
"id": "fdc23e58-dd3f-48d8-a767-944c1b6e030f",
422+
"metadata": {},
423+
"outputs": [],
424+
"source": [
425+
"start_context = \"Hello, I am\"\n",
426+
"\n",
427+
"encoded = tokenizer.encode(start_context)\n",
428+
"print(\"encoded:\", encoded)\n",
429+
"\n",
430+
"encoded_tensor = torch.tensor(encoded).unsqueeze(0)\n",
431+
"print(\"encoded_tensor.shape:\", encoded_tensor.shape)"
432+
]
433+
},
434+
{
435+
"cell_type": "code",
436+
"execution_count": null,
437+
"id": "599b0821-9755-4cf1-8da4-a1c0fec448b1",
438+
"metadata": {},
439+
"outputs": [],
440+
"source": [
441+
"out = generate_text_simple(\n",
442+
" model=model,\n",
443+
" idx=encoded_tensor, \n",
444+
" max_new_tokens=6, \n",
445+
" context_size=GPT_CONFIG_124M[\"context_length\"]\n",
446+
")\n",
447+
"\n",
448+
"print(\"Output:\", out)\n",
449+
"print(\"Output length:\", len(out[0]))"
450+
]
451+
},
452+
{
453+
"cell_type": "markdown",
454+
"id": "6de4f875-f967-4089-8410-b5cd2c200de8",
455+
"metadata": {},
456+
"source": [
457+
"- Remove batch dimension and convert back into text:"
458+
]
459+
},
460+
{
461+
"cell_type": "code",
462+
"execution_count": null,
463+
"id": "74c8d848-8ac1-41d4-b229-72ba7698297c",
464+
"metadata": {},
465+
"outputs": [],
466+
"source": [
467+
"decoded_text = tokenizer.decode(out.squeeze(0).tolist())\n",
468+
"print(decoded_text)"
469+
]
470+
},
471+
{
472+
"cell_type": "markdown",
473+
"id": "8c538bcd-a209-4273-9527-60d6fef1f6ab",
474+
"metadata": {},
475+
"source": [
476+
"- Note that the model is untrained; hence the random output texts above\n",
477+
"- We will train the model in the next notebook"
478+
]
402479
}
403480
],
404481
"metadata": {
@@ -417,7 +494,7 @@
417494
"name": "python",
418495
"nbconvert_exporter": "python",
419496
"pygments_lexer": "ipython3",
420-
"version": "3.10.6"
497+
"version": "3.10.10"
421498
}
422499
},
423500
"nbformat": 4,

‎04_pretraining/04.ipynb

+76-1
Original file line numberDiff line numberDiff line change
@@ -695,6 +695,81 @@
695695
"\n",
696696
"# Exercise 3 (Optional): Train the LLM on your own favorite texts"
697697
]
698+
},
699+
{
700+
"cell_type": "markdown",
701+
"id": "11f349d5-35e4-4502-8b86-ab57b5ca2f0c",
702+
"metadata": {},
703+
"source": [
704+
"<br>\n",
705+
"<br>\n",
706+
"<br>\n",
707+
"<br>\n",
708+
"\n",
709+
"\n",
710+
"# Solution to Exercise 1"
711+
]
712+
},
713+
{
714+
"cell_type": "code",
715+
"execution_count": null,
716+
"id": "f564c82a-49f7-46da-ad78-b9cb846eb5e3",
717+
"metadata": {},
718+
"outputs": [],
719+
"source": [
720+
"start_context = \"Every effort moves you\"\n",
721+
"tokenizer = tiktoken.get_encoding(\"gpt2\")\n",
722+
"\n",
723+
"token_ids = generate_text_simple(\n",
724+
" model=model,\n",
725+
" idx=text_to_token_ids(start_context, tokenizer),\n",
726+
" max_new_tokens=10,\n",
727+
" context_size=GPT_CONFIG_124M[\"context_length\"]\n",
728+
")"
729+
]
730+
},
731+
{
732+
"cell_type": "code",
733+
"execution_count": null,
734+
"id": "3e9d58e1-afba-44c7-9f82-7516adff359d",
735+
"metadata": {},
736+
"outputs": [],
737+
"source": [
738+
"print(\"Output text:\\n\", token_ids_to_text(token_ids, tokenizer))"
739+
]
740+
},
741+
{
742+
"cell_type": "markdown",
743+
"id": "b64b3b1f-c8d3-4755-a926-dc86eeae0ba0",
744+
"metadata": {},
745+
"source": [
746+
"<br>\n",
747+
"<br>\n",
748+
"<br>\n",
749+
"<br>\n",
750+
"\n",
751+
"\n",
752+
"# Solution to Exercise 2"
753+
]
754+
},
755+
{
756+
"cell_type": "code",
757+
"execution_count": null,
758+
"id": "a998656c-3615-4673-a9f9-c8eefb6b6611",
759+
"metadata": {},
760+
"outputs": [],
761+
"source": [
762+
"import torch\n",
763+
"\n",
764+
"# Imports from a local file\n",
765+
"from supplementary import GPTModel\n",
766+
"\n",
767+
"\n",
768+
"model = GPTModel(GPT_CONFIG_124M)\n",
769+
"device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n",
770+
"model.load_state_dict(torch.load(\"model.pth\", map_location=device))\n",
771+
"model.eval();"
772+
]
698773
}
699774
],
700775
"metadata": {
@@ -719,7 +794,7 @@
719794
"name": "python",
720795
"nbconvert_exporter": "python",
721796
"pygments_lexer": "ipython3",
722-
"version": "3.10.6"
797+
"version": "3.10.10"
723798
}
724799
},
725800
"nbformat": 4,

‎05_weightloading/05_part-1.ipynb

+1-1
Original file line numberDiff line numberDiff line change
@@ -407,7 +407,7 @@
407407
"name": "python",
408408
"nbconvert_exporter": "python",
409409
"pygments_lexer": "ipython3",
410-
"version": "3.10.6"
410+
"version": "3.10.10"
411411
}
412412
},
413413
"nbformat": 4,

‎06_finetuning/06_part-1.ipynb

+1-1
Original file line numberDiff line numberDiff line change
@@ -389,7 +389,7 @@
389389
"name": "python",
390390
"nbconvert_exporter": "python",
391391
"pygments_lexer": "ipython3",
392-
"version": "3.10.6"
392+
"version": "3.10.10"
393393
}
394394
},
395395
"nbformat": 4,

‎06_finetuning/06_part-2.ipynb

+15-2
Original file line numberDiff line numberDiff line change
@@ -369,6 +369,19 @@
369369
"- Save the resulting `test_data` dictionary as `test_base_and_finetuned_model.json`"
370370
]
371371
},
372+
{
373+
"cell_type": "markdown",
374+
"id": "fbdeb748-e3a9-4671-ae9b-046e6f298044",
375+
"metadata": {},
376+
"source": [
377+
"<br>\n",
378+
"<br>\n",
379+
"<br>\n",
380+
"<br>\n",
381+
"\n",
382+
"# Solution"
383+
]
384+
},
372385
{
373386
"cell_type": "code",
374387
"execution_count": null,
@@ -379,7 +392,7 @@
379392
"from litgpt import LLM\n",
380393
"\n",
381394
"del llm\n",
382-
"llm2 = LLM.load(\"/teamspace/studios/this_studio/out/finetune/lora/final/\")"
395+
"llm2 = LLM.load(\"out/finetune/lora/final/\")"
383396
]
384397
},
385398
{
@@ -418,7 +431,7 @@
418431
"name": "python",
419432
"nbconvert_exporter": "python",
420433
"pygments_lexer": "ipython3",
421-
"version": "3.10.6"
434+
"version": "3.10.10"
422435
}
423436
},
424437
"nbformat": 4,

‎06_finetuning/06_part-3.ipynb

+59-1
Original file line numberDiff line numberDiff line change
@@ -137,6 +137,64 @@
137137
"\n",
138138
"# Exercise 3: Evaluate the finetuned LLM"
139139
]
140+
},
141+
{
142+
"cell_type": "markdown",
143+
"id": "773a25be-5a02-477b-bfea-ffd53e44647b",
144+
"metadata": {},
145+
"source": [
146+
"<br>\n",
147+
"<br>\n",
148+
"<br>\n",
149+
"<br>\n",
150+
"\n",
151+
"# Solution"
152+
]
153+
},
154+
{
155+
"cell_type": "code",
156+
"execution_count": 13,
157+
"id": "6cd718c4-0e83-4a83-84f8-59e3fc4c3404",
158+
"metadata": {},
159+
"outputs": [
160+
{
161+
"name": "stdout",
162+
"output_type": "stream",
163+
"text": [
164+
"{'batch_size': 4,\n",
165+
" 'checkpoint_dir': PosixPath('out/finetune/lora/final'),\n",
166+
" 'device': None,\n",
167+
" 'dtype': None,\n",
168+
" 'force_conversion': False,\n",
169+
" 'limit': None,\n",
170+
" 'num_fewshot': None,\n",
171+
" 'out_dir': None,\n",
172+
" 'save_filepath': None,\n",
173+
" 'seed': 1234,\n",
174+
" 'tasks': 'mmlu_philosophy'}\n",
175+
"2024-07-04:00:57:13,332 INFO [huggingface.py:170] Using device 'cuda'\n",
176+
"Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n",
177+
"2024-07-04:00:57:18,981 INFO [evaluator.py:152] Setting random seed to 1234 | Setting numpy seed to 1234 | Setting torch manual seed to 1234\n",
178+
"2024-07-04:00:57:18,981 INFO [evaluator.py:203] Using pre-initialized model\n",
179+
"2024-07-04:00:57:24,808 INFO [evaluator.py:261] Setting fewshot random generator seed to 1234\n",
180+
"2024-07-04:00:57:24,809 INFO [task.py:411] Building contexts for mmlu_philosophy on rank 0...\n",
181+
"100%|████████████████████████████████████████| 311/311 [00:00<00:00, 807.98it/s]\n",
182+
"2024-07-04:00:57:25,206 INFO [evaluator.py:438] Running loglikelihood requests\n",
183+
"Running loglikelihood requests: 0%| | 0/1244 [00:00<?, ?it/s]We detected that you are passing `past_key_values` as a tuple and this is deprecated and will be removed in v4.43. Please use an appropriate `Cache` class (https://huggingface.co/docs/transformers/v4.41.3/en/internal/generation_utils#transformers.Cache)\n",
184+
"Running loglikelihood requests: 100%|██████| 1244/1244 [00:07<00:00, 158.49it/s]\n",
185+
"2024-07-04:00:57:33,515 WARNING [huggingface.py:1315] Failed to get model SHA for /teamspace/studios/this_studio/out/finetune/lora/final/evaluate at revision main. Error: Repo id must be in the form 'repo_name' or 'namespace/repo_name': '/teamspace/studios/this_studio/out/finetune/lora/final/evaluate'. Use `repo_type` argument if needed.\n",
186+
"fatal: not a git repository (or any parent up to mount point /teamspace/studios)\n",
187+
"Stopping at filesystem boundary (GIT_DISCOVERY_ACROSS_FILESYSTEM not set).\n",
188+
"| Tasks |Version|Filter|n-shot|Metric| |Value | |Stderr|\n",
189+
"|----------|------:|------|-----:|------|---|-----:|---|-----:|\n",
190+
"|philosophy| 0|none | 0|acc |↑ |0.5691|± |0.0281|\n",
191+
"\n"
192+
]
193+
}
194+
],
195+
"source": [
196+
"!litgpt evaluate out/finetune/lora/final --tasks \"mmlu_philosophy\" --batch_size 4"
197+
]
140198
}
141199
],
142200
"metadata": {
@@ -160,7 +218,7 @@
160218
"name": "python",
161219
"nbconvert_exporter": "python",
162220
"pygments_lexer": "ipython3",
163-
"version": "3.10.6"
221+
"version": "3.10.10"
164222
}
165223
},
166224
"nbformat": 4,

0 commit comments

Comments
 (0)