diff --git a/README.md b/README.md index 7b941c5..453e07e 100644 --- a/README.md +++ b/README.md @@ -14,6 +14,12 @@ Alongside it sits a parallel PyTorch implementation in [engine/main.py](engine/m The point of this repo is the C++ core. The PyTorch, FastAPI, and frontend layers exist to make the model usable, but if you're here to learn how a GPT is actually built and trained without a framework doing the work for you, [include/backward.h](include/backward.h) is where to start reading. +

+image + + +

+ ***technical notes***: [docs](https://eamon2009.github.io/LLMs/) diff --git a/assets/run_2026-07-16 165731.png b/assets/run_2026-07-16 165731.png new file mode 100644 index 0000000..3372892 Binary files /dev/null and b/assets/run_2026-07-16 165731.png differ diff --git a/engine/main.py b/engine/main.py index 5c15109..0725493 100644 --- a/engine/main.py +++ b/engine/main.py @@ -66,23 +66,23 @@ def success(msg): log(f" ok {msg}") device = 'cuda' if torch.cuda.is_available() else 'cpu' dropout = 0.1 -block_size = 256 -n_embd = 192 -n_head = 6 -n_layer = 6 -batch_size = 64 -max_iters = 5000 -eval_interval = 250 -learning_rate = 6e-4 -eval_iters = 200 -dropout = 0.1 +block_size = 256 +n_embd = 192 +n_head = 6 +n_layer = 6 +batch_size = 64 +max_iters = 5000 +eval_interval = 250 +learning_rate = 6e-4 +eval_iters = 200 +dropout = 0.1 torch.manual_seed(seed) # tokenizer -def get_tokenizer(encoding_name="o200k"): +def get_tokenizer(encoding_name="o200k_base"): tokenizer = tiktoken.get_encoding(encoding_name) vocab_size = tokenizer.n_vocab return tokenizer, vocab_size @@ -96,7 +96,7 @@ def decode(tokens, tokenizer): return tokenizer.decode(tokens) with open(cleaned_path, 'r', encoding='utf-8') as f: text = f.read() -tokenizer, vocab_size = get_tokenizer("o200k") +tokenizer, vocab_size = get_tokenizer("o200k_base") encoded_data = encode(text, tokenizer) data = torch.tensor(encoded_data, dtype=torch.long)