From 860a77f90aef648a8edd03ffbf3618d3dcef355f Mon Sep 17 00:00:00 2001 From: Eamon Date: Tue, 14 Apr 2026 11:42:20 +0530 Subject: [PATCH 01/27] CMake file removed ,not needed --- generate/CMakeLists.txt | 17 ----------------- 1 file changed, 17 deletions(-) delete mode 100644 generate/CMakeLists.txt diff --git a/generate/CMakeLists.txt b/generate/CMakeLists.txt deleted file mode 100644 index 392332e..0000000 --- a/generate/CMakeLists.txt +++ /dev/null @@ -1,17 +0,0 @@ -cmake_minimum_required(VERSION 3.18) -project(gpt_generate) - -set(CMAKE_CXX_STANDARD 17) -set(CMAKE_PREFIX_PATH "C:/libtorch") - -find_package(Torch REQUIRED) - -add_executable(generate main.cpp) -target_link_libraries(generate ${TORCH_LIBRARIES}) - -if(WIN32) - file(GLOB TORCH_DLLS "${TORCH_INSTALL_PREFIX}/lib/*.dll") - add_custom_command(TARGET generate POST_BUILD - COMMAND ${CMAKE_COMMAND} -E copy_if_different - ${TORCH_DLLS} $) -endif() \ No newline at end of file From 915df4371299aca6899cd00560965d1a44123183 Mon Sep 17 00:00:00 2001 From: Eamon Date: Tue, 14 Apr 2026 11:42:44 +0530 Subject: [PATCH 02/27] Update .gitignore --- .gitignore | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/.gitignore b/.gitignore index 59ba071..ab59c09 100644 --- a/.gitignore +++ b/.gitignore @@ -1,9 +1,14 @@ -# Virtual environments .venv eamon_env __pycache__ best_model.pt .ipynb_checkpoints .exe - - +checkpoints/*.pt +checkpoints/*.pth +checkpoints/*.ckpt +!checkpoints/.gitkeep +!checkpoints/README.md +!logs/.gitkeep +!logs/README.md +!logs/train_loss.csv \ No newline at end of file From d51023bd4bf6c2c8ea75b1c78c1b6ee3ef4b09a9 Mon Sep 17 00:00:00 2001 From: Eamon Date: Tue, 14 Apr 2026 22:09:35 +0530 Subject: [PATCH 03/27] Update GitHub Actions workflow for train tests --- .github/workflows/train_test.yml | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/.github/workflows/train_test.yml b/.github/workflows/train_test.yml index 71ba2b2..203a41a 100644 --- a/.github/workflows/train_test.yml +++ b/.github/workflows/train_test.yml @@ -1,6 +1,12 @@ -name: Run Train Test +name: Test Train Scripts -on: [push, pull_request] +on: + push: + paths: + - 'train_test/**' + pull_request: + paths: + - 'train_test/**' jobs: test: @@ -11,13 +17,10 @@ jobs: - name: Set up Python uses: actions/setup-python@v4 with: - python-version-file: '.python-version' + python-version-file: '.python-version' - name: Install dependencies run: pip install -r requirements.txt - - name: Run transformer tests - run: python transformer.py - - name: Run train_test scripts - run: python train_test/ \ No newline at end of file + run: python -m train_test From e0943f2c2265fb4759347b1f875e329041c60f33 Mon Sep 17 00:00:00 2001 From: Eamon Date: Tue, 14 Apr 2026 22:10:42 +0530 Subject: [PATCH 04/27] Add GitHub Actions workflow for transformer tests --- .github/workflows/test_transformer.yml | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) create mode 100644 .github/workflows/test_transformer.yml diff --git a/.github/workflows/test_transformer.yml b/.github/workflows/test_transformer.yml new file mode 100644 index 0000000..c57c774 --- /dev/null +++ b/.github/workflows/test_transformer.yml @@ -0,0 +1,26 @@ +name: Test Transformer + +on: + push: + paths: + - 'transformer.py' + pull_request: + paths: + - 'transformer.py' + +jobs: + test: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + + - name: Set up Python + uses: actions/setup-python@v4 + with: + python-version-file: '.python-version' + + - name: Install dependencies + run: pip install -r requirements.txt + + - name: Run transformer tests + run: python transformer.py From 94fe5510c1857855556fadaa4e3c1ded26de5a64 Mon Sep 17 00:00:00 2001 From: Eamon Date: Tue, 14 Apr 2026 22:11:33 +0530 Subject: [PATCH 05/27] Add general CI workflow for linting and checks --- .github/workflows/general.yml | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) create mode 100644 .github/workflows/general.yml diff --git a/.github/workflows/general.yml b/.github/workflows/general.yml new file mode 100644 index 0000000..4477adf --- /dev/null +++ b/.github/workflows/general.yml @@ -0,0 +1,30 @@ +name: General CI + +on: + push: + paths-ignore: + - 'transformer.py' + - 'train_test/**' + - '**.md' + pull_request: + paths-ignore: + - 'transformer.py' + - 'train_test/**' + - '**.md' + +jobs: + lint: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + + - name: Set up Python + uses: actions/setup-python@v4 + with: + python-version-file: '.python-version' + + - name: Install dependencies + run: pip install -r requirements.txt + + - name: General check + run: echo "General CI passed for this commit" From 2b0f85ff60796849d1c573146ed798f8d260b86b Mon Sep 17 00:00:00 2001 From: codeenthusiasm23 Date: Tue, 14 Apr 2026 22:30:24 +0530 Subject: [PATCH 06/27] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index b481125..2daffc7 100644 --- a/README.md +++ b/README.md @@ -418,7 +418,7 @@ All gaps are small and healthy. Run 3 sits between Run 1 and Run 2, which matche Screenshot 2026-03-17 171921 -### What Are Scaling Laws? +### What is Scaling Law? Scaling laws describe a predictable relationship between model size, dataset size, compute, and output quality: From ee7f02b1e6a4f574a350b9d910437e79153b4f46 Mon Sep 17 00:00:00 2001 From: Eamon Date: Tue, 14 Apr 2026 22:34:45 +0530 Subject: [PATCH 07/27] Correct title to 'What Are Scaling Laws?' --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 2daffc7..b481125 100644 --- a/README.md +++ b/README.md @@ -418,7 +418,7 @@ All gaps are small and healthy. Run 3 sits between Run 1 and Run 2, which matche Screenshot 2026-03-17 171921 -### What is Scaling Law? +### What Are Scaling Laws? Scaling laws describe a predictable relationship between model size, dataset size, compute, and output quality: From 0b7db40ab7d7700c504fc36b61834a291b13a49c Mon Sep 17 00:00:00 2001 From: Eamon Date: Wed, 15 Apr 2026 16:27:57 +0530 Subject: [PATCH 08/27] Create .gitkeep --- checkpoints/.gitkeep | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 checkpoints/.gitkeep diff --git a/checkpoints/.gitkeep b/checkpoints/.gitkeep new file mode 100644 index 0000000..e69de29 From 9b9ab1776f6395e49ef271f6c4068e3e573e388c Mon Sep 17 00:00:00 2001 From: Eamon Date: Wed, 15 Apr 2026 16:28:03 +0530 Subject: [PATCH 09/27] Create read.md --- checkpoints/read.md | 62 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 62 insertions(+) create mode 100644 checkpoints/read.md diff --git a/checkpoints/read.md b/checkpoints/read.md new file mode 100644 index 0000000..d4ade29 --- /dev/null +++ b/checkpoints/read.md @@ -0,0 +1,62 @@ +# Checkpoints + +This folder stores saved model weights during and after training. + +## Structure + +``` +checkpoints/ +├── best_model.pt ← best checkpoint by validation loss +├── latest_model.pt ← most recent checkpoint +└── epoch_XX_loss_X.XX.pt ← epoch-specific saves +``` + +## How to Save (add to your transformer.py / train script) + +```python +import torch +import os + +def save_checkpoint(model, optimizer, epoch, loss, path="checkpoints/"): + os.makedirs(path, exist_ok=True) + checkpoint = { + "epoch": epoch, + "model_state_dict": model.state_dict(), + "optimizer_state_dict": optimizer.state_dict(), + "loss": loss, + } + # Save latest + torch.save(checkpoint, os.path.join(path, "latest_model.pt")) + + # Save best if loss improved + best_path = os.path.join(path, "best_model.pt") + if not os.path.exists(best_path): + torch.save(checkpoint, best_path) + else: + best = torch.load(best_path) + if loss < best["loss"]: + torch.save(checkpoint, best_path) + print(f" New best model saved at epoch {epoch} with loss {loss:.4f}") + + # Save per epoch (optional) + torch.save(checkpoint, os.path.join(path, f"epoch_{epoch:03d}_loss_{loss:.4f}.pt")) +``` + +## How to Load + +```python +def load_checkpoint(model, optimizer, path="checkpoints/latest_model.pt"): + checkpoint = torch.load(path) + model.load_state_dict(checkpoint["model_state_dict"]) + optimizer.load_state_dict(checkpoint["optimizer_state_dict"]) + epoch = checkpoint["epoch"] + loss = checkpoint["loss"] + print(f" Loaded checkpoint from epoch {epoch}, loss: {loss:.4f}") + return model, optimizer, epoch, loss +``` + +## Notes + +- `.pt` files are listed in `.gitignore` — they are too large for GitHub +- Push only this README and `.gitkeep` to keep the folder tracked by git +- For long GPU runs, save every N epochs so you can resume if it crashes \ No newline at end of file From 302cc79768008f3920e8b1cc78d91a5cbd71ade7 Mon Sep 17 00:00:00 2001 From: Eamon Date: Wed, 15 Apr 2026 22:59:17 +0530 Subject: [PATCH 10/27] Rename project to Quadtrix and update images Updated project name and images in README. --- README.md | 16 ++-------------- 1 file changed, 2 insertions(+), 14 deletions(-) diff --git a/README.md b/README.md index b481125..79916be 100644 --- a/README.md +++ b/README.md @@ -1,19 +1,7 @@ -# Transformer Language Model [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/drive/1Zs84ZQf-0VPbQxHce1mlSMD-Jr22xJqZ#scrollTo=VdohdZ8imygv) ![GitHub](https://img.shields.io/github/license/Eamon2009/Transformer-language-model) -image - - +# Quadtrix ![GitHub](https://img.shields.io/github/license/Eamon2009/Transformer-language-model) +image A character-level GPT transformer built from scratch in PyTorch, trained on children's stories to generate simple English narrative text character by character. No pre-trained weights. No fine-tuning. Pure architecture and training from zero. -> **Latest run:** 1.99M parameter model trained on Tesla T4 GPU — val loss **0.9250** in just **6.1 minutes.** -Screenshot 2026-04-10 172442 - - -# GPU RUN-2 -Screenshot 2026-03-22 215122 - -## CPU RUN-1 -Screenshot 2026-03-21 001248 - --- ## Table of Contents From d94d6b4bf2d0d1ff52093a56d81817a9429ced5b Mon Sep 17 00:00:00 2001 From: Eamon Date: Wed, 15 Apr 2026 23:20:53 +0530 Subject: [PATCH 11/27] Update README with additional image Added an additional image to the README for better visual representation. --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index 79916be..94d4dfd 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,7 @@ # Quadtrix ![GitHub](https://img.shields.io/github/license/Eamon2009/Transformer-language-model) image +image + A character-level GPT transformer built from scratch in PyTorch, trained on children's stories to generate simple English narrative text character by character. No pre-trained weights. No fine-tuning. Pure architecture and training from zero. --- From 2de747b324eee1231a167e595dab86e390e2fbf1 Mon Sep 17 00:00:00 2001 From: Eamon Date: Thu, 16 Apr 2026 15:53:51 +0530 Subject: [PATCH 12/27] Update README.md --- README.md | 742 ++++++++++++++++++++++++++---------------------------- 1 file changed, 350 insertions(+), 392 deletions(-) diff --git a/README.md b/README.md index 94d4dfd..284dca0 100644 --- a/README.md +++ b/README.md @@ -1,315 +1,237 @@ -# Quadtrix ![GitHub](https://img.shields.io/github/license/Eamon2009/Transformer-language-model) -image -image +# Quadtrix [![License](https://img.shields.io/github/license/Eamon2009/Transformer-language-model)](LICENSE) +Screenshot 2026-04-15 225925 +Screenshot 2026-04-15 231851 -A character-level GPT transformer built from scratch in PyTorch, trained on children's stories to generate simple English narrative text character by character. No pre-trained weights. No fine-tuning. Pure architecture and training from zero. ---- +A minimal, educational GPT-style transformer trained character-by-character on children's stories. No pre-trained weights. No fine-tuning. Just raw PyTorch, from init to generation. +Character-level Transformer is a sequence-to-sequence architecture that operates on the granularity of individual characters rather than words or subword tokens. By utilizing a self-attention mechanism over long sequences of characters, the model learns to construct internal representations of morphology, syntax, and semantics from the ground up, effectively eliminating "out-of-vocabulary" (OOV) issues. While this approach allows for high fidelity in modeling rare words, spelling variations, and creative linguistics, it significantly increases the computational complexity—typically $O(L^2)$ where $L$ is the sequence length—as a single sentence requires many more steps than its word-level equivalent. Consequently, character-level Transformers often require deeper architectures or auxiliary losses to capture the long-range dependencies necessary to match the semantic performance of traditional token-based models -## Table of Contents - -1. [What This Project Does](#what-this-project-does) -2. [Project Structure](#project-structure) -3. [How It Works](#how-it-works) -4. [Setup & Requirements](#setup--requirements) -5. [How to Run](#how-to-run) -6. [Configuration](#configuration) -7. [Training Runs — All Results](#training-runs--all-results) -8. [Head-to-Head Comparison](#head-to-head-comparison) -9. [Model Output Comparison](#model-output-comparison) -10. [Loss Curve Analysis](#loss-curve-analysis) -11. [Overfitting Analysis](#overfitting-analysis) -12. [Scaling Laws — And Where Your Model Sits](#scaling-laws--and-where-your-model-sits) -13. [How Weights Produce Output](#how-weights-produce-output) -14. [Known Limitations](#known-limitations) +> **The goal**: Understand how language models learn patterns. See it happen on your machine. Train in minutes on a GPU. --- -## What This Project Does +## Quick Start -This project trains a small GPT-style transformer model on children's stories and then generates new story-like text character by character. It is a learning project — the goal is not to produce publishable stories, but to understand how language models learn patterns from text and to see that process happen live on your own machine and cloud GPU. +```bash +# Install PyTorch +pip install torch ---- +# Run training +python transformer.py -## How It Works +# The script will train, save best weights, then generate text forever +# Press Ctrl+C to stop +``` -The model is a **character-level transformer**. This means: +**That's it.** No complex setup, no data pipelines, no credentials. -- It reads your text file one character at a time -- It learns which characters tend to follow other characters in which contexts -- At generation time it predicts the next character, then the next, then the next — forever +--- -It is the same core architecture as GPT, just much smaller and trained on much less data. +## What Is This? -**The pipeline in order:** +Quadtrix is a learning project. It trains a tiny transformer on text—character by character—and learns which characters tend to follow others. At generation time, it predicts the next character, feeds it back in, and repeats. -``` -data.txt (children's stories) - ↓ -Characters encoded as integers (vocab size varies by run) - ↓ -Model trains on sequences of tokens at a time - ↓ -Every N steps: loss is measured and printed - ↓ -Best weights saved to best_model.pt whenever val loss improves - ↓ -After training: text generation begins - ↓ -Press Ctrl+C to stop -``` +**It's the same architecture as GPT**, just much smaller (1M–11M parameters instead of 175B) and trained on much less data (thousands of stories instead of the internet). ---- +The magic: in just 6 minutes on a Tesla T4 GPU, a 2M-parameter model learns meaningful patterns about narrative structure, dialogue, and storytelling. The output isn't Shakespeare, but it *is* recognizable as a story. -## Setup & Requirements +--- -**Python version:** 3.8 or higher +## How It Works -**Install dependencies:** +### The Pipeline -```bash -pip install torch ``` +1. Load children's stories from disk + ↓ +2. Encode each character as an integer (vocab size: 28–110) + ↓ +3. Split into train/val chunks (80/20) + ↓ +4. Build a GPT-style transformer with: + - Embedding layer (character → vector) + - Transformer blocks (self-attention + feedforward) + - Output projection (vector → logits over vocab) + ↓ +5. Train for N steps, measuring loss every eval_interval + ↓ +6. Save best weights whenever validation loss improves + ↓ +7. Load best model and generate text forever + ↓ +8. Press Ctrl+C to stop +``` + +### What the Model Actually Does + +Each forward pass: +1. Takes a sequence of character indices (e.g., "Once upon...") +2. Embeds each into a dense vector +3. Passes through transformer layers (multi-head attention learns which characters to attend to) +4. Outputs logits (scores) for every possible next character +5. Samples the next character according to those probabilities +6. Feeds it back in and repeats -No other dependencies needed. The project uses only PyTorch and Python standard library modules. +**Loss function**: Cross-entropy. The model minimizes surprise on unseen text. + +**Training**: Adam optimizer with a learning rate schedule. Dropout prevents overfitting. --- -## How to Run +## Project Structure -```bash -python transformer.py +``` +transformer.py ← Everything. One file. +best_model.pt ← Saved weights (after first training run) +data.txt ← Your text file (any UTF-8 file works) ``` -The script will: - -1. Print a startup banner with device and timestamp -2. Load and report stats on your dataset -3. Build the model and print parameter count -4. Train for the configured number of steps, printing progress at each eval interval -5. Save best weights to `best_model.pt` automatically -6. Start text generation when done +That's all. No fancy folder structure. No config files. Edit hyperparameters directly in the script. --- -## Configuration +## Configuration & Hardware -### Run 3 — GPU Configuration (Tesla T4, Latest) ⭐ +Quadtrix is designed to work anywhere: laptop CPU to cloud GPU. Edit these hyperparameters in `transformer.py`: ```python -batch_size = 64 # Sequences trained on at once -block_size = 128 # Context window (tokens) -max_iters = 5000 # Total training steps -eval_interval = 200 # Print progress every N steps +# ============================================================================ +# Hyperparameters +# ============================================================================ +batch_size = 64 # Sequences per batch +block_size = 128 # Context window (tokens) +max_iters = 5000 # Total training steps +eval_interval = 200 # Print loss every N steps learning_rate = 3e-4 -n_embd = 200 # Size of internal representations -n_head = 4 # Number of attention heads -n_layer = 4 # Number of transformer blocks -dropout = 0.2 +n_embd = 200 # Embedding dimension +n_head = 4 # Attention heads per layer +n_layer = 4 # Number of transformer blocks +dropout = 0.2 # Regularization ``` -**Parameter count: 1.99M parameters** - -### Run 2 — GPU Configuration (Google Colab) +### Three Pre-Tuned Configurations +**CPU (Laptop) — Fast Feedback** ```python -batch_size = 64 -block_size = 256 -max_iters = 5000 -eval_interval = 250 -learning_rate = 3e-4 -n_embd = 384 -n_head = 6 -n_layer = 6 -dropout = 0.2 +batch_size, block_size, max_iters = 16, 128, 3000 +n_embd, n_head, n_layer = 128, 4, 4 +# ~0.82M parameters +# Trains in ~40 min on AMD Ryzen ``` -**Parameter count: 10.82M parameters** - -### Run 1 — CPU Configuration (Laptop, Minimal Setup) - +**GPU (Google Colab) — Best Quality** ```python -batch_size = 16 -block_size = 128 -max_iters = 3000 -eval_interval = 200 -learning_rate = 3e-4 -n_embd = 128 -n_head = 4 -n_layer = 4 -dropout = 0.2 +batch_size, block_size, max_iters = 64, 256, 5000 +n_embd, n_head, n_layer = 384, 6, 6 +# ~10.82M parameters +# Trains in ~60 min on Colab GPU +# Best output quality ``` -**Parameter count: 0.82M parameters** +**GPU (Tesla T4) — Efficient** +```python +batch_size, block_size, max_iters = 64, 128, 5000 +n_embd, n_head, n_layer = 200, 4, 4 +# ~1.99M parameters +# Trains in **6.1 minutes** ← Fastest +# Best parameter/data balance +``` --- -## Training Runs — All Results +## Training Results -### Run 3 — GPU (Tesla T4, 1.99M Parameters) ⭐ Latest +Three runs. Three different hardware setups. All converged well. -| Field | Value | -|---|---| -| Device | Tesla T4 (CUDA 13.0, Driver 580.82.07) | -| Dataset | ~31.4M characters | -| Vocab size | 100 | -| Train tokens | 28,274,093 | -| Val tokens | 3,141,566 | -| Parameters | 1.99M | -| Architecture | 4 layers × 4 heads × 200 embd dim | -| Training time | **6.1 minutes (367s)** | -| Best val loss | **0.9250** | -| Final train loss | 0.9307 | -| Overfitting | None — `best!` at most checkpoints | +### Run 3 — Tesla T4 (Latest) ⭐ -**Full training log:** +**Configuration**: 4 layers × 4 heads × 200 dim = **1.99M params** -``` -[ 0/5000] train=4.6207 val=4.6202 elapsed=2s ETA=0s << best! -[ 200/5000] train=2.2058 val=2.1986 elapsed=17s ETA=405s << best! -[ 400/5000] train=1.6111 val=1.6039 elapsed=32s ETA=367s << best! -[ 600/5000] train=1.4109 val=1.4183 elapsed=47s ETA=342s << best! -[ 800/5000] train=1.3230 val=1.3231 elapsed=61s ETA=322s << best! -[ 1000/5000] train=1.2495 val=1.2567 elapsed=76s ETA=303s << best! -[ 1200/5000] train=1.1960 val=1.1948 elapsed=90s ETA=286s << best! -[ 1400/5000] train=1.1569 val=1.1642 elapsed=105s ETA=270s << best! -[ 1600/5000] train=1.1283 val=1.1283 elapsed=120s ETA=254s << best! -[ 1800/5000] train=1.0894 val=1.1023 elapsed=134s ETA=238s << best! -[ 2000/5000] train=1.0731 val=1.0765 elapsed=149s ETA=223s << best! -[ 2200/5000] train=1.0584 val=1.0550 elapsed=163s ETA=208s << best! -[ 2400/5000] train=1.0415 val=1.0346 elapsed=178s ETA=192s << best! -[ 2600/5000] train=1.0261 val=1.0199 elapsed=192s ETA=177s << best! -[ 2800/5000] train=1.0106 val=1.0117 elapsed=207s ETA=162s << best! -[ 3000/5000] train=1.0000 val=0.9956 elapsed=221s ETA=148s << best! -[ 3200/5000] train=0.9913 val=0.9924 elapsed=236s ETA=133s << best! -[ 3400/5000] train=0.9727 val=0.9782 elapsed=251s ETA=118s << best! -[ 3600/5000] train=0.9656 val=0.9720 elapsed=265s ETA=103s << best! -[ 3800/5000] train=0.9685 val=0.9632 elapsed=280s ETA=88s << best! -[ 4000/5000] train=0.9601 val=0.9642 elapsed=294s ETA=74s -[ 4200/5000] train=0.9515 val=0.9489 elapsed=309s ETA=59s << best! -[ 4400/5000] train=0.9433 val=0.9431 elapsed=323s ETA=44s << best! -[ 4600/5000] train=0.9384 val=0.9459 elapsed=338s ETA=29s -[ 4800/5000] train=0.9331 val=0.9250 elapsed=353s ETA=15s << best! -[ 4999/5000] train=0.9307 val=0.9430 elapsed=367s ETA=0s +| Metric | Value | +|--------|-------| +| Device | Tesla T4 (CUDA 13.0) | +| Dataset | ~31.4M characters (children's stories) | +| Train tokens | 28.3M | +| Val tokens | 3.1M | +| Training time | **6.1 minutes** | +| Best val loss | **0.9250** | +| Final train loss | 0.9307 | +| Overfitting | None detected | + +**Training curve** (every 200 steps): +``` +[ 0/5000] train=4.6207 val=4.6202 elapsed=2s << best! +[ 200/5000] train=2.2058 val=2.1986 elapsed=17s << best! +[ 400/5000] train=1.6111 val=1.6039 elapsed=32s << best! +[ 1000/5000] train=1.2495 val=1.2567 elapsed=76s << best! +[ 2000/5000] train=1.0731 val=1.0765 elapsed=149s << best! +[ 3000/5000] train=1.0000 val=0.9956 elapsed=221s << best! +[ 4000/5000] train=0.9601 val=0.9642 elapsed=294s +[ 4200/5000] train=0.9515 val=0.9489 elapsed=309s << best! +[ 4400/5000] train=0.9433 val=0.9431 elapsed=323s << best! +[ 4800/5000] train=0.9331 val=0.9250 elapsed=353s << best! +[ 4999/5000] train=0.9307 val=0.9430 elapsed=367s [DONE] Training finished in 367.0s (6.1 min) | Best val loss: 0.9250 ``` +**Key insight**: This run hit the **sweet spot**—large enough to learn coherent patterns, small enough to train fast. It's the reference configuration. + --- -### Run 2 — GPU (Google Colab, 10.82M Parameters) +### Run 2 — Google Colab (10.82M Parameters) -| Field | Value | -|---|---| +**Configuration**: 6 layers × 6 heads × 384 dim + +| Metric | Value | +|--------|-------| | Device | CUDA (Google Colab GPU) | -| Dataset | 88,406,739 characters | -| Vocab size | 110 | -| Train tokens | 79,566,065 | -| Val tokens | 8,840,674 | -| Parameters | 10.82M (10,823,534) | -| Architecture | 6 layers × 6 heads × 384 embd dim | -| Training time | **61.3 minutes** | +| Dataset | ~88.4M characters | +| Parameters | 10.82M | +| Training time | 61.3 minutes | | Best val loss | **0.7176** | -| Final train loss | 0.7259 | -| Overfitting | None — `best!` at every checkpoint | - -**Full training log:** +| Overfitting | None | -``` -[ 0/5000] 0.0% train=4.9244 val=4.9262 elapsed=31s ETA=0s best! -[ 250/5000] 5.0% train=2.1218 val=2.1169 elapsed=206s ETA=3901s best! -[ 500/5000] 10.0% train=1.3606 val=1.3500 elapsed=391s ETA=3510s best! -[ 750/5000] 15.0% train=1.1540 val=1.1411 elapsed=575s ETA=3250s best! -[ 1000/5000] 20.0% train=1.0332 val=1.0296 elapsed=757s ETA=3024s best! -[ 1250/5000] 25.0% train=0.9657 val=0.9556 elapsed=941s ETA=2819s best! -[ 1500/5000] 30.0% train=0.9305 val=0.9189 elapsed=1124s ETA=2619s best! -[ 1750/5000] 35.0% train=0.8935 val=0.8853 elapsed=1306s ETA=2424s best! -[ 2000/5000] 40.0% train=0.8673 val=0.8602 elapsed=1490s ETA=2233s best! -[ 2250/5000] 45.0% train=0.8413 val=0.8367 elapsed=1672s ETA=2042s best! -[ 2500/5000] 50.0% train=0.8162 val=0.8141 elapsed=1855s ETA=1854s best! -[ 2750/5000] 55.0% train=0.8058 val=0.7995 elapsed=2038s ETA=1666s best! -[ 3000/5000] 60.0% train=0.7888 val=0.7803 elapsed=2221s ETA=1479s best! -[ 3250/5000] 65.0% train=0.7798 val=0.7730 elapsed=2403s ETA=1293s best! -[ 3500/5000] 70.0% train=0.7634 val=0.7551 elapsed=2585s ETA=1107s best! -[ 3750/5000] 75.0% train=0.7588 val=0.7528 elapsed=2768s ETA=922s best! -[ 4000/5000] 80.0% train=0.7480 val=0.7434 elapsed=2951s ETA=737s best! -[ 4250/5000] 85.0% train=0.7381 val=0.7351 elapsed=3134s ETA=552s best! -[ 4500/5000] 90.0% train=0.7371 val=0.7314 elapsed=3316s ETA=368s best! -[ 4750/5000] 95.0% train=0.7282 val=0.7239 elapsed=3498s ETA=183s best! -[ 4999/5000] 100.0% train=0.7259 val=0.7176 elapsed=3680s ETA=0s best! - -[DONE] Training finished in 3680.1s (61.3 min) -[DONE] Best val loss: 0.7176 -[SAVE] Best weights saved to: /content/best_model.pt -``` +**Result**: Larger model, more data = **best output quality**. Slower to train but produces recognizable narratives. --- -### Run 1 — CPU (Laptop, 0.82M Parameters) +### Run 1 — CPU Laptop (0.82M Parameters) + +**Configuration**: 4 layers × 4 heads × 128 dim -| Field | Value | -|---|---| +| Metric | Value | +|--------|-------| | Device | AMD Ryzen 5 PRO 3500U (CPU only) | -| Dataset | 201,570 characters | -| Vocab size | 28 | | Parameters | 0.82M | -| Architecture | 4 layers × 4 heads × 128 embd dim | -| Training time | **39.4 minutes** | -| Best val loss | **1.3145** | -| Final train loss | 1.3191 | -| Overfitting | None — `best!` at every checkpoint | +| Dataset | ~201K characters | +| Training time | 39.4 minutes | +| Best val loss | 1.3145 | +| Overfitting | None | -**Full training log:** - -``` -[ 0/3000] 0.0% train=3.2961 val=3.2981 elapsed=12s ETA=0s best! -[ 200/3000] 6.7% train=2.3038 val=2.2490 elapsed=141s ETA=1959s best! -[ 400/3000] 13.3% train=2.2469 val=2.1950 elapsed=292s ETA=1891s best! -[ 600/3000] 20.0% train=2.1842 val=2.1318 elapsed=436s ETA=1739s best! -[ 800/3000] 26.7% train=1.9742 val=1.9103 elapsed=581s ETA=1594s best! -[ 1000/3000] 33.3% train=1.7628 val=1.7002 elapsed=723s ETA=1443s best! -[ 1200/3000] 40.0% train=1.6714 val=1.6040 elapsed=863s ETA=1293s best! -[ 1400/3000] 46.7% train=1.5889 val=1.5360 elapsed=1015s ETA=1158s best! -[ 1600/3000] 53.3% train=1.5375 val=1.4723 elapsed=1166s ETA=1019s best! -[ 1800/3000] 60.0% train=1.4847 val=1.4525 elapsed=1320s ETA=879s best! -[ 2000/3000] 66.7% train=1.4604 val=1.4081 elapsed=1472s ETA=735s best! -[ 2200/3000] 73.3% train=1.4113 val=1.3857 elapsed=1653s ETA=600s best! -[ 2400/3000] 80.0% train=1.3923 val=1.3725 elapsed=1820s ETA=454s best! -[ 2600/3000] 86.7% train=1.3501 val=1.3446 elapsed=1998s ETA=307s best! -[ 2800/3000] 93.3% train=1.3336 val=1.3334 elapsed=2174s ETA=154s best! -[ 2999/3000] 100.0% train=1.3191 val=1.3145 elapsed=2363s ETA=0s best! - -[DONE] Training finished in 2364.1s (39.4 min) -[DONE] Best val loss: 1.3145 -[SAVE] Best weights saved to best_model.pt -``` +**Result**: Smallest model, tiniest dataset. Trains fastest on CPU. Output is fragmented but shows the model learned *something*. --- ## Head-to-Head Comparison -| Metric | Run 1 — CPU Laptop | Run 2 — GPU Colab | Run 3 — Tesla T4 ⭐ | -|---|---|---|---| -| **Device** | AMD Ryzen 5 CPU | CUDA GPU (Colab) | Tesla T4 (CUDA 13.0) | -| **Parameters** | 0.82M | 10.82M | **1.99M** | -| **Architecture** | 4L × 4H × 128d | 6L × 6H × 384d | 4L × 4H × 200d | -| **Dataset size** | 201,570 chars | 88,406,739 chars | ~31.4M chars | -| **Vocab size** | 28 | 110 | **100** | -| **Block size** | 128 tokens | 256 tokens | 128 tokens | -| **Batch size** | 16 | 64 | 64 | -| **Training steps** | 3,000 | 5,000 | 5,000 | +| Metric | Run 1 — CPU | Run 2 — Colab | Run 3 — T4 ⭐ | +|--------|-------------|--------------|------------| +| **Parameters** | 0.82M | 10.82M | 1.99M | +| **Training data** | 200K chars | 88.4M chars | 31.4M chars | | **Training time** | 39.4 min | 61.3 min | **6.1 min** | | **Best val loss** | 1.3145 | **0.7176** | 0.9250 | -| **Overfitting** | None | None | None | -| **Still improving at end?** | Yes | Yes | Yes | +| **Output coherence** | Fragmented | Coherent paragraphs | Basic sentences | +| **Overfitting** | ✓ None | ✓ None | ✓ None | +| **Still improving?** | Yes | Yes | Yes | -> **Key insight on Run 3:** A 1.99M parameter model on a Tesla T4 reached val loss 0.9250 in just 6.1 minutes — faster than any previous run by a large margin. This shows GPU acceleration pays off even for small models. Run 2 still holds the best quality due to its larger size and more data, but Run 3 shows what efficient GPU use looks like. +> **Observation**: All three were still improving at the final checkpoint. More training steps = better loss. --- -## Model Output Comparison +## Example Outputs -### Run 2 — GPU (10.82M params, val loss 0.7176) +### Run 2 Output (10.82M params) — Best Quality ``` Upon a time, there were two friends, Jack and Tom. They had a cold doll in @@ -321,223 +243,259 @@ to share his happy with them. Nack knew it was feeling important to his passion in their rooms. He knew that night, he had never seen a small boy just soon could drink. - -He kept helping her passion and seing this boy. As he kept walking, he saw -a girl. -``` - -### Run 1 — CPU (0.82M params, val loss 1.3145) - -``` -when years me told be found a big ea reak abig driendly they named not she -rabbit smiled by aded he what in again -one smiled the mushrought boy -one day and was arroom him that she rabbing animals the dreezed at neard had -to there man owl them with o box and said you s mom that je animable went her -somethings of they ballike i wanted a big taught jill hone was and -he rabbit to havin after the but help and nelpft but it was surpring take to ``` -### Output Quality Analysis - -| Quality Dimension | Run 1 (CPU, 0.82M) | Run 2 (GPU, 10.82M) | Run 3 (T4, 1.99M) | -|---|---|---|---| -| **Sentence structure** | Fragmented | Full sentences | Partial sentences | -| **Story arc** | Weak | Clear narrative flow | Basic flow | -| **Character names** | Inconsistent | Consistent | Moderate | -| **Spelling** | Many errors | Mostly correct | Mostly correct | -| **Word spacing** | Mostly correct | Correct | Correct | -| **Coherence** | Low | Moderate | Low-moderate | -| **Story phrases** | Partial | Natural | Present | -| **Paragraph breaks** | None | Present | Partial | +**Analysis**: Clear sentence structure. Named characters. Logical progression. Some linguistic oddities ("felt dizzy and wanted to share his happy") but unmistakably a story. --- -## Loss Curve Analysis - -All three runs showed the same characteristic loss curve shape: +### Run 3 Output (1.99M params, Tesla T4) — Efficient ``` -Phase 1 — Rapid Drop (0–20% of training): - Run 1 (CPU): 3.30 → 1.70 (model learns basic structure fast) - Run 2 (Colab): 4.92 → 1.03 (steeper — larger model, more to learn) - Run 3 (T4): 4.62 → 1.25 (fast drop in just seconds on GPU) - -Phase 2 — Steady Descent (20–80%): - Run 1: 1.70 → 1.39 - Run 2: 1.03 → 0.74 - Run 3: 1.25 → 0.96 (consistent improvement throughout) - -Phase 3 — Diminishing Returns (80–100%): - Run 1: 1.39 → 1.31 - Run 2: 0.74 → 0.72 - Run 3: 0.96 → 0.93 (still falling — more steps would help) +Timmy and elsed him to tell being jumping things. They were tired and making +some pinkets and help paper me. They had to see them, drain and ran ar her +mommy. They also fast with the stretch and sacks the changer. + +Lily's truck laughed and saw a rock. She said, "You can't here some wet +sicks. You have something new favorite toys, I do yours." ``` -All three models were **still improving at the final checkpoint**. None hit a plateau. This means more training steps would reduce loss further in every run. +**Analysis**: Narrative present. Dialogue structure intact. Characters named. Some word-order errors and made-up words, but the *shape* of a story is clear. --- -## Overfitting Analysis - -**No run showed any overfitting.** - -In all three cases, val loss tracked train loss closely and improved monotonically across most checkpoints. +### Run 1 Output (0.82M params) — Minimal ``` -Healthy training (all runs): - train loss ↓ and val loss ↓ together → model is generalizing - -Overfitting would look like: - train loss ↓ but val loss ↑ → model is memorizing +when years me told be found a big ea reak abig driendly they named not she +rabbit smiled by aded he what in again one smiled the mushrought boy one day +and was arroom him that she rabbing animals the dreezed at neard had to there +man owl them with o box ``` -The train/val gap at the end of each run: +**Analysis**: Word boundaries mostly intact. Character names present (rabbit, owl). But syntax falls apart—the model is struggling to hold sentence structure. This is a 0.82M model trained on tiny data; it's learning *something* but hasn't converged to coherent text. + +--- +## Project Structure + +``` +. +├── .github/ +│ ├── ISSUE_TEMPLATE/ # GitHub issue templates +│ └── workflows/ # CI/CD workflows +├── .vscode/ # VS Code configuration +├── GPU train/ # GPU training scripts and configurations +├── assets/ # Images, diagrams, and other assets +├── checkpoints/ # Saved model checkpoints +├── config/ # Configuration files +├── data_set/ # Training and validation datasets +├── evaluate/ # Evaluation scripts and metrics +├── generate/ # Text generation scripts +├── logs/ # Training logs and tensorboard files +├── train_test/ # Training and testing utilities +├── .gitattributes # Git attributes configuration +├── .gitignore # Git ignore rules +├── .python-version # Python version specification +├── LICENSE # MIT License +├── README.md # This file +├── cleaned.txt # Cleaned training data +├── gpt-from-scratch.ipynb # Jupyter notebook implementation +├── requirements.txt # Python dependencies +├── traindata.txt # Raw training data +└── transformer.py # Main transformer model implementation +``` + +## Loss Curves & Training Dynamics + +All three runs showed the classic learning curve: + +``` +Phase 1: Rapid drop (0–20% of training) + - Model learns basic character transitions + - Loss halves or better + +Phase 2: Steady descent (20–80%) + - Model learns longer-range patterns + - Character names, sentence boundaries + - Loss continues down but more gradually + +Phase 3: Diminishing returns (80–100%) + - Model learning slows + - Val loss still improving but incremental + - More data or capacity would help here +``` + +**Train/Val Gap Analysis** (indicator of overfitting): | Run | Final Train | Final Val | Gap | -|---|---|---|---| -| Run 1 (CPU, 0.82M) | 1.3191 | 1.3145 | 0.0046 | -| Run 2 (Colab, 10.82M) | 0.7259 | 0.7176 | 0.0083 | -| Run 3 (T4, 1.99M) | 0.9307 | 0.9250 | 0.0057 | +|-----|-------------|-----------|-----| +| CPU | 1.3191 | 1.3145 | 0.0046 | +| Colab | 0.7259 | 0.7176 | 0.0083 | +| T4 | 0.9307 | 0.9250 | 0.0057 | -All gaps are small and healthy. Run 3 sits between Run 1 and Run 2, which matches its middle-sized architecture. +All gaps are tiny. **No overfitting detected.** The model is generalizing well to unseen text. --- -## Scaling Laws — And Where Your Model Sits +## Scaling Laws: Where Quadtrix Sits -Screenshot 2026-03-17 171921 +The Chinchilla (2022) scaling law suggests: **~20 tokens of training data per parameter is optimal**. -### What Are Scaling Laws? +Let's see how our runs align: -Scaling laws describe a predictable relationship between model size, dataset size, compute, and output quality: +| Model | Parameters | Training Data | Optimal (20×) | Coverage | +|-------|------------|---------------|--------------|----------| +| Run 1 | 0.82M | 200K tokens | 16.4M | **1.2%** | +| Run 3 | 1.99M | 28.3M tokens | 39.8M | **71.1%** ← Best balanced | +| Run 2 | 10.82M | 79.6M tokens | 216M | **36.8%** | +| GPT-2 Small | 117M | 40B tokens | 2.3B | ~1700% | +| GPT-3 | 175B | 600B tokens | 3.5T | ~17% | -> The more parameters, the more data, and the more compute you use — the better the model gets. And this follows a consistent, measurable curve. +**Insight**: Run 3 is the most *balanced*—the model size and data quantity are well-matched. Run 2 has the largest model but is only at 37% optimal data coverage, meaning it would benefit more from adding data than adding parameters. -The key finding (Chinchilla, 2022) is that the optimal ratio is roughly **20 tokens of training data per parameter.** +**Next steps for any run**: +1. **More training steps** — All three were still falling at the final checkpoint +2. **More data** — Run 3 is closest to optimal; Run 2 would benefit most +3. **Larger model** — Only worth doing once data coverage exceeds 50% -### The Three Axes of Scaling +--- -``` -Parameters (N) → How much the model can remember -Data (D) → How much it has learned from -Compute (C) → Parameters × Data × Training steps -``` +## How Generation Works -### Where All Three Runs Sit +Once training is done, `best_model.pt` contains frozen weights. Generation is simple: -| Model | Parameters | Training Data | Optimal Data (20× rule) | Data Coverage | -|---|---|---|---|---| -| Run 1 — CPU | 0.82M | ~200K tokens | ~16.4M tokens | **1.2%** | -| Run 3 — T4 | 1.99M | ~28.3M tokens | ~39.8M tokens | **71.1%** | -| Run 2 — Colab | 10.82M | ~79.6M tokens | ~216M tokens | **36.8%** | -| GPT-2 Small | 117M | ~40B tokens | ~2.3B tokens | ~1700% | -| GPT-3 | 175B | ~600B tokens | ~3.5T tokens | ~17% | +```python +# Pseudocode for generation +seed = torch.tensor([[start_token]]) # e.g., start_token = 0 + +for _ in range(num_chars_to_generate): + # Forward pass through all layers + logits = model(seed)[-1, :] # Get last token's logits + + # Convert logits to probabilities + probs = softmax(logits / temperature) + + # Sample next token + next_token = sample(probs) + + # Append and continue + seed = torch.cat([seed, next_token], dim=-1) + + # Trim to context window if needed + seed = seed[:, -block_size:] +``` + +**Why output differs each run**: The sampling step is random. Same weights, different random seeds = different output. Add `torch.manual_seed(42)` for deterministic output. + +--- -> **Run 3 stands out here.** At 71.1% of optimal data coverage it is the best-balanced run so far — the model size and dataset size are close to the ideal Chinchilla ratio. Run 2 has the most parameters but sits at only 36.8% coverage, meaning it would benefit more from additional data than additional capacity. +## Known Limitations -### Full Model Landscape +1. **Character-level learning** — the model learns characters, not words. It cannot reliably spell or track meaning across paragraphs. -``` -Model Parameters Data Val Loss Output Quality -────────────────────────────────────────────────────────────────────────────────────── -Run 1 — CPU (this repo) 0.82M ~200K tokens 1.3145 Word fragments -Run 3 — T4 (this repo) 1.99M ~28.3M tokens 0.9250 Basic sentences ← Efficient run -Run 2 — Colab (this repo)10.82M ~79.6M tokens 0.7176 Story sentences ← Best quality -GPT-2 Small 117M ~40B tokens ~3.0* Coherent English -GPT-2 Large 774M ~40B tokens ~2.5* Strong English -GPT-3 175B ~600B tokens — Near-human text - -* GPT-2 losses are on a different (larger) vocabulary and not directly comparable. -``` +2. **Output coherence** — especially on smaller runs. Sentences drift logically. Names disappear. Tense breaks. This is expected at this scale. -### What This Tells Us +3. **All models undertrained** — validation loss was still improving at iteration N. More training steps would help all three. -Run 3 proves that a small, well-balanced model on a fast GPU can converge in minutes. The next logical steps across any run: +4. **Limited data** — Run 2 is only at 37% optimal data coverage. A larger story corpus would meaningfully improve output quality. -``` -1. More training steps → all three were still falling at the final checkpoint -2. More data → Run 3 is closest to optimal ratio; Run 2 benefits most -3. Larger model → only worth it once data coverage is above 50% -``` +5. **No long-range memory** — transformer context window is fixed (128 or 256 tokens). The model cannot reference events from 10 paragraphs ago. --- -## How Weights Produce Output +## Technical Details -After training, the model is frozen. The weights file (`best_model.pt`) contains all the numbers that encode everything the model learned from your children's stories. +### Model Architecture -**The generation loop:** +```python +class GPTModel(nn.Module): + def __init__(self, vocab_size, n_embd, n_head, n_layer, block_size, dropout): + self.token_embedding = nn.Embedding(vocab_size, n_embd) + self.position_embedding = nn.Embedding(block_size, n_embd) + self.transformer = nn.Sequential( + *[TransformerBlock(n_embd, n_head, dropout) for _ in range(n_layer)] + ) + self.ln_final = nn.LayerNorm(n_embd) + self.lm_head = nn.Linear(n_embd, vocab_size) + self.dropout = nn.Dropout(dropout) + + def forward(self, x): + B, T = x.shape + tok_emb = self.token_embedding(x) # (B, T, n_embd) + pos_emb = self.position_embedding(torch.arange(T)) + x = self.dropout(tok_emb + pos_emb) + x = self.transformer(x) + x = self.ln_final(x) + logits = self.lm_head(x) + return logits +``` + +### Transformer Block + +Each block contains: +- **Multi-head self-attention**: `(B, T, n_embd) → (B, T, n_embd)` +- **Feedforward network**: Two linear layers with ReLU +- **Layer normalization & residual connections** +- **Dropout for regularization** + +### Training Loop +```python +for step in range(max_iters): + # Batch a random chunk of training data + batch_x, batch_y = get_batch('train') + + # Forward pass + logits = model(batch_x) + loss = F.cross_entropy(logits.view(-1, vocab_size), batch_y.view(-1)) + + # Backward pass + optimizer.zero_grad() + loss.backward() + torch.nn.utils.clip_grad_norm_(model.parameters(), 1.0) + optimizer.step() + + # Evaluate and save + if step % eval_interval == 0: + val_loss = estimate_loss('val') + print(f"train={train_loss:.4f} val={val_loss:.4f}") + + if val_loss < best_val_loss: + best_val_loss = val_loss + torch.save(model.state_dict(), 'best_model.pt') ``` -Step 1 — Start with a seed token (start of text) - ↓ -Step 2 — Feed it through all transformer layers - Each layer does matrix multiplications - using the saved weight numbers - ↓ -Step 3 — Output is N numbers (one per vocab character) - Each number = probability of that character being next - e.g. 'e' = 0.18 't' = 0.14 'a' = 0.12 - ↓ -Step 4 — Sample randomly from those probabilities - ↓ -Step 5 — That character becomes the new input - Go back to Step 2 - ↓ -Step 6 — Repeat forever -``` - -**Why output is different every run:** - -The sampling step picks randomly from the probabilities. Same weights, different random draws = different output each time. To get deterministic output, add `torch.manual_seed(42)` before generation. --- -## Known Limitations +## Why This Project Matters -- **Character-level only** — the model learns characters not words, so it cannot spell reliably or track meaning across sentences -- **Output will not be fully coherent** — story fragments are recognizable in Run 2 but still logically drift across paragraphs -- **All models undertrained** — val loss was still falling at the final checkpoint in all three runs; more iterations would help -- **Run 2 still at ~37% optimal data** — a larger story dataset would meaningfully improve output quality -- **No memory between runs** — each generation starts from scratch with no prior context +1. **Educational**: See exactly how a language model learns. No black boxes. ---- +2. **Verifiable**: You can trace loss, inspect weights, understand every line of code. -## Real Model Output (Run 1 — CPU) +3. **Fast**: Even on CPU, training finishes in under an hour. On GPU, minutes. -``` -when years me told be found a big ea reak abig driendly they named not she -rabbit smiled by aded he what in again -one smiled the mushrought boy -one day and was arroom him that she rabbing animals the dreezed at neard had -to there man owl them with o box and said you s mom that je animable went her -``` +4. **Runnable**: One script, one dependency (PyTorch). No cloud account required. -## Real Model Output (Run 2 — GPU Colab) +Quadtrix is to GPT what nanoGPT is to transformers: a stripped-down, readable, educational version that teaches the core ideas. -``` -Upon a time, there were two friends, Jack and Tom. They had a cold doll in -the sunshine. +--- -One day, Jack saw that he was universed. He used the sky at past it to march -around the garden. He had a small ball on his face. He felt dizzy and wanted -to share his happy with them. +## Getting Started -Nack knew it was feeling important to his passion in their rooms. He knew -that night, he had never seen a small boy just soon could drink. +1. **Clone or download** `transformer.py` +2. **Install PyTorch**: `pip install torch` +3. **Add your data**: Place a UTF-8 text file at `data.txt` (or edit the filename in the script) +4. **Run**: `python transformer.py` +5. **Watch**: Loss decreases. Weights save. Text generates. +6. **Tweak**: Edit hyperparameters and re-run to see how each affects training speed and output quality. -``` -## Real Model Output (Run 3 — GPU ) - -``` -Timmy and elsed him to tell being jumping things. They were tired and making some pinkets and help paper me. They had to see them, drain and ran ar her mommy. They also fast with the stretch and sacks the changer. They play and them together or day. They tlike to need to stay and cut fun and have to catch him. But the bird is pretty. You have to make your legs and it's some people truck in it." +--- -Lily's truck laughed and saw a rock. She said, "You can't here some wet sicks. You have something new favorite toys, I do yours. All of fun!" From that day on, Lily always callimbed the slide, and Tom were playing with the surprise, loved to play in the park. One day, they went to the park with most in the bathting to girl and dinner. One day, the family went off another floor and quickly made Jack the toys far away. +## License -In a weath came and dancelet every day out for righting. It was a lot of big ball towers and eggs and make him lots of them. The man is a perfect of the bad lettersser on him. -``` +MIT --- -*Built with PyTorch. — [https://github.com/Eamon2009/Transformer-language-model]* +*Built with PyTorch. | [GitHub](https://github.com/Eamon2009/Transformer-language-model)* From b4f56e51f19d346a274ef362eb20fe526f4c4d6a Mon Sep 17 00:00:00 2001 From: Eamon Date: Thu, 16 Apr 2026 16:52:34 +0530 Subject: [PATCH 13/27] Update README with scaling laws details Added information about scaling laws and Quadtrix alignment. --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index 284dca0..84defa7 100644 --- a/README.md +++ b/README.md @@ -338,6 +338,8 @@ All gaps are tiny. **No overfitting detected.** The model is generalizing well t --- ## Scaling Laws: Where Quadtrix Sits +Screenshot 2026-03-17 171921 + The Chinchilla (2022) scaling law suggests: **~20 tokens of training data per parameter is optimal**. From 814173f77399e370101e931fc5454b0ff3186e0d Mon Sep 17 00:00:00 2001 From: Eamon Date: Thu, 16 Apr 2026 19:41:11 +0530 Subject: [PATCH 14/27] GPU information --- GPU train/image.png | Bin 0 -> 55279 bytes 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 GPU train/image.png diff --git a/GPU train/image.png b/GPU train/image.png new file mode 100644 index 0000000000000000000000000000000000000000..d4898edfdca8b03111a75667e6626b9cbad4c7f1 GIT binary patch literal 55279 zcmcG#byOTr`!5I~!7U-Uhae%i+u(r^f;)o*cXx-N2{Kr4cPF^JySof-gUjH|lCQk` z?%DmFJ@=k-|6rJ!uI{SpuC97M^8ApOmBc_LLWP5a!;tzct_TN*Pyh!9-;0d+{EHh! z-pA*E@D7TSqHtxSB>T^WmnI@IB5-gOAhZX)SI=b>+s_&faB!GifBxY6YzhqF;N}9P z#6^@{bq*iBY?ahn*B-scFzRAac0(eX414!2g=CG22OJps3BO(!R{UHRk;;=#`VcFN zt1yL|;z&B}h!M1GMsk43wTsE6G?yUoDkNVtRB1oqtyy4^<5!uG0@SDzoG)oam{4?z ztH9ag@0+RCh{X|2*}R71N?Q|;1~yW_rAkI(;-S-f<?lgZa)x z;>9Ik!~=4U$+=16@qyJ>$Ho((OOvKP zzkN*MEw46oP#4Y^roG}hiH}h5B*5$5tBZ#{NA~mCG^Zz_QG6?X5g3egATg$x9aZPM zF`3^r!JkhsABwejj_mL}YF4moS`wz?}HG`H7m;{pDkxx?R$CG4h`iS`KB1xGW z?Ew0#lJ0oHL*X{}B6oF*N^*U2;4`xCBTOW^kY2!n?L_4^MbzV}V&#>W_$ubQm3?%R z*>YWC3eH>+g;(D_jSaHu+bmc%3nhegd3fFO6nhyzZ^-^&+DZY_`Z|lp!LXWR|0`zx z!EWjGeJHa+^Zh3JA=;W5ohNtdAr;dy^YJ&>**7zjJl-a`zb8V0-tE2^9eGR6?v?a} zr)eb*Anha>n;S-H>-oMMLUr4}YATouxy{U8MKQyBS4|$i)79kfR?c^It58P>2DS-C zlg|?md(h_;@>w1ZNUk0Y*0t>oOc9Sk8q?%b(Pcj8TZe812W8ztRpP9g5YLm!cCQ=& zIFoYtMf|g(%kqNFt<@{A(a1A^T(!*nNvvM}trxTSxrhVsOe^9Nzb;TV@4T^WH+U6g zYo^mA6zuSxd_i3g9D~S7&L!dx_2Tfd>z1}jQI`ahgiV~fdVC@Dw0?Ks{#_BYlped`gZ{B~7q!9W366MYN zY14#eBqzMMqfp%7xC6D|kT{H$Ln<9T0&8$aL|J~wXgv}}`_RP*Q~y(3=zWkB7%_K& zt9y#1h{T8gv+Hk;SVeyFTmvu4sM{6C&8Ew?o&4ReAFRw-1UA<*){jIMJ+j|(c!a4$ z)_HIANX3bt(7d?MdVeHkyx~ZjneK@5WO>kH+9z6Fi-@9IYssHr{I)$;mV~CIHh5lz z^wB<p?g}4&s!=q~IU?Rh3&>F{~zGIeJsBvbXZuQQKr|1qB>Zi^q*g;q?Tk;E?Rh z@Kp;*>*HA>VE>pl#dq`+3E^#98XjfZ@C&Bt6MQaJF{;X%U$bgmagdw|7pJ$r+cJSC z`}qwh{W@ElUV&{y*_*>GyzNuxR0dNEvJb^xjw!V%^;K#Fq(}#E1$x3AJk$SPMHI4k ze)~0&oJe3Ta|BseDt9!sgxUroj-)6caC~eJfBX85;E8m6tE1#8-b@SpDT7k-PLl|L z7D2uB)XK`W-A-M8~IpchHLAC5_wN1CE@F@ za@}@xl+-aCqu+;Sd5(T5xf8uH(y+ ziSK!XC<%|m=<|!Jao}^)-AR~Exlx(%J=2MH@lUk=9f1^{OT4F(2D@_GO*f{`WA_T8 z?1wci_^WD~^1fklza=~FR}OjSKiH>C*sMa6l_`xY8O-!%zUevYMz_D#T zyKAyobNu#%?}z8x%nPv7Vz#Zoi&NahVv%?FVQI%VOFH!yz090X$xr@vy2A-s*ZrPAG1C z|B^n*)0hNIIeXDq22og6dR&9x$(_Zwh>ip1D%!RIK%4eLgENWo550e&`}zIcUsj*n zUVeA`>UOWDRQxE*!T&`AM|xVqd<+T^LNNcnfF&UfKU$_s&>na@TQK?k*#O*G+uQ)> z?LHq_emT$b;)cD@ijDJ>4dG9%V@wz8#sY4&0L|m1dBxCX*oS=*9<^X(tb@HfiV3Ts zo1^WN2?URoryq$5*UlR@-`;+rmThahq5aJI9=Ek#PGZ*J|-Fe|%}8bg5= zJA$wOx(WJeBYFt9$AwpH29%mfB(^Suj6N3VZeevEw#%<1m4>?AVvWZ%{hR^PcxzEV z0A>d6nIA2^XM~I&tRA)SGj)@yP?@iB_}ronJ+BrQ3q?BHEW0-Qqn-*tqZiPZ4u)0< zbKG=1mjp+}k1u^@WtXskW~-#MRns!IF5rOl;KWt?SSLMatD`kx8>DjUedgXR>tvWi zEdm&e|A%+fdoM&$M%^XFd`o}SXTO^Svl?gSqtpQhpiBc& zKnQS-cW>WaW@n0nUx46$xc`{ndxO)PB`gSmn!UebSj*8-nHZHsArFu49vaL~5k`EB zras&D=JP)6yZ(*^1Mw5w+aZ^uea1Ko)o4{dvjo|j`@jXFSNLb1Qx5iVTI-Q0g}gt@ z)_b?hIlrCvnj%|iR<0Crk00N-&zyE)=<)E*F6U5t>^u0Msk}^kUE#+k0p(hAjWr4^ zW2-?LW58cIYu)R3^&a%GH&T+l3H*k; zw;%p^MyTe2EdNWe4~Fbo@z)}T$7KbSL&9%cmq1tYv1oycP#NG5@uzL&7yKT33uZt{ zH=5qcEx7zXH7*my6=NnkNB*vWZ0#9|xy-VkXz*89`QZ+A&L;cIR_0yBaGP!0SyuIo za7FRsnLJ3`O?DaVwgnD94blRLG9YtQ-Oe6%Ua2{--DG6asuOC714Q|{pm=9o8|TJZ zx3FKqBfL{^Y25F+g-~#`;O|V3T(6K)w=r&Bwfqw;)VDniKeQy81%&7vbV`3eHu0}j z(HW_4^+yctvgHa(NS{P{uFTF*VyTPU(iyRc)CDq8hjc(|;dx zi^Fo6Je?#EtU6shs6ThFLKDBN^JKU(gh6uHUEsn&y2!5)=gB3(`W)j`?nyx1_@h@x zLbu4IwU(}>>C--!>+hu>z}!=+wepHJeFwR_T0&MINqP*nE4J%#X(Oq;vM|7D=V_2M z2R^Hhz7~&k%=7qP&ZoEuXbSL-X@B}OdFpz*3m&%bTIW33Aw^KPTcyrtGU5`&O?~vv zZ_>B~uWZO_SniV)FJ+~yZuUq;ggtjVj&sAA_nUz1b(NI_8GV{sztLpx*pT}0lGnSx zLN@X6Mi8KYXa!c@)oruYUbyDn7{$K}Iy)00su;!9(F7i*CyKA{8eP6cfd0I_9g|uu zIdxzkxZZ=MB3z?p>!&yIeub^dW@nxTn4`b8X4$#pB?_bL`j||kZQ(?8RWLRmM7r5- zg!zf-h+A~&n5w%5oi}*s#(I>5ka09>rOMRtZA6zn!}Nz8t{DQIFSoWP`HDAbxfy~A zKatjgM)$1NpmcW_K&_d+qzs28rgyq~hVcY_2dXJsY-SsM&aW#uW#J_x+2IT&!oh@$ z_}K;)v=hqD@$%2Jy0(Exff=(UHFU`q;%&Qrqv-d7e8TrZvl^VTAIiuq$u)J5a~(=Z z(JCRPF94AC+(x@55<{E!Je5n9DT8BLq^rWch)5f<%KeSYj7(IjZKF2A?4z#d4!C2?6%xTo`EdbF`f$? zfdL&!PSpp|2XH|xR($cjEW*PR%_cO2&548Kk-hb{56lh~I&LVBzG51#pV$b>LMO^vSKcP96V0B*U zCK!0Jl)=7}KoIO6@psh8Lm+L>r<`i}RJBO_I%=G%1u;hs5EmGUiZeT{Q;S?ziM)fF z{sVlmIm@6+PHg|fW0T*;*2O(R#A0H5FC>+*ko_O%T)${Dr?T)L$V|BtkAC*QkDvd0 zMu`5xP-f12d=AFMT#pp~oqu?Ke>Z5=TSqDU%^zg!5&I8{5BZewJOCB0Kd@fF^x9PMxF znA|BoKrM{PtgMDx6%*^ZaXkSj3#it-7Uci#ymJ0HiZyO>>pg1_a>U7f9Io-y+EB_Z zxZZ@{l}m>M(&x*yFvjX>L-G9w)4N)|mxUew_>}GA5}T>~LED@~^q$_M=%66^K1BPs z78vqDV)XJgLlX(TAKC z2YYl(A$h0bIhz~OHYIx-Q}3?w3uf-iwB#cF7li=jHCZ%Ch=s+RH@~KFrZg%(5#Og5 z5E`j0dw8Os@k3oIE}g?&_VAM7goIFub1iOu0q3g=0Zk#AXZ0lYsx_7MUr7JqI(}}8 z=iYG_yu6kif;i1_N{lW-d}fksb!d;9o}Ue)T@9ky^ZPrF_<3L0KX`X;abvbDrvkG+ zYD*2yfJm(@S4vkprZo|P!e#2=nloHssjrGVjOP)w_6+yuxrLkSF5yULP}67r<}XGb zbsJb!YnkOy1d7{_7#5n=pb!0cx{S>0u$ExqDl|z|X0Nt9yzd@z*bA0&Sm7fQ#z&@h zOWA8!Np9$7mv#n9>-AG=oQIMMHA{&b-e@BCVjuV%Hz@dZ95>y&1x)^Lwjyq_a4Cyq z`S_FFPQR(yWFm>|%oyYr z?}S2Pnn`wBuBo8?J$kZ(KEK8X{107aI}+KU@dLbeHm;#qa>MnP<&_Uu6EKB@$g8H6 ze`yp8JVFzRF_sUop&O6CSrgk%@{v2)S-+|Q={=jK-;O|Kt48-!J*EQ-B7_g5w(D*a zf(B!MMV?7pNVNrl_5z}pCsXL@KYxdOah_ik%W*+^QejGg@`SFPDP3BfZFrNgxi1#f zZMM&lS-VO0~!?tNlMH7*Va7OlI=AtL8F@oA%r}0{_SnF#9&PY*&d^%#DJa zM=*NM2|y0keR8YePJ8|;wHtqzy{e#nrmi68k941)s0iK*y9F zO0;k1odD(O`vcb)2X2oC!mS3PN%Y^%(_*5YJjW|nE+lx!7j~JSnrzW4*s0teMcy`_|Fm4o`=(S0odVk0=^gGcso*8PVc3f;1z z++hcnvAgTcDcwHJ4b~BFr^S)abDpukF(%lle9$Ex#oB*9sDH~e^R=#(nY`_pcY;)S z%IN`XGKhY`2c=_tgqEKraf(a zc)$wvF7&ZZZ!et8JaX?wwyqj^i^{FSwj}9ih*7WVGi_HbU||7?f#HDL{wD2PC_bRa zqazIzwl-FGVl{?TvfXQ8}{!rA!`D>onkYYMURD)9yA!e=s!$wv)Dc~iHUC$19ICQVyrzss5HusU<;m1=c& z)Z*FZFdO*gfh-^wtLE6Ddtk9vcAE-!q#iOm zOR+u_D4s$3l^Afy#f1iUM|N&8JkbQ;=u+c2bs?!*hg<}OA5 zNEu`N0y4>>LZ$2S`uEmcV(gEDUZy)F$#Dj;87FN?^26-hYDoj46i)BxLhUr}&h0J@ z$-}PI5ijkZlX_keTBPszG&EXj(xV`Jcb@|%Ue55XNbS+C1PE2FQvK>iK0#xbAAZQ- z;?+PjEY~sAKXRkLRxHQ4*ni@dzx}xmi}P!<%*~(dnjvZ3HZ`wU-lE*>V0i*Er7=kZ zEG9ZW8p3ZUO0g}y6@_(%`=c-76aB^GoUfdWZQwDzb^V!sI>7XOYNTgcF>F;uekn=8 zI$vtFnl^!Ua44tD+*FOT8s&of{zsZjj6<2;(g4$<4hptwcBo%CS%2>B1C>L@iUsNr ze6QmmVk5>|qR_!%R6kt_sW5*JuYgl1oS)i@>b5F`asY>To?+Fw-?kobC8~N^OL90Knk`N1@02vBgI!$X6Fe4I^(K9@2gHDON#9w+#w&)$ zB1Ww3xx@5AcvfC|*G`#QswR-JcAUX88T&qOnzNL$7XHu7P&s$D5c0$}h2bHRGfN+}sX$9#iE*kHkP;I@nU|Hr#Tfg3g9_Nn-dXBZ*uX;1_H)7xcmwGL-o1NkynD z=OMklxX~9KUk;xL?jl>U#d`F>LY$yADtWvegOmG(hPOv1OJTe8ScQMJ*I3|%g#+Ei zoNVl_D2ZoNY*o1;EzX_sbLQ8b@)uzRTe|& z7fv--e0E?)VFX+WpG0BZjZ7zt++Y*(t~P{&)}f?M!^?1sn=%}G?jV!-jD*g6gXI^l z-Fl$Jpxe0Cadp8|<4dmkq;%#h4rNyl9WvBVo=dd>3 z(cJA-gBWeZOiVAGgx-!uckOpp6@rxdq}e6(o4l$?zuNAWi(QhS;;iC#vXf=sV~~>u za%Y2xJo=6Li%#+f1(hUP4@lg*(A>l$1 zv8zRDYhJlMuF1tC)!eHBXm#v4T%QOgoGu}Dce6=0m+?GdZaDe;D6MN$_4yoVqfINJ z#QoaR%X1JKGQ;e!a0DJ(xb;#&PiD_ix;6Eh=w&2$)$)sV;3Ha)4T5ZZMM|@q&z;Wc zI&t;bEOW@4dm9NG@rxiuUhQH2ctOrx*C12yTCW|q%%#alb?aK!+~U{0YD{V1X|mBn zdm2S6{_5rzJX3WsgUXoj=T=H75gor3IvD>-x=1Hx-b)gAdtW-7h7RDAABLIRVC#|^ z1~+Q7E%NNl5J#|h635n9?MRQaWNtLrj-RXXW+kfzGL8O?HMX2vGquxA|7t5qZDOBikM-pno>n^^i0jD zN+?U;(zX>iDjf$-$X%}@T&DB1r+Ga>`!yQo&cSEM!V#7dxsb34hnTzHq$7tYP6Axn zlQ6PDALb4dTnc|KLr}6r&7b(rYO$ak;RZ){L;=cI*=!H@hPif;J&Y_My-?rG%!%XYdZ_dX5ZT&?beqJ!992*Q`0gdrJge zby@|b60*=glw(e?MTV*Fy+9hGg~5WQ{KvR7#G$2&!OZHA7^nz;E5?zypmaYk=;u@U5m%Et&{;RwPfEMPT<7$;~v_<;SZ?>6m#S0nLo$8sV$$ zg|#r9(@%1`?-AhJl>#bT1eJoAW;*<=$@-4>5sf=pERAgySY@vJ2)u(Jx#*P zC}N`brM&~LC~N$|U~|~0Z4_R%`A7`Cc}c8~xK*QEbm&?|OZYD#qBUh!M&K91+i9om zfht(r%f}waY)?+9CVXLj(mV&bTkLwm|QcKf%$ z)*!RN?uwEL;-c2!!YYM#9!qECsfU>9vcv~5aAt3Uzkr$C-}zzkq-mKE55yU}%9+($ zaa`nA&f81fUDr%K4($i+j;0=y)6xA*uU(}ZXzK0s%U?|#u?JQa&i%NpWx-H>LqhhI zsH-n-Ve?B@9cppuZMwpacVKn+&chg?A~%79ndvEx5I(Ip(f;@y-0?9v$+by^^Cgh1FS+>>bh6mrjpG-@DV~!VU+0{#$I(NMhy@WA_dM zTyFcc@lG1>UX?Fw?j)4zsHu2a(V(_*C=fL0jh4#NO8XCb!IoW())zE&cF?Vps>M?p zzY37*%h`k(B5IB3JOIJ(&O0dXsPCI6%n7os;0D>CxBSa`cvK7KqR}@Km@ksUqeRP` za^p?d*{0l66I{@ygRee2vP^L7?I7JQNQ!MR{j)=W6pFNqX6pzQwG`ra1r_jBh#)Py))o{1fnk)@_*uDLSO*fVqwfiCOuvcV+;|pQ zg1&jybxxXHZ>_QHH|^NA)L7Wg$)q4glMTJ#z3lh_wpzacdX}PMx>?@Qji4fh<$l?p^We-~khwISLBzPkLQlJwrjmJPG!CO43kIe$bi-!kJ z`cI7N2)Ly&H`^p-#&UsYD?6v9x#9KKwQD-&K*}Ss!sIT;Qtq8ht%s6HDkC=1Y5|Pp zu!%A{x}I?vea~p{cJkmIMF)x)r*a%GiG;{`4dX$peZ@`>oF5~<8QAzX0^(!diZM#m z6+5y{D+>ZeQYSW!9BMSAb`)gweqLdrS!t^-j+D=}NJR=h!Z4gD;%dZnQb~zPek5v_ z8Csh4MiYAWFc3JTqAn@F@IWNseZ*Vst9mbYs6RDZxICK~=ef~D!?b@7lMZZVmwZExnT?PZH#eg4*(pe< zGPYau#SzV#xp>3vha;QX_`p#E2NiQQ{o#Rpkscdp?0mkTxMFYf+6Ds;A4l<0yzk?rAC(suIE2oWUW+6NEt;THLW}PCNKbih0&Flla zajSAiXjHmQY03lu3uXttA75vcG%*-O4o$hOgm`*;4jFcir$A8Zm{~o0U7bSt;KI1z z+7`XW)r6~@V2>?HT=W5eL&M2V2ST&Cvtdui(edC`I!?%3D$ zM&$fRV(f%cx zVX{DVBM8vQ-nPa(HoYLt=X5R^m3c^0+7HjRaTa6p_uiCSh9`5vRec*e^=L^PwZW>) zgS4|dMNW6)jKELT4GWKYM7{q&!VTl;j#A?{iJf?8vFl7;;W^&vUnba=^9=2!!H=Ns zeR)57#A?vf=bSVD~$dETHdKUfm!>JO1HEJu(X>85nWvkFKD(Ai_HeUwe-Pgt3=o zi`;t~WC^#;>aD!-o7OBk5F=(u>dxK1NaJ& zyOZ9|JK|MjxDHizA5GidZd9AgFT(~K^eK*qlv^~~em^NT_se8bR7ff`R<2H1N_a$lRT$kGMoI`tzmzE6jje7)F zbu}gQysaM8l)LLD6?kuxbncV)C&rild5t5?C3cs`ntlevuqJ_(+S(xf(Aw()fBUNR z+2r`t)StM=EKCQvlzqoZRi!)`a;K?)grMd!9z7IY>d+bR2t2-O;$`EOyq%hBrvmHW z&ZSR2_{&NYDtt<;e`Z$F%eO}3B}iYBZ3AUcnSvK?p1k|%z>HTysAKR$Ub+lp&!En2 zNu92+1k>RU*a{3B89vWtl9_H4^lce6Qz@~?-B#5q(6MLZqYvo&g@ z0SDJFk&bQ-skZV4=Ge>T0N8o%LDNBj5Rq3r4T{z3SO@(f)?^X9Mh)Os_AzLBl!8K8){7QC4fk z%AHY72^~UcU;oob`kB+u(I=VLUf1JJrJdej!OI<;SLU(4W#n5|kC%G0LR_RCGVmPPsr}pr`bV|BA$L4Kz-kjY%R+;kAyBBSCK3_uX=tl-i9Q z4TKNf?A=zEcr^BPpEjZYRf;pTpUCFT)#+DdU1>(a$~Rl%uvNv8Xp`CvhX@kOM1SH!h^IJ@Pf2`NAOE_J$K}Je_5t zn~tg+m&v>oeFcK|EyGr~<*#qgm?D(LeHb_yQ2Rk4_S(&9z`dmcZN4=s*$U}Mfu*6Cgn31CBTinWIj7P%vMIRN ze;SbBlcf%CRw{5QU?Ks{8KB#zi6jvggkq4n1($bZH9Y4Y+to)IAn+d;*=o+b-&CajBLYy|9 z{##SWlRc_~;MM*|dJSNCR#ZvFnv!U|RDjJNJyHIaXuHKq{|KCz|It?|chsI0`+t?u z?uY3A2;l!4w&!Qz6yeg7fc8lb3;r)nm38jfK=SM!Trhj~Su{&l)T`g?qA4Jlw`r>x{l_mIkf8hDrs^m3{w3}g zzd65{m*eMqc__qnw;F#pf40Ib{SjwHl_f*7N+20F^0WnuRE*VhgN+;$(%P<&!>H}^ z9RyiqR`REE`a7OUaa?lcU**O9FwqbVmj4y;{;PJ@v{yd7w@-|nhj=0%QPLA3_+j&e z@SYsXQ_M#cOZ^Zd*c&^La^-14XqYc`jX%^Twh|2+5PI-Fu=!K!N(VmzxM34yN@d6V zFtiAM7>i6}Ag}848N1BK=l6e!^@=MMqAM<~UD1}feVQY@8W{#&*k!Oz5{Un^Hov7# zo(J)HH1Gs=RsXeH?{Jw=9U~$&B%4W8%{uY{$d5Cwmg2^icp-%_6Ie_IQ0Z~vU~C|T z%%WzlZ4kmYcYU|41EU9B={3$3187yqAX>UGbI>oFIR%(cgq#<;spo?za?#*i?St+B z2|46X<#GrK7~cY9RiAWy=k{oC3* zY_}KwmmS#SW4BZ!g`OSaz}&OpSGeHO`Pr+>{*N_YG~{0Nzl`;rRPC^(R=ce7`rB<5 zn51VU>>TKo)R6N$PrCs`Qw>koc7pdXo$WhDj~r}$g=kMkag09~+zNgF&qNQvzK;4o z<|63UTACKQkxz(_fj)OAi~3eGgyFpok9~l2%^+b`+ zY@ZG8toBX;^MY-E3@STmMDOecKOJ}mM4E%%VhJ=niGH;O6Bqm=L;NupM0jgcIZ;Z0 zjoH<#!=ooh>P3I&!|#P>L-@{CCuw}g2b6n`9H_)Eqi<73q4IZJUfoWlUcbv3YOu4~ zqg@z8q{Fd2_)Q0-(Z=d#{VfX7#x&Cb=EnbdMO}T>hal|QJH1T$f#+VCL*f^_3V79T z#b}$587^hi_BGVd?eYX2jJ^L#LU@BVT!Iz5;_&SEPu)lZ?shG0buVo7)Qyrg{9g1a zP3(($*EmDg1c!Q?r@pFQu+z|nP5BL!JRIQ3!`-*wEg<`#72xb4P938I`!O~$J#Zhj zNn1FDH~Pz~i7wGrXyN0yC?7o{^_b8{{t=|4hX zg(OqSvdl$VQ(lZ$_m6X}5dYvNJZMjULhmn8o%%%o%9BD?b2(@FZP`zy% z!Mg{AYJ04OGw;=V2C4I^>pdZBkk)6)W;ywS7FkLpylx zaA7X5D0q)$Lxx)xhb1yzkMN>r`2S1d?>oj?UA%1+p#Gl0Dxost*B56Wn-o7UcmWLA zo9rl^F+N)^HL>TkToPfLax6 zlEw&YxoTetq!EwfK-DB&fzDGaTY-gP05l4mJeB$q%}sB-pgOkIB;+@sMP19<+gfYL zso2cBBDEmSpO4vwv|u_PFZ+bn`$1@*${3{inRxDWg|A;L#NLQRYh-M${X4xEIt)|& zp!Y)4?dzUyy6gGg}0Ij4ws1t_|cz*1_M3m&VAbs>`1%2rW z>c8>CDSzE6KBP#2OMQ6(|3yV8+%Fx;j6aD%+8Y1RLO=5H)tTb|5%P zz3H~x!t^g6&EV=haaYu)IGB`mGW>FCf>`p|*Z#g;M& zs6GtI6Kit^1zNEyzHM7F=7u{S*}i8?_LlFdz-t8_7$N~o!akrkmSsP`UKOj+m~cX* zC%uFfBo?`qeN_NOF0JR(px54A22!kIK3~q4?C{g@{Yw;54Qn`D4$aJEn1%?{gC>4q z6_*NV8{R>Lu9aWhBM~%cHcIN>$1ULe(9FXaui*T0&cyR{eycy0UPHnzx}CYA@=Gt! zOwI-g1xr> z$oL~|Q+4D-X>Uze5MbVNku zR%Gf^dv>ElHb({oaW)GqDDFovWCt=me#Inn*<5mt{9F#0YSdV84OFK`V{<7viz__iG4prbK}1da!nu;plk|9b?nGb5#FkNC+V5P+E;;bGgunBpFmq?C zc(f!Ia8?aymKMi?n;z@6BPYTrL+S-yLO&soRlY+rQ`G&QRXb6#^;wrI$-5@YPCMnM zf#=LYEJtujWQbcTD;QH*2z^|6J=~-Uu=1Gy zx5g{IDSws41;-uzQ1shMDFEmGg{u}O)ybX)2@_VwDo)i$NM7k&c-F69NW5&b5F_G7 z-I=R$&5);m$GNad7w^D1?(U~n(=DdvN*QI_3-km6hgiU&&!k{mELd^lh{a$WH zy}y3CRdA1HD8yb0c4PT*F~ zz6UP3xdkFr(`6bv5D5QCaUm&we3;gW!PSqFue&;pMqMV`9^5}TxjI!PZ^aihd23su zDRt}*S3GxnTKe>=s}4SYNS)71n>`#;IJC0N&FNM?nQ{EA$vk9HM)_8>;9*HT|rEeZ%iiWQ{r@iStlhTSFn=FJNPG*~<6Oa_-TE#+ec!fv=daz!t`Cs+p1v0$;U0Z>bG% zaCq1*xwDx2Wp{*o1U~P2e<&zo14EOEjFO!@HLZj~hT)RS^89udKDgW2Q6i{G&oT zMhLNw+Fc;ByC2EwFrzCyCTe;f7YK#2dGZeW)s1LAu5vwK_zHd##QBp(LWMxb{7_xL zrtCE4bD@bI@d8uxiC)q%CW=`Vw!*^uEz#o1`&-Yx+luE0O=JakE?o#t+`2;rEWi6%;nZ&iSPGDy```JN^(9vunA$`+L(L6B5z~XiDT0D98BCKQ>6o&~ zyt1pQR_?FQG41Ici2=7CpV7fO9(bPH_wFk|mV$R)FEJf^^VCNcVMxK;az==fW?O3j z5-1=Pt8WY9Wq!P5MjnIejO9ulGNMQAQ=Nu=KevoTZdZ>Z@x5cqAC>rVXw}y?yLw)% z1BH;RbuS5aw>JSCo7a>1gyi&k``ir@Mu!4@#0je~sLd?wHsL0B`F^PD)SCB+-uP%G zENzJuB3aXfkX3!H=DGQRU$y4hNf4(so2|vFB~s-Sjpxo;U-+6Cq-nd4_4aKKGNhJ- zX8-aHAK%5N>!QXdpKh>c10G(DcF*FsI*wZ+ByrPre&MxcT+BbBcfx#?iVYrKw@h^*5Kw=)o%rO5CvD{`H zK$I{NiCkHV=g--<#wz7aSe`ONe&|22-sLL%)Fe;W85Gq97!L@_O`@NCz)%V|ep1F8 zx-))D!B3LgMbe1c`!>BDi|=Qb@{+Pc&bE)BmOD+_0c|FNG_i(;<(kIVtA{$8sW-X1 zyZ=b29!0#$`3cOQ(DbOyTL4iBgI*RHd9cV=Dbw7E#0M&>n#`;0^hR@|_I9`?5n*_= z=NIy~BQ5Qn6E72U$G98GLb6 zO^^D;f`4O#d-ORrI z#->4?A)e$dXkPPuTARTz$c1MJk}{RFvFzY5k`nQ3+bPu5 zCK_W|E4%tmJzP2JS2g)6`QX-uHBT+Z{MfK+i2qg2?E(h}3L5`FPp&4f0W$o%Qvq*QMt z_`>yxKINw!^=8KQtEHx@R3tZx)X?Q{f|cYsT#!L+OyVYNg~ugv=vjL*#3ILOGlxCr zrJD7Z4o#6^=D0@Mk@7YHH{0k9d{`YVNVKGnEFbL$r1sSyg(cV8Wj0B>8IRC1Uwe$6Hs5iM_iv5u9iW+>Uf)<*x53_ zSB6QN%&uai$sA>Dn}zYFvOj!CB>bfZZ5a58L)*^H?v|~`#PgCT`3ot^7gG6;Lfj6v z376UqM(ck>Kau0!-?_UCCYJ(YW`zj8HK234*-w*29DP@f4RKtk&X*mi+jo)`C=S1h zH(aaFi$>TRX=N|MuUH{c|KMa4&xbK@5chIbQQ9ictzOzw($mzMlTHkx9zQ5R4-*$xkLr=#1M=eT>^QUX4Z7ce*3*ar^*oKUOY=)0%brh>Gr-NQd z^_-8a_&6J!wI-Meus^N+oc~$6FK%@)bq0l0GXyT%1bqp;8CJ$MO1M~v-=o9h@xE)d znUsF0Hei4VO=`WB(f^Y%ap;#0X!`>^t?24HX50^l%kzS|lQOJYyW;f*3vqYyg z^hctbeiGPc_pZz_H_P6YHKR0ji@Qm;>P9gv)2*gSTh%bQ3pTidh}WsyL4&+6xJ6fB z_9lfoZ3Bn7OeIRJ<$3)$4wh?Gp}iNd9wtP~;zQu$LEHNQ__Pt)kmaE-_o3s@YhIR< z!&Wv2FG)RjAmP{Wgf0mk6Qfjs9c~0xWIYNU{JZ#|u!f>SWcXL(g1eh;ZBZCMu!3oy zR>9<9yi~^+V&Y&F9Pj7MKw6PFlS*N#F<4`IniV_UsvP;+JibPOg`qB+zT5l05 zeVwW;{1bSTHy<32y z+@HcM_;@-OtilaX>=bZW$56K?*N_V#uboSrMY*J4u>2(gR>fM(K1hk9{xlZqQbe+g z@Gm2E5w2b1KH42G&kyiyo4i2 zz%h5sD2d0VR4`dhvm0_fj-Us;Y2@rUZ;38KcEifM&3-nozCyoO#)m;B?@N_ zlcu~Q?Pm(CES&<|k#I-52gc{OFGJfKws@k*Z(}{|1;78S7dk+JdhmMIz#tSKUOPDM zi!D~0d~|IL50MpS*7Ou*@}2)JGO$d*z>;Fe7$2MqjboY5)2MY3W@PCrYo~OhrUJ@C zt;nf{B313LA9Xw*AWL2W7Q+PceSiVpdQaDi*jdX8jv)Xeu@g&cp_V}8)Nt-n^@0B9 zHLBen!Rxr5sG>J>wT;6N>4CDrj~;{Puv~m1KYBJTu`g-_TIFjB$H%?_zUB+Ejsd-g z!1hbx5`!7p4Y0FRZbnO8@4o9HezTr8$ukEd4ADWUdu!C0~&5;=fA`G{-@TU;H zVPO~Tl6mqgo~XRIlmubtXmU?D1ESO%+)}p%K!n+2>;0M|qjdRN*hX^WT&%L`^HxhD zsc)TvK*Wd%5=+U{Tf0NX-c<|2TBM;EAE6y&YlVWQ2Y40A&xy^k^S{%@*zaD=>`V4FR}0Y_$2rWR zN^9BHeH}%58LOQbW5G-=Y5S6z{*~g!DfD@j_Dp@bkpNwq?kEovG6uGAyn{MTjsHAH z593X@0Ch&P6pwvzn=4kLGSPi?S#+l-HJ=OOMn=q8qwBkPG;zO(zbG4<9A{ksr313o zgtZEwH>8OB?|yLM#p5c90&OEtpR>IL6c>9<@IOc!zaF{2B_!T8t=h-ZJ8f(WW&VBZ z|IlbI4fGZNBIPIrIjwCaEC8FK?CWj}{X`_7H2&s4$T?}fq|Lym{I2+S+~t?M@4T(# z6LWCOedd~|417AAg)G8~pcdEr88WYzfzj(Gq{bfrQ7hjx_5Ir1%h#o?-D0jyvcLlk0bRkor&&r_7h@6uKGQ*xP_j**rccsjuQBu>NLej4!(#Ff9Tak&@kr~EI5&&?Cj z2Tm@BY-VKr;pO2jdoueE?cRyxnu1BJnno`=Gn1B8aOR0O$C^}(ti5?MOvpM=t5(Wf z)%-YLa>Y?E8k99{b~TmxUezh*OHPM2Sr+fjc_;Z+3o;4GZ0IT7Ere*V6=^4pMe7gt zLGd)EF4nA8A#W;UEQ(FlX^3Osrt%e>_0Ta=O7bFDZYD4F%V+?bu$JdX`W6c~d3+%*ztKNWm>A=jYu$J=vD-;S>9ct>`re zL$|p;C*m^{p3TSR>oLmBhA_EpF6Xe}Ky$Uw$Yw!iUSmlr<5rIaN;FE_B~f65(JvKg zX{F=0^?5}*xd$+%0pJ}nps(I$q(-r8(f@jE+I0Oq1iOY;#E<%Vn}9GCf`wMx{Y?nZ zPMq@A5BV~Xn?q@mqhg%1N{mVTN=;I{zsW>yp;j@Mb(V~plbH18(#S{h)72+suj_>9 zsnJY}&0g)zg&WXn%K>R=uXjYyvGY^~M{m56z8^nbV0b@f?CSe7;r=2r9UEX1ok$^_Ogf>>i=S+}1!31zlQVLvIXzGF*q$1;kF9^0vIV~(&EDW0|0Knh ze2eftUkL9)fM$RI?IrY0Z*oi318QN`V4XSJ0E1lwE<$%ejNzQ2a5b)>G?(iZ+0iLg z^fBXOqjP*WhXbGHAo%&=^HsYKc3Uv6wpG#yCT98z2d>XT4$}7FZVf*ANBYYue}A@f zR;a#IpUuQPEjcAL_rd*dKPpi071^#jmFAj^p;X7EcE5+yYnPOPWaxEk5gH_JHINC# zhr4n)tr-GJldazutzQ)zO5CozfCF_Nl7;J%U}6K)IfLqB0V9eptx02Ac##-;3~`dG z`Gi}qf*;RPI`{rm`_Zt|gik4>p&i(nx(9lPQniya@QnetU6Tv9Rt4W&kElM1YsAM* zGf0k9Gjr98avhn0*MQANova$Fv6z746J?ZjD>oJV4rz|Ir=O<)u+!MS0Q{_O-YXKx z`{=9QY06V78@t>&S6Hk?aQ(1MdobsRfm7SH z03mWU`uUNximJ#iz`QcejMS_`zqKB7O8H$+R>=AC!smJZ@PmmC zNtw$kPB*J7IKmx5^Rb+OoL`RJ#SgC$kOBNd$e z!(LixV|*nzLf@+u;D5bbA|ZF4d1cEfHWzBP5ufx2A~XkR8Kb;7a9gy?@~CN z?Yr7TAbnl$8^u3RNL&_Z{fMU8_mr3g>tww!{$3m*SnX^^WX$X7td1*mZx?p7ta}b^ zbEI*Lp7mA1*-ZpDalk^X)%74w&8Hvb^;QM*DF|{y_6)6lnNlFoPki3mM|8o$Pacd; zIXThvaHR_dp*~`SF1&D@w|wl~ygXG^EP3@&jm9fD{&ML^+c7l!Ukl!j=<7g#vE#ia z2kxzUybT3{EwRV#n=>WiuBr^nFwSqvYtVd7%14Kfk0jrJIebp)@J0Kfi&f^BfxhP; zJhB8h%Vn5ox4!2;%bggcLapC17GuB)Rk3cH;C=h96PH4iu&{0(d)>B8aa561&HUk$ zktGZ&e$)(~m`Ar{Y6Q6`J?emvOL`|hx8 z_)h4MI@fG~s%DJbViu!+S)=5%^kkRgIT7IiziZ{*o99H9)ura1ar%F1-R1S1XTuw$ z(_jzFG7wXM0wMnPUofXZtkcThET&$w35-LM7_=n4s>GR|M>44KB1aZ{4X=rjXkM zXJLXgNnLx172j4YhBXXjXo&(xjH~%lNtZFIb59kY!}y`b_>uh8C(P|6viKRY{+w95 zobWl0Xn;>S5@Pof#2e^rX8g#bYW-EJRRMvF1LZ<5W05AV5+P* zpFv&4OU;YBWf^c}CY@!kwVD^lxR2x9tE`F}HX@3yUS2Y8il|V#E)5>eQvDDeNb>Bt ziw!(WL8P_or^eEjiCs?SA1s4?(cZkXBI9PfjhRnsKG%JV&mZdfnf%r0%BCG3KO)K0 z6rfJMU`mA(O=69Ym{|k9+Y}xPdt&kw-KAU>swP*z+K+F_LiY7-Ef>2ea$^SaC4X-IPQuK zDd6gJg{iJSJwZY+lol8$y3#lrnKfNU=r-o#D?0l+e}>J#2A3zfU9GShzhm`ehHs(V zG_hkw!eeq}dUKcqV$H8nlhfIw=t&x{8mwiri-s1_?ta@@4No8V$?O`dhhSNG!)NJ5 zOz5kx*_Ss|DNx3bcTU(}POpaqpAHdh2=Xf?kl>`IygwY6)aM%1hkri_;#h(7NnusU+JEY;pir>8McgZp6thg=&8prwzw>n0+Tq+OWZv>?BPwm~fn(F_YHC^p zToN6Hx7Rly4?2ktp9v~IOK3@y6gUnwEw7y>$i?1ZQy1ZVC2BSSof>~qZT|-IzzX^r zN?mhIr=hX&eXgBw=IdtB3-lkHmW8fyNqR}NW02h^-DKOV4@etNZ9y9Jv|MuV8H3yH z_HDWL1`&r*8)eJMZDj+B$+6jD&VGPj*Aw(i!&!YL1n;%r-EcOU!E1&uX-i~br zs*@wvCacIMo0D=Vo;ansb37K`aVR-QbKJBrD=RoDMZ0hx^jbLA8Tr8l9_d$GjFc4( zinUhku}$K(0UOEr-&$d_I1X9%D5~33Fcg0_%Y+Jru$B_)sVjejne_yzt>cE+&Tdi<3{#VsHGqS3rkj75 zQXtcQPAXk>itk7ptrB!^u z;Wz2;^d)i;ye+RGg3k6(w-Y;C*bG}!iCqm#DCk>+!$-5nxrlh9W3~{K^Z2se3{v)i zH}y{&H9mBS)MmY$F$%7-2|x)+ZPFQxyIbW&O{vLF=u*?1^6j|{PM-WNDy+*EB{)zc zQ021pPQBH;rHRHE&Q;jzuU9;crp%_tAkjx2{veJakBMobG$kFNAb1vv^2(+AYQGui z8G4kXC)LO6Z8r&;<{TYdUq8ey!KCRXr~YqP36AeLFT!siO%A3fNL$KOf{{de1CW zEkT-zk2O`&_3DDR?XkK)$FsdSBMe>I)g_TNbSw)dy1nsNpE9*J0$NFgaPMBXsj3bz z?*Xy_7IK+tkTO_9*s}Y*Pv7NGaLxFmfUt+aF8;%MoLDE>h=PB-*>EnHsrBWF_s!H>e*cb!n63L!eYM5&9 z=^fDl3`e4zN{O{tN+YWFrptLGXz0sWfGBHxHA1+!doLum>NXMQijPiV}ooCM?$rc#?^1pg!QU z8Dgi#k;9R^5JMzMDH%qV%PPOjcH3v(40e`@y$Ny zS*?Smm~{$D5VYqi#Q-xpG?d9n4WgjVOA`Y^0u*HPg}8V2zMStSi9#h}ef54tQCp}) z4w6)l-bH#qDU%4VsoWo`;5O=rY>1c*viEs@sqU(+n|Ew2HWoV+js#StZH(jKpu-(u za|H&jLKkwM>e5U0v%`_Nh}{`G-Yg%)9MQV3Q!ZdY;E5!eoNyfKy_0*mv|uKEwrBUc zm_TCdkcU+!8#igI6ZbSzx1!qqc&itY658Rh+f5)g(dMef1QU`C>F;XMA}yx&kPFN# z|4HGyZ>VY-9^CE&#Pi>i%XxP7w??ue`p2r}nsq)4UMUt(?Rt1Qk##T_{F-n?Jm58y zFkn)YRgc%ZazJ6&g+jbjbqOm6gY ze;VJMxLVp{P#s3dr-d4%#_!f|y)e=Rz-|VOrEy{_vB)pv$IAKG59$+b?|tG*_t83y z8pGm3xvaz?{s^)IY*GXy_AnXbuGpkn(N{a5HPTW;!{?O=-wV(;HghV0-bZY%Q1N4! z#DE<=11l|rs9|Ipt0!^0IDKOi>DOG8tF7t5;hnW4 zrX_|RE|6%Jx<Yom|@d;c$tN z&^a}!I-HJJl|d+vEI!?VXMO2{yPlQPPAXLlK_cIGg}p~AHm9?a7Qk9;c}T#dR6 zpXu2SGvEBHp%{BczqYGPs3sv~@fl{YjUQ8v55^H#Q@pa#ten0=dJxrdH65`U$kpXr zFjlsa$?Z^8`?#L!(sl8yp&9}9}%S`pP+utmj{$W05vMO)?U(qabVEaIB7-||;~rTD zGXdx*i@UNs?fq$@gE^CtMYL*iaE{t1T>YMToA|jIZvC+48eAlAzfp-n>-kj*aD)!*{MeLqb7Y2Of}M5fWO=x}7wZ%Fnm&g~n1aAAkRdUvPi=4c z6r;kif6-zhF#?8{q{}}AoD?Oo>-fL>FZeBS=3a(~$BT zy~N%8sStenisPEbO=-?*BC{ZY`t4%6S)c0Ok_kyj=5Zi@#^}t6#Sk;IVZS_ALj^q0 zvv;=-&U(b_J{Dyi(0Ix1 z_RMA$)U%eUrnxGx0%x^j=ta0wcV7nHu05Q@|ByejuI2JrHnRNd58tXp3+PjE^;oh5 zA0#0JEy0?ej9Jf?x1^z00!MJU<$NZWpVw1!eoX`vcM_2f`JqwvEP(h{0_gtL%`8LZ zreZuWEt-%IvadO`H^Y+9*DPX@ z57>%X9`tRXJT%QG53bbVe-AWOHO6^&iy~ju687aTnM$f> z9ZPaLAJFWiyvISnm&5>r5r``K=P&2ZAeqGHc(2EE zzc5~WMi}JMDb6eN`argW1iX-btq3X;W5vLmp^zx6KRj%)8RU}*Dl?Ftn;I;}XxDAF zTb3E96H!j;7*T&3iZu!b+#IH*{r1LG0~uyZ78Kl#frGNb<)QB!*qnE6!fdpWIjom{;zpO>6Eov71s25G4^2(5P$Y;#1BpR(0Pjyh9!uboGy z$@AC0cL3V1B%y0wtxa!Lyq0NxoAj@{q-QlfBlg6TOyMk>lMXVTp8;?XhG&R{n(Z}b zR0xarMaOoo-Z)UFT?WR*)iHZ*XL$oxhVo24&YvL*r%(lDo-g@LQw28E9nnp2;%3F7NE0RBNrT}voxof+lHT-MSW}$Pt>-p^=nvvIg8o z_oWf;eR$WBg8zk|~CCHyh49?D=WR!ZjVzr$^M-`_pb>k0OLY27L~$4zOdy78L*0p){5K2}sUwmTi&v0bMh%;o6NMT2qK zg7*{Pa)iiU#D40%TWF0#ox;!tRg$ou04*&-L(ky59X#Op4ioN=I^bVTlh6$q5#saT z7NTjwtyhL#d*2)6ZF)XxB|ldUr^L4uqNH^AeXc~pj$nM6oz(y=J6>f$-rcX62j?lF zDuG#*7pEK@iXLA0NNH%Yj_PA4v%jQVmy<4>KP1KKGkF0Pg0lXRl7w#ST)M-&Fa-~< zcogQ2%QQqc{fB>}5>oRCS9kvwZ))SC0C*yk#e3VAAk1u?2wosc!BDsvJN?=CZ=R+x zOY;lZCj`z)IC|KC`m_1zFGs|rW(B$ct@(dAg8rl=2=p`j6Qw}>^3lSeJ~hJd5fSF) z>u;^|k7`TUx8ZpO(o6|#8NBge^)s}PRCV?-wW;5+1j=-x>`A~?99wi;#1mY01VUr&&Y1rf_z!46MV=o>*JPBBI(_qe%;C>F&?`+SGPso4sVy=#w zj))b@KLCyLO`CN>)`I%qXm}0H_65YW}-@BAYvxYuB8Z|CSEY zeWU+LnLvvr2Wl_o+o|nih??1WRg`&Mi!p}ENSWzguAuJ}~YH`Ldv zd}>U-O1}|Z~usK9dMLsdX}tLM{JzKC46$ z-}}eY?F+#A%|m~H%t8~hf_Xmzt0qpHFYl&ED@ztr81rV*&eYn)FW>louQ3($WQdAt z=&AmKVBCWswx!IHu6gdG#1q``GFdvW#wA`5Y@s;%9>d8NO z9II(#u8oW_XSNDlun-Xqc1X0)Wde(6l{o+&pzwve%x3h=3m-Mu*#jq{nc3`v1FPvQ z%G@jxcu(BP|9qJa!5nG2{N&eSR`pP`Cp{mKO>uUF5{1ji}uN1?!N!q zSoZom7DXETkKyQVPvA@W-@m=z?5S4((C>Tx{Lh#5%56)5b-eBJ{|%chs<6NTzVtyj z_9x{j>Em}oEQR&tX_8)862*chEfm#O^nnt3bZZ5ci)&GOAbPkvIIO!4y@>s0yL7E-Ie+bI zG)mWBwps|`cC(6EQnV@r4gotMwjm|ZCq%oRmcN<0LAx!lyKflPp}Qn&bNKX2?~Oja z|1jjsZFpa8n-E-)eB*Y(hu6S1K~nUP*uPz-0KPA1EVgzj8QMQDIRR{c;HFt?nT@i+ z^YfAmrE5E)b139$y!(4sH-i^d-%P}&i6I{`WFx1yCMs@CCuwe1bHg@i4eU}pBy8J# z^L2g~;QRt_w3?K?bR*GebU^uS^EbHQH%gUhKmdr>+ExkG2w5_c1uxUHAE)w3U@A8k zuqXi@sIr&8lSfae^whkBHK1fMb(X-&mhFcdY;Vk-x8BD1+9UDF|BtChj4W zBi)qk#4~#jfPwu~q?x*V3CE2`CoZn`5wr?#)Iyg^4wN8QHw?pNAjID=;?~*+W~|al*fxG1BqQaRY5X)b&_^>HTTPU3FR98+mZOP*sZF&_f4W_z|4mv z3a^t9Q74TS%mH7DNAuz1P_ia;S_2rLr^ky(RzL!YQsTeO5A10*kHBgrOG>bzTSZcG z#2?olqBQ*PDJ_>bW(~!e^}}8HpH)sk?muzE|CtB$2%y$6R2<7JF%FvfuK{vmp#NvY z_+88Eq5Vl3`Y+}*(0+0LYLb-}vtZwj>8Y^oDdNn--sS1-Gz#^1GbnvpKwZ zX69y4I+Uz?FtDX`UpeQ4guBE!=yH`U;D>i|ct8nC$2u*4|0>6=W+}Pw%fj?0=rW>8 zTHyqnHh4ep=c!Qaw6|PdVl7BOw}N+Kb#sYGk4l~zLDmy}R&hpFKB2hRAQnIKuyEr} zlSu2U8;ouA_V|EHN8uE-4ThIsKQCWWaiL7XBxFc93NW13xlYPE7EY)vNheEIp|2j= zF)9y6JJWT%VeHmh&@S^M$fZt;K$tmB^k9i01@aXrARSFBbRZt|a>`@^x8X0Eq6z$z zV4T{7PYmXw8TYLZX<+vJxZ9TSs?P%{afG|GF;^qeFz&WAX58okt*HFN$ZriO=R0pDz&p%OuTqrBJP^*au?8S-U3sGq z-`dkw?X2DP?9l-1Hsyvs>>FilbE&T z5203dTEVOyFrLgZP62a))I!+BDL5;2)4YPWUMG|ZqV*!Ln2TXQ^9t=@)!9Eg&~{Nl7wW~%=n@d z8Iw8ddqpaUHyOUAx!@&DVFyHcIBV69~SgKX+IfpSp_F} zlas4mu$e<7uZ=0jLcINVzni>m4rURa>Wa?^-aT`)V8RN|u(J8Nag5G*p`Zu+4ij(A z4iM{$Sx(HoDd!1OBGRjPJBoL<6m>Ru(-}Q=lGY5=qq95QK94j&K^YATnV`U$ZM46FcG z={a#`#s$igP3q%#8HpQ-=bLatvoFcQm;y$eyVszyDZ>1IW42k#V$g`2ojo&-}_A; zwP$%&q7YZyt5aocY6H7$Y+Bjbf6w{OSjJEdiPr3+QF*64xX}8pu;40b0Y`w$DtE*AS*ua z{|C>}g02!=#8uV0T%wZ|lUwrAfD!5jC;HRqtCa1$ler!@IBc9IxBb z9&_IXoCqwkv`h~Xw@&@UaQ3i-9O^HpN#42winkw*nD^_w42dU(5m+(|5O~*3OF9N}#V0B*<)m>4`_7V9;$!cs&C# z30gdc&2kCJ+`2cvAl%D=E)&;N^5o?6OmYHKF}QGoDU>uuPhjvpK`N?26L1E_j>C(w zMGN~h_knG__EfSj&mP~q;&iTypa(vuKCcAMmE}wK%;@HH+Eb&i*AXCV(P%YPOQ#u5 z)(_AJ)LD(h^BWfmv}Sg$Elx}dXG2Pwike?f`Vy#7a^jIq6i2t7nj3cjZF_$4Hk2`C z7Sys!m9H)zt4-!=n{dN3x_5I;04IdasRe(c?L9o?70@s%5RYZ9@O~ikM-#{Tfph0c zy`E6SPi2d<*9ubV^~Nb$_O;eypPEKwwChaDNtxuc`xaId1e->VRLT=?o;L+#0}q~k zVRqlWP0NfpR1f^IRV@VrCfL0_$%dFXwvo)TUMDHWK)tG4C98omyLaYs<_*%9ndIfb zc7&S{zL$MI3pFD^aB6sNPSX@#G%aY^Qpn=VTZ9xfL_>tKN-sJse!J-EW1(wBO?Oqj zmkL?Hy_q0Qm%UqW&z!tO2ZF%dmVn;{Aoafe@U>~q`IcL9GLH&IRCTMmQHyPH@cqeM z9ThLy5w0UCc7|T4;+4^Xz~PRW-(4-l96@r9@DR+ZUdm{2pMHbSkPwhAXzF9SyjV{8LE(#=wZ%;%o~x1Y3@kkXk|K)^G^Ac*~Y%^B<*O_jLseqg zU-teqn8sFQo6hujzYuOJAS(ByT;6XsY`o{mV2%Q|TldJaUe@YU=B=3CHy>NA2B@ZX zF$Ku|NnT)ZmbB8d2gBecx#MLPT;N5IR|p9T{b$xuZ;7Jq_K&T#59++;@?^*Su}68C zBiu>`&NVb$i}|988^BKGafki)P6Zeq?+69SXw1%^mE@pPhFyDF>3P2l?4FOd4eSGc zg>Ub@Q8`pKsOh*e?N&_eV2pksUlXYRFAmy9Uj*)t4!4=lx1lvhdCy8wrdX=Uy?qYL zojj_k47)a-c9k4$H&Y2)n;dDn+G~KNZfhbJf2nvL+r1Z;Goo;qe|SgQ;o-p78fEq2 zc*OjAK-N~38wCGLjnuV%4y#C#;$v^s;)034=y4kAuE3TJSWXSlc)4lr zYh7%+xf%?BHrGeOSuUe(POnap1fRbyw$zbX&D~}me!5H16W!XA88Q4u0Y4SS)AEDX zG6#ZO2-@i)j7%>#C+*6v1BFXR@AByGj!c*nMO4HEPU=%E0xZq6f zGt3`(z~YyUi%0tc2VeNbD>?Uhs&}{T5(Gj+F!Ut6WI{J-;h1n>7#ZDip-)GdYJ(F~j$`*6 zHAO6}lfQ4ULa2`STl6qBLQuE=0X>gRzL?@Z%ia;gl~SMktJgp9-?*Cl$(4}caE_*U zi55T<;^+ylV{K3)gTq(P4^h}rm$ zyJE`ZF@Mk&`+_9Rz~uXIGg%hImH%fdqQN}WV*v1NUfW;m)Jx9rk$UAi zvd}%QVm0moKv}$j5{b%9D{Bf#fO`ri2el2DjQ@+0V!;K zLjJ^DT^8N$T_`Nso+?t*hX8qj&+v0@qt_ft<>KO+k#u zK6&8O=*5ZJbo>~U;9%j+0mN0(4!*ut0cxm-K(#})<6nFdG5MeJi{^xWnQr4_VN~$< z$pbiU{6D(pAJ5dO`Fs3-0&jryn}2Fy0^^UTNDYD8cCSm{quIw(q+eZip65{-BT>R0 zx`5v&QDA0e`#ZBH;n)oE<@9pK46*Ifl~KuY1VBQPJ+?R`by6LLIcw`L^H#00AJ3{ew81Z**_!z^M*z+9`1QpYZVh+l zI|PXiOWXA!plO845jWw_z}_a(CI2UFjZfRCeX$?!3H}QH^tovZe*Ke>d7kGmGiymZpT9XOT}t zLn9Vjyrml1+aO0+~NBPd?Je&QUn}-29W2(Xhq&xzUuh8a^ac?!h(NyQvthRNl<1Xlj@Eq{ZzLhRpkROQv7u z0n|v5*su$z#Magp`2-DHB$aC1K+E-1^3I|-)fB?*o-pl`ztCNw%KChS2LK$VV*)>O zq1lQ$LBJv5yIf-XMY3;^#=IygcQOztRo+D#YtmLLaicQ(3LsOMQwkH(zU^Bq%@+PE zMu(1nCHnlb3vU|d=h(6rA8_w0P!q*91l;DlvwAe=fDmBh?mUBKCY)q(mqd`X!|MVs z31Z!DYotLnpR+2~xI~OgpPH-ESRuq9@W$0k1HP>{SBVHyI$uNPmu=(BuM=XTReeEL;&PVjDu_o+f% zfL?5}?Xrx^-ZV~}l!Bt{gJQj#?P>GG=XCP#e{yE#e<5s}a48ek#(SO-9#9|TTFtGX2!S7uz{ zi;i?WmUiFRZPDFD@1``zwXet>nJ;OY@?c+Gc791Z(CPNUj_f9l5w@l=8IX3p3Aub} zYPWKw!d_g^{;4n8^W%l0li8+#iv)Zl&(dD`=V;~O-J4Vy!2EbfI=t5w&bDi;nQ-00 zF7oRlF^(Bpz`Hj+=|K}i@zBEvn8jx=8qcNJWU^&A>~Ba~XO>AzExtP(k~6sHr`e&q za(hYAsW+Te0p$;H+hZAJ>Q-oySe~DEhsAt;D0duCM=bw;C+uK>1c}F;r|^J0VY2H7 z@e7U<_cby=r|bEAfq#p~wUZRK+3YJEBE0__BxaJseewZh!yGkrE(fZgxN&4#VmD0A zP~WVsTDc1=CNQc2_l`ch^gSiorE&I+M~;{m+`T;gb_F&&NWD4KLq`xuwAT z{R@3go@vGA1s(|(s>I9qIRJLG0puNr5;8a&Oz~xXfLGJQWI^X-9*$a!^Hv&`btMO^*QL|Oo03l zq3=9&LucyT*_6TYqWkn_BJEBp>%?(Zs!r{)g@yI=l=%(>s^8>zDfF_@POm-%k_0%D2Xp~^^o}ld~7TNzk@uQCDjLK(M6Z`+S?mQKM2^mtL%Y zNlrtl04<)ktoTTQ8Xz{>-e_SbWlk6EVGb^FnlOjgi zG1jFWf_?gPkj{4DeeG@8vspYrIi*#QQ#X;9(H4GGAh(5#*fsvP#|Bl@OJuzHwhINSiJoUun)=zU{ zc<>_iiP?)jcfy&Va|T`%ZJUR$Y|SZXN(Vi-ip2)7lp4R#QJ{0lm zyA*CXut4QLGDeGOXdKbLr+K}bk&8k@o;}xV_TD2LZ!`YI8-E;1pk+J(aW7E&{v*F? zAOS)c$SswF?E^P9MKN4))Rif3d5Rj)baE~($ysWw4mO;JuGB%q!Q08!ru17N|T z2mKAenWnSta_!^yLsTE4X*|Pw&!3wt))l7(cWf`|m{^~6(X79-Htr;srJ|6(0|R^3 z?_3P(la1VC_jJscE$+RFD9{>|J3Sg+`ATw63*6v}_2m_R!M$;q0VGSZPhKD&Hw0G4 zt8f4V^gi2aLnWSu9ZAQP8k6VebTG|^peKy239|Xz5q-(d6J|9Y^@ON7ux#z~Z|mj5 z*|Z4Dlfhq<^F!G(ZYtHnTj`!m&j)nF&ZJS@zi|0D)B(g?vm8lqy9aEDkTK-2RuSS&sP>N4MFzclr^XYjYaDDC)S4Y}|d&2a*r6YX=Mkzy<#b{wxG$Ha|x zuvNx5?3zwyk>t*)BJRKwdhe*8TSc^wC$|q3Z6{f%gfFN;KOeI*xY1@1mwIS#tr5cX z`gu}-Jw=xkV^JY2=drg()PJS2h&nnVg$*-v;5%Q%6K*Neum%Gt0^8}8%tRW;`^0dB zH5}GA1Urw&({PsyFBX=KQ#V*nRl9gSe`Vn$UGu#9Ny)hb#IkV4)@Mo6tuK?jnZIU) zvc33z@s5}M{-di|V#{<7XjUQWV#T%1Xr5iM2#={1ix;_8D!Iz^8SdyWTC1Y@(ozP2X=akzpAP!4OVe+V2$hA99VsI` zyBDsiiO9b(v&RMzb6+IW^jtg-K)$gzAt_3qSJPVqs| zp#W_OnN&ovFeyf5j{#Gpez0SU?+m*YGSBf4yS_U7DX2N&8yt?SXmzV+4n2^3hGc;b zJX448B+aeQ?+Z2n-C6Qt?A}8ry14!> zCDq#Hdq~DTiu1SytB~6fb}JUAPiR$zl?%cMejh&a&OMX4w8Qh%(V7{9AgSq=g?DFaYg8@LFb5E%C~#FMtzh_ds@p+z$MZkELM91Hp?!-Od6&qxRIkL9CVK?3QzkO zXJU@x|0Hqgy_38*mC^$pC5*XZ$ix0FP`|UCBSQ)G6m*G8IiSr9!dS$3bCn)&6NIO+ zN5O>33OvtG%n6zhL%MhPi@G_BGnVXRmgKhh?*sZLR#TGWZdpGbYg9t{csM9_gC8rK zo=p95Z~KC+wbK@u9(I+CSvM`qH?u${}3tG!zUO08*Puxqpr*y@fpcR$#9!4_R7|9BYfwxb_{!9h#|t^f_DxOl=D=UV|>{uEqUj?LC=~@TuDmLkkb-n9%AX z5Prq;8|Z$G!hB?emf&xCnSA|-uI0kT{ z_@-za09D98W^VkTzMJD>5epo_SI#-R%%9i>-1T zF$6e63wo7zMKjArm&>)tA=%Nh_%UYh96%}Qt1B*9@bJIxH?8}2g1zO1Vq~H5Z0B$E zT*wOkwR3O@_Wq9=@-A)HdUNznF$_cphSNt9ehXyA<6Jqmu*Vhi-e(si{UtB_io6Tj z(sUbYwljEn2PKRunOD{CXH`T^VD=SdyBIzc@7;Ubr2YbX4=1Mhi!ate(MR5*Y@^SC zzB{6+GQ!uMUJasciXOuYG8OS5bvvv?BTP|TPhE^W<#zV4-zv_L&SX^?q=7P+6Sqq{ z@b!1N&sIyeZf?8FFm5}9#Rz^0JMd z={1a(nC|Cw�X%Kw_zjCMkpp&z~BvOk#Fgnem)WB?}*RNgKT6s`Z>qx}AaEtyMlW z*4{ZK6aIw1Jvd$UQFwCc&;t{Mea?3*>r>)kVV1cXu2)Umc8Q03>78>2W~cw8Eg9S4yR7_x zSfpvJf4?09typg-cR5s`7`QkTv!pM(%@_9x~PL{(a+ajWUXHPP9do*#U0+2?tXJ5&;{I!@&!xX1NLIl_bxu$u#c z5oSMwFdPem?!9vZvwuaau?6eLH>1=Dql0-U1i_t`t*6rY zT1dCgdH+$9L~A*igTWLP!AgmV^Q>lG^Nvsm@V9aJ%Z)egOXRB zWD1GlM!huH7i9}PsD>LlBiVP;?3?Ru?8&4Be=BloAvv}20Q9SO9`kM`IKQ-$Y@1g% zZ>V?AtFwzWAAC@W&7PJUY!z^=YnAV}#O0)zWnCAe+&8ora+x1VW%%Cx z>Z>d?%UF7{H2XQvs&~Y(WMhTG`NJe3Azv|s?uW(s2!xi>H;Ex-_5!Z4bn^3q0H3<# zw6f|Nf6+Wj1$L^oUDzFCXh~U*1W5toePK-NDjJh*?VgCQR_O%*r^?vYsv%Dmt6sCB zYUy(-Ezdt(V9|wLYEL@jG(btPkjO5Bu{b5GjN#fjBp){%qKbEFAJH%u6`V9YP8Y8< z1siJJUhXmR)Gf`4IuC@tbS;e9SEZS|chGZe;JM!YppG%1QG3@=>vG~?52K@HMYuU_ zVgl*K{N!8KVAVd@VAYg(El!O1(%IbkqeUi1h6BBI7xAt3?DohaI6m)kR>$_UwPWKs zY$GyhN`p8Hu|nl55)NTgS(rldTeo3>Zn=sg$uejO<2Jna&Yfx-&vJ!qQ4XFNN5x&1 zf6Z%6F}RY2vu@}N`s+%dtSkC|xO>a6D8KG)7(_v(M5Mb_N=mvzDM6Hup$DWShaN&2 zq@+thK&3-EC5GC4Q)fx>coYK77P*zlaSU z4NE$3;oOurMztuKm`7xi>g-)M{0!4^gbYIssK60O?OhyIb;SEr)F*2Hk?*^KQcEnkrEYn~ zJwhr&Qr#wc>0=WV#{D|POmKvXJ?8p8i)JdlTCnip>nRrGT{|3UVCj_dG`Gb#e?B9e zaY_xjq%UdA3JmJE6*W0`UXvTkEOVuot}oWQ^TeHBku_SpetDV!2?26Vru-T_BC|-V zzc*DcP}x9^0#}|obJ-6r*_@Kil8kPNN9gW;r;Fefx}+Gpb=!T9uZ(hEQ@Dm-zU;>s zWYzT0%>2mMHZq>J92R+AcrGo`R;I0VF3&x)IG^3YVPqUV-XFPfz=2yz-i5w!Q#r89 zJ>Q(KM2N=>IvqL(K0}*Lf2Mj??>HbYv5so)TO)hm*Qm(B*WV>&Ju%+hiI~jWdpiv_ zvH_R6r)6?+DGwQD=DwDwZu!bE0w<-5cxxSl($I4+$`eNoHXJp*NPcKx8}|Y9*7hL| z?hQHLILj`c=pP4%8kA7>MkdmY&_<9_CZ%b6B=MowQV)Ge!Wjr*ot>HC#~Ytw89W_ay5`_}EqtXWq3vp>?M8dxHMw^k)6}IOLVTx{ z&lBlS@Z-A!!Rt@o6S620M80(r|Lk;j_CR>VkRmj-O-HwdEZghz$;HUY-eZTcUgO6& z*_736GPRG4=K|ka_ElkRV_3CVggPSnlUW#iZ864H5 zY`ck>jQS`UU z&@}~lNYvvQTt?2xa4+17L0(6(Aui7=?>iZWXScaPR$o-jE;>Gqo_rWZp%y!3c=iP^ z&h2sOU`Vu$w;SbU@ztH54en)dg+*VnT;Z5cHG~BZ&o5}Ut485yyH_A}W0w!Ru7ncM zzthOldfO+QuO>j8b=t^H5_R-KC}bvv&E~8Ey(nL>$dmdSABIl**ib^uszi~-+`)`m zTiun6WkAZ8HRpOfAV|QW%Zja!3#?aOOz@AUo+d8@ zr&?|cwsVGq**V7s$xyU=Iaf&tU=NM*&s zf+<*kunjXvyc0aRxA&;jlBA!+A^ju!LUaO_+97qv7bj&g)OmeqvqLQVo>wByZA7A} z_u#aX?;{mZc2K)+fpLcFgKXlTmNkX%NrE|m|Gp;)7!V-=+GLV3gWJqwoe6b@su|sU z4d>}34xL2J=lca3`Mi=5@6pH2R`psh9i?l79OsibD!g#Mhi?cN6=w{_P%9@95v9nY zp1u>zVmRI}(@)HP$V<0{;AUhT%Ulr+_u!J3V46oLZm;2o;HKQQvfB$XI zb?Xb`@3wMR0Q-w{wwGD^nZg~B0^lz#XY$-kxjU>D*9z~&3Mb$rG-SEy(3dO&qj2#QrkC8AnZ9>q3ydwOJ|~;@NvN= z%Rk@7*M2-n_|hx%XMTl1Xf#K+w&)X@?~+z`TTby{FD#9v{WfUcV;{%-^HLNs%K2Sf zwi~hY1F4FI3om-_5*m;*qw*@sGd3Ks$TS+<7^8whS{CIpqBdy$^^9f0Ht!EXd7ASA zQDjkK#O)XwSU7Ai-dEP9w8(U&1}5h(GP&Mq@W;6=I@{0pudisAUy&u{FJ6I-%6~z< z(3?cir+tQ0^VO35AoRuI7M4JLB-~oUPhO)JA0<*_sMW=q#Bfnwt;zvI2#Tx%S(oDq&Nqc%ZLEQewIDZ1rIstj@P1WgAZ&YHjHnmKk#eLNoMba-sb zRSWZ9qpGU1f6K~&07Lh(nv%TJecMw~5dObiq;Pt$P;xvFRHgbdpx4hOE-*hM0b}oG z65y<0>B2_3#ArCh(eahzzq?VV5RE6iDRTvw^LM`o_)FrvzGs69>c`-BidchZ9K zhva)3u{#4_UZ!gvzx!u~=zsKxO0wDHV~(c#+%wmx;pHdtXN7iusZf%WGtc|85HGyI zL7{!Rz$zqPm{qbe44uHzjQ4WDJcqf0uV^(?SBkB23A!{mH3OJ*Kr89e;#kp@_RA95l10!+}uBY1uwi0 zSR3ne#2pU8rDm8e1EX_dl?@TuTd0Ay*+2Jna{g!15NRVQ)!%a6GzV!+pu>rZHLA&r%HXDn zql~c$?JxS}?{3vnUh4Hic$IxJ2zQ`vmm-oA=)#jzH$8L~HB+Uq!t640^n$*@hN05O zR#}ura768q8|nPB>aw5Qy+-i0qI?uLDrLAAI9X_>Ii6zArf8=bs17^Ye41j27C1X~ zyDXZ#%ov-ROff=#KTBvX%t4{+d0pwA2v>346|kFpzt@~fvA@$-jHaC2!9!1gFGar= zl^m~D`yva`rVR>Fj`NPNUhqqk%ln+n1A4c1Y_soWsOM+@YP@Y0pmJkfEr!|>rHt_K zo;i-0V$IhE>T`6Hs!3T4e`cGL^1r804Hn{jAc)wfGj|dH7!;3nRUKVq-5Ccg5^wn& z(yXh;-Sy~o>MR2{t?;h&&wG09w|?D9LNV@-QSajX`~dXZst`He`o|8g-2K0uwjcl6 z2ss1kf80b(z~2@2|I-`(&z1VC^HQ*YP8}!zII-HyEwv;edHQP!|L?!}W3xi%4F27* zP1q08aWSXT=^I5uCu_Y`-nWWJ$z3THvfUU^`g^fJhkF#|YF*pFdWG2$hO3!iSe0hS z+?uHFl^qthaN(lC1|>!gUI(KKM@0NpwJ$7!eVo(UPmSBLL-*yU_rVs>Rx&8!!!Xoo zBB3CsRi`DZ=w+VmMiIE1u}gJypDH3ry?QC@VlV#{w9bLU=*MIUv?o_m$ma1l=LNgi zq_3!I{q#2bf@h-lsz}zv(4<~d{LJNDSLMV~dTm*d&BY7LuO-JO*Ml!Wo3P8QgryS>w(gRU5z^e97tzm`-FUUGNZjFC^$KpMOgRcJqI zg~kcZD#l3HHBm9_7n1(@^dx7fK|asF!kT~g+8dR_Q{uTHz(IT^4on_+^XJbUg#mL6 z#6EwGv0##h_n+YC|H#Sxo@3J* zf9___R4pL|hI#8tvSW~HrnkL%#DGYxPoSo zi~bCI<)`Mlx_BSFWpKE-aoa~!zZ0}WIO<^@dcxLm36$xlvCc1)e~G(S!b-u@s}b=m z?qvV2_GeN-yQp|-p5B=gVz0cfg}3~&4vLN9m{={h;iSq=AZOirRz1*9Fta$y*G(~g zN(1F)QU?`Y*^*^50S_F9YneJk$8hZgTIlM4iB%fTsXrq>hgqbYz+HE)4F0@qf~BMs z4KC5~n)4ISWhUNDPNzx#dcebf36j_5H7}%g_vV*8Hk%1A*%5R1+oHsBkWk+64z^v0 zD6^z0Tivtp2jhxk@l=UZw?LlpG*0y_AKtocCoE;FT{gX8^nTx#VJWaMg~80SbyCh< zEq&^;Ot7t4w9}uH;f4g^g)NU}txn9Sq@HVV( zMuO+%N?F+&FaO)&ZF(HgH`oZWt%(@TutT zWL~r!EKxrCGj5^H@9KMF-P)ooRzzq?FpBv$|j6q3?U2g-n@|^1E7fXYj@5 zvZA5J>Y{sGI~C#oYCU@f-(cqUkE>Ax-_CtACH{np5Bpn+>5Y!e-=jYySL_A642Dib z+-ae=E5O5Jk`rwS^!1k7-xWQX!f)ASOc}t!Mtwiap4h;krFZ@o=>C<)xaofGdN=JQ z=53QPtR!s32Hu4(u>%6I(^4bblPX(&@L~y`H(K>Fx_x_YMy9k!wP$YAqeZzC(79>X zTSHmf-VLMfiLtf0Bx&i_LEa%D_1>HI0DD?u>=}H)O^?p4NuTA|J@UOF z&t=ag$L|@oVhOAm!=(-#TKZBksg{GdLCfb&<$+rRhxhFI#$R$}x^Z#nsb9fQdpfeN zrwQoAardBJ`xsq3sVR>&aMj_rDD^@$Bgyudw~B11&v&`bvYy8pF6CIO;tppx-p3-{ zju)5fxiMb6M{|PeVT2?}AGa0R&WO8xmz*|o%U161u`jT=c7ay;Hp}BQ zjP>rmjv)1!$C7Fi8sJ0U@=MB9eWgnJ!Gs@Y?H}|M21}HYECcK7Yzyij$5-wnM@e+r-Ss?VZ7RRjrsR*cCQuzpKh)s_)Noc;9;ECJp zS=f&v_)$Q`6G^HEX;xW8lGX-cw-#lzYeiQR|F)?qy{v23FF5#O3%r@&FJ)(`Xae7` z;)?GSG^mxiG})1G^m5?Qgg<~aEJ!_y%Awgx(b3kr;%3Fwx=5bTpc^ZA;WeX+k~NBAV@% zSVB(IdJWpvrs}?!MSjZ~s8+jgxa?n`%iD}3UcA!6Sd{6)cP%?SbU!S%joFWHtb95H z(Q?)b?#;rQ>lTFfynU<5-iWc|^_({R-T_7}M~X7^#?nBYX!ljVW+iG4RWPjm%zYVT zw7QaDa~4qF)IEYtmw78je7Lg*jKq+fXHz?}nwRI=+_iKkBepuY;h|B2u%5IuBJAKsj>KhCS@@sLzf(INOe)CNfNi!Y! z?t^o9&gn!;=f95Wok=}`yRZ;mSegp)wgG10$XnAZ!4iE4KVzWw>lT=5$`^=CYT7G3Fl6s}rfzkk|Ay@aYM z?))bLjZG`){O58;!K8o1kwKJLWAwv~Jf>bbH#0Q*fU*jaJ8s}9yQ;?K&ixkq7&mw~>XnGXd5wqp4c6IjbUY>6<$ zv9!k5s~3`XcXSj-R^4H`W!VW+vrM1+4+kytLZ0v5HG?oItrQ+tL?^9DY*K}*n1Gyq zICNdMo11@3Iy)V$WKXELldXPTZJkHqA*!$wUVffRqA8J95qY}$RKu=OOCQb1==ADB z&}~HYNx(%nqN!%ZpkYS1u%bUQtK>X}yJxi?_9a@KEq2ONUAe#a&(gl`2ztM>aA zMHgS2d#=t(2k&mudZX^m60mF=8QLCrIeVkub{KuASND!Oz5q=+pE;a2$dht6kpS^Q z%4e_^R_4)O-pm;&m##Y2da^k*bdDdzdFMS{`28c~!CLs{YOeEPqoXIPs_@V@qeKvK z=hAB z`bH|G9KDZq93Z5AGAGaLjCHqVOr6&kw<$`$aW&`WPBbMZ`M9Cukl)7omF|$5<`)H6 z0GAF|$-L6dK!@$i*n*W4CV@-~;7Qn?-^RXv8B_me4-{oPu>$U;WwQ3x{f79z_3|TKle?uLz~8_SF-1I!55(QnOn1;WK`Tuv>=V+4#q{5t)pZR9de%_ zHu2YRU11P+2KbEZQnKvsb^W7KLEX{;e3D>RYPbVTW3K^b*x?FM7exc)5&fychqKiU z_aGoeCcda$g2O@lCHlC{9HVJHe_VH)7xcZPxU?)`i5GyHb`HtEmOnm@BA7WlP2wb~ zO;K6Nmte#}?EAFndIV~;)4~~3ZY2s3J0fXn$9aM|aBV&(h2}hVx4PY1PA+WACOo~Z zq3D?z8mkhWua6HziXDeDc}0ie%Qzd)0bi3!T{LI9a#@XhlXS9ERJps&W~d~q%odiA zn|0Qn<-02Co6KO zz9M8z7`K;~;!;tU`|4K&%8R;&##wGXXIl;fm4)`*s;B$p!4UJ}o=;QsVTDe7(`B7m3E z5BuO)(cyM6wxe5l__JSgJqVG;K!}Dl>Zp9`jV6rfdawwo8QTx$SRdZh`!YDaBM_aG zr#esl5WQD8$R~0JDSw+3od>HE_GJEf+nnKm8(%^sg|WKj*;b0B0*=O+!yC;nhTO5qrc7YI1Yjb>5FKQkLJBFP&h|WgnkP^F`z{$XTV}>=-0`yug0% zItt&pa-kKIa*6VP zIQsZH>iSWBY$B-P)jPclJGqiWw=v!HyMjynm@n8XI2)(YqawvR$Y0-rlgxk1Zx7$r&r{ zNumbNOfVP5qet%eWo5F-xOel^K*ZT6gKrlZc3%dzt?}Hx9*P=ORxv=A^)txqEUYpv z={ibY2>x&bX`wzW7x+y+V;Y*C2_P(e|IFHegt78l>{+~S%SRTyONl^RC~s(zSc+ga zf)0;JkIZ!4=jHuorC4`s=wUK(uAMB#;3-}hm`eBifN%4bBQELF(ehfC@R7)7%r5+q zb&z#6dqR}m38{WrRctP8i%MygS`PgZ zHqp=zsDW<@2)cdKgS+m)4yx{|$Zrn)eCX0PyjN=9HlV8lhWH}1LFLzPCssFyWf@?# z`dDU1rtnSV2-qKNfh(!??lJG?&Hck0!V26%Hg6nv1}2dbh-3pf!hKFi5t-i3=JRzCkI=heHalu6I%Ra zv9(qJTvYBnwKlw8R(Rj?*!%k6NlHP>c!W9YO&8Fzjy~zV%_7B>P@8jW(PWu`UrrUC?tJdU;M&7ti=90TPt06=ypvQ|= z*>C3sGa?K)6igrD;-*^p_DICZqgnkzsOhegYASfVU!n-(BcPSC4+b-)C}@p7ehSjL zNPboj|FQbYwPGiD0!lN^x;WUe0G!x?nTcK^v^Ih6lFThnb#&usF|MN}@6ijCT>Ch0^B|_t&mXeV;In^uUK#X zh@%oK9%SOeRgdp@CVkUt!K@?{5g$b!>|X{h{YkPghct9+Gy%%mfY)cV>U!9yZ?v>! zL{LYBA5Eq!+b2-L3)t~f%rPBaS^c25k4@t|R&bmZ#KVI!u{SR&z#~<;CVt39Nzg~= zikCZPv=8pq`>NVh6EpW=VR9SvKk=TLF1Ky493H9P%PDJA>wA)lbwt`-eYxeF7!R^v zTUDw5beh$ykbwLq)nwZMo?aP;II&4dXnRMhm1)ZQJ?l+XePZlLL7$e2XlxvZm5pwf z`13)(JJmKG%zQ9=>j1d&=o$_wfs1Xa zaIv8ixpdmi$PiFKBc_Lyp5arXz~&4othd*U8-wh#iz0lBle&&#-6_g`+;W?a{ap`_ zgR;6lz0kv38cC_;8(4J-SjO#02@#wg^zO}~D|8y4>c!%;K(?QX^K-K=vUMtgFvL!8 zlHxfX)IOc*>213BpfeV6A?kso8A=T+LB=V!v|7(A1%+jYESQWu>laV&eLsM#iKwo0@@!uKJpFJ|wsY;kH- z)u^TST1}IVF7PiR^(K_a*)KB+JJUN6)ix~5KVTLUi1nz2zLNbNrg?AyO{ddZQo>o7 z>}TlxGFT-ZO%DIw3vCW*R(R{wR2v{2^4=Zly4NV_(vZRaov5U$*|8Y%v7kDOm0_^6T}l%uL&ujK1{HSeNaehz%o!fv6|1FZJ_-$$28S!AX6J`*)%8o8{iv z?&xvSOHdRMQZc~1E(BQu-m}{}ym^=*7{ll6Em_i#6Q<4$XC}0yPk~CI*G5ci+58^#f+v(X zYOmZRuK3TrdI!Z>MYBpyYb2U$xTF9lG#Rw*&MyBn38qhJX71h%=9H*2X44<<+s)p) zFbd{g@9bp1AYCh6fTqwy!UCVqE%6qiX64E!rn{aJS-_Gv`TE)U$)dR~Lzld@HnQL~_y!=4icb+#wHc19>mj z%|t3hSL}6Z*7cwRqX=xos&8HqP_0%E;7-26i@Nn9LBca9DH66{k|Z__IxXLU_jXpM z+p<-jTEtly8}_TcS&sEnK5w4?%&IJO(?L{IlsV(LBw*27?i7h(}#pxIMfUf2<&ANIJDa1Q1* zDU-&Zx_!dPp)G-FU)d$1B>*)HWH^lvxxW3hjsV}(@O=2>EpU7=4k9nX7+pXYkJZo{ zSXtN{%T}8DZ=_`K;aLZ~%^A`d91`01@>*2(9ppT=?gP8%i>YXxvVoM2Wg>ypqT^Ev z@re`{yG7|Mp9q6P^5tV8q`C;*(*v%DlIO`+-c1v$m)@_Eo8)lADzsWf(jLmO0UJ@E z)qJxJCsP1qDyVWh8N)$8&C)^_hO*k8o!5WboOYg2@uIsU)L`>B{BmUR3t6ZxxJ5^L zE$DUW3jlhxU+rP5o>7=TA8yp*rBF2*0-_Yb#ZLSCvP{XFD%b?_*2TT?z`;UQX%arb zem%6WCe70T0yQ}QzET9}JW>j-#F=pzIT}-$Aj?b(qBE+9YL6ah&~-t91arqaZ`| zxc|I4}=_iUSal4;+{o)rS^t?ZO(-q4MTu?PqTjX5Zg|80WOF>maB#XgWLUu!g z*6%8JrFk&Tz=fZ#p#DSKD(=0R`>Tl9Ee@Qo9pFd*!bJ41AlS#}?>BNq_>}hv==?)+ z+831WWz7G6NN6bTL#Yf^m}qDrE)c&n@*9Wbkl}=*jmTIwtR%F~jhSq!S$`SEjavk^ z5dMbT%PseYla$4Zz@N%+5;P5?=g@G&>uNWaD8lO4C&%h!YeCm`?}LB$l2iRlKapoC zu-UMt*c)Q}K{FZS&Vc|luyOP2&eFwBNODf&Jz$Vyy9=#Y&dzRIkMfM|*aleXR%H4g znHXCuEW97C%Pz#k@crS1Bp24UJy%Rci4&R15-*lm+oZ3B3ZG^E+3WY{h3Y?&%4qAG z2e5|0a#p>J1m~dX)&mE>Ak0+Xt-=Sto>I#$b46Eor{rLnB|?*O#trB~8AmH}49>M@ z%C4lk4G67w+o)PI@~I8`Gl}s?2ZpsmfJyPVA-T(3`K1uQhE=;3wLxNrhBflJ`pg8Ih&=*=co_Xj z+Yz#GMd2^CxW?KQJBYPFKO1tnH9z}er!78r`YDlTUdDbBpkACAxUTJ8^XuqU!nF9% z>40fY?%4$0)!gux*3gEb?!0^Q(_cA|*$80L8rxE+-QjmW2$GTk6E>gI@?{4}PObH! z;HEU&FA|{rQ$*E#BOa)0Ss?X}`1h!NZDiR1e(cmV?}DulVYl=3$^Nv$j+x1i=%70# zgzBewhdp0b70I%IJOXX<+kg2Av6a$z<8i?U)xfL>xq$E!>tI}Qsm|#NYa?c@QBQ%G zX1nbViR{QZm~k_s7o&+hn!I07aQW z_jHv(?Pv4#4_(n`l|!hNQGV`sS0y3I1-~a`ZKL_FhkB$XqeVQycW=leBYI?a%2B8M zqd3W(PkY_@?y#FW@8A-C3f@LbJF18YBdV^o{u*(JsP1{oc zK`JohDi@E;8G{w3G$=El)b7dn#Awq{$x7$jodoXrtTFBi3VIhgK$J5Y26Fa1-)P6Y4FI&{axm% zL`~*zxy$_bWWVL09g6#d!B&!Y5cS)VhYusT4L8(4Qzb}C*%pH5$B$M}q;NXOw228NET(EZju<*_5+akD9zaPYxpK=9rFg@dMS73ZmP*1 zF7{r(KM5+Y!{)xQ4Q>?_arf2=eC6kPb)FzmF_@|9{;323$3S^e_ba>l(k*b8n=hyh z9b(e%GYV8=fAhZayE!w9V0 z^GRvNUD79FywbrXY*&6i)2(8DT_%g)jH-6w;mH6$Kd?R^<{kO5@ zjR4#XoZx-`ep+*?#mQvg)yZP@F!EDQQJ?OA2oe7JGQDe)hTj)5t~mXn9;oiv-6meR zmp%MHoZKJ5BvEXd^8Z;ZBOqbPsR6O@VvxUd9{i9mMTRu^y{G_gl)9{_878z!A*DQ}I+G=WUQo3j5P8VV z2S7Kfvy2?IG!dOEEZ*0b`39GP6?q!-am_!R_o_!L(=wV?0EviT@v$|&Ptp1>=@MVV zFXhJnYpDlt!Q_l~!|B*HU5)|8PFn^Cs>)wb5Om}9-0YvvB1#Ddy#F1D*I%;6vMp*} zdYt^PuvA#qOugVsQAYi7^`>m`rOGCJa$=Ig2wgEYc~Oc2=a(FtT@NaOHajk}GJcCs z{!{PeYE+1x{v$s5eSys$J7wS>*NC0|Z#s~F>zID=PcQqNDR?(k z6#kQcawPpLp8EetUG-&?&KRJu3gmky0FcL(*l@29h}kevw=)35XD?F7#-OzyDS{@> zXg3NUZtB)$Vq`py_#HUCc4R{D+u1@NtCTXpK2|stA3)0YYPEjY;5h+`OGd`;5xd_0 zTV^I#xPw9kmWwL8I}?ymNmJeB=SbWLxNE zbH&k_tJlphsN3Rf&z?w@m2%+j%-#0!YU{g%8-qqrViUI;EwqdN4}#xtADXS&?(1C! z-79^jlEnG%fY`}S_j9ydFO}LoKnDRneOG$vAEHfpcA1Tp z39GTP1oG7O({-1+Zasf@R^{;P;Z8Mk01OP(oeK_!L&`&*cMvY9xobOE)SukG=2m!I zjSGFZ&!ni(XG+!kgSVivejd^0(xkWUts>-48!`R*h@^Au18jTQc! zxO$1G-8Q$_Qa`|+nbvb-C0B}`%*9!$EFv%9%50u>y{?bIg`7uoD(_P{h+5i(F7a|N zSV+s*(72j8%INZp;?Zm~iw#0vR7?BK04$z8W0ZtgmcIFb{+gIXYL;sk>yytAfTQ!vM#i3B?%sCQcU1G^Eg-zDp1&r}o?ypH0>N@>)fSf?;b3Z|x zI~?!k5~#;~LU5jHY4uW9)TOO)*Z+#w`oZ1lgU0J(h%1Jwy)Zss>o&EMF3hSTp3WJI zW1|~XMoU{et4WU-Ll!@8ex>)XXRxu@-J>LI>byOVBMO_52T8uLRSxwVlpW}a;i*`I z9}QQ2O`F;uSrC`GilWtn*Qrt8I4o?72}7!>Uc?=m?hTnVwB>_(CBF zE6>9C%=!KS?Y_t8%x6|cx2Hedc2rkR2uOaw5+tH*5zX^y@>3y~vNF4g?2486uH z8@|rU*MLL2=5W>Odu)dWoRS^vG`rx;qusr%%MhoOTtcskyZBQER5<`kKahN(!AGob z}g5GP$Tf#CdTrv8xzyh#CW0?y@xe%*DRU;a%|v<`UqVA`Q@ zzRW`Va(oF5e&#nil6-P4n5uDzp{BjTIb2S>I=v1A+xGM|$D`Af?ava|3TTA5-O(Q( zVNCK-i>CPB!c2t-QO<=yFfa-L+-gLN#?|^41_6BlNa6coe`P_a z%!3R`2+!X$k!05fG-CSls}V%B%>=Qt(S#_J-N>#|<$K)I=Zas{s-_B#2%I$AeFLwk@jAzy=CMfkAoj zQ-D*%69w@`FgBDn-6_s@-$Xl_f2e7BjjW?GWwk=r-&@`!CO$)gX7b3~KNCG~mYqx+ z=$Z8gmPqaAs#VY7-1FN-xKT+tWG0x#`0)>$Yx(Tz>1VU>R#?=29(W?%Ys#bE-5CpV zcTw~BlFW2!LJDsryqA}dXF&_Fxdo7)`J2x;E3dwU_<}WuJN!q$z{j?6^LdkqExt(y z3hO1MoMXPRamFBw5pINX1dVyKD2&|jJ|}i+&8RgHk#{rUi~cZznOET{%E>PF)+7e; zjyT*cwHe>w`Qc0$KPl}E=&QY%0lfM+#)Q+MD~C?Fk*Zzz1B(E$v8}*4So$=d%En{z zD6Z$-i6tMi-os0nPrO0XVXl&NK5hIAaZZ5Upu={kA&fD*RG4R^hdpD8 zr{UeGctLwXMQ8wI3C$S+m1fIBstoza>HFU)YP?q7)*`Xi)Es>dd)3()91xjnP6B_B zWCYgL(TcN&en4!ShED@Vv)MMw++gc9Qp8XGB%NAKlkeH(wnw(N0~w3sMC^*D-IOd3 z3CD%$^kzB72=T}Q`*(R6L>J#2$fK+R+_DcIB<6|B{ZOT*68t{$ykaluY;n6pNEpki zI>gF*L=CKaH{)9PhFlNQlqDs3s6KSR8un_vZ!|1S786K#WT&fp#liw9B-CHQO9+ni zj>{Z+G27fV_wv}WF?1}u8{OAiV?u!QEPto^S(P4;XSviG{-ESE6MVM$%4*( zCsk`;ysu0s@t0&$g~ajD8xwhtdy6Wzc4)nj9QK(-ou(LEpiu2fQA?wn#%0S_EP_Oy zg;AMIitpigKSCOJ-M<}uCoXJ?7sV{rQL(CJlykQs4D(-gk+}UeYwx3L+=ny|J7*?k zGK^;0%QuiK`>>MC?4OG?t5Q4`99f)d$34dD;gIlWzp#kbx*KoVQ0UU2#Jb_blcVNq z9?9ZFY2NWBf{zkiOuffuUvp_e#IfRzbui0Inhr7TOEKM}gpy3-?Q_t_j--mwRRC1u zkvT^x9ywplp9O{P(uV~ODEZ}zm9zJa6s%gGbf(%-Kg^qj-48VP&-*^h4vyod4MCjV zGc;)D@J*U7!0;$1`RbDAW2#%s)s+LqpSZx&A`t5hu$?7gM^X{3podgzytf8uBvPNA z9$YePS_I^ZT^it?)|x)1H%fi#>ilu0Zsd%cS5Q@`nCx7%!x=~agL?dA?1H{oJsl+n zHo0rM;7P&_vz#NbSrzBQI{)VHM5}Lt$8wL9X5~jyUWnD^+Qc)fu_$IzGG6aLO>KS- z(4Ue%1Hs2#4TS3A5DagHGlO>soHNbvwFv2lC~`4e8t!BgvM@s3p_*_yHL|%2QES81 zK-;*5$He|qU(5a@_r?X6*5>UYI8X;SRp-0M%nOy2>^@@En|>-k$uO8D%k1sGmCrHS z{mM|elppDcm-6!JkMF(Q_MUALjhC1S)}1kef8&N?i7f8tpXZW=7T#I9Fbn6C1$Vu! zOn!*a;ln;{8>c^q~$IJK**sA6|i6EU@@RG-zik zRdb0r_LZLV!o_T$Yz36{CqPm}fxt0aG5xelBxP|nV8W720)bu`!0jh2^= z+?i1lb&XEysfY)~)M2Qy-#AEH{svJnOH)R+Od=j`78b2p-@f-t+d0>wOtACgVQs;H z)^RT|8SEx&+`6ANPNF7F7hN#NiUi$V$RjXXi(W;4Pk%yeZc&s$=_Szi_V8UZSO|-p zUsB=}IS@K}CzxBxIAQfo-ROr$J-L|cq}#-2iINEWa1Pp3t!mM<9%@qID?EVTfLxLr zk_dPdUwiPJ8X^6bu9cC+Zv+e%1)l(d$#a(HhJ?Qm!u~83p7OaxMHn{L5ygVXyvJTy zM4}gdB)qyD9rXYqD+lt7ugS#SeiNVjWg!&cctN}e*Mi%U_w$`?a9gbjh;zWJC#2}5 zr3py$n~6Jla3dK+V6w}O>s4G)Fng2#5^8{zz1c*mphxzP3qND=J&<(_pS zFWUXBci%)>qg!di+ehcxc=O$dhuaHf63G+b*J!26>xe$ZSG1F`@7Fu5VgjGfUyO4> z_xMOBIe%a+6;;E(X<3XURlzd!vY0ax1UZg@q^l{5y;nGj9)WU%H2IL8e4jqMYDK>~ zax=R~nzY&J=8_Cp)=!#(?)vq+?c0IOqsC(7Cb1Rvef)gVi5h0QO1)oAMOsoj)@n7) zgm8p~V#-VM)E2_NzWNY0($I%BNj7|S-XT#buv|-!Y0<9*y@@zDf zG+T6)89R@%IR$+3qrEX{bPQQHt$ADs1ARcZt*y}Q9Sk_W{QKqlN)H2^sS=$d3v^&G0#%E!@eo-z*}0 zf-v|n;e*`vWbB!mR!lhjn2VcKQ1!w)tK^3m4Vx#uA}RN$w?)TIed5OwQmVRV%pAZ8 z3>(`w%Vh6p+^j_#amRFs?09$B@_6h8J|LHtfo>O8sEQJs^9UpcHJLt^0>q_RKDnEu zQOo=wI#$vpycwzCYioLqJZ@&W=BCeX))yuOGD?KYP_;ugH4!>@F8*38g zPrU5UfHIg!((cE5zw4>OLRTADAT2r)FmswJ_!%J^K7KQz=?p-z1qZq;6USHV(pWV> zX@&RNRavSV`^;_HKKue&$W~QBV)I7=PnFRVhp#c(o@{waLUuaelB#KWN2$Kv>i`VR zJ@1-3bTJ|Z(;?p-%%!5!QfB{WZ3rDo&~SI9cxTZC(TwtDChCPu6s@Sm0sb>2FN)KI z_5f4c*<$tN>K){i$LZ=;0A%xdV^g4N+Nwah5Hif(8-p$ez3P&K+QG zsZ10PofFSfd#Q>Ze$NO$d9RjEe32dNk9G@2VY6T zLKb9nC#x`;|%W zrM%*GEK_6sS9TNI~(~*8yPLvY5A1*73^{ zAB?x2zdCCdrZPQquz09?E}pa(R!K>l4Ojm_f6vGDrn9-#)wvLVKrNruVz(F=Uhu<< z7>JEL>bh~=YtdhO7S(fsfwQ2br9|V?0s8i}?uKD`{q2?hGX!E&CASOS@)P@d z4g%C$S$$}X1m77!NX&C^_oBtrb~eadv2j}3CFvMP2x)qKrioPYt%E*D^4N9O&^hnqQ-qVS^yu0Eky*i2}18Ur#22egf}If$z_kF|N*w4@b2 zRQoCT!MNag#pdaQ+7SnfW%>Kuxw~~i5~5dUSx9V7IG&69SlEAy0(mvecBT&HP^h0 z{YqWXbz!IwM%Os3-1&<7jdbR2iRbj``9O}!kIXP%e=@jCSEf(?o4VPh0YGPxP!3fy zo88y=i4X4GW)4Nw6I`-VPtJx(NXn-!GXIJJ7zBVAKqawihhJ6nIN`P^z=N-cBe}Ce z=7OY+7N$iG8@xnLefQ~*eH|vY?gt5pBy0if6j>dDQXgBfYB}0U;2-UFQwon?{b!m~ zzSC&$PaTskAW3RlhheKQjgLhqG^6TI8kF0_FRFV^Y8ps};{45Yo6Z-KLEt+xzSe-W zo8!bTOGn%2pPZ#lcT?Ml72yX^q* zlk;ZrpC3kqj^&*i)wug6n*k1f;D$DQNAF^QhOGPQ=ahEpse3l79`s10h+S;$+1!)^ zKzb`TX+^%yC(1)^J=NlA?~R7MR-OMvfL?q8w=k{@^3>$VK0mCTd0`h4FIMs9(L%kL zJ1K@X_O@rClchZokt5Z=as!>}zpzL}(W;J4?=oajv79dYw%TxavJTME19<>}|4t3J z!uy*p@qeF1J4XI7c>90NZk9^rmf0K7cO`!P@t#{?$9ey~=n|0m3Y0w->ls;}J28N# zJ|30o$xlu$@K+YB7kb^TiRZ z=jPLQvK`QH-pC<)`jN{2wf8?h@+$R~dT`l0rZ6mI#p1-9@gI)L+ogV)Kd1ERs|*9N z59!%cG-p@7j@-miba~5f;B>?l-5=?7zh7RN589mi;p^-9MKAf!%>tV8kL&!_`+k@6 z%6@eLCo}HvIJeyN%hqY{mu%bTba#@f&WWqw-KmkUUe|!Pn*IcCHC?*?a}s!ODsXyW z>ArLA;JvBxuj|8pS?PoKrUIu2fEwOz{@49c%Jf+2aasNOs=yPAL{au6LAQ}U1x^rU zVu_npsor3pv|z5z{e9 c`_%v2KVjIvYFmS|C<739y85}Sb4q9e0J2ozwg3PC literal 0 HcmV?d00001 From 629d724d6188495fbbe5a58a4bcd5f5abf94499a Mon Sep 17 00:00:00 2001 From: Eamon Date: Thu, 16 Apr 2026 19:43:50 +0530 Subject: [PATCH 15/27] Update README with new graph of Neural scaling Law --- README.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 84defa7..b96afe4 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,7 @@ # Quadtrix [![License](https://img.shields.io/github/license/Eamon2009/Transformer-language-model)](LICENSE) -Screenshot 2026-04-15 225925 -Screenshot 2026-04-15 231851 +image +image + A minimal, educational GPT-style transformer trained character-by-character on children's stories. No pre-trained weights. No fine-tuning. Just raw PyTorch, from init to generation. From c9e068429e734df7b91780cfe7fe0d2b8c93ab5e Mon Sep 17 00:00:00 2001 From: Eamon Date: Fri, 17 Apr 2026 16:33:43 +0530 Subject: [PATCH 16/27] training logs --- logs/read.md | 85 ++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 85 insertions(+) create mode 100644 logs/read.md diff --git a/logs/read.md b/logs/read.md new file mode 100644 index 0000000..3458e38 --- /dev/null +++ b/logs/read.md @@ -0,0 +1,85 @@ +# Logs + +This folder stores training logs, loss curves, and run history. + +## Structure + +``` +logs/ +├── train_loss.csv ← loss per epoch (easy to plot) +├── train_run_latest.log ← full console output of latest run +└── run_YYYYMMDD_HHMM.log ← timestamped logs per run +``` + +## How to Log (add to your transformer.py / train script) + +```python +import csv +import os +import logging +from datetime import datetime + +# --- Setup --- +os.makedirs("logs", exist_ok=True) +timestamp = datetime.now().strftime("%Y%m%d_%H%M") + +# Console + file logger +logging.basicConfig( + level=logging.INFO, + format="%(asctime)s | %(message)s", + handlers=[ + logging.FileHandler(f"logs/run_{timestamp}.log"), + logging.FileHandler("logs/train_run_latest.log", mode="w"), + logging.StreamHandler() # still prints to terminal + ] +) +logger = logging.getLogger(__name__) + +# CSV loss tracker +loss_csv = "logs/train_loss.csv" +if not os.path.exists(loss_csv): + with open(loss_csv, "w", newline="") as f: + writer = csv.writer(f) + writer.writerow(["epoch", "train_loss", "val_loss", "lr"]) + +def log_epoch(epoch, train_loss, val_loss=None, lr=None): + logger.info(f"Epoch {epoch:03d} | Train Loss: {train_loss:.4f}" + + (f" | Val Loss: {val_loss:.4f}" if val_loss else "") + + (f" | LR: {lr:.6f}" if lr else "")) + with open(loss_csv, "a", newline="") as f: + writer = csv.writer(f) + writer.writerow([epoch, train_loss, val_loss or "", lr or ""]) +``` + +## Usage in Training Loop + +```python +for epoch in range(num_epochs): + train_loss = train_one_epoch(model, optimizer) + log_epoch(epoch, train_loss, lr=scheduler.get_last_lr()[0]) + save_checkpoint(model, optimizer, epoch, train_loss) +``` + +## Plotting Loss Curve + +```python +import pandas as pd +import matplotlib.pyplot as plt + +df = pd.read_csv("logs/train_loss.csv") +plt.plot(df["epoch"], df["train_loss"], label="Train Loss") +if "val_loss" in df and df["val_loss"].notna().any(): + plt.plot(df["epoch"], df["val_loss"], label="Val Loss") +plt.xlabel("Epoch") +plt.ylabel("Loss") +plt.title("GPT Training Loss") +plt.legend() +plt.savefig("logs/loss_curve.png") +plt.show() +``` + +## Notes + +- `.log` files and `loss_curve.png` can be pushed to GitHub (they're small) +- `train_loss.csv` is very useful to track across GPU runs +- Timestamped logs help you compare different training runs \ No newline at end of file From ae370e8fd5cc641f9990156d8b9de5a45f8b73d5 Mon Sep 17 00:00:00 2001 From: Eamon Date: Sat, 18 Apr 2026 19:26:18 +0530 Subject: [PATCH 17/27] training logs summary --- logs/run1.log | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) create mode 100644 logs/run1.log diff --git a/logs/run1.log b/logs/run1.log new file mode 100644 index 0000000..347e456 --- /dev/null +++ b/logs/run1.log @@ -0,0 +1,16 @@ +[ 0/3000] 0.0% train=3.2961 val=3.2981 elapsed=12s ETA=0s best! +[ 200/3000] 6.7% train=2.3038 val=2.2490 elapsed=141s ETA=1959s best! +[ 400/3000] 13.3% train=2.2469 val=2.1950 elapsed=292s ETA=1891s best! +[ 600/3000] 20.0% train=2.1842 val=2.1318 elapsed=436s ETA=1739s best! +[ 800/3000] 26.7% train=1.9742 val=1.9103 elapsed=581s ETA=1594s best! +[ 1000/3000] 33.3% train=1.7628 val=1.7002 elapsed=723s ETA=1443s best! +[ 1200/3000] 40.0% train=1.6714 val=1.6040 elapsed=863s ETA=1293s best! +[ 1400/3000] 46.7% train=1.5889 val=1.5360 elapsed=1015s ETA=1158s best! +[ 1600/3000] 53.3% train=1.5375 val=1.4723 elapsed=1166s ETA=1019s best! +[ 1800/3000] 60.0% train=1.4847 val=1.4525 elapsed=1320s ETA=879s best! +[ 2000/3000] 66.7% train=1.4604 val=1.4081 elapsed=1472s ETA=735s best! +[ 2200/3000] 73.3% train=1.4113 val=1.3857 elapsed=1653s ETA=600s best! +[ 2400/3000] 80.0% train=1.3923 val=1.3725 elapsed=1820s ETA=454s best! +[ 2600/3000] 86.7% train=1.3501 val=1.3446 elapsed=1998s ETA=307s best! +[ 2800/3000] 93.3% train=1.3336 val=1.3334 elapsed=2174s ETA=154s best! +[ 2999/3000] 100.0% train=1.3191 val=1.3145 elapsed=2363s ETA=0s best! From 95d4e00ba566833ee4be8ea025e4987fe4aa1c94 Mon Sep 17 00:00:00 2001 From: Eamon Date: Sun, 19 Apr 2026 08:13:30 +0530 Subject: [PATCH 18/27] run2 logs --- logs/run2.log | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) create mode 100644 logs/run2.log diff --git a/logs/run2.log b/logs/run2.log new file mode 100644 index 0000000..c836ca8 --- /dev/null +++ b/logs/run2.log @@ -0,0 +1,21 @@ +[ 0/5000] 0.0% train=4.9244 val=4.9262 elapsed=31s ETA=0s best! +[ 250/5000] 5.0% train=2.1218 val=2.1169 elapsed=206s ETA=3901s best! +[ 500/5000] 10.0% train=1.3606 val=1.3500 elapsed=391s ETA=3510s best! +[ 750/5000] 15.0% train=1.1540 val=1.1411 elapsed=575s ETA=3250s best! +[ 1000/5000] 20.0% train=1.0332 val=1.0296 elapsed=757s ETA=3024s best! +[ 1250/5000] 25.0% train=0.9657 val=0.9556 elapsed=941s ETA=2819s best! +[ 1500/5000] 30.0% train=0.9305 val=0.9189 elapsed=1124s ETA=2619s best! +[ 1750/5000] 35.0% train=0.8935 val=0.8853 elapsed=1306s ETA=2424s best! +[ 2000/5000] 40.0% train=0.8673 val=0.8602 elapsed=1490s ETA=2233s best! +[ 2250/5000] 45.0% train=0.8413 val=0.8367 elapsed=1672s ETA=2042s best! +[ 2500/5000] 50.0% train=0.8162 val=0.8141 elapsed=1855s ETA=1854s best! +[ 2750/5000] 55.0% train=0.8058 val=0.7995 elapsed=2038s ETA=1666s best! +[ 3000/5000] 60.0% train=0.7888 val=0.7803 elapsed=2221s ETA=1479s best! +[ 3250/5000] 65.0% train=0.7798 val=0.7730 elapsed=2403s ETA=1293s best! +[ 3500/5000] 70.0% train=0.7634 val=0.7551 elapsed=2585s ETA=1107s best! +[ 3750/5000] 75.0% train=0.7588 val=0.7528 elapsed=2768s ETA=922s best! +[ 4000/5000] 80.0% train=0.7480 val=0.7434 elapsed=2951s ETA=737s best! +[ 4250/5000] 85.0% train=0.7381 val=0.7351 elapsed=3134s ETA=552s best! +[ 4500/5000] 90.0% train=0.7371 val=0.7314 elapsed=3316s ETA=368s best! +[ 4750/5000] 95.0% train=0.7282 val=0.7239 elapsed=3498s ETA=183s best! +[ 4999/5000] 100.0% train=0.7259 val=0.7176 elapsed=3680s ETA=0s best! From 7bb893ba7ecca3c36ca58f2d6e8fbeeb8b3a24c7 Mon Sep 17 00:00:00 2001 From: Eamon Date: Mon, 20 Apr 2026 22:27:19 +0530 Subject: [PATCH 19/27] Create CNAME --- CNAME | 1 + 1 file changed, 1 insertion(+) create mode 100644 CNAME diff --git a/CNAME b/CNAME new file mode 100644 index 0000000..3638c6d --- /dev/null +++ b/CNAME @@ -0,0 +1 @@ +quadtrix.io \ No newline at end of file From 899e51a8f28990146d7cc4dede69dffd21ef6497 Mon Sep 17 00:00:00 2001 From: Eamon Date: Mon, 20 Apr 2026 22:30:36 +0530 Subject: [PATCH 20/27] Update CNAME --- CNAME | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CNAME b/CNAME index 3638c6d..e381adc 100644 --- a/CNAME +++ b/CNAME @@ -1 +1 @@ -quadtrix.io \ No newline at end of file +www.quadtrix.io \ No newline at end of file From 2d09df19891e288642c980f879e2de22adda9974 Mon Sep 17 00:00:00 2001 From: Eamon Date: Mon, 20 Apr 2026 22:34:27 +0530 Subject: [PATCH 21/27] Delete CNAME --- CNAME | 1 - 1 file changed, 1 deletion(-) delete mode 100644 CNAME diff --git a/CNAME b/CNAME deleted file mode 100644 index e381adc..0000000 --- a/CNAME +++ /dev/null @@ -1 +0,0 @@ -www.quadtrix.io \ No newline at end of file From b55ff5ffe9200d26d2557c337f6bb2c2d6b2a01c Mon Sep 17 00:00:00 2001 From: Eamon Date: Mon, 20 Apr 2026 23:08:48 +0530 Subject: [PATCH 22/27] Delete README.md --- README.md | 504 ------------------------------------------------------ 1 file changed, 504 deletions(-) delete mode 100644 README.md diff --git a/README.md b/README.md deleted file mode 100644 index b96afe4..0000000 --- a/README.md +++ /dev/null @@ -1,504 +0,0 @@ -# Quadtrix [![License](https://img.shields.io/github/license/Eamon2009/Transformer-language-model)](LICENSE) -image -image - - - -A minimal, educational GPT-style transformer trained character-by-character on children's stories. No pre-trained weights. No fine-tuning. Just raw PyTorch, from init to generation. -Character-level Transformer is a sequence-to-sequence architecture that operates on the granularity of individual characters rather than words or subword tokens. By utilizing a self-attention mechanism over long sequences of characters, the model learns to construct internal representations of morphology, syntax, and semantics from the ground up, effectively eliminating "out-of-vocabulary" (OOV) issues. While this approach allows for high fidelity in modeling rare words, spelling variations, and creative linguistics, it significantly increases the computational complexity—typically $O(L^2)$ where $L$ is the sequence length—as a single sentence requires many more steps than its word-level equivalent. Consequently, character-level Transformers often require deeper architectures or auxiliary losses to capture the long-range dependencies necessary to match the semantic performance of traditional token-based models - -> **The goal**: Understand how language models learn patterns. See it happen on your machine. Train in minutes on a GPU. - ---- - -## Quick Start - -```bash -# Install PyTorch -pip install torch - -# Run training -python transformer.py - -# The script will train, save best weights, then generate text forever -# Press Ctrl+C to stop -``` - -**That's it.** No complex setup, no data pipelines, no credentials. - ---- - -## What Is This? - -Quadtrix is a learning project. It trains a tiny transformer on text—character by character—and learns which characters tend to follow others. At generation time, it predicts the next character, feeds it back in, and repeats. - -**It's the same architecture as GPT**, just much smaller (1M–11M parameters instead of 175B) and trained on much less data (thousands of stories instead of the internet). - -The magic: in just 6 minutes on a Tesla T4 GPU, a 2M-parameter model learns meaningful patterns about narrative structure, dialogue, and storytelling. The output isn't Shakespeare, but it *is* recognizable as a story. - ---- - -## How It Works - -### The Pipeline - -``` -1. Load children's stories from disk - ↓ -2. Encode each character as an integer (vocab size: 28–110) - ↓ -3. Split into train/val chunks (80/20) - ↓ -4. Build a GPT-style transformer with: - - Embedding layer (character → vector) - - Transformer blocks (self-attention + feedforward) - - Output projection (vector → logits over vocab) - ↓ -5. Train for N steps, measuring loss every eval_interval - ↓ -6. Save best weights whenever validation loss improves - ↓ -7. Load best model and generate text forever - ↓ -8. Press Ctrl+C to stop -``` - -### What the Model Actually Does - -Each forward pass: -1. Takes a sequence of character indices (e.g., "Once upon...") -2. Embeds each into a dense vector -3. Passes through transformer layers (multi-head attention learns which characters to attend to) -4. Outputs logits (scores) for every possible next character -5. Samples the next character according to those probabilities -6. Feeds it back in and repeats - -**Loss function**: Cross-entropy. The model minimizes surprise on unseen text. - -**Training**: Adam optimizer with a learning rate schedule. Dropout prevents overfitting. - ---- - -## Project Structure - -``` -transformer.py ← Everything. One file. -best_model.pt ← Saved weights (after first training run) -data.txt ← Your text file (any UTF-8 file works) -``` - -That's all. No fancy folder structure. No config files. Edit hyperparameters directly in the script. - ---- - -## Configuration & Hardware - -Quadtrix is designed to work anywhere: laptop CPU to cloud GPU. Edit these hyperparameters in `transformer.py`: - -```python -# ============================================================================ -# Hyperparameters -# ============================================================================ -batch_size = 64 # Sequences per batch -block_size = 128 # Context window (tokens) -max_iters = 5000 # Total training steps -eval_interval = 200 # Print loss every N steps -learning_rate = 3e-4 -n_embd = 200 # Embedding dimension -n_head = 4 # Attention heads per layer -n_layer = 4 # Number of transformer blocks -dropout = 0.2 # Regularization -``` - -### Three Pre-Tuned Configurations - -**CPU (Laptop) — Fast Feedback** -```python -batch_size, block_size, max_iters = 16, 128, 3000 -n_embd, n_head, n_layer = 128, 4, 4 -# ~0.82M parameters -# Trains in ~40 min on AMD Ryzen -``` - -**GPU (Google Colab) — Best Quality** -```python -batch_size, block_size, max_iters = 64, 256, 5000 -n_embd, n_head, n_layer = 384, 6, 6 -# ~10.82M parameters -# Trains in ~60 min on Colab GPU -# Best output quality -``` - -**GPU (Tesla T4) — Efficient** -```python -batch_size, block_size, max_iters = 64, 128, 5000 -n_embd, n_head, n_layer = 200, 4, 4 -# ~1.99M parameters -# Trains in **6.1 minutes** ← Fastest -# Best parameter/data balance -``` - ---- - -## Training Results - -Three runs. Three different hardware setups. All converged well. - -### Run 3 — Tesla T4 (Latest) ⭐ - -**Configuration**: 4 layers × 4 heads × 200 dim = **1.99M params** - -| Metric | Value | -|--------|-------| -| Device | Tesla T4 (CUDA 13.0) | -| Dataset | ~31.4M characters (children's stories) | -| Train tokens | 28.3M | -| Val tokens | 3.1M | -| Training time | **6.1 minutes** | -| Best val loss | **0.9250** | -| Final train loss | 0.9307 | -| Overfitting | None detected | - -**Training curve** (every 200 steps): -``` -[ 0/5000] train=4.6207 val=4.6202 elapsed=2s << best! -[ 200/5000] train=2.2058 val=2.1986 elapsed=17s << best! -[ 400/5000] train=1.6111 val=1.6039 elapsed=32s << best! -[ 1000/5000] train=1.2495 val=1.2567 elapsed=76s << best! -[ 2000/5000] train=1.0731 val=1.0765 elapsed=149s << best! -[ 3000/5000] train=1.0000 val=0.9956 elapsed=221s << best! -[ 4000/5000] train=0.9601 val=0.9642 elapsed=294s -[ 4200/5000] train=0.9515 val=0.9489 elapsed=309s << best! -[ 4400/5000] train=0.9433 val=0.9431 elapsed=323s << best! -[ 4800/5000] train=0.9331 val=0.9250 elapsed=353s << best! -[ 4999/5000] train=0.9307 val=0.9430 elapsed=367s - -[DONE] Training finished in 367.0s (6.1 min) | Best val loss: 0.9250 -``` - -**Key insight**: This run hit the **sweet spot**—large enough to learn coherent patterns, small enough to train fast. It's the reference configuration. - ---- - -### Run 2 — Google Colab (10.82M Parameters) - -**Configuration**: 6 layers × 6 heads × 384 dim - -| Metric | Value | -|--------|-------| -| Device | CUDA (Google Colab GPU) | -| Dataset | ~88.4M characters | -| Parameters | 10.82M | -| Training time | 61.3 minutes | -| Best val loss | **0.7176** | -| Overfitting | None | - -**Result**: Larger model, more data = **best output quality**. Slower to train but produces recognizable narratives. - ---- - -### Run 1 — CPU Laptop (0.82M Parameters) - -**Configuration**: 4 layers × 4 heads × 128 dim - -| Metric | Value | -|--------|-------| -| Device | AMD Ryzen 5 PRO 3500U (CPU only) | -| Parameters | 0.82M | -| Dataset | ~201K characters | -| Training time | 39.4 minutes | -| Best val loss | 1.3145 | -| Overfitting | None | - -**Result**: Smallest model, tiniest dataset. Trains fastest on CPU. Output is fragmented but shows the model learned *something*. - ---- - -## Head-to-Head Comparison - -| Metric | Run 1 — CPU | Run 2 — Colab | Run 3 — T4 ⭐ | -|--------|-------------|--------------|------------| -| **Parameters** | 0.82M | 10.82M | 1.99M | -| **Training data** | 200K chars | 88.4M chars | 31.4M chars | -| **Training time** | 39.4 min | 61.3 min | **6.1 min** | -| **Best val loss** | 1.3145 | **0.7176** | 0.9250 | -| **Output coherence** | Fragmented | Coherent paragraphs | Basic sentences | -| **Overfitting** | ✓ None | ✓ None | ✓ None | -| **Still improving?** | Yes | Yes | Yes | - -> **Observation**: All three were still improving at the final checkpoint. More training steps = better loss. - ---- - -## Example Outputs - -### Run 2 Output (10.82M params) — Best Quality - -``` -Upon a time, there were two friends, Jack and Tom. They had a cold doll in -the sunshine. - -One day, Jack saw that he was universed. He used the sky at past it to march -around the garden. He had a small ball on his face. He felt dizzy and wanted -to share his happy with them. - -Nack knew it was feeling important to his passion in their rooms. He knew -that night, he had never seen a small boy just soon could drink. -``` - -**Analysis**: Clear sentence structure. Named characters. Logical progression. Some linguistic oddities ("felt dizzy and wanted to share his happy") but unmistakably a story. - ---- - -### Run 3 Output (1.99M params, Tesla T4) — Efficient - -``` -Timmy and elsed him to tell being jumping things. They were tired and making -some pinkets and help paper me. They had to see them, drain and ran ar her -mommy. They also fast with the stretch and sacks the changer. - -Lily's truck laughed and saw a rock. She said, "You can't here some wet -sicks. You have something new favorite toys, I do yours." -``` - -**Analysis**: Narrative present. Dialogue structure intact. Characters named. Some word-order errors and made-up words, but the *shape* of a story is clear. - ---- - -### Run 1 Output (0.82M params) — Minimal - -``` -when years me told be found a big ea reak abig driendly they named not she -rabbit smiled by aded he what in again one smiled the mushrought boy one day -and was arroom him that she rabbing animals the dreezed at neard had to there -man owl them with o box -``` - -**Analysis**: Word boundaries mostly intact. Character names present (rabbit, owl). But syntax falls apart—the model is struggling to hold sentence structure. This is a 0.82M model trained on tiny data; it's learning *something* but hasn't converged to coherent text. - ---- -## Project Structure - -``` -. -├── .github/ -│ ├── ISSUE_TEMPLATE/ # GitHub issue templates -│ └── workflows/ # CI/CD workflows -├── .vscode/ # VS Code configuration -├── GPU train/ # GPU training scripts and configurations -├── assets/ # Images, diagrams, and other assets -├── checkpoints/ # Saved model checkpoints -├── config/ # Configuration files -├── data_set/ # Training and validation datasets -├── evaluate/ # Evaluation scripts and metrics -├── generate/ # Text generation scripts -├── logs/ # Training logs and tensorboard files -├── train_test/ # Training and testing utilities -├── .gitattributes # Git attributes configuration -├── .gitignore # Git ignore rules -├── .python-version # Python version specification -├── LICENSE # MIT License -├── README.md # This file -├── cleaned.txt # Cleaned training data -├── gpt-from-scratch.ipynb # Jupyter notebook implementation -├── requirements.txt # Python dependencies -├── traindata.txt # Raw training data -└── transformer.py # Main transformer model implementation -``` - -## Loss Curves & Training Dynamics - -All three runs showed the classic learning curve: - -``` -Phase 1: Rapid drop (0–20% of training) - - Model learns basic character transitions - - Loss halves or better - -Phase 2: Steady descent (20–80%) - - Model learns longer-range patterns - - Character names, sentence boundaries - - Loss continues down but more gradually - -Phase 3: Diminishing returns (80–100%) - - Model learning slows - - Val loss still improving but incremental - - More data or capacity would help here -``` - -**Train/Val Gap Analysis** (indicator of overfitting): - -| Run | Final Train | Final Val | Gap | -|-----|-------------|-----------|-----| -| CPU | 1.3191 | 1.3145 | 0.0046 | -| Colab | 0.7259 | 0.7176 | 0.0083 | -| T4 | 0.9307 | 0.9250 | 0.0057 | - -All gaps are tiny. **No overfitting detected.** The model is generalizing well to unseen text. - ---- - -## Scaling Laws: Where Quadtrix Sits -Screenshot 2026-03-17 171921 - - -The Chinchilla (2022) scaling law suggests: **~20 tokens of training data per parameter is optimal**. - -Let's see how our runs align: - -| Model | Parameters | Training Data | Optimal (20×) | Coverage | -|-------|------------|---------------|--------------|----------| -| Run 1 | 0.82M | 200K tokens | 16.4M | **1.2%** | -| Run 3 | 1.99M | 28.3M tokens | 39.8M | **71.1%** ← Best balanced | -| Run 2 | 10.82M | 79.6M tokens | 216M | **36.8%** | -| GPT-2 Small | 117M | 40B tokens | 2.3B | ~1700% | -| GPT-3 | 175B | 600B tokens | 3.5T | ~17% | - -**Insight**: Run 3 is the most *balanced*—the model size and data quantity are well-matched. Run 2 has the largest model but is only at 37% optimal data coverage, meaning it would benefit more from adding data than adding parameters. - -**Next steps for any run**: -1. **More training steps** — All three were still falling at the final checkpoint -2. **More data** — Run 3 is closest to optimal; Run 2 would benefit most -3. **Larger model** — Only worth doing once data coverage exceeds 50% - ---- - -## How Generation Works - -Once training is done, `best_model.pt` contains frozen weights. Generation is simple: - -```python -# Pseudocode for generation -seed = torch.tensor([[start_token]]) # e.g., start_token = 0 - -for _ in range(num_chars_to_generate): - # Forward pass through all layers - logits = model(seed)[-1, :] # Get last token's logits - - # Convert logits to probabilities - probs = softmax(logits / temperature) - - # Sample next token - next_token = sample(probs) - - # Append and continue - seed = torch.cat([seed, next_token], dim=-1) - - # Trim to context window if needed - seed = seed[:, -block_size:] -``` - -**Why output differs each run**: The sampling step is random. Same weights, different random seeds = different output. Add `torch.manual_seed(42)` for deterministic output. - ---- - -## Known Limitations - -1. **Character-level learning** — the model learns characters, not words. It cannot reliably spell or track meaning across paragraphs. - -2. **Output coherence** — especially on smaller runs. Sentences drift logically. Names disappear. Tense breaks. This is expected at this scale. - -3. **All models undertrained** — validation loss was still improving at iteration N. More training steps would help all three. - -4. **Limited data** — Run 2 is only at 37% optimal data coverage. A larger story corpus would meaningfully improve output quality. - -5. **No long-range memory** — transformer context window is fixed (128 or 256 tokens). The model cannot reference events from 10 paragraphs ago. - ---- - -## Technical Details - -### Model Architecture - -```python -class GPTModel(nn.Module): - def __init__(self, vocab_size, n_embd, n_head, n_layer, block_size, dropout): - self.token_embedding = nn.Embedding(vocab_size, n_embd) - self.position_embedding = nn.Embedding(block_size, n_embd) - self.transformer = nn.Sequential( - *[TransformerBlock(n_embd, n_head, dropout) for _ in range(n_layer)] - ) - self.ln_final = nn.LayerNorm(n_embd) - self.lm_head = nn.Linear(n_embd, vocab_size) - self.dropout = nn.Dropout(dropout) - - def forward(self, x): - B, T = x.shape - tok_emb = self.token_embedding(x) # (B, T, n_embd) - pos_emb = self.position_embedding(torch.arange(T)) - x = self.dropout(tok_emb + pos_emb) - x = self.transformer(x) - x = self.ln_final(x) - logits = self.lm_head(x) - return logits -``` - -### Transformer Block - -Each block contains: -- **Multi-head self-attention**: `(B, T, n_embd) → (B, T, n_embd)` -- **Feedforward network**: Two linear layers with ReLU -- **Layer normalization & residual connections** -- **Dropout for regularization** - -### Training Loop - -```python -for step in range(max_iters): - # Batch a random chunk of training data - batch_x, batch_y = get_batch('train') - - # Forward pass - logits = model(batch_x) - loss = F.cross_entropy(logits.view(-1, vocab_size), batch_y.view(-1)) - - # Backward pass - optimizer.zero_grad() - loss.backward() - torch.nn.utils.clip_grad_norm_(model.parameters(), 1.0) - optimizer.step() - - # Evaluate and save - if step % eval_interval == 0: - val_loss = estimate_loss('val') - print(f"train={train_loss:.4f} val={val_loss:.4f}") - - if val_loss < best_val_loss: - best_val_loss = val_loss - torch.save(model.state_dict(), 'best_model.pt') -``` - ---- - -## Why This Project Matters - -1. **Educational**: See exactly how a language model learns. No black boxes. - -2. **Verifiable**: You can trace loss, inspect weights, understand every line of code. - -3. **Fast**: Even on CPU, training finishes in under an hour. On GPU, minutes. - -4. **Runnable**: One script, one dependency (PyTorch). No cloud account required. - -Quadtrix is to GPT what nanoGPT is to transformers: a stripped-down, readable, educational version that teaches the core ideas. - ---- - -## Getting Started - -1. **Clone or download** `transformer.py` -2. **Install PyTorch**: `pip install torch` -3. **Add your data**: Place a UTF-8 text file at `data.txt` (or edit the filename in the script) -4. **Run**: `python transformer.py` -5. **Watch**: Loss decreases. Weights save. Text generates. -6. **Tweak**: Edit hyperparameters and re-run to see how each affects training speed and output quality. - ---- - -## License - -MIT - ---- - -*Built with PyTorch. | [GitHub](https://github.com/Eamon2009/Transformer-language-model)* From 3a168f0e83a82ce69692f011b351161bbf39ba83 Mon Sep 17 00:00:00 2001 From: Eamon Date: Mon, 20 Apr 2026 23:09:21 +0530 Subject: [PATCH 23/27] Update print statement from 'Hello' to 'Goodbye' --- index.html | 1345 ++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 1345 insertions(+) create mode 100644 index.html diff --git a/index.html b/index.html new file mode 100644 index 0000000..e9967f6 --- /dev/null +++ b/index.html @@ -0,0 +1,1345 @@ + + + + + +Quadtrix — Educational GPT-Style Transformer + + + + + + + + + +
+
+
+
+
+
+ Educational GPT-Style Transformer — PyTorch from scratch +
+

Quadtrix

+

A minimal, educational GPT-style transformer trained character-by-character on children's stories.

+

No pre-trained weights. No fine-tuning. Just raw PyTorch, from init to generation.

+
+ Get Started + Explore Architecture + + + View on GitHub + +
+ +
+
+
+ model_output.txt — generated text +
+
+
// sampling at temperature=1.0
+

+ | +

+
+
+ +
+
0.82M
Min Parameters
CPU run
+
0.7176
Best Val Loss
10.82M model
+
6.1 min
Fastest Training
Tesla T4 GPU
+
10.82M
Max Parameters
Colab GPU
+
+
+
Scroll
+
+ + +
+
+
+
/ overview
+

What Is Quadtrix?

+

Trains a tiny transformer on text — character by character — and learns which characters tend to follow others. Same architecture as GPT, just scaled way down.

+
+ +
+
⚡
Same Architecture as GPT
Identical transformer structure to GPT — just 1M–11M params instead of 175B. Scale, not magic.
+
📖
Character-Level Learning
Operates on individual characters. No vocabulary limits, no OOV (out-of-vocabulary) problems.
+
🔬
Fully Transparent
Every line of code is readable. Trace loss, inspect weights, understand every operation.
+
🚀
Train in Minutes
6 minutes on a Tesla T4. Under an hour on CPU. No cloud account or API keys needed.
+
+ +
+
+

+ // The Pipeline +

+
    +
  • 1
    Load children's stories from disk
    • +
  • 2
    Encode each character as an integer (vocab: 28–110)
    • +
  • 3
    Split into train/val chunks (80/20)
    • +
  • 4
    Build GPT-style transformer
    • +
  • 5
    Train for N steps with loss tracking
    • +
  • 6
    Save best weights on val improvement
    • +
  • 7
    Load best model and generate text
    • +
+
+
+

+ // Each Forward Pass +

+
+
Input
Sequence of character indices — e.g., "Once upon..."
+
Embed
Each character → dense vector of floats
+
Attend
Transformer layers learn which chars to attend to
+
Project
Output logits = scores over all possible next chars
+
Sample
Softmax → probability → random sample next char
+
Loop
Feed next char back in and repeat indefinitely
+
+
+
Loss function
+

Cross-entropy — the model minimizes surprise on unseen text. Adam optimizer with learning rate schedule and dropout for regularization.

+
+
+
+
+
+ + +
+
+
+
/ architecture
+

Model Architecture

+

GPT-style decoder-only transformer. Identical conceptual structure to GPT-2, scaled down to run on commodity hardware.

+
+
+
+
+ + +
+
+ +
+
+
+

+ // Transformer Block Components +

+
+ +
+
+
hyperparameters.py
+
+
batch_size16–64
+
block_size128–256
+
n_embd128–384
+
n_head4–6
+
n_layer4–6
+
dropout0.2
+
learning_rate3e-4
+
max_iters3000–5000
+
+
+
+
+
+
+ + +
+
+
+
/ deep dive
+

How Transformers Really Work

+

The key mechanisms that make transformers powerful. Understanding these is understanding every modern language model.

+
+
+
+
+
+
+
⚠ Character-Level Complexity
+
+
Complexity

Self-attention scales as O(L²). One sentence ≈ 250 chars = 25× more attention pairs than word-level.

+
Advantage

No vocabulary limits. Can spell any word, handle typos, creative linguistics. OOV problems simply don't exist.

+
Trade-off

Harder to learn long-range semantic dependencies. Context windows fill up faster than word-level.

+
+
+
+
+ + +
+
+
+
/ results
+

Training Results

+

Three runs across three hardware setups. All converged well — none overfit, none underfit catastrophically.

+
+
+ + + +
+
+
+
+
+
+ loss_curve.svg +
+ + train + + + val + +
+
+
+ +
+
+
+
✓
+

No overfitting detected. Train/val gap is 0.0057 — the model generalizes to unseen text.

+
+
+
+
+
Run 3 — Tesla T4 output
+

+
+
+
head_to_head.md
+ + + + + + + +
MetricT4 ⭐ColabCPU
Parameters1.99M10.82M0.82M
Best Val Loss0.92500.71761.3145
Training Time6.1 min61.3 min39.4 min
+
+
+
Key Observation
+

All three runs were still improving at the final checkpoint. Validation loss was still falling. More training steps or data would help all runs.

+
+
+
+
+
+ + +
+
+
+
/ scaling laws
+

Where Quadtrix Sits

+

The Chinchilla (2022) scaling law: ~20 tokens of training data per parameter is optimal. Here's how our runs align with the frontier.

+
+
+
+

// Chinchilla Coverage

+
+ + + + + + + + + +
ModelParamsDataCoverage
Run 1 — CPU0.82M200K
1.2%
Run 3 — T4 ⭐1.99M28.3M
71.1%
Run 2 — Colab10.82M79.6M
36.8%
GPT-2 Small117M40B
1700%
GPT-3175B600B
17%
+
+
+
+

// Scaling Law Insights

+
+
+
1
+
The Chinchilla Scaling Law
DeepMind's 2022 paper established the optimal ratio: ~20 training tokens per parameter. Undertrained models waste compute; overtrained models waste data.
+
+
+
2
+
Run 3 is the Sweet Spot
At 71% of Chinchilla-optimal coverage, Run 3 has the best parameter-to-data ratio. It learns the most per unit of compute of all three runs.
+
+
+
3
+
GPT-3 Violates Chinchilla
GPT-3 (175B params) was trained on only 600B tokens — ~17% of what Chinchilla recommends. This is why smaller Chinchilla-trained models often match GPT-3.
+
+
+
4
+
What to Do Next
All runs benefit from more training steps first. Only after data coverage exceeds ~50% should you scale up model size.
+
+
+
+
+
+
+ + +
+
+
+
/ generation
+

How Generation Works

+

Once training finishes, best_model.pt contains frozen weights. Generation is a simple loop — predict, sample, feed back.

+
+
+
+
generate.py
+
+
+
+

// Temperature Effect

+
+
+
+ output — temperature=1.0 + +
+

+
+
+
Deterministic generation
+

Same weights + different random seed = different output. Add torch.manual_seed(42) for reproducible results.

+
+
+
+

// Known Limitations

+
+
⚠
Character-level trade-off
Learns characters, not words. Can't reliably spell or track meaning across paragraphs.
+
📉
Output coherence
Sentences drift logically, names disappear, tense breaks — expected at this scale.
+
⏳
All models undertrained
Val loss was still improving at the final checkpoint. More steps would help all three.
+
📦
Limited data
Run 2 is only at 37% optimal data coverage. A larger story corpus would help significantly.
+
🔭
No long-range memory
Fixed context window (128–256 tokens). Cannot reference events from earlier in the story.
+
🔤
No subword tokenization
'fox' = 3 char tokens vs 1 word token. Context fills up faster, harder to learn semantics.
+
+
+
+ + +
+
+
+
/ quick start
+

Get Running in Minutes

+

One script. One dependency. No cloud account, no credentials, no pipeline.

+
+
+
+

// Installation Steps

+
+
+
01 — Clone or download
+
git clone https://github.com/Eamon2009/Transformer-language-model
+
+
+
02 — Install PyTorch
+
pip install torch
+
+
+
03 — Add your data
+
# Place any UTF-8 text file at data.txt# (or edit the filename in transformer.py)
+
+
+
04 — Run training
+
python transformer.py
+
+
+
05 — Watch it learn
+
+ [ 0/5000] train=4.6207 val=4.6202 << best! + [ 200/5000] train=2.2058 val=2.1986 << best! + [ 400/5000] train=1.6111 val=1.6039 << best! + ... + [DONE] Training finished in 367.0s | Best val loss: 0.9250 +
+
+
+
+
+

// Choose Your Config

+
+ + + +
+
+
+
GPU — Tesla T4
⭐ Recommended · 6.1 min · 1.99M params
+
+ 

+        

+        

+          
project structure

+          
transformer.py    ← Everything. One file.
+best_model.pt     ← Saved weights (after first run)
+data.txt          ← Your text (any UTF-8 file)

+          
No config files. Edit hyperparameters directly in the script.

+        

+        

+          
After training finishes...

+          
The script loads best_model.pt and generates text indefinitely. Press Ctrl+C to stop. Output differs each run because sampling is random.

+        

+      

+    

+  

+

+
+
+
+
+
+
+
+

From 1fb6087e97317d979d53605fae196accd820c39d Mon Sep 17 00:00:00 2001
From: Eamon 
Date: Mon, 20 Apr 2026 23:10:56 +0530
Subject: [PATCH 24/27] Delete index.html

---
 index.html | 1345 ----------------------------------------------------
 1 file changed, 1345 deletions(-)
 delete mode 100644 index.html

diff --git a/index.html b/index.html
deleted file mode 100644
index e9967f6..0000000
--- a/index.html
+++ /dev/null
@@ -1,1345 +0,0 @@
-
-
-
-
-
-Quadtrix — Educational GPT-Style Transformer
-
-
-
-
-
-
-
-
-
-

-  

-  

-  

-    

-      

-      Educational GPT-Style Transformer — PyTorch from scratch
-    

-    
Quadtrix

-    
A minimal, educational GPT-style transformer trained character-by-character on children's stories.

-    
No pre-trained weights. No fine-tuning. Just raw PyTorch, from init to generation.

-    

-      Get Started
-      Explore Architecture
-      
-        
-        View on GitHub
-      
-    

-
-    

-      

-        

-        model_output.txt — generated text
-      

-      

-        
// sampling at temperature=1.0

-        

-          |
-        

-      

-    

-
-    

-      
0.82M
Min Parameters
CPU run

-      
0.7176
Best Val Loss
10.82M model

-      
6.1 min
Fastest Training
Tesla T4 GPU

-      
10.82M
Max Parameters
Colab GPU

-    

-  

-  
Scroll

-

-
-
-

-  

-    

-      
/ overview

-      
What Is Quadtrix?

-      
Trains a tiny transformer on text — character by character — and learns which characters tend to follow others. Same architecture as GPT, just scaled way down.

-    

-
-    

-      
⚡
Same Architecture as GPT
Identical transformer structure to GPT — just 1M–11M params instead of 175B. Scale, not magic.

-      
📖
Character-Level Learning
Operates on individual characters. No vocabulary limits, no OOV (out-of-vocabulary) problems.

-      
🔬
Fully Transparent
Every line of code is readable. Trace loss, inspect weights, understand every operation.

-      
🚀
Train in Minutes
6 minutes on a Tesla T4. Under an hour on CPU. No cloud account or API keys needed.

-    

-
-    

-      

-        

-          // The Pipeline
-        

-        
    
-          
  • 1
    Load children's stories from disk
    • 
-          
  • 2
    Encode each character as an integer (vocab: 28–110)
    • 
-          
  • 3
    Split into train/val chunks (80/20)
    • 
-          
  • 4
    Build GPT-style transformer
    • 
-          
  • 5
    Train for N steps with loss tracking
    • 
-          
  • 6
    Save best weights on val improvement
    • 
-          
  • 7
    Load best model and generate text
    • 
-        

-      

-      

-        

-          // Each Forward Pass
-        

-        

-          
Input
Sequence of character indices — e.g., "Once upon..."

-          
Embed
Each character → dense vector of floats

-          
Attend
Transformer layers learn which chars to attend to

-          
Project
Output logits = scores over all possible next chars

-          
Sample
Softmax → probability → random sample next char

-          
Loop
Feed next char back in and repeat indefinitely

-        

-        

-          
Loss function

-          
Cross-entropy — the model minimizes surprise on unseen text. Adam optimizer with learning rate schedule and dropout for regularization.

-        

-      

-    

-  

-

-
-
-

-  

-    

-      
/ architecture

-      
Model Architecture

-      
GPT-style decoder-only transformer. Identical conceptual structure to GPT-2, scaled down to run on commodity hardware.

-    

-    

-      

-        

-          
-          
-        

-        

-          
-        

-      

-      

-        

-          // Transformer Block Components
-        

-        

-          
-        

-        

-          
hyperparameters.py

-          

-            
batch_size16–64

-            
block_size128–256

-            
n_embd128–384

-            
n_head4–6

-            
n_layer4–6

-            
dropout0.2

-            
learning_rate3e-4

-            
max_iters3000–5000

-          

-        

-      

-    

-  

-

-
-
-

-  

-    

-      
/ deep dive

-      
How Transformers Really Work

-      
The key mechanisms that make transformers powerful. Understanding these is understanding every modern language model.

-    

-    

-      

-      

-    

-    

-      
⚠ Character-Level Complexity

-      

-        
Complexity
Self-attention scales as O(L²). One sentence ≈ 250 chars = 25× more attention pairs than word-level.

-        
Advantage
No vocabulary limits. Can spell any word, handle typos, creative linguistics. OOV problems simply don't exist.

-        
Trade-off
Harder to learn long-range semantic dependencies. Context windows fill up faster than word-level.

-      

-    

-  

-

-
-
-

-  

-    

-      
/ results

-      
Training Results

-      
Three runs across three hardware setups. All converged well — none overfit, none underfit catastrophically.

-    

-    

-      
-      
-      
-    

-    

-      

-        

-        

-          

-            loss_curve.svg
-            

-              
-                train
-              
-              
-                val
-              
-            

-          

-          

-            
-          

-        

-        

-          
✓

-          
No overfitting detected. Train/val gap is 0.0057 — the model generalizes to unseen text.

-        

-      

-      

-        

-          
Run 3 — Tesla T4 output

-          

-        

-        

-          
head_to_head.md

-          
-            
-            
-              
-              
-              
-            
-          
MetricT4 ⭐ColabCPU
Parameters1.99M10.82M0.82M
Best Val Loss0.92500.71761.3145
Training Time6.1 min61.3 min39.4 min

-        

-        

-          
Key Observation

-          
All three runs were still improving at the final checkpoint. Validation loss was still falling. More training steps or data would help all runs.

-        

-      

-    

-  

-

-
-
-

-  

-    

-      
/ scaling laws

-      
Where Quadtrix Sits

-      
The Chinchilla (2022) scaling law: ~20 tokens of training data per parameter is optimal. Here's how our runs align with the frontier.

-    

-    

-      

-        
// Chinchilla Coverage

-        

-          
-            
-            
-              
-              
-              
-              
-              
-            
-          
ModelParamsDataCoverage
Run 1 — CPU0.82M200K
1.2%
Run 3 — T4 ⭐1.99M28.3M
71.1%
Run 2 — Colab10.82M79.6M
36.8%
GPT-2 Small117M40B
1700%
GPT-3175B600B
17%

-        

-      

-      

-        
// Scaling Law Insights

-        

-          

-            
1

-            
The Chinchilla Scaling Law
DeepMind's 2022 paper established the optimal ratio: ~20 training tokens per parameter. Undertrained models waste compute; overtrained models waste data.

-          

-          

-            
2

-            
Run 3 is the Sweet Spot
At 71% of Chinchilla-optimal coverage, Run 3 has the best parameter-to-data ratio. It learns the most per unit of compute of all three runs.

-          

-          

-            
3

-            
GPT-3 Violates Chinchilla
GPT-3 (175B params) was trained on only 600B tokens — ~17% of what Chinchilla recommends. This is why smaller Chinchilla-trained models often match GPT-3.

-          

-          

-            
4

-            
What to Do Next
All runs benefit from more training steps first. Only after data coverage exceeds ~50% should you scale up model size.

-          

-        

-      

-    

-  

-

-
-
-

-  

-    

-      
/ generation

-      
How Generation Works

-      
Once training finishes, best_model.pt contains frozen weights. Generation is a simple loop — predict, sample, feed back.

-    

-    

-      

-        
generate.py

-        

-      

-      

-        
// Temperature Effect

-        

-        

-          

-            output — temperature=1.0
-            
-          

-          

-        

-        

-          
Deterministic generation

-          
Same weights + different random seed = different output. Add torch.manual_seed(42) for reproducible results.

-        

-      

-    

-    
// Known Limitations

-    

-      
⚠
Character-level trade-off
Learns characters, not words. Can't reliably spell or track meaning across paragraphs.

-      
📉
Output coherence
Sentences drift logically, names disappear, tense breaks — expected at this scale.

-      
⏳
All models undertrained
Val loss was still improving at the final checkpoint. More steps would help all three.

-      
📦
Limited data
Run 2 is only at 37% optimal data coverage. A larger story corpus would help significantly.

-      
🔭
No long-range memory
Fixed context window (128–256 tokens). Cannot reference events from earlier in the story.

-      
🔤
No subword tokenization
'fox' = 3 char tokens vs 1 word token. Context fills up faster, harder to learn semantics.

-    

-  

-

-
-
-

-  

-    

-      
/ quick start

-      
Get Running in Minutes

-      
One script. One dependency. No cloud account, no credentials, no pipeline.

-    

-    

-      

-        
// Installation Steps

-        

-          

-            
01 — Clone or download

-            
git clone https://github.com/Eamon2009/Transformer-language-model

-          

-          

-            
02 — Install PyTorch

-            
pip install torch

-          

-          

-            
03 — Add your data

-            
# Place any UTF-8 text file at data.txt# (or edit the filename in transformer.py)

-          

-          

-            
04 — Run training

-            
python transformer.py

-          

-          

-            
05 — Watch it learn

-            

-              [    0/5000]   train=4.6207   val=4.6202   << best!
-              [  200/5000]   train=2.2058   val=2.1986   << best!
-              [  400/5000]   train=1.6111   val=1.6039   << best!
-              ...
-              [DONE] Training finished in 367.0s | Best val loss: 0.9250
-            

-          

-        

-      

-      

-        
// Choose Your Config

-        

-          
-          
-          
-        

-        

-          

-            
GPU — Tesla T4
⭐ Recommended · 6.1 min · 1.99M params

-          

-          

-        

-        

-          
project structure

-          
transformer.py    ← Everything. One file.
-best_model.pt     ← Saved weights (after first run)
-data.txt          ← Your text (any UTF-8 file)

-          
No config files. Edit hyperparameters directly in the script.

-        

-        

-          
After training finishes...

-          
The script loads best_model.pt and generates text indefinitely. Press Ctrl+C to stop. Output differs each run because sampling is random.

-        

-      

-    

-  

-

-
-
-
-
-
-
-
-

From 6796fe506701ae4e78b4132d06151c1d1afb8ee4 Mon Sep 17 00:00:00 2001
From: Eamon 
Date: Mon, 20 Apr 2026 23:11:21 +0530
Subject: [PATCH 25/27] Create README.md for Quadtrix project

Add detailed README for Quadtrix project including installation, usage, architecture, and training results.
---
 README.md | 504 ++++++++++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 504 insertions(+)
 create mode 100644 README.md

diff --git a/README.md b/README.md
new file mode 100644
index 0000000..b96afe4
--- /dev/null
+++ b/README.md
@@ -0,0 +1,504 @@
+# Quadtrix [![License](https://img.shields.io/github/license/Eamon2009/Transformer-language-model)](LICENSE)
+image
+image
+
+
+
+A minimal, educational GPT-style transformer trained character-by-character on children's stories. No pre-trained weights. No fine-tuning. Just raw PyTorch, from init to generation.
+Character-level Transformer is a sequence-to-sequence architecture that operates on the granularity of individual characters rather than words or subword tokens. By utilizing a self-attention mechanism over long sequences of characters, the model learns to construct internal representations of morphology, syntax, and semantics from the ground up, effectively eliminating "out-of-vocabulary" (OOV) issues. While this approach allows for high fidelity in modeling rare words, spelling variations, and creative linguistics, it significantly increases the computational complexity—typically $O(L^2)$ where $L$ is the sequence length—as a single sentence requires many more steps than its word-level equivalent. Consequently, character-level Transformers often require deeper architectures or auxiliary losses to capture the long-range dependencies necessary to match the semantic performance of traditional token-based models
+
+> **The goal**: Understand how language models learn patterns. See it happen on your machine. Train in minutes on a GPU.
+
+---
+
+## Quick Start
+
+```bash
+# Install PyTorch
+pip install torch
+
+# Run training
+python transformer.py
+
+# The script will train, save best weights, then generate text forever
+# Press Ctrl+C to stop
+```
+
+**That's it.** No complex setup, no data pipelines, no credentials.
+
+---
+
+## What Is This?
+
+Quadtrix is a learning project. It trains a tiny transformer on text—character by character—and learns which characters tend to follow others. At generation time, it predicts the next character, feeds it back in, and repeats.
+
+**It's the same architecture as GPT**, just much smaller (1M–11M parameters instead of 175B) and trained on much less data (thousands of stories instead of the internet).
+
+The magic: in just 6 minutes on a Tesla T4 GPU, a 2M-parameter model learns meaningful patterns about narrative structure, dialogue, and storytelling. The output isn't Shakespeare, but it *is* recognizable as a story.
+
+---
+
+## How It Works
+
+### The Pipeline
+
+```
+1. Load children's stories from disk
+   ↓
+2. Encode each character as an integer (vocab size: 28–110)
+   ↓
+3. Split into train/val chunks (80/20)
+   ↓
+4. Build a GPT-style transformer with:
+   - Embedding layer (character → vector)
+   - Transformer blocks (self-attention + feedforward)
+   - Output projection (vector → logits over vocab)
+   ↓
+5. Train for N steps, measuring loss every eval_interval
+   ↓
+6. Save best weights whenever validation loss improves
+   ↓
+7. Load best model and generate text forever
+   ↓
+8. Press Ctrl+C to stop
+```
+
+### What the Model Actually Does
+
+Each forward pass:
+1. Takes a sequence of character indices (e.g., "Once upon...")
+2. Embeds each into a dense vector
+3. Passes through transformer layers (multi-head attention learns which characters to attend to)
+4. Outputs logits (scores) for every possible next character
+5. Samples the next character according to those probabilities
+6. Feeds it back in and repeats
+
+**Loss function**: Cross-entropy. The model minimizes surprise on unseen text.
+
+**Training**: Adam optimizer with a learning rate schedule. Dropout prevents overfitting.
+
+---
+
+## Project Structure
+
+```
+transformer.py          ← Everything. One file.
+best_model.pt           ← Saved weights (after first training run)
+data.txt                ← Your text file (any UTF-8 file works)
+```
+
+That's all. No fancy folder structure. No config files. Edit hyperparameters directly in the script.
+
+---
+
+## Configuration & Hardware
+
+Quadtrix is designed to work anywhere: laptop CPU to cloud GPU. Edit these hyperparameters in `transformer.py`:
+
+```python
+# ============================================================================
+# Hyperparameters
+# ============================================================================
+batch_size    = 64         # Sequences per batch
+block_size    = 128        # Context window (tokens)
+max_iters     = 5000       # Total training steps
+eval_interval = 200        # Print loss every N steps
+learning_rate = 3e-4
+n_embd        = 200        # Embedding dimension
+n_head        = 4          # Attention heads per layer
+n_layer       = 4          # Number of transformer blocks
+dropout       = 0.2        # Regularization
+```
+
+### Three Pre-Tuned Configurations
+
+**CPU (Laptop) — Fast Feedback**
+```python
+batch_size, block_size, max_iters = 16, 128, 3000
+n_embd, n_head, n_layer = 128, 4, 4
+# ~0.82M parameters
+# Trains in ~40 min on AMD Ryzen
+```
+
+**GPU (Google Colab) — Best Quality**
+```python
+batch_size, block_size, max_iters = 64, 256, 5000
+n_embd, n_head, n_layer = 384, 6, 6
+# ~10.82M parameters
+# Trains in ~60 min on Colab GPU
+# Best output quality
+```
+
+**GPU (Tesla T4) — Efficient**
+```python
+batch_size, block_size, max_iters = 64, 128, 5000
+n_embd, n_head, n_layer = 200, 4, 4
+# ~1.99M parameters
+# Trains in **6.1 minutes** ← Fastest
+# Best parameter/data balance
+```
+
+---
+
+## Training Results
+
+Three runs. Three different hardware setups. All converged well.
+
+### Run 3 — Tesla T4 (Latest) ⭐
+
+**Configuration**: 4 layers × 4 heads × 200 dim = **1.99M params**
+
+| Metric | Value |
+|--------|-------|
+| Device | Tesla T4 (CUDA 13.0) |
+| Dataset | ~31.4M characters (children's stories) |
+| Train tokens | 28.3M |
+| Val tokens | 3.1M |
+| Training time | **6.1 minutes** |
+| Best val loss | **0.9250** |
+| Final train loss | 0.9307 |
+| Overfitting | None detected |
+
+**Training curve** (every 200 steps):
+```
+[    0/5000]   train=4.6207   val=4.6202   elapsed=2s    << best!
+[  200/5000]   train=2.2058   val=2.1986   elapsed=17s   << best!
+[  400/5000]   train=1.6111   val=1.6039   elapsed=32s   << best!
+[ 1000/5000]   train=1.2495   val=1.2567   elapsed=76s   << best!
+[ 2000/5000]   train=1.0731   val=1.0765   elapsed=149s  << best!
+[ 3000/5000]   train=1.0000   val=0.9956   elapsed=221s  << best!
+[ 4000/5000]   train=0.9601   val=0.9642   elapsed=294s
+[ 4200/5000]   train=0.9515   val=0.9489   elapsed=309s  << best!
+[ 4400/5000]   train=0.9433   val=0.9431   elapsed=323s  << best!
+[ 4800/5000]   train=0.9331   val=0.9250   elapsed=353s  << best!
+[ 4999/5000]   train=0.9307   val=0.9430   elapsed=367s
+
+[DONE] Training finished in 367.0s (6.1 min) | Best val loss: 0.9250
+```
+
+**Key insight**: This run hit the **sweet spot**—large enough to learn coherent patterns, small enough to train fast. It's the reference configuration.
+
+---
+
+### Run 2 — Google Colab (10.82M Parameters)
+
+**Configuration**: 6 layers × 6 heads × 384 dim
+
+| Metric | Value |
+|--------|-------|
+| Device | CUDA (Google Colab GPU) |
+| Dataset | ~88.4M characters |
+| Parameters | 10.82M |
+| Training time | 61.3 minutes |
+| Best val loss | **0.7176** |
+| Overfitting | None |
+
+**Result**: Larger model, more data = **best output quality**. Slower to train but produces recognizable narratives.
+
+---
+
+### Run 1 — CPU Laptop (0.82M Parameters)
+
+**Configuration**: 4 layers × 4 heads × 128 dim
+
+| Metric | Value |
+|--------|-------|
+| Device | AMD Ryzen 5 PRO 3500U (CPU only) |
+| Parameters | 0.82M |
+| Dataset | ~201K characters |
+| Training time | 39.4 minutes |
+| Best val loss | 1.3145 |
+| Overfitting | None |
+
+**Result**: Smallest model, tiniest dataset. Trains fastest on CPU. Output is fragmented but shows the model learned *something*.
+
+---
+
+## Head-to-Head Comparison
+
+| Metric | Run 1 — CPU | Run 2 — Colab | Run 3 — T4 ⭐ |
+|--------|-------------|--------------|------------|
+| **Parameters** | 0.82M | 10.82M | 1.99M |
+| **Training data** | 200K chars | 88.4M chars | 31.4M chars |
+| **Training time** | 39.4 min | 61.3 min | **6.1 min** |
+| **Best val loss** | 1.3145 | **0.7176** | 0.9250 |
+| **Output coherence** | Fragmented | Coherent paragraphs | Basic sentences |
+| **Overfitting** | ✓ None | ✓ None | ✓ None |
+| **Still improving?** | Yes | Yes | Yes |
+
+> **Observation**: All three were still improving at the final checkpoint. More training steps = better loss.
+
+---
+
+## Example Outputs
+
+### Run 2 Output (10.82M params) — Best Quality
+
+```
+Upon a time, there were two friends, Jack and Tom. They had a cold doll in
+the sunshine.
+
+One day, Jack saw that he was universed. He used the sky at past it to march
+around the garden. He had a small ball on his face. He felt dizzy and wanted
+to share his happy with them.
+
+Nack knew it was feeling important to his passion in their rooms. He knew
+that night, he had never seen a small boy just soon could drink.
+```
+
+**Analysis**: Clear sentence structure. Named characters. Logical progression. Some linguistic oddities ("felt dizzy and wanted to share his happy") but unmistakably a story.
+
+---
+
+### Run 3 Output (1.99M params, Tesla T4) — Efficient
+
+```
+Timmy and elsed him to tell being jumping things. They were tired and making
+some pinkets and help paper me. They had to see them, drain and ran ar her
+mommy. They also fast with the stretch and sacks the changer.
+
+Lily's truck laughed and saw a rock. She said, "You can't here some wet
+sicks. You have something new favorite toys, I do yours."
+```
+
+**Analysis**: Narrative present. Dialogue structure intact. Characters named. Some word-order errors and made-up words, but the *shape* of a story is clear.
+
+---
+
+### Run 1 Output (0.82M params) — Minimal
+
+```
+when years me told be found a big ea reak abig driendly they named not she
+rabbit smiled by aded he what in again one smiled the mushrought boy one day
+and was arroom him that she rabbing animals the dreezed at neard had to there
+man owl them with o box
+```
+
+**Analysis**: Word boundaries mostly intact. Character names present (rabbit, owl). But syntax falls apart—the model is struggling to hold sentence structure. This is a 0.82M model trained on tiny data; it's learning *something* but hasn't converged to coherent text.
+
+---
+## Project Structure
+ 
+```
+.
+├── .github/
+│   ├── ISSUE_TEMPLATE/      # GitHub issue templates
+│   └── workflows/           # CI/CD workflows
+├── .vscode/                 # VS Code configuration
+├── GPU train/               # GPU training scripts and configurations
+├── assets/                  # Images, diagrams, and other assets
+├── checkpoints/             # Saved model checkpoints
+├── config/                  # Configuration files
+├── data_set/                # Training and validation datasets
+├── evaluate/                # Evaluation scripts and metrics
+├── generate/                # Text generation scripts
+├── logs/                    # Training logs and tensorboard files
+├── train_test/              # Training and testing utilities
+├── .gitattributes          # Git attributes configuration
+├── .gitignore              # Git ignore rules
+├── .python-version         # Python version specification
+├── LICENSE                 # MIT License
+├── README.md               # This file
+├── cleaned.txt             # Cleaned training data
+├── gpt-from-scratch.ipynb  # Jupyter notebook implementation
+├── requirements.txt        # Python dependencies
+├── traindata.txt           # Raw training data
+└── transformer.py          # Main transformer model implementation
+```
+
+## Loss Curves & Training Dynamics
+
+All three runs showed the classic learning curve:
+
+```
+Phase 1: Rapid drop (0–20% of training)
+  - Model learns basic character transitions
+  - Loss halves or better
+  
+Phase 2: Steady descent (20–80%)
+  - Model learns longer-range patterns
+  - Character names, sentence boundaries
+  - Loss continues down but more gradually
+  
+Phase 3: Diminishing returns (80–100%)
+  - Model learning slows
+  - Val loss still improving but incremental
+  - More data or capacity would help here
+```
+
+**Train/Val Gap Analysis** (indicator of overfitting):
+
+| Run | Final Train | Final Val | Gap |
+|-----|-------------|-----------|-----|
+| CPU | 1.3191 | 1.3145 | 0.0046 |
+| Colab | 0.7259 | 0.7176 | 0.0083 |
+| T4 | 0.9307 | 0.9250 | 0.0057 |
+
+All gaps are tiny. **No overfitting detected.** The model is generalizing well to unseen text.
+
+---
+
+## Scaling Laws: Where Quadtrix Sits
+Screenshot 2026-03-17 171921
+
+
+The Chinchilla (2022) scaling law suggests: **~20 tokens of training data per parameter is optimal**.
+
+Let's see how our runs align:
+
+| Model | Parameters | Training Data | Optimal (20×) | Coverage |
+|-------|------------|---------------|--------------|----------|
+| Run 1 | 0.82M | 200K tokens | 16.4M | **1.2%** |
+| Run 3 | 1.99M | 28.3M tokens | 39.8M | **71.1%** ← Best balanced |
+| Run 2 | 10.82M | 79.6M tokens | 216M | **36.8%** |
+| GPT-2 Small | 117M | 40B tokens | 2.3B | ~1700% |
+| GPT-3 | 175B | 600B tokens | 3.5T | ~17% |
+
+**Insight**: Run 3 is the most *balanced*—the model size and data quantity are well-matched. Run 2 has the largest model but is only at 37% optimal data coverage, meaning it would benefit more from adding data than adding parameters.
+
+**Next steps for any run**:
+1. **More training steps** — All three were still falling at the final checkpoint
+2. **More data** — Run 3 is closest to optimal; Run 2 would benefit most
+3. **Larger model** — Only worth doing once data coverage exceeds 50%
+
+---
+
+## How Generation Works
+
+Once training is done, `best_model.pt` contains frozen weights. Generation is simple:
+
+```python
+# Pseudocode for generation
+seed = torch.tensor([[start_token]])  # e.g., start_token = 0
+
+for _ in range(num_chars_to_generate):
+    # Forward pass through all layers
+    logits = model(seed)[-1, :]  # Get last token's logits
+    
+    # Convert logits to probabilities
+    probs = softmax(logits / temperature)
+    
+    # Sample next token
+    next_token = sample(probs)
+    
+    # Append and continue
+    seed = torch.cat([seed, next_token], dim=-1)
+    
+    # Trim to context window if needed
+    seed = seed[:, -block_size:]
+```
+
+**Why output differs each run**: The sampling step is random. Same weights, different random seeds = different output. Add `torch.manual_seed(42)` for deterministic output.
+
+---
+
+## Known Limitations
+
+1. **Character-level learning** — the model learns characters, not words. It cannot reliably spell or track meaning across paragraphs.
+
+2. **Output coherence** — especially on smaller runs. Sentences drift logically. Names disappear. Tense breaks. This is expected at this scale.
+
+3. **All models undertrained** — validation loss was still improving at iteration N. More training steps would help all three.
+
+4. **Limited data** — Run 2 is only at 37% optimal data coverage. A larger story corpus would meaningfully improve output quality.
+
+5. **No long-range memory** — transformer context window is fixed (128 or 256 tokens). The model cannot reference events from 10 paragraphs ago.
+
+---
+
+## Technical Details
+
+### Model Architecture
+
+```python
+class GPTModel(nn.Module):
+    def __init__(self, vocab_size, n_embd, n_head, n_layer, block_size, dropout):
+        self.token_embedding = nn.Embedding(vocab_size, n_embd)
+        self.position_embedding = nn.Embedding(block_size, n_embd)
+        self.transformer = nn.Sequential(
+            *[TransformerBlock(n_embd, n_head, dropout) for _ in range(n_layer)]
+        )
+        self.ln_final = nn.LayerNorm(n_embd)
+        self.lm_head = nn.Linear(n_embd, vocab_size)
+        self.dropout = nn.Dropout(dropout)
+
+    def forward(self, x):
+        B, T = x.shape
+        tok_emb = self.token_embedding(x)  # (B, T, n_embd)
+        pos_emb = self.position_embedding(torch.arange(T))
+        x = self.dropout(tok_emb + pos_emb)
+        x = self.transformer(x)
+        x = self.ln_final(x)
+        logits = self.lm_head(x)
+        return logits
+```
+
+### Transformer Block
+
+Each block contains:
+- **Multi-head self-attention**: `(B, T, n_embd) → (B, T, n_embd)`
+- **Feedforward network**: Two linear layers with ReLU
+- **Layer normalization & residual connections**
+- **Dropout for regularization**
+
+### Training Loop
+
+```python
+for step in range(max_iters):
+    # Batch a random chunk of training data
+    batch_x, batch_y = get_batch('train')
+    
+    # Forward pass
+    logits = model(batch_x)
+    loss = F.cross_entropy(logits.view(-1, vocab_size), batch_y.view(-1))
+    
+    # Backward pass
+    optimizer.zero_grad()
+    loss.backward()
+    torch.nn.utils.clip_grad_norm_(model.parameters(), 1.0)
+    optimizer.step()
+    
+    # Evaluate and save
+    if step % eval_interval == 0:
+        val_loss = estimate_loss('val')
+        print(f"train={train_loss:.4f} val={val_loss:.4f}")
+        
+        if val_loss < best_val_loss:
+            best_val_loss = val_loss
+            torch.save(model.state_dict(), 'best_model.pt')
+```
+
+---
+
+## Why This Project Matters
+
+1. **Educational**: See exactly how a language model learns. No black boxes.
+
+2. **Verifiable**: You can trace loss, inspect weights, understand every line of code.
+
+3. **Fast**: Even on CPU, training finishes in under an hour. On GPU, minutes.
+
+4. **Runnable**: One script, one dependency (PyTorch). No cloud account required.
+
+Quadtrix is to GPT what nanoGPT is to transformers: a stripped-down, readable, educational version that teaches the core ideas.
+
+---
+
+## Getting Started
+
+1. **Clone or download** `transformer.py`
+2. **Install PyTorch**: `pip install torch`
+3. **Add your data**: Place a UTF-8 text file at `data.txt` (or edit the filename in the script)
+4. **Run**: `python transformer.py`
+5. **Watch**: Loss decreases. Weights save. Text generates.
+6. **Tweak**: Edit hyperparameters and re-run to see how each affects training speed and output quality.
+
+---
+
+## License
+
+MIT
+
+---
+
+*Built with PyTorch. | [GitHub](https://github.com/Eamon2009/Transformer-language-model)*

From 766ffba93153dcd55cae3c1915e4c8a3164f2631 Mon Sep 17 00:00:00 2001
From: Eamon 
Date: Wed, 22 Apr 2026 12:56:01 +0530
Subject: [PATCH 26/27] Delete train_test/infer.cu

---
 train_test/infer.cu | 568 --------------------------------------------
 1 file changed, 568 deletions(-)
 delete mode 100644 train_test/infer.cu

diff --git a/train_test/infer.cu b/train_test/infer.cu
deleted file mode 100644
index 66dd660..0000000
--- a/train_test/infer.cu
+++ /dev/null
@@ -1,568 +0,0 @@
-// This need to be tested on a real gpu
-#include 
-#include 
-#include 
-#include 
-#include 
-#include 
-#include 
-#include 
-
-// Config
-typedef struct
-{
-      int vocab_size;
-      int block_size;
-      int n_embd;
-      int n_head;
-      int n_layer;
-} Config;
-
-// ── Error checking
-#define CUDA_CHECK(call)                                              \
-      do                                                              \
-      {                                                               \
-            cudaError_t err = call;                                   \
-            if (err != cudaSuccess)                                   \
-            {                                                         \
-                  fprintf(stderr, "[CUDA Error] %s at line %d: %s\n", \
-                          #call, __LINE__, cudaGetErrorString(err));  \
-                  exit(1);                                            \
-            }                                                         \
-      } while (0)
-
-#define CUBLAS_CHECK(call)                                              \
-      do                                                                \
-      {                                                                 \
-            cublasStatus_t err = call;                                  \
-            if (err != CUBLAS_STATUS_SUCCESS)                           \
-            {                                                           \
-                  fprintf(stderr, "[cuBLAS Error] %s at line %d: %d\n", \
-                          #call, __LINE__, err);                        \
-                  exit(1);                                              \
-            }                                                           \
-      } while (0)
-
-// Signal handler
-static volatile int running = 1;
-void handle_sigint(int s)
-{
-      (void)s;
-      printf("\n\n[Stopped by user]\n");
-      running = 0;
-}
-
-// Build x[T x C] from token + position embeddings
-__global__ void embed_kernel(float *x, float *tok_emb, float *pos_emb,
-                             int *tokens, int T, int C)
-{
-      int t = blockIdx.x;
-      int c = threadIdx.x;
-      if (t < T && c < C)
-            x[t * C + c] = tok_emb[tokens[t] * C + c] + pos_emb[t * C + c];
-}
-
-// LayerNorm over one row of length C
-__global__ void layernorm_kernel(float *out, float *x, float *w, float *b,
-                                 int T, int C)
-{
-      int t = blockIdx.x;
-      if (t >= T)
-            return;
-
-      float *xr = x + t * C;
-      float *outr = out + t * C;
-
-      float mean = 0.0f;
-      for (int i = 0; i < C; i++)
-            mean += xr[i];
-      mean /= C;
-
-      float var = 0.0f;
-      for (int i = 0; i < C; i++)
-            var += (xr[i] - mean) * (xr[i] - mean);
-      var = var / C + 1e-5f;
-      float inv = rsqrtf(var);
-
-      for (int i = 0; i < C; i++)
-            outr[i] = (xr[i] - mean) * inv * w[i] + b[i];
-}
-
-// Add bias in-place: x[T x N] += b[N]
-__global__ void add_bias_kernel(float *x, float *b, int T, int N)
-{
-      int t = blockIdx.x;
-      int n = threadIdx.x;
-      if (t < T && n < N)
-            x[t * N + n] += b[n];
-}
-
-// ReLU in-place
-__global__ void relu_kernel(float *x, int n)
-{
-      int i = blockIdx.x * blockDim.x + threadIdx.x;
-      if (i < n && x[i] < 0.0f)
-            x[i] = 0.0f;
-}
-
-// Residual add: a += b
-__global__ void residual_kernel(float *a, float *b, int n)
-{
-      int i = blockIdx.x * blockDim.x + threadIdx.x;
-      if (i < n)
-            a[i] += b[i];
-}
-
-// Causal attention scores + softmax for one head
-__global__ void attention_kernel(float *att, float *Q, float *K,
-                                 int T, int hs)
-{
-      int i = blockIdx.x;  // query position
-      int j = threadIdx.x; // key position
-      if (i >= T || j >= T)
-            return;
-
-      if (j > i)
-      {
-            att[i * T + j] = -1e9f;
-            return;
-      }
-      float scale = rsqrtf((float)hs);
-      float dot = 0.0f;
-      for (int k = 0; k < hs; k++)
-            dot += Q[i * hs + k] * K[j * hs + k];
-      att[i * T + j] = dot * scale;
-
-      // Softmax (done per row, only thread 0 of each row)
-      __syncthreads();
-      if (j == 0)
-      {
-            float *row = att + i * T;
-            float mx = -1e9f;
-            for (int x = 0; x <= i; x++)
-                  if (row[x] > mx)
-                        mx = row[x];
-            float sum = 0.0f;
-            for (int x = 0; x < T; x++)
-            {
-                  row[x] = (x <= i) ? expf(row[x] - mx) : 0.0f;
-                  sum += row[x];
-            }
-            for (int x = 0; x < T; x++)
-                  row[x] /= sum;
-      }
-}
-
-// Weighted sum: hv[T x hs] = att[T x T] @ V[T x hs]
-__global__ void attn_value_kernel(float *hv, float *att, float *V,
-                                  int T, int hs)
-{
-      int i = blockIdx.x;
-      int k = threadIdx.x;
-      if (i >= T || k >= hs)
-            return;
-      float s = 0.0f;
-      for (int j = 0; j <= i; j++)
-            s += att[i * T + j] * V[j * hs + k];
-      hv[i * hs + k] = s;
-}
-
-// Scatter head output into full [T x C] buffer at offset h*hs
-__global__ void scatter_head_kernel(float *head_out, float *hv,
-                                    int T, int C, int hs, int h_offset)
-{
-      int t = blockIdx.x;
-      int k = threadIdx.x;
-      if (t < T && k < hs)
-            head_out[t * C + h_offset + k] = hv[t * hs + k];
-}
-
-// Softmax over logits (single row, run on CPU side for simplicity)
-static void softmax_cpu(float *x, int n)
-{
-      float mx = x[0];
-      for (int i = 1; i < n; i++)
-            if (x[i] > mx)
-                  mx = x[i];
-      float sum = 0.0f;
-      for (int i = 0; i < n; i++)
-      {
-            x[i] = expf(x[i] - mx);
-            sum += x[i];
-      }
-      for (int i = 0; i < n; i++)
-            x[i] /= sum;
-}
-
-static int sample(float *probs, int n)
-{
-      float r = (float)rand() / ((float)RAND_MAX + 1.0f);
-      float cdf = 0.0f;
-      for (int i = 0; i < n; i++)
-      {
-            cdf += probs[i];
-            if (r < cdf)
-                  return i;
-      }
-      return n - 1;
-}
-
-// Weight struct (GPU pointers,need to be tested on a GPU )
-typedef struct
-{
-      float *tok_emb; // [vocab x C]
-      float *pos_emb; // [block x C]
-      float **head_k; // [n_layer x n_head][hs x C]
-      float **head_q;
-      float **head_v;
-      float **sa_proj_w; // [n_layer][C x C]
-      float **sa_proj_b; // [n_layer][C]
-      float **ff_w1;     // [n_layer][4C x C]
-      float **ff_b1;     // [n_layer][4C]
-      float **ff_w2;     // [n_layer][C x 4C]
-      float **ff_b2;     // [n_layer][C]
-      float **ln1_w;     // [n_layer][C]
-      float **ln1_b;
-      float **ln2_w;
-      float **ln2_b;
-      float *ln_f_w; // [C]
-      float *ln_f_b;
-      float *lm_w; // [vocab x C]
-      float *lm_b; // [vocab]
-} Weights;
-
-// Read tensor from file and upload to GPU
-static float *read_upload(FILE *f)
-{
-      int ndim;
-      fread(&ndim, sizeof(int), 1, f);
-      int total = 1;
-      for (int i = 0; i < ndim; i++)
-      {
-            int d;
-            fread(&d, sizeof(int), 1, f);
-            total *= d;
-      }
-      float *cpu = (float *)malloc(total * sizeof(float));
-      fread(cpu, sizeof(float), total, f);
-      float *gpu;
-      CUDA_CHECK(cudaMalloc(&gpu, total * sizeof(float)));
-      CUDA_CHECK(cudaMemcpy(gpu, cpu, total * sizeof(float), cudaMemcpyHostToDevice));
-      free(cpu);
-      return gpu;
-}
-
-//  Forward pass
-static void forward(float *d_logits, int *d_tokens, int T,
-                    Config *cfg, Weights *W, cublasHandle_t cublas)
-{
-      int C = cfg->n_embd;
-      int nh = cfg->n_head;
-      int hs = C / nh;
-      int ff = 4 * C;
-
-      float one = 1.0f, zero = 0.0f;
-
-      // Allocate working buffers
-      float *d_x, *d_ln_out, *d_head_out, *d_attn_out;
-      float *d_K, *d_Q, *d_V, *d_att, *d_hv;
-      float *d_ff1, *d_ff2;
-
-      CUDA_CHECK(cudaMalloc(&d_x, T * C * sizeof(float)));
-      CUDA_CHECK(cudaMalloc(&d_ln_out, T * C * sizeof(float)));
-      CUDA_CHECK(cudaMalloc(&d_head_out, T * C * sizeof(float)));
-      CUDA_CHECK(cudaMalloc(&d_attn_out, T * C * sizeof(float)));
-      CUDA_CHECK(cudaMalloc(&d_K, T * hs * sizeof(float)));
-      CUDA_CHECK(cudaMalloc(&d_Q, T * hs * sizeof(float)));
-      CUDA_CHECK(cudaMalloc(&d_V, T * hs * sizeof(float)));
-      CUDA_CHECK(cudaMalloc(&d_att, T * T * sizeof(float)));
-      CUDA_CHECK(cudaMalloc(&d_hv, T * hs * sizeof(float)));
-      CUDA_CHECK(cudaMalloc(&d_ff1, T * ff * sizeof(float)));
-      CUDA_CHECK(cudaMalloc(&d_ff2, T * C * sizeof(float)));
-
-      // Embed
-      embed_kernel<<>>(d_x, W->tok_emb, W->pos_emb, d_tokens, T, C);
-
-      for (int l = 0; l < cfg->n_layer; l++)
-      {
-            int base = l * nh;
-
-            // LayerNorm 1
-            layernorm_kernel<<>>(d_ln_out, d_x,
-                                       W->ln1_w[l], W->ln1_b[l], T, C);
-
-            // Multi-head attention
-            CUDA_CHECK(cudaMemset(d_head_out, 0, T * C * sizeof(float)));
-
-            for (int h = 0; h < nh; h++)
-            {
-                  CUBLAS_CHECK(cublasSgemm(cublas, CUBLAS_OP_T, CUBLAS_OP_N,
-                                           hs, T, C, &one,
-                                           W->head_k[base + h], C,
-                                           d_ln_out, C,
-                                           &zero, d_K, hs));
-
-                  CUBLAS_CHECK(cublasSgemm(cublas, CUBLAS_OP_T, CUBLAS_OP_N,
-                                           hs, T, C, &one,
-                                           W->head_q[base + h], C,
-                                           d_ln_out, C,
-                                           &zero, d_Q, hs));
-
-                  CUBLAS_CHECK(cublasSgemm(cublas, CUBLAS_OP_T, CUBLAS_OP_N,
-                                           hs, T, C, &one,
-                                           W->head_v[base + h], C,
-                                           d_ln_out, C,
-                                           &zero, d_V, hs));
-
-                  // Attention scores + softmax
-                  attention_kernel<<>>(d_att, d_Q, d_K, T, hs);
-
-                  // Weighted V
-                  attn_value_kernel<<>>(d_hv, d_att, d_V, T, hs);
-
-                  // Scatter into head_out
-                  scatter_head_kernel<<>>(d_head_out, d_hv, T, C, hs, h * hs);
-            }
-
-            // SA projection
-            CUBLAS_CHECK(cublasSgemm(cublas, CUBLAS_OP_T, CUBLAS_OP_N,
-                                     C, T, C, &one,
-                                     W->sa_proj_w[l], C,
-                                     d_head_out, C,
-                                     &zero, d_attn_out, C));
-            add_bias_kernel<<>>(d_attn_out, W->sa_proj_b[l], T, C);
-
-            // Residual
-            residual_kernel<<<(T * C + 255) / 256, 256>>>(d_x, d_attn_out, T * C);
-
-            // LayerNorm 2
-            layernorm_kernel<<>>(d_ln_out, d_x,
-                                       W->ln2_w[l], W->ln2_b[l], T, C);
-
-            // FF layer 1
-            CUBLAS_CHECK(cublasSgemm(cublas, CUBLAS_OP_T, CUBLAS_OP_N,
-                                     ff, T, C, &one,
-                                     W->ff_w1[l], C,
-                                     d_ln_out, C,
-                                     &zero, d_ff1, ff));
-            add_bias_kernel<<>>(d_ff1, W->ff_b1[l], T, ff);
-            relu_kernel<<<(T * ff + 255) / 256, 256>>>(d_ff1, T * ff);
-
-            // FF layer 2
-            CUBLAS_CHECK(cublasSgemm(cublas, CUBLAS_OP_T, CUBLAS_OP_N,
-                                     C, T, ff, &one,
-                                     W->ff_w2[l], ff,
-                                     d_ff1, ff,
-                                     &zero, d_ff2, C));
-            add_bias_kernel<<>>(d_ff2, W->ff_b2[l], T, C);
-
-            // Residual
-            residual_kernel<<<(T * C + 255) / 256, 256>>>(d_x, d_ff2, T * C);
-      }
-
-      // Final layernorm on last token only
-      float *d_last;
-      CUDA_CHECK(cudaMalloc(&d_last, C * sizeof(float)));
-      float *d_xf;
-      CUDA_CHECK(cudaMalloc(&d_xf, C * sizeof(float)));
-
-      CUDA_CHECK(cudaMemcpy(d_last, d_x + (T - 1) * C,
-                            C * sizeof(float), cudaMemcpyDeviceToDevice));
-      layernorm_kernel<<<1, 1>>>(d_xf, d_last, W->ln_f_w, W->ln_f_b, 1, C);
-
-      // LM head: logits[vocab] = lm_w[vocab x C] @ xf[C]
-      CUBLAS_CHECK(cublasSgemv(cublas, CUBLAS_OP_T,
-                               C, cfg->vocab_size, &one,
-                               W->lm_w, C,
-                               d_xf, 1,
-                               &zero, d_logits, 1));
-
-      // Add lm bias
-      float *d_lm_b_scaled;
-      CUDA_CHECK(cudaMalloc(&d_lm_b_scaled, cfg->vocab_size * sizeof(float)));
-      CUDA_CHECK(cudaMemcpy(d_lm_b_scaled, W->lm_b,
-                            cfg->vocab_size * sizeof(float),
-                            cudaMemcpyDeviceToDevice));
-      residual_kernel<<<(cfg->vocab_size + 255) / 256, 256>>>(
-          d_logits, d_lm_b_scaled, cfg->vocab_size);
-
-      // Free buffers
-      cudaFree(d_x);
-      cudaFree(d_ln_out);
-      cudaFree(d_head_out);
-      cudaFree(d_attn_out);
-      cudaFree(d_K);
-      cudaFree(d_Q);
-      cudaFree(d_V);
-      cudaFree(d_att);
-      cudaFree(d_hv);
-      cudaFree(d_ff1);
-      cudaFree(d_ff2);
-      cudaFree(d_last);
-      cudaFree(d_xf);
-      cudaFree(d_lm_b_scaled);
-}
-
-// Main
-int main(void)
-{
-      signal(SIGINT, handle_sigint);
-      srand((unsigned)time(NULL));
-
-      // Check GPU
-      int dev_count = 0;
-      CUDA_CHECK(cudaGetDeviceCount(&dev_count));
-      if (dev_count == 0)
-      {
-            fprintf(stderr, "[Error] No CUDA GPU found.\n");
-            return 1;
-      }
-      cudaDeviceProp prop;
-      CUDA_CHECK(cudaGetDeviceProperties(&prop, 0));
-      printf("[INFO] GPU: %s\n", prop.name);
-
-      // Load vocab
-      FILE *fv = fopen("../vocab.bin", "rb");
-      if (!fv)
-      {
-            fprintf(stderr, "[Error] Cannot open vocab.bin\n");
-            return 1;
-      }
-      int vocab_size;
-      fread(&vocab_size, sizeof(int), 1, fv);
-      char *vocab = (char *)malloc(vocab_size);
-      for (int i = 0; i < vocab_size; i++)
-      {
-            unsigned char c;
-            fread(&c, 1, 1, fv);
-            vocab[i] = (char)c;
-      }
-      fclose(fv);
-
-      // Load weights
-      FILE *fw = fopen("../weights.bin", "rb");
-      if (!fw)
-      {
-            fprintf(stderr, "[Error] Cannot open weights.bin\n");
-            return 1;
-      }
-
-      Config cfg;
-      fread(&cfg.vocab_size, sizeof(int), 1, fw);
-      fread(&cfg.block_size, sizeof(int), 1, fw);
-      fread(&cfg.n_embd, sizeof(int), 1, fw);
-      fread(&cfg.n_head, sizeof(int), 1, fw);
-      fread(&cfg.n_layer, sizeof(int), 1, fw);
-
-      int nl = cfg.n_layer, nh = cfg.n_head;
-
-      Weights W = {0};
-      W.tok_emb = read_upload(fw);
-      W.pos_emb = read_upload(fw);
-
-      W.head_k = (float **)malloc(nl * nh * sizeof(float *));
-      W.head_q = (float **)malloc(nl * nh * sizeof(float *));
-      W.head_v = (float **)malloc(nl * nh * sizeof(float *));
-      W.sa_proj_w = (float **)malloc(nl * sizeof(float *));
-      W.sa_proj_b = (float **)malloc(nl * sizeof(float *));
-      W.ff_w1 = (float **)malloc(nl * sizeof(float *));
-      W.ff_b1 = (float **)malloc(nl * sizeof(float *));
-      W.ff_w2 = (float **)malloc(nl * sizeof(float *));
-      W.ff_b2 = (float **)malloc(nl * sizeof(float *));
-      W.ln1_w = (float **)malloc(nl * sizeof(float *));
-      W.ln1_b = (float **)malloc(nl * sizeof(float *));
-      W.ln2_w = (float **)malloc(nl * sizeof(float *));
-      W.ln2_b = (float **)malloc(nl * sizeof(float *));
-
-      for (int l = 0; l < nl; l++)
-      {
-            for (int h = 0; h < nh; h++)
-            {
-                  W.head_k[l * nh + h] = read_upload(fw);
-                  W.head_q[l * nh + h] = read_upload(fw);
-                  W.head_v[l * nh + h] = read_upload(fw);
-            }
-            W.sa_proj_w[l] = read_upload(fw);
-            W.sa_proj_b[l] = read_upload(fw);
-            W.ff_w1[l] = read_upload(fw);
-            W.ff_b1[l] = read_upload(fw);
-            W.ff_w2[l] = read_upload(fw);
-            W.ff_b2[l] = read_upload(fw);
-            W.ln1_w[l] = read_upload(fw);
-            W.ln1_b[l] = read_upload(fw);
-            W.ln2_w[l] = read_upload(fw);
-            W.ln2_b[l] = read_upload(fw);
-      }
-
-      W.ln_f_w = read_upload(fw);
-      W.ln_f_b = read_upload(fw);
-      W.lm_w = read_upload(fw);
-      W.lm_b = read_upload(fw);
-      fclose(fw);
-
-      printf("--- Model loaded ---\n");
-      printf("[INFO] vocab=%d  block=%d  embd=%d  heads=%d  layers=%d\n",
-             cfg.vocab_size, cfg.block_size, cfg.n_embd, cfg.n_head, cfg.n_layer);
-      printf("Generating text (Ctrl+C to stop)...\n\n");
-      printf("--------------------------------------------------\n");
-
-      // cuBLAS handle
-      cublasHandle_t cublas;
-      CUBLAS_CHECK(cublasCreate(&cublas));
-
-      // GPU token buffer
-      int *d_tokens;
-      CUDA_CHECK(cudaMalloc(&d_tokens, cfg.block_size * sizeof(int)));
-
-      // GPU logits buffer
-      float *d_logits;
-      CUDA_CHECK(cudaMalloc(&d_logits, cfg.vocab_size * sizeof(float)));
-
-      // CPU logits for sampling
-      float *logits = (float *)malloc(cfg.vocab_size * sizeof(float));
-
-      // Context window
-      int *ctx = (int *)calloc(cfg.block_size, sizeof(int));
-      int ctx_len = 1;
-      ctx[0] = 0;
-
-      while (running)
-      {
-            int T = ctx_len < cfg.block_size ? ctx_len : cfg.block_size;
-            int *window = ctx + (ctx_len - T);
-
-            // Upload tokens to GPU
-            CUDA_CHECK(cudaMemcpy(d_tokens, window,
-                                  T * sizeof(int), cudaMemcpyHostToDevice));
-
-            forward(d_logits, d_tokens, T, &cfg, &W, cublas);
-
-            // Download logits
-            CUDA_CHECK(cudaMemcpy(logits, d_logits,
-                                  cfg.vocab_size * sizeof(float),
-                                  cudaMemcpyDeviceToHost));
-
-            softmax_cpu(logits, cfg.vocab_size);
-            int next = sample(logits, cfg.vocab_size);
-
-            printf("%c", vocab[next]);
-            fflush(stdout);
-
-            if (ctx_len < cfg.block_size)
-                  ctx[ctx_len++] = next;
-            else
-            {
-                  memmove(ctx, ctx + 1, (cfg.block_size - 1) * sizeof(int));
-                  ctx[cfg.block_size - 1] = next;
-            }
-      }
-
-      cublasDestroy(cublas);
-      cudaFree(d_tokens);
-      cudaFree(d_logits);
-      free(logits);
-      free(ctx);
-      free(vocab);
-
-      return 0;
-}
\ No newline at end of file

From fa3f93b28e5d19907e42777911db1ce095518542 Mon Sep 17 00:00:00 2001
From: Eamon 
Date: Fri, 24 Apr 2026 16:04:42 +0530
Subject: [PATCH 27/27] training logs in a file to analyze model

---
 logs/train_run_latest.log | 26 ++++++++++++++++++++++++++
 1 file changed, 26 insertions(+)
 create mode 100644 logs/train_run_latest.log

diff --git a/logs/train_run_latest.log b/logs/train_run_latest.log
new file mode 100644
index 0000000..5c42780
--- /dev/null
+++ b/logs/train_run_latest.log
@@ -0,0 +1,26 @@
+[    0/5000]   train=4.6207   val=4.6202   elapsed=2s    ETA=0s    << best!
+[  200/5000]   train=2.2058   val=2.1986   elapsed=17s   ETA=405s  << best!
+[  400/5000]   train=1.6111   val=1.6039   elapsed=32s   ETA=367s  << best!
+[  600/5000]   train=1.4109   val=1.4183   elapsed=47s   ETA=342s  << best!
+[  800/5000]   train=1.3230   val=1.3231   elapsed=61s   ETA=322s  << best!
+[ 1000/5000]   train=1.2495   val=1.2567   elapsed=76s   ETA=303s  << best!
+[ 1200/5000]   train=1.1960   val=1.1948   elapsed=90s   ETA=286s  << best!
+[ 1400/5000]   train=1.1569   val=1.1642   elapsed=105s  ETA=270s  << best!
+[ 1600/5000]   train=1.1283   val=1.1283   elapsed=120s  ETA=254s  << best!
+[ 1800/5000]   train=1.0894   val=1.1023   elapsed=134s  ETA=238s  << best!
+[ 2000/5000]   train=1.0731   val=1.0765   elapsed=149s  ETA=223s  << best!
+[ 2200/5000]   train=1.0584   val=1.0550   elapsed=163s  ETA=208s  << best!
+[ 2400/5000]   train=1.0415   val=1.0346   elapsed=178s  ETA=192s  << best!
+[ 2600/5000]   train=1.0261   val=1.0199   elapsed=192s  ETA=177s  << best!
+[ 2800/5000]   train=1.0106   val=1.0117   elapsed=207s  ETA=162s  << best!
+[ 3000/5000]   train=1.0000   val=0.9956   elapsed=221s  ETA=148s  << best!
+[ 3200/5000]   train=0.9913   val=0.9924   elapsed=236s  ETA=133s  << best!
+[ 3400/5000]   train=0.9727   val=0.9782   elapsed=251s  ETA=118s  << best!
+[ 3600/5000]   train=0.9656   val=0.9720   elapsed=265s  ETA=103s  << best!
+[ 3800/5000]   train=0.9685   val=0.9632   elapsed=280s  ETA=88s   << best!
+[ 4000/5000]   train=0.9601   val=0.9642   elapsed=294s  ETA=74s
+[ 4200/5000]   train=0.9515   val=0.9489   elapsed=309s  ETA=59s   << best!
+[ 4400/5000]   train=0.9433   val=0.9431   elapsed=323s  ETA=44s   << best!
+[ 4600/5000]   train=0.9384   val=0.9459   elapsed=338s  ETA=29s
+[ 4800/5000]   train=0.9331   val=0.9250   elapsed=353s  ETA=15s   << best!
+[ 4999/5000]   train=0.9307   val=0.9430   elapsed=367s  ETA=0s