Ai Engineering Core / Transformers, Tokens & Context Windows

level1 β†’ part3 full code

πŸ› οΈ Tokenization Using tiktoken

import tiktoken

encoding = tiktoken.encoding_for_model("gpt-4.1-mini")

tokens = encoding.encode("Hi my name is Ed and I like banoffee pie")
tokens
for token_id in tokens:
    token_text = encoding.decode([token_id])
    print(f"{token_id} = {token_text}")

🧠 The Illusion of Memory

import os
from dotenv import load_dotenv

load_dotenv(override=True)
api_key = os.getenv('OPENAI_API_KEY')

if not api_key:
    print("No API key was found - please head over to the troubleshooting notebook in this folder to identify & fix!")
elif not api_key.startswith("sk-proj-"):
    print("An API key was found, but it doesn't start sk-proj-; please check you're using the right key - see troubleshooting notebook")
else:
    print("API key found and looks good so far!")2
from openai import OpenAI

openai = OpenAI()
messages = [
    {"role": "system", "content": "You are a helpful assistant"},
    {"role": "user", "content": "Hi! I'm Bhaskar!"}
    ]
response = openai.chat.completions.create(model="gpt-4.1-mini", messages=messages)
response.choices[0].message.content
messages = [
    {"role": "system", "content": "You are a helpful assistant"},
    {"role": "user", "content": "What's my name?"}
    ]
response = openai.chat.completions.create(model="gpt-4.1-mini", messages=messages)
response.choices[0].message.content