[ { "slide_number": 1, "title": "Rapid Application Development using Large Language Models", "slide_body": [ "Speaker Name, Title | Event/Date" ], "notes": [ "- \"Hello and welcome to the NVIDIA Deep Learning Institute (DLI) course on rapid application development using large language models (LLMs).\"", "- We will explore LLMs, generative AI, and software paradigms built around these technologies.", "- Expect hands-on experience, practical learning, and insights into advanced AI concepts.", "- Let’s dive into the course and the exciting content ahead." ], "raw_notes": "- \"Hello and welcome to the NVIDIA Deep Learning Institute (DLI) course on rapid application development using large language models (LLMs).\"\n- We will explore LLMs, generative AI, and software paradigms built around these technologies.\n- Expect hands-on experience, practical learning, and insights into advanced AI concepts.\n- Let’s dive into the course and the exciting content ahead." }, { "slide_number": 2, "title": "Large Language Models", "slide_body": [ "Backbones for Language Understanding", "Backbones for language tasks, including classification and generation." ], "notes": [ "- \"LLMs enable computers to understand language without human intervention.\"", "- LLMs can handle various tasks: text classification, text generation, and even image-to-text conversion.", "- We’ll work with multiple models like semantic classification models, question-answering models, and more.", "- We will narrow down on text generation towards the end of the course." ], "raw_notes": "- \"LLMs enable computers to understand language without human intervention.\"\n- LLMs can handle various tasks: text classification, text generation, and even image-to-text conversion.\n- We’ll work with multiple models like semantic classification models, question-answering models, and more.\n- We will narrow down on text generation towards the end of the course." }, { "slide_number": 3, "title": "Language-Reasoning Systems", "slide_body": [ "Language-In and/or Language-Out", "?", "GenAI", "Model", "GenAI", "Orchestration" ], "notes": [ "- \"We distinguish between generative AI models and orchestration.\"", "- **Generative AI models**handle single tasks such as text understanding and generation.", "- **Orchestration**combines multiple models to create integrated AI systems.", "- By the end of the course, you'll see how to develop such systems easily." ], "raw_notes": "- \"We distinguish between generative AI models and orchestration.\"\n- **Generative AI models**handle single tasks such as text understanding and generation.\n- **Orchestration**combines multiple models to create integrated AI systems.\n- By the end of the course, you'll see how to develop such systems easily." }, { "slide_number": 4, "title": "Sequence Modeling", "slide_body": [ "Core Component of Generative AI", "3" ], "notes": [ "- \"Our focus will be on sequence modeling, particularly transformers, but our exploration will lead us to other domains as our understanding continues to evolve.\"", "- Sequence models are central to many AI tasks: from text understanding to generating images and animations.", "- We’ll also touch on complementary architectures like autoencoders and diffusion models to help us reason with more complex input formats like images.", "- You’ll learn how these models connect across various tasks and domains, and will approach an understanding of more general generative AI principles along the way." ], "raw_notes": "- \"Our focus will be on sequence modeling, particularly transformers, but our exploration will lead us to other domains as our understanding continues to evolve.\"\n- Sequence models are central to many AI tasks: from text understanding to generating images and animations.\n- We’ll also touch on complementary architectures like autoencoders and diffusion models to help us reason with more complex input formats like images.\n- You’ll learn how these models connect across various tasks and domains, and will approach an understanding of more general generative AI principles along the way." }, { "slide_number": 5, "title": "Course Objectives", "slide_body": [ "Understand How LLMs Work", "Architectures", "Intuitions", "Design Paradigms", "Use LLMs To Solve Interesting Problems", "Language Reasoning", "Text Generation", "Chat Assistants", "Understand How to Deploy LLM Servers", "Rapid LLM Application Development" ], "notes": [ "- \"This course is structured to help you understand and build LLM-based applications.\"", "- **Understand LLM architecture**: Deep dive into model architectures and design paradigms.", "- **Apply LLMs to problems**: Language reasoning, text generation, chatbot development, etc.", "- **Deploy and scale**: Learn how to deploy LLM servers and optimize them." ], "raw_notes": "- \"This course is structured to help you understand and build LLM-based applications.\"\n- **Understand LLM architecture**: Deep dive into model architectures and design paradigms.\n- **Apply LLMs to problems**: Language reasoning, text generation, chatbot development, etc.\n- **Deploy and scale**: Learn how to deploy LLM servers and optimize them." }, { "slide_number": 6, "title": "Prerequisites", "slide_body": [ "Fundamentals of Deep Learning", "Intermediate Python Required" ], "notes": [ "- \"Before starting, it's crucial you have a background in deep learning and Python.\"", "- Python knowledge is required, and familiarity with libraries like NumPy,Keras, andPyTorchis recommended.", "- Prior experience in deep learning frameworks is helpful.", "- We will build on machine learning and deep learning fundamentals throughout the course." ], "raw_notes": "- \"Before starting, it's crucial you have a background in deep learning and Python.\"\n- Python knowledge is required, and familiarity with libraries like NumPy,Keras, andPyTorchis recommended.\n- Prior experience in deep learning frameworks is helpful.\n- We will build on machine learning and deep learning fundamentals throughout the course." }, { "slide_number": 7, "title": "Core Tools", "slide_body": [ "Open-Source LLM Ecosystem" ], "notes": [ "- \"Our main tools for this course will be Transformers andLangChain.\"", "- **Transformers**: The foundation for model specification and inference.", "- **LangChain**: The orchestration framework for integrating different AI models.", "- These tools will be our primary focus, and we’ll explore their usage in depth." ], "raw_notes": "- \"Our main tools for this course will be Transformers andLangChain.\"\n- **Transformers**: The foundation for model specification and inference.\n- **LangChain**: The orchestration framework for integrating different AI models.\n- These tools will be our primary focus, and we’ll explore their usage in depth." }, { "slide_number": 8, "title": "Course Schedule", "slide_body": [ "Transformer Intuitions", "Getting Started", "Encoder Tasks", "Seq2Seq", "Inference Servers", "Multimodality", "Intro Agentics", "Final Assessment", "What’s The Plan?" ], "notes": [ "- \"The course will cover the following main topics:\"", "- **Getting Started**: Introduction to the tools and setup.", "- **Transformer Intuitions**: Understanding how transformers work.", "- **Encoder Tasks**: Text classification and feature extraction.", "- **Seq2Seq Models**: Sequence-to-sequence modeling.", "- **Multimodality**: Integrating different data types (text, images, audio).", "- **Inference Servers**: How to deploy models.", "- **Agentics**: Introduction to agent-based systems.", "- **Final Assessment**: Testing your understanding of the course." ], "raw_notes": "- \"The course will cover the following main topics:\"\n- **Getting Started**: Introduction to the tools and setup.\n- **Transformer Intuitions**: Understanding how transformers work.\n- **Encoder Tasks**: Text classification and feature extraction.\n- **Seq2Seq Models**: Sequence-to-sequence modeling.\n- **Multimodality**: Integrating different data types (text, images, audio).\n- **Inference Servers**: How to deploy models.\n- **Agentics**: Introduction to agent-based systems.\n- **Final Assessment**: Testing your understanding of the course." }, { "slide_number": 9, "title": "Part 1: From DL To LLMs", "slide_body": [ "•", "From Deep Learning to Language Modeling", "•", "Pulling In Our First Model", "•", "Model Architecture Intuitions", "•", "Using Encoders For Encoding" ], "notes": [ "- \"Welcome to Part 1: From Deep Learning to Language Models (LLMs). We’ll start by laying the foundation with a brief overview of deep learning concepts, then move towards applying them in language modeling.\"" ], "raw_notes": "- \"Welcome to Part 1: From Deep Learning to Language Models (LLMs). We’ll start by laying the foundation with a brief overview of deep learning concepts, then move towards applying them in language modeling.\"" }, { "slide_number": 10, "title": "Deep Learning", "slide_body": [ "Machine Learning with Deep Models", "High-Performance Data Science with RAPIDS | NVIDIA" ], "notes": [ "- \"At the heart of machine learning is deep learning, a paradigm shift in how we approach solving problems with complex data like images and text.\"", "- \"The process typically begins with raw data, which we extract, transform, and load into a system of deep learning models.\"", "- \"The goal is to iteratively train models using this data, continually refining their predictions until we have a model capable of generalizing well on unseen data.\"" ], "raw_notes": "- \"At the heart of machine learning is deep learning, a paradigm shift in how we approach solving problems with complex data like images and text.\"\n- \"The process typically begins with raw data, which we extract, transform, and load into a system of deep learning models.\"\n- \"The goal is to iteratively train models using this data, continually refining their predictions until we have a model capable of generalizing well on unseen data.\"" }, { "slide_number": 11, "title": "Deep Learning", "slide_body": [ "Machine Learning with Deep Models", "Cat \t [100%]", "Dog [ 0%]", "Bird [ 0%]", "Cat \t\t[80%]", "Dog \t\t[15%]", "Bird \t\t[ 5%]", "1", "2", "3", "4", "5", "0" ], "notes": [ "- \"Let’s look at a simple example of deep learning applied to image classification. Here, we’re asking the model to classify whether an image is a cat, dog, or bird.\"", "- \"The model takes in an image, represented as a grid of pixels, and processes it through multiple layers, where each layer transforms the data step-by-step, from a highly complex image to a final prediction like 'Cat: 80%, Dog: 15%, Bird: 5%'.\"" ], "raw_notes": "- \"Let’s look at a simple example of deep learning applied to image classification. Here, we’re asking the model to classify whether an image is a cat, dog, or bird.\"\n- \"The model takes in an image, represented as a grid of pixels, and processes it through multiple layers, where each layer transforms the data step-by-step, from a highly complex image to a final prediction like 'Cat: 80%, Dog: 15%, Bird: 5%'.\"" }, { "slide_number": 12, "title": "Deep Learning", "slide_body": [ "Machine Learning with Deep Models", "Cat \t\t[80%]", "Dog \t\t[15%]", "Bird \t\t[ 5%]", "1", "2", "3", "4", "5", "Convolution Layers", "Sparse", "Dense", "Dense Layers", "0", "Cat \t [100%]", "Dog [ 0%]", "Bird [ 0%]" ], "notes": [ "- \"Deep learning models can have different types of connections between neurons. Two common types are**Sparse**and**Dense**.\"", "- \"**Sparse connections**only interact with specific parts of the input, like certain regions of an image, while**Dense connections**work on the entire input at once.\"", "- \"You may have encountered sparse connections in**convolutional layers**, which focus on small regions of the input, while dense layers aggregate all the data.\"" ], "raw_notes": "- \"Deep learning models can have different types of connections between neurons. Two common types are**Sparse**and**Dense**.\"\n- \"**Sparse connections**only interact with specific parts of the input, like certain regions of an image, while**Dense connections**work on the entire input at once.\"\n- \"You may have encountered sparse connections in**convolutional layers**, which focus on small regions of the input, while dense layers aggregate all the data.\"" }, { "slide_number": 13, "title": "Deep Learning", "slide_body": [ "Machine Learning with Deep Models", "Cat \t\t[80%]", "Dog \t\t[15%]", "Bird \t\t[ 5%]", "0", "1", "2", "3", "4", "5", "General Image Logic", "Sparse", "Dense", "Feature Extractors", "Task Head", "Cat? Dog?", "Cat \t [100%]", "Dog [ 0%]", "Bird [ 0%]" ], "notes": [ "- \"Deep learning models often break down tasks into stages. The first part of the model, known as the**feature extractor**, learns to recognize general patterns in data.\"", " - \"For instance, the feature extractor might focus on identifying the edges and shapes in an image.\"", "- \"The second part, known as the**task head**, focuses on the specific problem, such as classifying the image into one of the three categories: cat, dog, or bird.”" ], "raw_notes": "- \"Deep learning models often break down tasks into stages. The first part of the model, known as the**feature extractor**, learns to recognize general patterns in data.\"\n - \"For instance, the feature extractor might focus on identifying the edges and shapes in an image.\"\n- \"The second part, known as the**task head**, focuses on the specific problem, such as classifying the image into one of the three categories: cat, dog, or bird.”" }, { "slide_number": 14, "title": "Deep Learning", "slide_body": [ "Machine Learning with Deep Models", "Cat \t\t[80%]", "Dog \t\t[15%]", "Bird \t\t[ 5%]", "0", "1", "2", "3", "4", "5", "Sparse", "Dense", "Feature Extractors", "Task Head", "Differentiable", "Cat \t [100%]", "Dog [ 0%]", "Bird [ 0%]" ], "notes": [ "- \"A crucial feature of deep learning models is that they are**differentiable**. This allows us to calculate how small changes to the model’s parameters (its weights) impact its predictions.\"", "- \"This process, called**backpropagation**, is what enables the model to learn from its mistakes and optimize itself.\"", "- \"Through backpropagation, we calculate the gradient of the loss function with respect to the model’s parameters and use this to adjust the model, making it perform better.\"" ], "raw_notes": "- \"A crucial feature of deep learning models is that they are**differentiable**. This allows us to calculate how small changes to the model’s parameters (its weights) impact its predictions.\"\n- \"This process, called**backpropagation**, is what enables the model to learn from its mistakes and optimize itself.\"\n- \"Through backpropagation, we calculate the gradient of the loss function with respect to the model’s parameters and use this to adjust the model, making it perform better.\"" }, { "slide_number": 15, "title": "Deep Learning", "slide_body": [ "Machine Learning with Deep Models", "Cat \t\t[ 80%]", "Dog \t\t[ 15%]", "Bird \t\t[ 5%]", "Cat \t [100%]", "Dog [ 0%]", "Bird [ 0%]" ], "notes": [ "- \"The model’s prediction is called a hypothesis. For example, it might predict that an image is 80% likely to be a cat and 20% likely to be a dog.\"", "- \"We compare this hypothesis to the true label—what we know the image actually is.”", "(next slide)", "- \"The difference between the model’s prediction and the true label is called the**loss**. The goal of training is to minimize this loss." ], "raw_notes": "- \"The model’s prediction is called a hypothesis. For example, it might predict that an image is 80% likely to be a cat and 20% likely to be a dog.\"\n- \"We compare this hypothesis to the true label—what we know the image actually is.”\n(next slide)\n- \"The difference between the model’s prediction and the true label is called the**loss**. The goal of training is to minimize this loss." }, { "slide_number": 16, "title": "Deep Learning", "slide_body": [ "Machine Learning with Deep Models", "Cat \t\t[ 80%]", "Dog \t\t[ 15%]", "Bird \t\t[ 5%]", "Cat \t [100%]", "Dog [ 0%]", "Bird [ 0%]", "Visualizing the Loss Landscape of Neural Nets | UMD" ], "notes": [ "- \"The difference between the model’s prediction and the true label is called the**loss**. The goal of training is to minimize this loss.”", "- (Next Slide)", "- \"To reduce the loss, we use a process called**gradient descent**. Essentially, we calculate how the loss changes with respect to the model’s parameters, then adjust those parameters in the direction that reduces the loss.\"", "- \"This allows the model to improve with each iteration, gradually finding better parameters that make better predictions.\"" ], "raw_notes": "- \"The difference between the model’s prediction and the true label is called the**loss**. The goal of training is to minimize this loss.”\n- (Next Slide)\n- \"To reduce the loss, we use a process called**gradient descent**. Essentially, we calculate how the loss changes with respect to the model’s parameters, then adjust those parameters in the direction that reduces the loss.\"\n- \"This allows the model to improve with each iteration, gradually finding better parameters that make better predictions.\"" }, { "slide_number": 17, "title": "Deep Learning", "slide_body": [ "Machine Learning with Deep Models", "Cat \t[ 86%]", "Dog \t[ 10%]", "Bird \t[ 4%]", "Cat \t\t[ 80%]", "Dog \t\t[ 15%]", "Bird \t\t[ 5%]", "Cat \t [100%]", "Dog [ 0%]", "Bird [ 0%]", "Visualizing the Loss Landscape of Neural Nets | UMD" ], "notes": [ "- \"To reduce the loss, we use a process called**gradient descent**. Essentially, we calculate how the loss changes with respect to the model’s parameters, then adjust those parameters in the direction that reduces the loss.\"", "- \"This allows the model to improve with each iteration, gradually finding better parameters that make better predictions", "- \"By repeatedly applying gradient descent, the model gets better and better at making accurate predictions. With each iteration, the loss decreases, and the model’s performance improves.\"", "- \"Eventually, the model reaches a point where its predictions are consistently very close to the true values, which may hopefully indicate that it can generalize well to new data - assuming everything is set up nicely.\"" ], "raw_notes": "- \"To reduce the loss, we use a process called**gradient descent**. Essentially, we calculate how the loss changes with respect to the model’s parameters, then adjust those parameters in the direction that reduces the loss.\"\n- \"This allows the model to improve with each iteration, gradually finding better parameters that make better predictions\n- \"By repeatedly applying gradient descent, the model gets better and better at making accurate predictions. With each iteration, the loss decreases, and the model’s performance improves.\"\n- \"Eventually, the model reaches a point where its predictions are consistently very close to the true values, which may hopefully indicate that it can generalize well to new data - assuming everything is set up nicely.\"" }, { "slide_number": 18, "title": "From Deep Learning to LLMs", "slide_body": [ "Our Goal For The Course", "Fundamentals of Deep Learning", "Language Reasoning" ], "notes": [ "- \"At the core of this course, we aim to transition from understanding**deep learning fundamentals**to tackling more complex language tasks with**large language models (LLMs)**.\"" ], "raw_notes": "- \"At the core of this course, we aim to transition from understanding**deep learning fundamentals**to tackling more complex language tasks with**large language models (LLMs)**.\"" }, { "slide_number": 19, "title": "What Is Language?", "slide_body": [ "Discrete Labels?" ], "notes": [ "- \"So, what exactly is**language**? It’s not just a string of letters or characters but a**sequence**of information that carries meaning.\"", "- \"We start with raw sequences of letters, as in this sentence: 'Hello! How is it going?!'.\"", "- \"Using a process called**tokenization**, we break these sequences into smaller, understandable units—like words, punctuation marks, or even sub-word components.\"", "- \"These tokens form an**ordered sequence of classes**that a model can process and understand. The goal is not just to recognize words but also to capture the**relationships**between them.\"" ], "raw_notes": "- \"So, what exactly is**language**? It’s not just a string of letters or characters but a**sequence**of information that carries meaning.\"\n- \"We start with raw sequences of letters, as in this sentence: 'Hello! How is it going?!'.\"\n- \"Using a process called**tokenization**, we break these sequences into smaller, understandable units—like words, punctuation marks, or even sub-word components.\"\n- \"These tokens form an**ordered sequence of classes**that a model can process and understand. The goal is not just to recognize words but also to capture the**relationships**between them.\"" }, { "slide_number": 20, "title": "The Base Formulation", "slide_body": [ "Dealing With Language As An Input", "The Illustrated GPT-2 (Visualizing Transformer Language Models)" ], "notes": [ "- \"When dealing with language, we can approach it from different angles. Consider the phrase 'Hello! How is it going?!'.\"", "- \"Some models, like**BERT**, treat language as a sequence that can be encoded. The encoder processes the entire sequence of tokens and extracts useful representations.\"", "- \"Other models, like**GPT-2**, use decoders to generate new sequences, predicting what comes next or generating responses.\"", "- \"Depending on the task, we might use**sequence prediction**(e.g., predicting the sentiment of a sentence: 90% positive, 10% negative),**pre-token prediction**(like understanding each token’s role in the sentence), or**forecasting predictions**(e.g., generating a response like 'It’s going great!’).”", "In the next notebook, we’ll explore some of the easier flavors of these models and will learn how we can start using them in our environment." ], "raw_notes": "- \"When dealing with language, we can approach it from different angles. Consider the phrase 'Hello! How is it going?!'.\"\n- \"Some models, like**BERT**, treat language as a sequence that can be encoded. The encoder processes the entire sequence of tokens and extracts useful representations.\"\n- \"Other models, like**GPT-2**, use decoders to generate new sequences, predicting what comes next or generating responses.\"\n- \"Depending on the task, we might use**sequence prediction**(e.g., predicting the sentiment of a sentence: 90% positive, 10% negative),**pre-token prediction**(like understanding each token’s role in the sentence), or**forecasting predictions**(e.g., generating a response like 'It’s going great!’).”\nIn the next notebook, we’ll explore some of the easier flavors of these models and will learn how we can start using them in our environment." }, { "slide_number": 21, "title": "Part 1: From DL To LLMs", "slide_body": [ "•", "From Deep Learning to Language Modeling", "•", "Pulling In Our First Model", "•", "Model Architecture Intuitions", "•", "Using Encoders For Encoding" ], "notes": [ "**NOTE: Ask people to start their notebooks now.**", "- Welcome back to the course! In the previous section, we introduced deep learning and its fundamentals to set the stage for more advanced topics.", "- Now, we’re going to dive deeper into language models and start pulling in models from the Hugging Face platform." ], "raw_notes": "**NOTE: Ask people to start their notebooks now.**\n- Welcome back to the course! In the previous section, we introduced deep learning and its fundamentals to set the stage for more advanced topics.\n- Now, we’re going to dive deeper into language models and start pulling in models from the Hugging Face platform." }, { "slide_number": 22, "title": "Hugging Face", "slide_body": [ "Open-Source Large Model Community" ], "notes": [ "- To kick things off, we’ll be working with**Hugging Face**, which is a large open-source community dedicated to sharing machine learning models, datasets, and tools.", "- Hugging Face offers a broad range of pre-trained models for various tasks, from text generation to image classification.", "- The platform allows you to pull in models, fine-tune them, or even deploy them directly in production." ], "raw_notes": "- To kick things off, we’ll be working with**Hugging Face**, which is a large open-source community dedicated to sharing machine learning models, datasets, and tools.\n- Hugging Face offers a broad range of pre-trained models for various tasks, from text generation to image classification.\n- The platform allows you to pull in models, fine-tune them, or even deploy them directly in production." }, { "slide_number": 23, "title": "Hugging Face", "slide_body": [ "Large Open-Sourced Repository", "Very easy to use, write to, pull from", "So many models!!", "Amazing For Experimentation", "See what all can be done!", "See how others have done things!", "Use their models for your use-cases sometimes", "Open-Source Large Models", "Models | Hugging Face" ], "notes": [ "- Hugging Face’s repository is particularly useful for experimentation. You can browse thousands of models uploaded by everyone from large corporations to individual developers.", "- The platform also offers an intuitive way to search, upload, and experiment with models, making it accessible even for newcomers.", "- (Next slide talks about EULAs and restrictions)" ], "raw_notes": "- Hugging Face’s repository is particularly useful for experimentation. You can browse thousands of models uploaded by everyone from large corporations to individual developers.\n- The platform also offers an intuitive way to search, upload, and experiment with models, making it accessible even for newcomers.\n- (Next slide talks about EULAs and restrictions)" }, { "slide_number": 24, "title": "Hugging Face", "slide_body": [ "Large Open-Sourced Repository", "Very easy to use, write to, pull from", "So many models!!", "Amazing For Experimentation", "See what all can be done!", "See how others have done things!", "Use their models for your use-cases sometimes", "Big Catch", "Check EULA licenses per-model per-dataset", "Might have to recreate system or train yourself", "Open-Source Large Models", "Models | Hugging Face" ], "notes": [ "- One thing to keep in mind when working with Hugging Face models is licensing.", "- Not every model is suitable for every purpose, so it’s essential to check the End User License Agreements (EULA) for models and datasets to ensure they align with your project needs.", "- Some models are only intended for research or have specific restrictions, so always do your due diligence." ], "raw_notes": "- One thing to keep in mind when working with Hugging Face models is licensing.\n- Not every model is suitable for every purpose, so it’s essential to check the End User License Agreements (EULA) for models and datasets to ensure they align with your project needs.\n- Some models are only intended for research or have specific restrictions, so always do your due diligence." }, { "slide_number": 25, "title": "Pulling In A Simple Model", "slide_body": [ "Lightweight BERT Model", "nlptown/bert-base-multilingual-uncased-sentiment | Hugging Face" ], "notes": [ "- In this section, we’ll show how easy it is to pull in a simple**LLM**from Hugging Face.", "- The first step is searching for a relevant model by filtering in the Hugging Face model directory for what you want." ], "raw_notes": "- In this section, we’ll show how easy it is to pull in a simple**LLM**from Hugging Face.\n- The first step is searching for a relevant model by filtering in the Hugging Face model directory for what you want." }, { "slide_number": 26, "title": "Pulling In A Simple Model", "slide_body": [ "Lightweight BERT Model", "nlptown/bert-base-multilingual-uncased-sentiment | Hugging Face" ], "notes": [ "- As an example, we can selected a**lightweight BERT model**for multilingual sentiment analysis.", "- You’ll see some key details like the model’s architecture (BERT), the licensing (MIT), and what frameworks it supports (e.g.,PyTorch, TensorFlow, etc.).", "- It’s always a good practice to review this information before proceeding." ], "raw_notes": "- As an example, we can selected a**lightweight BERT model**for multilingual sentiment analysis.\n- You’ll see some key details like the model’s architecture (BERT), the licensing (MIT), and what frameworks it supports (e.g.,PyTorch, TensorFlow, etc.).\n- It’s always a good practice to review this information before proceeding." }, { "slide_number": 27, "title": "Pulling In A Simple Model", "slide_body": [ "Lightweight BERT Model", "nlptown/bert-base-multilingual-uncased-sentiment | Hugging Face" ], "notes": [ "- Once you’ve selected a model, you can test it using Hugging Face’s**hosted inference API**.", "- This allows you to experiment with the model directly in the browser without needing to download it.", "- For instance, you can input a sentence like “I like you. I love you” and see that it returns a positive sentiment score." ], "raw_notes": "- Once you’ve selected a model, you can test it using Hugging Face’s**hosted inference API**.\n- This allows you to experiment with the model directly in the browser without needing to download it.\n- For instance, you can input a sentence like “I like you. I love you” and see that it returns a positive sentiment score." }, { "slide_number": 28, "title": "Pulling In A Simple Model", "slide_body": [ "Lightweight BERT Model", "nlptown/bert-base-multilingual-uncased-sentiment | Hugging Face" ], "notes": [ "- Next, let’s walk through how to import the model into your environment.", "- Using Hugging Face’s**Transformers library**, you can import the model with just a few lines of code.", "- The pipeline function simplifies this by abstracting away the complex logic behind pre-processing, model loading, and post-processing." ], "raw_notes": "- Next, let’s walk through how to import the model into your environment.\n- Using Hugging Face’s**Transformers library**, you can import the model with just a few lines of code.\n- The pipeline function simplifies this by abstracting away the complex logic behind pre-processing, model loading, and post-processing." }, { "slide_number": 29, "title": "Pulling In A Simple Model", "slide_body": [ "Lightweight BERT Model" ], "notes": [ "- Now that we've seen how easy it is to navigate Hugging Face and find models, let's pull one in and try it out.", "- Here, we use the transformers library to import a BERT-based sentiment classification model.", "- You can see how it downloads everything necessary to run the model – the configuration, tokenizer, model weights, and more.", "- This all comes from the model repository, which you can think of like a git repo associated with eachhuggingfacemodel." ], "raw_notes": "- Now that we've seen how easy it is to navigate Hugging Face and find models, let's pull one in and try it out.\n- Here, we use the transformers library to import a BERT-based sentiment classification model.\n- You can see how it downloads everything necessary to run the model – the configuration, tokenizer, model weights, and more.\n- This all comes from the model repository, which you can think of like a git repo associated with eachhuggingfacemodel." }, { "slide_number": 30, "title": "Running Your Simple Model", "slide_body": [ "Lightweight BERT Model" ], "notes": [ "- We can now feed simple words into the model and observe how it classifies them.", "- For example, common positive words like 'hello' or 'awesome' are rated with 5 stars, while words like 'uh' and 'eh' get lower ratings.", "- The model is making these classifications based on patterns it has learned during training, and each result comes with a confidence score that tells us how strongly the model associates the input with the corresponding label." ], "raw_notes": "- We can now feed simple words into the model and observe how it classifies them.\n- For example, common positive words like 'hello' or 'awesome' are rated with 5 stars, while words like 'uh' and 'eh' get lower ratings.\n- The model is making these classifications based on patterns it has learned during training, and each result comes with a confidence score that tells us how strongly the model associates the input with the corresponding label." }, { "slide_number": 31, "title": "Running Your Simple Model", "slide_body": [ "Lightweight BERT Model" ], "notes": [ "- Next, we test the model with some more complex inputs.", "- Sentences like 'I loved my old pillow more than anything!' are still classified as 5-star reviews with high confidence, while more negative-sounding inputs, such as 'I hated my old pillow so much,' receive 1 star.", "- You’ll notice that the model isn’t really great at handling tricky inputs, but you can see that it is trying to leverage semantic meanings to take guesses at the overall sentiment." ], "raw_notes": "- Next, we test the model with some more complex inputs.\n- Sentences like 'I loved my old pillow more than anything!' are still classified as 5-star reviews with high confidence, while more negative-sounding inputs, such as 'I hated my old pillow so much,' receive 1 star.\n- You’ll notice that the model isn’t really great at handling tricky inputs, but you can see that it is trying to leverage semantic meanings to take guesses at the overall sentiment." }, { "slide_number": 32, "title": "Pipeline Components", "slide_body": [ "Input -> Tensor -> Tensor -> Output" ], "notes": [ "- To break down the process a bit more, the pipeline is divided into three major stages: preprocessing, forward pass, and postprocessing.", "- In preprocessing, the input is tokenized and converted into tensors, which are the numerical representations the model can work with.", "- The forward pass sends these tensors through the model, where the actual predictions are made.", "- Finally, postprocessing takes the model's output and turns it back into a format that's understandable for us—such as star ratings or sentiment labels.", "- This structure makes it easy to work with language models, hiding the complexity of what's happening under the hood." ], "raw_notes": "- To break down the process a bit more, the pipeline is divided into three major stages: preprocessing, forward pass, and postprocessing.\n- In preprocessing, the input is tokenized and converted into tensors, which are the numerical representations the model can work with.\n- The forward pass sends these tensors through the model, where the actual predictions are made.\n- Finally, postprocessing takes the model's output and turns it back into a format that's understandable for us—such as star ratings or sentiment labels.\n- This structure makes it easy to work with language models, hiding the complexity of what's happening under the hood." }, { "slide_number": 33, "title": "Pipeline Components", "slide_body": [ "Input -> Tensor -> Tensor -> Output" ], "notes": [ "- Let's see the pipeline in action.", "- The input is the sentence, 'It’s so fluffy I’mgonnadie.’", "- First, it goes through preprocessing, where the sentence is tokenized into input IDs and tensors.", "- Then, the forward pass sends it through the model, generating logits—the raw predictions.", "- Finally, in the postprocessing step, the logits are converted back into human-readable output, in this case a star rating of 1 star and a score of around 0.49.", "- This process happens seamlessly when using the pipeline, and shows the overall flow of data in such a pipeline; from text to tensor, from tensor to tensor, and from tensor to output." ], "raw_notes": "- Let's see the pipeline in action.\n- The input is the sentence, 'It’s so fluffy I’mgonnadie.’\n- First, it goes through preprocessing, where the sentence is tokenized into input IDs and tensors.\n- Then, the forward pass sends it through the model, generating logits—the raw predictions.\n- Finally, in the postprocessing step, the logits are converted back into human-readable output, in this case a star rating of 1 star and a score of around 0.49.\n- This process happens seamlessly when using the pipeline, and shows the overall flow of data in such a pipeline; from text to tensor, from tensor to tensor, and from tensor to output." }, { "slide_number": 34, "title": "Pipeline Components", "slide_body": [ "Input -> Tensor -> Tensor -> Output" ], "notes": [ "- This diagram shows how each stage of the pipeline transforms data.", "- The message goes from a string to a dictionary of tensors, back to a dictionary of logits, and finally back to a string output.", "- Each step is tied to a specific component of the pipeline, be it the tokenizer, the model, or the pipeline itself.", "- All of these components are constructed in your environment from the config files which are present in the model repo combined with the settings you change in constructing your pipeline." ], "raw_notes": "- This diagram shows how each stage of the pipeline transforms data.\n- The message goes from a string to a dictionary of tensors, back to a dictionary of logits, and finally back to a string output.\n- Each step is tied to a specific component of the pipeline, be it the tokenizer, the model, or the pipeline itself.\n- All of these components are constructed in your environment from the config files which are present in the model repo combined with the settings you change in constructing your pipeline." }, { "slide_number": 35, "title": "Pipeline Components", "slide_body": [ "Input -> Tensor -> Tensor -> Output" ], "notes": [ "**Now that we know the specific components, let’s look a little bit more closely about how they were constructed for our specific model.**", "**(Subsequent sections)**", "- For the preprocessing phase, we can see that the config settings specify various tokenization properties and also point out to BERT, which loads in a default tokenizer if one is not specified.", "- For the forward, we can see that some pipeline details are also provided in the config andreguardingwhat kind of forward pipeline to use along with some additional architectural settings.", "- And then lastly, the postprocessing comes naturally out of the pipeline definition, which is also selected in the config file and is aliased and directed to appropriate python objects by the transformers library." ], "raw_notes": "**Now that we know the specific components, let’s look a little bit more closely about how they were constructed for our specific model.**\n**(Subsequent sections)**\n- For the preprocessing phase, we can see that the config settings specify various tokenization properties and also point out to BERT, which loads in a default tokenizer if one is not specified.\n- For the forward, we can see that some pipeline details are also provided in the config andreguardingwhat kind of forward pipeline to use along with some additional architectural settings.\n- And then lastly, the postprocessing comes naturally out of the pipeline definition, which is also selected in the config file and is aliased and directed to appropriate python objects by the transformers library." }, { "slide_number": 36, "title": "Pipeline Components", "slide_body": [ "Input -> Tensor -> Tensor -> Output", "nlptown/bert-base-multilingual-uncased-sentiment | Hugging Face" ], "notes": [ "Now that we know the specific components, let’s look a little bit more closely about how they were constructed for our specific model.", "- **For the preprocessing phase, we can see that the config settings specify various tokenization properties and also point out to BERT, which loads in a default tokenizer if one is not specified.**", "- (Subsequent sections)", "- For the forward, we can see that some pipeline details are also provided in the config andreguardingwhat kind of forward pipeline to use along with some additional architectural settings.", "- And then lastly, the postprocessing comes naturally out of the pipeline definition, which is also selected in the config file and is aliased and directed to appropriate python objects by the transformers library." ], "raw_notes": "Now that we know the specific components, let’s look a little bit more closely about how they were constructed for our specific model.\n- **For the preprocessing phase, we can see that the config settings specify various tokenization properties and also point out to BERT, which loads in a default tokenizer if one is not specified.**\n- (Subsequent sections)\n- For the forward, we can see that some pipeline details are also provided in the config andreguardingwhat kind of forward pipeline to use along with some additional architectural settings.\n- And then lastly, the postprocessing comes naturally out of the pipeline definition, which is also selected in the config file and is aliased and directed to appropriate python objects by the transformers library." }, { "slide_number": 37, "title": "Pipeline Components", "slide_body": [ "Input -> Tensor -> Tensor -> Output", "nlptown/bert-base-multilingual-uncased-sentiment | Hugging Face" ], "notes": [ "Now that we know the specific components, let’s look a little bit more closely about how they were constructed for our specific model.", "- For the preprocessing phase, we can see that the config settings specify various tokenization properties and also point out to BERT, which loads in a default tokenizer if one is not specified.", "- **For the forward, we can see that some pipeline details are also provided in the config and****reguarding****what kind of forward pipeline to use along with some additional architectural settings.**", "- (Subsequent sections)", "- And then lastly, the postprocessing comes naturally out of the pipeline definition, which is also selected in the config file and is aliased and directed to appropriate python objects by the transformers library." ], "raw_notes": "Now that we know the specific components, let’s look a little bit more closely about how they were constructed for our specific model.\n- For the preprocessing phase, we can see that the config settings specify various tokenization properties and also point out to BERT, which loads in a default tokenizer if one is not specified.\n- **For the forward, we can see that some pipeline details are also provided in the config and****reguarding****what kind of forward pipeline to use along with some additional architectural settings.**\n- (Subsequent sections)\n- And then lastly, the postprocessing comes naturally out of the pipeline definition, which is also selected in the config file and is aliased and directed to appropriate python objects by the transformers library." }, { "slide_number": 38, "title": "Pipeline Components", "slide_body": [ "Input -> Tensor -> Tensor -> Output", "nlptown/bert-base-multilingual-uncased-sentiment | Hugging Face" ], "notes": [ "Now that we know the specific components, let’s look a little bit more closely about how they were constructed for our specific model.", "- For the preprocessing phase, we can see that the config settings specify various tokenization properties and also point out to BERT, which loads in a default tokenizer if one is not specified.", "- For the forward, we can see that some pipeline details are also provided in the config andreguardingwhat kind of forward pipeline to use along with some additional architectural settings.", "- **And then lastly, the postprocessing comes naturally out of the pipeline definition, which is also selected in the config file and is aliased and directed to appropriate python objects by the transformers library.**" ], "raw_notes": "Now that we know the specific components, let’s look a little bit more closely about how they were constructed for our specific model.\n- For the preprocessing phase, we can see that the config settings specify various tokenization properties and also point out to BERT, which loads in a default tokenizer if one is not specified.\n- For the forward, we can see that some pipeline details are also provided in the config andreguardingwhat kind of forward pipeline to use along with some additional architectural settings.\n- **And then lastly, the postprocessing comes naturally out of the pipeline definition, which is also selected in the config file and is aliased and directed to appropriate python objects by the transformers library.**" }, { "slide_number": 39, "title": "HF Pipeline Options", "slide_body": [ "HuggingFace Automatic Task Support", "Based on a given task:", "Pulls in default models, tokenizers, etc.", "Pulls in default Pipeline/postprocess.", "Offers Backend Options (“pt”, “tf”, etc.)", "Recommends Default Model", "Automatic End-to-End Pipelines" ], "notes": [ "In most context, theHuggingFaceecosystem can pick default models based on the task at hand.", "- Based on the task you specify—whether it’s text classification, token classification, or even audio processing—HuggingFaceautomatically pulls in the appropriate model, tokenizer, and postprocessing steps if one is not specified.", "- This also includes flexibility between backends likePyTorchand TensorFlow.", "The task support abstracts much of the complexity away, enabling rapid experimentation and integration of state-of-the-art models.\"" ], "raw_notes": "In most context, theHuggingFaceecosystem can pick default models based on the task at hand.\n- Based on the task you specify—whether it’s text classification, token classification, or even audio processing—HuggingFaceautomatically pulls in the appropriate model, tokenizer, and postprocessing steps if one is not specified.\n- This also includes flexibility between backends likePyTorchand TensorFlow.\nThe task support abstracts much of the complexity away, enabling rapid experimentation and integration of state-of-the-art models.\"" }, { "slide_number": 40, "title": "HF Pipeline Options", "slide_body": [ "Automatic End-to-End Pipelines" ], "notes": [ "HuggingFacesupports a variety of models, and we’ll only be experimenting with a subset of its capabilities throughout this course.", "- Beyond NLP tasks like text classification, token classification, and question answering,HuggingFacealso provides pipelines for audio tasks like speech recognition, as well as vision tasks like object detection and image segmentation.", "- It also encompasses diffusers which we will use a bit later for image generation.", "**All-in-all, this should be enough to help you get started with Notebook 1, which should involve just making sure the environment is up and doing a simple model test to confirm that everything is working as intended.**" ], "raw_notes": "HuggingFacesupports a variety of models, and we’ll only be experimenting with a subset of its capabilities throughout this course.\n- Beyond NLP tasks like text classification, token classification, and question answering,HuggingFacealso provides pipelines for audio tasks like speech recognition, as well as vision tasks like object detection and image segmentation.\n- It also encompasses diffusers which we will use a bit later for image generation.\n**All-in-all, this should be enough to help you get started with Notebook 1, which should involve just making sure the environment is up and doing a simple model test to confirm that everything is working as intended.**" }, { "slide_number": 41, "title": "Part 1: From DL To LLMs", "slide_body": [ "•", "From Deep Learning to Language Modeling", "•", "Pulling In Our First Model", "•", "Model Architecture Intuitions", "•", "Using Encoders For Encoding" ], "notes": [ "- Welcome back to the course!", "- In the last section, we introduced you to Hugging Face and the Transformers library, helping you get comfortable with the environment.", "- Now, we’re going to dive deeper under the hood to explore sequence reasoning using transformers.", "- We’ll connect concepts from deep learning fundamentals to transformers, and show how they handle ordered sequences of classes.", "- Let’s get started!" ], "raw_notes": "- Welcome back to the course!\n- In the last section, we introduced you to Hugging Face and the Transformers library, helping you get comfortable with the environment.\n- Now, we’re going to dive deeper under the hood to explore sequence reasoning using transformers.\n- We’ll connect concepts from deep learning fundamentals to transformers, and show how they handle ordered sequences of classes.\n- Let’s get started!" }, { "slide_number": 42, "title": "Our Problem Space", "slide_body": [ "Ordered Sequence of Classes" ], "notes": [ "- **Let's recall the problem space.**", "- **We are transitioning from general sequence models to handling ordered sequences of classes.**", "- We know that models like BERT are effective for tasks like masked language modeling and sentence classification. But how are they doing this?", "- We need to dive into the architecture and see how transformers handle tokens and classes.”" ], "raw_notes": "- **Let's recall the problem space.**\n- **We are transitioning from general sequence models to handling ordered sequences of classes.**\n- We know that models like BERT are effective for tasks like masked language modeling and sentence classification. But how are they doing this?\n- We need to dive into the architecture and see how transformers handle tokens and classes.”" }, { "slide_number": 43, "title": "The Base Formulation", "slide_body": [ "Dealing With Language As An Input", "Per-Token Prediction", "Sequence-Level Prediction", "Generation (Forecasting)", "The Illustrated GPT-2 (Visualizing Transformer Language Models)" ], "notes": [ "- Let's recall the problem space.", "- We are transitioning from general sequence models to handling ordered sequences of classes.", "- **We know that models like BERT are effective for tasks like masked language modeling and sentence classification. But how are they doing this?**", "- **We need to dive into the architecture and see how transformers handle tokens and classes.**" ], "raw_notes": "- Let's recall the problem space.\n- We are transitioning from general sequence models to handling ordered sequences of classes.\n- **We know that models like BERT are effective for tasks like masked language modeling and sentence classification. But how are they doing this?**\n- **We need to dive into the architecture and see how transformers handle tokens and classes.**" }, { "slide_number": 44, "title": "Dealing With Classes", "slide_body": [ "Ordered Sequence of Classes" ], "notes": [ "We can predict classes from a variety of inputs, just like in standard deep learning.", "- The key is to map input data to classes and train the network to reinforce this mapping.", "- When generating tokens, we can extend this same principle.", "- Essentially, the transformer processes tokens as classes, predicting probabilities for each token.", "- This is the foundation of token classification." ], "raw_notes": "We can predict classes from a variety of inputs, just like in standard deep learning.\n- The key is to map input data to classes and train the network to reinforce this mapping.\n- When generating tokens, we can extend this same principle.\n- Essentially, the transformer processes tokens as classes, predicting probabilities for each token.\n- This is the foundation of token classification." }, { "slide_number": 45, "title": "Dealing With Classes", "slide_body": [ "Ordered Sequence of Classes" ], "notes": [ "- **While we know that this class-based approach works for outputting tokens, does it work as well for taking them in?**", "- **It turns out that it still works, but there may be better ways to take in high-dimensional vocabulary vectors.**", "- Instead of dense layers, which become problematic with large vocabularies, we use embedding layers.", "- Embedding layers map tokens to dense vector representations, reducing computational overhead.", "- This is more efficient for handling the large number of possible tokens in tasks like natural language processing, where we may have tens or hundreds of thousands of unique tokens." ], "raw_notes": "- **While we know that this class-based approach works for outputting tokens, does it work as well for taking them in?**\n- **It turns out that it still works, but there may be better ways to take in high-dimensional vocabulary vectors.**\n- Instead of dense layers, which become problematic with large vocabularies, we use embedding layers.\n- Embedding layers map tokens to dense vector representations, reducing computational overhead.\n- This is more efficient for handling the large number of possible tokens in tasks like natural language processing, where we may have tens or hundreds of thousands of unique tokens." }, { "slide_number": 46, "title": "Dealing With Classes", "slide_body": [ "Ordered Sequence of Classes" ], "notes": [ "- While we know that this class-based approach works for outputting tokens, does it work as well for taking them in?", "- It turns out that it still works, but there may be better ways to take in high-dimensional vocabulary vectors.", "- **Instead of dense layers, which become problematic with large vocabularies, we use embedding layers.**", "- **Embedding layers map tokens to dense vector representations, reducing computational overhead.**", "- **This is more efficient for handling the large number of possible tokens in tasks like natural language processing, where we may have tens or hundreds of thousands of unique tokens.**" ], "raw_notes": "- While we know that this class-based approach works for outputting tokens, does it work as well for taking them in?\n- It turns out that it still works, but there may be better ways to take in high-dimensional vocabulary vectors.\n- **Instead of dense layers, which become problematic with large vocabularies, we use embedding layers.**\n- **Embedding layers map tokens to dense vector representations, reducing computational overhead.**\n- **This is more efficient for handling the large number of possible tokens in tasks like natural language processing, where we may have tens or hundreds of thousands of unique tokens.**" }, { "slide_number": 47, "title": "Dealing With Classes", "slide_body": [ "Ordered Sequence of Classes" ], "notes": [ "- **We can think of sequences as many classes linked together. A good analogy is how convolutional layers process grids of pixels in images.**", "- **We know from deep learning that treating a large image as a single block is not a good idea, and the intuition persists here:**", " - **we probably don’t want to reason about text using densely-connected architectures.**", "- Instead, position-wise feed-forward networks in transformers process sequences token by token. This helps break down the reasoning process at the token level, which will be much easier to train." ], "raw_notes": "- **We can think of sequences as many classes linked together. A good analogy is how convolutional layers process grids of pixels in images.**\n- **We know from deep learning that treating a large image as a single block is not a good idea, and the intuition persists here:**\n - **we probably don’t want to reason about text using densely-connected architectures.**\n- Instead, position-wise feed-forward networks in transformers process sequences token by token. This helps break down the reasoning process at the token level, which will be much easier to train." }, { "slide_number": 48, "title": "Dealing With Classes", "slide_body": [ "Ordered Sequence of Classes" ], "notes": [ "- We can think of sequences as many classes linked together. A good analogy is how convolutional layers process grids of pixels in images.", "- We know from deep learning that treating a large image as a single block is not a good idea, and the intuition persists here:", " - we probably don’t want to reason about text using densely-connected architectures.", "- **Instead, position-wise feed-forward networks in transformers process sequences token by token. This helps break down the reasoning process at the token level, which will be much easier to train.**" ], "raw_notes": "- We can think of sequences as many classes linked together. A good analogy is how convolutional layers process grids of pixels in images.\n- We know from deep learning that treating a large image as a single block is not a good idea, and the intuition persists here:\n - we probably don’t want to reason about text using densely-connected architectures.\n- **Instead, position-wise feed-forward networks in transformers process sequences token by token. This helps break down the reasoning process at the token level, which will be much easier to train.**" }, { "slide_number": 49, "title": "Dealing With Ordered Classes", "slide_body": [ "Ordered Sequence of Classes", "Per-Token Predictions", "Per-Token", "Feed-Forward" ], "notes": [ "**Token-level prediction is easy enough to train, but it won’t give us sequence-level reasoning.**", "**A step in the right direction may be to provide more information, like information about word ordering and maybe other info.**", "But it still isn’t quite enough…" ], "raw_notes": "**Token-level prediction is easy enough to train, but it won’t give us sequence-level reasoning.**\n**A step in the right direction may be to provide more information, like information about word ordering and maybe other info.**\nBut it still isn’t quite enough…" }, { "slide_number": 50, "title": "Dealing With Ordered Classes", "slide_body": [ "Per-Token Predictions", "Per-Token", "Feed-Forward", "Ordered Sequence of Classes", "Token-Level Reasoning Ok", "Sequence-Level Reasoning Necessary" ], "notes": [ "Token-level prediction is easy enough to train, but it won’t give us sequence-level reasoning.", "A step in the right direction may be to provide more information, like information about word ordering and maybe other info.", "**But it still isn’t quite enough…**" ], "raw_notes": "Token-level prediction is easy enough to train, but it won’t give us sequence-level reasoning.\nA step in the right direction may be to provide more information, like information about word ordering and maybe other info.\n**But it still isn’t quite enough…**" }, { "slide_number": 51, "title": "Transformers for Sequence Reasonings", "slide_body": [ "Ordered Sequence of Classes" ], "notes": [ "**To reason with ordered SEQUENCES of classes:**", "- **Start with your per-sequence-entry feed-forward. Good for per-entry logic, but insufficient.**", "- You really want per-entry logic to persist, so why not add it as a residual connection?", "- From your input embeddings, derive 3 matrices, each being per-entry.", "- Use the key and value to compute attention (more details in notebook)", " - Lightweight interface produces a square matrix; self-attention", "- Feed the sequence-level logic back into the per-token pathway by multiplying with softmax-ed attention.", "- These components are known as transformer encoder blocks, and they generate per-token encodings with sequence-level knowledge" ], "raw_notes": "**To reason with ordered SEQUENCES of classes:**\n- **Start with your per-sequence-entry feed-forward. Good for per-entry logic, but insufficient.**\n- You really want per-entry logic to persist, so why not add it as a residual connection?\n- From your input embeddings, derive 3 matrices, each being per-entry.\n- Use the key and value to compute attention (more details in notebook)\n - Lightweight interface produces a square matrix; self-attention\n- Feed the sequence-level logic back into the per-token pathway by multiplying with softmax-ed attention.\n- These components are known as transformer encoder blocks, and they generate per-token encodings with sequence-level knowledge" }, { "slide_number": 52, "title": "Transformers for Sequence Reasonings", "slide_body": [ "Ordered Sequence of Classes" ], "notes": [ "**To reason with ordered SEQUENCES of classes:**", "- Start with your per-sequence-entry feed-forward. Good for per-entry logic, but insufficient.", "- **You really want per-entry logic to persist, so why not add it as a residual connection?**", "- From your input embeddings, derive 3 matrices, each being per-entry.", "- Use the key and value to compute attention (more details in notebook)", " - Lightweight interface produces a square matrix; self-attention", "- Feed the sequence-level logic back into the per-token pathway by multiplying with softmax-ed attention.", "- These components are known as transformer encoder blocks, and they generate per-token encodings with sequence-level knowledge" ], "raw_notes": "**To reason with ordered SEQUENCES of classes:**\n- Start with your per-sequence-entry feed-forward. Good for per-entry logic, but insufficient.\n- **You really want per-entry logic to persist, so why not add it as a residual connection?**\n- From your input embeddings, derive 3 matrices, each being per-entry.\n- Use the key and value to compute attention (more details in notebook)\n - Lightweight interface produces a square matrix; self-attention\n- Feed the sequence-level logic back into the per-token pathway by multiplying with softmax-ed attention.\n- These components are known as transformer encoder blocks, and they generate per-token encodings with sequence-level knowledge" }, { "slide_number": 53, "title": "Transformers for Sequence Reasonings", "slide_body": [ "Ordered Sequence of Classes" ], "notes": [ "**To reason with ordered SEQUENCES of classes:**", "- Start with your per-sequence-entry feed-forward. Good for per-entry logic, but insufficient.", "- **From your input embeddings, derive 3 matrices, each being per-entry.**", " - **Key and Query extracted to a latent dimension of choice**", " - **Value extracted to whatever dimension (why not keep it the same as input dimension for simplicity)**", " - **Move some of your per-token dense logic after the value computation. Why not?**", "- Use the key and value to compute attention (more details in notebook)", " - Lightweight interface produces a square matrix; self-attention", "- Feed the sequence-level logic back into the per-token pathway by multiplying with softmax-ed attention.", "- These components are known as transformer encoder blocks, and they generate per-token encodings with sequence-level knowledge" ], "raw_notes": "**To reason with ordered SEQUENCES of classes:**\n- Start with your per-sequence-entry feed-forward. Good for per-entry logic, but insufficient.\n- **From your input embeddings, derive 3 matrices, each being per-entry.**\n - **Key and Query extracted to a latent dimension of choice**\n - **Value extracted to whatever dimension (why not keep it the same as input dimension for simplicity)**\n - **Move some of your per-token dense logic after the value computation. Why not?**\n- Use the key and value to compute attention (more details in notebook)\n - Lightweight interface produces a square matrix; self-attention\n- Feed the sequence-level logic back into the per-token pathway by multiplying with softmax-ed attention.\n- These components are known as transformer encoder blocks, and they generate per-token encodings with sequence-level knowledge" }, { "slide_number": 54, "title": "Transformers for Sequence Reasonings", "slide_body": [ "Ordered Sequence of Classes" ], "notes": [ "**To reason with ordered SEQUENCES of classes:**", "- Start with your per-sequence-entry feed-forward. Good for per-entry logic, but insufficient.", "- From your input embeddings, derive 3 matrices, each being per-entry.", "- **Use the key and value to compute attention (more details in notebook)**", " - **Lightweight interface produces a square matrix; self-attention**", "- Feed the sequence-level logic back into the per-token pathway by multiplying with softmax-ed attention.", "- These components are known as transformer encoder blocks, and they generate per-token encodings with sequence-level knowledge" ], "raw_notes": "**To reason with ordered SEQUENCES of classes:**\n- Start with your per-sequence-entry feed-forward. Good for per-entry logic, but insufficient.\n- From your input embeddings, derive 3 matrices, each being per-entry.\n- **Use the key and value to compute attention (more details in notebook)**\n - **Lightweight interface produces a square matrix; self-attention**\n- Feed the sequence-level logic back into the per-token pathway by multiplying with softmax-ed attention.\n- These components are known as transformer encoder blocks, and they generate per-token encodings with sequence-level knowledge" }, { "slide_number": 55, "title": "Transformers for Sequence Reasonings", "slide_body": [ "Ordered Sequence of Classes" ], "notes": [ "**To reason with ordered SEQUENCES of classes:**", "- Start with your per-sequence-entry feed-forward. Good for per-entry logic, but insufficient.", "- From your input embeddings, derive 3 matrices, each being per-entry.", "- Use the key and value to compute attention (more details in notebook)", " - Lightweight interface produces a square matrix; self-attention", "- **Feed the sequence-level logic back into the per-token pathway by multiplying with softmax-ed attention.**", "- These components are known as transformer encoder blocks, and they generate per-token encodings with sequence-level knowledge" ], "raw_notes": "**To reason with ordered SEQUENCES of classes:**\n- Start with your per-sequence-entry feed-forward. Good for per-entry logic, but insufficient.\n- From your input embeddings, derive 3 matrices, each being per-entry.\n- Use the key and value to compute attention (more details in notebook)\n - Lightweight interface produces a square matrix; self-attention\n- **Feed the sequence-level logic back into the per-token pathway by multiplying with softmax-ed attention.**\n- These components are known as transformer encoder blocks, and they generate per-token encodings with sequence-level knowledge" }, { "slide_number": 56, "title": "Transformers for Sequence Reasonings", "slide_body": [ "Ordered Sequence of Classes" ], "notes": [ "**To reason with ordered SEQUENCES of classes:**", "- Start with your per-sequence-entry feed-forward. Good for per-entry logic, but insufficient.", "- From your input embeddings, derive 3 matrices, each being per-entry.", "- Use the key and value to compute attention (more details in notebook)", " - Lightweight interface produces a square matrix; self-attention", "- **Feed the sequence-level logic back into the per-token pathway by multiplying with softmax-ed attention.**", "- These components are known as transformer encoder blocks, and they generate per-token encodings with sequence-level knowledge" ], "raw_notes": "**To reason with ordered SEQUENCES of classes:**\n- Start with your per-sequence-entry feed-forward. Good for per-entry logic, but insufficient.\n- From your input embeddings, derive 3 matrices, each being per-entry.\n- Use the key and value to compute attention (more details in notebook)\n - Lightweight interface produces a square matrix; self-attention\n- **Feed the sequence-level logic back into the per-token pathway by multiplying with softmax-ed attention.**\n- These components are known as transformer encoder blocks, and they generate per-token encodings with sequence-level knowledge" }, { "slide_number": 57, "title": "Transformers for Sequence Reasonings", "slide_body": [ "Ordered Sequence of Classes" ], "notes": [ "**To reason with ordered SEQUENCES of classes:**", "- Start with your per-sequence-entry feed-forward. Good for per-entry logic, but insufficient.", "- From your input embeddings, derive 3 matrices, each being per-entry.", "- Use the key and value to compute attention (more details in notebook)", " - Lightweight interface produces a square matrix; self-attention", "- Feed the sequence-level logic back into the per-token pathway by multiplying withsoftmax-ed attention.", "- **These components are known as transformer encoder blocks, and they generate per-token encodings with sequence-level knowledge**", "(Next slide): This leads to entry-on-entry attention mechanism!" ], "raw_notes": "**To reason with ordered SEQUENCES of classes:**\n- Start with your per-sequence-entry feed-forward. Good for per-entry logic, but insufficient.\n- From your input embeddings, derive 3 matrices, each being per-entry.\n- Use the key and value to compute attention (more details in notebook)\n - Lightweight interface produces a square matrix; self-attention\n- Feed the sequence-level logic back into the per-token pathway by multiplying withsoftmax-ed attention.\n- **These components are known as transformer encoder blocks, and they generate per-token encodings with sequence-level knowledge**\n(Next slide): This leads to entry-on-entry attention mechanism!" }, { "slide_number": 58, "title": "Transformers Intuition", "slide_body": [ "Attention Is All You Need", "Transformer: A Novel Neural Network Architecture for Language Understandin (2017)" ], "notes": [ "**This leads to entry-on-entry attention mechanism!**", "(Next slide: why is this good)" ], "raw_notes": "**This leads to entry-on-entry attention mechanism!**\n(Next slide: why is this good)" }, { "slide_number": 59, "title": "Transformer Value Add", "slide_body": [ "Lightweight Entry Interface via Attention", "Residual Connection To Enforce Ordering", "Interpretable and Transferable Representation", "Natural Sequence Reasoners" ], "notes": [ "**Why is it good? Read off the slide.**", "(Next slide: How do the big models use attention?)" ], "raw_notes": "**Why is it good? Read off the slide.**\n(Next slide: How do the big models use attention?)" }, { "slide_number": 60, "title": "“Encoder” Transformers In Practice", "slide_body": [ "What Does SOTA Look Like?", "BERT 101 – Hugging Face" ], "notes": [ "**Big models don’t have one attention mechanism, but many multi-headed attention mechanisms, and they’re trained for bigger things.**", "- Encoder-style architectures like BERT are giant, trained on large datasets to predict masked out tokens.", "- They are used as backbones to perform more complex reasoning." ], "raw_notes": "**Big models don’t have one attention mechanism, but many multi-headed attention mechanisms, and they’re trained for bigger things.**\n- Encoder-style architectures like BERT are giant, trained on large datasets to predict masked out tokens.\n- They are used as backbones to perform more complex reasoning." }, { "slide_number": 61, "title": "BERT Model", "slide_body": [ "Encoder: Encodes n-sequence into n-encoding.", "Trained on Masked Language Modeling (MLM).", "Internet-Scale Pre-Training.", "Just embeddings and transformer blocks.", "Used as language-reasoning backbone.", "Powerful Encoder Model", "https://arxiv.org/pdf/1810.04805v2.pdf", "Pretraining", "Fine-tuning", "BERT: Pre-training of Deep Bidirectional Transformers for Language Understanding (2019)" ], "notes": [ "Big models don’t have one attention mechanism, but many multi-headed attention mechanisms, and they’re trained for bigger things.", "- **Encoder-style architectures like BERT are giant, trained on large datasets to predict masked out tokens.**", "- **They are used as backbones to perform more complex reasoning.**", "What do they do that makes them work well for such complex tasks?" ], "raw_notes": "Big models don’t have one attention mechanism, but many multi-headed attention mechanisms, and they’re trained for bigger things.\n- **Encoder-style architectures like BERT are giant, trained on large datasets to predict masked out tokens.**\n- **They are used as backbones to perform more complex reasoning.**\nWhat do they do that makes them work well for such complex tasks?" }, { "slide_number": 62, "title": "From Transformer to BERT", "slide_body": [ "Necessary Modifications" ], "notes": [ "**What do they do that makes them work well for such complex tasks?**", "- Grow the network width by adding more attention mechanisms per layer.", "- Add more transformer layers to grow deep network depth.", "- Add regularization efforts to make training smoother and overfitting less likely.", "- Add residual connections to help enforce per-token reasoning and reduce vanishing gradient." ], "raw_notes": "**What do they do that makes them work well for such complex tasks?**\n- Grow the network width by adding more attention mechanisms per layer.\n- Add more transformer layers to grow deep network depth.\n- Add regularization efforts to make training smoother and overfitting less likely.\n- Add residual connections to help enforce per-token reasoning and reduce vanishing gradient." }, { "slide_number": 63, "title": "From Transformer to BERT", "slide_body": [ "Problem: Need Deeper Reasoning", "On same data representation? Multi-Headed Attention", "Necessary Modifications" ], "notes": [ "What do they do that makes them work well for such complex tasks?", "- **Grow the network width by adding more attention mechanisms per layer.**", "- Add more transformer layers to grow deep network depth.", "- Add regularization efforts to make training smoother and overfitting less likely.", "- Add residual connections to help enforce per-token reasoning and reduce vanishing gradient." ], "raw_notes": "What do they do that makes them work well for such complex tasks?\n- **Grow the network width by adding more attention mechanisms per layer.**\n- Add more transformer layers to grow deep network depth.\n- Add regularization efforts to make training smoother and overfitting less likely.\n- Add residual connections to help enforce per-token reasoning and reduce vanishing gradient." }, { "slide_number": 64, "title": "From Transformer to BERT", "slide_body": [ "Problem: Need Deeper Reasoning", "On same data representation? Multi-Headed Attention", "Need more representations? Multiple Transformer Layers", "Necessary Modifications" ], "notes": [ "What do they do that makes them work well for such complex tasks?", "- Grow the network width by adding more attention mechanisms per layer.", "- **Add more transformer layers to grow deep network depth.**", "- Add regularization efforts to make training smoother and overfitting less likely.", "- Add residual connections to help enforce per-token reasoning and reduce vanishing gradient." ], "raw_notes": "What do they do that makes them work well for such complex tasks?\n- Grow the network width by adding more attention mechanisms per layer.\n- **Add more transformer layers to grow deep network depth.**\n- Add regularization efforts to make training smoother and overfitting less likely.\n- Add residual connections to help enforce per-token reasoning and reduce vanishing gradient." }, { "slide_number": 65, "title": "From Transformer to BERT", "slide_body": [ "Problem: Need Deeper Reasoning", "On same data representation? Multi-Headed Attention", "Need more representations? Multiple Transformer Layers", "Problem: Volatile gradients behavior, distribution shifts.", "Layer Normalization (as seen in BERT)", "Gaussian Error Linear Unit (GELU) (as seen in BERT)", "Dropout During Training (as seen in BERT)", "Necessary Modifications" ], "notes": [ "What do they do that makes them work well for such complex tasks?", "- Grow the network width by adding more attention mechanisms per layer.", "- Add more transformer layers to grow deep network depth.", "- **Add regularization efforts to make training smoother and overfitting less likely.**", "- Add residual connections to help enforce per-token reasoning and reduce vanishing gradient." ], "raw_notes": "What do they do that makes them work well for such complex tasks?\n- Grow the network width by adding more attention mechanisms per layer.\n- Add more transformer layers to grow deep network depth.\n- **Add regularization efforts to make training smoother and overfitting less likely.**\n- Add residual connections to help enforce per-token reasoning and reduce vanishing gradient." }, { "slide_number": 66, "title": "From Transformer to BERT", "slide_body": [ "Problem: Need Deeper Reasoning", "On same data representation? Multi-Headed Attention", "Need more representations? Multiple Transformer Layers", "Problem: Volatile gradients behavior, distribution shifts.", "Layer Normalization (as seen in BERT)", "Gaussian Error Linear Unit (GELU) (as seen in BERT)", "Dropout During Training (as seen in BERT)", "Problem: Network too deep; Gradients are vanishing", "Residual Connections", "Necessary Modifications" ], "notes": [ "What do they do that makes them work well for such complex tasks?", "- Grow the network width by adding more attention mechanisms per layer.", "- Add more transformer layers to grow deep network depth.", "- Add regularization efforts to make training smoother and overfitting less likely.", "- **Add residual connections to help enforce per-token reasoning and reduce vanishing gradient.**", "Now you have a BERT-like model!" ], "raw_notes": "What do they do that makes them work well for such complex tasks?\n- Grow the network width by adding more attention mechanisms per layer.\n- Add more transformer layers to grow deep network depth.\n- Add regularization efforts to make training smoother and overfitting less likely.\n- **Add residual connections to help enforce per-token reasoning and reduce vanishing gradient.**\nNow you have a BERT-like model!" }, { "slide_number": 67, "title": "BERT Constructed", "slide_body": [ "From Transformer to BERT", "BERT 101 – Hugging Face" ], "notes": [ "**Now you have a BERT-like model!**", "**So please check it out in the Notebook 2!**" ], "raw_notes": "**Now you have a BERT-like model!**\n**So please check it out in the Notebook 2!**" }, { "slide_number": 68, "title": "Part 1: From DL To LLMs", "slide_body": [ "•", "From Deep Learning to Language Modeling", "•", "Pulling In Our First Model", "•", "Model Architecture Intuitions", "•", "Using Encoders For Encoding" ], "notes": [ "We’ve pulling in some models, experimented with them, and now have an understanding of the architecture and intuitions!", "**Now let’s see how this backbone is used to solve other tasks aside from Masked Language Modeling!**", "For this part, we’re going to start off by considering the basic NLP tasks listed in the graphic!" ], "raw_notes": "We’ve pulling in some models, experimented with them, and now have an understanding of the architecture and intuitions!\n**Now let’s see how this backbone is used to solve other tasks aside from Masked Language Modeling!**\nFor this part, we’re going to start off by considering the basic NLP tasks listed in the graphic!" }, { "slide_number": 69, "title": "HF Pipeline Options", "slide_body": [ "Automatic End-to-End Pipelines" ], "notes": [ "We’ve pulling in some models, experimented with them, and now have an understanding of the architecture and intuitions!", "Now let’s see how this backbone is used to solve other tasks aside from Masked Language Modeling!", "**For this part, we’re going to start off by considering the basic NLP tasks!**", "- Feature extraction: Just semantically-dense embeddings of the text that comes in. Sequence-in sequence-out, not human-interpretable.", "- Token classification: From a semantically-rich embedding, just predict something for each token, and we have token classification.", " - If the output space is the space of tokens and we’re trying to predict unknown words, then this is mask filling (like most encoder cores)", "- Text classification: Alternatively, query the model out of just a single sequence entry, and perhaps you can derive insight about the whole passage?" ], "raw_notes": "We’ve pulling in some models, experimented with them, and now have an understanding of the architecture and intuitions!\nNow let’s see how this backbone is used to solve other tasks aside from Masked Language Modeling!\n**For this part, we’re going to start off by considering the basic NLP tasks!**\n- Feature extraction: Just semantically-dense embeddings of the text that comes in. Sequence-in sequence-out, not human-interpretable.\n- Token classification: From a semantically-rich embedding, just predict something for each token, and we have token classification.\n - If the output space is the space of tokens and we’re trying to predict unknown words, then this is mask filling (like most encoder cores)\n- Text classification: Alternatively, query the model out of just a single sequence entry, and perhaps you can derive insight about the whole passage?" }, { "slide_number": 70, "title": "Natural Encoder Tasks", "slide_body": [ "From n-sequence Encodings", "feature extraction", "token classification", "mask filling", "text classification", "Tasks | Hugging Face" ], "notes": [ "Now let’s see how this backbone is used to solve other tasks aside from Masked Language Modeling!", "**For this part, we’re going to start off by considering the basic NLP tasks!**", "- **Feature extraction: Just semantically-dense embeddings of the text that comes in. Sequence-in sequence-out, not human-interpretable.**", "- Token classification: From a semantically-rich embedding, just predict something for each token, and we have token classification.", " - If the output space is the space of tokens and we’re trying to predict unknown words, then this is mask filling (like most encoder cores)", "- Text classification: Alternatively, query the model out of just a single sequence entry, and perhaps you can derive insight about the whole passage?" ], "raw_notes": "Now let’s see how this backbone is used to solve other tasks aside from Masked Language Modeling!\n**For this part, we’re going to start off by considering the basic NLP tasks!**\n- **Feature extraction: Just semantically-dense embeddings of the text that comes in. Sequence-in sequence-out, not human-interpretable.**\n- Token classification: From a semantically-rich embedding, just predict something for each token, and we have token classification.\n - If the output space is the space of tokens and we’re trying to predict unknown words, then this is mask filling (like most encoder cores)\n- Text classification: Alternatively, query the model out of just a single sequence entry, and perhaps you can derive insight about the whole passage?" }, { "slide_number": 71, "title": "Natural Encoder Tasks", "slide_body": [ "From n-sequence Encodings", "feature extraction", "token classification", "mask filling", "text classification", "Tasks | Hugging Face" ], "notes": [ "Now let’s see how this backbone is used to solve other tasks aside from Masked Language Modeling!", "**For this part, we’re going to start off by considering the basic NLP tasks!**", "- Feature extraction: Just semantically-dense embeddings of the text that comes in. Sequence-in sequence-out, not human-interpretable.", "- **Token classification: From a semantically-rich embedding, just predict something for each token, and we have token classification.**", " - If the output space is the space of tokens and we’re trying to predict unknown words, then this is mask filling (like most encoder cores)", "- Text classification: Alternatively, query the model out of just a single sequence entry, and perhaps you can derive insight about the whole passage?" ], "raw_notes": "Now let’s see how this backbone is used to solve other tasks aside from Masked Language Modeling!\n**For this part, we’re going to start off by considering the basic NLP tasks!**\n- Feature extraction: Just semantically-dense embeddings of the text that comes in. Sequence-in sequence-out, not human-interpretable.\n- **Token classification: From a semantically-rich embedding, just predict something for each token, and we have token classification.**\n - If the output space is the space of tokens and we’re trying to predict unknown words, then this is mask filling (like most encoder cores)\n- Text classification: Alternatively, query the model out of just a single sequence entry, and perhaps you can derive insight about the whole passage?" }, { "slide_number": 72, "title": "Natural Encoder Tasks", "slide_body": [ "From n-sequence Encodings", "feature extraction", "token classification", "mask filling", "text classification", "Tasks | Hugging Face" ], "notes": [ "Now let’s see how this backbone is used to solve other tasks aside from Masked Language Modeling!", "**For this part, we’re going to start off by considering the basic NLP tasks!**", "- Feature extraction: Just semantically-dense embeddings of the text that comes in. Sequence-in sequence-out, not human-interpretable.", "- Token classification: From a semantically-rich embedding, just predict something for each token, and we have token classification.", " - **If the output space is the space of tokens and we’re trying to predict unknown words, then this is mask filling (like most encoder cores)**", "- Text classification: Alternatively, query the model out of just a single sequence entry, and perhaps you can derive insight about the whole passage?" ], "raw_notes": "Now let’s see how this backbone is used to solve other tasks aside from Masked Language Modeling!\n**For this part, we’re going to start off by considering the basic NLP tasks!**\n- Feature extraction: Just semantically-dense embeddings of the text that comes in. Sequence-in sequence-out, not human-interpretable.\n- Token classification: From a semantically-rich embedding, just predict something for each token, and we have token classification.\n - **If the output space is the space of tokens and we’re trying to predict unknown words, then this is mask filling (like most encoder cores)**\n- Text classification: Alternatively, query the model out of just a single sequence entry, and perhaps you can derive insight about the whole passage?" }, { "slide_number": 73, "title": "Natural Encoder Tasks", "slide_body": [ "From n-sequence Encodings", "feature extraction", "token classification", "mask filling", "text classification", "Tasks | Hugging Face" ], "notes": [ "Now let’s see how this backbone is used to solve other tasks aside from Masked Language Modeling!", "**For this part, we’re going to start off by considering the basic NLP tasks!**", "- Feature extraction: Just semantically-dense embeddings of the text that comes in. Sequence-in sequence-out, not human-interpretable.", "- Token classification: From a semantically-rich embedding, just predict something for each token, and we have token classification.", " - If the output space is the space of tokens and we’re trying to predict unknown words, then this is mask filling (like most encoder cores)", "- **Text classification: Alternatively, query the model out of just a single sequence entry, and perhaps you can derive insight about the whole passage?**", "Turns out, all of this can be achieved from the same underlying architectural backbone!" ], "raw_notes": "Now let’s see how this backbone is used to solve other tasks aside from Masked Language Modeling!\n**For this part, we’re going to start off by considering the basic NLP tasks!**\n- Feature extraction: Just semantically-dense embeddings of the text that comes in. Sequence-in sequence-out, not human-interpretable.\n- Token classification: From a semantically-rich embedding, just predict something for each token, and we have token classification.\n - If the output space is the space of tokens and we’re trying to predict unknown words, then this is mask filling (like most encoder cores)\n- **Text classification: Alternatively, query the model out of just a single sequence entry, and perhaps you can derive insight about the whole passage?**\nTurns out, all of this can be achieved from the same underlying architectural backbone!" }, { "slide_number": 74, "title": "Common Encoder Backbone", "slide_body": [ "From n-sequence Encodings", "BERT: Pre-training of Deep Bidirectional Transformers for Language Understanding (2019)" ], "notes": [ "**Turns out, all of this can be achieved from the same underlying architectural backbone!**", "**It’s just either a subset (feature ext), entirety of (mask fill), or added layer on top of (text/token class) a single pretrained LLM**(or maybe derivatives of one)" ], "raw_notes": "**Turns out, all of this can be achieved from the same underlying architectural backbone!**\n**It’s just either a subset (feature ext), entirety of (mask fill), or added layer on top of (text/token class) a single pretrained LLM**(or maybe derivatives of one)" }, { "slide_number": 75, "title": "Common Encoder Backbone", "slide_body": [ "From n-sequence Encodings", "BERT: Pre-training of Deep Bidirectional Transformers for Language Understanding (2019)" ], "notes": [ "Turns out, all of this can be achieved from the same underlying architectural backbone!", "It’s just either a subset (feature ext), entirety of (mask fill), or added layer on top of (text/token class) a single pretrained LLM**(or maybe derivatives of one)**", "**Let’s talk about some of the derivatives!**" ], "raw_notes": "Turns out, all of this can be achieved from the same underlying architectural backbone!\nIt’s just either a subset (feature ext), entirety of (mask fill), or added layer on top of (text/token class) a single pretrained LLM**(or maybe derivatives of one)**\n**Let’s talk about some of the derivatives!**" }, { "slide_number": 76, "title": "Encoder Model Options", "slide_body": [ "Common Derivatives/Flavors" ], "notes": [ "**Let’s talk about some of the derivatives!**", "(See slides)" ], "raw_notes": "**Let’s talk about some of the derivatives!**\n(See slides)" }, { "slide_number": 77, "title": "Encoder Model Options", "slide_body": [ "Common Derivatives/Flavors" ], "notes": [ "**Let’s talk about some of the derivatives!**", "(See slides)" ], "raw_notes": "**Let’s talk about some of the derivatives!**\n(See slides)" }, { "slide_number": 78, "title": "Encoder Model Options", "slide_body": [ "Common Derivatives/Flavors" ], "notes": [ "**Let’s talk about some of the derivatives!**", "(See slides)", "NEXT: Now let’s talk about what they all have in common: What they’re good for!" ], "raw_notes": "**Let’s talk about some of the derivatives!**\n(See slides)\nNEXT: Now let’s talk about what they all have in common: What they’re good for!" }, { "slide_number": 79, "title": "BERT Benefits", "slide_body": [ "Key Features: Simple and Small, But Powerful", "Simple input-output schema; strong supervision.", "Top-performing models usually <1B parameters.", "Transferable embeddings/attention interfaces.", "Constant bidirectional enforcement", "BEST WHEN", "When n->n, n->1, or subsetting is sufficient.", "When BERT-Style Encoders Shine", "BERT 101 – Hugging Face" ], "notes": [ "**Now let’s talk about what they all have in common: What they’re good for!**", "(in slides)", "NEXT: But what about when you want to predict multiple things, but you don’t want them to be n->n or n->1? You can do zero-shot classification!" ], "raw_notes": "**Now let’s talk about what they all have in common: What they’re good for!**\n(in slides)\nNEXT: But what about when you want to predict multiple things, but you don’t want them to be n->n or n->1? You can do zero-shot classification!" }, { "slide_number": 80, "title": "Multiple-Query Pipelines", "slide_body": [ "Towards Zero-Shot Classification for Any Class", "Dune is the best movie ever", "This is about Cinema -> 0.6", "This is about Art -> 0.3", "This is about Music -> 0.1", "Contradiction/Entailment Task" ], "notes": [ "But what about when you want to predict multiple things, but you don’t want them to be n->n or n->1?", "**You can do zero-shot classification! Derived from contradiction/entailment task.**", "**Great for pulling m classes, one at a time, from an n-entry sequence…**", "but doesn’t generate sequences…" ], "raw_notes": "But what about when you want to predict multiple things, but you don’t want them to be n->n or n->1?\n**You can do zero-shot classification! Derived from contradiction/entailment task.**\n**Great for pulling m classes, one at a time, from an n-entry sequence…**\nbut doesn’t generate sequences…" }, { "slide_number": 81, "title": "Variable-Length Outputs", "slide_body": [ "Range Selection Sufficient, But Limited", "Tasks | Hugging Face" ], "notes": [ "What aboutsubsettingsequences?", "**Sure, you can do that with range prediction. Predict for each sequence entry:**", "**- Whether the token is a start**", "**- Whether the token is an end**", "But what about when that’s not enough and we want more than a subset?" ], "raw_notes": "What aboutsubsettingsequences?\n**Sure, you can do that with range prediction. Predict for each sequence entry:**\n**- Whether the token is a start**\n**- Whether the token is an end**\nBut what about when that’s not enough and we want more than a subset?" }, { "slide_number": 82, "title": "BERT Benefits", "slide_body": [ "Key Features: Simple and Small, But Powerful", "Simple input-output schema; strong supervision.", "Top-performing models usually <1B parameters.", "Transferable embeddings/attention interfaces.", "Constant bidirectional enforcement", "BEST WHEN", "n->n, n->1, n->(1,1, ...).", "BUT NOT GOOD WHEN", "Need to generate new tokens/sequences.", "When BERT-Style Encoders Shine", "BERT 101 – Hugging Face" ], "notes": [ "Still, we’re running up on some limitations. In the next notebook, we’ll explore these limitations in more detail and will propose a better solution for novel text generation.", "**Start Notebook 3!!**" ], "raw_notes": "Still, we’re running up on some limitations. In the next notebook, we’ll explore these limitations in more detail and will propose a better solution for novel text generation.\n**Start Notebook 3!!**" }, { "slide_number": 83, "title": "Part 2: GenAI Models", "slide_body": [ "•", "Decoders for Sequence Forecasting", "•", "Multimodal Reasoning and Generation", "•", "GenAI Servers and Deployment" ], "notes": [ "- Welcome back to the course!", "- This section covers decoder models, which generate language sequences.", "- We'll contrast decoders with encoders and dive into language generation." ], "raw_notes": "- Welcome back to the course!\n- This section covers decoder models, which generate language sequences.\n- We'll contrast decoders with encoders and dive into language generation." }, { "slide_number": 84, "title": "Variable-Length Outputs", "slide_body": [ "Range Selection Sufficient, But Limited", "Tasks | Hugging Face" ], "notes": [ "We’re going from very natural n->n and n->1 tasks, and even a bit of n->m tasks…", "but let’s look at some new tasks that might not be so good for our architecture.", "**Question-answering is our first such task, but we said earlier that it’s actually partially solvable by range prediction.**", "**Added structure is good for small models!**" ], "raw_notes": "We’re going from very natural n->n and n->1 tasks, and even a bit of n->m tasks…\nbut let’s look at some new tasks that might not be so good for our architecture.\n**Question-answering is our first such task, but we said earlier that it’s actually partially solvable by range prediction.**\n**Added structure is good for small models!**" }, { "slide_number": 85, "title": "Translation Tasks", "slide_body": [ "Encoder-Only Insufficient for General m->n", "Tasks | Hugging Face" ], "notes": [ "We’re going from very natural n->n and n->1 tasks, and even a bit of n->m tasks…", "but let’s look at some new tasks that might not be so good for our architecture.", "**How about translation though:**", "- **Not necessarily 1-to-1 input-token-to-output-token correspondence.**", "- **n->n is a poor constraint. You can do it, but it’s complicated and we won’t talk about it.**", "**We need a new formulation!**" ], "raw_notes": "We’re going from very natural n->n and n->1 tasks, and even a bit of n->m tasks…\nbut let’s look at some new tasks that might not be so good for our architecture.\n**How about translation though:**\n- **Not necessarily 1-to-1 input-token-to-output-token correspondence.**\n- **n->n is a poor constraint. You can do it, but it’s complicated and we won’t talk about it.**\n**We need a new formulation!**" }, { "slide_number": 86, "title": "From Encoding to Decoding", "slide_body": [ "Predict ?->1 To Forecast n Predictions" ], "notes": [ "**We’ve been dealing with encoders, so going from a sequence to a latent sequence.**", "Now we go to Decoder: From a sequence, predict the next entry", "- Go back to zero-shot classification’s n->1 formulation", "- Train the model to predict a single token; the next token!" ], "raw_notes": "**We’ve been dealing with encoders, so going from a sequence to a latent sequence.**\nNow we go to Decoder: From a sequence, predict the next entry\n- Go back to zero-shot classification’s n->1 formulation\n- Train the model to predict a single token; the next token!" }, { "slide_number": 87, "title": "From Encoding to Decoding", "slide_body": [ "Predict ?->1 To Forecast n Predictions" ], "notes": [ "We’ve been dealing with encoders, so going from a sequence to a latent sequence.", "**Now we go to Decoder: From a sequence, predict the next entry**", "- **Go back to zero-shot classification’s n->1 formulation**", "- **Train the model to predict a single token; the next token!**", "Given this kind of formulation, we can generate text by predicting one word after another after another.", "Thereby, n->m." ], "raw_notes": "We’ve been dealing with encoders, so going from a sequence to a latent sequence.\n**Now we go to Decoder: From a sequence, predict the next entry**\n- **Go back to zero-shot classification’s n->1 formulation**\n- **Train the model to predict a single token; the next token!**\nGiven this kind of formulation, we can generate text by predicting one word after another after another.\nThereby, n->m." }, { "slide_number": 88, "title": "From Encoding to Decoding", "slide_body": [ "Predict ?->1 To Forecast n Predictions" ], "notes": [ "**Given this kind of formulation, we can generate text by predicting one word after another after another.**", "**Thereby, n->m.**", "Great, but isn’t this just n->1? Isn’t this just an encoder?" ], "raw_notes": "**Given this kind of formulation, we can generate text by predicting one word after another after another.**\n**Thereby, n->m.**\nGreat, but isn’t this just n->1? Isn’t this just an encoder?" }, { "slide_number": 89, "title": "Decoder Training Parallelization", "slide_body": [ "Why Unidirectional Reasoning Is Standard", "Exploring the Limits of Transfer Learning with a Unified Text-to-Text Transformer (2020)" ], "notes": [ "**Great, but isn’t this just n->1? Isn’t this just an encoder?**", "**We can just do it with an encoder, but it’s not a good idea:**", "- **For a dataset of strings, we’d be training the network for a lot of substrings to do 1->1, 2->1, 3->1 etc.**", "- **Otherwise, a large number of inputs would be OOD (out of domain) for the model. Same with short, medium, etc.**", "To solve this, we can restrict our encoder model to only reason about the previous sequence entries." ], "raw_notes": "**Great, but isn’t this just n->1? Isn’t this just an encoder?**\n**We can just do it with an encoder, but it’s not a good idea:**\n- **For a dataset of strings, we’d be training the network for a lot of substrings to do 1->1, 2->1, 3->1 etc.**\n- **Otherwise, a large number of inputs would be OOD (out of domain) for the model. Same with short, medium, etc.**\nTo solve this, we can restrict our encoder model to only reason about the previous sequence entries." }, { "slide_number": 90, "title": "Decoder Training Parallelization", "slide_body": [ "Why Unidirectional Reasoning Is Standard", "Exploring the Limits of Transfer Learning with a Unified Text-to-Text Transformer (2020)" ], "notes": [ "**Great, but isn’t this just n->1? Isn’t this just an encoder?**", "We can just do it with an encoder, but it’s not a good idea:", "- For a dataset of strings, we’d be training the network for a lot of substrings to do 1->1, 2->1, 3->1 etc.", "- Otherwise, a large number of inputs would be OOD (out of domain) for the model. Same with short, medium, etc.", "**To solve this, we can restrict our encoder model to only reason about the previous sequence entries.**", "- **Means single pass on long string is enough to train for all subsets.**", "- **All we need to do is mask the attention!**", "**This solves a lot of practical problems, though it makes our decoder weaker. Still, doesn’t solve all of our problems.**", "**Going from encoder to decoder solves our short-medium-long-range training domain problem, but what about super-long domain?**" ], "raw_notes": "**Great, but isn’t this just n->1? Isn’t this just an encoder?**\nWe can just do it with an encoder, but it’s not a good idea:\n- For a dataset of strings, we’d be training the network for a lot of substrings to do 1->1, 2->1, 3->1 etc.\n- Otherwise, a large number of inputs would be OOD (out of domain) for the model. Same with short, medium, etc.\n**To solve this, we can restrict our encoder model to only reason about the previous sequence entries.**\n- **Means single pass on long string is enough to train for all subsets.**\n- **All we need to do is mask the attention!**\n**This solves a lot of practical problems, though it makes our decoder weaker. Still, doesn’t solve all of our problems.**\n**Going from encoder to decoder solves our short-medium-long-range training domain problem, but what about super-long domain?**" }, { "slide_number": 91, "title": "Context Limits", "slide_body": [ "Why Autoregressing Indefinitely Is Not Great", "Like Training", "Out Of Distribution", "Slower Training For Longer Sequences" ], "notes": [ "To solve this, we can restrict our encoder model to only reason about the previous sequence entries.", "**This solves a lot of practical problems, though it makes our decoder weaker. Still, doesn’t solve all of our problems.**", "**Going from encoder to decoder solves our short-medium-long-range training domain problem, but what about super-long domain?**", "- **Can’t train for super-long because that requires a lot of compute and time.**", "- **This manifests in settling for training data length maximums, which creates context limits.**", "- **After context limits, generation goes beyond distribution and deteriorates rapidly.**" ], "raw_notes": "To solve this, we can restrict our encoder model to only reason about the previous sequence entries.\n**This solves a lot of practical problems, though it makes our decoder weaker. Still, doesn’t solve all of our problems.**\n**Going from encoder to decoder solves our short-medium-long-range training domain problem, but what about super-long domain?**\n- **Can’t train for super-long because that requires a lot of compute and time.**\n- **This manifests in settling for training data length maximums, which creates context limits.**\n- **After context limits, generation goes beyond distribution and deteriorates rapidly.**" }, { "slide_number": 92, "title": "Decoder-Guided Generation", "slide_body": [ "Continue The Thought", "HARD PROS", "Everything goes in one path", "Decoder gets much more training", "HARD CONS", "Needs way more training", "Needs to handle way more inputs" ], "notes": [ "- Pros: Decoders process everything in one path, gaining more training efficiency.", "- Cons: Requires more data and can lead to derailment if early predictions are off.", "- More training improves results, but the need for robust handling of inputs remains." ], "raw_notes": "- Pros: Decoders process everything in one path, gaining more training efficiency.\n- Cons: Requires more data and can lead to derailment if early predictions are off.\n- More training improves results, but the need for robust handling of inputs remains." }, { "slide_number": 93, "title": "Encoders vs Decoders", "slide_body": [ "Both Options Have Pros and Cons" ], "notes": [ "**So now we have out encoders and our decoders (read off the slide).**", "Both have limitations, so why not combine them?" ], "raw_notes": "**So now we have out encoders and our decoders (read off the slide).**\nBoth have limitations, so why not combine them?" }, { "slide_number": 94, "title": "Encoders vs Decoders", "slide_body": [ "Both Options Have Pros and Cons", "Both Have Limitations", "Why Not Combine Them?" ], "notes": [ "**So now we have out encoders and our decoders (read off the slide).**", "**Both have limitations, so why not combine them?**" ], "raw_notes": "**So now we have out encoders and our decoders (read off the slide).**\n**Both have limitations, so why not combine them?**" }, { "slide_number": 95, "title": "Encoder-Decoder", "slide_body": [ "Best of Both Worlds (Possibly)", "Attention is All You Need (2017)" ], "notes": [ "**Encoder-Decoder from Attention Is All You Need (2017)**", "- **Combines both encoder and decoder components together via an interface called cross-attention.**", "- **In other words, use an n-element sequence to condition an m-element sequence, resolving sequence length discrepancy.**", "- Feed an n-sequence key and value and an m-sequence query, and the math will work out! (N+M)->1 conditioning!" ], "raw_notes": "**Encoder-Decoder from Attention Is All You Need (2017)**\n- **Combines both encoder and decoder components together via an interface called cross-attention.**\n- **In other words, use an n-element sequence to condition an m-element sequence, resolving sequence length discrepancy.**\n- Feed an n-sequence key and value and an m-sequence query, and the math will work out! (N+M)->1 conditioning!" }, { "slide_number": 96, "title": "Encoder-Decoder", "slide_body": [ "Best of Both Worlds (Possibly)", "Attention is All You Need (2017)" ], "notes": [ "**Encoder-Decoder from Attention Is All You Need (2017)**", "- Combines both encoder and decoder components together via an interface called cross-attention.", "- In other words, use an n-element sequence to condition an m-element sequence, resolving sequence length discrepancy.", "- **Feed an n-sequence key and value and an m-sequence query, and the math will work out!**", " - Q is from encoder (n), so attn row is n.", " - K is transposed and from decoder (m), so attn column is m.", " - V is also from encoder (n), so row of V is n.", " - MultiplyMxNandNxKleadsMxK, where k is some latent value.", " - **Math Works! (N+M)->1 Conditioning**" ], "raw_notes": "**Encoder-Decoder from Attention Is All You Need (2017)**\n- Combines both encoder and decoder components together via an interface called cross-attention.\n- In other words, use an n-element sequence to condition an m-element sequence, resolving sequence length discrepancy.\n- **Feed an n-sequence key and value and an m-sequence query, and the math will work out!**\n - Q is from encoder (n), so attn row is n.\n - K is transposed and from decoder (m), so attn column is m.\n - V is also from encoder (n), so row of V is n.\n - MultiplyMxNandNxKleadsMxK, where k is some latent value.\n - **Math Works! (N+M)->1 Conditioning**" }, { "slide_number": 97, "title": "Encoder-Decoder Benefits", "slide_body": [ "Key Features: Powerful and Efficient", "Grounds harder-to-train decoder with context", "Static context also allows KV caching", "Leverages decoder’s forecasting formulation", "BEST WHEN", "Context and response are strongly separated", "BUT NOT IDEAL WHEN", "There is little-to-no context/response division", "When T5-Style Seq2Seq Shine", "Attention is All You Need (2017)" ], "notes": [ "Overall Encoder-Decoder benefits.", "- Decoder supervised with context from encoder which is held constant.", "- (Good time to note KV caching, where the encoder outputs are often-times not recomputed)", "- Takes advantage of decoder to generate sequences autoregressively.", "Best when context and response are strongly separated (i.e. vocabulary gap, semantic gap,etc)", "Not ideal when there is little-to-no context/response gap (i.e. same vocabulary, fuzzy boundaries)" ], "raw_notes": "Overall Encoder-Decoder benefits.\n- Decoder supervised with context from encoder which is held constant.\n- (Good time to note KV caching, where the encoder outputs are often-times not recomputed)\n- Takes advantage of decoder to generate sequences autoregressively.\nBest when context and response are strongly separated (i.e. vocabulary gap, semantic gap,etc)\nNot ideal when there is little-to-no context/response gap (i.e. same vocabulary, fuzzy boundaries)" }, { "slide_number": 98, "title": "Conversational Tasks", "slide_body": [ "Forecasting Sentences", "Tasks | Hugging Face" ], "notes": [ "**Text-generation and conversation are both forecasting tasks, one on text generation and the other on response.**", "**This is the area that has seen a lot of interest, so it’s a great one to finish off on.**" ], "raw_notes": "**Text-generation and conversation are both forecasting tasks, one on text generation and the other on response.**\n**This is the area that has seen a lot of interest, so it’s a great one to finish off on.**" }, { "slide_number": 99, "title": "Progressing Towards Zero-Shot", "slide_body": [ "Understanding Tasks Without Explicitly Training To Them", "Exploring the Limits of Transfer Learning with a Unified Text-to-Text Transformer (2020)" ], "notes": [ "We can train a variety of pipelines to do whatever seq2seq task we want.", "**However, this isn’t really how the modern seq2seq landscape has evolved.**", "**Instead of training single-task seq2seq pipelines, many modern formulations train a single model to do many tasks.**", "- **More language understanding in both the encoder and decoder.**", "- **Potential transferability of intuition from one task to another.**", "T5: An encoder-decoder that’s commonly trained on many tasks, both useful on their own and for core language understanding." ], "raw_notes": "We can train a variety of pipelines to do whatever seq2seq task we want.\n**However, this isn’t really how the modern seq2seq landscape has evolved.**\n**Instead of training single-task seq2seq pipelines, many modern formulations train a single model to do many tasks.**\n- **More language understanding in both the encoder and decoder.**\n- **Potential transferability of intuition from one task to another.**\nT5: An encoder-decoder that’s commonly trained on many tasks, both useful on their own and for core language understanding." }, { "slide_number": 100, "title": "Question-Understanding Seq2Seq", "slide_body": [ "Arbitrary Translation", "Exploring the Limits of Transfer Learning with a Unified Text-to-Text Transformer (2020)" ], "notes": [ "We can train a variety of pipelines to do whatever seq2seq task we want.", "**However, this isn’t really how the modern seq2seq landscape has evolved.**", "**Instead of training single-task seq2seq pipelines, many modern formulations train a single model to do many tasks.**", "**T5: An encoder-decoder that’s commonly trained on many tasks, both useful on their own and for core language understanding.**", "**The goal is for eventual understanding of the language and nuanced meaning of encoder inputs to generate decoder outputs.**", "(But the T5 on its own is insufficient. It does not transfer well beyond training tasks, but is compact at least)" ], "raw_notes": "We can train a variety of pipelines to do whatever seq2seq task we want.\n**However, this isn’t really how the modern seq2seq landscape has evolved.**\n**Instead of training single-task seq2seq pipelines, many modern formulations train a single model to do many tasks.**\n**T5: An encoder-decoder that’s commonly trained on many tasks, both useful on their own and for core language understanding.**\n**The goal is for eventual understanding of the language and nuanced meaning of encoder inputs to generate decoder outputs.**\n(But the T5 on its own is insufficient. It does not transfer well beyond training tasks, but is compact at least)" }, { "slide_number": 101, "title": "Instruction-Following Seq2Seq", "slide_body": [ "Arbitrary Translation", "Scaling Instruction-Finetuned Language Models (2022)" ], "notes": [ "**The goal is for eventual understanding of the language and nuanced meaning of encoder inputs to generate decoder outputs.**", "But the T5 on its own is insufficient. It does not transfer well beyond training tasks, but is compact at least.", "**Flan, on the other hand, has shown to be pretty good.**", "**Leverages the same multi-task formulation but trains on even more complicated tasks that require nuanced understanding.**", "**Transfers well to new tasks based on language instruction alone.**", "**You will investigate this formulation and more in the next notebook, so good luck!**" ], "raw_notes": "**The goal is for eventual understanding of the language and nuanced meaning of encoder inputs to generate decoder outputs.**\nBut the T5 on its own is insufficient. It does not transfer well beyond training tasks, but is compact at least.\n**Flan, on the other hand, has shown to be pretty good.**\n**Leverages the same multi-task formulation but trains on even more complicated tasks that require nuanced understanding.**\n**Transfers well to new tasks based on language instruction alone.**\n**You will investigate this formulation and more in the next notebook, so good luck!**" }, { "slide_number": 102, "title": "Part 2: GenAI Models", "slide_body": [ "•", "Decoders for Sequence Forecasting", "•", "Multimodal Reasoning and Generation", "•", "GenAI Servers and Deployment" ], "notes": [ "Hello and welcome back to the course!", "- If you haven't taken a break yet, we highly recommend you do so before continuing.", "- But if you're ready, let's dive into the next section." ], "raw_notes": "Hello and welcome back to the course!\n- If you haven't taken a break yet, we highly recommend you do so before continuing.\n- But if you're ready, let's dive into the next section." }, { "slide_number": 103, "title": "Large Language Models", "slide_body": [ "Backbones for Language Understanding", "Backbones for language tasks, including classification and generation." ], "notes": [ "We promised earlier that we'd discuss how data is processed, the intuitions at play, and what the architectures are actually doing as the pipeline progresses." ], "raw_notes": "We promised earlier that we'd discuss how data is processed, the intuitions at play, and what the architectures are actually doing as the pipeline progresses." }, { "slide_number": 104, "title": "Encoders vs Decoders", "slide_body": [ "Both Options Have Pros and Cons" ], "notes": [ "Now, we have our components: encoders for taking in sequences and decoders for outputting sequences, using transformers as our architecture.", "But what about other modalities other than language?" ], "raw_notes": "Now, we have our components: encoders for taking in sequences and decoders for outputting sequences, using transformers as our architecture.\nBut what about other modalities other than language?" }, { "slide_number": 105, "title": "Modular Modalities", "slide_body": [ "Where Can The Transformer Fit?" ], "notes": [ "- At the beginning, we mentioned that natural language isn't just for reasoning with language in isolation—it's useful to incorporate other modalities like images, audio, 3D video, DNA, protein molecules, etc.", "- We've been working with text input and text output, but what about these other things? Before diving into each individually, let's discuss what we mean by 'different modalities.'\"" ], "raw_notes": "- At the beginning, we mentioned that natural language isn't just for reasoning with language in isolation—it's useful to incorporate other modalities like images, audio, 3D video, DNA, protein molecules, etc.\n- We've been working with text input and text output, but what about these other things? Before diving into each individually, let's discuss what we mean by 'different modalities.'\"" }, { "slide_number": 106, "title": "Information Modalities", "slide_body": [ "Core Component of Generative AI" ], "notes": [ "An information modality is data in a particular format. You're familiar with classes, images, and text.", "- We've reasoned with images as two-dimensional grids of pixels with spatial locality—hence why convolutional networks are common.", "- For natural language, while you could use convolutional neural networks, people often prefer transformers for their attention mechanisms.", "- Other modalities exist, and as far as processing is concerned, these are just different data distributions with different modeling assumptions. But let’s define them a bit more rigorously…" ], "raw_notes": "An information modality is data in a particular format. You're familiar with classes, images, and text.\n- We've reasoned with images as two-dimensional grids of pixels with spatial locality—hence why convolutional networks are common.\n- For natural language, while you could use convolutional neural networks, people often prefer transformers for their attention mechanisms.\n- Other modalities exist, and as far as processing is concerned, these are just different data distributions with different modeling assumptions. But let’s define them a bit more rigorously…" }, { "slide_number": 107, "title": "What Makes Modalities Different?", "slide_body": [ "Core Component of Generative AI", "Surface", "Technical", "Broad", "Narrow", "Data communicated", "in different ways", "Data that follows", "different distributions", "Data produced/ingested", "in different ways", "Data that is structurally", "different to model" ], "notes": [ "Modalities can be communicated in different ways—different file types, sensory types, etc.", "- Whether they're explicit files processed differently by an operating system or data frequencies processed internally by humans through senses like sight, hearing, touch—they're different.", "- That's the surface-level definition. Technically, all this data is just data. We can think of modalities as different data distributions—maybe Gaussian, Poisson, or the distribution of images or employee data.”", "- But we also have a narrower technical definition when motivating architectures. When we think about images or natural language, we consider how to understand the data—what structure does it elicit, and how do we process it effectively?", " - For images, we used convolutional neural networks to extract features, then dense layers to reason about them. For language, we use encoder-decoder structures.", " - Depending on the data's structure and important intuitions, we choose different models. This can define a different modality or data distribution, radically changing our approach to processing the domain.\"" ], "raw_notes": "Modalities can be communicated in different ways—different file types, sensory types, etc.\n- Whether they're explicit files processed differently by an operating system or data frequencies processed internally by humans through senses like sight, hearing, touch—they're different.\n- That's the surface-level definition. Technically, all this data is just data. We can think of modalities as different data distributions—maybe Gaussian, Poisson, or the distribution of images or employee data.”\n- But we also have a narrower technical definition when motivating architectures. When we think about images or natural language, we consider how to understand the data—what structure does it elicit, and how do we process it effectively?\n - For images, we used convolutional neural networks to extract features, then dense layers to reason about them. For language, we use encoder-decoder structures.\n - Depending on the data's structure and important intuitions, we choose different models. This can define a different modality or data distribution, radically changing our approach to processing the domain.\"" }, { "slide_number": 108, "title": "Multimodal Connections", "slide_body": [ "Core Components of Generative AI", "Image of Objects", "Sequence of Letters", "Sequence of Images", "Sequence of Frequencies", "3D Field", "Collection of Faces", "Volume of Values", "Sequence of Classes", "Sequence of Images", "Sequence of Classes", "Grid of Pixels" ], "notes": [ "- So all this data—text, audio, images, 3D, etc.— may be modeled differently depending on how it's captured and represented.", "- Text can be modeled as a sequence of letters or classes—that's an engineering decision that works well.", "- Audio—is it a sequence of frequencies? Frequencies can be considered as vectors; maybe we can model it as images if we window it.", "- For images, we've thought of grids of pixels with spatial locality, but perhaps there are other ways—chunking into patches, for example.", "- 3D data can be modeled as volumes, point clouds, or collections of faces (meshes).", "- Video is a sequence of images—but should we model it that way? Maybe as sequences of windows or 3D blocks.", "- DNA is also a sequence of classes with a limited vocabulary—it can be modeled similarly.", "- So, how do we link these modalities together?\"" ], "raw_notes": "- So all this data—text, audio, images, 3D, etc.— may be modeled differently depending on how it's captured and represented.\n- Text can be modeled as a sequence of letters or classes—that's an engineering decision that works well.\n- Audio—is it a sequence of frequencies? Frequencies can be considered as vectors; maybe we can model it as images if we window it.\n- For images, we've thought of grids of pixels with spatial locality, but perhaps there are other ways—chunking into patches, for example.\n- 3D data can be modeled as volumes, point clouds, or collections of faces (meshes).\n- Video is a sequence of images—but should we model it that way? Maybe as sequences of windows or 3D blocks.\n- DNA is also a sequence of classes with a limited vocabulary—it can be modeled similarly.\n- So, how do we link these modalities together?\"" }, { "slide_number": 109, "title": "Multimodal Connections", "slide_body": [ "Core Components of Generative AI" ], "notes": [ "At an architectural level, we can link modalities by encoding them through intuitive architectures and combining them if their dimensionalities align. We've seen in previous notebooks that we can pass from natural language representations into latent sequences repeatedly. Is it possible to combine information from one sequence with another?", "- Mathematically, yes—if we have vectors of lengths N and M, we can perform operations on them.", "- So, if we can encode multiple modalities in a way that they can be connected mathematically, or adjust their structures to align, that works.", "- In practice, creating multimodal connections involves modeling different input pathways, different decoder or output pathways, and linking them together.\"" ], "raw_notes": "At an architectural level, we can link modalities by encoding them through intuitive architectures and combining them if their dimensionalities align. We've seen in previous notebooks that we can pass from natural language representations into latent sequences repeatedly. Is it possible to combine information from one sequence with another?\n- Mathematically, yes—if we have vectors of lengths N and M, we can perform operations on them.\n- So, if we can encode multiple modalities in a way that they can be connected mathematically, or adjust their structures to align, that works.\n- In practice, creating multimodal connections involves modeling different input pathways, different decoder or output pathways, and linking them together.\"" }, { "slide_number": 110, "title": "Multimodal Connections", "slide_body": [ "Core Components of Generative AI", "Volume of Values", "Sequence of Classes", "Grid of Pixels", "Sequence of Images", "Sequence of Classes", "Volume of Values" ], "notes": [ "This sets a natural definition for encoders and decoders.", "- The bidirectional transformer architecture is great for encoding language but not for decoding it—encoders aren't well-suited for generating sequences one token at a time.", "- In contrast, decoder-style architectures are great at generating text.", "- The same applies to other modalities—for images, convolutional neural networks are great encoders, and transpose convolutions can serve as decoders. Linking them creates encoder-decoder architectures, which we'll see in image generation.", "So, multimodality involves linking components suited for their domains.\"" ], "raw_notes": "This sets a natural definition for encoders and decoders.\n- The bidirectional transformer architecture is great for encoding language but not for decoding it—encoders aren't well-suited for generating sequences one token at a time.\n- In contrast, decoder-style architectures are great at generating text.\n- The same applies to other modalities—for images, convolutional neural networks are great encoders, and transpose convolutions can serve as decoders. Linking them creates encoder-decoder architectures, which we'll see in image generation.\nSo, multimodality involves linking components suited for their domains.\"" }, { "slide_number": 111, "title": "Multimodal Connections", "slide_body": [ "Core Components of Generative AI", "Image of Objects", "Sequence of Letters", "Sequence of Images", "Sequence of Frequencies", "3D Field", "Collection of Faces", "Volume of Values", "Sequence of Classes", "Sequence of Images", "Sequence of Classes", "Grid of Pixels" ], "notes": [ "Now that we understand we have these components and can link them, let's focus on the ones we'll discuss in the course and notebooks: audio and images, in addition to text.", "- Text, as a sequence of letters or classes, makes sense.", "- Audio has similar assumptions—it's a sequence of frequencies.", "- Images are also relatively easy to reason about as sequences but will present more issues when we discuss generation.", "Other modalities like 3D, video, DNA, robotics—you can explore on your own, but they also involve modeling assumptions and architectures suited to process that data efficiently." ], "raw_notes": "Now that we understand we have these components and can link them, let's focus on the ones we'll discuss in the course and notebooks: audio and images, in addition to text.\n- Text, as a sequence of letters or classes, makes sense.\n- Audio has similar assumptions—it's a sequence of frequencies.\n- Images are also relatively easy to reason about as sequences but will present more issues when we discuss generation.\nOther modalities like 3D, video, DNA, robotics—you can explore on your own, but they also involve modeling assumptions and architectures suited to process that data efficiently." }, { "slide_number": 112, "title": "Transformer Benefits", "slide_body": [ "Misconception:", "Transformers are only good for language", "Recall:", "Language Text = Ordered Sequence of Classes", "Resolution:", "Transformers are good for ordered sequences", "What Are They Good For", "Seq", "Seq", "Seq" ], "notes": [ "**Reiterating the value proposition of transformers—they're great for text because text is an ordered sequence of classes. So, if we can think of data as a sequence and reason about it as a sequence, transformers might be a good fit.**", "Moving to audio, people use transformers a lot for reasoning with audio and speech because audio is a sequence over time—a time series of frequencies. You sample audio over time, getting frequency samples at each timestep. You can normalize them tomel-frequency spectrograms, but you’re inherently reasoning with a time series. So, transformers can be applied here.", "\"What about images? We've learned that convolution is great for encoding images. Can we use transformers? Yes—Vision Transformers (ViTs) take an image, split it into patches, and reason about the embeddings of patches. This provides benefits, allowing us to synergize with existing transformer architectures.ViTsare still used today for image processing.\"" ], "raw_notes": "**Reiterating the value proposition of transformers—they're great for text because text is an ordered sequence of classes. So, if we can think of data as a sequence and reason about it as a sequence, transformers might be a good fit.**\nMoving to audio, people use transformers a lot for reasoning with audio and speech because audio is a sequence over time—a time series of frequencies. You sample audio over time, getting frequency samples at each timestep. You can normalize them tomel-frequency spectrograms, but you’re inherently reasoning with a time series. So, transformers can be applied here.\n\"What about images? We've learned that convolution is great for encoding images. Can we use transformers? Yes—Vision Transformers (ViTs) take an image, split it into patches, and reason about the embeddings of patches. This provides benefits, allowing us to synergize with existing transformer architectures.ViTsare still used today for image processing.\"" }, { "slide_number": 113, "title": "Speech-Guided Encoders", "slide_body": [ "Audio-In Systems", "https://docs.nvidia.com/deeplearning/nemo/user-guide/docs/zh/latest/voice_activity_detection/tutorial.html" ], "notes": [ "Reiterating the value proposition of transformers—they're great for text because text is an ordered sequence of classes. So, if we can think of data as a sequence and reason about it as a sequence, transformers might be a good fit.", "**Moving to audio, people use transformers a lot for reasoning with audio and speech because audio is a sequence over time—a time series of frequencies. You sample audio over time, getting frequency samples at each timestep. You can normalize them to****mel****-frequency spectrograms, but you’re inherently reasoning with a time series. So, transformers can be applied here.**", "\"What about images? We've learned that convolution is great for encoding images. Can we use transformers? Yes—Vision Transformers (ViTs) take an image, split it into patches, and reason about the embeddings of patches. This provides benefits, allowing us to synergize with existing transformer architectures.ViTsare still used today for image processing." ], "raw_notes": "Reiterating the value proposition of transformers—they're great for text because text is an ordered sequence of classes. So, if we can think of data as a sequence and reason about it as a sequence, transformers might be a good fit.\n**Moving to audio, people use transformers a lot for reasoning with audio and speech because audio is a sequence over time—a time series of frequencies. You sample audio over time, getting frequency samples at each timestep. You can normalize them to****mel****-frequency spectrograms, but you’re inherently reasoning with a time series. So, transformers can be applied here.**\n\"What about images? We've learned that convolution is great for encoding images. Can we use transformers? Yes—Vision Transformers (ViTs) take an image, split it into patches, and reason about the embeddings of patches. This provides benefits, allowing us to synergize with existing transformer architectures.ViTsare still used today for image processing." }, { "slide_number": 114, "title": "Image-Guided Encoders", "slide_body": [ "Image-In Systems", "Improve Accuracy and Robustness of Vision AI Apps with Vision Transformers and NVIDIA TAO | NVIDIA Blog" ], "notes": [ "Reiterating the value proposition of transformers—they're great for text because text is an ordered sequence of classes. So, if we can think of data as a sequence and reason about it as a sequence, transformers might be a good fit.", "Moving to audio, people use transformers a lot for reasoning with audio and speech because audio is a sequence over time—a time series of frequencies. You sample audio over time, getting frequency samples at each timestep. You can normalize them tomel-frequency spectrograms, but you’re inherently reasoning with a time series. So, transformers can be applied here.", "**\"What about images? We've learned that convolution is great for encoding images. Can we use transformers? Yes—Vision Transformers (****ViTs****) take an image, split it into patches, and reason about the embeddings of patches. This provides benefits, allowing us to synergize with existing transformer architectures.****ViTs****are still used today for image processing.**" ], "raw_notes": "Reiterating the value proposition of transformers—they're great for text because text is an ordered sequence of classes. So, if we can think of data as a sequence and reason about it as a sequence, transformers might be a good fit.\nMoving to audio, people use transformers a lot for reasoning with audio and speech because audio is a sequence over time—a time series of frequencies. You sample audio over time, getting frequency samples at each timestep. You can normalize them tomel-frequency spectrograms, but you’re inherently reasoning with a time series. So, transformers can be applied here.\n**\"What about images? We've learned that convolution is great for encoding images. Can we use transformers? Yes—Vision Transformers (****ViTs****) take an image, split it into patches, and reason about the embeddings of patches. This provides benefits, allowing us to synergize with existing transformer architectures.****ViTs****are still used today for image processing.**" }, { "slide_number": 115, "title": "Synergizing Encoders", "slide_body": [ "Resolving Modality Gaps via Similarity", "CLIP: Connecting text and images | OpenAI" ], "notes": [ "- Given that we can create different encoders based on the transformer architecture, can we get use out of them without an explicit decoder?", "- Yes—Contrastive Language-Image Pre-training (CLIP) shows this. CLIP uses multiple encoders for different modalities and synergizes their outputs.", "- When we have a pair of an image and a caption, we train the network to produce embeddings that are close together. This synergizes a text encoder and an image encoder.", "- You'll see this in the notebook—the embeddings become useful representations for images and text, allowing for tasks like retrieval or swapping between images and text in analysis." ], "raw_notes": "- Given that we can create different encoders based on the transformer architecture, can we get use out of them without an explicit decoder?\n- Yes—Contrastive Language-Image Pre-training (CLIP) shows this. CLIP uses multiple encoders for different modalities and synergizes their outputs.\n- When we have a pair of an image and a caption, we train the network to produce embeddings that are close together. This synergizes a text encoder and an image encoder.\n- You'll see this in the notebook—the embeddings become useful representations for images and text, allowing for tasks like retrieval or swapping between images and text in analysis." }, { "slide_number": 116, "title": "Multimodal Retrievers", "slide_body": [ "Resolving Modality Gaps via Similarity", "RA-CM3: Retrieval-Augmented Multimodal Modeling (2023)" ], "notes": [ "- Going further into retrieval, synergized encoders are useful even within natural language.", "- If you have two types of text or documents, you can use encoders to retrieve what's relevant, similar, or useful in a context. This is common in retrieval-augmented generation.", "- For example, with a query 'Labrador sitting on bench near water,' if you have synergized image and text encoders, you can embed the query and retrieve images from a pool that match.", "- Similarly, with text documents, you can use different encoders to retrieve longer passages or summaries." ], "raw_notes": "- Going further into retrieval, synergized encoders are useful even within natural language.\n- If you have two types of text or documents, you can use encoders to retrieve what's relevant, similar, or useful in a context. This is common in retrieval-augmented generation.\n- For example, with a query 'Labrador sitting on bench near water,' if you have synergized image and text encoders, you can embed the query and retrieve images from a pool that match.\n- Similarly, with text documents, you can use different encoders to retrieve longer passages or summaries." }, { "slide_number": 117, "title": "Synergizing Encoders", "slide_body": [ "Resolving Modality Gaps via Similarity", "By aligning six modalities’ embedding into a common space, ImageBind enables cross-modal retrieval of different types of content that aren’t observed together, the addition of embeddings from different modalities to naturally compose their semantics, and audio-to-image generation by using our audio embeddings with a pretrained DALLE-2 decoder to work with CLIP text embeddings.", "ImageBind: Holistic AI learning across six modalities (2023)" ], "notes": [ "This idea of synergizing encoders can be pushed to extremes.", "TheImageBindresearch from Meta trained encoders mapping various modalities (depth, heatmaps, IMUs, audio, text) into a joint embedding space.", "They could swap out modalities, generate new data, retrieve from pools, and explore emergent properties." ], "raw_notes": "This idea of synergizing encoders can be pushed to extremes.\nTheImageBindresearch from Meta trained encoders mapping various modalities (depth, heatmaps, IMUs, audio, text) into a joint embedding space.\nThey could swap out modalities, generate new data, retrieve from pools, and explore emergent properties." }, { "slide_number": 118, "title": "Speech-Guided Text Generation", "slide_body": [ "Audio-In, Text-Out", "Introducing Whisper | OpenAI", "Audio", "Text", "Text" ], "notes": [ "- Let's step back to not only taking in interesting modalities but also creating interesting information.", "- **Is speech-guided text generation possible? Yes—if we have transformer components for ingesting audio and creating text, we can combine them in a classic encoder-decoder architecture. This makes sense due to the clear modality gap: input audio, output text. Encoder-decoder architectures work well here—OpenAI's Whisper model is an example, using encoder plus decoder with cross-attention. You'll get to play with that in the notebook.**", "Similarly, for image-guided text generation, we can take an image encoder and combine it with a text decoder using cross-attention. This works well in practice, and you can experiment with it. More recently, people have been training decoder-only models for the same objective." ], "raw_notes": "- Let's step back to not only taking in interesting modalities but also creating interesting information.\n- **Is speech-guided text generation possible? Yes—if we have transformer components for ingesting audio and creating text, we can combine them in a classic encoder-decoder architecture. This makes sense due to the clear modality gap: input audio, output text. Encoder-decoder architectures work well here—OpenAI's Whisper model is an example, using encoder plus decoder with cross-attention. You'll get to play with that in the notebook.**\nSimilarly, for image-guided text generation, we can take an image encoder and combine it with a text decoder using cross-attention. This works well in practice, and you can experiment with it. More recently, people have been training decoder-only models for the same objective." }, { "slide_number": 119, "title": "Image-Guided Text Generation", "slide_body": [ "Image-In, Text-Out", "Image", "Text", "Text", "An Image is Worth 16x16 Words: Transformers for Image Recognition at Scale (2021)" ], "notes": [ "- Let's step back to not only taking in interesting modalities but also creating interesting information.", "- Is speech-guided text generation possible? Yes—if we have transformer components for ingesting audio and creating text, we can combine them in a classic encoder-decoder architecture. This makes sense due to the clear modality gap: input audio, output text. Encoder-decoder architectures work well here—OpenAI's Whisper model is an example, using encoder plus decoder with cross-attention. You'll get to play with that in the notebook.", "**Similarly, for image-guided text generation, we can take an image encoder and combine it with a text decoder using cross-attention. This works well in practice, and you can experiment with it.**More recently, people have been training decoder-only models for the same objective." ], "raw_notes": "- Let's step back to not only taking in interesting modalities but also creating interesting information.\n- Is speech-guided text generation possible? Yes—if we have transformer components for ingesting audio and creating text, we can combine them in a classic encoder-decoder architecture. This makes sense due to the clear modality gap: input audio, output text. Encoder-decoder architectures work well here—OpenAI's Whisper model is an example, using encoder plus decoder with cross-attention. You'll get to play with that in the notebook.\n**Similarly, for image-guided text generation, we can take an image encoder and combine it with a text decoder using cross-attention. This works well in practice, and you can experiment with it.**More recently, people have been training decoder-only models for the same objective." }, { "slide_number": 120, "title": "Multi-Image Guided Text Generation", "slide_body": [ "Image- Systems", "VILA: On Pre-training for Visual Language Models" ], "notes": [ "Similarly, for image-guided text generation, we can take an image encoder and combine it with a text decoder using cross-attention. This works well in practice, and you can experiment with it.", "**More recently, people have been training decoder-only models for the same objective.**", "**If you look into models like****LLaVA****, VILA, or****InstructBLIP****, you'll notice that models taking both text and images often put them through the same input pathway. There's a text decoder running, and you can input natural language or train an auxiliary layer to embed images into the same input space as text. This allows for rich and even multi-image prompts. Of course, this requires different training formulations, but there's no reason you can't use a one-size-fits-all decoder taking in both text and images, as long as images are projected appropriately (e.g., using a****ViT****).**" ], "raw_notes": "Similarly, for image-guided text generation, we can take an image encoder and combine it with a text decoder using cross-attention. This works well in practice, and you can experiment with it.\n**More recently, people have been training decoder-only models for the same objective.**\n**If you look into models like****LLaVA****, VILA, or****InstructBLIP****, you'll notice that models taking both text and images often put them through the same input pathway. There's a text decoder running, and you can input natural language or train an auxiliary layer to embed images into the same input space as text. This allows for rich and even multi-image prompts. Of course, this requires different training formulations, but there's no reason you can't use a one-size-fits-all decoder taking in both text and images, as long as images are projected appropriately (e.g., using a****ViT****).**" }, { "slide_number": 121, "title": "NVLM: Open Frontier-Class Multimodal LLMS (2024)", "slide_body": [], "notes": [ "We've solidified that both encoder-decoder and decoder-only models can be used for multimodality. Models like NVLM exemplify this with three architectures for the same task of visual-language modeling: cross-attention-based, hybrid, and decoder-only schemes. Encoding is processed at the beginning, and inputs of images and text are fed through the same pathway.", "This validates that encoder-decoder vs. decoder-only is up to you, with pros and cons for each.\"" ], "raw_notes": "We've solidified that both encoder-decoder and decoder-only models can be used for multimodality. Models like NVLM exemplify this with three architectures for the same task of visual-language modeling: cross-attention-based, hybrid, and decoder-only schemes. Encoding is processed at the beginning, and inputs of images and text are fed through the same pathway.\nThis validates that encoder-decoder vs. decoder-only is up to you, with pros and cons for each.\"" }, { "slide_number": 122, "title": "Introducing Llama 3.1: Our most capable models to date | Meta (2024)", "slide_body": [], "notes": [ "Going further, people have been experimenting with creating complex pathways with arbitrary numbers of encoders and decoders to build the systems they want. This can be prohibitively expensive if starting from scratch due to the need to synergize components. But if you have the resources, you can create custom pipelines for video, image, speech, etc., and combine them. Most people benefit from these architectures' emergence. Conceptually, when reading papers, this is what's happening—experimenting with different modalities, input representations, and combining them into shared fused architectures." ], "raw_notes": "Going further, people have been experimenting with creating complex pathways with arbitrary numbers of encoders and decoders to build the systems they want. This can be prohibitively expensive if starting from scratch due to the need to synergize components. But if you have the resources, you can create custom pipelines for video, image, speech, etc., and combine them. Most people benefit from these architectures' emergence. Conceptually, when reading papers, this is what's happening—experimenting with different modalities, input representations, and combining them into shared fused architectures." }, { "slide_number": 123, "title": "Core Transformer Benefits", "slide_body": [ "Delivers Semantically-Dense Embeddings", "Easy to reuse. Easy to transfer", "Sequence logic and token logic", "Enforce Attention-Oriented Logic", "Lightweight supervisory interface", "Doubles as interpretable “I care about X”", "Synergize Semantically-Dense Embeddings", "Link two modalities for powerful results", "Good for zero-shot and training enforcement", "When Does It Shine", "Transformer Interpretability Beyond Attention Visualization (2021)" ], "notes": [ "- To wrap up this multimodal encoding discussion, what are the core benefits of transformers? It's not just language modeling—it's sequence modeling. If we're good at modeling sequences, we can model audio and image ingestion as well. We can combine them, use fusion formulations, and train mappings from one latent representation to another.", "- That's what you saw with decoder-only language modeling where there's an image encoder at the beginning mapping to the decoder's input. Lots of good stuff going on." ], "raw_notes": "- To wrap up this multimodal encoding discussion, what are the core benefits of transformers? It's not just language modeling—it's sequence modeling. If we're good at modeling sequences, we can model audio and image ingestion as well. We can combine them, use fusion formulations, and train mappings from one latent representation to another.\n- That's what you saw with decoder-only language modeling where there's an image encoder at the beginning mapping to the decoder's input. Lots of good stuff going on." }, { "slide_number": 124, "title": "Image Generation?", "slide_body": [ "Can Transformers Generate Images?" ], "notes": [ "- **It looks like transformers can reason with a lot of data… so does that mean they can generate images? Unfortunately, sort of, but not really.**", "That's why, when we looked at diagrams connecting modalities, there wasn't just the transformer—there was another structure, like an autoencoder or a U-Net, which we'll discuss. Generating images isn't the transformer's strong suit on its own.", "This is partly because we don't model images as sequences when generating them. Generating images as sequences is problematic—you miss high-frequency, pixel-level information. What's your sequence resolution—one pixel? Autoregressive image generation is inefficient.\"" ], "raw_notes": "- **It looks like transformers can reason with a lot of data… so does that mean they can generate images? Unfortunately, sort of, but not really.**\nThat's why, when we looked at diagrams connecting modalities, there wasn't just the transformer—there was another structure, like an autoencoder or a U-Net, which we'll discuss. Generating images isn't the transformer's strong suit on its own.\nThis is partly because we don't model images as sequences when generating them. Generating images as sequences is problematic—you miss high-frequency, pixel-level information. What's your sequence resolution—one pixel? Autoregressive image generation is inefficient.\"" }, { "slide_number": 125, "title": "Image Generation?", "slide_body": [ "Can Transformers Generate Images?" ], "notes": [ "- It looks like transformers can reason with a lot of data… so does that mean they can generate images? Unfortunately, sort of, but not really.", "**That's why, when we looked at diagrams connecting modalities, there wasn't just the transformer—there was another structure, like an autoencoder or a U-Net, which we'll discuss. Generating images isn't the transformer's strong suit on its own.**", "This is partly because we don't model images as sequences when generating them. Generating images as sequences is problematic—you miss high-frequency, pixel-level information. What's your sequence resolution—one pixel? Autoregressive image generation is inefficient.\"" ], "raw_notes": "- It looks like transformers can reason with a lot of data… so does that mean they can generate images? Unfortunately, sort of, but not really.\n**That's why, when we looked at diagrams connecting modalities, there wasn't just the transformer—there was another structure, like an autoencoder or a U-Net, which we'll discuss. Generating images isn't the transformer's strong suit on its own.**\nThis is partly because we don't model images as sequences when generating them. Generating images as sequences is problematic—you miss high-frequency, pixel-level information. What's your sequence resolution—one pixel? Autoregressive image generation is inefficient.\"" }, { "slide_number": 126, "title": "Image Generation?", "slide_body": [ "Can Transformers Generate Images?", "VAE in 3D | Differentiable Blog (2020)", "Denoising Diffusion Implicit Models | Keras (2022)" ], "notes": [ "- It looks like transformers can reason with a lot of data… so does that mean they can generate images? Unfortunately, sort of, but not really.", "That's why, when we looked at diagrams connecting modalities, there wasn't just the transformer—there was another structure, like an autoencoder or a U-Net, which we'll discuss. Generating images isn't the transformer's strong suit on its own.", "**This is partly because we don't model images as sequences when generating them. Generating images as sequences is problematic—you miss high-frequency, pixel-level information. What's your sequence resolution—one pixel? Autoregressive image generation is inefficient.\"**" ], "raw_notes": "- It looks like transformers can reason with a lot of data… so does that mean they can generate images? Unfortunately, sort of, but not really.\nThat's why, when we looked at diagrams connecting modalities, there wasn't just the transformer—there was another structure, like an autoencoder or a U-Net, which we'll discuss. Generating images isn't the transformer's strong suit on its own.\n**This is partly because we don't model images as sequences when generating them. Generating images as sequences is problematic—you miss high-frequency, pixel-level information. What's your sequence resolution—one pixel? Autoregressive image generation is inefficient.\"**" }, { "slide_number": 127, "title": "Making An Autoencoder", "slide_body": [ "Decoding Your Encoding", "Latents" ], "notes": [ "So, what do we do instead?", "- **We often use bottlenecking architectures where we go down in encoding and up in decoding.**", "- **Transpose convolutions can add details, maintaining spatial locality, allowing the decoder to learn how to add pixel information. This is useful for denoising or super-resolution.**", "- U-Nets are similar to autoencoders but include skip connections between encoder and decoder layers, allowing information at different resolutions to be communicated as the decoder progresses.\"" ], "raw_notes": "So, what do we do instead?\n- **We often use bottlenecking architectures where we go down in encoding and up in decoding.**\n- **Transpose convolutions can add details, maintaining spatial locality, allowing the decoder to learn how to add pixel information. This is useful for denoising or super-resolution.**\n- U-Nets are similar to autoencoders but include skip connections between encoder and decoder layers, allowing information at different resolutions to be communicated as the decoder progresses.\"" }, { "slide_number": 128, "title": "Making A U-Net", "slide_body": [ "Restyle An Image", "U-Net: Convolutional Networks for Biomedical Image Segmentation (2015)" ], "notes": [ "So, what do we do instead?", "- We often use bottlenecking architectures where we go down in encoding and up in decoding.", "- Transpose convolutions can add details, maintaining spatial locality, allowing the decoder to learn how to add pixel information. This is useful for denoising or super-resolution.", "- **U-Nets are similar to autoencoders but include skip connections between encoder and decoder layers, allowing information at different resolutions to be communicated as the decoder progresses.\"**" ], "raw_notes": "So, what do we do instead?\n- We often use bottlenecking architectures where we go down in encoding and up in decoding.\n- Transpose convolutions can add details, maintaining spatial locality, allowing the decoder to learn how to add pixel information. This is useful for denoising or super-resolution.\n- **U-Nets are similar to autoencoders but include skip connections between encoder and decoder layers, allowing information at different resolutions to be communicated as the decoder progresses.\"**" }, { "slide_number": 129, "title": "Making A Denoiser", "slide_body": [ "Data by Noising, Train/Inferending by Denoising", "Noising (Algorithmic)", "Denoising (Learned)", "Denoising Diffusion Probabilistic Models (2020)" ], "notes": [ "Another strategy is denoising or diffusion. Instead of going from implicit representations to explicit output suddenly, we progress the representation in a natural space, like image space or compressed image space. We go from noise to slightly less noise progressively until we get a clear image. This allows details to be added progressively, enforcing spatial locality and refining high-precision information. It's an all-in-one approach for modeling complex, high-frequency details progressively." ], "raw_notes": "Another strategy is denoising or diffusion. Instead of going from implicit representations to explicit output suddenly, we progress the representation in a natural space, like image space or compressed image space. We go from noise to slightly less noise progressively until we get a clear image. This allows details to be added progressively, enforcing spatial locality and refining high-precision information. It's an all-in-one approach for modeling complex, high-frequency details progressively." }, { "slide_number": 130, "title": "Policy Networks", "slide_body": [ "Where Direct, Autoregression, or Diffusion Can Work", "[Spatial] Delta Prediction From Previous Position/Delta", "[Sequential] Prediction From Previous Position(s)", "[Diffusion] Progression of points into final structured alignment", "Diffusion Policy: Visuomotor Policy Learning via Action Diffusion (2024)" ], "notes": [ "Considering that we have diffusion, autoregression, and other methods, there are situations where all are viable. For example, policy networks in robotics predict what a robot should do at a given time.", "You can model the entire trajectory directly, which works but is restrictive. Alternatively, you can model it progressively, predicting the next action step by step. This can be autoregressive. In some cases, diffusion is used to initialize possible paths and progressively denoise them into the final pathway. Despite treating it less like a sequence problem, this works fine." ], "raw_notes": "Considering that we have diffusion, autoregression, and other methods, there are situations where all are viable. For example, policy networks in robotics predict what a robot should do at a given time.\nYou can model the entire trajectory directly, which works but is restrictive. Alternatively, you can model it progressively, predicting the next action step by step. This can be autoregressive. In some cases, diffusion is used to initialize possible paths and progressively denoise them into the final pathway. Despite treating it less like a sequence problem, this works fine." }, { "slide_number": 131, "title": "Helping Image Generation?", "slide_body": [ "Can Transformers HELP Generate Images?" ], "notes": [ "- That was a deep dive, but hopefully you're still with us.", "- We’ve confirmed that transformers might not be ideal for generating complex images.", "- But, can transformers help generate images? The answer is very much yes." ], "raw_notes": "- That was a deep dive, but hopefully you're still with us.\n- We’ve confirmed that transformers might not be ideal for generating complex images.\n- But, can transformers help generate images? The answer is very much yes." }, { "slide_number": 132, "title": "Image Generation?", "slide_body": [ "Can Transformers Help Generate Images?", "High-Resolution Image Synthesis with Latent Diffusion Models" ], "notes": [ "- This is the architecture for a common image-generating model called Stable Diffusion.", "- As you can see, we can guide the denoising process with textual information or other representations using attention mechanisms. We inject context into the generation, impacting the output.", "- There's also an additional encoder-decoder on the far left, converting raw pixel space into a smaller representation before denoising. It's engineering, but it follows the logic of giving enough information, simplifying representations, modeling important aspects, and with proper training, it works.\"" ], "raw_notes": "- This is the architecture for a common image-generating model called Stable Diffusion.\n- As you can see, we can guide the denoising process with textual information or other representations using attention mechanisms. We inject context into the generation, impacting the output.\n- There's also an additional encoder-decoder on the far left, converting raw pixel space into a smaller representation before denoising. It's engineering, but it follows the logic of giving enough information, simplifying representations, modeling important aspects, and with proper training, it works.\"" }, { "slide_number": 133, "title": "Synergized Multi-Domain Encoders", "slide_body": [ "Image-In Systems", "X-VILA: Cross-Modality Alignment for Large Language Model (2024)" ], "notes": [ "Pushing this further, models like X-LAN have multiple embedding and encoding strategies optimized for various modalities, creating a visual embedding highway.", "Representations communicate with each other, and as long as we have something like a language model and the highway connecting information without decimating structure, we can learn effective relationships.", "We can have diffusion-guided image decoding, video decoding, audio decoding, language generation—all happening with shared representations synergized together in an end-to-end deep learning pipeline.\"" ], "raw_notes": "Pushing this further, models like X-LAN have multiple embedding and encoding strategies optimized for various modalities, creating a visual embedding highway.\nRepresentations communicate with each other, and as long as we have something like a language model and the highway connecting information without decimating structure, we can learn effective relationships.\nWe can have diffusion-guided image decoding, video decoding, audio decoding, language generation—all happening with shared representations synergized together in an end-to-end deep learning pipeline.\"" }, { "slide_number": 134, "title": "Modular Modalities", "slide_body": [ "Where Can The Transformer Fit?" ], "notes": [ "- Okay, that's it! This was probably more detailed than you wanted about the magic of generative AI, but hopefully you learned a lot.", "- You're now ready to tackle Notebook 5, which will showcase synergized encoders, decoders, image generation, and a few other things.", "- Good luck, and I'll see you in the next section for generative AI services." ], "raw_notes": "- Okay, that's it! This was probably more detailed than you wanted about the magic of generative AI, but hopefully you learned a lot.\n- You're now ready to tackle Notebook 5, which will showcase synergized encoders, decoders, image generation, and a few other things.\n- Good luck, and I'll see you in the next section for generative AI services." }, { "slide_number": 135, "title": "Part 2: GenAI Models", "slide_body": [ "•", "Decoders for Sequence Forecasting", "•", "Multimodal Reasoning and Generation", "•", "GenAI Servers and Deployment" ], "notes": [ "- Welcome back!", "- Last section:**Multimodality**—synergizing encoders and decoders.", "- Now, shifting focus to**scaling, optimization, and building production-level systems**for real applications." ], "raw_notes": "- Welcome back!\n- Last section:**Multimodality**—synergizing encoders and decoders.\n- Now, shifting focus to**scaling, optimization, and building production-level systems**for real applications." }, { "slide_number": 136, "title": "Decoder-Guided Generation", "slide_body": [ "Continue The Thought", "HARD PROS", "Everything goes in one path", "Decoder gets much more training", "HARD CONS", "Needs way more training", "Needs to handle way more inputs" ], "notes": [ "Main focus:**Decoder models for natural language**.", "We can build multitask systems with decoders that take in context/questions and generate responses. This is great, but you’ve also definitely seen this in action before…" ], "raw_notes": "Main focus:**Decoder models for natural language**.\nWe can build multitask systems with decoders that take in context/questions and generate responses. This is great, but you’ve also definitely seen this in action before…" }, { "slide_number": 137, "title": "LLMs in the Wild: ChatGPT", "slide_body": [ "GPT-Powered Chat Application", "ChatGPT by OpenAI" ], "notes": [ "**ChatGPT**is a prime example. A decoder-style model wrapped with a user interface that \"remembers\" (feeds previous info as context)." ], "raw_notes": "**ChatGPT**is a prime example. A decoder-style model wrapped with a user interface that \"remembers\" (feeds previous info as context)." }, { "slide_number": 138, "title": "Productized LLM Application", "slide_body": [ "Well-Behaved Model", "Support Arbitrary Conversation Options", "Follow Along And Reason Well", "Be Nice, Inoffensive, and Helpful", "Don’t Fall Apart", "Be As Correct As Possible", "Solve Every Language Problem", "ChatGPT by OpenAI" ], "notes": [ "**GPT-4**is the powerhouse behind ChatGPT.", "- Handles various tasks and text generation options.", "- Supports conversations, tasks, continuations—generates until it stops.", "- It's helpful, reasons well, doesn't derail easily.", "- High expectations—we want it to be correct and reliable.", "- Techniques beyond training help ensure this.", "- Such models are powerful, solving many language problems." ], "raw_notes": "**GPT-4**is the powerhouse behind ChatGPT.\n- Handles various tasks and text generation options.\n- Supports conversations, tasks, continuations—generates until it stops.\n- It's helpful, reasons well, doesn't derail easily.\n- High expectations—we want it to be correct and reliable.\n- Techniques beyond training help ensure this.\n- Such models are powerful, solving many language problems." }, { "slide_number": 139, "title": "Productized LLM Application", "slide_body": [ "Well-Behaved Model", "Support Arbitrary Conversation Options", "Follow Along And Reason Well", "Be Nice, Inoffensive, and Helpful", "Don’t Fall Apart", "Be As Correct As Possible", "Solve Every Language Problem", "meta-llama/Llama-3.1-405B-Instruct | Hugging Face", "ChatGPT by OpenAI" ], "notes": [ "GPT-4 is closed-source, so you can't see its internals.", "- But there are open models like**Llama 3.1 40B**.", "- Huge models—you probably can't run them locally due to resource limits.", "- In theory, you can use the same pipeline APIs." ], "raw_notes": "GPT-4 is closed-source, so you can't see its internals.\n- But there are open models like**Llama 3.1 40B**.\n- Huge models—you probably can't run them locally due to resource limits.\n- In theory, you can use the same pipeline APIs." }, { "slide_number": 140, "title": "Productized LLM Application", "slide_body": [ "Well-Behaved Model", "Delivered at Scale", "Support Arbitrary Conversation Options", "Follow Along And Reason Well", "Be Nice, Inoffensive, and Helpful", "Don’t Fall Apart", "Be As Correct As Possible", "Solve Every Language Problem", "ChatGPT by OpenAI" ], "notes": [ "Despite not being able to host it locally, you can still access giant and/or closed models through an**API interface**using POST requests.", "Set up inputs as system and user messages and pass them in, and the server will format the input correctly for the model and send back a generated result." ], "raw_notes": "Despite not being able to host it locally, you can still access giant and/or closed models through an**API interface**using POST requests.\nSet up inputs as system and user messages and pass them in, and the server will format the input correctly for the model and send back a generated result." }, { "slide_number": 141, "title": "Productized LLM Application", "slide_body": [ "Support Arbitrary Conversation Options", "Follow Along And Reason Well", "Be Nice, Inoffensive, and Helpful", "Don’t Fall Apart", "Be As Correct As Possible", "Solve Every Language Problem", "Well-Behaved Model", "Delivered at Scale", "Wrapped in an Interface", "ChatGPT by OpenAI" ], "notes": [ "- Using such an API which can scale and stay resilient against load, systems like ChatGPT can scale to many concurrent users.", "- This is the penultimate example of a generative AI application scaled to the masses without requiring the masses to be able to host it themselves." ], "raw_notes": "- Using such an API which can scale and stay resilient against load, systems like ChatGPT can scale to many concurrent users.\n- This is the penultimate example of a generative AI application scaled to the masses without requiring the masses to be able to host it themselves." }, { "slide_number": 142, "title": "The Problem Landscape", "slide_body": [ "openai-community/gpt2-xl | Hugging Face", "Your Device", "Other Devices", "Web Browser", "Small Model to Big Model", "ChatGPT by OpenAI" ], "notes": [ "Challenge: Going from a local decoder model to an app like ChatGPT.", "- **Limited resources**—hard to load large models locally, or even fit big models on typical systems.", "- Lacking chat interfaces and functionalities, so need**software and orchestration**to bridge the gap.", "(NEXT SLIDE) Also hard to scale to many users and many instances." ], "raw_notes": "Challenge: Going from a local decoder model to an app like ChatGPT.\n- **Limited resources**—hard to load large models locally, or even fit big models on typical systems.\n- Lacking chat interfaces and functionalities, so need**software and orchestration**to bridge the gap.\n(NEXT SLIDE) Also hard to scale to many users and many instances." }, { "slide_number": 143, "title": "The Problem Landscape", "slide_body": [ "Your Device", "Other Devices", "Web Browser", "Your Device", "Other Devices", "Web Browser", "Your Device", "Other Devices", "Web Browser", "Your Device", "Other Devices", "Web Browser", "Your Device", "Your Device", "Your Device", "Small Model to Big Model", "Single-User to Many-User", "openai-community/gpt2-xl | Hugging Face", "ChatGPT by OpenAI" ], "notes": [ "Challenge: Going from a local decoder model to an app like ChatGPT.", "- **Limited resources**—hard to load large models locally, or even fit big models on typical systems.", "- Lacking chat interfaces and functionalities, so need**software and orchestration**to bridge the gap.", "**Also hard to scale to many users and many instances.**", "- Even if you have access to big models, sharing them is tough. Can't expect everyone to host massive models, and doing so would be grossly inefficient and expensive.", "- Need solutions beyond individual environments to provide access." ], "raw_notes": "Challenge: Going from a local decoder model to an app like ChatGPT.\n- **Limited resources**—hard to load large models locally, or even fit big models on typical systems.\n- Lacking chat interfaces and functionalities, so need**software and orchestration**to bridge the gap.\n**Also hard to scale to many users and many instances.**\n- Even if you have access to big models, sharing them is tough. Can't expect everyone to host massive models, and doing so would be grossly inefficient and expensive.\n- Need solutions beyond individual environments to provide access." }, { "slide_number": 144, "title": "Model Deployment", "slide_body": [ "Node 1", "Node 2", "Node 3", "Node 4", "LLM Pipeline", "Prep", "Post", "LLM Pipeline (Quantization)", "LLM Pipeline (Compiled and Optimized)", "Construction Perspective" ], "notes": [ "**Model optimization is necessary for deployment.**", "- Make models smaller, optimize them for specific hardware.", "- Fuse components to improve efficiency.", "- Sometimes need to**distribute models over multiple GPUs**.", "- Various routines and tools can help with this." ], "raw_notes": "**Model optimization is necessary for deployment.**\n- Make models smaller, optimize them for specific hardware.\n- Fuse components to improve efficiency.\n- Sometimes need to**distribute models over multiple GPUs**.\n- Various routines and tools can help with this." }, { "slide_number": 145, "title": "Model Deployment", "slide_body": [ "Query Perspective", "Model Replica", "Scheduler", "Your Device", "Other Devices", "Web Browser", "Your Device", "Other Devices", "Web Browser", "Your Device", "Other Devices", "Web Browser", "Your Device", "Other Devices", "Web Browser", "Your Device", "Your Device", "Your Device", "Backend Services", "Request Batcher", "Model Replica", "Request Batcher", "Model Replica", "Request Batcher", "Model Replica", "Request Batcher", "Model Replica", "Request Batcher" ], "notes": [ "**Model scaling is necessary for deployment.**", "- Allow models to grow, get and prioritize requests from many users, and replicate.", "- The goal is to use some access path to the model as a primitive to build more interesting applications.", "- (NEXT) How OpenAI does this." ], "raw_notes": "**Model scaling is necessary for deployment.**\n- Allow models to grow, get and prioritize requests from many users, and replicate.\n- The goal is to use some access path to the model as a primitive to build more interesting applications.\n- (NEXT) How OpenAI does this." }, { "slide_number": 146, "title": "Large Model Hosting Platforms - OpenAI-Style", "slide_body": [ "Responses", "Messages", "Model Name", "Settings", "API Key", "https://developer.nvidia.com/blog/simplifying-ai-inference-in-production-with-triton/", "OpenAI Gateway", "GPT", "(3.5/4)", "embed-<>", "images/generate", "chat/completion", "completion", "models", "embeddings", "Dalle-3", "End-User Application", "Retriever Microservice", "Server Management", "Query Router", "Endpoint Selector" ], "notes": [ "- **lide****146: OpenAI's Solution**", "- OpenAI uses an**OpenAI gateway system using a standard format**.", "- Models deployed in data centers, replicating as needed.", "- Users access models via**endpoints**—submit requests, specify models, use API keys.", "- Abstracts complexity, allows for scaling and monitoring automatically so they can focus on building applications." ], "raw_notes": "- **lide****146: OpenAI's Solution**\n- OpenAI uses an**OpenAI gateway system using a standard format**.\n- Models deployed in data centers, replicating as needed.\n- Users access models via**endpoints**—submit requests, specify models, use API keys.\n- Abstracts complexity, allows for scaling and monitoring automatically so they can focus on building applications." }, { "slide_number": 147, "title": "Query Router Access", "slide_body": [ "[INST]<>", "{{system_message}}", "<>", "{{instruction}} [/INST] {{primer}}", "{", "“context”:{{context}}", "“model”:“query”/”doc”", "}", "{", "\"messages\": [{", "\"content\": \"...\",", "\"role\": \"system\"", "},{", "\"content\": \"...\",", "\"role\": \"user\"", "}],", "\"model\": ”mixtral”,", "\"temperature\": 0.2,", "\"top_p\": 0.7,", "\"max_tokens\": 1024,", "\"stream\": True", "}", "SDXL", "Llama", "Dalle-3", "Embed", "GPT4", "Facilitate", "Monitor", "Optimize", "Load Balance", "/chat/completions", "VLM" ], "notes": [ "Review of sending a request:", "- Users send messages and settings to an endpoint.", "- Endpoint handles transaction facilitation, monitoring, optimization.", "- Inputs formatted for the model, processed in batches.", "- Load balancing and other optimizations occur behind the scenes." ], "raw_notes": "Review of sending a request:\n- Users send messages and settings to an endpoint.\n- Endpoint handles transaction facilitation, monitoring, optimization.\n- Inputs formatted for the model, processed in batches.\n- Load balancing and other optimizations occur behind the scenes." }, { "slide_number": 148, "title": "NVIDIA NIM Optimized Inference Microservices", "slide_body": [ "Pre-Optimized Deployment Primitive", "SDXL", "E5", "Mixtral", "VideoGen", "RIVA", "NIM", "Server", "Custom Modules", "General Chatbot", "Document", "Copilot", "Image Generator", "K8s", "Azure", "/AWS", "Org", "Cluster", "TensorRT-LLM / vLLM", "NVIDIA NIM" ], "notes": [ "In this course, we'll use**NVIDIA NIM**Inference Microservices to deploy large models efficiently.", "- Offers prebuilt containers, scaling tools like Helm charts.", "- Supports**in-flight batching**—smoothly integrates new requests.", "- Allows deploying models on your cloud infrastructure.", "- Creates endpoints for client-side processing or integration into larger applications." ], "raw_notes": "In this course, we'll use**NVIDIA NIM**Inference Microservices to deploy large models efficiently.\n- Offers prebuilt containers, scaling tools like Helm charts.\n- Supports**in-flight batching**—smoothly integrates new requests.\n- Allows deploying models on your cloud infrastructure.\n- Creates endpoints for client-side processing or integration into larger applications." }, { "slide_number": 149, "title": "NIM In Your Course", "slide_body": [ "Your Inference API Options", "build.nvidia.com (llm_client)", "Llama-8b-3.1 (nim)", "Llama-3.1-405b", "NV-Embed", "SDXL", "Phi-3.5-Vision (vLLM)", "A100 #2", "(dedicated)", "A100 #1", "(same as jupyter)", "Scaled Deployment" ], "notes": [ "This course will have you interact with 3 levels of deployment:", "- VLLM-based; non-quantizing deployment of a small model manually. Kickstarted in complementary notebook.", "- NIM-based: Model kickstarted in the background which is optimized for our A100-backed environment. Accessed through nim microservice.", "- Build.nvidia.com: Also NIM-backed, but hosted externally. Check outbuild.nvidia.comfor more info. Accessed throughllm_clientmicroservice." ], "raw_notes": "This course will have you interact with 3 levels of deployment:\n- VLLM-based; non-quantizing deployment of a small model manually. Kickstarted in complementary notebook.\n- NIM-based: Model kickstarted in the background which is optimized for our A100-backed environment. Accessed through nim microservice.\n- Build.nvidia.com: Also NIM-backed, but hosted externally. Check outbuild.nvidia.comfor more info. Accessed throughllm_clientmicroservice." }, { "slide_number": 150, "title": "Notebook Objective: Endpoint Orchestration", "slide_body": [ "GenAI Servers", "chat/completion", "completion", "models", "embeddings", "Endpoint Orchestration", "Applying light-weight state management on a more conservative client/server environment", "Endpoint Deployment", "Delivering semi-stateless AI systems at scale" ], "notes": [ "The goal for Notebook 6 is to explore**model deployment solutions**and introduce you to**endpoint orchestration.**More details on orchestration in the coming notebooks." ], "raw_notes": "The goal for Notebook 6 is to explore**model deployment solutions**and introduce you to**endpoint orchestration.**More details on orchestration in the coming notebooks." }, { "slide_number": 151, "title": "Part 3: Stateful Systems", "slide_body": [ "•", "Towards Orchestration and Agentics", "•", "[Optional] Towards Multi-Agent Systems", "•", "[Assessment] Image Generator Pipeline" ], "notes": [ "Welcome back!", "Now that we have our AI primitives deployed, we're moving into**Stateful Systems**.", "Focus:**Orchestration and****Agentics**.", "We'll explore how to add interesting state to our AI systems and do some very cool things with them" ], "raw_notes": "Welcome back!\nNow that we have our AI primitives deployed, we're moving into**Stateful Systems**.\nFocus:**Orchestration and****Agentics**.\nWe'll explore how to add interesting state to our AI systems and do some very cool things with them" }, { "slide_number": 152, "title": "Endpoint Orchestration", "slide_body": [ "GenAI Servers", "chat/completion", "completion", "models", "embeddings", "Endpoint Orchestration", "Applying light-weight state management on a more conservative client/server environment", "Endpoint Deployment", "Delivering semi-stateless AI systems at scale" ], "notes": [ "We've introduced**Endpoint Orchestration**and**LLM Orchestration**.", "- Learned about how endpoints are created, and how they can be used to build interesting systems.", "- We've done basic document Q&A, tested our deployed models, and even created a chatbot interaction.", "- Let's delve deeper." ], "raw_notes": "We've introduced**Endpoint Orchestration**and**LLM Orchestration**.\n- Learned about how endpoints are created, and how they can be used to build interesting systems.\n- We've done basic document Q&A, tested our deployed models, and even created a chatbot interaction.\n- Let's delve deeper." }, { "slide_number": 153, "title": "Chatbot Workflow", "slide_body": [ "{{sys_msg}}", "{{user_msg}}", "…", "LLM", " {{bot_msg}}", " {{user_msg}}", "How Things Started" ], "notes": [ "- **Chatbot Workflow (How Things Started)**", "- The classic**Chatbot Workflow**is an iterative process.", "- We feed the model dialogue exchanges and ask it to generate the next agent response.", "- It's a simple loop and forms the foundation of many chatbots today." ], "raw_notes": "- **Chatbot Workflow (How Things Started)**\n- The classic**Chatbot Workflow**is an iterative process.\n- We feed the model dialogue exchanges and ask it to generate the next agent response.\n- It's a simple loop and forms the foundation of many chatbots today." }, { "slide_number": 154, "title": "General Chat Application", "slide_body": [ "🛡️", "🛡️", "How Things Progressed", "LLM", "LLM", "LLM", "Prompt", "Context", "Question", "LLM Data Retriever", "TOOLSET", "Prompt", "Progress towards solution", "Here are some common tools", "Ask user for input" ], "notes": [ "- **General Chat Application (How Things Progressed)**", "- Chatbots have evolved. Beyond accumulating correspondence, we need**state management**.", "- This includes:", " - Scale deployment aspects.", " - Multiple endpoints for different use cases.", " - Support for calling tools.", " - Finding more context and stateful information over time.", " - Directing or intercepting outputs to create richer interfaces." ], "raw_notes": "- **General Chat Application (How Things Progressed)**\n- Chatbots have evolved. Beyond accumulating correspondence, we need**state management**.\n- This includes:\n - Scale deployment aspects.\n - Multiple endpoints for different use cases.\n - Support for calling tools.\n - Finding more context and stateful information over time.\n - Directing or intercepting outputs to create richer interfaces." }, { "slide_number": 155, "title": "Dialog/Retrieval Agents", "slide_body": [ "Demo: ChatGPT" ], "notes": [ "- **Dialog/Retrieval Agents (Demo: ChatGPT)**", "- **ChatGPT**showcases advanced capabilities.", "- It can display graphs, write code, and even execute code.", "- When it writes code, it attempts to run it, providing outputs like plots.", "- This is possible by combining LLMs with additional software tools." ], "raw_notes": "- **Dialog/Retrieval Agents (Demo: ChatGPT)**\n- **ChatGPT**showcases advanced capabilities.\n- It can display graphs, write code, and even execute code.\n- When it writes code, it attempts to run it, providing outputs like plots.\n- This is possible by combining LLMs with additional software tools." }, { "slide_number": 156, "title": "Dialog/Retrieval Agents", "slide_body": [ "Demo: ChatGPT" ], "notes": [ "- **Dialog/Retrieval Agents (Web Search)**", "- ChatGPT and similar systems can retrieve real-time information.", "- They can scrape data from websites, perform web searches, and incorporate that into responses.", "- Example: Asking about a recent conference and getting up-to-date information." ], "raw_notes": "- **Dialog/Retrieval Agents (Web Search)**\n- ChatGPT and similar systems can retrieve real-time information.\n- They can scrape data from websites, perform web searches, and incorporate that into responses.\n- Example: Asking about a recent conference and getting up-to-date information." }, { "slide_number": 157, "title": "Dialog/Retrieval Agents", "slide_body": [ "Demo: ChatGPT" ], "notes": [ "- **Dialog/Retrieval Agents (Memory and Context)**", "- These systems can remember information about you.", "- Not because the model weights change, but because they maintain a**state**or**memory**.", "- They can recall past interactions, updating their knowledge base dynamically." ], "raw_notes": "- **Dialog/Retrieval Agents (Memory and Context)**\n- These systems can remember information about you.\n- Not because the model weights change, but because they maintain a**state**or**memory**.\n- They can recall past interactions, updating their knowledge base dynamically." }, { "slide_number": 158, "title": "Dialog/Retrieval Agents", "slide_body": [ "LLMs with Context and Control", "LLM Orchestration: Software + LLM helps to route to software and LLMs.", "User Asks Something", "Agent Responds", "RAG 101: Demystifying Retrieval-Augmented Generation Pipelines (2023)" ], "notes": [ "- **LLM Orchestration with Context and Control**", "- This leads us to**LLM Orchestration**, which involves combining LLMs with software to route, direct, and influence both the software and the LLM.", "- Key components:", " - Retrieving context.", " - Updating context.", " - Using classical algorithms or heuristics.", " - Incorporating state management." ], "raw_notes": "- **LLM Orchestration with Context and Control**\n- This leads us to**LLM Orchestration**, which involves combining LLMs with software to route, direct, and influence both the software and the LLM.\n- Key components:\n - Retrieving context.\n - Updating context.\n - Using classical algorithms or heuristics.\n - Incorporating state management." }, { "slide_number": 159, "title": "Dialog/Retrieval Agents", "slide_body": [ "LLMs with Context and Control", "LLM Orchestration: Software + LLM helps to route to software and LLMs.", "Retrieve: Tool runs algorithms (database, code execution, semantic search, return a constant value, etc) to provide context", "User Asks Something", "Agent Responds", "RAG 101: Demystifying Retrieval-Augmented Generation Pipelines (2023)" ], "notes": [ " - Retrieving context.", " - Beyond just user input, we retrieve additional context relevant to the generation.", " - This can involve database queries, code execution, or semantic searches.", " - Helps the LLM provide more informed responses." ], "raw_notes": " - Retrieving context.\n - Beyond just user input, we retrieve additional context relevant to the generation.\n - This can involve database queries, code execution, or semantic searches.\n - Helps the LLM provide more informed responses." }, { "slide_number": 160, "title": "Dialog/Retrieval Agents", "slide_body": [ "LLMs with Context and Control", "LLM Orchestration: Software + LLM helps to route to software and LLMs.", "Retrieve: Tool runs algorithms (database, code execution, semantic search, return a constant value, etc) to provide context", "User Asks Something", "Agent Responds", "Augment: Based on tool responses, the software pipeline synthesizes some “context”", "to feed to LLM w/ question.", "RAG 101: Demystifying Retrieval-Augmented Generation Pipelines (2023)" ], "notes": [ "- **Augment Stage**:", " - We synthesize the retrieved context and prepare it to feed into the LLM.", " - Enhances the LLM's ability to generate precise responses based on the question." ], "raw_notes": "- **Augment Stage**:\n - We synthesize the retrieved context and prepare it to feed into the LLM.\n - Enhances the LLM's ability to generate precise responses based on the question." }, { "slide_number": 161, "title": "Dialog/Retrieval Agents", "slide_body": [ "LLMs with Context and Control", "LLM Orchestration: Software + LLM helps to route to software and LLMs.", "Retrieve: Tool runs algorithms (database, code execution, semantic search, return a constant value, etc) to provide context", "Generate: Based on question, instructions, and enhanced context, the LLM returns a response.", "User Asks Something", "Agent Responds", "Augment: Based on tool responses, the software pipeline synthesizes some “context”", "to feed to LLM w/ question.", "RAG 101: Demystifying Retrieval-Augmented Generation Pipelines (2023)" ], "notes": [ "- **Generate Stage**:", " - The LLM takes the augmented context and generates a response.", " - The process integrates retrieval, augmentation, and generation.", " - Results in a well-contextualized answer to the user's query." ], "raw_notes": "- **Generate Stage**:\n - The LLM takes the augmented context and generates a response.\n - The process integrates retrieval, augmentation, and generation.\n - Results in a well-contextualized answer to the user's query." }, { "slide_number": 162, "title": "LLM Orchestration Terminology", "slide_body": [ "(Mind The Overlap)", "Give the LLM retrieval", "from a database.", "Retrieval", "Let the LLM", "call tools/code.", "Tooling", "Let the LLM delegate", "work to specialized", "expert systems.", "Agentics", "Let the LLM swap", "prompts/directives.", "Routing" ], "notes": [ "- **LLM Orchestration Terminology**", "- Key terms in LLM orchestration:", " - **Retrieval**: Fetching data from a database or source.", " - **Tooling**: Allowing the LLM to call external tools or execute code.", " - **Routing**: Switching between different prompts or instructions.", " - **Agentics**: Delegating tasks to specialized expert systems or agents." ], "raw_notes": "- **LLM Orchestration Terminology**\n- Key terms in LLM orchestration:\n - **Retrieval**: Fetching data from a database or source.\n - **Tooling**: Allowing the LLM to call external tools or execute code.\n - **Routing**: Switching between different prompts or instructions.\n - **Agentics**: Delegating tasks to specialized expert systems or agents." }, { "slide_number": 163, "title": "Semantic Tools", "slide_body": [ "Similarity Queries", "High-performance computing.", "Happy Holidays!", "DLSS Gaming Statistics", "Any cool video games lately?", "Biological vision structure", "What’s with GPUs these days?", "Mitochondria, powerhouse", "Accelerating Vector Search: Using GPU-Powered Indexes with RAPIDS RAFT (2023)" ], "notes": [ "- **Similarity Queries**", "- Using**semantic tools**for similarity queries.", "- Embedding documents and performing semantic searches.", "- Example: Retrieving relevant scientific papers based on a user's query." ], "raw_notes": "- **Similarity Queries**\n- Using**semantic tools**for similarity queries.\n- Embedding documents and performing semantic searches.\n- Example: Retrieving relevant scientific papers based on a user's query." }, { "slide_number": 164, "title": "Semantic Tools", "slide_body": [ "LLM", "Prompt", "History", "Context", "Question", "It’s Jane!", "What’s My Name?", "VDB", "Retriever", "Hello Jane! How are you?", "Hello! My name is Jane", "Retriever", "VDB", "LLM Orchestration Technique", "Company", "Database" ], "notes": [ "- **LLM Orchestration Technique**", "- LLMs use orchestration to perform tasks like recalling a user's name from a database.", "- Involves retrieving relevant info (e.g., via a vector database retriever) and synthesizing it into the response." ], "raw_notes": "- **LLM Orchestration Technique**\n- LLMs use orchestration to perform tasks like recalling a user's name from a database.\n- Involves retrieving relevant info (e.g., via a vector database retriever) and synthesizing it into the response." }, { "slide_number": 165, "title": "Semantic Tools", "slide_body": [ "LLM", "Prompt", "Answer Please Question", "How’s the weather?", "No. I shouldn’t answer that", "Classifier", "Branch", "Prompt", "Don’t Answer question", "Routing Systems" ], "notes": [ "- **Routing Systems**", "- **Routing Systems**decide how the LLM should respond by routing to different prompts, states, and workflows.", "- They can determine whether to answer a query based on policies or classifiers.", "- Example: If a question is disallowed, the LLM will refuse to answer by being routed to a prompt that forces it to react negatively and abruptly." ], "raw_notes": "- **Routing Systems**\n- **Routing Systems**decide how the LLM should respond by routing to different prompts, states, and workflows.\n- They can determine whether to answer a query based on policies or classifiers.\n- Example: If a question is disallowed, the LLM will refuse to answer by being routed to a prompt that forces it to react negatively and abruptly." }, { "slide_number": 166, "title": "Hybrid Semantic/Structured", "slide_body": [ "Agentic Systems", "Tell Me About", "My Files", "File Reader", "“Query Files”", "File Reasoner", "“Summary”", "Search Request", "“File System”" ], "notes": [ "- **Hybrid Semantic/Structured Agentic Systems**", "- Combining LLMs with structured data sources.", "- Allows handling both structured and semantic queries.", "- Interaction with file systems, databases, or APIs to perform tasks like searching files or reasoning over data." ], "raw_notes": "- **Hybrid Semantic/Structured Agentic Systems**\n- Combining LLMs with structured data sources.\n- Allows handling both structured and semantic queries.\n- Interaction with file systems, databases, or APIs to perform tasks like searching files or reasoning over data." }, { "slide_number": 167, "title": "Structured Retrieval", "slide_body": [ "Prompt", "LLM", "Input", "Unstructured Generation", "Interacting Like A Human", "167" ], "notes": [ "- **Interacting Like a Human**", "- LLMs can interact with data in a human-like manner by just outputting text.", "- Generating unstructured text from input queries, which can be fed into other LLMs (or humans)", "- Good for some interactions, but not necessarily for environment access." ], "raw_notes": "- **Interacting Like a Human**\n- LLMs can interact with data in a human-like manner by just outputting text.\n- Generating unstructured text from input queries, which can be fed into other LLMs (or humans)\n- Good for some interactions, but not necessarily for environment access." }, { "slide_number": 168, "title": "Structured Retrieval", "slide_body": [ "Prompt", "LLM", "Input", "Unstructured Generation", "Prompt", "Code", "LLM", "Structured Retrieval", "Environment", "Interacting Like A Human", "Build a Question/Answering system over SQL data | LangChain" ], "notes": [ "- **Structured Retrieval With Direct Environment Queries**", "- LLMs can retrieve structured data by interacting with databases or programmatic environments.", "- Example: The LLM generates an SQL query to retrieve data from a database and answers the user's question." ], "raw_notes": "- **Structured Retrieval With Direct Environment Queries**\n- LLMs can retrieve structured data by interacting with databases or programmatic environments.\n- Example: The LLM generates an SQL query to retrieve data from a database and answers the user's question." }, { "slide_number": 169, "title": "Structured Retrieval", "slide_body": [ "Prompt", "LLM", "Input", "Unstructured Generation", "Prompt", "Code", "LLM", "Structured Retrieval", "Environment", "Prompt", "LLM", "Guided Generation", "Grammar", "/Schema", "{“first_name” : “Jane”,", "“last_name” : “Doe”,", "“confirmation” : -1}", "{“first_name” : “unknown”,", "“last_name” : “unknown”,", "“confirmation” : -1}", "Update given new info:", "“Sure, my name is Jane Doe!”", "LLM", "Interacting Like A Human" ], "notes": [ "- **Guided Structure Generation**", "- LLMs can be forced to output schema in a particular grammar with sampling-level tricks. More on this later.", "- Since natural input to functions is a key-value pair (variable + value), this is extremely powerful." ], "raw_notes": "- **Guided Structure Generation**\n- LLMs can be forced to output schema in a particular grammar with sampling-level tricks. More on this later.\n- Since natural input to functions is a key-value pair (variable + value), this is extremely powerful." }, { "slide_number": 170, "title": "Structured Retrieval", "slide_body": [ "Prompt", "LLM", "Input", "Unstructured Generation", "Prompt", "Code", "LLM", "Structured Retrieval", "Environment", "Prompt", "LLM", "Guided Generation", "Grammar", "/Schema", "Prompt", "LLM", "Tool Choice", "Tooling", "Tool", "Tool Schema", "Interacting Like A Human" ], "notes": [ "- **Agentic Tooling**", "- If LLMs can be forced to output a specific schema, maybe they can do it twice; once to decide what to do, and another to do it.", "- Backbone logic of LLMs selecting tools, since LLM can be forced to predict both from the set of tool options and then the arguments for said tool." ], "raw_notes": "- **Agentic Tooling**\n- If LLMs can be forced to output a specific schema, maybe they can do it twice; once to decide what to do, and another to do it.\n- Backbone logic of LLMs selecting tools, since LLM can be forced to predict both from the set of tool options and then the arguments for said tool." }, { "slide_number": 171, "title": "How To Predict Valid Schema?", "slide_body": [ "LLM", "Sure, I think…", "{“first_name” : “Jane”,", "“last_name” : “Doe”,", "“confirmation” : -1}", "{“first_name” : “unknown”,", "“last_name” : “unknown”,", "“confirmation” : -1}", "Update given new info:", "“Sure, my name is Jane Doe!”", "Desired", "Prompt Engineering?" ], "notes": [ "- **Schema Prediction via Prompt Engineering**", "- Making valid schema predictions using prompt engineering.", "- System fills in missing information (e.g., retrieving a name like \"Jane Doe\").", "- Refines outputs through iterative attempts and algorithmic postprocessing, but likely to be rough." ], "raw_notes": "- **Schema Prediction via Prompt Engineering**\n- Making valid schema predictions using prompt engineering.\n- System fills in missing information (e.g., retrieving a name like \"Jane Doe\").\n- Refines outputs through iterative attempts and algorithmic postprocessing, but likely to be rough." }, { "slide_number": 172, "title": "How To Predict Valid Schema?", "slide_body": [ "LLM", "“Jane”, “...", "{“first_name” : “Jane”,", "“last_name” : “Doe”,", "“confirmation” : -1}", "{“first_name” : “unknown”,", "“last_name” : “unknown”,", "“confirmation” : -1}", "Update given new info:", "“Sure, my name is Jane Doe!”", "Desired", "{“first_name” : “unknown”,", "“last_name” : “unknown”,", "“confirmation” : -1}", "Update given new info:", "“Sure, my name is Jane Doe!”", "[/INST] Sure, here you go:", "{“first_name”:", "Prompt Engineering + LLM Priming" ], "notes": [ "- **LLM Priming**", "- Using**LLM Priming**to improve schema predictions.", "- Providing well-prepared prompts to guide the LLM.", "- Helps the LLM fill in expected values based on structured schema inputs, but still can derail or miss." ], "raw_notes": "- **LLM Priming**\n- Using**LLM Priming**to improve schema predictions.\n- Providing well-prepared prompts to guide the LLM.\n- Helps the LLM fill in expected values based on structured schema inputs, but still can derail or miss." }, { "slide_number": 173, "title": "How To Predict Valid Schema?", "slide_body": [ "LLM", "“Jane”, “...", "{“first_name” : “Jane”,", "“last_name” : “Doe”,", "“confirmation” : -1}", "{“first_name” : “unknown”,", "“last_name” : “unknown”,", "“confirmation” : -1}", "Update given new info:", "“Sure, my name is Jane Doe!”", "Desired", "{“first_name” : “unknown”,", "“last_name” : “unknown”,", "“confirmation” : -1}", "Update given new info:", "“Sure, my name is Jane Doe!”", "{“first_name”:", "Prompt Engineering + LLM Priming + Tool Training" ], "notes": [ "- **Tool Training for Accurate Schema Generation**", "- Enhancing schema prediction with**Tool Training**.", "- Integrating tools into the LLM's flow for more accurate outputs.", "- Ensures consistency in retrieval processes, possibly via external APIs or code functions." ], "raw_notes": "- **Tool Training for Accurate Schema Generation**\n- Enhancing schema prediction with**Tool Training**.\n- Integrating tools into the LLM's flow for more accurate outputs.\n- Ensures consistency in retrieval processes, possibly via external APIs or code functions." }, { "slide_number": 174, "title": "How To Predict Valid Schema?", "slide_body": [ "LLM", "“Jane”, …", "Legal Tokens: [“]", "{“first_name” : “Jane”,", "“last_name” : “Doe”,", "“confirmation” : -1}", "{“first_name” : “unknown”,", "“last_name” : “unknown”,", "“confirmation” : -1}", "Update given new info:", "“Sure, my name is Jane Doe!”", "Desired", "{“first_name” : “unknown”,", "“last_name” : “unknown”,", "“confirmation” : -1}", "Update given new info:", "“Sure, my name is Jane Doe!”", "{“first_name”:", "Prompt Engineering + LLM Priming + Tool Training + Constrained Decoding" ], "notes": [ "- **Constrained Decoding**", "- By restricting the output space to valid tokens, the LLM produces outputs that fit expected data formats.", "- Reduces errors in the prediction pipeline, but still requires other tools in the arsenal (i.e. prompt engineering, function-calling training).", "- NOTE: sometimes models that do function-calling are called “action models.”" ], "raw_notes": "- **Constrained Decoding**\n- By restricting the output space to valid tokens, the LLM produces outputs that fit expected data formats.\n- Reduces errors in the prediction pipeline, but still requires other tools in the arsenal (i.e. prompt engineering, function-calling training).\n- NOTE: sometimes models that do function-calling are called “action models.”" }, { "slide_number": 175, "title": "Tool-Calling", "slide_body": [ "LLM Orchestration Technique", "General", "LLM Loop", "Create Python Code", "Make Wikipedia Query", "Make SQL Code" ], "notes": [ "- **Tool-Calling Mechanism in Action**", "- **Tools can be introduced to the main event loop directly, and the main system can try to pick them as it goes along.**", "- (Next) alternatively, each tool can be wrapped with its own agentic loop or some LLM-backed interface which keeps the main loop from getting blasted with complexity/context." ], "raw_notes": "- **Tool-Calling Mechanism in Action**\n- **Tools can be introduced to the main event loop directly, and the main system can try to pick them as it goes along.**\n- (Next) alternatively, each tool can be wrapped with its own agentic loop or some LLM-backed interface which keeps the main loop from getting blasted with complexity/context." }, { "slide_number": 176, "title": "Agentic Tool-Calling", "slide_body": [ "LLM Orchestration Technique", "Code LLM", "Function LLM", "SQL LLM", "Main Agent Loop" ], "notes": [ "- **Tool-Calling Mechanism in Action**", "- Tools can be introduced to the main event loop directly, and the main system can try to pick them as it goes along.", "- **Alternatively, each tool can be wrapped with its own agentic loop or some LLM-backed interface which keeps the main loop from getting blasted with complexity/context.**" ], "raw_notes": "- **Tool-Calling Mechanism in Action**\n- Tools can be introduced to the main event loop directly, and the main system can try to pick them as it goes along.\n- **Alternatively, each tool can be wrapped with its own agentic loop or some LLM-backed interface which keeps the main loop from getting blasted with complexity/context.**" }, { "slide_number": 177, "title": "Tooled ReAct Loop", "slide_body": [ "Reason And Act Loop", "Code LLM", "Function LLM", "SQL LLM", "Main Agent Loop", "A tool", "wasn’t used?" ], "notes": [ "- **Multi-Turn With****ReAct****:**", "- **- Tools can also be abstracted to include the user, either explicitly or implicitly.**", "- User here would be a tool returns dialog or supervision as context… just like any other tool.", "- If all or most tools are properly honored, their results are fed back into the agent loop, and multiple tools can be called between user interactions, it’s often known as the “ReAct(Reason and Act)” paradigm (which is one of many).", "Notebook 7 will explore these ideas more, and will culminate in aReActloop." ], "raw_notes": "- **Multi-Turn With****ReAct****:**\n- **- Tools can also be abstracted to include the user, either explicitly or implicitly.**\n- User here would be a tool returns dialog or supervision as context… just like any other tool.\n- If all or most tools are properly honored, their results are fed back into the agent loop, and multiple tools can be called between user interactions, it’s often known as the “ReAct(Reason and Act)” paradigm (which is one of many).\nNotebook 7 will explore these ideas more, and will culminate in aReActloop." }, { "slide_number": 178, "title": "Part 3: Stateful Systems", "slide_body": [ "•", "Towards Orchestration and Agentics", "•", "[Optional] Towards Multi-Agent Systems", "•", "[Assessment] Image Generator Pipeline" ], "notes": [ "**Optional Section on Multi-Agent Systems**", "Intended to prime the optionalLangGraphnotebook and discuss more about multi-agent systems." ], "raw_notes": "**Optional Section on Multi-Agent Systems**\nIntended to prime the optionalLangGraphnotebook and discuss more about multi-agent systems." }, { "slide_number": 179, "title": "Tooled ReAct Loop", "slide_body": [ "Reason And Act Loop", "Code LLM", "Function LLM", "SQL LLM", "Main Agent Loop", "Was a tool used?" ], "notes": [ "We left off at theReActloop, and previously defended the multi-agent abstraction as useful for maintaining simplicity and modularity. Let’s further investigate this proposition…" ], "raw_notes": "We left off at theReActloop, and previously defended the multi-agent abstraction as useful for maintaining simplicity and modularity. Let’s further investigate this proposition…" }, { "slide_number": 180, "title": "Multi-Agent System", "slide_body": [ "Generic Definition", "Agent Loop 1", "Agent Loop 2" ], "notes": [ "In contrast to single-agent systems, MAS involve multiple systems that performagentically, which could mean agents that kickstart other agents or multiple concurrent agents that interact through a shared interface.", "The main thing here is the notion of disjoint states and potentially-disjoint policies, or data pipelines, for dealing with their inputs…." ], "raw_notes": "In contrast to single-agent systems, MAS involve multiple systems that performagentically, which could mean agents that kickstart other agents or multiple concurrent agents that interact through a shared interface.\nThe main thing here is the notion of disjoint states and potentially-disjoint policies, or data pipelines, for dealing with their inputs…." }, { "slide_number": 181, "title": "Data Pipeline", "slide_body": [ "Per-Agent Behavior", "🤖", "🤖", "🤖", "🤖", "Update", "Answer", "Update", "Answer", "Update", "Answer", "Question", "Iterative Refinement", "Dive into Deep Learning (d2l.ai)" ], "notes": [ "The main thing here is the notion of disjoint states and potentially-disjoint policies, or data pipelines, for dealing with their inputs….", "- **A particular agent could have a refinement pipeline similar to the dialog management chatbot or the long document processing examples in the previous notebook.**", "- Another might have more of a baked-in retrieval heuristic where they query an external resource by default as part of their operations.", "- Another one yet may construct a hierarchy or other abstraction to deal with information.", "All these have been explored in the previous notebook in isolation, butagenticswould in theory allow any of these processes to be modularized behind an “agent interface” and be called as a sub-process that managed its own state.", "The only thing we would really need, is a buffer system that facilitates this transfer of information and a way to inform our systems how the other agents function. If we just do that, we can treat agents as tools and make arbitrarily modularized or arbitrarily complex pipelines for dealing with language and interacting with environments." ], "raw_notes": "The main thing here is the notion of disjoint states and potentially-disjoint policies, or data pipelines, for dealing with their inputs….\n- **A particular agent could have a refinement pipeline similar to the dialog management chatbot or the long document processing examples in the previous notebook.**\n- Another might have more of a baked-in retrieval heuristic where they query an external resource by default as part of their operations.\n- Another one yet may construct a hierarchy or other abstraction to deal with information.\nAll these have been explored in the previous notebook in isolation, butagenticswould in theory allow any of these processes to be modularized behind an “agent interface” and be called as a sub-process that managed its own state.\nThe only thing we would really need, is a buffer system that facilitates this transfer of information and a way to inform our systems how the other agents function. If we just do that, we can treat agents as tools and make arbitrarily modularized or arbitrarily complex pipelines for dealing with language and interacting with environments." }, { "slide_number": 182, "title": "Data Pipeline", "slide_body": [ "🤖", "Question", "Synthesize Answer", "Retrieval", "🤖", "🤖", "🤖", "🤖", "Update", "Answer", "Update", "Answer", "Update", "Answer", "Question", "Iterative Refinement", "Per-Agent Behavior", "Dive into Deep Learning (d2l.ai)" ], "notes": [ "The main thing here is the notion of disjoint states and potentially-disjoint policies, or data pipelines, for dealing with their inputs….", "- A particular agent could have a refinement pipeline similar to the dialog management chatbot or the long document processing examples in the previous notebook.", "- **Another might have more of a baked-in retrieval heuristic where they query an external resource by default as part of their operations.**", "- Another one yet may construct a hierarchy or other abstraction to deal with information.", "All these have been explored in the previous notebook in isolation, butagenticswould in theory allow any of these processes to be modularized behind an “agent interface” and be called as a sub-process that managed its own state.", "The only thing we would really need, is a buffer system that facilitates this transfer of information and a way to inform our systems how the other agents function. If we just do that, we can treat agents as tools and make arbitrarily modularized or arbitrarily complex pipelines for dealing with language and interacting with environments." ], "raw_notes": "The main thing here is the notion of disjoint states and potentially-disjoint policies, or data pipelines, for dealing with their inputs….\n- A particular agent could have a refinement pipeline similar to the dialog management chatbot or the long document processing examples in the previous notebook.\n- **Another might have more of a baked-in retrieval heuristic where they query an external resource by default as part of their operations.**\n- Another one yet may construct a hierarchy or other abstraction to deal with information.\nAll these have been explored in the previous notebook in isolation, butagenticswould in theory allow any of these processes to be modularized behind an “agent interface” and be called as a sub-process that managed its own state.\nThe only thing we would really need, is a buffer system that facilitates this transfer of information and a way to inform our systems how the other agents function. If we just do that, we can treat agents as tools and make arbitrarily modularized or arbitrarily complex pipelines for dealing with language and interacting with environments." }, { "slide_number": 183, "title": "Data Pipeline", "slide_body": [ "🤖", "Question", "Synthesize Answer", "🤖", "🤖", "Concurrent Processes", "Retrieval", "🤖", "🤖", "🤖", "🤖", "Update", "Answer", "Update", "Answer", "Update", "Answer", "Question", "Iterative Refinement", "Combine", "Per-Agent Behavior", "Dive into Deep Learning (d2l.ai)" ], "notes": [ "The main thing here is the notion of disjoint states and potentially-disjoint policies, or data pipelines, for dealing with their inputs….", "- A particular agent could have a refinement pipeline similar to the dialog management chatbot or the long document processing examples in the previous notebook.", "- Another might have more of a baked-in retrieval heuristic where they query an external resource by default as part of their operations.", "- **Another one yet may construct a hierarchy or other abstraction to deal with information.**", "All these have been explored in the previous notebook in isolation, butagenticswould in theory allow any of these processes to be modularized behind an “agent interface” and be called as a sub-process that managed its own state.", "The only thing we would really need, is a buffer system that facilitates this transfer of information and a way to inform our systems how the other agents function. If we just do that, we can treat agents as tools and make arbitrarily modularized or arbitrarily complex pipelines for dealing with language and interacting with environments." ], "raw_notes": "The main thing here is the notion of disjoint states and potentially-disjoint policies, or data pipelines, for dealing with their inputs….\n- A particular agent could have a refinement pipeline similar to the dialog management chatbot or the long document processing examples in the previous notebook.\n- Another might have more of a baked-in retrieval heuristic where they query an external resource by default as part of their operations.\n- **Another one yet may construct a hierarchy or other abstraction to deal with information.**\nAll these have been explored in the previous notebook in isolation, butagenticswould in theory allow any of these processes to be modularized behind an “agent interface” and be called as a sub-process that managed its own state.\nThe only thing we would really need, is a buffer system that facilitates this transfer of information and a way to inform our systems how the other agents function. If we just do that, we can treat agents as tools and make arbitrarily modularized or arbitrarily complex pipelines for dealing with language and interacting with environments." }, { "slide_number": 184, "title": "Local State Accumulation", "slide_body": [ "LLM", "Prompt", "Update Knowledge", "Input", "DB Lookup", "Prompt", "Format", "LLM", "Prompt", "Update Knowledge", "History", "User", "+Knowledge Base", "+Customer Info", "+Response", "184", "Per-Agent Behavior" ], "notes": [ "The main thing here is the notion of disjoint states and potentially-disjoint policies, or data pipelines, for dealing with their inputs….", "- A particular agent could have a refinement pipeline similar to the dialog management chatbot or the long document processing examples in the previous notebook.", "- Another might have more of a baked-in retrieval heuristic where they query an external resource by default as part of their operations.", "- Another one yet may construct a hierarchy or other abstraction to deal with information.", "**All these have been explored in the previous notebook in isolation, but****agentics****would in theory allow any of these processes to be modularized behind an “agent interface” and be called as a sub-process that managed its own state.**", "The only thing we would really need, is a buffer system that facilitates this transfer of information and a way to inform our systems how the other agents function. If we just do that, we can treat agents as tools and make arbitrarily modularized or arbitrarily complex pipelines for dealing with language and interacting with environments." ], "raw_notes": "The main thing here is the notion of disjoint states and potentially-disjoint policies, or data pipelines, for dealing with their inputs….\n- A particular agent could have a refinement pipeline similar to the dialog management chatbot or the long document processing examples in the previous notebook.\n- Another might have more of a baked-in retrieval heuristic where they query an external resource by default as part of their operations.\n- Another one yet may construct a hierarchy or other abstraction to deal with information.\n**All these have been explored in the previous notebook in isolation, but****agentics****would in theory allow any of these processes to be modularized behind an “agent interface” and be called as a sub-process that managed its own state.**\nThe only thing we would really need, is a buffer system that facilitates this transfer of information and a way to inform our systems how the other agents function. If we just do that, we can treat agents as tools and make arbitrarily modularized or arbitrarily complex pipelines for dealing with language and interacting with environments." }, { "slide_number": 185, "title": "Communication Protocols", "slide_body": [ "Agentic Behavior", "Agent Loop 1", "Agent Loop 2", "Interact", "Observe" ], "notes": [ "The main thing here is the notion of disjoint states and potentially-disjoint policies, or data pipelines, for dealing with their inputs….", "- A particular agent could have a refinement pipeline similar to the dialog management chatbot or the long document processing examples in the previous notebook.", "- Another might have more of a baked-in retrieval heuristic where they query an external resource by default as part of their operations.", "- Another one yet may construct a hierarchy or other abstraction to deal with information.", "All these have been explored in the previous notebook in isolation, butagenticswould in theory allow any of these processes to be modularized behind an “agent interface” and be called as a sub-process that managed its own state.", "**The only thing we would really need, is a buffer system that facilitates this transfer of information and a way to inform our systems how the other agents function. If we just do that, we can treat agents as tools and make arbitrarily modularized or arbitrarily complex pipelines for dealing with language and interacting with environments.**" ], "raw_notes": "The main thing here is the notion of disjoint states and potentially-disjoint policies, or data pipelines, for dealing with their inputs….\n- A particular agent could have a refinement pipeline similar to the dialog management chatbot or the long document processing examples in the previous notebook.\n- Another might have more of a baked-in retrieval heuristic where they query an external resource by default as part of their operations.\n- Another one yet may construct a hierarchy or other abstraction to deal with information.\nAll these have been explored in the previous notebook in isolation, butagenticswould in theory allow any of these processes to be modularized behind an “agent interface” and be called as a sub-process that managed its own state.\n**The only thing we would really need, is a buffer system that facilitates this transfer of information and a way to inform our systems how the other agents function. If we just do that, we can treat agents as tools and make arbitrarily modularized or arbitrarily complex pipelines for dealing with language and interacting with environments.**" }, { "slide_number": 186, "title": "Case For Agents", "slide_body": [ "Why Even Bother", "Compartmentalization", "Context bottlenecks", "Modularity/Independence", "Intuitive Design", "Main Agent Loop" ], "notes": [ "- **Case for Agents (Why Even Bother)**", "- Modular systems reduce context bottlenecks.", "- Each agent handles its part—SQL queries, document processing, etc.", "- Focus on dividing tasks into manageable modules.", "- Agents reduce the workload on the main loop by handling specialized logic independently." ], "raw_notes": "- **Case for Agents (Why Even Bother)**\n- Modular systems reduce context bottlenecks.\n- Each agent handles its part—SQL queries, document processing, etc.\n- Focus on dividing tasks into manageable modules.\n- Agents reduce the workload on the main loop by handling specialized logic independently." }, { "slide_number": 187, "title": "LangGraph", "slide_body": [ "Agent Development/Orchestration", "LangGraph" ], "notes": [ "LangGraphwill be used in the notebook. Please check it out to see roughly how it works, and consider checking out examples after the class regarding example implementations and off-the-shelf solutions from theLangChainteam!" ], "raw_notes": "LangGraphwill be used in the notebook. Please check it out to see roughly how it works, and consider checking out examples after the class regarding example implementations and off-the-shelf solutions from theLangChainteam!" }, { "slide_number": 188, "title": "Part 3: Stateful Systems", "slide_body": [ "•", "Towards Orchestration and Agentics", "•", "[Optional] Towards Multi-Agent Systems", "•", "[Assessment] Image Generator Pipeline" ], "notes": [ "**Welcome back—and congratulations!**", "We've**completed the course content**.", "- Covered how large language models are developed, implemented, and deployed.", "- Explored their use in cognitive architectures, multi-agent systems, chatbots, and processing applications.", "In this section, we'll introduce the**final assessment**—**Notebook 8**.", "We want you to go through it, finish it, and earn your certification." ], "raw_notes": "**Welcome back—and congratulations!**\nWe've**completed the course content**.\n- Covered how large language models are developed, implemented, and deployed.\n- Explored their use in cognitive architectures, multi-agent systems, chatbots, and processing applications.\nIn this section, we'll introduce the**final assessment**—**Notebook 8**.\nWe want you to go through it, finish it, and earn your certification." }, { "slide_number": 189, "title": "Data Pipeline", "slide_body": [ "🤖", "Question", "Synthesize Answer", "🤖", "🤖", "Concurrent Processes", "Retrieval", "🤖", "🤖", "🤖", "🤖", "Update", "Answer", "Update", "Answer", "Update", "Answer", "Question", "Iterative Refinement", "Combine", "Per-Agent Behavior", "Dive into Deep Learning (d2l.ai)" ], "notes": [ "**Recall**: We've talked a lot about**data pipelines**.", "- Learned how to set up systems that pass from one agent component to another.", "- It's all software—combining traditional software with**LLM interfaces**.", "- If you have an LLM predicting actions, you instruct it to predict in a specific structure.", "- Then, fulfill the request, provide necessary context, or transition to the next step." ], "raw_notes": "**Recall**: We've talked a lot about**data pipelines**.\n- Learned how to set up systems that pass from one agent component to another.\n- It's all software—combining traditional software with**LLM interfaces**.\n- If you have an LLM predicting actions, you instruct it to predict in a specific structure.\n- Then, fulfill the request, provide necessary context, or transition to the next step." }, { "slide_number": 190, "title": "Local State Accumulation", "slide_body": [ "LLM", "Prompt", "Update Knowledge", "Input", "DB Lookup", "Prompt", "Format", "LLM", "Prompt", "Update Knowledge", "History", "User", "+Knowledge Base", "+Customer Info", "+Response", "190", "Per-Agent Behavior" ], "notes": [ "- **Applying Concepts in Practice**", "- **You can build a cognitive architecture:**", " - Queries update states, accumulate context, and provide feedback.", " - Like a**dialogue-style loop**.", "- (Next) Or keep it simple:", " - Transition from input to processed versions sequentially.", " - Accumulate states, let states drop, or move between states as needed.", " - Define your pathways—for example:", " - Natural language input → model → output.", " - LLM output → algorithm for further handling.", "- The**final assessment**tests your ability to apply this in practice." ], "raw_notes": "- **Applying Concepts in Practice**\n- **You can build a cognitive architecture:**\n - Queries update states, accumulate context, and provide feedback.\n - Like a**dialogue-style loop**.\n- (Next) Or keep it simple:\n - Transition from input to processed versions sequentially.\n - Accumulate states, let states drop, or move between states as needed.\n - Define your pathways—for example:\n - Natural language input → model → output.\n - LLM output → algorithm for further handling.\n- The**final assessment**tests your ability to apply this in practice." }, { "slide_number": 191, "title": "Typical Data Pipeline", "slide_body": [ "LLM", "Prompt", "Update Knowledge", "Input", "DB Lookup", "Prompt", "Format", "LLM", "Prompt", "Update Knowledge", "+Knowledge Base", "+Customer Info", "+Response", "Transitioning From One Representation To Another" ], "notes": [ "- **Applying Concepts in Practice**", "- You can build a cognitive architecture:", " - Queries update states, accumulate context, and provide feedback.", " - Like a**dialogue-style loop**.", "- Or keep it simple:", " - Transition from input to processed versions sequentially.", " - Accumulate states, let states drop, or move between states as needed.", " - Define your pathways—for example:", " - Natural language input → model → output.", " - LLM output → algorithm for further handling.", "- (Up Next) The**final assessment**tests your ability to apply this in practice." ], "raw_notes": "- **Applying Concepts in Practice**\n- You can build a cognitive architecture:\n - Queries update states, accumulate context, and provide feedback.\n - Like a**dialogue-style loop**.\n- Or keep it simple:\n - Transition from input to processed versions sequentially.\n - Accumulate states, let states drop, or move between states as needed.\n - Define your pathways—for example:\n - Natural language input → model → output.\n - LLM output → algorithm for further handling.\n- (Up Next) The**final assessment**tests your ability to apply this in practice." }, { "slide_number": 192, "title": "Assessment Diffusion Pipeline", "slide_body": [ "VLM", "Prompt", "Describe", "Image", "SDXL", "+ Image Description", "+ Diffusion Prompt", "+ Diffusion", "Images", "Transitioning From Image To Images", "LLM", "Prompt", "Synthesize", "Image", "Image", "Image", "Image" ], "notes": [ "- **Final Assessment—Implementing a Diffusion Pipeline**", "- **Your task**:", " - Implement a**diffusion pipeline**that:", " - Takes in an image.", " - **Summarizes**the image.", " - Generates prompts to create new images.", " - Generates those new images.", "- Use skills from**Notebooks 4 through 7**.", "- **Instructions are in the notebook**.", "- Expected outcome:", " - **Four images per input image**.", " - Input images are from the file system.", " - Each is described using a**visual language model**.", " - Descriptions generate four synthetic prompts for the diffusion pipeline.", " - The diffusion model generates four images from those prompts.", "- **Total**: Four input images and**16 output images**(or more if you like).", "- Follow the instructions to get started.", "- After finishing and receiving a**success message**:", " - **Exit the environment**.", " - Go to the bottom where you set up your environment.", " - Click the**\"Assess Task\"**checkbox.", " - You'll receive a success message if everything works correctly.", "- **Hope you enjoyed the course and the assignment**." ], "raw_notes": "- **Final Assessment—Implementing a Diffusion Pipeline**\n- **Your task**:\n - Implement a**diffusion pipeline**that:\n - Takes in an image.\n - **Summarizes**the image.\n - Generates prompts to create new images.\n - Generates those new images.\n- Use skills from**Notebooks 4 through 7**.\n- **Instructions are in the notebook**.\n- Expected outcome:\n - **Four images per input image**.\n - Input images are from the file system.\n - Each is described using a**visual language model**.\n - Descriptions generate four synthetic prompts for the diffusion pipeline.\n - The diffusion model generates four images from those prompts.\n- **Total**: Four input images and**16 output images**(or more if you like).\n- Follow the instructions to get started.\n- After finishing and receiving a**success message**:\n - **Exit the environment**.\n - Go to the bottom where you set up your environment.\n - Click the**\"Assess Task\"**checkbox.\n - You'll receive a success message if everything works correctly.\n- **Hope you enjoyed the course and the assignment**." }, { "slide_number": 193, "title": "Thank You!", "slide_body": [], "notes": [], "raw_notes": "" } ]