{"url": "https://maven.com/p/2fe5a8/mastering-llm-application-testing", "core": {"title": "Mastering LLM Application Testing", "slug": "2fe5a8", "id": 3288, "page_type": "workshop", "description": "LLM-powered applications require constant iteration and evaluation to ensure robust and reliable outputs. In this Lightning Lesson, you’ll learn how to apply pytest to systematically test and refine your LLM apps. We’ll cover how to identify and address failure modes, evaluate outputs, and build confidence in your app’s performance.", "start_datetime": "2025-03-24T23:00:00Z", "duration_min": 30, "is_recording_public": true, "has_internal_recording": true, "instructor_name": "Hugo Bowne-Anderson", "instructor_title": "Podcaster, Educator, DS & ML expert", "instructor_headline": null, "course_name": "Building AI Applications for Data Scientists and Software Engineers", "course_slug": "building-ai-apps-ds-and-swe-from-first-principles", "school_name": "Hugo & Stefan", "school_slug": "hugo-stefan", "chapter_count": 13}, "next_data": {"props": {"pageProps": {"apiSchool": {"id": 10564, "slug": "hugo-stefan", "name": "Hugo & Stefan", "is_stripe_connected": true, "stripe_connect_account_id": "acct_1QHfdd4K0Yuq5oze", "zoom_auth_version": 2, "verified": true, "timezone": "America/Los_Angeles", "support_email": null, "school_tags": [{"id": 254, "tag_type": "internal_collection", "label": "Upmarket CLP", "slug": "upmarket-clp", "description": null, "parent_tag_id": null}], "instructor_bio": null}, "pageType": "workshop", "school": {"id": 10564, "slug": "hugo-stefan", "name": "Hugo & Stefan", "is_stripe_connected": true, "stripe_connect_account_id": "acct_1QHfdd4K0Yuq5oze", "zoom_auth_version": 2, "verified": true, "timezone": "America/Los_Angeles", "support_email": null, "school_tags": [{"id": 254, "tag_type": "internal_collection", "label": "Upmarket CLP", "slug": "upmarket-clp", "description": null, "parent_tag_id": null}], "instructor_bio": null}, "course": {"slug": "building-ai-apps-ds-and-swe-from-first-principles", "school_id": 10564, "name": "Building AI Applications for Data Scientists and Software Engineers", "id": 13822, "format": "full_course", "description": "Build AI-powered software reliably & from first principles. Learn the GenAI software development lifecycle: agents, evals, iteration & more", "attrs": {"courses_page_highlight_order_index": null, "custom_landing_page_url": null, "facebook_pixel_id": null, "is_course_application_enabled": false, "is_demoted_on_courses_page": false, "is_hidden_by_maven_staff": false, "is_live_on_courses_page": true, "short_description": null, "skip_landing_page": null, "satisfaction_guarantee_status": "unset"}, "social_image_url": "https://d2426xcxuh3ht5.cloudfront.net/GlniSWl9TSANvYOOW20g_Maven-LP.jpg", "course_image_url": null, "clp_hero_media": {"media_type": "image_url", "image_url": "https://d2426xcxuh3ht5.cloudfront.net/I0f0eOFTN6AyTKppPVla_maven-LP.png", "youtube_video_id": "dQw4w9WgXcQ", "vimeo_video_id": null}, "instructor_infos": [{"name": "Hugo Bowne-Anderson", "image_url": "https://d2426xcxuh3ht5.cloudfront.net/ognNAYkvSq4G08s4igqQ_hba-headshot.jpg", "image_url_no_bg": "https://user-assets.maven.com/instructor-no-bg/ognNAYkvSq4G08s4igqQ_hba-headshot.jpg", "uuid": "ae5c5ace-9dce-484e-970c-1efb020f858f", "bio_html": "<p>Hugo Bowne-Anderson is an independent data &amp; AI consultant with extensive experience in the tech industry. He has advised and taught teams building AI-powered systems, including engineers from Netflix, Meta, &amp; Amazon. He is the host of the industry Vanishing Gradients, where he explores cutting-edge developments in data science and artificial intelligence, &amp; has written for publications such as Harvard Business Review &amp; VentureBeat. Previously, Hugo served as Head of Developer Relations at Outerbounds &amp; held roles at Coiled &amp; DataCamp, where his work in data science education reached over 6 million learners. He has taught at Yale University, Cold Spring Harbor Laboratory, and conferences like SciPy and PyCon, &amp; is a passionate advocate for democratizing AI skills and open-source tools. </p>", "headline": "AI Builder/Educator (6 Million Students)", "title": "AI & data engineer, consultant, educator of 6+ million students (ex-Yale)", "linkedin_url": "https://www.linkedin.com/in/hugo-bowne-anderson-045939a5/", "twitter_handle": "https://x.com/hugobowne", "custom_social_link": null, "logos": {"logos": [], "label": null}, "highlights": null, "highlights_html": "<ul><li></li></ul>", "preferred_name": null}, {"name": "Stefan Krawczyk", "image_url": "https://d2426xcxuh3ht5.cloudfront.net/6WbbhIrxQLCj0VlK64FN_stefan_right_cropped_reduced.jpg", "image_url_no_bg": "https://user-assets.maven.com/instructor-no-bg/6WbbhIrxQLCj0VlK64FN_stefan_right_cropped_reduced.jpg", "uuid": "904e4b2d-2f8d-462f-9202-126821a742ba", "bio_html": "<p><a target=\"_blank\" rel=\"noopener noreferrer nofollow\" href=\"https://www.linkedin.com/in/skrawczyk?miniProfileUrn=urn%3Ali%3Afs_miniProfile%3AACoAAADNVx8BpkwwfnSHaBYK3Wfh9twVCOiSyGw\">Stefan Krawczyk</a> is a principal member of staff on Agentforce at Salesforce, working on building an enterprise agent platform. He is also the co-creator of two Apache Projects: <a target=\"_blank\" rel=\"noopener noreferrer nofollow\" href=\"https://github.com/dagworks-inc/hamilton\">Apache Ham</a>i<a target=\"_blank\" rel=\"noopener noreferrer nofollow\" href=\"https://github.com/dagworks-inc/hamilton\">lton</a> &amp; <a target=\"_blank\" rel=\"noopener noreferrer nofollow\" href=\"https://github.com/dagworks-inc/burr\">Apache Burr</a>. He was formerly the co-founder and CEO of <a target=\"_blank\" rel=\"noopener noreferrer nofollow\" href=\"http://ycombinator.com/companies/dagworks-inc\">DAGWorks</a> (YC23) whose mission was to empower developers to build reliable AI agents &amp; applications. He is a Y Combinator alum, StartX alum, and a Stanford graduate with a Master of Science in Computer Science with Distinction in Research. He has over thirteen years of experience in building and leading data &amp; ML-related systems and teams, at companies like Stitch Fix, Idibon, Nextdoor, and Linkedin.</p>", "headline": "AI Builder with 15 years of experience", "title": "Agentforce; Ex-Stitch Fix; 13+ Years building / productionizing data, ML, Agents", "linkedin_url": null, "twitter_handle": null, "custom_social_link": null, "logos": {"logos": [], "label": null}, "highlights": null, "highlights_html": "<ul><li></li></ul>", "preferred_name": null}], "instructor_caption": "AI & data engineer, consultant, educator of 6+ million students (ex-Yale)", "course_tags": [{"id": 31, "tag_type": "category", "label": "Data & engineering", "slug": "engineering-ll", "description": null, "parent_tag_id": null}, {"id": 35, "tag_type": "category", "label": "AI", "slug": "ai-ll", "description": null, "parent_tag_id": null}, {"id": 69, "tag_type": "internal_collection", "label": "Maven Selects", "slug": "maven-selects", "description": null, "parent_tag_id": null}, {"id": 80, "tag_type": "deprecated", "label": "AI/ML engineering", "slug": "x-subcategory-ai-ml-engineering", "description": null, "parent_tag_id": 35}, {"id": 81, "tag_type": "deprecated", "label": "Data science and engineering", "slug": "x-subcategory-data-science-and-engineering", "description": null, "parent_tag_id": 35}, {"id": 108, "tag_type": "deprecated", "label": "AI/ML engineering", "slug": "x-subcategory-ai-ml-engineering-1", "description": null, "parent_tag_id": 31}, {"id": 109, "tag_type": "deprecated", "label": "Data science and engineering", "slug": "x-subcategory-data-science-and-engineering-1", "description": null, "parent_tag_id": 31}, {"id": 110, "tag_type": "deprecated", "label": "Software engineering", "slug": "x-subcategory-software-engineering", "description": null, "parent_tag_id": 31}, {"id": 167, "tag_type": "public_collection", "label": "AI-Powered Professional", "slug": "ai-powered-professional", "description": null, "parent_tag_id": 35}, {"id": 172, "tag_type": "internal_collection", "label": "AI Affiliates", "slug": "ai-affiliates-nov-24", "description": null, "parent_tag_id": null}, {"id": 176, "tag_type": "public_collection", "label": "Fast Track 2024", "slug": "fast-track-2024", "description": null, "parent_tag_id": null}, {"id": 209, "tag_type": "internal_collection", "label": "Affiliate Hamel Husain", "slug": "affiliate-hamel-husain", "description": null, "parent_tag_id": null}, {"id": 213, "tag_type": "internal_collection", "label": "The AI-Powered Super IC", "slug": "ai-powered-super-ic", "description": null, "parent_tag_id": null}, {"id": 216, "tag_type": "internal_collection", "label": "[LEGACY] Instructor Owned CLC Opt-in", "slug": "legacy-instructor-owned-clc", "description": null, "parent_tag_id": null}, {"id": 234, "tag_type": "public_collection", "label": "Maven 100 - AI 2025", "slug": "maven-100-ai-2025", "description": null, "parent_tag_id": null}, {"id": 235, "tag_type": "public_collection", "label": "Maven 100 - Engineering 2025", "slug": "maven-100-engineering-2025", "description": null, "parent_tag_id": null}, {"id": 244, "tag_type": "internal_collection", "label": "Courses filtered for recommendations", "slug": "courses-filtered-for-recommendations", "description": null, "parent_tag_id": null}, {"id": 245, "tag_type": "internal_collection", "label": "maven-100-2025", "slug": "maven-100-2025", "description": null, "parent_tag_id": null}, {"id": 293, "tag_type": "persona", "label": "For Engineers", "slug": "for-engineers", "description": null, "parent_tag_id": null}, {"id": 299, "tag_type": "persona", "label": "For Data Scientists", "slug": "for-data-scientists", "description": null, "parent_tag_id": null}, {"id": 313, "tag_type": "topic", "label": "AI", "slug": "ai", "description": null, "parent_tag_id": null}, {"id": 315, "tag_type": "topic", "label": "Agentic AI", "slug": "agentic-ai", "description": null, "parent_tag_id": null}, {"id": 336, "tag_type": "topic", "label": "Working with LLMs", "slug": "llms", "description": null, "parent_tag_id": null}]}, "contentPageSlug": "2fe5a8", "contentPage": {"id": 3288, "school_id": 10564, "slug": "2fe5a8", "content_page_type": "workshop", "sections": [{"uuid": "e40cd4cb-44bf-4c5b-a81b-5d2329d89fb1", "title": "Mastering LLM Application Testing", "version": 1, "image_url": "https://d2426xcxuh3ht5.cloudfront.net/jfBoTpWwQpCBRBOJp66L_sZkURASaTSmJDbVUGBbh_DALL·E 2024-10-30 22.webp", "is_visible": true, "topic_desc": "LLM-powered applications require constant iteration and evaluation to ensure robust and reliable outputs. In this Lightning Lesson, you’ll learn how to apply pytest to systematically test and refine your LLM apps. We’ll cover how to identify and address failure modes, evaluate outputs, and build confidence in your app’s performance.", "brand_logos": {"label": "Previously at", "logos": [{"name": "LinkedIn", "type": "logo", "uuid": "780c4bc5-bada-4afe-9853-d7ecc8bd7bf6", "file_type": "svg", "image_url": "https://asset.brandfetch.io/idJFz6sAsl/idnwC5pGyX.svg"}, {"name": "Yale University", "type": "icon", "uuid": "71b8d37e-7cbb-44c5-954c-46cf304601f9", "file_type": "jpeg", "image_url": "https://asset.brandfetch.io/idLT-yPqwp/idyZishAwX.jpeg"}, {"name": "Stitch Fix", "type": "logo", "uuid": "b83c2924-5a5e-49d6-96c7-23f2d6de3975", "file_type": "svg", "image_url": "https://asset.brandfetch.io/idTgrRl8nA/idVnuVYfek.svg"}, {"name": "Stanford University", "type": "logo", "uuid": "2ddfef72-f6b3-437c-81e6-25a4543c062e", "file_type": "svg", "image_url": "https://asset.brandfetch.io/idPv3iQPET/idBFw-L5SX.svg"}, {"name": "New York University", "type": "logo", "uuid": "a3a91a21-9e77-43c9-a45b-d0f80d262f0a", "file_type": "svg", "image_url": "https://asset.brandfetch.io/idMj8iq0ev/idShvt3ozm.svg"}]}, "section_type": "main", "instructor_infos": [{"name": "Hugo Bowne-Anderson", "uuid": "dbc818e9-5568-4c42-924c-13c1767fac23", "title": "Podcaster, Educator, DS & ML expert", "bio_html": "<p>Hugo Bowne-Anderson is an independent data and AI consultant with extensive experience in the tech industry. He is the host of the industry Vanishing Gradients, where he explores cutting-edge developments in data science and artificial intelligence. As a data scientist, educator, evangelist, content marketer, and strategist, Hugo has worked with leading companies in the field. His past roles include Head of Developer Relations at Outerbounds, a company committed to building infrastructure for machine learning applications, and positions at Coiled and DataCamp, where he focused on scaling data science and online education respectively. Hugo's teaching experience spans from institutions like Yale University and Cold Spring Harbor Laboratory to conferences such as SciPy, PyCon, and ODSC. He has also worked with organizations like Data Carpentry to promote data literacy. His impact on data science education is significant, having developed over 30 courses on the DataCamp platform that have reached more than 3 million learners worldwide. Hugo also created and hosted the popular weekly data industry podcast DataFramed for two years. Committed to democratizing data skills and access to data science tools, Hugo advocates for open source software both for individuals and enterprises.</p>", "image_url": "https://d2426xcxuh3ht5.cloudfront.net/vzk3M8kRRmUnvD1XJolI_hba-headshot.jpg"}, {"name": "Stefan Krawczyk", "uuid": "bf30c55c-cb7c-44db-9b2f-21ed786a65c2", "title": "13+years in MLOps: Ex-Stitch Fix, Ex-Nextdoor, Ex-LinkedIn", "bio_html": "<p><a href=\"https://www.linkedin.com/in/skrawczyk?miniProfileUrn=urn%3Ali%3Afs_miniProfile%3AACoAAADNVx8BpkwwfnSHaBYK3Wfh9twVCOiSyGw\" rel=\"noopener noreferrer\" target=\"_blank\">Stefan Krawczyk</a>&nbsp;is the co-founder and CEO of <a href=\"https://www.dagworks.io/\" rel=\"noopener noreferrer\" target=\"_blank\">DAGWorks</a>, an open-source company driving two projects: <a href=\"https://github.com/dagworks-inc/hamilton\" rel=\"noopener noreferrer\" target=\"_blank\">Hamilton</a> &amp; <a href=\"https://github.com/dagworks-inc/burr\" rel=\"noopener noreferrer\" target=\"_blank\">Burr</a>, whose mission to empower developers to build reliable AI agents &amp; applications. He is a Y Combinator alum,&nbsp;StartX alum, and a Stanford graduate with a Master of Science in Computer Science with Distinction in Research. He has over thirteen years of experience in building and leading data &amp; ML-related systems and teams, at companies like Stitch Fix, Idibon, Nextdoor, and Linkedin, his passion is to make others more successful with data by bridging the engineering gap between data science, machine learning, artificial intelligence, and the business.</p>", "image_url": "https://d2426xcxuh3ht5.cloudfront.net/xN6E2t25R8aqn2vGSjLm_stefan.jpg"}], "learning_outcomes": [{"uuid": "b4f76615-1766-4ee5-a2a8-a34864874012", "title": "Write effective pytest cases for testing LLM outputs.", "description": ""}, {"uuid": "e55731d7-b9dd-4ded-a583-556da32a75e4", "title": "Build a framework for iterative improvement of LLM apps", "description": ""}, {"uuid": "32328c8b-7206-44b1-b066-f06ef202f756", "title": "Evaluate LLM results systematically to improve stability", "description": ""}]}], "created_at": "2026-02-12T20:58:48.419367Z", "updated_at": null, "theme": null, "attrs": {"type": "workshop", "is_canceled": false, "is_delisted": false, "connected_course_id": 13822, "course_stripe_promo_code_id": null}, "school_event": {"id": 3288, "slug": null, "title": "Mastering LLM Application Testing", "description": null, "start_datetime": "2025-03-24T23:00:00Z", "end_datetime": "2025-03-24T23:30:00Z", "start_date": "2025-03-25", "start_time": "10:00:00", "duration_min": 30, "timezone": "Australia/Sydney", "has_internal_recording": true, "is_recording_public": true}, "num_signups": 239}, "relatedWorkshops": [{"id": 361, "school_id": 10334, "is_visible_on_discovery_page": true, "is_canceled": false, "is_delisted": false, "connected_course_id": 11227, "workshop_tags": [{"id": 35, "tag_type": "category", "label": "AI", "slug": "ai-ll", "description": null, "parent_tag_id": null}, {"id": 137, "tag_type": "internal_collection", "label": "Instructor", "slug": "Instructor", "description": null, "parent_tag_id": null}, {"id": 271, "tag_type": "internal_collection", "label": "AI Evals", "slug": "ai-evals", "description": null, "parent_tag_id": null}, {"id": 290, "tag_type": "persona", "label": "For Product Managers", "slug": "for-product-managers", "description": null, "parent_tag_id": null}, {"id": 313, "tag_type": "topic", "label": "AI", "slug": "ai", "description": null, "parent_tag_id": null}], "published_content_page": {"id": 1190, "school_id": 10334, "slug": "45a0c2", "content_page_type": "workshop", "sections": [{"uuid": "12a7bc34-a308-4837-bcde-e5a93e962760", "title": "Evaluating LLMs for Your Applications", "version": 1, "image_url": "https://d2426xcxuh3ht5.cloudfront.net/0zds7xZLTNWfND9mcBNs_LLM Lightning Lesson Image .png", "is_visible": true, "topic_desc": "Evaluating GenAI applications is crucial for product managers to ensure that the chosen AI model meets not only technical specification but also business goals and user expectations. Understanding how to select the right model will  help you get started on the path to define model specific evaluation metrics, this will allow teams to iterate quickly and create impactful solutions.", "brand_logos": {"label": "Previously at", "logos": []}, "section_type": "main", "instructor_infos": [{"name": "Mahesh Yadav", "uuid": "09cb5b23-2d7d-4258-8f60-c79d5d007853", "title": "GenAI Product Lead at Google, previously at Meta, Amazon, and Microsoft", "bio_html": "<p>Mahesh Yadav is&nbsp;a&nbsp;<strong>Product Leader at Google GenAI team</strong>. Mahesh&nbsp;is one of the world's&nbsp;top AI executives&nbsp;and an&nbsp;<strong>award-winning AI Product Educator</strong>. His work on AI has been featured in the&nbsp;<strong>Nvidia GTC conference, Microsoft Build, and Meta blogs</strong>.</p><p><br></p><p>Mahesh has 20 years of experience in building products&nbsp;at <strong>Meta, Microsoft and AWS AI teams</strong>. Mahesh has worked in all layers of the AI stack from AI chips to LLM and has a deep understanding of how GenAI companies ship value to customers.</p><p><br></p><p>Currently, he leads&nbsp;<strong>AI agent at Google</strong> Cloud&nbsp;where it is used extensively for&nbsp;<strong>Gemini&nbsp;</strong>and other key Google products.</p>", "image_url": "https://d2426xcxuh3ht5.cloudfront.net/tAQehNkQOCcDjleGur5Q_MyImage.png"}], "learning_outcomes": [{"uuid": "750875e7-3e61-4e0f-81a2-187017715891", "title": "Framework for choosing the right LLMs", "description": "Identify the best GenAI model for your needs based on budget, latency and team expertise etc"}, {"uuid": "794fc156-3ba7-4fc9-ac76-0738795a0b45", "title": "Setting Evaluation Criteria", "description": "Learn how to set clear, actionable metrics despite challenges with benchmarks and real-world examples."}, {"uuid": "602124cf-8fd8-4f21-9029-82cc0b12d0a9", "title": "Case Study: Contract Processing Application", "description": "We'll use a contract processing application to illustrate how to apply the principles you just learn."}]}], "created_at": "2026-02-12T20:53:17.582460Z", "updated_at": null, "theme": null, "attrs": {"type": "workshop", "is_canceled": false, "is_delisted": false, "connected_course_id": 11227, "course_stripe_promo_code_id": null}, "school_event": {"id": 1190, "slug": null, "title": "Evaluating LLMs for Your Applications", "description": null, "start_datetime": "2024-07-09T16:00:00Z", "end_datetime": "2024-07-09T16:30:00Z", "start_date": "2024-07-09", "start_time": "09:00:00", "duration_min": 30, "timezone": "America/Los_Angeles", "has_internal_recording": true, "is_recording_public": true}, "num_signups": 1015}}, {"id": 184, "school_id": 10905, "is_visible_on_discovery_page": true, "is_canceled": false, "is_delisted": false, "connected_course_id": 11912, "workshop_tags": [{"id": 35, "tag_type": "category", "label": "AI", "slug": "ai-ll", "description": null, "parent_tag_id": null}, {"id": 136, "tag_type": "internal_collection", "label": "LL Partner", "slug": "ll-partner", "description": null, "parent_tag_id": null}, {"id": 271, "tag_type": "internal_collection", "label": "AI Evals", "slug": "ai-evals", "description": null, "parent_tag_id": null}, {"id": 293, "tag_type": "persona", "label": "For Engineers", "slug": "for-engineers", "description": null, "parent_tag_id": null}, {"id": 299, "tag_type": "persona", "label": "For Data Scientists", "slug": "for-data-scientists", "description": null, "parent_tag_id": null}, {"id": 313, "tag_type": "topic", "label": "AI", "slug": "ai", "description": null, "parent_tag_id": null}], "published_content_page": {"id": 1013, "school_id": 10905, "slug": "71bee3", "content_page_type": "workshop", "sections": [{"uuid": "b3b15233-2dee-43cc-add5-ca5be1e4ccfd", "title": "Master Evaluation Techniques for LLM Apps", "version": 1, "image_url": "https://d2426xcxuh3ht5.cloudfront.net/cXdxvxLRT2eMzRzmdMH1_Lightning Lesson Template (1).png", "is_visible": true, "topic_desc": "Evaluating LLM applications is crucial for AI teams to ensure the effectiveness and reliability of AI systems.\n\nMastering evaluation techniques helps AI teams increase development speed, drive better business outcomes, and maintain a competitive edge.\n\nAn ureliable LLM app can lead to poor decision-making, user dissatisfaction, and potential security vulnerabilities.", "brand_logos": {"label": "Based on best practices from AI teams at", "logos": [{"name": "OpenAI", "type": "logo", "uuid": "b74607c5-a317-4ccb-805d-665cc27cf1d2", "file_type": "svg", "image_url": "https://asset.brandfetch.io/idR3duQxYl/idpkYOpXVV.svg"}, {"name": " Intercom", "type": "logo", "uuid": "7223454a-ad38-43f5-af99-0adc4d897b8e", "file_type": "svg", "image_url": "https://asset.brandfetch.io/idJRrn3vuU/idDUlDMkm1.svg"}, {"name": "Retool", "type": "logo", "uuid": "c18e57a3-344a-4bf6-98f1-a6cbf4123850", "file_type": "svg", "image_url": "https://asset.brandfetch.io/id3V8wH0I2/idwcsAhBma.svg"}, {"name": "Hex", "type": "icon", "uuid": "5f1ab69b-370c-4682-a361-03eaf8e0036e", "file_type": "png", "image_url": "https://asset.brandfetch.io/id0DD2N_HT/idrF-A6V86.png"}, {"name": "Airtable", "type": "logo", "uuid": "dae4f3d7-5fcb-41ed-8396-b11bfd716c46", "file_type": "svg", "image_url": "https://asset.brandfetch.io/iddsnRzkxS/idcxnDcOV0.svg"}]}, "section_type": "main", "instructor_infos": [{"name": "Haroon Choudery", "uuid": "89d8f99e-7e37-4a44-90f1-6d453ce9d79a", "title": "CEO, Autoblocks AI", "bio_html": "<p>Haroon is the Co-Founder &amp; CEO at Autoblocks AI, one of the first-ever AI evaluation platforms. </p><p>He is also the host of the Building With AI podcast, where he interviews &amp; distills best practices from top AI product teams at companies like OpenAI, Intercom, and Retool. </p><p>Haroon previously founded one of the top AI literacy nonprofits in the US, AI For Anyone, and is passionate about democratizing knowledge about AI.</p>", "image_url": "https://d2426xcxuh3ht5.cloudfront.net/xBTXPHEbR3avcbWsrI32_main-headshot copy.jpeg"}], "learning_outcomes": [{"uuid": "e42c567c-4bcc-4d31-8fe4-5a20b3adf87e", "title": "Why evaluations are necessary", "description": "Learn what role evaluations play in LLM apps and why they are crucial ensuring their effectiveness & reliability."}, {"uuid": "0673118a-1b32-4182-8d4f-36029f3919c9", "title": "How to choose the right evaluations", "description": "Explore how to select the best evaluation techniques (i.e., rules-based evals or LLM judges) for your LLM use case"}, {"uuid": "ca83b58f-79b9-470a-8733-ae9b898f7d52", "title": "Improving evaluation reliability", "description": "Discover methods, like fine-tuning & expert alignment, for improving the consistency and accuracy of your evaluations."}]}], "created_at": "2026-02-12T20:52:46.288885Z", "updated_at": null, "theme": null, "attrs": {"type": "workshop", "is_canceled": false, "is_delisted": false, "connected_course_id": 11912, "course_stripe_promo_code_id": null}, "school_event": {"id": 1013, "slug": null, "title": "Master Evaluation Techniques for LLM Apps", "description": null, "start_datetime": "2024-06-04T17:30:00Z", "end_datetime": "2024-06-04T18:00:00Z", "start_date": "2024-06-04", "start_time": "13:30:00", "duration_min": 30, "timezone": "America/New_York", "has_internal_recording": true, "is_recording_public": true}, "num_signups": 411}}, {"id": 76, "school_id": 6332, "is_visible_on_discovery_page": true, "is_canceled": false, "is_delisted": false, "connected_course_id": 6702, "workshop_tags": [{"id": 35, "tag_type": "category", "label": "AI", "slug": "ai-ll", "description": null, "parent_tag_id": null}, {"id": 293, "tag_type": "persona", "label": "For Engineers", "slug": "for-engineers", "description": null, "parent_tag_id": null}, {"id": 299, "tag_type": "persona", "label": "For Data Scientists", "slug": "for-data-scientists", "description": null, "parent_tag_id": null}, {"id": 313, "tag_type": "topic", "label": "AI", "slug": "ai", "description": null, "parent_tag_id": null}], "published_content_page": {"id": 902, "school_id": 6332, "slug": "a13872", "content_page_type": "workshop", "sections": [{"uuid": "da135695-cad8-4026-8ccc-f2c62c6bc978", "title": "Building Advanced Applications with LLMs", "version": 1, "image_url": "https://d2426xcxuh3ht5.cloudfront.net/Xl8LMZEoQSe54O6RgajN_lightning.png", "is_visible": true, "topic_desc": "Building out common use cases and applications with LLMs requires an understanding of how to leverage advanced prompting techniques and tools. From there, we will show you how to effectively build complex applications (e.g., agentic workflows and RAG systems) with LLMs.", "brand_logos": {"label": "Previously at", "logos": [{"name": "Meta", "type": "logo", "uuid": "d8cb8658-388d-47c1-bf67-9bdb9da70b57", "file_type": "svg", "image_url": "https://asset.brandfetch.io/idWvz5T3V7/idpvCl_Ipb.svg"}, {"name": "Elastic", "type": "logo", "uuid": "226807f4-eb10-4edf-bbf6-f2ea93457860", "file_type": "svg", "image_url": "https://asset.brandfetch.io/idRKAFOKVG/idoEJANxs3.svg"}, {"name": "Papers with Code", "type": "logo", "uuid": "023f88a4-0280-4eb9-9d52-552bf1718c3d", "file_type": "svg", "image_url": "https://asset.brandfetch.io/id-lOQ3HNi/idoBQX6msj.svg"}]}, "section_type": "main", "instructor_infos": [{"name": "Elvis Saravia", "uuid": "738463a2-698a-4f05-8c49-07899983c2df", "title": "Founder & Lead AI Scientist @ DAIR.AI | Prev: Meta AI, Galactica LLM, Elastic\n ", "bio_html": "<p><br></p>", "image_url": "https://d2426xcxuh3ht5.cloudfront.net/NT1OLyIQ62QNcoJeEu3E_elvis-photo-200.jpg"}], "learning_outcomes": [{"uuid": "a96064f9-978a-4320-8086-de836b8997f1", "title": "Getting Started", "description": "Learn advanced prompting techniques like chain-of-thought and few-shot learning for building LLM applications"}, {"uuid": "2384fefa-c856-45d1-90a3-86a235758ad4", "title": "The Tools", "description": "Familiarize yourself with tools like Flowise AI and LangChain for building applications with LLMs"}, {"uuid": "88563c6f-d5e9-484c-a244-c9297babfd7c", "title": "Building Chatflows", "description": "Bringing everything together to effectively build applications that combine LLMs with external tools and data sources"}]}], "created_at": "2026-02-12T20:52:33.340542Z", "updated_at": null, "theme": null, "attrs": {"type": "workshop", "is_canceled": false, "is_delisted": false, "connected_course_id": 6702, "course_stripe_promo_code_id": null}, "school_event": {"id": 902, "slug": null, "title": "Building Advanced Applications with LLMs", "description": null, "start_datetime": "2024-05-23T18:00:00Z", "end_datetime": "2024-05-23T18:30:00Z", "start_date": "2024-05-23", "start_time": "12:00:00", "duration_min": 30, "timezone": "America/Belize", "has_internal_recording": true, "is_recording_public": true}, "num_signups": 502}}], "exploreCourses": null, "workshopTags": [{"id": 137, "tag_type": "internal_collection", "label": "Instructor", "slug": "Instructor", "description": null, "parent_tag_id": null}, {"id": 31, "tag_type": "category", "label": "Data & engineering", "slug": "engineering-ll", "description": null, "parent_tag_id": null}, {"id": 35, "tag_type": "category", "label": "AI", "slug": "ai-ll", "description": null, "parent_tag_id": null}, {"id": 271, "tag_type": "internal_collection", "label": "AI Evals", "slug": "ai-evals", "description": null, "parent_tag_id": null}, {"id": 293, "tag_type": "persona", "label": "For Engineers", "slug": "for-engineers", "description": null, "parent_tag_id": null}, {"id": 313, "tag_type": "topic", "label": "AI", "slug": "ai", "description": null, "parent_tag_id": null}], "videoChapters": [{"title": "Introduction and Welcome", "start_seconds": 0}, {"title": "Why Testing LLM Applications is Crucial", "start_seconds": 70}, {"title": "Agenda and Course Overview", "start_seconds": 196}, {"title": "The Iterative Software Development Life Cycle for AI", "start_seconds": 245}, {"title": "Case Study: LinkedIn Profile Scraper", "start_seconds": 377}, {"title": "Challenges in Structured Data Extraction", "start_seconds": 478}, {"title": "From Prototyping to Production: The Development Workflow", "start_seconds": 653}, {"title": "Introduction to PyTest for LLM Testing", "start_seconds": 795}, {"title": "Live Coding Demo: Using PyTest to Test an LLM Call", "start_seconds": 944}, {"title": "Scaling Tests and Collaborating with Domain Experts", "start_seconds": 1323}, {"title": "Course Promotion and Q&A", "start_seconds": 1446}, {"title": "Discussion: Evaluation vs. Testing", "start_seconds": 1602}, {"title": "Conclusion and Final Remarks", "start_seconds": 1770}], "navPages": [{"category": {"slug": "ai", "href": "/courses/ai", "title": "AI", "palette": {"--palette-category-base": "#460952", "--palette-category-highlight": "#931FAA", "--palette-category-border": "#EDA6FB"}}, "topics": [{"title": "Agentic AI", "slug": "ai/agentic-ai", "href": "/courses/ai/agentic-ai"}, {"title": "Coding with AI", "slug": "ai/ai-coding", "href": "/courses/ai/ai-coding"}, {"title": "AI Workflows", "slug": "ai/ai-workflows", "href": "/courses/ai/ai-workflows"}, {"title": "Claude Code", "slug": "ai/claude-code", "href": "/courses/ai/claude-code"}, {"title": "OpenClaw", "slug": "ai/openclaw", "href": "/courses/ai/openclaw"}, {"title": "Vibe Coding", "slug": "ai/vibe-coding", "href": "/courses/ai/vibe-coding"}, {"title": "AI Evals", "slug": "ai/evals", "href": "/courses/ai/evals"}, {"title": "AI Transformation", "slug": "ai/ai-transformation", "href": "/courses/ai/ai-transformation"}, {"title": "RAG & Search", "slug": "ai/rag-search", "href": "/courses/ai/rag-search"}, {"title": "MCP", "slug": "ai/mcp", "href": "/courses/ai/mcp"}, {"title": "AI for PMs", "slug": "ai/for-product-managers", "href": "/courses/ai/for-product-managers"}, {"title": "AI for Engineers", "slug": "ai/for-engineers", "href": "/courses/ai/for-engineers"}, {"title": "AI for Designers", "slug": "ai/for-designers", "href": "/courses/ai/for-designers"}, {"title": "AI for Marketers", "slug": "ai/for-marketers", "href": "/courses/ai/for-marketers"}, {"title": "AI for Founders", "slug": "ai/for-founders", "href": "/courses/ai/for-founders"}]}, {"category": {"slug": "product", "href": "/courses/product", "title": "Product", "palette": {"--palette-category-base": "#510B00", "--palette-category-highlight": "#B33600", "--palette-category-border": "#F7804D"}}, "topics": [{"title": "AI for PMs", "slug": "for-product-managers/ai", "href": "/courses/for-product-managers/ai"}, {"title": "Agentic AI", "slug": "for-product-managers/agentic-ai", "href": "/courses/for-product-managers/agentic-ai"}, {"title": "AI Evals", "slug": "for-product-managers/ai-evals", "href": "/courses/for-product-managers/ai-evals"}, {"title": "Vibe Coding", "slug": "for-product-managers/vibe-coding", "href": "/courses/for-product-managers/vibe-coding"}, {"title": "Product Sense", "slug": "for-product-managers/product-sense", "href": "/courses/for-product-managers/product-sense"}, {"title": "Product Discovery", "slug": "for-product-managers/product-discovery", "href": "/courses/for-product-managers/product-discovery"}, {"title": "User Research", "slug": "for-product-managers/user-research", "href": "/courses/for-product-managers/user-research"}, {"title": "Prototyping", "slug": "for-product-managers/prototyping", "href": "/courses/for-product-managers/prototyping"}, {"title": "Growth", "slug": "for-product-managers/growth", "href": "/courses/for-product-managers/growth"}, {"title": "Analytics", "slug": "for-product-managers/analytics", "href": "/courses/for-product-managers/analytics"}, {"title": "Tech Foundations", "slug": "for-product-managers/technical-foundations", "href": "/courses/for-product-managers/technical-foundations"}, {"title": "Strategy", "slug": "for-product-managers/strategy", "href": "/courses/for-product-managers/strategy"}, {"title": "Influence", "slug": "for-product-managers/influence", "href": "/courses/for-product-managers/influence"}, {"title": "Leadership", "slug": "for-product-managers/leadership", "href": "/courses/for-product-managers/leadership"}, {"title": "Career Growth", "slug": "for-product-managers/career-growth", "href": "/courses/for-product-managers/career-growth"}]}, {"category": {"slug": "engineering", "href": "/courses/engineering", "title": "Engineering", "palette": {"--palette-category-base": "#00332F", "--palette-category-highlight": "#00746B", "--palette-category-border": "#88B7B2"}}, "topics": [{"title": "AI for Engineers", "slug": "for-engineers/ai", "href": "/courses/for-engineers/ai"}, {"title": "Agentic AI", "slug": "for-engineers/agentic-ai", "href": "/courses/for-engineers/agentic-ai"}, {"title": "Coding with AI", "slug": "for-engineers/ai-coding", "href": "/courses/for-engineers/ai-coding"}, {"title": "Claude Code", "slug": "for-engineers/claude-code", "href": "/courses/for-engineers/claude-code"}, {"title": "OpenClaw", "slug": "for-engineers/openclaw", "href": "/courses/for-engineers/openclaw"}, {"title": "MCP", "slug": "for-engineers/mcp", "href": "/courses/for-engineers/mcp"}, {"title": "RAG & Search", "slug": "for-engineers/rag-search", "href": "/courses/for-engineers/rag-search"}, {"title": "AI Evals", "slug": "for-engineers/ai-evals", "href": "/courses/for-engineers/ai-evals"}, {"title": "Machine Learning", "slug": "for-engineers/ml", "href": "/courses/for-engineers/ml"}, {"title": "LLM Ops", "slug": "for-engineers/llm-ops", "href": "/courses/for-engineers/llm-ops"}, {"title": "Context Eng", "slug": "for-engineers/context-engineering", "href": "/courses/for-engineers/context-engineering"}, {"title": "Security", "slug": "for-engineers/security", "href": "/courses/for-engineers/security"}, {"title": "System Design", "slug": "for-engineers/system-design", "href": "/courses/for-engineers/system-design"}, {"title": "Leadership", "slug": "for-engineers/leadership", "href": "/courses/for-engineers/leadership"}, {"title": "Career Growth", "slug": "for-engineers/career-growth", "href": "/courses/for-engineers/career-growth"}]}, {"category": {"slug": "design", "href": "/courses/design", "title": "Design", "palette": {"--palette-category-base": "#38085F", "--palette-category-highlight": "#772CBB", "--palette-category-border": "#B5A2D0"}}, "topics": [{"title": "AI for Designers", "slug": "for-designers/ai", "href": "/courses/for-designers/ai"}, {"title": "Agentic AI", "slug": "for-designers/agentic-ai", "href": "/courses/for-designers/agentic-ai"}, {"title": "Vibe Coding", "slug": "for-designers/vibe-coding", "href": "/courses/for-designers/vibe-coding"}, {"title": "Prototyping", "slug": "for-designers/prototyping", "href": "/courses/for-designers/prototyping"}, {"title": "Figma", "slug": "for-designers/figma", "href": "/courses/for-designers/figma"}, {"title": "Design Systems", "slug": "for-designers/design-systems", "href": "/courses/for-designers/design-systems"}, {"title": "User Research", "slug": "for-designers/user-research", "href": "/courses/for-designers/user-research"}, {"title": "Product Discovery", "slug": "for-designers/product-discovery", "href": "/courses/for-designers/product-discovery"}, {"title": "UX", "slug": "for-designers/ux", "href": "/courses/for-designers/ux"}, {"title": "UI", "slug": "for-designers/ui", "href": "/courses/for-designers/ui"}, {"title": "Visual Design", "slug": "for-designers/visual-design", "href": "/courses/for-designers/visual-design"}, {"title": "Design Strategy", "slug": "for-designers/strategy", "href": "/courses/for-designers/strategy"}, {"title": "Influence", "slug": "for-designers/influence", "href": "/courses/for-designers/influence"}, {"title": "Leadership", "slug": "for-designers/leadership", "href": "/courses/for-designers/leadership"}, {"title": "Career Growth", "slug": "for-designers/career-growth", "href": "/courses/for-designers/career-growth"}]}, {"category": {"slug": "marketing", "href": "/courses/marketing", "title": "Marketing", "palette": {"--palette-category-base": "#431532", "--palette-category-highlight": "#943170", "--palette-category-border": "#C99EB5"}}, "topics": [{"title": "AI for Marketers", "slug": "for-marketers/ai", "href": "/courses/for-marketers/ai"}, {"title": "Agentic AI", "slug": "for-marketers/agentic-ai", "href": "/courses/for-marketers/agentic-ai"}, {"title": "Vibe Coding", "slug": "for-marketers/vibe-coding", "href": "/courses/for-marketers/vibe-coding"}, {"title": "Automation", "slug": "for-marketers/marketing-automation", "href": "/courses/for-marketers/marketing-automation"}, {"title": "Content Marketing", "slug": "for-marketers/content-marketing", "href": "/courses/for-marketers/content-marketing"}, {"title": "Demand Gen", "slug": "for-marketers/demand-generation", "href": "/courses/for-marketers/demand-generation"}, {"title": "Go-to-Market", "slug": "for-marketers/gtm", "href": "/courses/for-marketers/gtm"}, {"title": "Product Marketing", "slug": "for-marketers/product-marketing", "href": "/courses/for-marketers/product-marketing"}, {"title": "Positioning", "slug": "for-marketers/positioning", "href": "/courses/for-marketers/positioning"}, {"title": "Social Media", "slug": "for-marketers/social-media", "href": "/courses/for-marketers/social-media"}, {"title": "Brand", "slug": "for-marketers/brand", "href": "/courses/for-marketers/brand"}, {"title": "B2B Marketing", "slug": "for-marketers/b2b", "href": "/courses/for-marketers/b2b"}, {"title": "SEO & AEO", "slug": "for-marketers/seo-aeo", "href": "/courses/for-marketers/seo-aeo"}, {"title": "Strategy", "slug": "for-marketers/strategy", "href": "/courses/for-marketers/strategy"}, {"title": "Leadership", "slug": "for-marketers/leadership", "href": "/courses/for-marketers/leadership"}]}, {"category": {"slug": "leadership", "href": "/courses/leadership", "title": "Leadership", "palette": {"--palette-category-base": "#50090E", "--palette-category-highlight": "#A9202A", "--palette-category-border": "#D49C98"}}, "topics": [{"title": "AI for Leaders", "slug": "for-leaders/ai", "href": "/courses/for-leaders/ai"}, {"title": "Agentic AI", "slug": "for-leaders/agentic-ai", "href": "/courses/for-leaders/agentic-ai"}, {"title": "AI Transformation", "slug": "for-leaders/ai-transformation", "href": "/courses/for-leaders/ai-transformation"}, {"title": "AI Governance", "slug": "for-leaders/ai-governance", "href": "/courses/for-leaders/ai-governance"}, {"title": "Communication", "slug": "for-leaders/communication", "href": "/courses/for-leaders/communication"}, {"title": "Influence", "slug": "for-leaders/influence", "href": "/courses/for-leaders/influence"}, {"title": "Strategy", "slug": "for-leaders/strategy", "href": "/courses/for-leaders/strategy"}, {"title": "Management", "slug": "for-leaders/management", "href": "/courses/for-leaders/management"}, {"title": "People Operations", "slug": "for-leaders/people-operations", "href": "/courses/for-leaders/people-operations"}, {"title": "Exec Presence", "slug": "for-leaders/executive-presence", "href": "/courses/for-leaders/executive-presence"}, {"title": "Storytelling", "slug": "for-leaders/storytelling", "href": "/courses/for-leaders/storytelling"}, {"title": "Goal-setting", "slug": "for-leaders/goal-setting", "href": "/courses/for-leaders/goal-setting"}, {"title": "Personal Brand", "slug": "for-leaders/personal-brand", "href": "/courses/for-leaders/personal-brand"}, {"title": "Career Growth", "slug": "for-leaders/career-growth", "href": "/courses/for-leaders/career-growth"}]}, {"category": {"slug": "founders", "href": "/courses/founders", "title": "Founders", "palette": {"--palette-category-base": "#0C3209", "--palette-category-highlight": "#077202", "--palette-category-border": "#69C662"}}, "topics": [{"title": "AI for Founders", "slug": "for-founders/ai", "href": "/courses/for-founders/ai"}, {"title": "Agentic AI", "slug": "for-founders/agentic-ai", "href": "/courses/for-founders/agentic-ai"}, {"title": "AI Workflows", "slug": "for-founders/ai-workflows", "href": "/courses/for-founders/ai-workflows"}, {"title": "Vibe Coding", "slug": "for-founders/vibe-coding", "href": "/courses/for-founders/vibe-coding"}, {"title": "Prototyping", "slug": "for-founders/prototyping", "href": "/courses/for-founders/prototyping"}, {"title": "Product Sense", "slug": "for-founders/product-sense", "href": "/courses/for-founders/product-sense"}, {"title": "Positioning", "slug": "for-founders/positioning", "href": "/courses/for-founders/positioning"}, {"title": "Product Discovery", "slug": "for-founders/product-discovery", "href": "/courses/for-founders/product-discovery"}, {"title": "Management", "slug": "for-founders/management", "href": "/courses/for-founders/management"}, {"title": "Strategy", "slug": "for-founders/strategy", "href": "/courses/for-founders/strategy"}, {"title": "Go-to-Market", "slug": "for-founders/gtm", "href": "/courses/for-founders/gtm"}, {"title": "Personal Brand", "slug": "for-founders/personal-brand", "href": "/courses/for-founders/personal-brand"}, {"title": "Leadership", "slug": "for-founders/leadership", "href": "/courses/for-founders/leadership"}, {"title": "Fundraising", "slug": "for-founders/fundraising", "href": "/courses/for-founders/fundraising"}, {"title": "PMF", "slug": "for-founders/product-market-fit", "href": "/courses/for-founders/product-market-fit"}]}, {"category": {"slug": "more", "href": "/courses/more", "title": "More", "palette": {"--palette-category-base": "#1E2C2B", "--palette-category-highlight": "#4A6361", "--palette-category-border": "#B9C1BE"}}, "topics": [{"title": "Everyone", "slug": "for-everyone", "href": "/courses/for-everyone"}, {"title": "Operators", "slug": "for-operators", "href": "/courses/for-operators"}, {"title": "Data Scientists", "slug": "for-data-scientists", "href": "/courses/for-data-scientists"}, {"title": "Business Analysts", "slug": "for-business-analysts", "href": "/courses/for-business-analysts"}, {"title": "User Researchers", "slug": "for-user-researchers", "href": "/courses/for-user-researchers"}, {"title": "Customer Success", "slug": "for-customer-success", "href": "/courses/for-customer-success"}, {"title": "Project Managers", "slug": "for-project-managers", "href": "/courses/for-project-managers"}, {"title": "HR Professionals", "slug": "for-hr-professionals", "href": "/courses/for-hr-professionals"}, {"title": "Sales People", "slug": "for-sales-people", "href": "/courses/for-sales-people"}, {"title": "Lawyers", "slug": "for-lawyers", "href": "/courses/for-lawyers"}, {"title": "Finance", "slug": "for-finance-professionals", "href": "/courses/for-finance-professionals"}, {"title": "Investors", "slug": "for-investors", "href": "/courses/for-investors"}, {"title": "Real Estate", "slug": "for-real-estate-agents", "href": "/courses/for-real-estate-agents"}, {"title": "Educators", "slug": "for-educators", "href": "/courses/for-educators"}, {"title": "Creators", "slug": "for-creators", "href": "/courses/for-creators"}]}], "navTagLinks": [{"tag": {"id": 293, "tag_type": "persona", "label": "For Engineers", "slug": "for-engineers", "description": null, "parent_tag_id": null}, "link_slug": "for-engineers"}], "now": "2026-05-19T22:54:28.103Z"}, "__N_SSG": true}, "page": "/p/[slug]/[...ignored]", "query": {"slug": "2fe5a8", "ignored": ["mastering-llm-application-testing"]}, "buildId": "qPlLjBFLQ9fnkFKJ7n7E-", "isFallback": false, "isExperimentalCompile": false, "gsp": true, "scriptLoader": []}}