{"url": "https://maven.com/p/063cd6/evaluation-driven-development-for-agentic-ai-systems", "core": {"title": "Evaluation Driven Development for Agentic AI Systems", "slug": "063cd6", "id": 5325, "page_type": "workshop", "description": "Evals are notoriously difficult to implement in real world Agentic AI applications. Nevertheless, we cannot build reliable and maintainable Agentic AI Systems without implementing robust evaluation strategies. As an AI Engineer you will spend a large portion of your day-to-day making sure that these strategies maintain the robustness and correctness of the systems you are building out.", "start_datetime": "2025-09-25T16:00:00Z", "duration_min": 45, "is_recording_public": true, "has_internal_recording": true, "instructor_name": "Aurimas Griciūnas", "instructor_title": "LinkedIn Top Voice in AI • Founder & CEO @ SwirlAI • Former CPO @ Neptune.ai", "instructor_headline": null, "course_name": "End-to-End AI Engineering Bootcamp", "course_slug": "end-to-end-ai-engineering", "school_name": "SwirlAI", "school_slug": "swirl-ai", "chapter_count": 24}, "next_data": {"props": {"pageProps": {"apiSchool": {"id": 14202, "slug": "swirl-ai", "name": "SwirlAI", "is_stripe_connected": true, "stripe_connect_account_id": "acct_1Rxjfd3x3DkVdYB4", "zoom_auth_version": 2, "verified": true, "timezone": "Europe/Vilnius", "support_email": null, "school_tags": [{"id": 413, "tag_type": "internal_collection", "label": "[Beta] New SKU's Opt-in (2026)", "slug": "beta-new-skus-2026", "description": null, "parent_tag_id": null}], "instructor_bio": null}, "pageType": "workshop", "school": {"id": 14202, "slug": "swirl-ai", "name": "SwirlAI", "is_stripe_connected": true, "stripe_connect_account_id": "acct_1Rxjfd3x3DkVdYB4", "zoom_auth_version": 2, "verified": true, "timezone": "Europe/Vilnius", "support_email": null, "school_tags": [{"id": 413, "tag_type": "internal_collection", "label": "[Beta] New SKU's Opt-in (2026)", "slug": "beta-new-skus-2026", "description": null, "parent_tag_id": null}], "instructor_bio": null}, "course": {"slug": "end-to-end-ai-engineering", "school_id": 14202, "name": "End-to-End AI Engineering Bootcamp", "id": 15766, "format": "full_course", "description": "Master end-to-end AI engineering - transform prototypes into production-ready apps with LLMs, RAG & agents in just 8 weeks.", "attrs": {"courses_page_highlight_order_index": null, "custom_landing_page_url": null, "facebook_pixel_id": null, "is_course_application_enabled": false, "is_demoted_on_courses_page": false, "is_hidden_by_maven_staff": false, "is_live_on_courses_page": true, "short_description": null, "skip_landing_page": null, "satisfaction_guarantee_status": "unset"}, "social_image_url": "https://d2426xcxuh3ht5.cloudfront.net/nvJPVRRSR6YrWbrhbmxH_Copy%20of%20Copy%20of%20Course%20landing.jpg", "course_image_url": null, "clp_hero_media": {"media_type": "image_url", "image_url": "https://d2426xcxuh3ht5.cloudfront.net/RBAjVfKFSTWEcxNlGQHs_Copy of Course landing (2).jpg", "youtube_video_id": "dQw4w9WgXcQ", "vimeo_video_id": null}, "instructor_infos": [{"name": "Aurimas Griciunas", "image_url": "https://user-assets.maven.com/instructor-avatars/end-to-end-ai-engineering/aurimas-griciunas-8100b589.png", "image_url_no_bg": "https://user-assets.maven.com/instructor-avatars/end-to-end-ai-engineering/aurimas-griciunas-bbabfdd1-no-bg.png", "uuid": "e2a0279e-860d-4a6c-bc20-80837728651a", "bio_html": "<p>Aurimas Griciūnas is a recognized AI expert, LinkedIn Top Voice in AI, and the founder of SwirlAI. He previously served as Chief Product Officer at Neptune.ai where he worked closely with top ML teams to scale infrastructure, evaluation, and LLMOps practices across industries. With over a decade of experience at the intersection of data science, machine learning, and software engineering, Aurimas has led AI initiatives in both startups and enterprise environments. His mission is to bridge the gap between hype and reality by teaching engineers how to build systems that work in the real world. Students will benefit from his hands-on knowledge, technical depth, and product-first mindset - gained by solving actual engineering problems.</p>", "headline": "Founder @ SwirlAI • Ex CPO @ neptune.ai", "title": "LinkedIn Top Voice in AI • Founder & CEO @ SwirlAI", "linkedin_url": "www.linkedin.com/in/aurimas-griciunas", "twitter_handle": "@Aurimas_Gr", "custom_social_link": "https://www.swirlai.com", "logos": {"logos": [{"image_url": "https://asset.brandfetch.io/idiqc62DHm/id1FFCYumh.svg", "type": "logo", "file_type": "svg", "uuid": "2a8f45b6-6e30-44e7-9a2d-3d1dc5b66b91", "name": "neptune.ai"}], "label": "Former CPO @ Neptune.ai (acquired by OpenAI)"}, "highlights": null, "highlights_html": "<ul><li></li></ul>", "preferred_name": null}], "instructor_caption": "LinkedIn Top Voice in AI • Founder & CEO @ SwirlAI", "course_tags": [{"id": 31, "tag_type": "category", "label": "Data & engineering", "slug": "engineering-ll", "description": null, "parent_tag_id": null}, {"id": 35, "tag_type": "category", "label": "AI", "slug": "ai-ll", "description": null, "parent_tag_id": null}, {"id": 80, "tag_type": "deprecated", "label": "AI/ML engineering", "slug": "x-subcategory-ai-ml-engineering", "description": null, "parent_tag_id": 35}, {"id": 108, "tag_type": "deprecated", "label": "AI/ML engineering", "slug": "x-subcategory-ai-ml-engineering-1", "description": null, "parent_tag_id": 31}, {"id": 110, "tag_type": "deprecated", "label": "Software engineering", "slug": "x-subcategory-software-engineering", "description": null, "parent_tag_id": 31}, {"id": 172, "tag_type": "internal_collection", "label": "AI Affiliates", "slug": "ai-affiliates-nov-24", "description": null, "parent_tag_id": null}, {"id": 226, "tag_type": "internal_collection", "label": "Building AI-Native Products", "slug": "building-ai-native-products", "description": null, "parent_tag_id": null}, {"id": 243, "tag_type": "public_collection", "label": "Lenny's List AI", "slug": "lennys-list-ai", "description": null, "parent_tag_id": null}, {"id": 244, "tag_type": "internal_collection", "label": "Courses filtered for recommendations", "slug": "courses-filtered-for-recommendations", "description": null, "parent_tag_id": null}, {"id": 293, "tag_type": "persona", "label": "For Engineers", "slug": "for-engineers", "description": null, "parent_tag_id": null}, {"id": 299, "tag_type": "persona", "label": "For Data Scientists", "slug": "for-data-scientists", "description": null, "parent_tag_id": null}, {"id": 313, "tag_type": "topic", "label": "AI", "slug": "ai", "description": null, "parent_tag_id": null}, {"id": 315, "tag_type": "topic", "label": "Agentic AI", "slug": "agentic-ai", "description": null, "parent_tag_id": null}, {"id": 332, "tag_type": "topic", "label": "Machine Learning", "slug": "ml", "description": null, "parent_tag_id": null}, {"id": 432, "tag_type": "topic", "label": "LLM Ops", "slug": "llm-ops", "description": null, "parent_tag_id": null}, {"id": 441, "tag_type": "topic", "label": "RAG & Search", "slug": "rag-search", "description": null, "parent_tag_id": null}]}, "contentPageSlug": "063cd6", "contentPage": {"id": 5325, "school_id": 14202, "slug": "063cd6", "content_page_type": "workshop", "sections": [{"uuid": "a9167549-1757-4ce1-b0bb-74ada134b241", "title": "Evaluation Driven Development for Agentic AI Systems", "version": 1, "image_url": "https://d2426xcxuh3ht5.cloudfront.net/uBQVmboTRNa0tZ2YOkv4_Eval_Driven_Development_Lightning_Lesson.png", "is_visible": true, "topic_desc": "Evals are notoriously difficult to implement in real world Agentic AI applications. Nevertheless, we cannot build reliable and maintainable Agentic AI Systems without implementing robust evaluation strategies. As an AI Engineer you will spend a large portion of your day-to-day making sure that these strategies maintain the robustness and correctness of the systems you are building out.", "brand_logos": {"label": "Previously at", "logos": []}, "section_type": "main", "instructor_infos": [{"name": "Aurimas Griciūnas", "uuid": "75fb2756-5c43-4cb2-b414-8342aec142a8", "logos": null, "title": "LinkedIn Top Voice in AI • Founder & CEO @ SwirlAI • Former CPO @ Neptune.ai", "bio_html": "<p>Aurimas Griciūnas is a recognized AI expert, LinkedIn Top Voice in AI, and the founder of SwirlAI. He previously served as Chief Product Officer at Neptune.ai where he worked closely with top ML teams to scale infrastructure, evaluation, and LLMOps practices across industries. With over a decade of experience at the intersection of data science, machine learning, and software engineering, Aurimas has led AI initiatives in both startups and enterprise environments. His mission is to bridge the gap between hype and reality by teaching engineers how to build systems that work in the real world. Students will benefit from his hands-on knowledge, technical depth, and product-first mindset - gained by solving actual engineering problems.</p>", "image_url": "https://d2426xcxuh3ht5.cloudfront.net/lyORS3Z5RBqqzhkE68Lo_Aurimas%20LinkedIn%20(2).jpg", "highlights": null, "linkedin_url": null, "twitter_handle": null, "image_url_no_bg": null, "custom_social_link": null}], "learning_outcomes": [{"uuid": "31aa7940-4bac-4319-9e57-36a3e1eacb67", "title": "The Flywheel of Agentic AI Systems Development", "description": "Learn the moving parts of a successful Agentic AI System development lifecycle."}, {"uuid": "600e4a85-3aa3-409b-95d8-80868cc145ac", "title": "Why Evals are Difficult in Real World Scenarios", "description": "Learn from real world failures so that you don't repeat them in your own projects."}, {"uuid": "a3e2db7b-8bf7-40e2-a1cd-3252de704173", "title": "How to Make Your Evaluation Strategy Succeed", "description": "Learn methods that will help you succeed in your next Agentic AI product build."}]}], "created_at": "2026-02-12T21:04:00.919762Z", "updated_at": null, "theme": null, "attrs": {"type": "workshop", "is_canceled": false, "is_delisted": false, "connected_course_id": 15766, "course_stripe_promo_code_id": null}, "school_event": {"id": 5325, "slug": null, "title": "Evaluation Driven Development for Agentic AI Systems", "description": null, "start_datetime": "2025-09-25T16:00:00Z", "end_datetime": "2025-09-25T16:45:00Z", "start_date": "2025-09-25", "start_time": "18:00:00", "duration_min": 45, "timezone": "Europe/Madrid", "has_internal_recording": true, "is_recording_public": true}, "num_signups": 582}, "relatedWorkshops": [{"id": 4996, "school_id": 15665, "is_visible_on_discovery_page": true, "is_canceled": false, "is_delisted": false, "connected_course_id": 17883, "workshop_tags": [{"id": 2, "tag_type": "category", "label": "Product", "slug": "product-ll", "description": null, "parent_tag_id": null}, {"id": 35, "tag_type": "category", "label": "AI", "slug": "ai-ll", "description": null, "parent_tag_id": null}, {"id": 271, "tag_type": "internal_collection", "label": "AI Evals", "slug": "ai-evals", "description": null, "parent_tag_id": null}, {"id": 290, "tag_type": "persona", "label": "For Product Managers", "slug": "for-product-managers", "description": null, "parent_tag_id": null}, {"id": 293, "tag_type": "persona", "label": "For Engineers", "slug": "for-engineers", "description": null, "parent_tag_id": null}, {"id": 313, "tag_type": "topic", "label": "AI", "slug": "ai", "description": null, "parent_tag_id": null}], "published_content_page": {"id": 5843, "school_id": 15665, "slug": "602b4f", "content_page_type": "workshop", "sections": [{"uuid": "24d0a2a2-5b91-41ac-a0f3-e26372da0046", "title": "How to Drive AI Evals Adoption", "version": 1, "image_url": "https://d2426xcxuh3ht5.cloudfront.net/txhogjeSTKmPgKkxe9FZ_Single%20Instructor.png", "is_visible": true, "topic_desc": "AI Evals & Analytics is excited to host Dr. Sebastian Fox, who has helped companies build reliable, scalable eval systems. With deep expertise in designing evals for LLM features and agents, he brings lessons learned from real-world deployments. In this lightning lesson, he’ll share the hurdles and patterns he’s seen and how to drive AI evals adoption across organizations.", "brand_logos": {"label": "Previously AI at McKinsey & QuantumBlack", "logos": [{"name": "McKinsey & Company", "type": "logo", "uuid": "a2044c40-8cf0-4c20-8e78-08963418833a", "file_type": "svg", "image_url": "https://asset.brandfetch.io/iddTc-lesG/idZ0K6UwLs.svg"}, {"name": "University of Oxford", "type": "logo", "uuid": "4f1a768f-a2b8-4247-b49f-6b81e83af781", "file_type": "png", "image_url": "https://asset.brandfetch.io/idRK9czrcr/idpEPRxdxK.png"}]}, "section_type": "main", "instructor_infos": [{"name": "Dr Sebastian Fox", "uuid": "85599000-aac2-4c23-9732-f549dfc1455f", "logos": {"label": "", "logos": []}, "title": "CEO & Co-founder at Composo.ai", "bio_html": "<p>Building the best evals for LLM features and agents at Composo. Medical doctor. Formerly AI at McKinsey &amp; QuantumBlack.</p>", "headline": "", "image_url": "https://d2426xcxuh3ht5.cloudfront.net/jXYQmhqKSAuWVntByqxv_Screenshot%202025-11-03%20at%2023.15.51.png", "highlights": [], "linkedin_url": "", "twitter_handle": "", "image_url_no_bg": null, "custom_social_link": ""}], "learning_outcomes": [{"uuid": "6bad2e3a-a565-4f33-b607-1d8bbb473815", "title": "Inside Look at Real-World Evals Challenges", "description": "AI eval adoption hurdles: from cultural pushback to tough technical trade-offs."}, {"uuid": "9b23a52e-d2b1-4ea4-bddc-e75e18da4d33", "title": "Lessons from Building Evals for LLM Features & Agents", "description": "Evals beyond benchmarks: capture user value, ensure consistency, and balance automation with human judgement."}, {"uuid": "77525986-8722-4e12-86ed-28d32ec70639", "title": "The Future Beyond “LLM-as-a-Judge”", "description": "Explores limits of LLM-as-judge methods and scalable AI evaluation alternatives"}]}], "created_at": "2026-02-12T21:05:19.681225Z", "updated_at": null, "theme": null, "attrs": {"type": "workshop", "is_canceled": false, "is_delisted": false, "connected_course_id": 17883, "course_stripe_promo_code_id": null}, "school_event": {"id": 5843, "slug": null, "title": "How to Drive AI Evals Adoption", "description": null, "start_datetime": "2025-11-20T17:00:00Z", "end_datetime": "2025-11-20T17:30:00Z", "start_date": "2025-11-21", "start_time": "01:00:00", "duration_min": 30, "timezone": "Asia/Taipei", "has_internal_recording": true, "is_recording_public": true}, "num_signups": 322}}, {"id": 2853, "school_id": 13230, "is_visible_on_discovery_page": true, "is_canceled": false, "is_delisted": false, "connected_course_id": 14554, "workshop_tags": [{"id": 35, "tag_type": "category", "label": "AI", "slug": "ai-ll", "description": null, "parent_tag_id": null}, {"id": 137, "tag_type": "internal_collection", "label": "Instructor", "slug": "Instructor", "description": null, "parent_tag_id": null}, {"id": 226, "tag_type": "internal_collection", "label": "Building AI-Native Products", "slug": "building-ai-native-products", "description": null, "parent_tag_id": null}, {"id": 271, "tag_type": "internal_collection", "label": "AI Evals", "slug": "ai-evals", "description": null, "parent_tag_id": null}, {"id": 293, "tag_type": "persona", "label": "For Engineers", "slug": "for-engineers", "description": null, "parent_tag_id": null}, {"id": 313, "tag_type": "topic", "label": "AI", "slug": "ai", "description": null, "parent_tag_id": null}, {"id": 315, "tag_type": "topic", "label": "Agentic AI", "slug": "agentic-ai", "description": null, "parent_tag_id": null}], "published_content_page": {"id": 3683, "school_id": 13230, "slug": "6f0e97", "content_page_type": "workshop", "sections": [{"uuid": "d6857d2d-6951-47b0-8721-58e08e926006", "title": "Evaluating Agentic AI Applications Beyond Vibe Checks", "version": 1, "image_url": "https://d2426xcxuh3ht5.cloudfront.net/mDToG6LTWSbrwN4DlhBc_Panel (4 People).png", "is_visible": true, "topic_desc": "As agentic applications become more widespread, evaluating their effectiveness is more critical than ever. Join us to learn how to run meaningful evaluations—plus a live demo using Comet Opik, an open-source platform built for this purpose.", "brand_logos": {"label": "", "logos": []}, "section_type": "main", "instructor_infos": [{"name": "Aishwarya Naresh Reganti", "uuid": "4a33ab84-f8f1-4cda-a4fc-3fbe1de363ec", "title": "Tech Lead @ AWS | AI Lecturer & Advisor", "bio_html": "<p>Aishwarya Naresh Reganti is an Applied Science Tech Lead at the AWS Generative AI Innovation Center (GenAIIC), where she leads initiatives to develop and deploy production-ready generative AI solutions for AWS clients. With over 9 years of experience in machine learning, she has published more than&nbsp;<a href=\"https://scholar.google.com/citations?user=gvgg4ksAAAAJ&amp;hl=en\" rel=\"noopener noreferrer\" target=\"_blank\">35 research papers</a>&nbsp;at top-tier AI conferences, including NeurIPS, AAAI, and CVPR.</p><p><br></p><p>Aishwarya has taught professional courses on generative AI at renowned institutions like MIT and Oxford. She has also designed&nbsp;<a href=\"https://github.com/aishwaryanr/awesome-generative-ai-guide/tree/main/free_courses/Applied_LLMs_Mastery_2024\" rel=\"noopener noreferrer\" target=\"_blank\">courses</a>&nbsp;that have reached over 10,000 students globally and have formed the foundation for several academic programs and industry training curricula.</p><p><br></p><p>Recognized as one of the most prominent voices in enterprise AI, with over 90,000 professionals following&nbsp;her on&nbsp;<a href=\"https://www.linkedin.com/in/areganti/\" rel=\"noopener noreferrer\" target=\"_blank\">LinkedIn</a>, she is a sought-after thought leader frequently invited to speak at leading conferences and events, including TEDx, MLOps World, and ReWork.</p><p><br></p><p>Aishwarya actively collaborates with leading research professors and provides strategic advisory to organizations, enabling them to harness AI effectively to address complex business challenges.</p>", "image_url": "https://d2426xcxuh3ht5.cloudfront.net/34ZV8qoTbelqJzyQEdly_UzyVmfTFo4Nj3PzwKzQA_Screenshot 2024-12-26 at 10_LE_upscale_balanced_x4.png"}, {"name": "Kiriti Badam", "uuid": "dda1c1d4-75bf-4388-9be4-efb50fe14599", "title": "Member of Technical Staff @ OpenAI | Ex-Google", "bio_html": "<p>Kiriti Badam is a member of the technical staff at OpenAI, with over a decade of experience designing high-impact enterprise AI systems. He specializes in AI-centric infrastructure, with deep expertise in large-scale compute, data engineering, and storage systems.</p><p>Prior to OpenAI, Kiriti was a founding engineer at Kumo.ai, a Forbes AI 50 startup, where he led the development of infrastructure that enabled training hundreds of models daily—driving significant ARR growth for enterprise clients.</p><p>Kiriti brings a rare blend of startup agility and enterprise-scale depth, having worked at companies like Google, Samsung, Databricks, and Kumo.ai. At Google Ads, he built globally distributed key-value stores that powered ad systems generating tens of billions in annual revenue.</p><p>He holds a Master’s degree from Carnegie Mellon University and a Bachelor’s from IIT Madras, where his research focused on advanced storage systems and distributed databases for AI workloads. A sought-after mentor and advisor, Kiriti helps startups and organizations design scalable AI infrastructure, reach product-market fit, and build long-term product strategy.</p>", "image_url": "https://d2426xcxuh3ht5.cloudfront.net/QYKlSOekTEuHdzTkimvh_uH5r8WQyQqyCsj7SokN9_1552866171925_LE_upscale_balanced_x4.png"}, {"name": "Claire Longo", "uuid": "25cdd70d-f2b2-4a2e-b840-3861eef7bdbc", "title": "Lead AI Researcher @ Comet", "bio_html": "<p>Claire&nbsp;Longo&nbsp;is an AI leader with over a decade of experience in Data Science, Machine Learning, and MLOps. From coding in R as a Statistician at DOE National Laboratories to building recommender systems at Trunk Club, she has navigated the evolving AI landscape—teaching herself Python and ML along the way. She has led cross-functional AI teams at Twilio, Opendoor, and Arize AI, focusing on the engineering best practices required to bring AI models from ideation to production at scale. Currently at Comet,&nbsp;Claire&nbsp;researches AI trends and shares best practices with the developer community. She holds a Bachelor’s in Applied Mathematics and a Master’s in Statistics from The University of New Mexico.</p>", "image_url": "https://d2426xcxuh3ht5.cloudfront.net/wm694bGfTkGit7ciSeLF_1723745075057.jpeg"}], "learning_outcomes": [{"uuid": "6c5f9c9e-e63d-49d4-a311-e7bf0e475824", "title": "Why Agentic AI Evaluation ?", "description": "Traditional evaluation methods fall short for agentic systems due to their autonomous and non-deterministic nature"}, {"uuid": "983a4b61-734a-467e-a19c-d6e0cbf49b86", "title": "Types of Evaluations for Agentic Systems", "description": "We’ll break down key evaluation types needed for robust, real-world agent behavior."}, {"uuid": "45d84dd7-1f34-431c-9948-915174cf6496", "title": "Live Demo: Evaluating with Comet Opik", "description": "Watch a hands-on demo using Comet Opik to run, monitor, and visualize agent evaluations in a production-like setup."}]}], "created_at": "2026-02-12T20:59:49.368770Z", "updated_at": null, "theme": null, "attrs": {"type": "workshop", "is_canceled": false, "is_delisted": false, "connected_course_id": 14554, "course_stripe_promo_code_id": null}, "school_event": {"id": 3683, "slug": null, "title": "Evaluating Agentic AI Applications Beyond Vibe Checks", "description": null, "start_datetime": "2025-05-18T16:00:00Z", "end_datetime": "2025-05-18T16:45:00Z", "start_date": "2025-05-18", "start_time": "09:00:00", "duration_min": 45, "timezone": "America/Los_Angeles", "has_internal_recording": true, "is_recording_public": true}, "num_signups": 1249}}, {"id": 2477, "school_id": 9364, "is_visible_on_discovery_page": true, "is_canceled": false, "is_delisted": false, "connected_course_id": 14937, "workshop_tags": [{"id": 2, "tag_type": "category", "label": "Product", "slug": "product-ll", "description": null, "parent_tag_id": null}, {"id": 35, "tag_type": "category", "label": "AI", "slug": "ai-ll", "description": null, "parent_tag_id": null}, {"id": 137, "tag_type": "internal_collection", "label": "Instructor", "slug": "Instructor", "description": null, "parent_tag_id": null}, {"id": 290, "tag_type": "persona", "label": "For Product Managers", "slug": "for-product-managers", "description": null, "parent_tag_id": null}, {"id": 291, "tag_type": "persona", "label": "For Leaders", "slug": "for-leaders", "description": null, "parent_tag_id": null}, {"id": 313, "tag_type": "topic", "label": "AI", "slug": "ai", "description": null, "parent_tag_id": null}], "published_content_page": {"id": 3300, "school_id": 9364, "slug": "9b1bab", "content_page_type": "workshop", "sections": [{"uuid": "0710794c-f92e-485c-9e59-b7550445f3e0", "title": "Stop Managing AI Projects Like Traditional Software", "version": 1, "image_url": "https://d2426xcxuh3ht5.cloudfront.net/YKrArMY2QFOaIf0m00nv_Duo Instructor (6).png", "is_visible": true, "topic_desc": "AI projects fail when managed like traditional software. Instead of fixed deadlines, successful AI teams focus on experiments. We'll share a proven framework that shifts your thinking from timelines to learning. This experimental mindset is what separates successful AI initiatives from struggling ones.", "brand_logos": {"label": "", "logos": [{"name": "Stitch Fix", "type": "logo", "uuid": "d861c508-1ed6-4cde-976d-7dc82c3ef44b", "file_type": "svg", "image_url": "https://asset.brandfetch.io/idTgrRl8nA/idVnuVYfek.svg"}, {"name": "GitHub", "type": "logo", "uuid": "196fa83b-76ae-4530-ab6f-4ecf9efc9cc7", "file_type": "png", "image_url": "https://asset.brandfetch.io/idZAyF9rlg/id8uXh7wzx.png"}]}, "section_type": "main", "instructor_infos": [{"name": "Bryan Bischof", "uuid": "372f7ed4-070a-434b-ac1a-ffcb1ee8a554", "title": "Head of AI at Theory Ventures", "bio_html": "<p>Bryan Bischof is currently Head of AI at Theory Ventures.  He formerly led AI Engineering at Hex, where he spearheaded the development of Magic—the data science and analytics copilot. Bryan has worked all over the data stack leading teams in analytics, machine learning engineering, data platform engineering, and AI engineering. He started the data team at Blue Bottle Coffee, led several projects at Stitch Fix, and built the data teams at Weights and Biases. Bryan previously co-authored the book Building Production Recommendation Systems with O’Reilly, and teaches Data Science and Analytics in the graduate school at Rutgers. His Ph.D. is in pure mathematics.</p>", "image_url": "https://d2426xcxuh3ht5.cloudfront.net/pjqeZa1TsaeOlGjjQ3jA_Untitled design (2).png"}, {"name": "Hamel Husain", "uuid": "df43f7ea-ddd1-491a-a214-dba57eb4244b", "title": "ML Engineer with 20 years of experience.", "bio_html": "<p>Hamel Husain is a ML Engineer with over 20 years of&nbsp;<a href=\"https://www.linkedin.com/in/hamelhusain/\" rel=\"noopener noreferrer\" target=\"_blank\">experience</a>. He has worked with innovative companies such as Airbnb and GitHub, which included&nbsp;<a href=\"https://openai.com/index/introducing-text-and-code-embeddings#:~:text=models%20on%20the-,CodeSearchNet,),-evaluation%20suite%20where\" rel=\"noopener noreferrer\" target=\"_blank\">early LLM research used by OpenAI</a>, for code understanding. He has also led and contributed to numerous popular&nbsp;<a href=\"https://hamel.dev/oss/opensource.html\" rel=\"noopener noreferrer\" target=\"_blank\">open-source machine-learning tools</a>. Hamel is currently an independent consultant helping companies operationalize Large Language Models (LLMs).</p><p><br></p>", "image_url": "https://d2426xcxuh3ht5.cloudfront.net/0DN5gm1xRxStDiehpYUK_1572471556692.jpg"}], "learning_outcomes": [{"uuid": "edfab190-7b52-48f4-87b2-a7b53440aa3e", "title": "Learn Why Traditional Software Roadmaps Fail in AI Projects", "description": "We'll show you why conventional approaches to product development break down when building AI  and what to do instead."}, {"uuid": "27d33067-8b40-458e-b901-1679ac744d26", "title": "Prioritizing Work Through Effective Evaluation", "description": "Explore methods for creating evals that pinpoint where your AI is struggling, and how to prioritize improvements."}, {"uuid": "efb83b0d-8bbf-41cf-b4ff-94495db29c85", "title": "How to Adopt an Experimental Mindset", "description": "Learn to build AI products through iterative experiments rather than rigid roadmaps, with clear, measurable objectives."}, {"uuid": "fecbed7b-ecd4-4131-86b3-57b10618f6c0", "title": "Turn Failures Into Actionable Insights", "description": "Learn to break down complex AI capabilities into measurable stages that help you identify where to focus."}]}], "created_at": "2026-02-12T20:58:48.419367Z", "updated_at": null, "theme": null, "attrs": {"type": "workshop", "is_canceled": false, "is_delisted": false, "connected_course_id": 14937, "course_stripe_promo_code_id": null}, "school_event": {"id": 3300, "slug": null, "title": "Stop Managing AI Projects Like Traditional Software", "description": null, "start_datetime": "2025-04-10T00:00:00Z", "end_datetime": "2025-04-10T01:00:00Z", "start_date": "2025-04-09", "start_time": "17:00:00", "duration_min": 60, "timezone": "America/Los_Angeles", "has_internal_recording": true, "is_recording_public": true}, "num_signups": 1556}}], "exploreCourses": null, "workshopTags": [{"id": 257, "tag_type": "internal_collection", "label": "Building AI Systems", "slug": "building-ai-systems", "description": null, "parent_tag_id": null}, {"id": 31, "tag_type": "category", "label": "Data & engineering", "slug": "engineering-ll", "description": null, "parent_tag_id": null}, {"id": 35, "tag_type": "category", "label": "AI", "slug": "ai-ll", "description": null, "parent_tag_id": null}, {"id": 271, "tag_type": "internal_collection", "label": "AI Evals", "slug": "ai-evals", "description": null, "parent_tag_id": null}, {"id": 293, "tag_type": "persona", "label": "For Engineers", "slug": "for-engineers", "description": null, "parent_tag_id": null}, {"id": 315, "tag_type": "topic", "label": "Agentic AI", "slug": "agentic-ai", "description": null, "parent_tag_id": null}, {"id": 313, "tag_type": "topic", "label": "AI", "slug": "ai", "description": null, "parent_tag_id": null}], "videoChapters": [{"title": "Introduction and Welcome", "start_seconds": 366}, {"title": "Eval-Driven Development for Agentic AI Systems", "start_seconds": 677}, {"title": "Speaker's Background and Experience", "start_seconds": 752}, {"title": "Defining Agentic AI Systems", "start_seconds": 846}, {"title": "The Flywheel of Continuous Agentic AI Development", "start_seconds": 952}, {"title": "Monitoring and Alerting in Production", "start_seconds": 1374}, {"title": "Defining Evals: The Cold Start Problem", "start_seconds": 1423}, {"title": "Defining Evals from Human Feedback and Traces", "start_seconds": 1775}, {"title": "Discovering Evals During Development", "start_seconds": 1991}, {"title": "The Importance of Proper Tracing", "start_seconds": 2053}, {"title": "Managing the Cost and Time of Evals", "start_seconds": 2235}, {"title": "Strategies for Running Evals Efficiently", "start_seconds": 2403}, {"title": "Granularity of Evals: End-to-End vs. Node-Specific", "start_seconds": 2793}, {"title": "Conclusion and Bootcamp Promotion", "start_seconds": 2982}, {"title": "Q&A: Measuring Eval Comprehensiveness", "start_seconds": 3069}, {"title": "Q&A: Mocking Tools and Infrastructure", "start_seconds": 3242}, {"title": "Q&A: Using Frameworks vs. Building Your Own Evals", "start_seconds": 3325}, {"title": "Q&A: Evals vs. Performance Metrics", "start_seconds": 3472}, {"title": "Q&A: Evals for a Financial Spreading System", "start_seconds": 3531}, {"title": "Q&A: How to Select Evals for Different Agents", "start_seconds": 3793}, {"title": "Q&A: Test-Driven Development Analogy", "start_seconds": 3980}, {"title": "Q&A: Estimating if You Have Enough Evals", "start_seconds": 4052}, {"title": "Q&A: Bootcamp Details and Failure Modes", "start_seconds": 4132}, {"title": "Final Remarks and Contact Information", "start_seconds": 4277}], "navPages": [{"category": {"slug": "ai", "href": "/courses/ai", "title": "AI", "palette": {"--palette-category-base": "#460952", "--palette-category-highlight": "#931FAA", "--palette-category-border": "#EDA6FB"}}, "topics": [{"title": "Agentic AI", "slug": "ai/agentic-ai", "href": "/courses/ai/agentic-ai"}, {"title": "Coding with AI", "slug": "ai/ai-coding", "href": "/courses/ai/ai-coding"}, {"title": "AI Workflows", "slug": "ai/ai-workflows", "href": "/courses/ai/ai-workflows"}, {"title": "Claude Code", "slug": "ai/claude-code", "href": "/courses/ai/claude-code"}, {"title": "OpenClaw", "slug": "ai/openclaw", "href": "/courses/ai/openclaw"}, {"title": "Vibe Coding", "slug": "ai/vibe-coding", "href": "/courses/ai/vibe-coding"}, {"title": "AI Evals", "slug": "ai/evals", "href": "/courses/ai/evals"}, {"title": "AI Transformation", "slug": "ai/ai-transformation", "href": "/courses/ai/ai-transformation"}, {"title": "RAG & Search", "slug": "ai/rag-search", "href": "/courses/ai/rag-search"}, {"title": "MCP", "slug": "ai/mcp", "href": "/courses/ai/mcp"}, {"title": "AI for PMs", "slug": "ai/for-product-managers", "href": "/courses/ai/for-product-managers"}, {"title": "AI for Engineers", "slug": "ai/for-engineers", "href": "/courses/ai/for-engineers"}, {"title": "AI for Designers", "slug": "ai/for-designers", "href": "/courses/ai/for-designers"}, {"title": "AI for Marketers", "slug": "ai/for-marketers", "href": "/courses/ai/for-marketers"}, {"title": "AI for Founders", "slug": "ai/for-founders", "href": "/courses/ai/for-founders"}]}, {"category": {"slug": "product", "href": "/courses/product", "title": "Product", "palette": {"--palette-category-base": "#510B00", "--palette-category-highlight": "#B33600", "--palette-category-border": "#F7804D"}}, "topics": [{"title": "AI for PMs", "slug": "for-product-managers/ai", "href": "/courses/for-product-managers/ai"}, {"title": "Agentic AI", "slug": "for-product-managers/agentic-ai", "href": "/courses/for-product-managers/agentic-ai"}, {"title": "AI Evals", "slug": "for-product-managers/ai-evals", "href": "/courses/for-product-managers/ai-evals"}, {"title": "Vibe Coding", "slug": "for-product-managers/vibe-coding", "href": "/courses/for-product-managers/vibe-coding"}, {"title": "Product Sense", "slug": "for-product-managers/product-sense", "href": "/courses/for-product-managers/product-sense"}, {"title": "Product Discovery", "slug": "for-product-managers/product-discovery", "href": "/courses/for-product-managers/product-discovery"}, {"title": "User Research", "slug": "for-product-managers/user-research", "href": "/courses/for-product-managers/user-research"}, {"title": "Prototyping", "slug": "for-product-managers/prototyping", "href": "/courses/for-product-managers/prototyping"}, {"title": "Growth", "slug": "for-product-managers/growth", "href": "/courses/for-product-managers/growth"}, {"title": "Analytics", "slug": "for-product-managers/analytics", "href": "/courses/for-product-managers/analytics"}, {"title": "Tech Foundations", "slug": "for-product-managers/technical-foundations", "href": "/courses/for-product-managers/technical-foundations"}, {"title": "Strategy", "slug": "for-product-managers/strategy", "href": "/courses/for-product-managers/strategy"}, {"title": "Influence", "slug": "for-product-managers/influence", "href": "/courses/for-product-managers/influence"}, {"title": "Leadership", "slug": "for-product-managers/leadership", "href": "/courses/for-product-managers/leadership"}, {"title": "Career Growth", "slug": "for-product-managers/career-growth", "href": "/courses/for-product-managers/career-growth"}]}, {"category": {"slug": "engineering", "href": "/courses/engineering", "title": "Engineering", "palette": {"--palette-category-base": "#00332F", "--palette-category-highlight": "#00746B", "--palette-category-border": "#88B7B2"}}, "topics": [{"title": "AI for Engineers", "slug": "for-engineers/ai", "href": "/courses/for-engineers/ai"}, {"title": "Agentic AI", "slug": "for-engineers/agentic-ai", "href": "/courses/for-engineers/agentic-ai"}, {"title": "Coding with AI", "slug": "for-engineers/ai-coding", "href": "/courses/for-engineers/ai-coding"}, {"title": "Claude Code", "slug": "for-engineers/claude-code", "href": "/courses/for-engineers/claude-code"}, {"title": "OpenClaw", "slug": "for-engineers/openclaw", "href": "/courses/for-engineers/openclaw"}, {"title": "MCP", "slug": "for-engineers/mcp", "href": "/courses/for-engineers/mcp"}, {"title": "RAG & Search", "slug": "for-engineers/rag-search", "href": "/courses/for-engineers/rag-search"}, {"title": "AI Evals", "slug": "for-engineers/ai-evals", "href": "/courses/for-engineers/ai-evals"}, {"title": "Machine Learning", "slug": "for-engineers/ml", "href": "/courses/for-engineers/ml"}, {"title": "LLM Ops", "slug": "for-engineers/llm-ops", "href": "/courses/for-engineers/llm-ops"}, {"title": "Context Eng", "slug": "for-engineers/context-engineering", "href": "/courses/for-engineers/context-engineering"}, {"title": "Security", "slug": "for-engineers/security", "href": "/courses/for-engineers/security"}, {"title": "System Design", "slug": "for-engineers/system-design", "href": "/courses/for-engineers/system-design"}, {"title": "Leadership", "slug": "for-engineers/leadership", "href": "/courses/for-engineers/leadership"}, {"title": "Career Growth", "slug": "for-engineers/career-growth", "href": "/courses/for-engineers/career-growth"}]}, {"category": {"slug": "design", "href": "/courses/design", "title": "Design", "palette": {"--palette-category-base": "#38085F", "--palette-category-highlight": "#772CBB", "--palette-category-border": "#B5A2D0"}}, "topics": [{"title": "AI for Designers", "slug": "for-designers/ai", "href": "/courses/for-designers/ai"}, {"title": "Agentic AI", "slug": "for-designers/agentic-ai", "href": "/courses/for-designers/agentic-ai"}, {"title": "Vibe Coding", "slug": "for-designers/vibe-coding", "href": "/courses/for-designers/vibe-coding"}, {"title": "Prototyping", "slug": "for-designers/prototyping", "href": "/courses/for-designers/prototyping"}, {"title": "Figma", "slug": "for-designers/figma", "href": "/courses/for-designers/figma"}, {"title": "Design Systems", "slug": "for-designers/design-systems", "href": "/courses/for-designers/design-systems"}, {"title": "User Research", "slug": "for-designers/user-research", "href": "/courses/for-designers/user-research"}, {"title": "Product Discovery", "slug": "for-designers/product-discovery", "href": "/courses/for-designers/product-discovery"}, {"title": "UX", "slug": "for-designers/ux", "href": "/courses/for-designers/ux"}, {"title": "UI", "slug": "for-designers/ui", "href": "/courses/for-designers/ui"}, {"title": "Visual Design", "slug": "for-designers/visual-design", "href": "/courses/for-designers/visual-design"}, {"title": "Design Strategy", "slug": "for-designers/strategy", "href": "/courses/for-designers/strategy"}, {"title": "Influence", "slug": "for-designers/influence", "href": "/courses/for-designers/influence"}, {"title": "Leadership", "slug": "for-designers/leadership", "href": "/courses/for-designers/leadership"}, {"title": "Career Growth", "slug": "for-designers/career-growth", "href": "/courses/for-designers/career-growth"}]}, {"category": {"slug": "marketing", "href": "/courses/marketing", "title": "Marketing", "palette": {"--palette-category-base": "#431532", "--palette-category-highlight": "#943170", "--palette-category-border": "#C99EB5"}}, "topics": [{"title": "AI for Marketers", "slug": "for-marketers/ai", "href": "/courses/for-marketers/ai"}, {"title": "Agentic AI", "slug": "for-marketers/agentic-ai", "href": "/courses/for-marketers/agentic-ai"}, {"title": "Vibe Coding", "slug": "for-marketers/vibe-coding", "href": "/courses/for-marketers/vibe-coding"}, {"title": "Automation", "slug": "for-marketers/marketing-automation", "href": "/courses/for-marketers/marketing-automation"}, {"title": "Content Marketing", "slug": "for-marketers/content-marketing", "href": "/courses/for-marketers/content-marketing"}, {"title": "Demand Gen", "slug": "for-marketers/demand-generation", "href": "/courses/for-marketers/demand-generation"}, {"title": "Go-to-Market", "slug": "for-marketers/gtm", "href": "/courses/for-marketers/gtm"}, {"title": "Product Marketing", "slug": "for-marketers/product-marketing", "href": "/courses/for-marketers/product-marketing"}, {"title": "Positioning", "slug": "for-marketers/positioning", "href": "/courses/for-marketers/positioning"}, {"title": "Social Media", "slug": "for-marketers/social-media", "href": "/courses/for-marketers/social-media"}, {"title": "Brand", "slug": "for-marketers/brand", "href": "/courses/for-marketers/brand"}, {"title": "B2B Marketing", "slug": "for-marketers/b2b", "href": "/courses/for-marketers/b2b"}, {"title": "SEO & AEO", "slug": "for-marketers/seo-aeo", "href": "/courses/for-marketers/seo-aeo"}, {"title": "Strategy", "slug": "for-marketers/strategy", "href": "/courses/for-marketers/strategy"}, {"title": "Leadership", "slug": "for-marketers/leadership", "href": "/courses/for-marketers/leadership"}]}, {"category": {"slug": "leadership", "href": "/courses/leadership", "title": "Leadership", "palette": {"--palette-category-base": "#50090E", "--palette-category-highlight": "#A9202A", "--palette-category-border": "#D49C98"}}, "topics": [{"title": "AI for Leaders", "slug": "for-leaders/ai", "href": "/courses/for-leaders/ai"}, {"title": "Agentic AI", "slug": "for-leaders/agentic-ai", "href": "/courses/for-leaders/agentic-ai"}, {"title": "AI Transformation", "slug": "for-leaders/ai-transformation", "href": "/courses/for-leaders/ai-transformation"}, {"title": "AI Governance", "slug": "for-leaders/ai-governance", "href": "/courses/for-leaders/ai-governance"}, {"title": "Communication", "slug": "for-leaders/communication", "href": "/courses/for-leaders/communication"}, {"title": "Influence", "slug": "for-leaders/influence", "href": "/courses/for-leaders/influence"}, {"title": "Strategy", "slug": "for-leaders/strategy", "href": "/courses/for-leaders/strategy"}, {"title": "Management", "slug": "for-leaders/management", "href": "/courses/for-leaders/management"}, {"title": "People Operations", "slug": "for-leaders/people-operations", "href": "/courses/for-leaders/people-operations"}, {"title": "Exec Presence", "slug": "for-leaders/executive-presence", "href": "/courses/for-leaders/executive-presence"}, {"title": "Storytelling", "slug": "for-leaders/storytelling", "href": "/courses/for-leaders/storytelling"}, {"title": "Goal-setting", "slug": "for-leaders/goal-setting", "href": "/courses/for-leaders/goal-setting"}, {"title": "Personal Brand", "slug": "for-leaders/personal-brand", "href": "/courses/for-leaders/personal-brand"}, {"title": "Career Growth", "slug": "for-leaders/career-growth", "href": "/courses/for-leaders/career-growth"}]}, {"category": {"slug": "founders", "href": "/courses/founders", "title": "Founders", "palette": {"--palette-category-base": "#0C3209", "--palette-category-highlight": "#077202", "--palette-category-border": "#69C662"}}, "topics": [{"title": "AI for Founders", "slug": "for-founders/ai", "href": "/courses/for-founders/ai"}, {"title": "Agentic AI", "slug": "for-founders/agentic-ai", "href": "/courses/for-founders/agentic-ai"}, {"title": "AI Workflows", "slug": "for-founders/ai-workflows", "href": "/courses/for-founders/ai-workflows"}, {"title": "Vibe Coding", "slug": "for-founders/vibe-coding", "href": "/courses/for-founders/vibe-coding"}, {"title": "Prototyping", "slug": "for-founders/prototyping", "href": "/courses/for-founders/prototyping"}, {"title": "Product Sense", "slug": "for-founders/product-sense", "href": "/courses/for-founders/product-sense"}, {"title": "Positioning", "slug": "for-founders/positioning", "href": "/courses/for-founders/positioning"}, {"title": "Product Discovery", "slug": "for-founders/product-discovery", "href": "/courses/for-founders/product-discovery"}, {"title": "Management", "slug": "for-founders/management", "href": "/courses/for-founders/management"}, {"title": "Strategy", "slug": "for-founders/strategy", "href": "/courses/for-founders/strategy"}, {"title": "Go-to-Market", "slug": "for-founders/gtm", "href": "/courses/for-founders/gtm"}, {"title": "Personal Brand", "slug": "for-founders/personal-brand", "href": "/courses/for-founders/personal-brand"}, {"title": "Leadership", "slug": "for-founders/leadership", "href": "/courses/for-founders/leadership"}, {"title": "Fundraising", "slug": "for-founders/fundraising", "href": "/courses/for-founders/fundraising"}, {"title": "PMF", "slug": "for-founders/product-market-fit", "href": "/courses/for-founders/product-market-fit"}]}, {"category": {"slug": "more", "href": "/courses/more", "title": "More", "palette": {"--palette-category-base": "#1E2C2B", "--palette-category-highlight": "#4A6361", "--palette-category-border": "#B9C1BE"}}, "topics": [{"title": "Everyone", "slug": "for-everyone", "href": "/courses/for-everyone"}, {"title": "Operators", "slug": "for-operators", "href": "/courses/for-operators"}, {"title": "Data Scientists", "slug": "for-data-scientists", "href": "/courses/for-data-scientists"}, {"title": "Business Analysts", "slug": "for-business-analysts", "href": "/courses/for-business-analysts"}, {"title": "User Researchers", "slug": "for-user-researchers", "href": "/courses/for-user-researchers"}, {"title": "Customer Success", "slug": "for-customer-success", "href": "/courses/for-customer-success"}, {"title": "Project Managers", "slug": "for-project-managers", "href": "/courses/for-project-managers"}, {"title": "HR Professionals", "slug": "for-hr-professionals", "href": "/courses/for-hr-professionals"}, {"title": "Sales People", "slug": "for-sales-people", "href": "/courses/for-sales-people"}, {"title": "Lawyers", "slug": "for-lawyers", "href": "/courses/for-lawyers"}, {"title": "Finance", "slug": "for-finance-professionals", "href": "/courses/for-finance-professionals"}, {"title": "Investors", "slug": "for-investors", "href": "/courses/for-investors"}, {"title": "Real Estate", "slug": "for-real-estate-agents", "href": "/courses/for-real-estate-agents"}, {"title": "Educators", "slug": "for-educators", "href": "/courses/for-educators"}, {"title": "Creators", "slug": "for-creators", "href": "/courses/for-creators"}]}], "navTagLinks": [{"tag": {"id": 315, "tag_type": "topic", "label": "Agentic AI", "slug": "agentic-ai", "description": null, "parent_tag_id": null}, "link_slug": "for-engineers/agentic-ai"}, {"tag": {"id": 293, "tag_type": "persona", "label": "For Engineers", "slug": "for-engineers", "description": null, "parent_tag_id": null}, "link_slug": "for-engineers"}], "now": "2026-05-19T22:53:51.620Z"}, "__N_SSG": true}, "page": "/p/[slug]/[...ignored]", "query": {"slug": "063cd6", "ignored": ["evaluation-driven-development-for-agentic-ai-systems"]}, "buildId": "qPlLjBFLQ9fnkFKJ7n7E-", "isFallback": false, "isExperimentalCompile": false, "gsp": true, "scriptLoader": []}}