", "headline": "", "title": "Founder at Seeko. Writes AI Ready to 50,000+ subs. Previously founded Autoblocks & AI For Anyone", "linkedin_url": "https://linkedin.com/in/haroonchoudery", "x_handle": "https://x.com/haroonc", "custom_social_link": "seeko.so", "logos": {"logos": [{"image_url": "https://asset.brandfetch.io/id519GJZCP/id9dCIwqqB.svg", "type": "logo", "file_type": "svg", "uuid": "618c840c-15ca-4ebc-804a-981ddd69487b", "name": "UC Berkeley"}, {"image_url": "https://asset.brandfetch.io/idpKX136kp/idDbwqFalU.png", "type": "logo", "file_type": "png", "uuid": "64bf0e19-9a43-429e-b5ee-2ebd05dca04a", "name": "Facebook"}, {"image_url": "https://asset.brandfetch.io/id0DD2N_HT/idWB8N7JuM.jpeg", "type": "icon", "file_type": "jpeg", "uuid": "07e3db9d-b2e5-4b25-be15-c41c821a6957", "name": "Hex"}, {"image_url": "https://asset.brandfetch.io/ideSyfm6Fo/idjcs34e4Q.svg", "type": "logo", "file_type": "svg", "uuid": "f3f4b2c8-e7f1-4842-9737-8d011bab4042", "name": "Mark Cuban Companies"}, {"image_url": "https://asset.brandfetch.io/idIu-Ji9Le/idO6MoschZ.svg", "type": "logo", "file_type": "svg", "uuid": "485051f4-a9a0-421c-b381-f31abc5646ab", "name": "Deloitte"}], "label": "PREVIOUSLY AT"}, "expert_tagline": "will help you get the most leverage out of AI", "primary_color": "#8ef59a", "contact_email": "haroon@seeko.so", "show_contact": true}}, "pageType": "workshop", "school": {"id": 10905, "slug": "haroon", "name": "Haroon Choudery", "is_stripe_connected": true, "stripe_connect_account_id": "acct_1Sn1UL3XH8TS1jJ0", "zoom_auth_version": 2, "verified": true, "timezone": "America/New_York", "support_email": null, "school_tags": [{"id": 96, "tag_type": "internal_collection", "label": "Free lesson beta", "slug": "free-lesson-beta", "description": null, "parent_tag_id": null}], "instructor_bio": {"name": "Haroon Choudery", "preferred_name": "Haroon", "image_url": "https://d2426xcxuh3ht5.cloudfront.net/vJHOrSISQKDwVZQ8BiDf_DSC00391-2.jpg", "image_url_no_bg": "https://user-assets.maven.com/instructor-no-bg/vJHOrSISQKDwVZQ8BiDf_DSC00391-2.jpg", "bio_html": "Haroon Choudery has spent over a decade in AI, building infrastructure and advising the teams deploying it. He previously founded Autoblocks, a VC-backed AI platform, as well as AI For Anyone, an AI education initiative that has taught over 70,000 people since 2016.
Now, he brings that experience to Seeko, an AI transformation consultancy that works with mid-market and enterprise companies to shape and execute AI transformations that actually work.
He also writes AI Ready, a newsletter read by more than 50,000 operators and thought leaders, covering the latest AI developments for anyone trying to keep up with the space.
", "headline": "", "title": "Founder at Seeko. Writes AI Ready to 50,000+ subs. Previously founded Autoblocks & AI For Anyone", "linkedin_url": "https://linkedin.com/in/haroonchoudery", "x_handle": "https://x.com/haroonc", "custom_social_link": "seeko.so", "logos": {"logos": [{"image_url": "https://asset.brandfetch.io/id519GJZCP/id9dCIwqqB.svg", "type": "logo", "file_type": "svg", "uuid": "618c840c-15ca-4ebc-804a-981ddd69487b", "name": "UC Berkeley"}, {"image_url": "https://asset.brandfetch.io/idpKX136kp/idDbwqFalU.png", "type": "logo", "file_type": "png", "uuid": "64bf0e19-9a43-429e-b5ee-2ebd05dca04a", "name": "Facebook"}, {"image_url": "https://asset.brandfetch.io/id0DD2N_HT/idWB8N7JuM.jpeg", "type": "icon", "file_type": "jpeg", "uuid": "07e3db9d-b2e5-4b25-be15-c41c821a6957", "name": "Hex"}, {"image_url": "https://asset.brandfetch.io/ideSyfm6Fo/idjcs34e4Q.svg", "type": "logo", "file_type": "svg", "uuid": "f3f4b2c8-e7f1-4842-9737-8d011bab4042", "name": "Mark Cuban Companies"}, {"image_url": "https://asset.brandfetch.io/idIu-Ji9Le/idO6MoschZ.svg", "type": "logo", "file_type": "svg", "uuid": "485051f4-a9a0-421c-b381-f31abc5646ab", "name": "Deloitte"}], "label": "PREVIOUSLY AT"}, "expert_tagline": "will help you get the most leverage out of AI", "primary_color": "#8ef59a", "contact_email": "haroon@seeko.so", "show_contact": true}}, "course": null, "contentPageSlug": "71bee3", "contentPage": {"id": 1013, "school_id": 10905, "slug": "71bee3", "content_page_type": "workshop", "sections": [{"uuid": "b3b15233-2dee-43cc-add5-ca5be1e4ccfd", "title": "Master Evaluation Techniques for LLM Apps", "version": 1, "image_url": "https://d2426xcxuh3ht5.cloudfront.net/cXdxvxLRT2eMzRzmdMH1_Lightning Lesson Template (1).png", "is_visible": true, "topic_desc": "Evaluating LLM applications is crucial for AI teams to ensure the effectiveness and reliability of AI systems.\n\nMastering evaluation techniques helps AI teams increase development speed, drive better business outcomes, and maintain a competitive edge.\n\nAn ureliable LLM app can lead to poor decision-making, user dissatisfaction, and potential security vulnerabilities.", "brand_logos": {"label": "Based on best practices from AI teams at", "logos": [{"name": "OpenAI", "type": "logo", "uuid": "b74607c5-a317-4ccb-805d-665cc27cf1d2", "file_type": "svg", "image_url": "https://asset.brandfetch.io/idR3duQxYl/idpkYOpXVV.svg"}, {"name": " Intercom", "type": "logo", "uuid": "7223454a-ad38-43f5-af99-0adc4d897b8e", "file_type": "svg", "image_url": "https://asset.brandfetch.io/idJRrn3vuU/idDUlDMkm1.svg"}, {"name": "Retool", "type": "logo", "uuid": "c18e57a3-344a-4bf6-98f1-a6cbf4123850", "file_type": "svg", "image_url": "https://asset.brandfetch.io/id3V8wH0I2/idwcsAhBma.svg"}, {"name": "Hex", "type": "icon", "uuid": "5f1ab69b-370c-4682-a361-03eaf8e0036e", "file_type": "png", "image_url": "https://asset.brandfetch.io/id0DD2N_HT/idrF-A6V86.png"}, {"name": "Airtable", "type": "logo", "uuid": "dae4f3d7-5fcb-41ed-8396-b11bfd716c46", "file_type": "svg", "image_url": "https://asset.brandfetch.io/iddsnRzkxS/idcxnDcOV0.svg"}]}, "section_type": "main", "instructor_infos": [{"name": "Haroon Choudery", "uuid": "89d8f99e-7e37-4a44-90f1-6d453ce9d79a", "title": "CEO, Autoblocks AI", "bio_html": "Haroon Choudery has spent over a decade in AI, building infrastructure and advising the teams deploying it. He previously founded Autoblocks, a VC-backed AI platform, as well as AI For Anyone, an AI education initiative that has taught over 70,000 people since 2016.
Now, he brings that experience to Seeko, an AI transformation consultancy that works with mid-market and enterprise companies to shape and execute AI transformations that actually work.
He also writes AI Ready, a newsletter read by more than 50,000 operators and thought leaders, covering the latest AI developments for anyone trying to keep up with the space.
Haroon is the Co-Founder & CEO at Autoblocks AI, one of the first-ever AI evaluation platforms.
He is also the host of the Building With AI podcast, where he interviews & distills best practices from top AI product teams at companies like OpenAI, Intercom, and Retool.
Haroon previously founded one of the top AI literacy nonprofits in the US, AI For Anyone, and is passionate about democratizing knowledge about AI.
", "image_url": "https://d2426xcxuh3ht5.cloudfront.net/xBTXPHEbR3avcbWsrI32_main-headshot copy.jpeg"}], "learning_outcomes": [{"uuid": "e42c567c-4bcc-4d31-8fe4-5a20b3adf87e", "title": "Why evaluations are necessary", "description": "Learn what role evaluations play in LLM apps and why they are crucial ensuring their effectiveness & reliability."}, {"uuid": "0673118a-1b32-4182-8d4f-36029f3919c9", "title": "How to choose the right evaluations", "description": "Explore how to select the best evaluation techniques (i.e., rules-based evals or LLM judges) for your LLM use case"}, {"uuid": "ca83b58f-79b9-470a-8733-ae9b898f7d52", "title": "Improving evaluation reliability", "description": "Discover methods, like fine-tuning & expert alignment, for improving the consistency and accuracy of your evaluations."}]}], "created_at": "2026-02-12T20:52:46.288885Z", "updated_at": null, "theme": null, "attrs": {"type": "workshop", "is_canceled": false, "is_delisted": false, "connected_course_id": 11912, "course_stripe_promo_code_id": null}, "school_event": {"id": 1013, "slug": null, "title": "Master Evaluation Techniques for LLM Apps", "description": null, "start_datetime": "2024-06-04T17:30:00Z", "end_datetime": "2024-06-04T18:00:00Z", "start_date": "2024-06-04", "start_time": "13:30:00", "duration_min": 30, "timezone": "America/New_York", "has_internal_recording": true, "is_recording_public": true}, "num_signups": 411}, "relatedWorkshops": [{"id": 361, "school_id": 10334, "is_visible_on_discovery_page": true, "is_canceled": false, "is_delisted": false, "connected_course_id": 11227, "workshop_tags": [{"id": 35, "tag_type": "category", "label": "AI", "slug": "ai-ll", "description": null, "parent_tag_id": null}, {"id": 137, "tag_type": "internal_collection", "label": "Instructor", "slug": "Instructor", "description": null, "parent_tag_id": null}, {"id": 271, "tag_type": "internal_collection", "label": "AI Evals", "slug": "ai-evals", "description": null, "parent_tag_id": null}, {"id": 290, "tag_type": "persona", "label": "For Product Managers", "slug": "for-product-managers", "description": null, "parent_tag_id": null}, {"id": 313, "tag_type": "topic", "label": "AI", "slug": "ai", "description": null, "parent_tag_id": null}], "published_content_page": {"id": 1190, "school_id": 10334, "slug": "45a0c2", "content_page_type": "workshop", "sections": [{"uuid": "12a7bc34-a308-4837-bcde-e5a93e962760", "title": "Evaluating LLMs for Your Applications", "version": 1, "image_url": "https://d2426xcxuh3ht5.cloudfront.net/0zds7xZLTNWfND9mcBNs_LLM Lightning Lesson Image .png", "is_visible": true, "topic_desc": "Evaluating GenAI applications is crucial for product managers to ensure that the chosen AI model meets not only technical specification but also business goals and user expectations. Understanding how to select the right model will help you get started on the path to define model specific evaluation metrics, this will allow teams to iterate quickly and create impactful solutions.", "brand_logos": {"label": "Previously at", "logos": []}, "section_type": "main", "instructor_infos": [{"name": "Mahesh Yadav", "uuid": "09cb5b23-2d7d-4258-8f60-c79d5d007853", "title": "GenAI Product Lead at Google, previously at Meta, Amazon, and Microsoft", "bio_html": "Mahesh Yadav is a Product Leader at Google GenAI team. Mahesh is one of the world's top AI executives and an award-winning AI Product Educator. His work on AI has been featured in the Nvidia GTC conference, Microsoft Build, and Meta blogs.
Mahesh has 20 years of experience in building products at Meta, Microsoft and AWS AI teams. Mahesh has worked in all layers of the AI stack from AI chips to LLM and has a deep understanding of how GenAI companies ship value to customers.
Currently, he leads AI agent at Google Cloud where it is used extensively for Gemini and other key Google products.
", "image_url": "https://d2426xcxuh3ht5.cloudfront.net/tAQehNkQOCcDjleGur5Q_MyImage.png"}], "learning_outcomes": [{"uuid": "750875e7-3e61-4e0f-81a2-187017715891", "title": "Framework for choosing the right LLMs", "description": "Identify the best GenAI model for your needs based on budget, latency and team expertise etc"}, {"uuid": "794fc156-3ba7-4fc9-ac76-0738795a0b45", "title": "Setting Evaluation Criteria", "description": "Learn how to set clear, actionable metrics despite challenges with benchmarks and real-world examples."}, {"uuid": "602124cf-8fd8-4f21-9029-82cc0b12d0a9", "title": "Case Study: Contract Processing Application", "description": "We'll use a contract processing application to illustrate how to apply the principles you just learn."}]}], "created_at": "2026-02-12T20:53:17.582460Z", "updated_at": null, "theme": null, "attrs": {"type": "workshop", "is_canceled": false, "is_delisted": false, "connected_course_id": 11227, "course_stripe_promo_code_id": null}, "school_event": {"id": 1190, "slug": null, "title": "Evaluating LLMs for Your Applications", "description": null, "start_datetime": "2024-07-09T16:00:00Z", "end_datetime": "2024-07-09T16:30:00Z", "start_date": "2024-07-09", "start_time": "09:00:00", "duration_min": 30, "timezone": "America/Los_Angeles", "has_internal_recording": true, "is_recording_public": true}, "num_signups": 1015}}, {"id": 4641, "school_id": 15363, "is_visible_on_discovery_page": true, "is_canceled": false, "is_delisted": false, "connected_course_id": 17433, "workshop_tags": [{"id": 31, "tag_type": "category", "label": "Data & engineering", "slug": "engineering-ll", "description": null, "parent_tag_id": null}, {"id": 35, "tag_type": "category", "label": "AI", "slug": "ai-ll", "description": null, "parent_tag_id": null}, {"id": 137, "tag_type": "internal_collection", "label": "Instructor", "slug": "Instructor", "description": null, "parent_tag_id": null}, {"id": 293, "tag_type": "persona", "label": "For Engineers", "slug": "for-engineers", "description": null, "parent_tag_id": null}, {"id": 299, "tag_type": "persona", "label": "For Data Scientists", "slug": "for-data-scientists", "description": null, "parent_tag_id": null}, {"id": 313, "tag_type": "topic", "label": "AI", "slug": "ai", "description": null, "parent_tag_id": null}, {"id": 432, "tag_type": "topic", "label": "LLM Ops", "slug": "llm-ops", "description": null, "parent_tag_id": null}, {"id": 441, "tag_type": "topic", "label": "RAG & Search", "slug": "rag-search", "description": null, "parent_tag_id": null}], "published_content_page": {"id": 5484, "school_id": 15363, "slug": "bdc051", "content_page_type": "workshop", "sections": [{"uuid": "551ebd35-af23-4b4e-a4b6-71fd9067a7c7", "title": "Use LLMs as Judges for Search Result Quality", "version": 1, "image_url": "https://d2426xcxuh3ht5.cloudfront.net/js84wiZsROSADh3O298K_rene-canva.png", "is_visible": true, "topic_desc": "Search result quality evaluation is at the core of any search improvement work. LLMs as Judges can be a game-changing tool for practitioners. They promise to overcome limitations of human or implicit judgments in terms of scalability, rapid availability, and also in terms of what aspects of result quality they cover - provided we get them right.", "brand_logos": {"label": "Built Search At", "logos": [{"name": "OpenSource Connections", "type": "icon", "uuid": "febe8e0f-2cae-4d4a-b068-e01d4058ea04", "file_type": "png", "image_url": "https://asset.brandfetch.io/id2VbA0wwD/idv8cqA42F.png"}, {"name": "Searchkernel", "type": "icon", "uuid": "eb62c864-d5df-4a8e-86fe-f5077f87b34a", "file_type": "jpeg", "image_url": "https://asset.brandfetch.io/idJTONuDZB/idvnLUMzvj.jpeg"}, {"name": "CareerBuilder", "type": "logo", "uuid": "086ba048-7f43-47ab-a58f-64cf3c19b5f4", "file_type": "svg", "image_url": "https://asset.brandfetch.io/idRrcMU04X/idPe8dAyv6.svg"}, {"name": "Lucidworks", "type": "icon", "uuid": "246d939e-345f-4198-a8fa-9ba139cf6311", "file_type": "jpeg", "image_url": "https://asset.brandfetch.io/idskYMg_Lc/idsgrCljLm.jpeg"}, {"name": "Presearch", "type": "logo", "uuid": "7c1cebd2-abc0-46fe-a5d7-b4df29259671", "file_type": "svg", "image_url": "https://asset.brandfetch.io/idTCWdN3V2/idDlXTsZYw.svg"}]}, "section_type": "main", "instructor_infos": [{"name": "René Kriegler", "uuid": "8a9a90a0-4570-4e6c-80d4-161b37ce2519", "logos": null, "title": "Chief Strategy Officer at OpenSource Connections", "bio_html": "René has worked in search for almost two decades, including in projects for some of the top 10 German e-commerce sites. He is co-founder and co-organiser of MICES (Mix-Camp E-commerce Search), an event that brings together the e-commerce search community each year. His technological focus is on open-source search technologies. He created and maintains the Querqy open source library for query rewriting. He believes that good search is not just a result of good technology but also of a team’s understanding of their search users, their experimentation capabilities and providing their users with a good search UX.
René works as Chief Strategy Officer at OpenSource Connections, where he and his team empower the company’s clients on search and AI.
", "image_url": "https://d2426xcxuh3ht5.cloudfront.net/ECgEgVGtTbiGgVrLYQXD_rene.jpeg", "highlights": null, "linkedin_url": null, "twitter_handle": null, "image_url_no_bg": null, "custom_social_link": null}, {"name": "Trey Grainger", "uuid": "b776a0a1-1a30-43ee-acda-6a2d4371aa37", "logos": {"label": "", "logos": []}, "title": "Author, \"AI-Powered Search\", Founder @ Searchkernel", "bio_html": "Trey is author of the book AI-Powered Search and is the founder of Searchkernel, a software company building the next generation of AI-powered search. He is an advisor to several startups and adjunct professor of computer science at Furman University. He previously served as CTO of Presearch, a decentralized web search engine, and as chief algorithms officer and SVP of engineering at Lucidworks, an search company whose search technology powers hundreds of the world’s leading organizations.
Trey is an instructor of Maven's AI-Powered Search course.
", "image_url": "https://d2426xcxuh3ht5.cloudfront.net/oWRCa6IQREKXkcwvEcUd_trey-headshot-2018.jpg", "highlights": [], "linkedin_url": "", "twitter_handle": "", "image_url_no_bg": null, "custom_social_link": ""}], "learning_outcomes": [{"uuid": "afdef458-6293-4cd0-8aec-99ad58ad0327", "title": "How can LLMs help us quantify search result quality?", "description": "Learn how LLMs as judges compare to human and implicit search evaluation and when to use them."}, {"uuid": "8ceab0ac-306d-45f7-b59f-3a85c64c8b76", "title": "LLMs need to judge based on rules", "description": "Learn why you should not just ask the LLM whether a document is relevant and how you can formulate those rules."}, {"uuid": "df1f6402-c26c-499d-97e6-e8ab10fb404a", "title": "Advanced: Using critique models and personas", "description": "Improving LLM judgments further and adapting to user segments by using critique models and personas"}]}], "created_at": "2026-02-12T21:04:32.198370Z", "updated_at": null, "theme": null, "attrs": {"type": "workshop", "is_canceled": false, "is_delisted": false, "connected_course_id": 17433, "course_stripe_promo_code_id": null}, "school_event": {"id": 5484, "slug": null, "title": "Use LLMs as Judges for Search Result Quality", "description": null, "start_datetime": "2025-10-28T15:00:00Z", "end_datetime": "2025-10-28T16:15:00Z", "start_date": "2025-10-28", "start_time": "11:00:00", "duration_min": 75, "timezone": "America/New_York", "has_internal_recording": true, "is_recording_public": true}, "num_signups": 210}}, {"id": 6877, "school_id": 15545, "is_visible_on_discovery_page": true, "is_canceled": false, "is_delisted": false, "connected_course_id": 18547, "workshop_tags": [{"id": 31, "tag_type": "category", "label": "Data & engineering", "slug": "engineering-ll", "description": null, "parent_tag_id": null}, {"id": 35, "tag_type": "category", "label": "AI", "slug": "ai-ll", "description": null, "parent_tag_id": null}, {"id": 137, "tag_type": "internal_collection", "label": "Instructor", "slug": "Instructor", "description": null, "parent_tag_id": null}, {"id": 271, "tag_type": "internal_collection", "label": "AI Evals", "slug": "ai-evals", "description": null, "parent_tag_id": null}, {"id": 293, "tag_type": "persona", "label": "For Engineers", "slug": "for-engineers", "description": null, "parent_tag_id": null}, {"id": 313, "tag_type": "topic", "label": "AI", "slug": "ai", "description": null, "parent_tag_id": null}], "published_content_page": {"id": 7675, "school_id": 15545, "slug": "fd511b", "content_page_type": "workshop", "sections": [{"uuid": "898819f3-a28b-4481-83e7-3496808e2841", "title": "Setting up your first AI eval with a LLM-as-judge", "version": 1, "image_url": "https://d2426xcxuh3ht5.cloudfront.net/APNicWPRyq1OY1KIMYgA_Maven345.png", "is_visible": true, "topic_desc": "Most teams building an LLM-as-a-judge make the same mistakes. They skip error analysis and ask the judge to look for hypothetical errors instead of real ones. They pack multiple criteria into one judge, creating high cognitive load that produces unreliable scores. They never validate the judge against human labels, so they don't know if it's accurate or noise.", "brand_logos": {"label": "2nd time founder", "logos": []}, "section_type": "main", "instructor_infos": [{"name": "Madalina Turlea", "uuid": "f019431e-d725-4140-a76c-8678f03f0c84", "logos": {"label": "", "logos": []}, "title": "Co-founder @Lovelaice, 10+ years in Product", "bio_html": "I'm co-founder of Lovelaice and a product leader with 10+ years building products across fintech, payments, and compliance. I hold a CFA charter and have led AI product development in highly regulated environments — where AI failures aren't just embarrassing, they're liabilities.
I've watched smart teams make the same mistakes: choosing models based on benchmarks that don't reflect their use case, writing prompts that work in testing but fail in production, and leaving domain experts out of the loop. These aren't edge cases — they're why 80% of AI projects underperform.
Through these failures (my own included), I developed a systematic approach to AI experimentation that puts domain expertise at the center. I teach what I've learned building Lovelaice: how to test, evaluate, and iterate on AI — before it reaches your users.
", "image_url": "https://d2426xcxuh3ht5.cloudfront.net/1tIUBUIARau45evs95td_WhatsApp%20Image%202025-09-05%20at%2021.24.26.jpeg", "highlights": [], "linkedin_url": "", "twitter_handle": "", "image_url_no_bg": null, "custom_social_link": ""}, {"name": "Catalina Turlea", "uuid": "84e106a0-dd75-4964-b600-15a68d889a7b", "logos": null, "title": "Founder @Lovelaice", "bio_html": "I bring over 14 years of software development expertise and a decade of startup experience to help teams build AI products that actually work. After founding my first company six years ago, I run a consultancy specializing in helping startups build MVPs, solve complex technical challenges, and integrate AI effectively.
I've seen firsthand how AI projects fail due to lack of systematic experimentation—teams treat AI like traditional software and struggle with inconsistent results. That's why I co-created Lovelace, a platform designed for non-technical professionals to experiment with AI agents systematically.
", "image_url": "https://d2426xcxuh3ht5.cloudfront.net/s45dRgjeRKzArParZX1o_IMG_5528.JPG", "highlights": null, "linkedin_url": null, "twitter_handle": null, "highlights_html": null, "image_url_no_bg": null, "custom_social_link": null}], "learning_outcomes": [{"uuid": "c78f508b-30a1-41ec-974c-b3d1a320ce21", "title": "Most common mistakes to avoid when building an LLM-as-judge", "description": "Understand why most teams' LLM judges don't work and the specific mistakes that make them unreliable."}, {"uuid": "97ea1fcc-f8a7-48fd-8844-534392b8f819", "title": "How to write your judge instructions", "description": "Learn how to identify what to check through LLM-as-judge, define specific rules, and build the judge prompt"}, {"uuid": "a4080945-0aac-40b4-82a9-2873adaa20e9", "title": "How to evaluate your LLM-as-judge", "description": "Know how to evaluate the evaluator by comparing judge scores to human labels and decide if you can trust the results."}]}], "created_at": "2026-02-25T10:21:35.614739Z", "updated_at": "2026-02-25T15:54:04.211766Z", "theme": null, "attrs": {"type": "workshop", "is_canceled": false, "is_delisted": false, "connected_course_id": 18547, "course_stripe_promo_code_id": null}, "school_event": {"id": 7675, "slug": null, "title": "Setting up your first AI eval with a LLM-as-judge", "description": null, "start_datetime": "2026-02-27T14:00:00Z", "end_datetime": "2026-02-27T14:45:00Z", "start_date": "2026-02-27", "start_time": "16:00:00", "duration_min": 45, "timezone": "Europe/Bucharest", "has_internal_recording": true, "is_recording_public": true}, "num_signups": 60}}], "exploreCourses": [{"objectID": "14937", "school_id": 9364, "school_name": "Hamel Husain & Shreya Shankar", "school_slug": "parlance-labs", "course_id": 14937, "course_name": "AI Evals For Engineers & PMs", "course_slug": "evals", "course_description": "Learn proven approaches for quickly improving AI applications. Build AI that works better than the competition, regardless of the use-case.", "clp_hero_media": {"media_type": "image_url", "image_url": "https://d2426xcxuh3ht5.cloudfront.net/FPbJ9HQJuNes7edtvAOg_Client%20Facing%20-%20AI%20Evals%20Course%20Image%20(1).png", "youtube_video_id": "dQw4w9WgXcQ", "vimeo_video_id": null}, "instructor_infos": [{"name": "Hamel Husain", "image_url": "https://user-assets.maven.com/instructor-avatars/evals/hamel-husain-1376a2cb.png", "image_url_no_bg": "https://user-assets.maven.com/instructor-avatars/evals/hamel-husain-f77002e5-no-bg.png", "uuid": "5efe32bb-d379-4954-b11d-fe8090530444", "bio_html": "Hamel Husain is a ML Engineer with over 20 years of experience. He has worked with innovative companies such as Airbnb and GitHub, which included early LLM research used by OpenAI, for code understanding. He has also led and contributed to numerous popular open-source machine-learning tools. Hamel is currently an independent consultant helping companies build AI products.
", "headline": "ML Engineer with 20 years of experience", "title": "ML Engineer with 20 years of experience.", "linkedin_url": "https://www.linkedin.com/in/hamelhusain/", "twitter_handle": "https://x.com/HamelHusain", "custom_social_link": "https://hamel.dev", "logos": {"logos": [{"image_url": "https://asset.brandfetch.io/idkuvXnjOH/idZUWqJuSO.svg", "type": "logo", "file_type": "svg", "uuid": "75592b77-ae32-4f3e-bae6-664342aa0612", "name": "Airbnb"}, {"image_url": "https://asset.brandfetch.io/idZAyF9rlg/id8uXh7wzx.png", "type": "logo", "file_type": "png", "uuid": "5c329210-dfd3-4795-9d21-010179b5c7d4", "name": "GitHub"}, {"image_url": "https://asset.brandfetch.io/idWT6nOiwd/idPNX7Z4_s.svg", "type": "logo", "file_type": "svg", "uuid": "e155fd46-c7ab-4171-b280-587d5c04b889", "name": "DataRobot"}, {"image_url": "https://asset.brandfetch.io/idLsWkUASh/idRHlXGSiV.svg", "type": "logo", "file_type": "svg", "uuid": "1300bf20-69e0-49a4-9350-3f169ae443b5", "name": "AlixPartners"}], "label": "Previously At"}, "highlights": null, "highlights_html": "Shreya builds open-source systems for AI-powered data processing. She is a final-year PhD at UC Berkeley. Shreya created DocETL, an open-source system for analyzing unstructured text at scale. DocETL has been deployed across journalism, law, medicine, policy, finance, and urban planning. Her research has been published at top computer science venues including VLDB, SIGMOD, and UIST (including a Best Paper award). Before her PhD, Shreya worked as a machine learning and data engineer at startups. She holds a BS in Computer Science from Stanford University.
", "headline": "ML Systems & Applied AI Evals Researcher", "title": "ML Systems Researcher Making AI Evaluation Work in Practice", "linkedin_url": "https://www.linkedin.com/in/shrshnk/", "twitter_handle": "https://x.com/sh_reya", "custom_social_link": "https://www.sh-reya.com/", "logos": {"logos": [{"image_url": "https://asset.brandfetch.io/id6O2oGzv-/idjOb2YcqW.svg", "type": "logo", "file_type": "svg", "uuid": "f682b22b-3a7a-4ca9-93b5-5e229b5ba270", "name": "Google"}, {"image_url": "https://asset.brandfetch.io/id519GJZCP/id9dCIwqqB.svg", "type": "logo", "file_type": "svg", "uuid": "0e7e677f-a863-4302-b91b-1a47d0bde282", "name": "UC Berkeley"}, {"image_url": "https://asset.brandfetch.io/idPv3iQPET/idBFw-L5SX.svg", "type": "logo", "file_type": "svg", "uuid": "954bba97-f64a-4292-b06a-9b36f0832c2f", "name": "Stanford University"}], "label": null}, "highlights": null, "highlights_html": "🚨 New: September 2026 cohort's material is completely refreshed to cover advancements in the field 🚨
All students get:
♾️ Unlimited access to future cohorts & office hours: never worry about timing or missing new material.
🗄️ Lifetime access to all materials!
🤖 6 months of unlimited access to our new AI Eval Assistant (more info below).
🧑🏫 10+ hours of office hours to maximize the value of live interaction.
🏫 A Discord community with continuous access to instructors to get unstuck (even after the course!).
---
Do you catch yourself asking any of the following questions while building AI applications?
1. How do I test applications when the outputs require subjective judgements?
2. If I change the prompt, how do I know I'm not breaking something else?
3. Where should I focus my engineering efforts? Do I need to test everything?
4. What if I have no data or customers, where do I start?
5. What metrics should I track? What tools should I use?
6. Can I automate testing and evaluation? If so, how do I trust it?
If so, this course is for you.
All sessions are live and recorded.
"}, "upmarket_clp_target_audience": {"items": [{"uuid": "7fffd146-76df-4458-a8ca-0d6f9da8f13b", "description_html": "Engineers and PMs who ship prompt changes and hope nothing breaks. (You'll learn to measure impact before and after every change.)
"}, {"uuid": "f6e06cbc-a58f-400c-9aa2-944d6eb9ee57", "description_html": "Teams still spot-checking AI outputs by hand instead of measuring systematically. (You'll learn how build automated evals you can trust.)
"}, {"uuid": "1813f77f-9acf-461a-8f52-24c4be08dc0f", "description_html": "Leaders who don't know where their AI is failing or where to invest resources. You'll learn how to systematically find & prioritize issues.
"}]}, "upmarket_clp_prerequisites": {"is_visible": false, "items": [{"uuid": "809d6402-312a-49a5-948b-c765f8ed1930", "name": "", "description": ""}]}, "upmarket_clp_outcomes": {"is_visible": false, "description": "Learn proven approaches for quickly improving AI applications. Build AI that works better than the competition, regardless of the use-case.", "items": [{"uuid": "c0b9d32c-d5a8-4ec4-ab65-019e923b3376", "title": "How To Collect Data For Evals", "items": ["Understand instrumentation and observability for tracking system behavior.", "Learn approaches for generating synthetic data to maximize error discovery and bootstrap product development.", "Understand how to choose the right tools and vendors for you, with deep dives into the most popular solutions in the evals space."]}, {"uuid": "f13edfd2-0053-47ae-aaed-623be18461a8", "title": "Get immediate clarity and direction with Error Analysis", "items": ["Apply data analysis techniques to rapidly find systematic issues in your product regardless of the use case.", "Master the processes and tools to annotate and analyze data quickly and efficiently.", "Learn how to analyze agentic systems (tool calls, RAG, etc.) to quickly identify systematic patterns and errors."]}, {"uuid": "3ce04a75-e8a4-4e2b-8286-12db770ee6ed", "title": "Implement Effective Evaluations", "items": ["Create evals that are customized to your product and provide immediate value, NOT generic off the shelf evals (which do not work). ", "Align evals with stakeholders & domain experts that allow you to scientifically trust the evals. ", "Create high-quality LLM-as-a-judge and code based evals with a systematic, iterative process."]}, {"uuid": "7c28868a-30d6-441a-9bc8-2b64f126801b", "title": "Master Architecture-Specific Eval Strategies", "items": ["Learn how to measure & debug RAG systems for retrieval relevance and factual accuracy.", "Understand how to tame multi-step pipelines to identify error propagation and root-causes of errors quickly.", "Master techniques that apply to multi-modal settings, including text, image, and audio interactions."]}, {"uuid": "d5178f99-429d-46a3-a791-fbff55ab277f", "title": "Run Evals In Production", "items": ["Learn how to set up automated evaluation gates in CI/CD pipelines.", "Understand methods for consistent comparison across experiments, including how to prepare and maintain datasets to prevent overfitting.", "Implement safety and quality control guardrails."]}, {"uuid": "3bb23500-d978-407a-a606-17da2311fd83", "title": "Ensure That Evals Lead To High ROI", "items": ["Develop a strong intuition of when to write an eval, and when NOT write an eval.", "Learn how to design interfaces to remove friction from reviewing data and collect higher quality data with less effort.", "Learn how to avoid common pitfalls surrounding team organization, collaboration, responsibilities, tools, automation, and metrics."]}]}, "vector_string": null, "vector": null, "score": 1.8217099}, {"objectID": "18328", "school_id": 14684, "school_name": "AI Product Hub", "school_slug": "aiproducthub", "course_id": 18328, "course_name": "AI Evals for PMs Certification", "course_slug": "genai-evals-certification", "course_description": "Acquire and develop a critical skill for product managers who are leading and contributing to AI products.", "clp_hero_media": null, "instructor_infos": [{"name": "Marily Nika, Ph.D AI/ML", "image_url": "https://d2426xcxuh3ht5.cloudfront.net/2PyoZup4Rqyxs6UnvHHL_marily-sh.png", "image_url_no_bg": "https://user-assets.maven.com/instructor-no-bg/2PyoZup4Rqyxs6UnvHHL_marily-sh.png", "uuid": "8845fba6-bed6-4c67-99f8-f140c0862a50", "bio_html": "AI PM Newsletter (50k+)
🔗 LinkedIn (120k+)
🎥 YouTube
❌ Twitter
Based in Silicon Valley, Dr. Marily Nika is an award-winning Gen AI Product Leader, best selling author and one of the world’s top AI educators. With 12+ years at Google & Meta and a PhD in Machine Learning, she’s been featured in Fortune, TechCrunch, is a Fellow at Harvard, and a TED AI / TEDx speaker.
Marily created Maven’s first & top-rated AI PM Certification and has taught 20k+ students worldwide. Recognized as Amplitude’s Most Influential Product Leader, Fortune 40 under 40, Top 100 Women in Tech and recipient of multiple global awards.
✉️ maven@aiproduct.com | Corporate & group discounts available
", "headline": "GenAI Product Builder @ Google, ex-Meta", "title": "GenAI Product Builder @ Google, ex-Meta", "linkedin_url": null, "twitter_handle": null, "custom_social_link": null, "logos": {"logos": [], "label": null}, "highlights": null, "highlights_html": "Diego Granados is a Product Manager with experience across web, mobile, AI, and machine learning. He collaborates closely with engineers, designers, marketers, and data scientists to build products that solve real user problems. Known for blending strategy, vision, and execution, Diego brings clarity and calm to complex, cross-functional teams. With a background in engineering and a passion for music, he combines logic with creativity. He also runs a YouTube channel to help aspiring PMs break into the field.
Specialties: Product management, AI/ML, consumer tech, agile, product design, data-driven decisions.
", "headline": "Product Manager AI&ML @ Google", "title": "Product Manager AI&ML @ Google", "linkedin_url": "https://www.linkedin.com/in/diegogranadosh/", "twitter_handle": null, "custom_social_link": null, "logos": {"logos": [], "label": ""}, "highlights": null, "highlights_html": "George Zoto is a Senior Solutions Architect specializing in AI, ML, and data science, known for turning complex technical ideas into practical business outcomes. At SHI, he guides enterprises on AI adoption, architecture, and responsible deployment. He also founded Deep Learning Adventures, a global community of 3,000+ practitioners focused on applied AI skills and emerging agentic systems.
Previously, George led data science and engineering teams at Share Our Strength, delivering scalable AI solutions that improved operational efficiency and decision making. His career spans Google Cloud–focused work, nonprofit innovation, and hands-on leadership across analytics, engineering, and ML strategy. George is recognized for bridging cutting-edge AI with real organizational needs.
", "headline": "Senior Data, ML and GenAI Scientist", "title": "Senior NLP/AI Scientist", "linkedin_url": "https://www.linkedin.com/in/george-zoto", "twitter_handle": null, "custom_social_link": null, "logos": {"logos": [], "label": ""}, "highlights": null, "highlights_html": "“Does it work?”... “Is it good enough?”... “Can we ship it?”...
How do you answer these questions for AI products? You’re responsible for “running evals” but what does that mean?
How do you choose the right metrics, interpret fuzzy results, and make a confident decision?
This course gives you a framework to do just that.
Map user value to evaluation (eval) objectives so your metrics aren’t abstract. Define success then translate it into measurable criteria.
Choose metrics you can actually maintain: capability, safety, UX friction, latency, cost and “does this reduce support tickets or increase activation.”
Set ship/no-ship thresholds you can defend to leadership.
Build lightweight workflows that work in real teams: human review where it matters, automation where it lasts, documentation that drives decisions.
Consider domain constraints (e.g., healthcare safety) and know what to avoid: silent failures, misleading proxy metrics and tests that don’t reflect production.
Tie everything to ROI: impact vs unit cost, eval coverage vs reliability, and the minimum viable monitoring you need post-launch.
Experience AI evals through a case-based approach with a real AI product that we evaluate together.
"}, "upmarket_clp_target_audience": {"items": [{"uuid": "8accc164-dcfd-49da-83ab-6f30aa8e4123", "description_html": "PMs leading AI features, growth, or platform initiatives
"}, {"uuid": "d65b52f1-83d6-4061-a36a-3bbbeb13d45a", "description_html": "PMs who partner with ML teams and want to set evaluation standards
"}, {"uuid": "08440536-98f7-4489-8ee4-b5c5b1fc4ef5", "description_html": "PMs who need to make clear “ship or hold” calls without doing the engineering
"}]}, "upmarket_clp_prerequisites": {"is_visible": false, "items": [{"uuid": "507fd4af-39ad-4c70-988b-f65cfdd16495", "name": "", "description": ""}]}, "upmarket_clp_outcomes": {"is_visible": false, "description": "Acquire and develop a critical skill for product managers who are leading and contributing to AI products.", "items": [{"uuid": "2d4dce09-f70d-4c73-90d0-9645b895445a", "title": "Make confident ship or hold decisions for AI features", "items": ["Learn a repeatable framework for deciding when an AI feature is ready to launch.", "Tie decisions to user value, business goals, and measurable evaluation criteria.", ""]}, {"uuid": "b147ad1b-c72d-4ca0-915c-3fb6e65ee3fd", "title": "Translate user value into clear evaluation goals", "items": ["Turn fuzzy product goals into concrete eval objectives and measurable success criteria.", "Define “good enough” in plain language before choosing metrics or tools.", ""]}, {"uuid": "4ff54795-1f30-44b6-bfed-1a6628fc2ca4", "title": "Choose the right metrics for capability, safety, UX, and cost", "items": ["Use a PM-friendly menu of metrics to avoid misleading proxies and anchor on business value.", "Balance capability, latency, UX friction, and cost without being an ML engineer.", ""]}, {"uuid": "e6292e3c-cf76-4f2f-8280-6afd278ed4ec", "title": "Set defensible thresholds leadership will trust", "items": ["Create ship/no-ship thresholds tied to KPIs, risk, and user impact.", "Know when to stop tweaking prompts and when launch should be paused.", ""]}, {"uuid": "ede68296-f582-4951-af7b-2470dcd3138f", "title": "Build lightweight evaluation workflows teams can maintain", "items": ["Learn what to automate, what to review manually, and how to design sustainable processes.", "Produce datasets, golden examples, and error taxonomies your team can reuse.", ""]}, {"uuid": "22c9f084-6639-4c8c-a498-144e442776f1", "title": "Navigate domain constraints and avoid common failures", "items": ["Understand risks in sensitive domains like healthcare and finance.", "Avoid silent failures, weak proxies, and tests that don’t reflect production.", ""]}]}, "vector_string": null, "vector": null, "score": 1.8188357}, {"objectID": "16252", "school_id": 14543, "school_name": "Actualize", "school_slug": "actualize", "course_id": 16252, "course_name": "Let's Code LLM Chatbots and Agents from Scratch", "course_slug": "lets-build-an-llm-app", "course_description": "Build robust LLM-powered apps, chatbots, and agents. Learn by writing real code, one line at a time.", "clp_hero_media": {"media_type": "image_url", "image_url": "https://d2426xcxuh3ht5.cloudfront.net/ECLUstVxRTapd75yiV6T_Let%E2%80%99s%20build%20an%20(1).png", "youtube_video_id": "dQw4w9WgXcQ", "vimeo_video_id": null}, "instructor_infos": [{"name": "Jay Wengrow", "image_url": "https://d2426xcxuh3ht5.cloudfront.net/D7qiQm1QQnSyfc2zlMWn_jay_headshot_2.jpg", "image_url_no_bg": "https://user-assets.maven.com/instructor-no-bg/D7qiQm1QQnSyfc2zlMWn_jay_headshot_2.jpg", "uuid": "54b3f405-4165-4772-9d81-aef4df628b99", "bio_html": "Jay Wengrow is an experienced educator and software engineer, and the author of A Common-Sense Guide to AI Engineering. He is also the founder of Actualize, a software and AI engineering education company, and specializes in making advanced technical topics approachable for professionals across industries. He also wrote the popular Common-Sense Guide to Data Structures and Algorithms book series.
", "headline": "CEO @ Actualize", "title": "Software Engineer and Educator, Author of A Common-Sense Guide to AI Engineering", "linkedin_url": "https://www.linkedin.com/in/jaywengrow", "twitter_handle": null, "custom_social_link": "https://commonsensedev.com", "logos": {"logos": [], "label": ""}, "highlights": null, "highlights_html": "Do you have a grasp of basic LLM principles but struggle to put it all into code? Tired of scattered documentation, outdated tutorials, and frameworks that impress in demos but collapse in production?
You’re not alone. Building with large language models doesn’t have to be confusing - it can be clear, structured, and deeply rewarding.
In this course, you’ll build real LLM-powered apps, one line of code at a time. You'll see how an app is built from scratch in real time, and follow along. Along the way, you'll discover not only how to implement these systems, but the why behind each line of code.
From chatbots to agents and RAG to evals, you’ll learn the core concepts of building atop LLMs. More importantly, you’ll get how LLMs “think” - allowing you to guide them to do what you want despite their nondeterministic nature. Additionally, you’ll be able to balance quality, latency, and cost, making big-picture decisions about AI-powered app architecture.
AI engineering isn’t just another branch of software development - it’s a different mindset altogether.
"}, "upmarket_clp_target_audience": {"items": [{"uuid": "2fa366ac-151a-4935-b9f9-9268822af55a", "description_html": "Software engineers who want to build LLM-powered chatbots and agents, or break into AI engineering more generally
"}, {"uuid": "b8e486b4-9c53-4058-bb05-74eefa0c001a", "description_html": "Data Scientists/Engineers who want to build their own LLM-powered apps
"}, {"uuid": "358711b3-1c12-4b65-b60f-8bc6fdc484f5", "description_html": "Product Managers who want to understand AI engineering from the coding perspective
"}]}, "upmarket_clp_prerequisites": {"is_visible": true, "items": [{"uuid": "bd5e660e-1376-4123-9d7d-06b967d6dd4a", "name": "Software Engineering", "description": "This course is designed for current software engineers who will use Python to build LLM-powered apps. We won't teach basic coding here."}, {"uuid": "3f29a4a6-aa0a-451f-a640-4b55dbd8cdf0", "name": "Note about Python", "description": "We'll be using Python, but we'll keep it simple in case you're more familiar with other coding languages."}, {"uuid": "c5ccfcc3-6fcf-4e0d-a2ac-ed1139d1b89e", "name": "Exception: Product Managers", "description": "If you want an inside look at what's involved with building AI apps but don't plan on coding yourself, you'll still follow what's going on."}]}, "upmarket_clp_outcomes": {"is_visible": false, "description": "Build robust LLM-powered apps, chatbots, and agents. Learn by writing real code, one line at a time.", "items": [{"uuid": "088fba96-7520-41a8-b40e-c306aacc4f49", "title": "Gain real-world intuition for working with LLMs", "items": ["You'll create LLM-powered apps from scratch without needing to rely on frameworks that abstract away the important details.", "You'll understand how LLMs work under the hood, and what they can and cannot do.", "Understanding the essential nature of LLMs as a statistical next-token predictor is the key for utilizing them effectively."]}, {"uuid": "5909c488-aa91-406f-8321-09507060bad9", "title": "Reliably guide LLMs instead of getting unpredictable results.", "items": ["LLMs are unpredictable by nature, but you'll know how to steer them into doing what you want and achieving your goals.", "You'll use prompt and context engineering to tame the nondeterministic LLM and reduce hallucinations.", ""]}, {"uuid": "0924b19f-ecd0-447a-9274-b05dbec0be1c", "title": "Build retrieval-augmented assistants that can access proprietary knowledge", "items": ["You'll build chatbots that converse accurately about your organization's data and help advise users appropriately.", "You'll create your own search engine that use vector databases to power semantic search.", ""]}, {"uuid": "677bb943-772b-4d9c-8b49-4acc88f05f37", "title": "Use evals to systematically optimize your LLM app’s performance over time", "items": ["Instead of simply hoping that your newest updates make things better, you'll use evals to consistently monitor your app's quality.", "You'll learn how to perform error analysis, annotate traces, and even automate your evals.", ""]}, {"uuid": "ca7e1dff-73e3-4acd-9a1b-0d52a26ec949", "title": "Assemble agents that act upon the real world", "items": ["You'll equip LLMs with tools that can do more than generate text - they'll trigger real code functions", "Your agents will write code, deploy websites, and even produce podcasts.", "You'll gain finer control of your agent by assembling agentic workflows, thereby reducing the ways in which the agent can go rogue."]}]}, "vector_string": null, "vector": null, "score": 1.8171184}, {"objectID": "18719", "school_id": 13230, "school_name": "Aishwarya & Kiriti", "school_slug": "aishwarya-kiriti", "course_id": 18719, "course_name": "Beyond Evals: Designing Improvement Flywheels for AI Products", "course_slug": "evals-problem-first", "course_description": "Evals aren't your product moat. A continuous improvement flywheel is.", "clp_hero_media": null, "instructor_infos": [{"name": "Aishwarya Naresh Reganti", "image_url": "https://d2426xcxuh3ht5.cloudfront.net/nu8rxGKURO6yZd7nioyW_Applied%20LLMs%20(59).png", "image_url_no_bg": "https://user-assets.maven.com/instructor-no-bg/nu8rxGKURO6yZd7nioyW_Applied%20LLMs%20(59).png", "uuid": "28cb7a06-5bca-41d9-bb86-fd690ca10b68", "bio_html": "Aishwarya Naresh Reganti is the CEO and Founder of LevelUp Labs, a startup that helps enterprises build bespoke AI solutions and transformation programs, clients include Hitachi Digital, Deloitte, several F500 companies etc. Prior to this, she worked as a tech lead at AWS and led initiatives to develop and deploy production-ready generative AI solutions enterprise clients.
With over 10 years of experience in machine learning, she has published more than 35 research papers at top-tier AI conferences, including NeurIPS, AAAI, and CVPR.
Aishwarya has taught professional courses on AI at renowned institutions like MIT and Oxford.
Aishwarya is also is a sought-after thought leader frequently invited to speak at leading conferences and events, including TEDx, MLOps World, and ReWork.
", "headline": "AI Founder & Advisor to F500s | Ex-AWS", "title": "AI Founder & Advisor to F500s | Ex-AWS", "linkedin_url": "https://www.linkedin.com/in/areganti/", "twitter_handle": "https://x.com/AishwaryaN17746", "custom_social_link": "https://levelup-labs.ai/", "logos": {"logos": [{"image_url": "https://asset.brandfetch.io/idVoqFQ-78/idAgLCF87x.svg", "type": "logo", "file_type": "svg", "uuid": "e440fd6c-ab31-42f8-9b80-b505d4b5cf54", "name": "Amazon Web Services"}, {"image_url": "https://asset.brandfetch.io/idchmboHEZ/idc7I0n_oy.svg", "type": "logo", "file_type": "svg", "uuid": "c353eece-7995-4c76-ab6a-36d83ce19863", "name": "Microsoft"}, {"image_url": "https://asset.brandfetch.io/idRK9czrcr/idpEPRxdxK.png", "type": "logo", "file_type": "png", "uuid": "9a72d629-e7a9-4885-ad54-e0d5bbc37b79", "name": "University of Oxford"}, {"image_url": "https://asset.brandfetch.io/idr1PnJb4A/ideLoQ3NtM.svg", "type": "logo", "file_type": "svg", "uuid": "7e54e175-3bd0-4014-be83-60d5c4a06f1b", "name": "MIT"}], "label": "Worked/Taught at"}, "highlights": null, "highlights_html": "Check out our free AI Evals Certification here (5000+) Learners!
Most teams building AI products today know about evals and LLM judges, but they’re deeply confused about how to actually turn them into actionable improvement signals post production. Just throwing in LLM judges won’t magically fix your product. It won’t.
This course shows how to use offline evals, online evals, and production monitoring to drive continuous improvement, based on real production experience.
We believe that going forward, a thoughtfully designed data flywheel will be the moat for AI applications. This course shows you how to build it.
This course is for people who’ve already built AI systems and are stuck with evals that are noisy, expensive, or useless
If you need something more foundational, check out our top-rated flagship enterprise AI course taken by over 1500+ builders and leaders.
For questions or bulk enrollments please email problemfirst.ai@gmail.com
We follow a flipped-classroom format with async lecture content and meet twice a week for office hour style sessions to optimize two-way/interactive live time.
Software/AI Engineers, Strategists, Data Professionals, Solution Architects and Consultants looking for systematic evaluation practices
"}, {"uuid": "b5c46a31-7cc4-4108-b1cc-fa4f7139c677", "description_html": "Business Leaders and Product Managers seeking to gain the technical understanding of AI Product Lifecycle and intentional iteration
"}, {"uuid": "b9502928-5de5-425b-b4dc-8ebfe0a51061", "description_html": "Founders & teams stuck at systematic evaluations and monitoring, unsure how to turn signals into real product improvements.
"}]}, "upmarket_clp_prerequisites": {"is_visible": true, "items": [{"uuid": "e5a71020-bfaf-4274-b060-977d09c17727", "name": "A good understanding of AI system design", "description": "You should have built small AI applications before and be familiar with concepts like RAG, MCP, and tool use."}, {"uuid": "bee90b1a-9b01-4a62-96da-a59938e6abb3", "name": "Working knowledge of Python/Coding (Optional)", "description": "The course includes optional Python assignments, you're free to use coding agents, but basic coding skills are recommended"}]}, "upmarket_clp_outcomes": {"is_visible": false, "description": "Evals aren't your product moat. A continuous improvement flywheel is.", "items": [{"uuid": "ef28044f-4c30-4ec1-adbe-c770c578167d", "title": "Understand what evals actually mean and why it’s misunderstood", "items": ["Learn why evals is often reduced to LLM judges, and why that breaks real AI products.", "Understand evaluation as a lifecycle practice, not a one time pre deployment check.", "Build an intuition for continuously improving AI products through a connected evals and monitoring loop."]}, {"uuid": "dfe80a39-5a4d-46c6-832b-122e27fc9494", "title": "Build a pre-deployment evaluation foundation from scratch", "items": ["Learn how to build reference datasets before your product ships.", "Identify key stakeholders (product, engineering, SMEs) who can help build an accurate behaviour estimate", "Design improvement setups that evolve as the product evolves, including when to add or retire evals."]}, {"uuid": "2d96c9ed-10d7-4f96-a0ec-532051917c00", "title": "Choose the right types of metrics for the system you’re building", "items": ["Understand how evaluation strategies differ across RAG systems, tool calling workflows, and multi turn systems.", "Learn which categories of evals matter for different system behaviors and failure modes.", "Learn when to use LLM judges, when not to, and how to avoid overly biasing on them."]}, {"uuid": "e812f71e-3303-4f2d-90c2-5c93beb57b44", "title": "Monitor in production and close the improvement loop", "items": ["Learn which production signals matter, both explicit and implicit, and how to use them for improvement.", "Understand how behavior drifts in production and how monitoring helps surface it early.", "Close the loop by feeding production signals back into evals and datasets, and learn how this data flywheel becomes a product moat."]}]}, "vector_string": null, "vector": null, "score": 1.8220851}], "workshopTags": [{"id": 136, "tag_type": "internal_collection", "label": "LL Partner", "slug": "ll-partner", "description": null, "parent_tag_id": null}, {"id": 35, "tag_type": "category", "label": "AI", "slug": "ai-ll", "description": null, "parent_tag_id": null}, {"id": 271, "tag_type": "internal_collection", "label": "AI Evals", "slug": "ai-evals", "description": null, "parent_tag_id": null}, {"id": 293, "tag_type": "persona", "label": "For Engineers", "slug": "for-engineers", "description": null, "parent_tag_id": null}, {"id": 299, "tag_type": "persona", "label": "For Data Scientists", "slug": "for-data-scientists", "description": null, "parent_tag_id": null}, {"id": 313, "tag_type": "topic", "label": "AI", "slug": "ai", "description": null, "parent_tag_id": null}], "videoChapters": [{"title": "Welcome and Agenda Overview", "start_seconds": 34}, {"title": "What Are Evaluations and Why Are They Necessary?", "start_seconds": 472}, {"title": "Applying Evaluations in RAG and Agent Applications", "start_seconds": 587}, {"title": "Categorizing Evaluation Techniques: Programmatic vs. Manual", "start_seconds": 771}, {"title": "A Framework for Choosing Evaluations: Reliability, Scalability, Cost, and Value", "start_seconds": 990}, {"title": "Techniques for Improving LLM Judge Reliability", "start_seconds": 1390}, {"title": "Auto Blocks Platform Demo", "start_seconds": 1538}, {"title": "Q&A and Closing Remarks", "start_seconds": 1746}], "navPages": [{"category": {"slug": "ai", "href": "/courses/ai", "title": "AI", "palette": {"--palette-category-base": "#460952", "--palette-category-highlight": "#931FAA", "--palette-category-border": "#EDA6FB"}}, "topics": [{"title": "Agentic AI", "slug": "ai/agentic-ai", "href": "/courses/ai/agentic-ai"}, {"title": "Coding with AI", "slug": "ai/ai-coding", "href": "/courses/ai/ai-coding"}, {"title": "AI Workflows", "slug": "ai/ai-workflows", "href": "/courses/ai/ai-workflows"}, {"title": "Claude Code", "slug": "ai/claude-code", "href": "/courses/ai/claude-code"}, {"title": "OpenClaw", "slug": "ai/openclaw", "href": "/courses/ai/openclaw"}, {"title": "Vibe Coding", "slug": "ai/vibe-coding", "href": "/courses/ai/vibe-coding"}, {"title": "AI Evals", "slug": "ai/evals", "href": "/courses/ai/evals"}, {"title": "AI Transformation", "slug": "ai/ai-transformation", "href": "/courses/ai/ai-transformation"}, {"title": "RAG & Search", "slug": "ai/rag-search", "href": "/courses/ai/rag-search"}, {"title": "MCP", "slug": "ai/mcp", "href": "/courses/ai/mcp"}, {"title": "AI for PMs", "slug": "ai/for-product-managers", "href": "/courses/ai/for-product-managers"}, {"title": "AI for Engineers", "slug": "ai/for-engineers", "href": "/courses/ai/for-engineers"}, {"title": "AI for Designers", "slug": "ai/for-designers", "href": "/courses/ai/for-designers"}, {"title": "AI for Marketers", "slug": "ai/for-marketers", "href": "/courses/ai/for-marketers"}, {"title": "AI for Founders", "slug": "ai/for-founders", "href": "/courses/ai/for-founders"}]}, {"category": {"slug": "product", "href": "/courses/product", "title": "Product", "palette": {"--palette-category-base": "#510B00", "--palette-category-highlight": "#B33600", "--palette-category-border": "#F7804D"}}, "topics": [{"title": "AI for PMs", "slug": "for-product-managers/ai", "href": "/courses/for-product-managers/ai"}, {"title": "Agentic AI", "slug": "for-product-managers/agentic-ai", "href": "/courses/for-product-managers/agentic-ai"}, {"title": "AI Evals", "slug": "for-product-managers/ai-evals", "href": "/courses/for-product-managers/ai-evals"}, {"title": "Vibe Coding", "slug": "for-product-managers/vibe-coding", "href": "/courses/for-product-managers/vibe-coding"}, {"title": "Product Sense", "slug": "for-product-managers/product-sense", "href": "/courses/for-product-managers/product-sense"}, {"title": "Product Discovery", "slug": "for-product-managers/product-discovery", "href": "/courses/for-product-managers/product-discovery"}, {"title": "User Research", "slug": "for-product-managers/user-research", "href": "/courses/for-product-managers/user-research"}, {"title": "Prototyping", "slug": "for-product-managers/prototyping", "href": "/courses/for-product-managers/prototyping"}, {"title": "Growth", "slug": "for-product-managers/growth", "href": "/courses/for-product-managers/growth"}, {"title": "Analytics", "slug": "for-product-managers/analytics", "href": "/courses/for-product-managers/analytics"}, {"title": "Tech Foundations", "slug": "for-product-managers/technical-foundations", "href": "/courses/for-product-managers/technical-foundations"}, {"title": "Strategy", "slug": "for-product-managers/strategy", "href": "/courses/for-product-managers/strategy"}, {"title": "Influence", "slug": "for-product-managers/influence", "href": "/courses/for-product-managers/influence"}, {"title": "Leadership", "slug": "for-product-managers/leadership", "href": "/courses/for-product-managers/leadership"}, {"title": "Career Growth", "slug": "for-product-managers/career-growth", "href": "/courses/for-product-managers/career-growth"}]}, {"category": {"slug": "engineering", "href": "/courses/engineering", "title": "Engineering", "palette": {"--palette-category-base": "#00332F", "--palette-category-highlight": "#00746B", "--palette-category-border": "#88B7B2"}}, "topics": [{"title": "AI for Engineers", "slug": "for-engineers/ai", "href": "/courses/for-engineers/ai"}, {"title": "Agentic AI", "slug": "for-engineers/agentic-ai", "href": "/courses/for-engineers/agentic-ai"}, {"title": "Coding with AI", "slug": "for-engineers/ai-coding", "href": "/courses/for-engineers/ai-coding"}, {"title": "Claude Code", "slug": "for-engineers/claude-code", "href": "/courses/for-engineers/claude-code"}, {"title": "OpenClaw", "slug": "for-engineers/openclaw", "href": "/courses/for-engineers/openclaw"}, {"title": "MCP", "slug": "for-engineers/mcp", "href": "/courses/for-engineers/mcp"}, {"title": "RAG & Search", "slug": "for-engineers/rag-search", "href": "/courses/for-engineers/rag-search"}, {"title": "AI Evals", "slug": "for-engineers/ai-evals", "href": "/courses/for-engineers/ai-evals"}, {"title": "Machine Learning", "slug": "for-engineers/ml", "href": "/courses/for-engineers/ml"}, {"title": "LLM Ops", "slug": "for-engineers/llm-ops", "href": "/courses/for-engineers/llm-ops"}, {"title": "Context Eng", "slug": "for-engineers/context-engineering", "href": "/courses/for-engineers/context-engineering"}, {"title": "Security", "slug": "for-engineers/security", "href": "/courses/for-engineers/security"}, {"title": "System Design", "slug": "for-engineers/system-design", "href": "/courses/for-engineers/system-design"}, {"title": "Leadership", "slug": "for-engineers/leadership", "href": "/courses/for-engineers/leadership"}, {"title": "Career Growth", "slug": "for-engineers/career-growth", "href": "/courses/for-engineers/career-growth"}]}, {"category": {"slug": "design", "href": "/courses/design", "title": "Design", "palette": {"--palette-category-base": "#38085F", "--palette-category-highlight": "#772CBB", "--palette-category-border": "#B5A2D0"}}, "topics": [{"title": "AI for Designers", "slug": "for-designers/ai", "href": "/courses/for-designers/ai"}, {"title": "Agentic AI", "slug": "for-designers/agentic-ai", "href": "/courses/for-designers/agentic-ai"}, {"title": "Vibe Coding", "slug": "for-designers/vibe-coding", "href": "/courses/for-designers/vibe-coding"}, {"title": "Prototyping", "slug": "for-designers/prototyping", "href": "/courses/for-designers/prototyping"}, {"title": "Figma", "slug": "for-designers/figma", "href": "/courses/for-designers/figma"}, {"title": "Design Systems", "slug": "for-designers/design-systems", "href": "/courses/for-designers/design-systems"}, {"title": "User Research", "slug": "for-designers/user-research", "href": "/courses/for-designers/user-research"}, {"title": "Product Discovery", "slug": "for-designers/product-discovery", "href": "/courses/for-designers/product-discovery"}, {"title": "UX", "slug": "for-designers/ux", "href": "/courses/for-designers/ux"}, {"title": "UI", "slug": "for-designers/ui", "href": "/courses/for-designers/ui"}, {"title": "Visual Design", "slug": "for-designers/visual-design", "href": "/courses/for-designers/visual-design"}, {"title": "Design Strategy", "slug": "for-designers/strategy", "href": "/courses/for-designers/strategy"}, {"title": "Influence", "slug": "for-designers/influence", "href": "/courses/for-designers/influence"}, {"title": "Leadership", "slug": "for-designers/leadership", "href": "/courses/for-designers/leadership"}, {"title": "Career Growth", "slug": "for-designers/career-growth", "href": "/courses/for-designers/career-growth"}]}, {"category": {"slug": "marketing", "href": "/courses/marketing", "title": "Marketing", "palette": {"--palette-category-base": "#431532", "--palette-category-highlight": "#943170", "--palette-category-border": "#C99EB5"}}, "topics": [{"title": "AI for Marketers", "slug": "for-marketers/ai", "href": "/courses/for-marketers/ai"}, {"title": "Agentic AI", "slug": "for-marketers/agentic-ai", "href": "/courses/for-marketers/agentic-ai"}, {"title": "Vibe Coding", "slug": "for-marketers/vibe-coding", "href": "/courses/for-marketers/vibe-coding"}, {"title": "Automation", "slug": "for-marketers/marketing-automation", "href": "/courses/for-marketers/marketing-automation"}, {"title": "Content Marketing", "slug": "for-marketers/content-marketing", "href": "/courses/for-marketers/content-marketing"}, {"title": "Demand Gen", "slug": "for-marketers/demand-generation", "href": "/courses/for-marketers/demand-generation"}, {"title": "Go-to-Market", "slug": "for-marketers/gtm", "href": "/courses/for-marketers/gtm"}, {"title": "Product Marketing", "slug": "for-marketers/product-marketing", "href": "/courses/for-marketers/product-marketing"}, {"title": "Positioning", "slug": "for-marketers/positioning", "href": "/courses/for-marketers/positioning"}, {"title": "Social Media", "slug": "for-marketers/social-media", "href": "/courses/for-marketers/social-media"}, {"title": "Brand", "slug": "for-marketers/brand", "href": "/courses/for-marketers/brand"}, {"title": "B2B Marketing", "slug": "for-marketers/b2b", "href": "/courses/for-marketers/b2b"}, {"title": "SEO & AEO", "slug": "for-marketers/seo-aeo", "href": "/courses/for-marketers/seo-aeo"}, {"title": "Strategy", "slug": "for-marketers/strategy", "href": "/courses/for-marketers/strategy"}, {"title": "Leadership", "slug": "for-marketers/leadership", "href": "/courses/for-marketers/leadership"}]}, {"category": {"slug": "leadership", "href": "/courses/leadership", "title": "Leadership", "palette": {"--palette-category-base": "#50090E", "--palette-category-highlight": "#A9202A", "--palette-category-border": "#D49C98"}}, "topics": [{"title": "AI for Leaders", "slug": "for-leaders/ai", "href": "/courses/for-leaders/ai"}, {"title": "Agentic AI", "slug": "for-leaders/agentic-ai", "href": "/courses/for-leaders/agentic-ai"}, {"title": "AI Transformation", "slug": "for-leaders/ai-transformation", "href": "/courses/for-leaders/ai-transformation"}, {"title": "AI Governance", "slug": "for-leaders/ai-governance", "href": "/courses/for-leaders/ai-governance"}, {"title": "Communication", "slug": "for-leaders/communication", "href": "/courses/for-leaders/communication"}, {"title": "Influence", "slug": "for-leaders/influence", "href": "/courses/for-leaders/influence"}, {"title": "Strategy", "slug": "for-leaders/strategy", "href": "/courses/for-leaders/strategy"}, {"title": "Management", "slug": "for-leaders/management", "href": "/courses/for-leaders/management"}, {"title": "People Operations", "slug": "for-leaders/people-operations", "href": "/courses/for-leaders/people-operations"}, {"title": "Exec Presence", "slug": "for-leaders/executive-presence", "href": "/courses/for-leaders/executive-presence"}, {"title": "Storytelling", "slug": "for-leaders/storytelling", "href": "/courses/for-leaders/storytelling"}, {"title": "Goal-setting", "slug": "for-leaders/goal-setting", "href": "/courses/for-leaders/goal-setting"}, {"title": "Personal Brand", "slug": "for-leaders/personal-brand", "href": "/courses/for-leaders/personal-brand"}, {"title": "Career Growth", "slug": "for-leaders/career-growth", "href": "/courses/for-leaders/career-growth"}]}, {"category": {"slug": "founders", "href": "/courses/founders", "title": "Founders", "palette": {"--palette-category-base": "#0C3209", "--palette-category-highlight": "#077202", "--palette-category-border": "#69C662"}}, "topics": [{"title": "AI for Founders", "slug": "for-founders/ai", "href": "/courses/for-founders/ai"}, {"title": "Agentic AI", "slug": "for-founders/agentic-ai", "href": "/courses/for-founders/agentic-ai"}, {"title": "AI Workflows", "slug": "for-founders/ai-workflows", "href": "/courses/for-founders/ai-workflows"}, {"title": "Vibe Coding", "slug": "for-founders/vibe-coding", "href": "/courses/for-founders/vibe-coding"}, {"title": "Prototyping", "slug": "for-founders/prototyping", "href": "/courses/for-founders/prototyping"}, {"title": "Product Sense", "slug": "for-founders/product-sense", "href": "/courses/for-founders/product-sense"}, {"title": "Positioning", "slug": "for-founders/positioning", "href": "/courses/for-founders/positioning"}, {"title": "Product Discovery", "slug": "for-founders/product-discovery", "href": "/courses/for-founders/product-discovery"}, {"title": "Management", "slug": "for-founders/management", "href": "/courses/for-founders/management"}, {"title": "Strategy", "slug": "for-founders/strategy", "href": "/courses/for-founders/strategy"}, {"title": "Go-to-Market", "slug": "for-founders/gtm", "href": "/courses/for-founders/gtm"}, {"title": "Personal Brand", "slug": "for-founders/personal-brand", "href": "/courses/for-founders/personal-brand"}, {"title": "Leadership", "slug": "for-founders/leadership", "href": "/courses/for-founders/leadership"}, {"title": "Fundraising", "slug": "for-founders/fundraising", "href": "/courses/for-founders/fundraising"}, {"title": "PMF", "slug": "for-founders/product-market-fit", "href": "/courses/for-founders/product-market-fit"}]}, {"category": {"slug": "more", "href": "/courses/more", "title": "More", "palette": {"--palette-category-base": "#1E2C2B", "--palette-category-highlight": "#4A6361", "--palette-category-border": "#B9C1BE"}}, "topics": [{"title": "Everyone", "slug": "for-everyone", "href": "/courses/for-everyone"}, {"title": "Operators", "slug": "for-operators", "href": "/courses/for-operators"}, {"title": "Data Scientists", "slug": "for-data-scientists", "href": "/courses/for-data-scientists"}, {"title": "Business Analysts", "slug": "for-business-analysts", "href": "/courses/for-business-analysts"}, {"title": "User Researchers", "slug": "for-user-researchers", "href": "/courses/for-user-researchers"}, {"title": "Customer Success", "slug": "for-customer-success", "href": "/courses/for-customer-success"}, {"title": "Project Managers", "slug": "for-project-managers", "href": "/courses/for-project-managers"}, {"title": "HR Professionals", "slug": "for-hr-professionals", "href": "/courses/for-hr-professionals"}, {"title": "Sales People", "slug": "for-sales-people", "href": "/courses/for-sales-people"}, {"title": "Lawyers", "slug": "for-lawyers", "href": "/courses/for-lawyers"}, {"title": "Finance", "slug": "for-finance-professionals", "href": "/courses/for-finance-professionals"}, {"title": "Investors", "slug": "for-investors", "href": "/courses/for-investors"}, {"title": "Real Estate", "slug": "for-real-estate-agents", "href": "/courses/for-real-estate-agents"}, {"title": "Educators", "slug": "for-educators", "href": "/courses/for-educators"}, {"title": "Creators", "slug": "for-creators", "href": "/courses/for-creators"}]}], "navTagLinks": [{"tag": {"id": 293, "tag_type": "persona", "label": "For Engineers", "slug": "for-engineers", "description": null, "parent_tag_id": null}, "link_slug": "for-engineers"}, {"tag": {"id": 299, "tag_type": "persona", "label": "For Data Scientists", "slug": "for-data-scientists", "description": null, "parent_tag_id": null}, "link_slug": "for-data-scientists"}], "now": "2026-05-19T22:54:22.483Z"}, "__N_SSG": true}, "page": "/p/[slug]/[...ignored]", "query": {"slug": "71bee3", "ignored": ["master-evaluation-techniques-for-llm-apps"]}, "buildId": "qPlLjBFLQ9fnkFKJ7n7E-", "isFallback": false, "isExperimentalCompile": false, "gsp": true, "scriptLoader": []}}