Shane Butler is a Principal Data Scientist at Ontra, where he leads AI evaluation strategy for product development in legal tech. He has 10+ years of experience in product data science, causal inference, and measurement across both B2B SaaS and consumer products, with prior roles at Stripe, Nextdoor, AppFolio, and PwC. His work focuses on practical, end-to-end methods for evaluating AI features in production, connecting offline quality signals to real user and business outcomes. Shane is also the co-host of the AI podcast Data Neighbor, where he interviews product, data, and engineering leaders shaping the next generation of analytics in an AI-driven landscape.
", "headline": "Principal Data Scientist, AI Evaluations", "title": "Principal Data Scientist, AI Evaluations at Ontra", "linkedin_url": "https://www.linkedin.com/in/shaneausleybutler/", "twitter_handle": "@ShaneAusley", "custom_social_link": "https://substack.com/@sabutler", "logos": {"logos": [{"image_url": "https://asset.brandfetch.io/idxAg10C0L/idcVaYaqmU.svg", "type": "logo", "file_type": "svg", "uuid": "1c46ef78-788f-4dd0-8629-a92a77ea67ab", "name": "Stripe"}, {"image_url": "https://asset.brandfetch.io/idQKUi61nl/idcQVi6Svx.svg", "type": "logo", "file_type": "svg", "uuid": "136e3e05-d836-402c-9dfc-7ddbd1feae77", "name": "Nextdoor"}, {"image_url": "https://asset.brandfetch.io/idDNlITzR1/idOW-eBZ3u.png", "type": "logo", "file_type": "png", "uuid": "21c11206-3c9b-4ef3-bb42-7c0a3570be51", "name": "PwC India"}, {"image_url": "https://asset.brandfetch.io/id8AJtesvE/idIM0mZ2DQ.svg", "type": "logo", "file_type": "svg", "uuid": "0acb3b0a-5f9f-4f92-a0e9-67378e37b092", "name": "Ontra"}, {"image_url": "https://asset.brandfetch.io/idVVIcX1XO/id2w1tGpDN.svg", "type": "logo", "file_type": "svg", "uuid": "94293e56-aa70-441d-bf00-24a4707ac321", "name": "AppFolio"}], "label": "Stripe, Nextdoor, PwC"}, "highlights": null, "highlights_html": "", "preferred_name": null}], "instructor_caption": "Principal Data Scientist, AI Evaluations at Ontra", "course_tags": [{"id": 244, "tag_type": "internal_collection", "label": "Courses filtered for recommendations", "slug": "courses-filtered-for-recommendations", "description": null, "parent_tag_id": null}, {"id": 271, "tag_type": "internal_collection", "label": "AI Evals", "slug": "ai-evals", "description": null, "parent_tag_id": null}, {"id": 290, "tag_type": "persona", "label": "For Product Managers", "slug": "for-product-managers", "description": null, "parent_tag_id": null}, {"id": 299, "tag_type": "persona", "label": "For Data Scientists", "slug": "for-data-scientists", "description": null, "parent_tag_id": null}, {"id": 312, "tag_type": "topic", "label": "Product Strategy", "slug": "product-strategy", "description": null, "parent_tag_id": null}, {"id": 313, "tag_type": "topic", "label": "AI", "slug": "ai", "description": null, "parent_tag_id": null}, {"id": 382, "tag_type": "topic", "label": "AI Evals", "slug": "evals", "description": null, "parent_tag_id": null}, {"id": 387, "tag_type": "topic", "label": "Data Analysis", "slug": "data-analysis", "description": null, "parent_tag_id": null}, {"id": 426, "tag_type": "topic", "label": "Claude Code", "slug": "claude-code", "description": null, "parent_tag_id": null}]}, "contentPageSlug": "75ecae", "contentPage": {"id": 10669, "school_id": 15975, "slug": "75ecae", "content_page_type": "workshop", "sections": [{"uuid": "afb83492-0cc6-4dc6-9207-f6a7fb5ee5a9", "title": "Agentic Experiment Design & Analysis", "version": 1, "image_url": "https://d2426xcxuh3ht5.cloudfront.net/t6exI9xSHmRrWSQLIMzQ_Screenshot%202026-05-08%20at%201.05.36%E2%80%AFPM.png", "is_visible": true, "topic_desc": "Most experiment readouts ship bad decisions: p-values without validity checks, no segment analysis, vague recommendations. AI makes it easier to crunch numbers, which makes it easier to crunch the wrong ones. Learn the Agentic Experiment Loop to design, validate, and analyze tests, then see how AI agents accelerate every stage without replacing judgment.", "brand_logos": {"label": "Previously at", "logos": []}, "section_type": "main", "instructor_infos": [{"name": "Sravya Madipalli", "uuid": "996a7c00-b07f-4d0d-998e-880d10cc851d", "logos": null, "title": "Senior DS Leader (Ex-Microsoft)", "bio_html": "Sravya Madipalli is a Senior Manager of Data Science with 14+ years of experience helping teams make better decisions with data. She has built and led data science and product analytics teams at Microsoft, eBay, Nextdoor, and Superhuman (prev. Grammarly), working closely with product, engineering, marketing, and leadership. Her expertise spans experimentation, metrics design, modeling, analytics, and translating complex user behavior into clear, actionable insights.
", "headline": "", "image_url": "https://d2426xcxuh3ht5.cloudfront.net/4sDV2jRaRpupgn80ttRV_sravya_headshot.png", "highlights": null, "linkedin_url": null, "preferred_name": null, "twitter_handle": null, "highlights_html": null, "image_url_no_bg": null, "custom_social_link": null}, {"name": "Hai Guan", "uuid": "2f708305-428c-49f1-b356-c600dde3618c", "logos": {"label": "", "logos": []}, "title": "Head of Data at Ontra, Ex-LinkedIn", "bio_html": "Hai Guan leads the data organization at Ontra, the leading legal tech AI solutions for private markets. He previously led Data Science & Analytics at LinkedIn, Nextdoor, Pinterest, and Meta. He's spent a decade teaching product development teams how to ask questions that actually drive decisions—and now teaches how to combine that judgment with AI to move 10x faster.
", "headline": "", "image_url": "https://d2426xcxuh3ht5.cloudfront.net/emUbu9BYSimXINxFIsfY_hai_headshot.png", "highlights": [], "linkedin_url": "", "preferred_name": null, "twitter_handle": "", "highlights_html": null, "image_url_no_bg": null, "custom_social_link": ""}, {"name": "Shane Butler", "uuid": "4b567796-4ede-422c-8480-0a628e69e7f4", "logos": {"label": "", "logos": []}, "title": "Principal Data Scientist at Ontra", "bio_html": "Shane Butler is a Principal Data Scientist at Ontra, where he leads evaluation strategy for AI product development in the legal tech domain. He has more than ten years of experience in product data science and causal inference, with prior roles at Stripe, Nextdoor, and PwC. Shane is also the co-host of the AI podcast Data Neighbor, where he interviews product, data, and engineering leaders who are pioneering the next generation of data science and analytics in an AI-driven landscape.
", "headline": "", "image_url": "https://d2426xcxuh3ht5.cloudfront.net/A3No3mB6RZ6hdqCmJeQQ_shane_butler_headshot.png", "highlights": [], "linkedin_url": "", "preferred_name": null, "twitter_handle": "", "highlights_html": null, "image_url_no_bg": null, "custom_social_link": ""}], "learning_outcomes": [{"uuid": "37d245e0-cf60-4932-9ff4-216529bc6fbb", "title": "Frame experiments using the Agentic Loop", "description": "Apply the 4 stages (Frame, Power, Validate, Decide) to design tests that hold up to scrutiny."}, {"uuid": "f0f0362e-10f4-4d55-a792-798297b6814e", "title": "Catch the validity gaps that ship bad decisions", "description": "Spot sample ratio mismatch, novelty effects, and underpowered tests before the readout meeting."}, {"uuid": "d1ad5233-f71d-4ef8-8371-7d0f0c88884f", "title": "Use AI agents to design and analyze A/B tests", "description": "Apply a design helper prompt and watch Claude Code produce a full experiment readout from raw data."}]}], "created_at": "2026-05-08T20:05:58.210357Z", "updated_at": "2026-05-08T20:06:01.606915Z", "theme": null, "attrs": {"type": "workshop", "is_canceled": false, "is_delisted": false, "connected_course_id": 18421, "course_stripe_promo_code_id": null}, "school_event": {"id": 10669, "slug": null, "title": "Agentic Experiment Design & Analysis", "description": null, "start_datetime": "2026-06-05T19:00:00Z", "end_datetime": "2026-06-05T20:00:00Z", "start_date": "2026-06-05", "start_time": "12:00:00", "duration_min": 60, "timezone": "America/Los_Angeles", "has_internal_recording": false, "is_recording_public": false}, "num_signups": 30}, "relatedWorkshops": [{"id": 8305, "school_id": 4779, "is_visible_on_discovery_page": true, "is_canceled": false, "is_delisted": false, "connected_course_id": 18262, "workshop_tags": [{"id": 299, "tag_type": "persona", "label": "For Data Scientists", "slug": "for-data-scientists", "description": null, "parent_tag_id": null}, {"id": 326, "tag_type": "topic", "label": "Experimentation", "slug": "experimentation", "description": null, "parent_tag_id": null}, {"id": 327, "tag_type": "topic", "label": "Analytics", "slug": "analytics", "description": null, "parent_tag_id": null}, {"id": 387, "tag_type": "topic", "label": "Data Analysis", "slug": "data-analysis", "description": null, "parent_tag_id": null}, {"id": 418, "tag_type": "persona", "label": "For Business Analysts", "slug": "for-business-analysts", "description": null, "parent_tag_id": null}, {"id": 425, "tag_type": "topic", "label": "AI Workflows", "slug": "ai-workflows", "description": null, "parent_tag_id": null}, {"id": 450, "tag_type": "internal_collection", "label": "Maven Marketing: Amplitude Series AI Analytics", "slug": "ai-analytics-for-builders", "description": null, "parent_tag_id": null}, {"id": 454, "tag_type": "internal_collection", "label": "Agent Powered Super IC Series", "slug": "agent-super-ic", "description": null, "parent_tag_id": null}], "published_content_page": {"id": 11038, "school_id": 4779, "slug": "93d1d0", "content_page_type": "workshop", "sections": [{"uuid": "7a681d01-b49b-4fa5-9a00-61afe3766744", "title": "Experimentation in the age of AI", "version": 1, "image_url": "https://d2426xcxuh3ht5.cloudfront.net/0KxyoPN4RhWLf0ImUbAf_Single%20Instructor%20-%20theme%20a%20(1).png", "is_visible": true, "topic_desc": "AI is accelerating product development at a pace that traditional experimentation frameworks weren't built for. \nTeams that can't adapt risk shipping AI features blind — without knowing what's actually moving the needle. Rigorous experimentation skills are no longer a nice-to-have; they're the difference between AI that drives growth and AI that creates noise.", "brand_logos": {"label": "Previously at", "logos": []}, "section_type": "main", "instructor_infos": [{"name": "Manisha Arora", "uuid": "d10bb9bf-fd3d-4f08-9941-132f1f5727f5", "logos": null, "title": "Data Science Lead at Google", "bio_html": "I am a seasoned Data Science professional with 10+ years of experience leading data science teams and driving business growth through data-driven decision making. Having navigated the job market both as a candidate and as an interviewer, I understand the intricacies of the hiring process from both perspectives.
I am passionate about democratizing data science and enabling others level up in their careers. I found PrepVector to enable aspiring professionals to excel in their data science careers. I have taught 350+ data professionals through my courses at Maven & PrepVector. I am committed to providing individuals with the skills and knowledge required to thrive in the industry.
", "headline": "", "image_url": "https://d2426xcxuh3ht5.cloudfront.net/VnfREONSeWtmNOhggk0z_00100lrPORTRAIT_00100_BURST20191225204859204_COVER-2%20(2).jpg", "highlights": null, "linkedin_url": null, "preferred_name": null, "twitter_handle": null, "highlights_html": null, "image_url_no_bg": null, "custom_social_link": null}], "learning_outcomes": [{"uuid": "9ce9f110-4d69-41c3-a7a9-96d94540c4b4", "title": "How AI is reshaping Experimentation", "description": "Understand how the rise of GenAI is changing what we test, how we test it, and what \"success\" means in a modern product"}, {"uuid": "69940709-c526-4848-85ce-506ab2c9b5d3", "title": "Where Traditional A/B testing Frameworks fall short", "description": "Explore why traditional A/B Testing breaks with AI-driven feature development & learn the mental models being used"}, {"uuid": "43bc8257-2d31-477c-bbae-a3a340bd9d6b", "title": "How to Design Experiments for AI-driven Development", "description": "Learn practical frameworks for structuring experiments, while maintaining & translating results into product decisions"}]}], "created_at": "2026-05-15T13:45:34.255177Z", "updated_at": "2026-05-15T17:31:28.340397Z", "theme": null, "attrs": {"type": "workshop", "is_canceled": false, "is_delisted": false, "connected_course_id": 18262, "course_stripe_promo_code_id": null}, "school_event": {"id": 11038, "slug": null, "title": "Experimentation in the age of AI", "description": null, "start_datetime": "2026-06-12T19:00:00Z", "end_datetime": "2026-06-12T19:30:00Z", "start_date": "2026-06-13", "start_time": "00:30:00", "duration_min": 30, "timezone": "Asia/Calcutta", "has_internal_recording": false, "is_recording_public": false}, "num_signups": 46}}, {"id": 8195, "school_id": 10770, "is_visible_on_discovery_page": true, "is_canceled": false, "is_delisted": false, "connected_course_id": 11750, "workshop_tags": [{"id": 137, "tag_type": "internal_collection", "label": "Instructor", "slug": "Instructor", "description": null, "parent_tag_id": null}, {"id": 296, "tag_type": "persona", "label": "For Designers", "slug": "for-designers", "description": null, "parent_tag_id": null}, {"id": 313, "tag_type": "topic", "label": "AI", "slug": "ai", "description": null, "parent_tag_id": null}, {"id": 424, "tag_type": "topic", "label": "AI Transformation", "slug": "ai-transformation", "description": null, "parent_tag_id": null}, {"id": 425, "tag_type": "topic", "label": "AI Workflows", "slug": "ai-workflows", "description": null, "parent_tag_id": null}, {"id": 454, "tag_type": "internal_collection", "label": "Agent Powered Super IC Series", "slug": "agent-super-ic", "description": null, "parent_tag_id": null}], "published_content_page": {"id": 10829, "school_id": 10770, "slug": "706f93", "content_page_type": "workshop", "sections": [{"uuid": "860cc232-189b-44c4-8049-849c7e67ac41", "title": "How to Become an Agentic Designer", "version": 1, "image_url": "https://d2426xcxuh3ht5.cloudfront.net/fXR23PL3ShKTIVjai9sP_Opening%20Slide%20-%20Duo%20(22).jpg", "is_visible": true, "topic_desc": "Your company isn't going to change overnight — but you can. Designers and product teams who start working differently will transform companies. I've spent 12+ months building with AI in every step of our product development. Good news. AI leads to more work, not less — when used correctly. This talk gives you the framing and tangible next steps to apply.", "brand_logos": {"label": "Trusted by", "logos": [{"name": "Amazon", "type": "logo", "uuid": "3f446e67-afd8-440b-9fd3-40c98b6a45ad", "file_type": "svg", "image_url": "https://asset.brandfetch.io/idawOgYOsG/idEqxL5YFw.svg"}, {"name": "Microsoft", "type": "logo", "uuid": "420e9580-fb35-4493-b6d6-3f8f3e37fa10", "file_type": "svg", "image_url": "https://asset.brandfetch.io/idchmboHEZ/idc7I0n_oy.svg"}, {"name": "Blackberry", "type": "logo", "uuid": "1ecfea7d-d379-4a8d-b2c0-6fb1fd15f2cd", "file_type": "svg", "image_url": "https://asset.brandfetch.io/idD0HYWPCK/idvUa8d6w6.svg"}, {"name": "Adobe", "type": "logo", "uuid": "8d268767-0fa5-44a6-9378-f615d3cd143f", "file_type": "svg", "image_url": "https://asset.brandfetch.io/id_KsyK7J9/idqgHTh9pg.svg"}, {"name": "Yara International", "type": "logo", "uuid": "7137c837-351f-4751-a792-6fd279c7bf5b", "file_type": "svg", "image_url": "https://asset.brandfetch.io/id6x21KF-p/idRiGSerTz.svg"}]}, "section_type": "main", "instructor_infos": [{"name": "Dominik Gmeiner", "uuid": "7758cce9-2f87-4c92-a077-fc33a383ca03", "logos": {"label": "", "logos": []}, "title": "Designer, Business Owner", "bio_html": "Dominik is a hands-on designer building products fully with AI for international clients, and bootstrapping a profitable business made possible by AI.
He's spent 14 years across design and product roles spanning consumer electronics, preventative health, decentralized finance, and agriculture.
He's built startups and teams inside large companies, including leading design for a global agriculture division spanning offices from Singapore to São Paulo. His work has been used by hundreds of thousands of users.
Dominik has seen what it takes to ship real products inside complex organizations, and how to bring a product to life. He's now building, writing, and sharing some of what he knows with us in this talk.
", "headline": "", "image_url": "https://d2426xcxuh3ht5.cloudfront.net/8NPUyBdpRDKgGGp0fM5n_CleanShot%202026-05-10%20at%2007.11.09%402x.png", "highlights": [], "linkedin_url": "", "preferred_name": null, "twitter_handle": "", "highlights_html": null, "image_url_no_bg": null, "custom_social_link": ""}, {"name": "Xinran Ma", "uuid": "221", "logos": {"label": "Trusted by", "logos": [{"name": "Microsoft", "type": "logo", "uuid": "8b745c36-eb32-4fe5-9463-32abd63d5f7e", "file_type": "svg", "image_url": "https://asset.brandfetch.io/idchmboHEZ/idc7I0n_oy.svg"}, {"name": "Adobe", "type": "logo", "uuid": "b734e9c2-208a-4f04-b0de-cdb7cd7f82c7", "file_type": "svg", "image_url": "https://asset.brandfetch.io/id_KsyK7J9/idqgHTh9pg.svg"}, {"name": "Etsy", "type": "logo", "uuid": "cd15a886-96e3-4152-ad5f-1c5944c48016", "file_type": "svg", "image_url": "https://asset.brandfetch.io/idzyTAzn6G/idEsdAbZSE.svg"}, {"name": "Workday", "type": "logo", "uuid": "75fd634b-3cd4-4f93-af25-72c14ada6a3d", "file_type": "svg", "image_url": "https://asset.brandfetch.io/id0V-YF4nE/idAi5TCbxO.svg"}, {"name": "Amazon", "type": "logo", "uuid": "a71bf18e-a516-4735-9634-9ed0f0b85cf3", "file_type": "svg", "image_url": "https://asset.brandfetch.io/idawOgYOsG/idEqxL5YFw.svg"}]}, "title": "Founder of Design with AI | 3x Author | Startup Advisor", "bio_html": "Xinran Ma is the founder and author of Design with AI, speaker, and award-winning designer.
Design with AI is one of the fastest-growing AI communities with over 40K subscribers from companies like Google, Amazon, Microsoft, and McKinsey.
Xinran has also written multiple Amazon #1 New Release books in UX and teaches a popular course on Maven: AI for Product Designers.
He regularly delivers lectures and conducts AI workshops at places such as Microsoft, Columbia Business School, UXPA, the City of Vancouver, Workday, and Pratt Institute.
One of his lightning sessions is the most watched design classes on Maven with about 7K signups.
", "headline": "", "image_url": "https://d2426xcxuh3ht5.cloudfront.net/3rXbb896SKWoVLUK4MQu_CleanShot%202026-04-16%20at%2012.00.02%402x.png", "highlights": null, "linkedin_url": "https://www.linkedin.com/in/davidmaxinran/", "preferred_name": "Xinran", "twitter_handle": null, "highlights_html": null, "image_url_no_bg": "https://user-assets.maven.com/instructor-no-bg/3rXbb896SKWoVLUK4MQu_CleanShot%202026-04-16%20at%2012.00.02%402x.png", "custom_social_link": "https://designwithai.substack.com/"}], "learning_outcomes": [{"uuid": "40b4911c-9be1-4575-b1df-fbd2f3457340", "title": "Apply your existing design training to get more from AI", "description": ""}, {"uuid": "aab1e679-3f89-4343-b6b0-9838439b4941", "title": "Train AI to conform to your design standards", "description": ""}, {"uuid": "bfc8a7b3-ea77-4231-a3bf-bb7801e1d3b0", "title": "Reframe a designer's value when anyone can prototype", "description": ""}]}], "created_at": "2026-05-12T16:35:27.772999Z", "updated_at": "2026-05-12T16:36:09.894107Z", "theme": null, "attrs": {"type": "workshop", "is_canceled": false, "is_delisted": false, "connected_course_id": 11750, "course_stripe_promo_code_id": null}, "school_event": {"id": 10829, "slug": null, "title": "How to Become an Agentic Designer", "description": null, "start_datetime": "2026-05-22T14:00:00Z", "end_datetime": "2026-05-22T15:00:00Z", "start_date": "2026-05-22", "start_time": "10:00:00", "duration_min": 60, "timezone": "America/New_York", "has_internal_recording": false, "is_recording_public": false}, "num_signups": 210}}, {"id": 8166, "school_id": 16736, "is_visible_on_discovery_page": true, "is_canceled": false, "is_delisted": false, "connected_course_id": 19834, "workshop_tags": [{"id": 159, "tag_type": "internal_collection", "label": "LL non-instructor", "slug": "ll-non-instructor", "description": null, "parent_tag_id": null}, {"id": 290, "tag_type": "persona", "label": "For Product Managers", "slug": "for-product-managers", "description": null, "parent_tag_id": null}, {"id": 299, "tag_type": "persona", "label": "For Data Scientists", "slug": "for-data-scientists", "description": null, "parent_tag_id": null}, {"id": 305, "tag_type": "persona", "label": "For Project Managers", "slug": "for-project-managers", "description": null, "parent_tag_id": null}, {"id": 313, "tag_type": "topic", "label": "AI", "slug": "ai", "description": null, "parent_tag_id": null}, {"id": 327, "tag_type": "topic", "label": "Analytics", "slug": "analytics", "description": null, "parent_tag_id": null}, {"id": 341, "tag_type": "topic", "label": "Growth", "slug": "growth", "description": null, "parent_tag_id": null}, {"id": 382, "tag_type": "topic", "label": "AI Evals", "slug": "evals", "description": null, "parent_tag_id": null}, {"id": 425, "tag_type": "topic", "label": "AI Workflows", "slug": "ai-workflows", "description": null, "parent_tag_id": null}, {"id": 450, "tag_type": "internal_collection", "label": "Maven Marketing: Amplitude Series AI Analytics", "slug": "ai-analytics-for-builders", "description": null, "parent_tag_id": null}, {"id": 454, "tag_type": "internal_collection", "label": "Agent Powered Super IC Series", "slug": "agent-super-ic", "description": null, "parent_tag_id": null}], "published_content_page": {"id": 11239, "school_id": 16736, "slug": "0fd335", "content_page_type": "workshop", "sections": [{"uuid": "e03751df-a544-4b95-92d2-da15dc388811", "title": "Build an AI Growth Optimization System", "version": 1, "image_url": "https://d2426xcxuh3ht5.cloudfront.net/Yauor2slTTawz6BMv8wu_Single%20Instructor%20-%20theme%20a%20(3).png", "is_visible": true, "topic_desc": "Evals tell you your model is working. They don't tell you it's working for your users. The jump from offline evals to live experiments is where most AI product teams stall, and where the teams that don't pull ahead. This session shows you how to build the full loop.", "brand_logos": {"label": "", "logos": [{"name": "Amplitude", "type": "logo", "uuid": "ea305f18-3ce6-4689-ad9d-082fef682933", "file_type": "svg", "image_url": "https://asset.brandfetch.io/idgF9FqCgW/idNL-u0qDf.svg"}, {"name": "Eppo", "type": "logo", "uuid": "fde69932-1995-41ea-baa2-94c457247170", "file_type": "svg", "image_url": "https://asset.brandfetch.io/idXRefEmqh/iddY5Bkldk.svg"}, {"name": "Cameo", "type": "logo", "uuid": "ff9b1538-4eda-4421-817a-feaced96a47d", "file_type": "svg", "image_url": "https://asset.brandfetch.io/id5wE2bxKj/id2zeVdYnh.svg"}, {"name": "SpotHero", "type": "logo", "uuid": "38bbf899-5218-4471-b2b9-1afd94e8c6b2", "file_type": "svg", "image_url": "https://asset.brandfetch.io/idG1kat0PS/idSBa8MHKN.svg"}, {"name": "PowerReviews", "type": "logo", "uuid": "8b37877a-ca9e-4e28-87dc-5ea759d86968", "file_type": "png", "image_url": "https://asset.brandfetch.io/idi6VZgusG/idpbXS9_Ab.png"}]}, "section_type": "main", "instructor_infos": [{"name": "Eric Metelka", "uuid": "0723e22a-5ec3-4ed9-9ffd-b905678545be", "logos": null, "title": "Product Leader, Amplitude Experiment |\nLenny's Top 25 Contributor", "bio_html": "Eric Metelka leads product for Amplitude Experiment, where his team builds the experimentation and feature flag platform. He has 11 years of experience building growth and experimentation systems across marketplaces, SaaS, and consumer platforms, and is a top 25 contributor to Lenny's Newsletter, where he advises PMs and growth professionals on product and career strategy.
Connect with Eric in Cohort, an Amplitude community for product leaders and AI builders.
", "headline": "", "image_url": "https://d2426xcxuh3ht5.cloudfront.net/cv8uWEkcTcqp21CY4ebU_Eric%20Metelka.jpg", "highlights": null, "linkedin_url": null, "preferred_name": null, "twitter_handle": null, "highlights_html": null, "image_url_no_bg": null, "custom_social_link": null}], "learning_outcomes": [{"uuid": "21e54f83-2dab-4308-a2ef-0e2b3adf6d9b", "title": "Identify where evals end and experiments begin", "description": "Recognize where evals stop giving signal and when to run a live experiment"}, {"uuid": "dba30408-f43a-4989-8830-8f5579646e94", "title": "Layer evals, experiments, and analytics", "description": "Apply a practical framework for sequencing eval checks, online A/B tests, and Amplitude analytics across the AI product"}, {"uuid": "40887cda-b1cc-4a24-9104-571193b3a1f9", "title": "Automate your experimentation loop", "description": "Use AI-native tooling to generate hypotheses, run tests, and surface analysis without manual overhead at each step."}]}], "created_at": "2026-05-19T21:54:50.003357Z", "updated_at": "2026-05-19T21:54:57.937369Z", "theme": null, "attrs": {"type": "workshop", "is_canceled": false, "is_delisted": false, "connected_course_id": 19834, "course_stripe_promo_code_id": null}, "school_event": {"id": 11239, "slug": null, "title": "Build an AI Growth Optimization System", "description": null, "start_datetime": "2026-06-17T17:00:00Z", "end_datetime": "2026-06-17T17:45:00Z", "start_date": "2026-06-17", "start_time": "12:00:00", "duration_min": 45, "timezone": "America/Chicago", "has_internal_recording": false, "is_recording_public": false}, "num_signups": 185}}], "exploreCourses": null, "workshopTags": [{"id": 418, "tag_type": "persona", "label": "For Business Analysts", "slug": "for-business-analysts", "description": null, "parent_tag_id": null}, {"id": 450, "tag_type": "internal_collection", "label": "Maven Marketing: Amplitude Series AI Analytics", "slug": "ai-analytics-for-builders", "description": null, "parent_tag_id": null}, {"id": 454, "tag_type": "internal_collection", "label": "Agent Powered Super IC Series", "slug": "agent-super-ic", "description": null, "parent_tag_id": null}, {"id": 137, "tag_type": "internal_collection", "label": "Instructor", "slug": "Instructor", "description": null, "parent_tag_id": null}, {"id": 425, "tag_type": "topic", "label": "AI Workflows", "slug": "ai-workflows", "description": null, "parent_tag_id": null}, {"id": 327, "tag_type": "topic", "label": "Analytics", "slug": "analytics", "description": null, "parent_tag_id": null}, {"id": 290, "tag_type": "persona", "label": "For Product Managers", "slug": "for-product-managers", "description": null, "parent_tag_id": null}, {"id": 293, "tag_type": "persona", "label": "For Engineers", "slug": "for-engineers", "description": null, "parent_tag_id": null}, {"id": 299, "tag_type": "persona", "label": "For Data Scientists", "slug": "for-data-scientists", "description": null, "parent_tag_id": null}, {"id": 315, "tag_type": "topic", "label": "Agentic AI", "slug": "agentic-ai", "description": null, "parent_tag_id": null}, {"id": 426, "tag_type": "topic", "label": "Claude Code", "slug": "claude-code", "description": null, "parent_tag_id": null}, {"id": 387, "tag_type": "topic", "label": "Data Analysis", "slug": "data-analysis", "description": null, "parent_tag_id": null}, {"id": 313, "tag_type": "topic", "label": "AI", "slug": "ai", "description": null, "parent_tag_id": null}], "videoChapters": [], "navPages": [{"category": {"slug": "ai", "href": "/courses/ai", "title": "AI", "palette": {"--palette-category-base": "#460952", "--palette-category-highlight": "#931FAA", "--palette-category-border": "#EDA6FB"}}, "topics": [{"title": "Agentic AI", "slug": "ai/agentic-ai", "href": "/courses/ai/agentic-ai"}, {"title": "Coding with AI", "slug": "ai/ai-coding", "href": "/courses/ai/ai-coding"}, {"title": "AI Workflows", "slug": "ai/ai-workflows", "href": "/courses/ai/ai-workflows"}, {"title": "Claude Code", "slug": "ai/claude-code", "href": "/courses/ai/claude-code"}, {"title": "OpenClaw", "slug": "ai/openclaw", "href": "/courses/ai/openclaw"}, {"title": "Vibe Coding", "slug": "ai/vibe-coding", "href": "/courses/ai/vibe-coding"}, {"title": "AI Evals", "slug": "ai/evals", "href": "/courses/ai/evals"}, {"title": "AI Transformation", "slug": "ai/ai-transformation", "href": "/courses/ai/ai-transformation"}, {"title": "RAG & Search", "slug": "ai/rag-search", "href": "/courses/ai/rag-search"}, {"title": "MCP", "slug": "ai/mcp", "href": "/courses/ai/mcp"}, {"title": "AI for PMs", "slug": "ai/for-product-managers", "href": "/courses/ai/for-product-managers"}, {"title": "AI for Engineers", "slug": "ai/for-engineers", "href": "/courses/ai/for-engineers"}, {"title": "AI for Designers", "slug": "ai/for-designers", "href": "/courses/ai/for-designers"}, {"title": "AI for Marketers", "slug": "ai/for-marketers", "href": "/courses/ai/for-marketers"}, {"title": "AI for Founders", "slug": "ai/for-founders", "href": "/courses/ai/for-founders"}]}, {"category": {"slug": "product", "href": "/courses/product", "title": "Product", "palette": {"--palette-category-base": "#510B00", "--palette-category-highlight": "#B33600", "--palette-category-border": "#F7804D"}}, "topics": [{"title": "AI for PMs", "slug": "for-product-managers/ai", "href": "/courses/for-product-managers/ai"}, {"title": "Agentic AI", "slug": "for-product-managers/agentic-ai", "href": "/courses/for-product-managers/agentic-ai"}, {"title": "AI Evals", "slug": "for-product-managers/ai-evals", "href": "/courses/for-product-managers/ai-evals"}, {"title": "Vibe Coding", "slug": "for-product-managers/vibe-coding", "href": "/courses/for-product-managers/vibe-coding"}, {"title": "Product Sense", "slug": "for-product-managers/product-sense", "href": "/courses/for-product-managers/product-sense"}, {"title": "Product Discovery", "slug": "for-product-managers/product-discovery", "href": "/courses/for-product-managers/product-discovery"}, {"title": "User Research", "slug": "for-product-managers/user-research", "href": "/courses/for-product-managers/user-research"}, {"title": "Prototyping", "slug": "for-product-managers/prototyping", "href": "/courses/for-product-managers/prototyping"}, {"title": "Growth", "slug": "for-product-managers/growth", "href": "/courses/for-product-managers/growth"}, {"title": "Analytics", "slug": "for-product-managers/analytics", "href": "/courses/for-product-managers/analytics"}, {"title": "Tech Foundations", "slug": "for-product-managers/technical-foundations", "href": "/courses/for-product-managers/technical-foundations"}, {"title": "Strategy", "slug": "for-product-managers/strategy", "href": "/courses/for-product-managers/strategy"}, {"title": "Influence", "slug": "for-product-managers/influence", "href": "/courses/for-product-managers/influence"}, {"title": "Leadership", "slug": "for-product-managers/leadership", "href": "/courses/for-product-managers/leadership"}, {"title": "Career Growth", "slug": "for-product-managers/career-growth", "href": "/courses/for-product-managers/career-growth"}]}, {"category": {"slug": "engineering", "href": "/courses/engineering", "title": "Engineering", "palette": {"--palette-category-base": "#00332F", "--palette-category-highlight": "#00746B", "--palette-category-border": "#88B7B2"}}, "topics": [{"title": "AI for Engineers", "slug": "for-engineers/ai", "href": "/courses/for-engineers/ai"}, {"title": "Agentic AI", "slug": "for-engineers/agentic-ai", "href": "/courses/for-engineers/agentic-ai"}, {"title": "Coding with AI", "slug": "for-engineers/ai-coding", "href": "/courses/for-engineers/ai-coding"}, {"title": "Claude Code", "slug": "for-engineers/claude-code", "href": "/courses/for-engineers/claude-code"}, {"title": "OpenClaw", "slug": "for-engineers/openclaw", "href": "/courses/for-engineers/openclaw"}, {"title": "MCP", "slug": "for-engineers/mcp", "href": "/courses/for-engineers/mcp"}, {"title": "RAG & Search", "slug": "for-engineers/rag-search", "href": "/courses/for-engineers/rag-search"}, {"title": "AI Evals", "slug": "for-engineers/ai-evals", "href": "/courses/for-engineers/ai-evals"}, {"title": "Machine Learning", "slug": "for-engineers/ml", "href": "/courses/for-engineers/ml"}, {"title": "LLM Ops", "slug": "for-engineers/llm-ops", "href": "/courses/for-engineers/llm-ops"}, {"title": "Context Eng", "slug": "for-engineers/context-engineering", "href": "/courses/for-engineers/context-engineering"}, {"title": "Security", "slug": "for-engineers/security", "href": "/courses/for-engineers/security"}, {"title": "System Design", "slug": "for-engineers/system-design", "href": "/courses/for-engineers/system-design"}, {"title": "Leadership", "slug": "for-engineers/leadership", "href": "/courses/for-engineers/leadership"}, {"title": "Career Growth", "slug": "for-engineers/career-growth", "href": "/courses/for-engineers/career-growth"}]}, {"category": {"slug": "design", "href": "/courses/design", "title": "Design", "palette": {"--palette-category-base": "#38085F", "--palette-category-highlight": "#772CBB", "--palette-category-border": "#B5A2D0"}}, "topics": [{"title": "AI for Designers", "slug": "for-designers/ai", "href": "/courses/for-designers/ai"}, {"title": "Agentic AI", "slug": "for-designers/agentic-ai", "href": "/courses/for-designers/agentic-ai"}, {"title": "Vibe Coding", "slug": "for-designers/vibe-coding", "href": "/courses/for-designers/vibe-coding"}, {"title": "Prototyping", "slug": "for-designers/prototyping", "href": "/courses/for-designers/prototyping"}, {"title": "Figma", "slug": "for-designers/figma", "href": "/courses/for-designers/figma"}, {"title": "Design Systems", "slug": "for-designers/design-systems", "href": "/courses/for-designers/design-systems"}, {"title": "User Research", "slug": "for-designers/user-research", "href": "/courses/for-designers/user-research"}, {"title": "Product Discovery", "slug": "for-designers/product-discovery", "href": "/courses/for-designers/product-discovery"}, {"title": "UX", "slug": "for-designers/ux", "href": "/courses/for-designers/ux"}, {"title": "UI", "slug": "for-designers/ui", "href": "/courses/for-designers/ui"}, {"title": "Visual Design", "slug": "for-designers/visual-design", "href": "/courses/for-designers/visual-design"}, {"title": "Design Strategy", "slug": "for-designers/strategy", "href": "/courses/for-designers/strategy"}, {"title": "Influence", "slug": "for-designers/influence", "href": "/courses/for-designers/influence"}, {"title": "Leadership", "slug": "for-designers/leadership", "href": "/courses/for-designers/leadership"}, {"title": "Career Growth", "slug": "for-designers/career-growth", "href": "/courses/for-designers/career-growth"}]}, {"category": {"slug": "marketing", "href": "/courses/marketing", "title": "Marketing", "palette": {"--palette-category-base": "#431532", "--palette-category-highlight": "#943170", "--palette-category-border": "#C99EB5"}}, "topics": [{"title": "AI for Marketers", "slug": "for-marketers/ai", "href": "/courses/for-marketers/ai"}, {"title": "Agentic AI", "slug": "for-marketers/agentic-ai", "href": "/courses/for-marketers/agentic-ai"}, {"title": "Vibe Coding", "slug": "for-marketers/vibe-coding", "href": "/courses/for-marketers/vibe-coding"}, {"title": "Automation", "slug": "for-marketers/marketing-automation", "href": "/courses/for-marketers/marketing-automation"}, {"title": "Content Marketing", "slug": "for-marketers/content-marketing", "href": "/courses/for-marketers/content-marketing"}, {"title": "Demand Gen", "slug": "for-marketers/demand-generation", "href": "/courses/for-marketers/demand-generation"}, {"title": "Go-to-Market", "slug": "for-marketers/gtm", "href": "/courses/for-marketers/gtm"}, {"title": "Product Marketing", "slug": "for-marketers/product-marketing", "href": "/courses/for-marketers/product-marketing"}, {"title": "Positioning", "slug": "for-marketers/positioning", "href": "/courses/for-marketers/positioning"}, {"title": "Social Media", "slug": "for-marketers/social-media", "href": "/courses/for-marketers/social-media"}, {"title": "Brand", "slug": "for-marketers/brand", "href": "/courses/for-marketers/brand"}, {"title": "B2B Marketing", "slug": "for-marketers/b2b", "href": "/courses/for-marketers/b2b"}, {"title": "SEO & AEO", "slug": "for-marketers/seo-aeo", "href": "/courses/for-marketers/seo-aeo"}, {"title": "Strategy", "slug": "for-marketers/strategy", "href": "/courses/for-marketers/strategy"}, {"title": "Leadership", "slug": "for-marketers/leadership", "href": "/courses/for-marketers/leadership"}]}, {"category": {"slug": "leadership", "href": "/courses/leadership", "title": "Leadership", "palette": {"--palette-category-base": "#50090E", "--palette-category-highlight": "#A9202A", "--palette-category-border": "#D49C98"}}, "topics": [{"title": "AI for Leaders", "slug": "for-leaders/ai", "href": "/courses/for-leaders/ai"}, {"title": "Agentic AI", "slug": "for-leaders/agentic-ai", "href": "/courses/for-leaders/agentic-ai"}, {"title": "AI Transformation", "slug": "for-leaders/ai-transformation", "href": "/courses/for-leaders/ai-transformation"}, {"title": "AI Governance", "slug": "for-leaders/ai-governance", "href": "/courses/for-leaders/ai-governance"}, {"title": "Communication", "slug": "for-leaders/communication", "href": "/courses/for-leaders/communication"}, {"title": "Influence", "slug": "for-leaders/influence", "href": "/courses/for-leaders/influence"}, {"title": "Strategy", "slug": "for-leaders/strategy", "href": "/courses/for-leaders/strategy"}, {"title": "Management", "slug": "for-leaders/management", "href": "/courses/for-leaders/management"}, {"title": "People Operations", "slug": "for-leaders/people-operations", "href": "/courses/for-leaders/people-operations"}, {"title": "Exec Presence", "slug": "for-leaders/executive-presence", "href": "/courses/for-leaders/executive-presence"}, {"title": "Storytelling", "slug": "for-leaders/storytelling", "href": "/courses/for-leaders/storytelling"}, {"title": "Goal-setting", "slug": "for-leaders/goal-setting", "href": "/courses/for-leaders/goal-setting"}, {"title": "Personal Brand", "slug": "for-leaders/personal-brand", "href": "/courses/for-leaders/personal-brand"}, {"title": "Career Growth", "slug": "for-leaders/career-growth", "href": "/courses/for-leaders/career-growth"}]}, {"category": {"slug": "founders", "href": "/courses/founders", "title": "Founders", "palette": {"--palette-category-base": "#0C3209", "--palette-category-highlight": "#077202", "--palette-category-border": "#69C662"}}, "topics": [{"title": "AI for Founders", "slug": "for-founders/ai", "href": "/courses/for-founders/ai"}, {"title": "Agentic AI", "slug": "for-founders/agentic-ai", "href": "/courses/for-founders/agentic-ai"}, {"title": "AI Workflows", "slug": "for-founders/ai-workflows", "href": "/courses/for-founders/ai-workflows"}, {"title": "Vibe Coding", "slug": "for-founders/vibe-coding", "href": "/courses/for-founders/vibe-coding"}, {"title": "Prototyping", "slug": "for-founders/prototyping", "href": "/courses/for-founders/prototyping"}, {"title": "Product Sense", "slug": "for-founders/product-sense", "href": "/courses/for-founders/product-sense"}, {"title": "Positioning", "slug": "for-founders/positioning", "href": "/courses/for-founders/positioning"}, {"title": "Product Discovery", "slug": "for-founders/product-discovery", "href": "/courses/for-founders/product-discovery"}, {"title": "Management", "slug": "for-founders/management", "href": "/courses/for-founders/management"}, {"title": "Strategy", "slug": "for-founders/strategy", "href": "/courses/for-founders/strategy"}, {"title": "Go-to-Market", "slug": "for-founders/gtm", "href": "/courses/for-founders/gtm"}, {"title": "Personal Brand", "slug": "for-founders/personal-brand", "href": "/courses/for-founders/personal-brand"}, {"title": "Leadership", "slug": "for-founders/leadership", "href": "/courses/for-founders/leadership"}, {"title": "Fundraising", "slug": "for-founders/fundraising", "href": "/courses/for-founders/fundraising"}, {"title": "PMF", "slug": "for-founders/product-market-fit", "href": "/courses/for-founders/product-market-fit"}]}, {"category": {"slug": "more", "href": "/courses/more", "title": "More", "palette": {"--palette-category-base": "#1E2C2B", "--palette-category-highlight": "#4A6361", "--palette-category-border": "#B9C1BE"}}, "topics": [{"title": "Everyone", "slug": "for-everyone", "href": "/courses/for-everyone"}, {"title": "Operators", "slug": "for-operators", "href": "/courses/for-operators"}, {"title": "Data Scientists", "slug": "for-data-scientists", "href": "/courses/for-data-scientists"}, {"title": "Business Analysts", "slug": "for-business-analysts", "href": "/courses/for-business-analysts"}, {"title": "User Researchers", "slug": "for-user-researchers", "href": "/courses/for-user-researchers"}, {"title": "Customer Success", "slug": "for-customer-success", "href": "/courses/for-customer-success"}, {"title": "Project Managers", "slug": "for-project-managers", "href": "/courses/for-project-managers"}, {"title": "HR Professionals", "slug": "for-hr-professionals", "href": "/courses/for-hr-professionals"}, {"title": "Sales People", "slug": "for-sales-people", "href": "/courses/for-sales-people"}, {"title": "Lawyers", "slug": "for-lawyers", "href": "/courses/for-lawyers"}, {"title": "Finance", "slug": "for-finance-professionals", "href": "/courses/for-finance-professionals"}, {"title": "Investors", "slug": "for-investors", "href": "/courses/for-investors"}, {"title": "Real Estate", "slug": "for-real-estate-agents", "href": "/courses/for-real-estate-agents"}, {"title": "Educators", "slug": "for-educators", "href": "/courses/for-educators"}, {"title": "Creators", "slug": "for-creators", "href": "/courses/for-creators"}]}], "navTagLinks": [{"tag": {"id": 425, "tag_type": "topic", "label": "AI Workflows", "slug": "ai-workflows", "description": null, "parent_tag_id": null}, "link_slug": "ai/ai-workflows"}, {"tag": {"id": 327, "tag_type": "topic", "label": "Analytics", "slug": "analytics", "description": null, "parent_tag_id": null}, "link_slug": "for-product-managers/analytics"}, {"tag": {"id": 315, "tag_type": "topic", "label": "Agentic AI", "slug": "agentic-ai", "description": null, "parent_tag_id": null}, "link_slug": "for-product-managers/agentic-ai"}, {"tag": {"id": 426, "tag_type": "topic", "label": "Claude Code", "slug": "claude-code", "description": null, "parent_tag_id": null}, "link_slug": "for-engineers/claude-code"}, {"tag": {"id": 418, "tag_type": "persona", "label": "For Business Analysts", "slug": "for-business-analysts", "description": null, "parent_tag_id": null}, "link_slug": "for-business-analysts"}, {"tag": {"id": 290, "tag_type": "persona", "label": "For Product Managers", "slug": "for-product-managers", "description": null, "parent_tag_id": null}, "link_slug": "for-product-managers"}, {"tag": {"id": 293, "tag_type": "persona", "label": "For Engineers", "slug": "for-engineers", "description": null, "parent_tag_id": null}, "link_slug": "for-engineers"}, {"tag": {"id": 299, "tag_type": "persona", "label": "For Data Scientists", "slug": "for-data-scientists", "description": null, "parent_tag_id": null}, "link_slug": "for-data-scientists"}], "now": "2026-05-19T22:53:24.812Z"}, "__N_SSG": true}, "page": "/p/[slug]/[...ignored]", "query": {"slug": "75ecae", "ignored": ["agentic-experiment-design-analysis"]}, "buildId": "qPlLjBFLQ9fnkFKJ7n7E-", "isFallback": false, "isExperimentalCompile": false, "gsp": true, "scriptLoader": []}}