{"url": "https://maven.com/p/30886a/understanding-embedding-performance-through-generative-evals", "core": {"title": "Understanding Embedding Performance through Generative Evals", "slug": "30886a", "id": 3646, "page_type": "workshop", "description": "Effective AI evaluation is critical as systems move from labs to production. Understanding generative benchmarking helps you build AI that performs well on real-world tasks, not just academic tests. This knowledge bridges the gap between theoretical capabilities and practical performance, giving you a competitive edge in developing AI solutions that deliver genuine value to users.", "start_datetime": "2025-05-21T17:00:00Z", "duration_min": 60, "is_recording_public": true, "has_internal_recording": true, "instructor_name": "Jason Liu", "instructor_title": "Consultant at the intersection of Information Retrieval and AI", "instructor_headline": null, "course_name": "Systematically Improving RAG Applications", "course_slug": "rag-playbook", "school_name": "Applied LLMs", "school_slug": "applied-llms", "chapter_count": 22}, "next_data": {"props": {"pageProps": {"apiSchool": {"id": 11275, "slug": "applied-llms", "name": "Applied LLMs", "is_stripe_connected": true, "stripe_connect_account_id": "acct_1PzkQhQLrr16GFS9", "zoom_auth_version": 2, "verified": true, "timezone": "America/Los_Angeles", "support_email": null, "school_tags": [{"id": 254, "tag_type": "internal_collection", "label": "Upmarket CLP", "slug": "upmarket-clp", "description": null, "parent_tag_id": null}], "instructor_bio": null}, "pageType": "workshop", "school": {"id": 11275, "slug": "applied-llms", "name": "Applied LLMs", "is_stripe_connected": true, "stripe_connect_account_id": "acct_1PzkQhQLrr16GFS9", "zoom_auth_version": 2, "verified": true, "timezone": "America/Los_Angeles", "support_email": null, "school_tags": [{"id": 254, "tag_type": "internal_collection", "label": "Upmarket CLP", "slug": "upmarket-clp", "description": null, "parent_tag_id": null}], "instructor_bio": null}, "course": {"slug": "rag-playbook", "school_id": 11275, "name": "Systematically Improving RAG Applications", "id": 12327, "format": "full_course", "description": "Follow a repeatable process to continually evaluate and improve your RAG application", "attrs": {"courses_page_highlight_order_index": null, "custom_landing_page_url": null, "facebook_pixel_id": null, "is_course_application_enabled": false, "is_demoted_on_courses_page": false, "is_hidden_by_maven_staff": false, "is_live_on_courses_page": true, "short_description": null, "skip_landing_page": null, "satisfaction_guarantee_status": "unset"}, "social_image_url": "https://d2426xcxuh3ht5.cloudfront.net/9GwwSK8sSwCVnlmSGcVs_Screenshot 2024-10-24 at 2.07.03 PM.jpeg", "course_image_url": null, "clp_hero_media": {"media_type": "image_url", "image_url": "https://d2426xcxuh3ht5.cloudfront.net/mGmpyJU8S3y58hbPhsAF_Sample YouTube Thumbnail (4) (1).png", "youtube_video_id": "dQw4w9WgXcQ", "vimeo_video_id": null}, "instructor_infos": [{"name": "Jason Liu", "image_url": "https://d2426xcxuh3ht5.cloudfront.net/rfSyvi24S8aT9hsfK1IY_jason liu.png", "image_url_no_bg": "https://user-assets.maven.com/instructor-no-bg/rfSyvi24S8aT9hsfK1IY_jason liu.png", "uuid": "32147213-3988-4414-8737-96df0ab8737a", "bio_html": "<p>Jason has built search and recommendation systems for the past 8 years. He has consulted and advised a dozens startups in the last year to improve their RAG systems. He is the creator of the Instructor Python library.</p>", "headline": "", "title": "Staff machine learning engineer, currently working as an AI consultant", "linkedin_url": "https://www.linkedin.com/in/jxnlco/", "twitter_handle": null, "custom_social_link": "567stud.io", "logos": {"logos": [{"image_url": "https://asset.brandfetch.io/idR3duQxYl/idS9nxrHVA.svg", "type": "logo", "file_type": "svg", "uuid": "67f09b6b-731d-4237-ac4b-e5a18fc22d15", "name": "OpenAI"}, {"image_url": "https://asset.brandfetch.io/idmJWF3N06/idS2edu_4T.svg", "type": "logo", "file_type": "svg", "uuid": "b9c32ec4-b299-4d7a-a9e0-5cd497195964", "name": "Anthropic"}, {"image_url": "https://asset.brandfetch.io/idchmboHEZ/idc7I0n_oy.svg", "type": "logo", "file_type": "svg", "uuid": "5340be5f-ee96-4da1-9eae-06cafd8918c7", "name": "Microsoft"}, {"image_url": "https://asset.brandfetch.io/id6O2oGzv-/idjOb2YcqW.svg", "type": "logo", "file_type": "svg", "uuid": "85797ac0-3112-4f0f-a70f-dc3e1d32d999", "name": "Google"}, {"image_url": "https://asset.brandfetch.io/idWvz5T3V7/idpvCl_Ipb.svg", "type": "logo", "file_type": "svg", "uuid": "196198b0-3164-42b4-ab07-598a5b51cb74", "name": "Meta"}], "label": "Students from "}, "highlights": [], "highlights_html": null, "preferred_name": null}], "instructor_caption": "Staff machine learning engineer, currently working as an AI consultant", "course_tags": [{"id": 31, "tag_type": "category", "label": "Data & engineering", "slug": "engineering-ll", "description": null, "parent_tag_id": null}, {"id": 35, "tag_type": "category", "label": "AI", "slug": "ai-ll", "description": null, "parent_tag_id": null}, {"id": 69, "tag_type": "internal_collection", "label": "Maven Selects", "slug": "maven-selects", "description": null, "parent_tag_id": null}, {"id": 71, "tag_type": "internal_collection", "label": "VIP", "slug": "vip", "description": null, "parent_tag_id": null}, {"id": 80, "tag_type": "deprecated", "label": "AI/ML engineering", "slug": "x-subcategory-ai-ml-engineering", "description": null, "parent_tag_id": 35}, {"id": 81, "tag_type": "deprecated", "label": "Data science and engineering", "slug": "x-subcategory-data-science-and-engineering", "description": null, "parent_tag_id": 35}, {"id": 109, "tag_type": "deprecated", "label": "Data science and engineering", "slug": "x-subcategory-data-science-and-engineering-1", "description": null, "parent_tag_id": 31}, {"id": 172, "tag_type": "internal_collection", "label": "AI Affiliates", "slug": "ai-affiliates-nov-24", "description": null, "parent_tag_id": null}, {"id": 176, "tag_type": "public_collection", "label": "Fast Track 2024", "slug": "fast-track-2024", "description": null, "parent_tag_id": null}, {"id": 209, "tag_type": "internal_collection", "label": "Affiliate Hamel Husain", "slug": "affiliate-hamel-husain", "description": null, "parent_tag_id": null}, {"id": 210, "tag_type": "internal_collection", "label": "Affiliate Jason Liu", "slug": "affiliate-jason-liu", "description": null, "parent_tag_id": null}, {"id": 216, "tag_type": "internal_collection", "label": "[LEGACY] Instructor Owned CLC Opt-in", "slug": "legacy-instructor-owned-clc", "description": null, "parent_tag_id": null}, {"id": 226, "tag_type": "internal_collection", "label": "Building AI-Native Products", "slug": "building-ai-native-products", "description": null, "parent_tag_id": null}, {"id": 234, "tag_type": "public_collection", "label": "Maven 100 - AI 2025", "slug": "maven-100-ai-2025", "description": null, "parent_tag_id": null}, {"id": 235, "tag_type": "public_collection", "label": "Maven 100 - Engineering 2025", "slug": "maven-100-engineering-2025", "description": null, "parent_tag_id": null}, {"id": 243, "tag_type": "public_collection", "label": "Lenny's List AI", "slug": "lennys-list-ai", "description": null, "parent_tag_id": null}, {"id": 244, "tag_type": "internal_collection", "label": "Courses filtered for recommendations", "slug": "courses-filtered-for-recommendations", "description": null, "parent_tag_id": null}, {"id": 245, "tag_type": "internal_collection", "label": "maven-100-2025", "slug": "maven-100-2025", "description": null, "parent_tag_id": null}, {"id": 271, "tag_type": "internal_collection", "label": "AI Evals", "slug": "ai-evals", "description": null, "parent_tag_id": null}, {"id": 275, "tag_type": "internal_collection", "label": "Maven Ads", "slug": "maven-ads", "description": null, "parent_tag_id": null}, {"id": 293, "tag_type": "persona", "label": "For Engineers", "slug": "for-engineers", "description": null, "parent_tag_id": null}, {"id": 299, "tag_type": "persona", "label": "For Data Scientists", "slug": "for-data-scientists", "description": null, "parent_tag_id": null}, {"id": 313, "tag_type": "topic", "label": "AI", "slug": "ai", "description": null, "parent_tag_id": null}, {"id": 320, "tag_type": "topic", "label": "RAG", "slug": "rag", "description": null, "parent_tag_id": null}, {"id": 441, "tag_type": "topic", "label": "RAG & Search", "slug": "rag-search", "description": null, "parent_tag_id": null}]}, "contentPageSlug": "30886a", "contentPage": {"id": 3646, "school_id": 11275, "slug": "30886a", "content_page_type": "workshop", "sections": [{"uuid": "0c5f8f2a-e6e5-4a95-a3b1-2115ca82d81a", "title": "Understanding Embedding Performance through Generative Evals", "version": 1, "image_url": "https://d2426xcxuh3ht5.cloudfront.net/UflmntqWS5iExTu3mPFI_Single Instructor (4).png", "is_visible": true, "topic_desc": "Effective AI evaluation is critical as systems move from labs to production. Understanding generative benchmarking helps you build AI that performs well on real-world tasks, not just academic tests. This knowledge bridges the gap between theoretical capabilities and practical performance, giving you a competitive edge in developing AI solutions that deliver genuine value to users.", "brand_logos": {"label": "Previously at", "logos": [{"name": "Stitch Fix", "type": "logo", "uuid": "a50d7104-f715-4b54-a0ce-03ba0b992971", "file_type": "svg", "image_url": "https://asset.brandfetch.io/idTgrRl8nA/idVnuVYfek.svg"}, {"name": "Meta", "type": "logo", "uuid": "01bd1baa-efac-4821-bd33-0a0dcc914c5e", "file_type": "svg", "image_url": "https://asset.brandfetch.io/idWvz5T3V7/idpvCl_Ipb.svg"}, {"name": "University of Waterloo", "type": "logo", "uuid": "591dbe57-cf71-46c4-9988-c59025744c3e", "file_type": "png", "image_url": "https://asset.brandfetch.io/idLK11nGS0/idd60XBB4R.png"}, {"name": "New York University", "type": "logo", "uuid": "6cc21f6b-b2e5-4b6c-9113-505aa8d8ca84", "file_type": "svg", "image_url": "https://asset.brandfetch.io/idMj8iq0ev/idShvt3ozm.svg"}]}, "section_type": "main", "instructor_infos": [{"name": "Jason Liu", "uuid": "989c0f9f-0b51-4654-9ff5-1a5fc4cc5185", "title": "Consultant at the intersection of Information Retrieval and AI", "bio_html": "<p>Jason has built search and recommendation systems for the past 6 years. He has consulted and advised a dozens startups in the last year to improve their RAG systems. He is the creator of the Instructor Python library.</p>", "image_url": "https://d2426xcxuh3ht5.cloudfront.net/QXS56kXBTraYPeAqeSnn_Jason.png"}, {"name": "Kelly Hong", "uuid": "a36c2da9-06cc-4652-a0d8-6fb770330b8e", "title": "Researcher at Chroma", "bio_html": "<p><br></p>", "image_url": "https://d2426xcxuh3ht5.cloudfront.net/2rGGIRswSma1yISbApOJ_headshot.png"}], "learning_outcomes": [{"uuid": "982b77e7-5d43-4443-9573-8200c31f699b", "title": "AI Evaluation Challenges", "description": "Discover why AI systems need specialized benchmarking beyond traditional testing methods."}, {"uuid": "5ff1a04a-cb36-4e86-bd55-c18378199cdd", "title": "Benchmark Limitations", "description": "Identify the shortcomings of public benchmarks, including clean datasets and potential training data contamination."}, {"uuid": "0d8a444b-21df-493f-b934-b8b6d0a91a58", "title": "Representativeness in Testing", "description": "Apply techniques to generate benchmark tests that accurately reflect real-world user queries and production conditions."}]}], "created_at": "2026-02-12T20:59:33.961229Z", "updated_at": null, "theme": null, "attrs": {"type": "workshop", "is_canceled": false, "is_delisted": false, "connected_course_id": 12327, "course_stripe_promo_code_id": null}, "school_event": {"id": 3646, "slug": null, "title": "Understanding Embedding Performance through Generative Evals", "description": null, "start_datetime": "2025-05-21T17:00:00Z", "end_datetime": "2025-05-21T18:00:00Z", "start_date": "2025-05-21", "start_time": "13:00:00", "duration_min": 60, "timezone": "America/New_York", "has_internal_recording": true, "is_recording_public": true}, "num_signups": 1181}, "relatedWorkshops": [{"id": 6528, "school_id": 13230, "is_visible_on_discovery_page": true, "is_canceled": false, "is_delisted": false, "connected_course_id": 18719, "workshop_tags": [{"id": 31, "tag_type": "category", "label": "Data & engineering", "slug": "engineering-ll", "description": null, "parent_tag_id": null}, {"id": 35, "tag_type": "category", "label": "AI", "slug": "ai-ll", "description": null, "parent_tag_id": null}, {"id": 271, "tag_type": "internal_collection", "label": "AI Evals", "slug": "ai-evals", "description": null, "parent_tag_id": null}, {"id": 290, "tag_type": "persona", "label": "For Product Managers", "slug": "for-product-managers", "description": null, "parent_tag_id": null}, {"id": 293, "tag_type": "persona", "label": "For Engineers", "slug": "for-engineers", "description": null, "parent_tag_id": null}, {"id": 313, "tag_type": "topic", "label": "AI", "slug": "ai", "description": null, "parent_tag_id": null}], "published_content_page": {"id": 477, "school_id": 13230, "slug": "11d5ee", "content_page_type": "workshop", "sections": [{"uuid": "77863aad-0d97-46ee-a69a-b8d6ea9ae415", "title": "Design Evals Users Will Trust", "version": 1, "image_url": "https://d2426xcxuh3ht5.cloudfront.net/t9WcTIRQYqwwg7N43jvK_Screenshot%202026-02-04%20at%206.23.59%E2%80%AFPM.png", "is_visible": true, "topic_desc": "Most AI teams ship products that pass benchmarks but fail users. The gap between \"model works\" and \"product works\" is where careers and products stall. This session gives you the systematic approach used by production AI teams at companies like OpenAI and Google to evaluate what actually matters before your users find the problems first.", "brand_logos": {"label": "Previously at", "logos": []}, "section_type": "main", "instructor_infos": [{"name": "Aishwarya Naresh Reganti", "uuid": "4de9f2bd-989e-4ff7-bc08-6a3df9a52e6b", "logos": {"label": "", "logos": []}, "title": "AI Founder & Advisor to F500 Leaders", "bio_html": "<p><a href=\"https://www.linkedin.com/in/areganti/\" rel=\"noopener noreferrer\" target=\"_blank\">Aishwarya Naresh Reganti</a>&nbsp;is an Applied Science Tech Lead and leads initiatives to develop and deploy production-ready generative AI solutions enterprise clients. With over 9 years of experience in machine learning, she has published more than&nbsp;<a href=\"https://scholar.google.com/citations?user=gvgg4ksAAAAJ&amp;hl=en\" rel=\"noopener noreferrer\" target=\"_blank\">35 research papers</a>&nbsp;at top-tier AI conferences, including NeurIPS, AAAI, and CVPR.</p><p>Aishwarya has taught professional courses on generative AI at renowned institutions like MIT and Oxford. She has also designed&nbsp;<a href=\"https://github.com/aishwaryanr/awesome-generative-ai-guide/tree/main/free_courses/Applied_LLMs_Mastery_2024\" rel=\"noopener noreferrer\" target=\"_blank\">free courses</a>&nbsp;that have reached over 8,000 students globally and have formed the foundation for several academic programs and industry training curricula.</p><p>Recognized as one of the most prominent voices in enterprise AI, with over 95,000 professionals following&nbsp;her on&nbsp;<a href=\"https://www.linkedin.com/in/areganti/\" rel=\"noopener noreferrer\" target=\"_blank\">LinkedIn</a>, she is a sought-after thought leader frequently invited to speak at leading conferences and events, including TEDx, MLOps World, and ReWork.</p><p>Aishwarya actively collaborates with leading research professors and provides strategic advisory to organizations, enabling them to harness AI effectively to address complex business challenges.</p>", "headline": "", "image_url": "https://d2426xcxuh3ht5.cloudfront.net/CbyVSNxxTsC9bcqM1Ki0_ais.jpeg", "highlights": [], "linkedin_url": "", "twitter_handle": "", "highlights_html": null, "image_url_no_bg": null, "custom_social_link": ""}], "learning_outcomes": [{"uuid": "1312a4bd-b67e-4ede-86a7-a0becf4f51bd", "title": "Distinguish model evals from product evals", "description": "Learn why benchmarks that look impressive on paper often fail in production, and what to measure instead"}, {"uuid": "76077ba0-4141-458e-ac14-d742f9da04b7", "title": "Design your evaluation framework", "description": "Apply a three-component structure (reference datasets, metrics, and scoring methods) that scales with your AI product"}, {"uuid": "1c277f87-91a1-488d-99ca-b78ad2032b4f", "title": "Create reference datasets before launch", "description": "Build evaluation datasets that catch real failure modes, not just the obvious ones"}]}], "created_at": "2026-02-04T23:24:23.256734Z", "updated_at": "2026-02-08T20:46:59.449631Z", "theme": null, "attrs": {"type": "workshop", "is_canceled": false, "is_delisted": false, "connected_course_id": 18719, "course_stripe_promo_code_id": null}, "school_event": {"id": 477, "slug": null, "title": "Design Evals Users Will Trust", "description": null, "start_datetime": "2026-02-18T17:00:00Z", "end_datetime": "2026-02-18T17:45:00Z", "start_date": "2026-02-18", "start_time": "12:00:00", "duration_min": 45, "timezone": "America/New_York", "has_internal_recording": true, "is_recording_public": true}, "num_signups": 768}}, {"id": 4996, "school_id": 15665, "is_visible_on_discovery_page": true, "is_canceled": false, "is_delisted": false, "connected_course_id": 17883, "workshop_tags": [{"id": 2, "tag_type": "category", "label": "Product", "slug": "product-ll", "description": null, "parent_tag_id": null}, {"id": 35, "tag_type": "category", "label": "AI", "slug": "ai-ll", "description": null, "parent_tag_id": null}, {"id": 271, "tag_type": "internal_collection", "label": "AI Evals", "slug": "ai-evals", "description": null, "parent_tag_id": null}, {"id": 290, "tag_type": "persona", "label": "For Product Managers", "slug": "for-product-managers", "description": null, "parent_tag_id": null}, {"id": 293, "tag_type": "persona", "label": "For Engineers", "slug": "for-engineers", "description": null, "parent_tag_id": null}, {"id": 313, "tag_type": "topic", "label": "AI", "slug": "ai", "description": null, "parent_tag_id": null}], "published_content_page": {"id": 5843, "school_id": 15665, "slug": "602b4f", "content_page_type": "workshop", "sections": [{"uuid": "24d0a2a2-5b91-41ac-a0f3-e26372da0046", "title": "How to Drive AI Evals Adoption", "version": 1, "image_url": "https://d2426xcxuh3ht5.cloudfront.net/txhogjeSTKmPgKkxe9FZ_Single%20Instructor.png", "is_visible": true, "topic_desc": "AI Evals & Analytics is excited to host Dr. Sebastian Fox, who has helped companies build reliable, scalable eval systems. With deep expertise in designing evals for LLM features and agents, he brings lessons learned from real-world deployments. In this lightning lesson, he’ll share the hurdles and patterns he’s seen and how to drive AI evals adoption across organizations.", "brand_logos": {"label": "Previously AI at McKinsey & QuantumBlack", "logos": [{"name": "McKinsey & Company", "type": "logo", "uuid": "a2044c40-8cf0-4c20-8e78-08963418833a", "file_type": "svg", "image_url": "https://asset.brandfetch.io/iddTc-lesG/idZ0K6UwLs.svg"}, {"name": "University of Oxford", "type": "logo", "uuid": "4f1a768f-a2b8-4247-b49f-6b81e83af781", "file_type": "png", "image_url": "https://asset.brandfetch.io/idRK9czrcr/idpEPRxdxK.png"}]}, "section_type": "main", "instructor_infos": [{"name": "Dr Sebastian Fox", "uuid": "85599000-aac2-4c23-9732-f549dfc1455f", "logos": {"label": "", "logos": []}, "title": "CEO & Co-founder at Composo.ai", "bio_html": "<p>Building the best evals for LLM features and agents at Composo. Medical doctor. Formerly AI at McKinsey &amp; QuantumBlack.</p>", "headline": "", "image_url": "https://d2426xcxuh3ht5.cloudfront.net/jXYQmhqKSAuWVntByqxv_Screenshot%202025-11-03%20at%2023.15.51.png", "highlights": [], "linkedin_url": "", "twitter_handle": "", "image_url_no_bg": null, "custom_social_link": ""}], "learning_outcomes": [{"uuid": "6bad2e3a-a565-4f33-b607-1d8bbb473815", "title": "Inside Look at Real-World Evals Challenges", "description": "AI eval adoption hurdles: from cultural pushback to tough technical trade-offs."}, {"uuid": "9b23a52e-d2b1-4ea4-bddc-e75e18da4d33", "title": "Lessons from Building Evals for LLM Features & Agents", "description": "Evals beyond benchmarks: capture user value, ensure consistency, and balance automation with human judgement."}, {"uuid": "77525986-8722-4e12-86ed-28d32ec70639", "title": "The Future Beyond “LLM-as-a-Judge”", "description": "Explores limits of LLM-as-judge methods and scalable AI evaluation alternatives"}]}], "created_at": "2026-02-12T21:05:19.681225Z", "updated_at": null, "theme": null, "attrs": {"type": "workshop", "is_canceled": false, "is_delisted": false, "connected_course_id": 17883, "course_stripe_promo_code_id": null}, "school_event": {"id": 5843, "slug": null, "title": "How to Drive AI Evals Adoption", "description": null, "start_datetime": "2025-11-20T17:00:00Z", "end_datetime": "2025-11-20T17:30:00Z", "start_date": "2025-11-21", "start_time": "01:00:00", "duration_min": 30, "timezone": "Asia/Taipei", "has_internal_recording": true, "is_recording_public": true}, "num_signups": 322}}, {"id": 1914, "school_id": 11275, "is_visible_on_discovery_page": true, "is_canceled": false, "is_delisted": false, "connected_course_id": 12327, "workshop_tags": [{"id": 35, "tag_type": "category", "label": "AI", "slug": "ai-ll", "description": null, "parent_tag_id": null}, {"id": 137, "tag_type": "internal_collection", "label": "Instructor", "slug": "Instructor", "description": null, "parent_tag_id": null}, {"id": 293, "tag_type": "persona", "label": "For Engineers", "slug": "for-engineers", "description": null, "parent_tag_id": null}, {"id": 299, "tag_type": "persona", "label": "For Data Scientists", "slug": "for-data-scientists", "description": null, "parent_tag_id": null}, {"id": 313, "tag_type": "topic", "label": "AI", "slug": "ai", "description": null, "parent_tag_id": null}, {"id": 332, "tag_type": "topic", "label": "Machine Learning", "slug": "ml", "description": null, "parent_tag_id": null}], "published_content_page": {"id": 2763, "school_id": 11275, "slug": "1ea9c9", "content_page_type": "workshop", "sections": [{"uuid": "251b4b9c-b7f3-4481-9f00-6d6c08caada6", "title": "Glean's Embedding Model for Enterprise-Adapted AI", "version": 1, "image_url": "https://d2426xcxuh3ht5.cloudfront.net/AYWrEgtTTOm8DgBUdXz2_Single Instructor (1).png", "is_visible": true, "topic_desc": "Custom embedding models are the key to making AI work effectively in business settings. By learning to adapt these models to specific company contexts, you'll be able to build AI systems that truly understand organizational language and deliver real value—a critical skill as enterprise AI adoption grows.", "brand_logos": {"label": "Representing", "logos": [{"name": "Glean", "type": "logo", "uuid": "62d4e89b-c03b-43d5-a0c5-3e91dff642fd", "file_type": "svg", "image_url": "https://asset.brandfetch.io/idJB_yEUFy/idJlIJp-ye.svg"}]}, "section_type": "main", "instructor_infos": [{"name": "Jason Liu", "uuid": "7dd252c4-a54c-460e-b9fd-a1a728b0515b", "title": "Consultant at the intersection of Information Retrieval and AI", "bio_html": "<p>Jason has built search and recommendation systems for the past 6 years. He has consulted and advised a dozens startups in the last year to improve their RAG systems. He is the creator of the Instructor Python library.</p>", "image_url": "https://d2426xcxuh3ht5.cloudfront.net/3ZLdqk4KRcaEy7pdUd3M_Jason.png"}, {"name": "Manav Rathod", "uuid": "b91d0d3a-212f-49c7-a960-11a0f2660445", "title": "Software Engineer at Glean", "bio_html": "<p>Manav is representing Glean and works on the Search and Intelligence Team.</p>", "image_url": "https://d2426xcxuh3ht5.cloudfront.net/awLseMDxTyuL3pwWkWvu_Manav.jpeg"}], "learning_outcomes": [{"uuid": "0d1e57c1-4a73-45f0-ae1c-c546a653b9d3", "title": "Enterprise Embedding Model Creation", "description": "Build and fine-tune custom embedding models using MLM, contrastive learning, and user feedback data."}, {"uuid": "918cd129-82c7-44d7-93bd-bb1d6b44b349", "title": "Data Adaptation Strategies", "description": "Apply synthetic and real company data to overcome cold-start problems and adapt models to specific contexts."}, {"uuid": "81292bcc-7d3e-4033-a145-cc55ad9bc084", "title": "RAG Architecture Integration", "description": "Implement embedding models within RAG systems to create company-specific AI search and generation tools."}]}], "created_at": "2026-02-12T20:57:30.282846Z", "updated_at": null, "theme": null, "attrs": {"type": "workshop", "is_canceled": false, "is_delisted": false, "connected_course_id": 12327, "course_stripe_promo_code_id": null}, "school_event": {"id": 2763, "slug": null, "title": "Glean's Embedding Model for Enterprise-Adapted AI", "description": null, "start_datetime": "2025-03-05T18:00:00Z", "end_datetime": "2025-03-05T19:00:00Z", "start_date": "2025-03-05", "start_time": "13:00:00", "duration_min": 60, "timezone": "America/New_York", "has_internal_recording": true, "is_recording_public": true}, "num_signups": 543}}], "exploreCourses": null, "workshopTags": [{"id": 137, "tag_type": "internal_collection", "label": "Instructor", "slug": "Instructor", "description": null, "parent_tag_id": null}, {"id": 226, "tag_type": "internal_collection", "label": "Building AI-Native Products", "slug": "building-ai-native-products", "description": null, "parent_tag_id": null}, {"id": 31, "tag_type": "category", "label": "Data & engineering", "slug": "engineering-ll", "description": null, "parent_tag_id": null}, {"id": 35, "tag_type": "category", "label": "AI", "slug": "ai-ll", "description": null, "parent_tag_id": null}, {"id": 239, "tag_type": "internal_collection", "label": "Maven 100 LL 2025", "slug": "maven-100-ll-2025", "description": null, "parent_tag_id": null}, {"id": 213, "tag_type": "internal_collection", "label": "The AI-Powered Super IC", "slug": "ai-powered-super-ic", "description": null, "parent_tag_id": null}, {"id": 266, "tag_type": "internal_collection", "label": "The AI Powered Super Engineer", "slug": "ai-powered-super-engineer", "description": null, "parent_tag_id": null}, {"id": 271, "tag_type": "internal_collection", "label": "AI Evals", "slug": "ai-evals", "description": null, "parent_tag_id": null}, {"id": 293, "tag_type": "persona", "label": "For Engineers", "slug": "for-engineers", "description": null, "parent_tag_id": null}, {"id": 299, "tag_type": "persona", "label": "For Data Scientists", "slug": "for-data-scientists", "description": null, "parent_tag_id": null}, {"id": 313, "tag_type": "topic", "label": "AI", "slug": "ai", "description": null, "parent_tag_id": null}], "videoChapters": [{"title": "Introduction to Generative Benchmarking", "start_seconds": 0}, {"title": "The Importance and Structure of Retrieval Evals", "start_seconds": 22}, {"title": "Understanding the MTEB Benchmark", "start_seconds": 58}, {"title": "Limitations of the MTEB Benchmark", "start_seconds": 136}, {"title": "How Generative Benchmarking Solves MTEB's Limitations", "start_seconds": 203}, {"title": "Case Study: Weights & Biases Support Bot", "start_seconds": 294}, {"title": "Step 1: Chunk Filtering with an Aligned LLM Judge", "start_seconds": 376}, {"title": "Step 2: Generating Realistic Queries", "start_seconds": 509}, {"title": "Running the Retrieval Task and Analyzing Results", "start_seconds": 582}, {"title": "Contradictions with MTEB Model Rankings", "start_seconds": 688}, {"title": "Key Insights and Practical Applications", "start_seconds": 792}, {"title": "Final Takeaways and Resources", "start_seconds": 937}, {"title": "Q&A: Upcoming Research on Long Context vs. Retrieval", "start_seconds": 1026}, {"title": "Q&A: Selecting Few-Shot Examples for Query Generation", "start_seconds": 1165}, {"title": "Q&A: KPIs and Metrics for Retrieval Evaluation", "start_seconds": 1366}, {"title": "Q&A: Common Pitfalls in Building Evals", "start_seconds": 1471}, {"title": "Q&A: Experience with Contextual Chunk Rewriting", "start_seconds": 1655}, {"title": "Q&A: Processing Metadata in the Pipeline", "start_seconds": 1810}, {"title": "Q&A: A Cheaper Approach to Contextualized Chunking", "start_seconds": 1940}, {"title": "Q&A: Evaluating Hybrid Search Models", "start_seconds": 2021}, {"title": "Q&A: Hyperparameter Tuning and Iteration Process", "start_seconds": 2186}, {"title": "Q&A: Evals as Unit Tests and DSPy Integration", "start_seconds": 2400}], "navPages": [{"category": {"slug": "ai", "href": "/courses/ai", "title": "AI", "palette": {"--palette-category-base": "#460952", "--palette-category-highlight": "#931FAA", "--palette-category-border": "#EDA6FB"}}, "topics": [{"title": "Agentic AI", "slug": "ai/agentic-ai", "href": "/courses/ai/agentic-ai"}, {"title": "Coding with AI", "slug": "ai/ai-coding", "href": "/courses/ai/ai-coding"}, {"title": "AI Workflows", "slug": "ai/ai-workflows", "href": "/courses/ai/ai-workflows"}, {"title": "Claude Code", "slug": "ai/claude-code", "href": "/courses/ai/claude-code"}, {"title": "OpenClaw", "slug": "ai/openclaw", "href": "/courses/ai/openclaw"}, {"title": "Vibe Coding", "slug": "ai/vibe-coding", "href": "/courses/ai/vibe-coding"}, {"title": "AI Evals", "slug": "ai/evals", "href": "/courses/ai/evals"}, {"title": "AI Transformation", "slug": "ai/ai-transformation", "href": "/courses/ai/ai-transformation"}, {"title": "RAG & Search", "slug": "ai/rag-search", "href": "/courses/ai/rag-search"}, {"title": "MCP", "slug": "ai/mcp", "href": "/courses/ai/mcp"}, {"title": "AI for PMs", "slug": "ai/for-product-managers", "href": "/courses/ai/for-product-managers"}, {"title": "AI for Engineers", "slug": "ai/for-engineers", "href": "/courses/ai/for-engineers"}, {"title": "AI for Designers", "slug": "ai/for-designers", "href": "/courses/ai/for-designers"}, {"title": "AI for Marketers", "slug": "ai/for-marketers", "href": "/courses/ai/for-marketers"}, {"title": "AI for Founders", "slug": "ai/for-founders", "href": "/courses/ai/for-founders"}]}, {"category": {"slug": "product", "href": "/courses/product", "title": "Product", "palette": {"--palette-category-base": "#510B00", "--palette-category-highlight": "#B33600", "--palette-category-border": "#F7804D"}}, "topics": [{"title": "AI for PMs", "slug": "for-product-managers/ai", "href": "/courses/for-product-managers/ai"}, {"title": "Agentic AI", "slug": "for-product-managers/agentic-ai", "href": "/courses/for-product-managers/agentic-ai"}, {"title": "AI Evals", "slug": "for-product-managers/ai-evals", "href": "/courses/for-product-managers/ai-evals"}, {"title": "Vibe Coding", "slug": "for-product-managers/vibe-coding", "href": "/courses/for-product-managers/vibe-coding"}, {"title": "Product Sense", "slug": "for-product-managers/product-sense", "href": "/courses/for-product-managers/product-sense"}, {"title": "Product Discovery", "slug": "for-product-managers/product-discovery", "href": "/courses/for-product-managers/product-discovery"}, {"title": "User Research", "slug": "for-product-managers/user-research", "href": "/courses/for-product-managers/user-research"}, {"title": "Prototyping", "slug": "for-product-managers/prototyping", "href": "/courses/for-product-managers/prototyping"}, {"title": "Growth", "slug": "for-product-managers/growth", "href": "/courses/for-product-managers/growth"}, {"title": "Analytics", "slug": "for-product-managers/analytics", "href": "/courses/for-product-managers/analytics"}, {"title": "Tech Foundations", "slug": "for-product-managers/technical-foundations", "href": "/courses/for-product-managers/technical-foundations"}, {"title": "Strategy", "slug": "for-product-managers/strategy", "href": "/courses/for-product-managers/strategy"}, {"title": "Influence", "slug": "for-product-managers/influence", "href": "/courses/for-product-managers/influence"}, {"title": "Leadership", "slug": "for-product-managers/leadership", "href": "/courses/for-product-managers/leadership"}, {"title": "Career Growth", "slug": "for-product-managers/career-growth", "href": "/courses/for-product-managers/career-growth"}]}, {"category": {"slug": "engineering", "href": "/courses/engineering", "title": "Engineering", "palette": {"--palette-category-base": "#00332F", "--palette-category-highlight": "#00746B", "--palette-category-border": "#88B7B2"}}, "topics": [{"title": "AI for Engineers", "slug": "for-engineers/ai", "href": "/courses/for-engineers/ai"}, {"title": "Agentic AI", "slug": "for-engineers/agentic-ai", "href": "/courses/for-engineers/agentic-ai"}, {"title": "Coding with AI", "slug": "for-engineers/ai-coding", "href": "/courses/for-engineers/ai-coding"}, {"title": "Claude Code", "slug": "for-engineers/claude-code", "href": "/courses/for-engineers/claude-code"}, {"title": "OpenClaw", "slug": "for-engineers/openclaw", "href": "/courses/for-engineers/openclaw"}, {"title": "MCP", "slug": "for-engineers/mcp", "href": "/courses/for-engineers/mcp"}, {"title": "RAG & Search", "slug": "for-engineers/rag-search", "href": "/courses/for-engineers/rag-search"}, {"title": "AI Evals", "slug": "for-engineers/ai-evals", "href": "/courses/for-engineers/ai-evals"}, {"title": "Machine Learning", "slug": "for-engineers/ml", "href": "/courses/for-engineers/ml"}, {"title": "LLM Ops", "slug": "for-engineers/llm-ops", "href": "/courses/for-engineers/llm-ops"}, {"title": "Context Eng", "slug": "for-engineers/context-engineering", "href": "/courses/for-engineers/context-engineering"}, {"title": "Security", "slug": "for-engineers/security", "href": "/courses/for-engineers/security"}, {"title": "System Design", "slug": "for-engineers/system-design", "href": "/courses/for-engineers/system-design"}, {"title": "Leadership", "slug": "for-engineers/leadership", "href": "/courses/for-engineers/leadership"}, {"title": "Career Growth", "slug": "for-engineers/career-growth", "href": "/courses/for-engineers/career-growth"}]}, {"category": {"slug": "design", "href": "/courses/design", "title": "Design", "palette": {"--palette-category-base": "#38085F", "--palette-category-highlight": "#772CBB", "--palette-category-border": "#B5A2D0"}}, "topics": [{"title": "AI for Designers", "slug": "for-designers/ai", "href": "/courses/for-designers/ai"}, {"title": "Agentic AI", "slug": "for-designers/agentic-ai", "href": "/courses/for-designers/agentic-ai"}, {"title": "Vibe Coding", "slug": "for-designers/vibe-coding", "href": "/courses/for-designers/vibe-coding"}, {"title": "Prototyping", "slug": "for-designers/prototyping", "href": "/courses/for-designers/prototyping"}, {"title": "Figma", "slug": "for-designers/figma", "href": "/courses/for-designers/figma"}, {"title": "Design Systems", "slug": "for-designers/design-systems", "href": "/courses/for-designers/design-systems"}, {"title": "User Research", "slug": "for-designers/user-research", "href": "/courses/for-designers/user-research"}, {"title": "Product Discovery", "slug": "for-designers/product-discovery", "href": "/courses/for-designers/product-discovery"}, {"title": "UX", "slug": "for-designers/ux", "href": "/courses/for-designers/ux"}, {"title": "UI", "slug": "for-designers/ui", "href": "/courses/for-designers/ui"}, {"title": "Visual Design", "slug": "for-designers/visual-design", "href": "/courses/for-designers/visual-design"}, {"title": "Design Strategy", "slug": "for-designers/strategy", "href": "/courses/for-designers/strategy"}, {"title": "Influence", "slug": "for-designers/influence", "href": "/courses/for-designers/influence"}, {"title": "Leadership", "slug": "for-designers/leadership", "href": "/courses/for-designers/leadership"}, {"title": "Career Growth", "slug": "for-designers/career-growth", "href": "/courses/for-designers/career-growth"}]}, {"category": {"slug": "marketing", "href": "/courses/marketing", "title": "Marketing", "palette": {"--palette-category-base": "#431532", "--palette-category-highlight": "#943170", "--palette-category-border": "#C99EB5"}}, "topics": [{"title": "AI for Marketers", "slug": "for-marketers/ai", "href": "/courses/for-marketers/ai"}, {"title": "Agentic AI", "slug": "for-marketers/agentic-ai", "href": "/courses/for-marketers/agentic-ai"}, {"title": "Vibe Coding", "slug": "for-marketers/vibe-coding", "href": "/courses/for-marketers/vibe-coding"}, {"title": "Automation", "slug": "for-marketers/marketing-automation", "href": "/courses/for-marketers/marketing-automation"}, {"title": "Content Marketing", "slug": "for-marketers/content-marketing", "href": "/courses/for-marketers/content-marketing"}, {"title": "Demand Gen", "slug": "for-marketers/demand-generation", "href": "/courses/for-marketers/demand-generation"}, {"title": "Go-to-Market", "slug": "for-marketers/gtm", "href": "/courses/for-marketers/gtm"}, {"title": "Product Marketing", "slug": "for-marketers/product-marketing", "href": "/courses/for-marketers/product-marketing"}, {"title": "Positioning", "slug": "for-marketers/positioning", "href": "/courses/for-marketers/positioning"}, {"title": "Social Media", "slug": "for-marketers/social-media", "href": "/courses/for-marketers/social-media"}, {"title": "Brand", "slug": "for-marketers/brand", "href": "/courses/for-marketers/brand"}, {"title": "B2B Marketing", "slug": "for-marketers/b2b", "href": "/courses/for-marketers/b2b"}, {"title": "SEO & AEO", "slug": "for-marketers/seo-aeo", "href": "/courses/for-marketers/seo-aeo"}, {"title": "Strategy", "slug": "for-marketers/strategy", "href": "/courses/for-marketers/strategy"}, {"title": "Leadership", "slug": "for-marketers/leadership", "href": "/courses/for-marketers/leadership"}]}, {"category": {"slug": "leadership", "href": "/courses/leadership", "title": "Leadership", "palette": {"--palette-category-base": "#50090E", "--palette-category-highlight": "#A9202A", "--palette-category-border": "#D49C98"}}, "topics": [{"title": "AI for Leaders", "slug": "for-leaders/ai", "href": "/courses/for-leaders/ai"}, {"title": "Agentic AI", "slug": "for-leaders/agentic-ai", "href": "/courses/for-leaders/agentic-ai"}, {"title": "AI Transformation", "slug": "for-leaders/ai-transformation", "href": "/courses/for-leaders/ai-transformation"}, {"title": "AI Governance", "slug": "for-leaders/ai-governance", "href": "/courses/for-leaders/ai-governance"}, {"title": "Communication", "slug": "for-leaders/communication", "href": "/courses/for-leaders/communication"}, {"title": "Influence", "slug": "for-leaders/influence", "href": "/courses/for-leaders/influence"}, {"title": "Strategy", "slug": "for-leaders/strategy", "href": "/courses/for-leaders/strategy"}, {"title": "Management", "slug": "for-leaders/management", "href": "/courses/for-leaders/management"}, {"title": "People Operations", "slug": "for-leaders/people-operations", "href": "/courses/for-leaders/people-operations"}, {"title": "Exec Presence", "slug": "for-leaders/executive-presence", "href": "/courses/for-leaders/executive-presence"}, {"title": "Storytelling", "slug": "for-leaders/storytelling", "href": "/courses/for-leaders/storytelling"}, {"title": "Goal-setting", "slug": "for-leaders/goal-setting", "href": "/courses/for-leaders/goal-setting"}, {"title": "Personal Brand", "slug": "for-leaders/personal-brand", "href": "/courses/for-leaders/personal-brand"}, {"title": "Career Growth", "slug": "for-leaders/career-growth", "href": "/courses/for-leaders/career-growth"}]}, {"category": {"slug": "founders", "href": "/courses/founders", "title": "Founders", "palette": {"--palette-category-base": "#0C3209", "--palette-category-highlight": "#077202", "--palette-category-border": "#69C662"}}, "topics": [{"title": "AI for Founders", "slug": "for-founders/ai", "href": "/courses/for-founders/ai"}, {"title": "Agentic AI", "slug": "for-founders/agentic-ai", "href": "/courses/for-founders/agentic-ai"}, {"title": "AI Workflows", "slug": "for-founders/ai-workflows", "href": "/courses/for-founders/ai-workflows"}, {"title": "Vibe Coding", "slug": "for-founders/vibe-coding", "href": "/courses/for-founders/vibe-coding"}, {"title": "Prototyping", "slug": "for-founders/prototyping", "href": "/courses/for-founders/prototyping"}, {"title": "Product Sense", "slug": "for-founders/product-sense", "href": "/courses/for-founders/product-sense"}, {"title": "Positioning", "slug": "for-founders/positioning", "href": "/courses/for-founders/positioning"}, {"title": "Product Discovery", "slug": "for-founders/product-discovery", "href": "/courses/for-founders/product-discovery"}, {"title": "Management", "slug": "for-founders/management", "href": "/courses/for-founders/management"}, {"title": "Strategy", "slug": "for-founders/strategy", "href": "/courses/for-founders/strategy"}, {"title": "Go-to-Market", "slug": "for-founders/gtm", "href": "/courses/for-founders/gtm"}, {"title": "Personal Brand", "slug": "for-founders/personal-brand", "href": "/courses/for-founders/personal-brand"}, {"title": "Leadership", "slug": "for-founders/leadership", "href": "/courses/for-founders/leadership"}, {"title": "Fundraising", "slug": "for-founders/fundraising", "href": "/courses/for-founders/fundraising"}, {"title": "PMF", "slug": "for-founders/product-market-fit", "href": "/courses/for-founders/product-market-fit"}]}, {"category": {"slug": "more", "href": "/courses/more", "title": "More", "palette": {"--palette-category-base": "#1E2C2B", "--palette-category-highlight": "#4A6361", "--palette-category-border": "#B9C1BE"}}, "topics": [{"title": "Everyone", "slug": "for-everyone", "href": "/courses/for-everyone"}, {"title": "Operators", "slug": "for-operators", "href": "/courses/for-operators"}, {"title": "Data Scientists", "slug": "for-data-scientists", "href": "/courses/for-data-scientists"}, {"title": "Business Analysts", "slug": "for-business-analysts", "href": "/courses/for-business-analysts"}, {"title": "User Researchers", "slug": "for-user-researchers", "href": "/courses/for-user-researchers"}, {"title": "Customer Success", "slug": "for-customer-success", "href": "/courses/for-customer-success"}, {"title": "Project Managers", "slug": "for-project-managers", "href": "/courses/for-project-managers"}, {"title": "HR Professionals", "slug": "for-hr-professionals", "href": "/courses/for-hr-professionals"}, {"title": "Sales People", "slug": "for-sales-people", "href": "/courses/for-sales-people"}, {"title": "Lawyers", "slug": "for-lawyers", "href": "/courses/for-lawyers"}, {"title": "Finance", "slug": "for-finance-professionals", "href": "/courses/for-finance-professionals"}, {"title": "Investors", "slug": "for-investors", "href": "/courses/for-investors"}, {"title": "Real Estate", "slug": "for-real-estate-agents", "href": "/courses/for-real-estate-agents"}, {"title": "Educators", "slug": "for-educators", "href": "/courses/for-educators"}, {"title": "Creators", "slug": "for-creators", "href": "/courses/for-creators"}]}], "navTagLinks": [{"tag": {"id": 293, "tag_type": "persona", "label": "For Engineers", "slug": "for-engineers", "description": null, "parent_tag_id": null}, "link_slug": "for-engineers"}, {"tag": {"id": 299, "tag_type": "persona", "label": "For Data Scientists", "slug": "for-data-scientists", "description": null, "parent_tag_id": null}, "link_slug": "for-data-scientists"}], "now": "2026-05-19T22:54:10.134Z"}, "__N_SSG": true}, "page": "/p/[slug]/[...ignored]", "query": {"slug": "30886a", "ignored": ["understanding-embedding-performance-through-generative-evals"]}, "buildId": "qPlLjBFLQ9fnkFKJ7n7E-", "isFallback": false, "isExperimentalCompile": false, "gsp": true, "scriptLoader": []}}