Collection of advanced experiments and benchmarks in LLM evaluation, instrumentation, and agent systems
Time Series Evals with OpenAI o1-preview
Prompt Caching Benchmarking
Multi-Agent Systems: Swarm
Instrumenting LLMs with OTel
Comparing Agent Frameworks
Testing Generation in RAG
Time Series Evals with OpenAI o1-preview
We benchmarked o1-preview on our hardest eval task - time series trend evaluations. This post compares that performance against GPT-4o-mini, Claude 3.5 sonnet, and GPT-4o.
o1-preview Time Series EvaluationsArize AI
Prompt Caching Benchmarking
We compare the performance and cost savings of prompt caching on Anthropic vs OpenAI.
How to Make Your AI App Feel Magical: Prompt CachingArize AI
Multi-Agent Systems: Swarm
We compare and contrast OpenAI’s experimental Swarm repo against other popular multi-agent frameworks: Autogen and CrewAI
Comparing OpenAI Swarm with other Multi Agent FrameworksArize AI
Instrumenting LLMs with OTel
Lessons learned from our journey to one million downloads of our OpenTelemetry wrapper, OpenInference.
Zero to a Million: Instrumenting LLMs with OTELArize AI
Comparing Agent Frameworks
We built the same agent in LangGraph, LlamaIndex Workflows, CrewAI, Autogen, and pure code. See how each framework compares.
Comparing Agent FrameworksArize AI
Testing Generation in RAG
Testing the generation stage of RAG across GPT-4 and Claude 2.1.
Evaluating the Generation Stage in RAGArize AI
Was this page helpful?
{"docsConfig":{"theme":"mint","$schema":"https://mintlify.com/docs.json","name":"Arize AX Docs","colors":{"primary":"#7A0047","light":"#FF8AC8","dark":"#FF74BF"},"logo":{"light":"https://mintcdn.com/arize-ax/V-uvingrGLOCK4VD/logo/light.svg?fit=max\u0026auto=format\u0026n=V-uvingrGLOCK4VD\u0026q=85\u0026s=cc58ee9388d5b5253d39b65cc662908f","dark":"https://mintcdn.com/arize-ax/V-uvingrGLOCK4VD/logo/dark.svg?fit=max\u0026auto=format\u0026n=V-uvingrGLOCK4VD\u0026q=85\u0026s=b16a0266f5913934131301a1b60af51e","href":"https://arize.com"},"favicon":"https://mintcdn.com/arize-ax/1gcABHkqP5TIGbru/favicon.ico?fit=max\u0026auto=format\u0026n=1gcABHkqP5TIGbru\u0026q=85\u0026s=aef538e87b54a3c819583bc40acfc2f8","appearance":{"default":"dark"},"background":{"color":{"light":"#FFFFFF","dark":"#0A0A12"}},"navbar":{"links":[],"primary":{"type":"button","label":"Get started","href":"https://app.arize.com/auth/join"}},"footer":{"socials":{"x":"https://x.com/arizeai","bluesky":"https://bsky.app/profile/arize.bsky.social","threads":"https://www.threads.com/@arize.ai","instagram":"https://www.instagram.com/arize_ai","linkedin":"https://www.linkedin.com/company/arizeai","youtube":"https://youtube.com/@arizeai","slack":"https://arize-ai.slack.com/ssb/redirect","website":"https://fosstodon.org/@arizeai"},"links":[{"header":"Support","items":[{"label":"Chat Us On Slack","href":"https://arize-ai.slack.com/ssb/redirect"},{"label":"support@arize.com","href":"mailto:support@arize.com"},{"label":"System Status","href":"https://status.arize.com"}]},{"header":"Get Started","items":[{"label":"Signup For Free","href":"https://arize.com/join"},{"label":"Book A Demo","href":"https://arize.com/request-a-demo"}]},{"header":"Resources","items":[{"label":"Blog","href":"https://arize.com/blog"}]}]},"seo":{"metatags":{"canonical":"https://arize.com/docs"},"indexing":"all"},"fonts":{"heading":{"family":"Geist","weight":600},"body":{"family":"Geist"}},"styling":{"codeblocks":{"theme":{"light":"github-light-default","dark":"github-dark-default"}},"latex":true},"integrations":{"gtm":{"tagId":"GTM-W96XM35"}},"contextual":{"options":["copy","view","chatgpt","claude"],"display":"header"},"interaction":{"drilldown":false},"navigation":{"pages":[]}},"deployment":{"deploymentMetadataValue":{"subdomain":"arize-ax","hasAudioNarration":false,"actualSubdomain":"arize-ax","customDomains":[],"gitSource":{"type":"github","owner":"arize-ai","repo":"docs","deployBranch":"main","contentDirectory":"","isPrivate":true,"mountPath":"","label":""},"feedback":{"thumbs":true,"contextualFeedback":{"allowEmail":true,"emailDisclaimer":"We will only use your email to follow up with you about this feedback."}},"entitlements":{"AI_CHAT":{"status":"ENABLED"},"REMOVE_BRANDING":{"status":"ENABLED"},"CONTEXTUAL_FEEDBACK":{"status":"ENABLED"},"CODE_SNIPPET_FEEDBACK":{"status":"ENABLED"},"WEB_MCP":{"status":"ENABLED"},"NARRATION":{"status":"ENABLED"}},"buildId":"6afa81bdfa6d377b6d1d4433","clientVersion":"0.0.4143","searchFilterCounts":[{"language":"default","tag":"APIs, SDKs, Skills \u0026 MCP","count":369},{"language":"default","tag":"Get Started","count":256},{"language":"default","tag":"API reference","count":214},{"language":"default","tag":"REST","count":181},{"language":"default","tag":"Endpoints","count":178},{"language":"default","tag":"API clients","count":143},{"language":"default","tag":"Machine Learning","count":139},{"language":"default","tag":"ML observability","count":139},{"language":"default","tag":"Integrations","count":129},{"language":"default","tag":"Python SDK","count":67}]},"assistantConfig":{"enableAskAiButton":true,"enableStarterQuestions":true,"enableAskAiSearchBar":true,"deflection":{"enabled":true,"email":"support@arize.com"}},"basePath":"/docs","hasChatPermissions":true,"hasWebMcpPermissions":true,"toggles":[{"name":"kill-isr","enabled":true,"variant":{"name":"disabled","enabled":false,"feature_enabled":true},"impressionData":false},{"name":"custom-rbac-roles","enabled":true,"variant":{"name":"disabled","enabled":false,"feature_enabled":true},"impressionData":false},{"name":"agent-mintlify-slack","enabled":true,"variant":{"name":"disabled","enabled":false,"feature_enabled":true},"impressionData":false},{"name":"agent-integrations-settings","enabled":true,"variant":{"name":"disabled","enabled":false,"feature_enabled":true},"impressionData":false},{"name":"rbac-expansion","enabled":true,"variant":{"name":"disabled","enabled":false,"feature_enabled":true},"impressionData":false},{"name":"automations-pr-grouping","enabled":true,"variant":{"name":"disabled","enabled":false,"feature_enabled":true},"impressionData":false},{"name":"custom-prefix","enabled":true,"variant":{"name":"disabled","enabled":false,"feature_enabled":true},"impressionData":false},{"name":"analytics-v7","enabled":true,"variant":{"name":"disabled","enabled":false,"feature_enabled":true},"impressionData":false},{"name":"user-flows-analytics","enabled":true,"variant":{"name":"disabled","enabled":false,"feature_enabled":true},"impressionData":false},{"name":"no-publish-flow","enabled":true,"variant":{"name":"disabled","enabled":false,"feature_enabled":true},"impressionData":false},{"name":"workos-onboarding","enabled":true,"variant":{"name":"disabled","enabled":false,"feature_enabled":true},"impressionData":false},{"name":"snapshot-serving-live","enabled":true,"variant":{"name":"disabled","enabled":false,"feature_enabled":true},"impressionData":false},{"name":"stripTemperatureMiddleware","enabled":true,"variant":{"name":"disabled","enabled":false,"feature_enabled":true},"impressionData":false},{"name":"support-platform-pylon-zendesk","enabled":true,"variant":{"name":"disabled","enabled":false,"feature_enabled":true},"impressionData":false},{"name":"snapshot-serving","enabled":true,"variant":{"name":"disabled","enabled":false,"feature_enabled":true},"impressionData":false},{"name":"instant-rollback","enabled":true,"variant":{"name":"disabled","enabled":false,"feature_enabled":true},"impressionData":false},{"name":"granular-notifications-server","enabled":true,"variant":{"name":"disabled","enabled":false,"feature_enabled":true},"impressionData":false},{"name":"granular-notifications","enabled":true,"variant":{"name":"disabled","enabled":false,"feature_enabled":true},"impressionData":false},{"name":"automations-fable","enabled":true,"variant":{"name":"disabled","enabled":false,"feature_enabled":true},"impressionData":false},{"name":"fixed-ai-action-pricing","enabled":true,"variant":{"name":"disabled","enabled":false,"feature_enabled":true},"impressionData":false},{"name":"automation-resolve-merge-conflict","enabled":true,"variant":{"name":"disabled","enabled":false,"feature_enabled":true},"impressionData":false},{"name":"scoped-publishing","enabled":true,"variant":{"name":"disabled","enabled":false,"feature_enabled":true},"impressionData":false},{"name":"mintql","enabled":true,"variant":{"name":"disabled","enabled":false,"feature_enabled":true},"impressionData":false},{"name":"page-level-analytics","enabled":true,"variant":{"name":"disabled","enabled":false,"feature_enabled":true},"impressionData":false},{"name":"editor-version-history-v2","enabled":true,"variant":{"name":"disabled","enabled":false,"feature_enabled":true},"impressionData":false},{"name":"billing-page-v2","enabled":true,"variant":{"name":"disabled","enabled":false,"feature_enabled":true},"impressionData":false},{"name":"full-page-search","enabled":true,"variant":{"name":"disabled","enabled":false,"feature_enabled":true},"impressionData":false},{"name":"workspace-projects","enabled":true,"variant":{"name":"disabled","enabled":false,"feature_enabled":true},"impressionData":false},{"name":"assistant-v5","enabled":true,"variant":{"name":"disabled","enabled":false,"feature_enabled":true},"impressionData":false},{"name":"fork-pr-previews","enabled":true,"variant":{"name":"disabled","enabled":false,"feature_enabled":true},"impressionData":false},{"name":"revalidation-jwt","enabled":true,"variant":{"name":"disabled","enabled":false,"feature_enabled":true},"impressionData":false},{"name":"workos-sso-scim","enabled":true,"variant":{"name":"disabled","enabled":false,"feature_enabled":true},"impressionData":false},{"name":"hobby-abuse-scan","enabled":true,"variant":{"name":"disabled","enabled":false,"feature_enabled":true},"impressionData":false},{"name":"dashboard-inbox","enabled":true,"variant":{"name":"disabled","enabled":false,"feature_enabled":true},"impressionData":false},{"name":"workos-signup","enabled":true,"variant":{"name":"disabled","enabled":false,"feature_enabled":true},"impressionData":false},{"name":"turbopuffer-dual-write","enabled":true,"variant":{"name":"disabled","enabled":false,"feature_enabled":true},"impressionData":false},{"name":"workos-enabled","enabled":true,"variant":{"name":"disabled","enabled":false,"feature_enabled":true},"impressionData":false},{"name":"approve-review-flow","enabled":true,"variant":{"name":"disabled","enabled":false,"feature_enabled":true},"impressionData":false},{"name":"editor-metadata","enabled":true,"variant":{"name":"disabled","enabled":false,"feature_enabled":true},"impressionData":false},{"name":"sso-setup-deadline-banner","enabled":true,"variant":{"name":"disabled","enabled":false,"feature_enabled":true},"impressionData":false},{"name":"turbopuffer-serve","enabled":true,"variant":{"name":"disabled","enabled":false,"feature_enabled":true},"impressionData":false},{"name":"live-agents","enabled":true,"variant":{"name":"disabled","enabled":false,"feature_enabled":true},"impressionData":false},{"name":"llms-txt-v2","enabled":true,"variant":{"name":"disabled","enabled":false,"feature_enabled":true},"impressionData":false}],"starterQuestions":[{"_id":"6944513fe59e82150b5700b3","deploymentId":"6894ef048eafbd2a7c20796e","questionText":"Need more help? Ask our team in #arize-ax-platform-support on community Slack.","status":"active","generatedBy":"manual","createdAt":"2025-12-18T19:08:47.802Z","updatedAt":"2025-12-18T19:15:01.436Z","__v":0}],"googleFontsInlined":true},"customCss":[{"filePath":"style.css","content":"/* cyrillic-ext */\n@font-face {\n font-family: 'Geist Mono';\n font-style: normal;\n font-weight: 300;\n font-display: swap;\n src: url(/docs/_mintlify/assets/fonts/google/s/geistmono/v6/or3nQ6H-1_WfwkMZI_qYFrodmgPn.woff2) format('woff2');\n unicode-range: U+0460-052F, U+1C80-1C8A, U+20B4, U+2DE0-2DFF, U+A640-A69F, U+FE2E-FE2F;\n}\n/* cyrillic */\n@font-face {\n font-family: 'Geist Mono';\n font-style: normal;\n font-weight: 300;\n font-display: swap;\n src: url(/docs/_mintlify/assets/fonts/google/s/geistmono/v6/or3nQ6H-1_WfwkMZI_qYFrMdmgPn.woff2) format('woff2');\n unicode-range: U+0301, U+0400-045F, U+0490-0491, U+04B0-04B1, U+2116;\n}\n/* symbols2 */\n@font-face {\n font-family: 'Geist Mono';\n font-style: normal;\n font-weight: 300;\n font-display: swap;\n src: url(/docs/_mintlify/assets/fonts/google/s/geistmono/v6/or3nQ6H-1_WfwkMZI_qYFg08vz7ehw.woff2) format('woff2');\n unicode-range: U+2000-2001, U+2004-2008, U+200A, U+23B8-23BD, U+2500-259F;\n}\n/* vietnamese */\n@font-face {\n font-family: 'Geist Mono';\n font-style: normal;\n font-weight: 300;\n font-display: swap;\n src: url(/docs/_mintlify/assets/fonts/google/s/geistmono/v6/or3nQ6H-1_WfwkMZI_qYFrgdmgPn.woff2) format('woff2');\n unicode-range: U+0102-0103, U+0110-0111, U+0128-0129, U+0168-0169, U+01A0-01A1, U+01AF-01B0, U+0300-0301, U+0303-0304, U+0308-0309, U+0323, U+0329, U+1EA0-1EF9, U+20AB;\n}\n/* latin-ext */\n@font-face {\n font-family: 'Geist Mono';\n font-style: normal;\n font-weight: 300;\n font-display: swap;\n src: url(/docs/_mintlify/assets/fonts/google/s/geistmono/v6/or3nQ6H-1_WfwkMZI_qYFrkdmgPn.woff2) format('woff2');\n unicode-range: U+0100-02BA, U+02BD-02C5, U+02C7-02CC, U+02CE-02D7, U+02DD-02FF, U+0304, U+0308, U+0329, U+1D00-1DBF, U+1E00-1E9F, U+1EF2-1EFF, U+2020, U+20A0-20AB, U+20AD-20C4, U+2113, U+2C60-2C7F, U+A720-A7FF;\n}\n/* latin */\n@font-face {\n font-family: 'Geist Mono';\n font-style: normal;\n font-weight: 300;\n font-display: swap;\n src: url(/docs/_mintlify/assets/fonts/google/s/geistmono/v6/or3nQ6H-1_WfwkMZI_qYFrcdmg.woff2) format('woff2');\n unicode-range: U+0000-00FF, U+0131, U+0152-0153, U+02BB-02BC, U+02C6, U+02DA, U+02DC, U+0304, U+0308, U+0329, U+2000-206F, U+20AC, U+2122, U+2191, U+2193, U+2212, U+2215, U+FEFF, U+FFFD;\n}\n/* cyrillic-ext */\n@font-face {\n font-family: 'Geist Mono';\n font-style: normal;\n font-weight: 400;\n font-display: swap;\n src: url(/docs/_mintlify/assets/fonts/google/s/geistmono/v6/or3nQ6H-1_WfwkMZI_qYFrodmgPn.woff2) format('woff2');\n unicode-range: U+0460-052F, U+1C80-1C8A, U+20B4, U+2DE0-2DFF, U+A640-A69F, U+FE2E-FE2F;\n}\n/* cyrillic */\n@font-face {\n font-family: 'Geist Mono';\n font-style: normal;\n font-weight: 400;\n font-display: swap;\n src: url(/docs/_mintlify/assets/fonts/google/s/geistmono/v6/or3nQ6H-1_WfwkMZI_qYFrMdmgPn.woff2) format('woff2');\n unicode-range: U+0301, U+0400-045F, U+0490-0491, U+04B0-04B1, U+2116;\n}\n/* symbols2 */\n@font-face {\n font-family: 'Geist Mono';\n font-style: normal;\n font-weight: 400;\n font-display: swap;\n src: url(/docs/_mintlify/assets/fonts/google/s/geistmono/v6/or3nQ6H-1_WfwkMZI_qYFg08vz7ehw.woff2) format('woff2');\n unicode-range: U+2000-2001, U+2004-2008, U+200A, U+23B8-23BD, U+2500-259F;\n}\n/* vietnamese */\n@font-face {\n font-family: 'Geist Mono';\n font-style: normal;\n font-weight: 400;\n font-display: swap;\n src: url(/docs/_mintlify/assets/fonts/google/s/geistmono/v6/or3nQ6H-1_WfwkMZI_qYFrgdmgPn.woff2) format('woff2');\n unicode-range: U+0102-0103, U+0110-0111, U+0128-0129, U+0168-0169, U+01A0-01A1, U+01AF-01B0, U+0300-0301, U+0303-0304, U+0308-0309, U+0323, U+0329, U+1EA0-1EF9, U+20AB;\n}\n/* latin-ext */\n@font-face {\n font-family: 'Geist Mono';\n font-style: normal;\n font-weight: 400;\n font-display: swap;\n src: url(/docs/_mintlify/assets/fonts/google/s/geistmono/v6/or3nQ6H-1_WfwkMZI_qYFrkdmgPn.woff2) format('woff2');\n unicode-range: U+0100-02BA, U+02BD-02C5, U+02C7-02CC, U+02CE-02D7, U+02DD-02FF, U+0304, U+0308, U+0329, U+1D00-1DBF, U+1E00-1E9F, U+1EF2-1EFF, U+2020, U+20A0-20AB, U+20AD-20C4, U+2113, U+2C60-2C7F, U+A720-A7FF;\n}\n/* latin */\n@font-face {\n font-family: 'Geist Mono';\n font-style: normal;\n font-weight: 400;\n font-display: swap;\n src: url(/docs/_mintlify/assets/fonts/google/s/geistmono/v6/or3nQ6H-1_WfwkMZI_qYFrcdmg.woff2) format('woff2');\n unicode-range: U+0000-00FF, U+0131, U+0152-0153, U+02BB-02BC, U+02C6, U+02DA, U+02DC, U+0304, U+0308, U+0329, U+2000-206F, U+20AC, U+2122, U+2191, U+2193, U+2212, U+2215, U+FEFF, U+FFFD;\n}\n/* cyrillic-ext */\n@font-face {\n font-family: 'Geist Mono';\n font-style: normal;\n font-weight: 500;\n font-display: swap;\n src: url(/docs/_mintlify/assets/fonts/google/s/geistmono/v6/or3nQ6H-1_WfwkMZI_qYFrodmgPn.woff2) format('woff2');\n unicode-range: U+0460-052F, U+1C80-1C8A, U+20B4, U+2DE0-2DFF, U+A640-A69F, U+FE2E-FE2F;\n}\n/* cyrillic */\n@font-face {\n font-family: 'Geist Mono';\n font-style: normal;\n font-weight: 500;\n font-display: swap;\n src: url(/docs/_mintlify/assets/fonts/google/s/geistmono/v6/or3nQ6H-1_WfwkMZI_qYFrMdmgPn.woff2) format('woff2');\n unicode-range: U+0301, U+0400-045F, U+0490-0491, U+04B0-04B1, U+2116;\n}\n/* symbols2 */\n@font-face {\n font-family: 'Geist Mono';\n font-style: normal;\n font-weight: 500;\n font-display: swap;\n src: url(/docs/_mintlify/assets/fonts/google/s/geistmono/v6/or3nQ6H-1_WfwkMZI_qYFg08vz7ehw.woff2) format('woff2');\n unicode-range: U+2000-2001, U+2004-2008, U+200A, U+23B8-23BD, U+2500-259F;\n}\n/* vietnamese */\n@font-face {\n font-family: 'Geist Mono';\n font-style: normal;\n font-weight: 500;\n font-display: swap;\n src: url(/docs/_mintlify/assets/fonts/google/s/geistmono/v6/or3nQ6H-1_WfwkMZI_qYFrgdmgPn.woff2) format('woff2');\n unicode-range: U+0102-0103, U+0110-0111, U+0128-0129, U+0168-0169, U+01A0-01A1, U+01AF-01B0, U+0300-0301, U+0303-0304, U+0308-0309, U+0323, U+0329, U+1EA0-1EF9, U+20AB;\n}\n/* latin-ext */\n@font-face {\n font-family: 'Geist Mono';\n font-style: normal;\n font-weight: 500;\n font-display: swap;\n src: url(/docs/_mintlify/assets/fonts/google/s/geistmono/v6/or3nQ6H-1_WfwkMZI_qYFrkdmgPn.woff2) format('woff2');\n unicode-range: U+0100-02BA, U+02BD-02C5, U+02C7-02CC, U+02CE-02D7, U+02DD-02FF, U+0304, U+0308, U+0329, U+1D00-1DBF, U+1E00-1E9F, U+1EF2-1EFF, U+2020, U+20A0-20AB, U+20AD-20C4, U+2113, U+2C60-2C7F, U+A720-A7FF;\n}\n/* latin */\n@font-face {\n font-family: 'Geist Mono';\n font-style: normal;\n font-weight: 500;\n font-display: swap;\n src: url(/docs/_mintlify/assets/fonts/google/s/geistmono/v6/or3nQ6H-1_WfwkMZI_qYFrcdmg.woff2) format('woff2');\n unicode-range: U+0000-00FF, U+0131, U+0152-0153, U+02BB-02BC, U+02C6, U+02DA, U+02DC, U+0304, U+0308, U+0329, U+2000-206F, U+20AC, U+2122, U+2191, U+2193, U+2212, U+2215, U+FEFF, U+FFFD;\n}\n/* cyrillic-ext */\n@font-face {\n font-family: 'Geist Mono';\n font-style: normal;\n font-weight: 600;\n font-display: swap;\n src: url(/docs/_mintlify/assets/fonts/google/s/geistmono/v6/or3nQ6H-1_WfwkMZI_qYFrodmgPn.woff2) format('woff2');\n unicode-range: U+0460-052F, U+1C80-1C8A, U+20B4, U+2DE0-2DFF, U+A640-A69F, U+FE2E-FE2F;\n}\n/* cyrillic */\n@font-face {\n font-family: 'Geist Mono';\n font-style: normal;\n font-weight: 600;\n font-display: swap;\n src: url(/docs/_mintlify/assets/fonts/google/s/geistmono/v6/or3nQ6H-1_WfwkMZI_qYFrMdmgPn.woff2) format('woff2');\n unicode-range: U+0301, U+0400-045F, U+0490-0491, U+04B0-04B1, U+2116;\n}\n/* symbols2 */\n@font-face {\n font-family: 'Geist Mono';\n font-style: normal;\n font-weight: 600;\n font-display: swap;\n src: url(/docs/_mintlify/assets/fonts/google/s/geistmono/v6/or3nQ6H-1_WfwkMZI_qYFg08vz7ehw.woff2) format('woff2');\n unicode-range: U+2000-2001, U+2004-2008, U+200A, U+23B8-23BD, U+2500-259F;\n}\n/* vietnamese */\n@font-face {\n font-family: 'Geist Mono';\n font-style: normal;\n font-weight: 600;\n font-display: swap;\n src: url(/docs/_mintlify/assets/fonts/google/s/geistmono/v6/or3nQ6H-1_WfwkMZI_qYFrgdmgPn.woff2) format('woff2');\n unicode-range: U+0102-0103, U+0110-0111, U+0128-0129, U+0168-0169, U+01A0-01A1, U+01AF-01B0, U+0300-0301, U+0303-0304, U+0308-0309, U+0323, U+0329, U+1EA0-1EF9, U+20AB;\n}\n/* latin-ext */\n@font-face {\n font-family: 'Geist Mono';\n font-style: normal;\n font-weight: 600;\n font-display: swap;\n src: url(/docs/_mintlify/assets/fonts/google/s/geistmono/v6/or3nQ6H-1_WfwkMZI_qYFrkdmgPn.woff2) format('woff2');\n unicode-range: U+0100-02BA, U+02BD-02C5, U+02C7-02CC, U+02CE-02D7, U+02DD-02FF, U+0304, U+0308, U+0329, U+1D00-1DBF, U+1E00-1E9F, U+1EF2-1EFF, U+2020, U+20A0-20AB, U+20AD-20C4, U+2113, U+2C60-2C7F, U+A720-A7FF;\n}\n/* latin */\n@font-face {\n font-family: 'Geist Mono';\n font-style: normal;\n font-weight: 600;\n font-display: swap;\n src: url(/docs/_mintlify/assets/fonts/google/s/geistmono/v6/or3nQ6H-1_WfwkMZI_qYFrcdmg.woff2) format('woff2');\n unicode-range: U+0000-00FF, U+0131, U+0152-0153, U+02BB-02BC, U+02C6, U+02DA, U+02DC, U+0304, U+0308, U+0329, U+2000-206F, U+20AC, U+2122, U+2191, U+2193, U+2212, U+2215, U+FEFF, U+FFFD;\n}\nhtml:root{--font-family-mono-custom: 'Geist Mono'}html:root{--gray-50: 248 248 246;--gray-100: 242 242 237;--gray-200: 230 230 223;--gray-300: 205 205 200;--gray-400: 179 179 191;--gray-500: 138 138 150;--gray-600: 91 91 105;--gray-700: 62 62 78;--gray-800: 46 46 61;--gray-900: 21 21 31;--gray-950: 10 10 18}html:root{--ax-radius-sm: 2px;--ax-radius-md: 6px;--ax-radius-lg: 8px;--ax-accent-bg: #FF74BF;--ax-panel: #CDBEF7;--ax-panel-2: #E3DBFC;--ax-panel-code: #FFFFFF;--ax-panel-border: rgba(19, 22, 64, 0.22);--ax-panel-icon: #4827C1;--ax-panel-t1: #121221;--ax-panel-t2: #2A2A38;--ax-panel-accent: #6E1046;--ax-panel-code-kw: #3F4110;--ax-panel-code-flag: #5A3200;--ax-panel-code-url: #004A70;--ax-panel-selection: #FFE9F4}html:not(.dark){--ax-bg: #FFFFFF;--ax-surface: #F8F8F4;--ax-code-bg: #F5F5ED;--ax-chip: #EDEDE2;--ax-line: rgba(19, 22, 64, 0.20);--ax-t1: #121221;--ax-t2: #2E2E3D;--ax-t3: #3E3E4E;--ax-accent: #7A0047;--ax-accent-line: rgba(255, 98, 182, 0.55);--ax-accent-soft: #FFE9F4;--ax-on-accent: #121221;--ax-note: #E8F3EC;--ax-note-line: rgba(30, 70, 50, 0.24);--ax-note-t: #1B4230;--ax-warn: #FBEEE0;--ax-warn-line: rgba(90, 50, 0, 0.24);--ax-warn-t: #54300A;--ax-blue: #E7F3FD;--ax-blue-line: rgba(0, 82, 125, 0.24);--ax-blue-t: #00527D;--ax-danger: #FFEBE9;--ax-danger-line: rgba(207, 34, 46, 0.28);--ax-danger-t: #A40E26}html.dark{--ax-bg: #0A0A12;--ax-surface: #15151F;--ax-code-bg: #05050B;--ax-chip: #221B3A;--ax-line: rgba(247, 247, 252, 0.16);--ax-t1: #F5F5FA;--ax-t2: #DCDCE6;--ax-t3: #B3B3C4;--ax-accent: #FF8AC8;--ax-accent-line: rgba(255, 138, 200, 0.45);--ax-accent-soft: #2A1425;--ax-on-accent: #0A0A12;--ax-note: #10241A;--ax-note-line: rgba(159, 224, 188, 0.30);--ax-note-t: #9FE0BC;--ax-warn: #2A1B0A;--ax-warn-line: rgba(255, 201, 138, 0.30);--ax-warn-t: #FFC98A;--ax-blue: #0A1A24;--ax-blue-line: rgba(57, 195, 255, 0.30);--ax-blue-t: #97D8FF;--ax-danger: #2A0F12;--ax-danger-line: rgba(255, 138, 148, 0.30);--ax-danger-t: #FFB3B8}#content{line-height:1.75}#page-title,#content h1{font-size:clamp(2rem, 1.4rem + 2.1vw, 2.875rem);line-height:1.12;letter-spacing:-0.02em;font-weight:600}#content h2,#content h3,#content h4{letter-spacing:-0.01em;font-weight:600}#content code,#content kbd,#content samp{font-feature-settings:'tnum' 1}#content a:not(.card){color:var(--ax-accent);text-decoration-thickness:1px;text-underline-offset:3px}:focus-visible{outline:2px solid var(--ax-accent-line);outline-offset:2px}html.dark :focus-visible{outline-color:var(--ax-accent-line)}::selection{background:var(--ax-accent-soft)}#navbar{background:var(--ax-bg);border-bottom:1px solid var(--ax-line)}#navbar div:has(>.nav-tabs){position:relative}#navbar div:has(>.nav-tabs)::before{content:'';position:absolute;top:0;left:50%;width:100vw;margin-left:-50vw;border-top:1px solid var(--ax-line);pointer-events:none}.nav-tabs-item[data-active]{box-shadow:inset 0 -2px 0 var(--ax-accent)}.ax-product-selector{position:relative;display:inline-flex;align-items:stretch;flex:0 0 auto}.ax-product-selector--desktop{margin-left:0.75rem;align-self:center}.page-context-dropdown__label{font-family:inherit;font-weight:400}.page-context-dropdown__chevron>span{display:inline-block;transition:transform 150ms ease}.page-context-dropdown__chevron svg{display:block;width:1rem;height:1rem}.page-context-dropdown__chevron[aria-expanded=\"true\"]>span{transform:rotate(180deg)}.page-context-dropdown__menu{position:absolute;z-index:60;top:calc(100% + 0.55rem);right:0;min-width:12.5rem;padding:0.4rem;background:var(--ax-bg);border:1px solid var(--ax-line);border-radius:var(--ax-radius-md);box-shadow:0 0.75rem 1.5rem rgb(10 10 18 / 0.18)}.ax-product-selector--desktop .page-context-dropdown__menu{right:auto;left:0}.page-context-dropdown__menu[hidden]{display:none}.page-context-dropdown__item{display:flex;align-items:center;justify-content:space-between;min-height:2.5rem;padding:0.55rem 0.65rem;color:var(--ax-t1);border-radius:var(--ax-radius-sm);font-size:0.9375rem;font-weight:400;text-decoration:none;transition:background-color 150ms ease, color 150ms ease}.page-context-dropdown__item:hover,.page-context-dropdown__item:focus-visible{color:var(--ax-accent);background:var(--ax-accent-soft)}.page-context-dropdown__item[data-selected=\"true\"]{color:var(--ax-accent);font-weight:500}.page-context-dropdown__item[data-selected=\"true\"]::after{content:'✓';margin-left:1.5rem;color:var(--ax-accent);font-size:0.875rem}@media (prefers-reduced-motion: reduce){.page-context-dropdown__chevron>span,.page-context-dropdown__item{transition:none}}@media (max-width: 767px){#navbar .ax-product-selector--desktop{display:none}#mobile-nav-content .ax-product-selector--mobile{margin:0.5rem 0 1rem}#mobile-nav-content .page-context-dropdown__label{flex:1 1 auto}#mobile-nav-content .page-context-dropdown__chevron{flex:0 0 auto}#mobile-nav-content .page-context-dropdown__menu{position:static;margin-top:0.4rem}}#topbar-cta-button{background:var(--ax-accent-bg);color:var(--ax-on-accent);border-radius:var(--ax-radius-md);border:none}#topbar-cta-button span,#topbar-cta-button svg{color:var(--ax-on-accent)!important}#search-bar-entry{background:var(--ax-surface);border:1px solid var(--ax-line);border-radius:var(--ax-radius-md)}#assistant-entry{background:var(--ax-surface);border:1px solid var(--ax-accent-line);border-radius:var(--ax-radius-md)}#assistant-entry svg{color:var(--ax-accent)}#page-context-menu,.page-context-dropdown{font-size:0.9375rem;align-items:stretch}#page-context-menu-button,.page-context-dropdown__label{display:inline-flex;align-items:center;padding:10px 14px;border-radius:var(--ax-radius-md) 0 0 var(--ax-radius-md);background:var(--ax-surface);border-color:var(--ax-line);color:var(--ax-t1);border:1px solid var(--ax-line);text-decoration:none}#page-context-menu-button>div{font-size:0.9375rem;gap:9px}#page-context-menu>button[aria-haspopup],.page-context-dropdown__chevron{height:auto;align-self:stretch;aspect-ratio:auto;padding:10px 11px;border-radius:0 var(--ax-radius-md) var(--ax-radius-md) 0;background:var(--ax-surface);border-color:var(--ax-line);color:var(--ax-t3);border:1px solid var(--ax-line);border-left:0;cursor:pointer}#sidebar-content li[data-active]>a{background:var(--ax-accent-soft);color:var(--ax-accent);border-radius:var(--ax-radius-md);font-weight:500}.sidebar-group-header{font-weight:600;color:var(--ax-t1)}.toc-item[data-active],.toc-item[data-active] a{color:var(--ax-accent);font-weight:400}.card.card{background:var(--ax-surface);border-color:var(--ax-line);border-radius:var(--ax-radius-lg);box-shadow:none}.frame.frame{background:var(--ax-surface);border:1px solid var(--ax-line);border-radius:var(--ax-radius-lg)}.accordion.accordion,.accordion-group.accordion-group{border-radius:var(--ax-radius-md);border-color:var(--ax-line)}.accordion.accordion{background:var(--ax-surface)}.code-block.code-block,.code-group.code-group{background:var(--ax-code-bg);border-color:var(--ax-line);border-radius:var(--ax-radius-md)}.code-block-background{background:var(--ax-code-bg)!important}html.dark .code-block-background{background:var(--ax-code-bg)!important}#content :not(pre)>code{background:var(--ax-chip);border-radius:var(--ax-radius-sm)}.code-block:has([data-floating-buttons]) [data-fade-overlay]{background-image:none}[data-floating-buttons] button{background:var(--ax-chip)}.callout .code-block.code-block,.callout .code-group.code-group,.callout .code-block-background,html.dark .callout .code-block-background{background:var(--ax-bg)!important}.callout{border-radius:var(--ax-radius-lg)}.callout[data-callout-type=\"callout\"],.callout[data-callout-type=\"info\"]{background:var(--ax-surface);border-color:var(--ax-line)}.callout[data-callout-type=\"callout\"] [data-component-part=\"callout-content\"],.callout[data-callout-type=\"info\"] [data-component-part=\"callout-content\"]{color:var(--ax-t2)!important}.callout[data-callout-type=\"tip\"],.callout[data-callout-type=\"check\"]{background:var(--ax-note);border-color:var(--ax-note-line)}.callout[data-callout-type=\"tip\"] [data-component-part=\"callout-content\"],.callout[data-callout-type=\"check\"] [data-component-part=\"callout-content\"]{color:var(--ax-note-t)!important}.callout[data-callout-type=\"warning\"]{background:var(--ax-warn);border-color:var(--ax-warn-line)}.callout[data-callout-type=\"warning\"] [data-component-part=\"callout-content\"]{color:var(--ax-warn-t)!important}.callout[data-callout-type=\"note\"]{background:var(--ax-blue);border-color:var(--ax-blue-line)}.callout[data-callout-type=\"note\"] [data-component-part=\"callout-content\"]{color:var(--ax-blue-t)!important}.callout[data-callout-type=\"danger\"]{background:var(--ax-danger);border-color:var(--ax-danger-line)}.callout[data-callout-type=\"danger\"] [data-component-part=\"callout-content\"]{color:var(--ax-danger-t)!important}[data-component-part=\"step-number\"]>div{background:var(--ax-accent-soft);border:1px solid var(--ax-accent-line);color:var(--ax-accent);font-weight:500}.update.update-container{flex-direction:column;gap:0.5rem;border-top:1px solid var(--ax-line)}.update.update-container:first-of-type{border-top:none}.update.update-container>div:first-child{width:100%;position:static}.update.update-container>div:last-child{width:100%;max-width:none}#content th{color:var(--ax-t1);font-weight:600;border-bottom:1px solid var(--ax-line)}#content td{border-bottom:1px solid var(--ax-line)}footer{border-top:1px solid var(--ax-line)}.ax-hero .callout{background:var(--ax-panel);border-color:var(--ax-panel-border);border-radius:var(--ax-radius-lg)}.ax-hero .callout [data-component-part=\"callout-content\"],.ax-hero .callout p,.ax-hero .callout li{color:var(--ax-panel-t2)!important}.ax-hero .callout h2,.ax-hero .callout h3,.ax-hero .callout strong{color:var(--ax-panel-t1)!important}.ax-hero .callout [data-component-part=\"callout-icon\"] svg{background:var(--ax-panel-icon)!important}.ax-hero .code-block,.ax-hero .code-group,.ax-hero .code-block-background{background:var(--ax-panel-code)!important;border-color:var(--ax-panel-border)}.ax-hero .code-block,.ax-hero .code-group{padding:0;overflow:hidden}html.dark .ax-hero .code-block,html.dark .ax-hero .code-group,html.dark .ax-hero .code-block-background,html.dark .ax-hero .shiki,html:not(.dark) .ax-hero .code-block,html:not(.dark) .ax-hero .code-group,html:not(.dark) .ax-hero .code-block-background{background:var(--ax-panel-code)!important}html.dark .ax-hero .shiki,html.dark .ax-hero .shiki span{color:var(--ax-panel-t1)!important}html.dark .ax-hero .shiki span[style*=\"#FFA657\"]{color:var(--ax-panel-code-kw)!important}html.dark .ax-hero .shiki span[style*=\"#79C0FF\"]{color:var(--ax-panel-code-flag)!important}html.dark .ax-hero .shiki span[style*=\"#A5D6FF\"]{color:var(--ax-panel-code-url)!important}.ax-hero .code-group>div:first-child,html.dark .ax-hero .code-group>div:first-child{background:var(--ax-panel-2)!important;border-bottom:1px solid var(--ax-panel-border)}.ax-hero .code-group [role=\"tab\"]{color:var(--ax-panel-t2)!important}.ax-hero .code-group [role=\"tab\"][aria-selected=\"true\"]{color:var(--ax-panel-accent)!important}.ax-hero .code-group [role=\"tab\"][aria-selected=\"true\"]>div:last-child{background:var(--ax-panel-accent)!important}.ax-hero .code-block svg,.ax-hero .code-group svg{color:var(--ax-panel-t2)}html.dark .ax-hero [data-fade-overlay]{background-image:linear-gradient(\n to right,\n rgb(255 255 255 / 0) 0,\n rgb(255 255 255 / 0.2) 10px,\n rgb(255 255 255 / 0.5) 25px,\n rgb(255 255 255 / 0.8) 35px,\n rgb(255 255 255) 45px\n )}.ax-hero .callout a{color:var(--ax-panel-accent)!important}.ax-hero::selection,.ax-hero ::selection{background:var(--ax-panel-selection)}.card-group .card img{max-height:197px!important}.card-group[class$=\"-2\"] .card img{height:160px!important}.card-group[class$=\"-3\"] .card img{height:150px!important}table{display:table!important}.step-title{font-size:1.5rem!important}"}],"page":{"pagePath":"ax/cookbooks/ai-research","slug":"/ax/cookbooks/ai-research","pageMetadata":{"title":"AI Research","description":"Collection of advanced experiments and benchmarks in LLM evaluation, instrumentation, and agent systems","href":"/ax/cookbooks/ai-research"},"description":{"compiledSource":"\"use strict\";\nconst {jsx: _jsx} = arguments[0];\nconst {useMDXComponents: _provideComponents} = arguments[0];\nfunction _createMdxContent(props) {\n const _components = {\n p: \"p\",\n ..._provideComponents(),\n ...props.components\n };\n return _jsx(_components.p, {\n children: \"Collection of advanced experiments and benchmarks in LLM evaluation, instrumentation, and agent systems\"\n });\n}\nfunction MDXContent(props = {}) {\n const {wrapper: MDXLayout} = {\n ..._provideComponents(),\n ...props.components\n };\n return MDXLayout ? _jsx(MDXLayout, {\n ...props,\n children: _jsx(_createMdxContent, {\n ...props\n })\n }) : _createMdxContent(props);\n}\nreturn {\n default: MDXContent\n};\n","frontmatter":{},"scope":{}},"pageType":"default","mdx":{"mdxSource":{"compiledSource":"\"use strict\";const{Fragment:_Fragment,jsx:_jsx,jsxs:_jsxs}=arguments[0];const{useMDXComponents:_provideComponents}=arguments[0];function _createMdxContent(props){const _components={a:`a`,p:`p`,strong:`strong`,..._provideComponents(),...props.components},{Card,CardGroup,Frame,Heading,OptimizedFrame}=_components;if(!Card)_missingMdxReference(`Card`,true);if(!CardGroup)_missingMdxReference(`CardGroup`,true);if(!Frame)_missingMdxReference(`Frame`,true);if(!Heading)_missingMdxReference(`Heading`,true);if(!OptimizedFrame)_missingMdxReference(`OptimizedFrame`,true);return _jsxs(_Fragment,{children:[_jsxs(CardGroup,{children:[_jsx(Card,{icon:`clock`,href:`#time-series-evals-with-openai-o1-preview`,title:`Time Series Evals with OpenAI o1-preview`,horizontal:true}),_jsx(Card,{icon:`brain`,href:`#prompt-caching-benchmarking`,title:`Prompt Caching Benchmarking`,horizontal:true}),_jsx(Card,{icon:`robot`,href:`#multi-agent-systems-swarm`,title:`Multi-Agent Systems: Swarm`,horizontal:true}),_jsx(Card,{icon:`microchip`,href:`#instrumenting-llms-with-otel`,title:`Instrumenting LLMs with OTel`,horizontal:true}),_jsx(Card,{icon:`robot`,href:`#comparing-agent-frameworks`,title:`Comparing Agent Frameworks`,horizontal:true}),_jsx(Card,{icon:`code`,href:`#testing-generation-in-rag`,title:`Testing Generation in RAG`,horizontal:true})]}),`\n`,_jsx(Heading,{level:`2`,id:`time-series-evals-with-openai-o1-preview`,children:`Time Series Evals with OpenAI o1-preview`}),`\n`,_jsx(Frame,{children:_jsx(OptimizedFrame,{width:`100%`,height:`420`,src:`https://cdn.iframe.ly/wo1LxVY7`,title:`LLM Evals and LLM as a Judge: Fundamentals`,frameborder:`0`,allow:`accelerometer; autoplay; clipboard-write; encrypted-media; gyroscope; picture-in-picture; web-share`,referrerpolicy:`strict-origin-when-cross-origin`,allowfullscreen:true})}),`\n`,_jsxs(_components.p,{children:[`We benchmarked o1-preview on our hardest eval task - `,_jsx(_components.strong,{children:`time series trend evaluations`}),`. This post compares that performance against GPT-4o-mini, Claude 3.5 sonnet, and GPT-4o.`]}),`\n`,_jsx(CardGroup,{children:_jsx(Card,{title:`o1-preview Time Series EvaluationsArize AI`,href:`https://arize.com/blog/o1-preview-time-series-evaluations`,cta:`Arize AI`,icon:`clock`})}),`\n`,_jsx(Heading,{level:`2`,id:`prompt-caching-benchmarking`,children:`Prompt Caching Benchmarking`}),`\n`,_jsx(Frame,{children:_jsx(OptimizedFrame,{width:`100%`,height:`420`,src:`https://cdn.iframe.ly/Qea8uRSv`,title:`LLM Evals and LLM as a Judge: Fundamentals`,frameborder:`0`,allow:`accelerometer; autoplay; clipboard-write; encrypted-media; gyroscope; picture-in-picture; web-share`,referrerpolicy:`strict-origin-when-cross-origin`,allowfullscreen:true})}),`\n`,_jsxs(_components.p,{children:[`We compare the performance and cost savings of `,_jsx(_components.strong,{children:`prompt caching on Anthropic vs OpenAI`}),`.`]}),`\n`,_jsx(CardGroup,{children:_jsx(Card,{title:`How to Make Your AI App Feel Magical: Prompt CachingArize AI`,href:`https://arize.com/blog/prompt-caching-analysis`,cta:`Arize AI`,icon:`brain`})}),`\n`,_jsx(Heading,{level:`2`,id:`multi-agent-systems-swarm`,children:`Multi-Agent Systems: Swarm`}),`\n`,_jsx(Frame,{children:_jsx(OptimizedFrame,{width:`100%`,height:`420`,src:`https://cdn.iframe.ly/IUEEfxxN`,title:`LLM Evals and LLM as a Judge: Fundamentals`,frameborder:`0`,allow:`accelerometer; autoplay; clipboard-write; encrypted-media; gyroscope; picture-in-picture; web-share`,referrerpolicy:`strict-origin-when-cross-origin`,allowfullscreen:true})}),`\n`,_jsxs(_components.p,{children:[`We compare and contrast `,_jsx(_components.strong,{children:`OpenAI’s experimental Swarm repo`}),` against other popular multi-agent frameworks: `,_jsx(_components.strong,{children:`Autogen and CrewAI`})]}),`\n`,_jsx(CardGroup,{children:_jsx(Card,{title:`Comparing OpenAI Swarm with other Multi Agent FrameworksArize AI`,href:`https://arize.com/blog/comparing-openai-swarm`,cta:`Arize AI`,icon:`robot`})}),`\n`,_jsx(Heading,{level:`2`,id:`instrumenting-llms-with-otel`,children:`Instrumenting LLMs with OTel`}),`\n`,_jsx(Frame,{children:_jsx(OptimizedFrame,{width:`100%`,height:`420`,src:`https://cdn.iframe.ly/rmqkMGSE`,title:`LLM Evals and LLM as a Judge: Fundamentals`,frameborder:`0`,allow:`accelerometer; autoplay; clipboard-write; encrypted-media; gyroscope; picture-in-picture; web-share`,referrerpolicy:`strict-origin-when-cross-origin`,allowfullscreen:true})}),`\n`,_jsxs(_components.p,{children:[`Lessons learned from our journey to one million downloads of our OpenTelemetry wrapper, `,_jsx(_components.a,{href:`https://github.com/Arize-ai/openinference`,children:`OpenInference`}),`.`]}),`\n`,_jsx(CardGroup,{children:_jsx(Card,{title:`Zero to a Million: Instrumenting LLMs with OTELArize AI`,href:`https://arize.com/blog/zero-to-a-million-instrumenting-llms-with-otel`,cta:`Arize AI`,icon:`robot`})}),`\n`,_jsx(Heading,{level:`2`,id:`comparing-agent-frameworks`,children:`Comparing Agent Frameworks`}),`\n`,_jsx(Frame,{children:_jsx(OptimizedFrame,{width:`100%`,height:`420`,src:`https://cdn.iframe.ly/AUHGLGQQ`,title:`LLM Evals and LLM as a Judge: Fundamentals`,frameborder:`0`,allow:`accelerometer; autoplay; clipboard-write; encrypted-media; gyroscope; picture-in-picture; web-share`,referrerpolicy:`strict-origin-when-cross-origin`,allowfullscreen:true})}),`\n`,_jsxs(_components.p,{children:[`We built the `,_jsx(_components.a,{href:`https://github.com/Arize-ai/phoenix/tree/main/examples/agent_framework_comparison`,children:`same agent`}),` in LangGraph, LlamaIndex Workflows, CrewAI, Autogen, and pure code. See how each framework compares.`]}),`\n`,_jsx(CardGroup,{children:_jsx(Card,{title:`Comparing Agent FrameworksArize AI`,href:`https://arize.com/blog-course/llm-agent-how-to-set-up/comparing-agent-frameworks`,cta:`Arize AI`,icon:`microchip`})}),`\n`,_jsx(Heading,{level:`2`,id:`testing-generation-in-rag`,children:`Testing Generation in RAG`}),`\n`,_jsx(Frame,{children:_jsx(OptimizedFrame,{width:`100%`,height:`420`,src:`https://cdn.iframe.ly/ij3ahLhV`,title:`LLM Evals and LLM as a Judge: Fundamentals`,frameborder:`0`,allow:`accelerometer; autoplay; clipboard-write; encrypted-media; gyroscope; picture-in-picture; web-share`,referrerpolicy:`strict-origin-when-cross-origin`,allowfullscreen:true})}),`\n`,_jsx(_components.p,{children:`Testing the generation stage of RAG across GPT-4 and Claude 2.1.`}),`\n`,_jsx(CardGroup,{children:_jsx(Card,{title:`Evaluating the Generation Stage in RAGArize AI`,href:`https://arize.com/blog/evaluating-the-generation-stage-in-rag`,cta:`Arize AI`,icon:`code`})})]})}function MDXContent(props={}){const{wrapper:MDXLayout}={..._provideComponents(),...props.components};return MDXLayout?_jsx(MDXLayout,{...props,children:_jsx(_createMdxContent,{...props})}):_createMdxContent(props)}return{default:MDXContent};function _missingMdxReference(id,component){throw new Error(`Expected `+(component?`component`:`object`)+\" `\"+id+\"` to be defined: you likely forgot to import, pass, or provide it.\")}","frontmatter":{},"scope":{"codeStyling":{"theme":{"light":"github-light-default","dark":"github-dark-default"}},"config":{},"pageMetadata":{"title":"AI Research","description":"Collection of advanced experiments and benchmarks in LLM evaluation, instrumentation, and agent systems","href":"/ax/cookbooks/ai-research"}}},"mdxSourceWithNoJs":null},"mdxExtracts":{"tableOfContents":[{"title":"Time Series Evals with OpenAI o1-preview","slug":"time-series-evals-with-openai-o1-preview","depth":2,"children":[]},{"title":"Prompt Caching Benchmarking","slug":"prompt-caching-benchmarking","depth":2,"children":[]},{"title":"Multi-Agent Systems: Swarm","slug":"multi-agent-systems-swarm","depth":2,"children":[]},{"title":"Instrumenting LLMs with OTel","slug":"instrumenting-llms-with-otel","depth":2,"children":[]},{"title":"Comparing Agent Frameworks","slug":"comparing-agent-frameworks","depth":2,"children":[]},{"title":"Testing Generation in RAG","slug":"testing-generation-in-rag","depth":2,"children":[]}],"codeExamples":{}},"jsLinks":["/docs/_mintlify/custom/arize-ax/6a30b7a67da02161/analytics.js","/docs/_mintlify/custom/arize-ax/67a4d27b62fababa/product-selector.js"],"subdomain":"arize-ax","siteName":"Arize AX Docs","title":"AI Research - Arize AX Docs","navMaps":{"firstHrefInVersion":{},"firstHrefInLanguage":{}},"lastModified":"2026-01-29T00:37:11.973Z","originalFileLocation":"ax/cookbooks/ai-research.mdx"},"headHtml":"\u003cscript data-mint-head>(function(){var d=document.documentElement,p=null;if(!p){try{p=localStorage.getItem(\"isDarkMode\")}catch(e){}if(p==='true')p='dark';else if(p==='false')p='light';if(p!=='light'\u0026\u0026p!=='dark'\u0026\u0026p!=='system')p=\"dark\"}var r=p==='system'?(matchMedia(\"(prefers-color-scheme: dark)\").matches?'dark':'light'):p;d.classList.remove(r==='dark'?'light':'dark');d.classList.add(r);d.setAttribute(\"data-theme-preference\",p)})();\u003c/script>\u003cmeta data-mint-head name=\"description\" content=\"Collection of advanced experiments and benchmarks in LLM evaluation, instrumentation, and agent systems\">\u003cmeta data-mint-head name=\"application-name\" content=\"Arize AX Docs\">\u003cmeta data-mint-head name=\"generator\" content=\"Mintlify\">\u003clink data-mint-head rel=\"icon\" type=\"image/png\" sizes=\"192x192\" href=\"/docs/mintlify-assets/_mintlify/favicons/arize-ax/vxzD83QsbAzzm43O/_generated/favicon/android-chrome-192x192.png\" media=\"(prefers-color-scheme: light)\">\u003clink data-mint-head rel=\"apple-touch-icon\" type=\"image/png\" sizes=\"180x180\" href=\"/docs/mintlify-assets/_mintlify/favicons/arize-ax/vxzD83QsbAzzm43O/_generated/favicon/apple-touch-icon.png\">\u003clink data-mint-head rel=\"icon\" type=\"image/png\" sizes=\"16x16\" href=\"/docs/mintlify-assets/_mintlify/favicons/arize-ax/vxzD83QsbAzzm43O/_generated/favicon/favicon-16x16.png\" media=\"(prefers-color-scheme: light)\">\u003clink data-mint-head rel=\"icon\" type=\"image/png\" sizes=\"32x32\" href=\"/docs/mintlify-assets/_mintlify/favicons/arize-ax/vxzD83QsbAzzm43O/_generated/favicon/favicon-32x32.png\" media=\"(prefers-color-scheme: light)\">\u003clink data-mint-head rel=\"shortcut icon\" type=\"image/x-icon\" href=\"/docs/mintlify-assets/_mintlify/favicons/arize-ax/vxzD83QsbAzzm43O/_generated/favicon/favicon.ico\" media=\"(prefers-color-scheme: light)\">\u003clink data-mint-head rel=\"icon\" type=\"image/png\" sizes=\"192x192\" href=\"/docs/mintlify-assets/_mintlify/favicons/arize-ax/vxzD83QsbAzzm43O/_generated/favicon-dark/android-chrome-192x192.png\" media=\"(prefers-color-scheme: dark)\">\u003clink data-mint-head rel=\"icon\" type=\"image/png\" sizes=\"16x16\" href=\"/docs/mintlify-assets/_mintlify/favicons/arize-ax/vxzD83QsbAzzm43O/_generated/favicon-dark/favicon-16x16.png\" media=\"(prefers-color-scheme: dark)\">\u003clink data-mint-head rel=\"icon\" type=\"image/png\" sizes=\"32x32\" href=\"/docs/mintlify-assets/_mintlify/favicons/arize-ax/vxzD83QsbAzzm43O/_generated/favicon-dark/favicon-32x32.png\" media=\"(prefers-color-scheme: dark)\">\u003clink data-mint-head rel=\"shortcut icon\" type=\"image/x-icon\" href=\"/docs/mintlify-assets/_mintlify/favicons/arize-ax/vxzD83QsbAzzm43O/_generated/favicon-dark/favicon.ico\" media=\"(prefers-color-scheme: dark)\">\u003cmeta data-mint-head property=\"og:title\" content=\"AI Research - Arize AX Docs\">\u003cmeta data-mint-head property=\"og:description\" content=\"Collection of advanced experiments and benchmarks in LLM evaluation, instrumentation, and agent systems\">\u003cmeta data-mint-head property=\"og:site_name\" content=\"Arize AX Docs\">\u003cmeta data-mint-head property=\"og:url\" content=\"https://arize.com/docs/ax/cookbooks/ai-research\">\u003cmeta data-mint-head property=\"og:type\" content=\"website\">\u003cmeta data-mint-head property=\"og:image\" content=\"https://arize-ax.mintlify.site/docs/_mintlify/api/og/v2?division=Advanced+Workflows\u0026amp;title=AI+Research\u0026amp;description=Collection+of+advanced+experiments+and+benchmarks+in+LLM+evaluation%2C+instrumentation%2C+and+agent+systems\u0026amp;theme=6b2e1c1a14346acb1262754f\u0026amp;signature=5a483c01e2127a8617c88770ad087c7de81e8b467e4700219579eca7c05a7039\">\u003cmeta data-mint-head property=\"og:image:width\" content=\"1200\">\u003cmeta data-mint-head property=\"og:image:height\" content=\"630\">\u003cmeta data-mint-head name=\"twitter:card\" content=\"summary_large_image\">\u003cmeta data-mint-head name=\"twitter:title\" content=\"AI Research - Arize AX Docs\">\u003cmeta data-mint-head name=\"twitter:description\" content=\"Collection of advanced experiments and benchmarks in LLM evaluation, instrumentation, and agent systems\">\u003cmeta data-mint-head name=\"twitter:image\" content=\"https://arize-ax.mintlify.site/docs/_mintlify/api/og/v2?division=Advanced+Workflows\u0026amp;title=AI+Research\u0026amp;description=Collection+of+advanced+experiments+and+benchmarks+in+LLM+evaluation%2C+instrumentation%2C+and+agent+systems\u0026amp;theme=6b2e1c1a14346acb1262754f\u0026amp;signature=5a483c01e2127a8617c88770ad087c7de81e8b467e4700219579eca7c05a7039\">\u003cmeta data-mint-head name=\"twitter:image:width\" content=\"1200\">\u003cmeta data-mint-head name=\"twitter:image:height\" content=\"630\">\u003cmeta data-mint-head name=\"msapplication-config\" content=\"/docs/mintlify-assets/_mintlify/favicons/arize-ax/vxzD83QsbAzzm43O/_generated/favicon/browserconfig.xml\">\u003cmeta data-mint-head name=\"apple-mobile-web-app-title\" content=\"Arize AX Docs\">\u003cmeta data-mint-head name=\"msapplication-TileColor\" content=\"#7A0047\">\u003clink data-mint-head rel=\"canonical\" href=\"https://arize.com/docs/ax/cookbooks/ai-research\">\u003clink data-mint-head rel=\"alternate\" type=\"application/xml\" href=\"/docs/sitemap.xml\">\u003clink data-mint-head rel=\"alternate\" type=\"text/markdown\" href=\"/docs/ax/cookbooks/ai-research.md\">\u003cscript data-mint-head type=\"application/ld+json\">{\"@context\":\"https://schema.org\",\"@graph\":[{\"@type\":\"Organization\",\"@id\":\"https://arize.com/#organization\",\"name\":\"Arize AX Docs\",\"url\":\"https://arize.com\",\"logo\":{\"@type\":\"ImageObject\",\"url\":\"https://mintcdn.com/arize-ax/V-uvingrGLOCK4VD/logo/light.svg?fit=max\u0026auto=format\u0026n=V-uvingrGLOCK4VD\u0026q=85\u0026s=cc58ee9388d5b5253d39b65cc662908f\"}},{\"@type\":\"WebSite\",\"@id\":\"https://arize.com/docs#website\",\"name\":\"Arize AX Docs\",\"url\":\"https://arize.com/docs\",\"publisher\":{\"@id\":\"https://arize.com/#organization\"}},{\"@type\":\"WebPage\",\"@id\":\"https://arize.com/docs/ax/cookbooks/ai-research#webpage\",\"url\":\"https://arize.com/docs/ax/cookbooks/ai-research\",\"name\":\"AI Research\",\"description\":\"Collection of advanced experiments and benchmarks in LLM evaluation, instrumentation, and agent systems\",\"dateModified\":\"2026-01-29T00:37:11.973Z\",\"isPartOf\":{\"@id\":\"https://arize.com/docs#website\"},\"breadcrumb\":{\"@id\":\"https://arize.com/docs/ax/cookbooks/ai-research#breadcrumb\"}},{\"@type\":\"BreadcrumbList\",\"@id\":\"https://arize.com/docs/ax/cookbooks/ai-research#breadcrumb\",\"itemListElement\":[{\"@type\":\"ListItem\",\"position\":1,\"name\":\"Advanced Workflows\",\"item\":\"https://arize.com/docs/ax/cookbooks/advanced-workflows/overview\"},{\"@type\":\"ListItem\",\"position\":2,\"name\":\"AI Research\",\"item\":\"https://arize.com/docs/ax/cookbooks/ai-research\"}]},{\"@type\":[\"Article\",\"TechArticle\"],\"@id\":\"https://arize.com/docs/ax/cookbooks/ai-research#article\",\"headline\":\"AI Research\",\"name\":\"AI Research\",\"description\":\"Collection of advanced experiments and benchmarks in LLM evaluation, instrumentation, and agent systems\",\"url\":\"https://arize.com/docs/ax/cookbooks/ai-research\",\"mainEntityOfPage\":{\"@id\":\"https://arize.com/docs/ax/cookbooks/ai-research#webpage\"},\"dateModified\":\"2026-01-29T00:37:11.973Z\",\"publisher\":{\"@id\":\"https://arize.com/#organization\"},\"isPartOf\":{\"@id\":\"https://arize.com/docs#website\"}}]}\u003c/script>","agentPointerHref":"/docs/ax/cookbooks/ai-research.md","siteName":"Arize AX Docs","nav":{"global":null,"tabs":[{"tab":"Get Started","pages":[{"title":"Get Started","href":"/ax"}]},{"tab":"Guides","pages":[{"title":"Overview","sidebarTitle":"Overview","href":"/ax/cookbooks"},{"group":"Instrument","pages":[{"title":"Instrument","sidebarTitle":"Overview","href":"/ax/cookbooks/instrument/overview"},{"group":"Instrument Agents Built with Anthropic","pages":[{"title":"Agents Built with the Claude Agent SDK","href":"/ax/cookbooks/instrument/claude-agent-sdk-guide"},{"title":"Trace and Evaluate Claude Managed Agents","href":"/ax/cookbooks/instrument/claude-managed-agents-guide"},{"title":"Agents Built with the Anthropic SDK","href":"/ax/cookbooks/instrument/anthropic-sdk-guide"}]},{"group":"Instrument agents built with OpenAI","pages":[{"title":"Trace and Evaluate an Agent Built with the OpenAI Agents SDK","sidebarTitle":"Agents built with the OpenAI Agents SDK","href":"/ax/cookbooks/instrument/openai-agents-cookbook"},{"title":"Trace an Agent Built with the OpenAI SDK","sidebarTitle":"Agents built with the OpenAI SDK","href":"/ax/cookbooks/instrument/openai-sdk-guide"}]},{"title":"Instrument LangChain agents","href":"/ax/cookbooks/instrument/langchain-guide"},{"title":"Instrument AI Coding Agents","href":"/ax/cookbooks/instrument/tracing-coding-agents"},{"title":"Instrument Vercel Eve Agents","href":"/ax/cookbooks/instrument/tracing-a-vercel-eve-agent"},{"title":"Instrument Mastra agents","href":"/ax/cookbooks/instrument/mastra-guide"},{"title":"Instrument Groq agents","href":"/ax/cookbooks/instrument/groq-sdk-guide"},{"title":"Instrument LiteLLM agents","href":"/ax/cookbooks/instrument/litellm-guide"},{"title":"Instrument Google ADK agents","href":"/ax/cookbooks/instrument/google-adk-guide"},{"title":"Instrument \u0026 Evaluate a Customer Support Agent","href":"/ax/cookbooks/instrument/tracing-and-evaluating-agents"},{"title":"OpenInference Best Practices","href":"/ax/cookbooks/instrument/openinference-best-practice"},{"title":"Send Telemetry to Arize AX and Dynatrace","sidebarTitle":"Export to Arize AX and Dynatrace","href":"/ax/cookbooks/instrument/send-telemetry-to-arize-and-dynatrace"},{"title":"Tracing Integrations","href":"/ax/cookbooks/tracing-integrations"}]},{"group":"Observe","pages":[{"title":"Observe","sidebarTitle":"Overview","href":"/ax/cookbooks/observe/overview"},{"title":"Cluster trace patterns with managed agents","href":"/ax/cookbooks/observe/cluster-trace-patterns-with-managed-agents"},{"title":"Online Evals \u0026 Monitoring for Agents in Production (Mosaic AI)","href":"/ax/cookbooks/observe/arize-+-mosaic-ai-agent-framework"},{"title":"Designing Realtime Guardrails: Input, Output, and the Cost of Blocking","href":"/ax/cookbooks/observe/designing-realtime-guardrails"}]},{"group":"Evaluate","pages":[{"title":"Evaluate","sidebarTitle":"Overview","href":"/ax/cookbooks/evaluate/overview"},{"title":"Evaluations Quickstart","href":"/ax/cookbooks/evaluate/evaluations-quickstart"},{"title":"Align LLM Evals with Human Judgment","href":"/ax/cookbooks/evaluate/align-llm-evals-with-human-judgment"},{"title":"Why Public Benchmarks Lie: Building Your Own Eval Harness","href":"/ax/cookbooks/evaluate/model-comparison-for-an-email-text-extraction-service"},{"title":"Trace-Level Evaluation: Beyond Input/Output Checks","href":"/ax/cookbooks/evaluate/trace-level-evaluations-for-a-recommendation-agent"},{"title":"Session-Level Evaluation: Scoring the Whole Conversation","href":"/ax/cookbooks/evaluate/session-level-evaluations-for-an-ai-tutor"},{"title":"Evaluating RAG Retrieval Quality and Correctness","href":"/ax/cookbooks/evaluate/evaluating-rag"},{"title":"Evaluate a Math Problem-Solving Agent Using Ragas","href":"/ax/cookbooks/evaluate/ragas-agents-cookbook"},{"title":"Pydantic Evals","href":"/ax/cookbooks/evaluate/pydantic-evals"},{"title":"Tracing and Evaluating Voice Applications","href":"/ax/cookbooks/evaluate/tracing-and-evaluating-audio"},{"title":"Audio Transcription And Evaluation With Gemini Flash","href":"/ax/cookbooks/evaluate/gemini-audio-evals"},{"title":"Evaluate Receipt Agents with an Image Judge","href":"/ax/cookbooks/evaluate/evaluate-receipt-agents-with-image-judge"},{"title":"Use Jev as a Remote Evaluator","href":"/ax/cookbooks/evaluate/jev-remote-evaluator"}]},{"group":"Improve","pages":[{"title":"Improve","sidebarTitle":"Overview","href":"/ax/cookbooks/improve/overview"},{"title":"Build, Test, and Optimize a Trip-Planner Prompt","href":"/ax/cookbooks/improve/trip-planner-build-test-optimize"},{"title":"Prompt Experimentation For Summarization Task","href":"/ax/cookbooks/improve/summarization"},{"title":"Text2SQL Application for Database Querying","href":"/ax/cookbooks/improve/text2sql"},{"title":"Improving Structured Output Generation with Prompt Learning","href":"/ax/cookbooks/improve/improving-structured-output-generation-with-prompt-learning"},{"title":"Optimizing Coding Agent Prompts for Planning","href":"/ax/cookbooks/improve/optimizing-coding-agent-prompts-for-planning"},{"title":"Optimizing Coding Agent Prompts for Execution","href":"/ax/cookbooks/improve/optimizing-coding-agent-prompts-for-execution"},{"title":"Observe and Optimize Coding Agent Workflows","sidebarTitle":"Observe and Optimize Coding Agent Workflows","href":"/ax/cookbooks/improve/observe-and-optimize-coding-agent-workflows"},{"title":"Find and Fix Issues in Your AI App with a Coding Agent","sidebarTitle":"Find and Fix Issues with a Coding Agent","href":"/ax/cookbooks/improve/find-and-fix-issues-with-a-coding-agent"},{"title":"Optimizing Your Eval Prompts","href":"/ax/cookbooks/improve/optimizing-your-eval-prompts"}]},{"group":"Advanced Workflows","pages":[{"title":"Advanced Workflows","sidebarTitle":"Overview","href":"/ax/cookbooks/advanced-workflows/overview"},{"title":"Evaluating Agentic RAG Using Arize AX And Couchbase","href":"/ax/cookbooks/advanced-workflows/evaluating-agentic-rag-using-arize-and-couchbase"},{"title":"Product Recommendation Agent: Google Agent Engine \u0026 LangGraph","href":"/ax/cookbooks/advanced-workflows/tracing-a-langgraph-application-with-agent-engine-in-vertex-ai"},{"title":"A2A Financial Trading Agents - Google ADK / MCP / Llama","href":"/ax/cookbooks/advanced-workflows/tracing-a2a-agent"},{"title":"Multi-modal Autonomous Browser Agent with LLama Models ","href":"/ax/cookbooks/advanced-workflows/trace-evaluate-browser-use-agent-with-l-lama4"},{"title":"Trace LangChain Agent \u0026 Microsoft Risk+Safety Evaluators (Microsoft Foundry)","href":"/ax/cookbooks/advanced-workflows/evaluating-and-improving-ai-agents-at-scale-with-microsoft-foundry"},{"title":"Trace Red Teaming Agent (Microsoft Foundry)","href":"/ax/cookbooks/advanced-workflows/foundry-red-team"},{"title":"Jailbreak and Prompt Injection Defense","href":"/ax/cookbooks/advanced-workflows/jailbreak-and-prompt-injection-defense"},{"title":"LLM Red Teaming","href":"/ax/security-and-settings/llm-security/llm-red-teaming"},{"title":"Dual Tracing into Databricks Unity Catalog and Arize AX","description":null,"href":"/ax/cookbooks/instrument/dual-o-tel-ingest-into-unity-catalog-and-arize-ax"},{"title":"AI Research","href":"/ax/cookbooks/ai-research"}]}]},{"tab":"Integrations","pages":[{"title":"Integrations","href":"/ax/integrations"}]},{"tab":"APIs, SDKs, Skills \u0026 MCP","pages":[{"title":"APIs, SDKs, Skills \u0026 MCP","href":"/api-reference/overview"}]},{"tab":"Courses","pages":[{"title":"Courses","href":"/ax/learn/courses/overview"}]},{"tab":"Concepts","pages":[{"title":"Concepts","href":"/ax/concepts/otel-openinference/overview"}]},{"tab":"Self-Hosting","pages":[{"title":"Self-Hosting","href":"/ax/selfhosting"}]},{"tab":"Changelog","pages":[{"title":"Changelog","href":"/ax/release-notes"}]}]},"basePath":"","title":"AI Research - Arize AX Docs","buildId":"dpl_Aw98ji4XvBsgtZwL1VrkUF9ehFqz","deploymentId":"dpl_Aw98ji4XvBsgtZwL1VrkUF9ehFqz","dynamics":["07c022d89b"],"chunks":["Frame-0domioUa.js","DynamicMarkdown-BcGRTrw6.js"]}