95 lines
70 KiB
HTML
95 lines
70 KiB
HTML
<!doctype html><script>(function(e,t,n,s,o){e[s]=e[s]||[],e[s].push({"gtm.start":(new Date).getTime(),event:"gtm.js"});var a=t.getElementsByTagName(n)[0],i=t.createElement(n),r=s!="dataLayer"?"&l="+s:"";i.async=!0,i.src="https://www.googletagmanager.com/gtm.js?id="+o+r,a.parentNode.insertBefore(i,a)})(window,document,"script","dataLayer","GTM-W9HPZZ22")</script><html lang=zh-cn><head><meta charset=utf-8><title>使用 Xinference 接入本地模型 | FastGPT</title><meta name=viewport content="width=device-width,initial-scale=1"><meta name=description content="一站式本地 LLM 私有化部署"><meta name=keywords content="Documentation,Hugo,Hugo Theme,Bootstrap"><meta name=author content="Colin Wilson - Lotus Labs"><meta name=email content="support@aigis.uk"><meta name=website content="https://lotusdocs.dev"><meta name=Version content="v0.1.0"><link rel=icon href=https://doc.tryfastgpt.ai/favicon.ico sizes=any><link rel=icon type=image/svg+xml href=https://doc.tryfastgpt.ai/favicon.svg><link rel=apple-touch-icon sizes=180x180 href=https://doc.tryfastgpt.ai/apple-touch-icon.png><link rel=icon type=image/png sizes=32x32 href=https://doc.tryfastgpt.ai/favicon-32x32.png><link rel=icon type=image/png sizes=16x16 href=https://doc.tryfastgpt.ai/favicon-16x16.png><link rel=manifest crossorigin=use-credentials href=https://doc.tryfastgpt.ai/site.webmanifest><meta property="og:title" content="使用 Xinference 接入本地模型"><meta property="og:description" content="一站式本地 LLM 私有化部署"><meta property="og:type" content="article"><meta property="og:url" content="https://doc.tryfastgpt.ai/docs/development/custom-models/xinference/"><meta property="og:image" content="https://doc.tryfastgpt.ai/opengraph/card-base-2_hu06b1a92291a380a0d2e0ec03dab66b2f_17642_filter_12080930744490214278.png"><meta property="article:section" content="docs"><meta name=twitter:card content="summary_large_image"><meta name=twitter:image content="https://doc.tryfastgpt.ai/opengraph/card-base-2_hu06b1a92291a380a0d2e0ec03dab66b2f_17642_filter_12080930744490214278.png"><meta name=twitter:title content="使用 Xinference 接入本地模型"><meta name=twitter:description content="一站式本地 LLM 私有化部署"><script>(()=>{var t=window.matchMedia&&window.matchMedia("(prefers-color-scheme: dark)").matches,e=localStorage.getItem("theme");t&&e===null&&(localStorage.setItem("theme","dark"),document.documentElement.setAttribute("data-dark-mode","")),t&&e==="dark"&&document.documentElement.setAttribute("data-dark-mode",""),e==="dark"&&document.documentElement.setAttribute("data-dark-mode","")})()</script><link rel=preconnect href=https://gstatic.loli.net/><link rel=preconnect href=https://gstatic.loli.net/ crossorigin><link href="https://fonts.loli.net/css?family=Inter:300,400,600,700|Fira+Code:500,700|JetBrains+Mono:500,700&display=block" rel=stylesheet><link rel=stylesheet href=/docs/scss/style.min.0ec826a837202d1bb0cc402457fc3b09de1d00673bd1fea8591ce85da15706ba66db8c4b0243747a2179524215798bcd.css integrity=sha384-DsgmqDcgLRuwzEAkV/w7Cd4dAGc70f6oWRzoXaFXBrpm24xLAkN0eiF5UkIVeYvN crossorigin=anonymous><script data-id=umami-script async src=https://umami.fastgpt.in/fastgpt data-website-id=69b3f2c6-39a5-4937-b92d-6fdd5c8f5958></script><script type=text/javascript>document.querySelector('script[data-id="umami-script"]').addEventListener("load",function(){const e=document.head.querySelector('meta[property = "og:type"]').getAttribute("content");let t=document.head.querySelector('meta[property = "og:title"]').getAttribute("content"),n=document.head.querySelector('meta[property = "og:url"]').getAttribute("content");umami.track(e+":"+t,{url:n})})</script><script defer>function loadAnalyticsScript(){var e=document.createElement("script");e.async=!0,e.src="https://gaxy.fastgpt.in/gtag/js?id=G-KGYF44Y7TT",document.head.appendChild(e),window.dataLayer=window.dataLayer||[];function t(){dataLayer.push(arguments)}t("js",new Date),t("config","G-KGYF44Y7TT"),window.removeEventListener("scroll",onScrollLoadAnalytics)}function onScrollLoadAnalytics(){window.scrollY>100&&loadAnalyticsScript()}window.addEventListener("scroll",onScrollLoadAnalytics)</script><script defer>function loadScriptOnScroll(){var e,t,n,s=window.scrollY||window.pageYOffset;s>100&&(n=n||[],e=document.createElement("script"),e.src="https://hm.baidu.com/hm.js?537c7867df8f9ae2e7147284ec0dbfdd",t=document.getElementsByTagName("script")[0],t.parentNode.insertBefore(e,t),window.removeEventListener("scroll",loadScriptOnScroll))}window.addEventListener("scroll",loadScriptOnScroll)</script><script defer>function loadClarityScript(){(function(e,t,n,s,o,i,a){e[n]=e[n]||function(){(e[n].q=e[n].q||[]).push(arguments)},i=t.createElement(s),i.async=1,i.src="https://www.clarity.ms/tag/"+o,a=t.getElementsByTagName(s)[0],a.parentNode.insertBefore(i,a)})(window,document,"clarity","script","mjgug8s1pf"),window.removeEventListener("scroll",onScrollLoadClarity)}function onScrollLoadClarity(){window.scrollY>100&&loadClarityScript()}window.addEventListener("scroll",onScrollLoadClarity)</script><link rel=preload href=https://cdn.jsdelivr.net/npm/lxgw-wenkai-screen-webfont@1.1.0/style.css as=style><link rel=stylesheet href=https://cdn.jsdelivr.net/npm/lxgw-wenkai-screen-webfont@1.1.0/style.css></head><body><noscript><iframe src="https://www.googletagmanager.com/ns.html?id=GTM-W9HPZZ22" height=0 width=0 style=display:none;visibility:hidden></iframe></noscript><div class=content><div class="page-wrapper toggled"><nav id=sidebar class=sidebar-wrapper><div class=sidebar-brand><a href=https://tryfastgpt.ai/ aria-label=HomePage alt=HomePage style=text-transform:unset><svg width="26" height="26" viewBox="0 0 1041 1348" fill="none" xmlns="http://www.w3.org/2000/svg"><path d="M340.837.33933 681.068.338989V.455643C684.032.378397 686.999.339702 689.967.339702 735.961.3397 781.504 9.62899 823.997 27.6772 866.49 45.7254 905.099 72.1791 937.622 105.528 970.144 138.877 995.942 178.467 1013.54 222.04 1031.14 265.612 1040.2 312.312 1040.2 359.474H340.836V1347.84C296.157 1347.84 251.914 1338.55 210.636 1320.49 169.357 1302.43 131.85 1275.95 100.257 1242.58 68.6636 1209.21 43.6023 1169.59 26.5041 1125.99 11.3834 1087.43 2.75216 1046.42.957956 1004.81H.605869L.605897 368.098H.70363c-.597878-26.267 1.53378-52.655 6.43943-78.687C20.2709 219.745 52.6748 155.754 100.257 105.528 147.839 55.3017 208.462 21.0975 274.461 7.24017 296.426 2.62833 318.657.339101 340.837.33933z" fill="url(#paint0_linear_1172_228)"/><path d="M633.639 904.645H513.029V576.37H635.422V576.377C678.161 576.607 720.454 585.093 759.951 601.37 799.997 617.874 836.384 642.064 867.033 672.559c30.65 30.495 54.963 66.698 71.55 106.542C955.171 818.944 963.709 861.648 963.709 904.775H633.639V904.645z" fill="url(#paint1_linear_1172_228)"/><defs><linearGradient id="paint0_linear_1172_228" x1="520.404" y1=".338989" x2="520.404" y2="1347.84" gradientUnits="userSpaceOnUse"><stop stop-color="#326dff"/><stop offset="1" stop-color="#8eaeff"/></linearGradient><linearGradient id="paint1_linear_1172_228" x1="738.369" y1="576.37" x2="738.369" y2="904.775" gradientUnits="userSpaceOnUse"><stop stop-color="#326dff"/><stop offset="1" stop-color="#8eaeff"/></linearGradient></defs></svg>
|
||
FastGPT</a></div><div class=sidebar-content style="height:calc(100% - 131px)"><ul class=sidebar-menu><li><a class=sidebar-root-link href=https://doc.tryfastgpt.ai/docs/intro/><i class="material-icons me-2">rocket_launch</i>
|
||
快速了解 FastGPT</a></li><li class=sidebar-dropdown><button class=btn>
|
||
<i class="material-icons me-2">import_contacts</i>
|
||
功能介绍</button><div class=sidebar-submenu><ul><li class="sidebar-dropdown nested"><button class=btn>
|
||
基础教程</button><div class=sidebar-submenu><ul><li><a class=sidebar-nested-link href=https://doc.tryfastgpt.ai/docs/guide/course/quick-start/>快速上手</a></li><li><a class=sidebar-nested-link href=https://doc.tryfastgpt.ai/docs/guide/course/ai_settings/>AI 相关参数配置说明</a></li><li><a class=sidebar-nested-link href=https://doc.tryfastgpt.ai/docs/guide/course/chat_input_guide/>对话问题引导</a></li><li><a class=sidebar-nested-link href=https://doc.tryfastgpt.ai/docs/guide/course/collection_tags/>知识库集合标签</a></li><li><a class=sidebar-nested-link href=https://doc.tryfastgpt.ai/docs/guide/course/fileinput/>文件输入功能介绍</a></li></ul></div></li><li class="sidebar-dropdown nested"><button class=btn>
|
||
工作台</button><div class=sidebar-submenu><ul><li><a class=sidebar-nested-link href=https://doc.tryfastgpt.ai/docs/guide/dashboard/basic-mode/>简易模式</a></li><li><a class=sidebar-nested-link href=https://doc.tryfastgpt.ai/docs/guide/dashboard/intro/>工作流&插件</a></li><li class="sidebar-dropdown nested"><button class=btn>
|
||
工作流节点</button><div class=sidebar-submenu><ul><li><a class=sidebar-nested-link href=https://doc.tryfastgpt.ai/docs/guide/dashboard/workflow/ai_chat/>AI 对话</a></li><li><a class=sidebar-nested-link href=https://doc.tryfastgpt.ai/docs/guide/dashboard/workflow/dataset_search/>知识库搜索</a></li><li><a class=sidebar-nested-link href=https://doc.tryfastgpt.ai/docs/guide/dashboard/workflow/tool/>工具调用&终止</a></li><li><a class=sidebar-nested-link href=https://doc.tryfastgpt.ai/docs/guide/dashboard/workflow/question_classify/>问题分类</a></li><li><a class=sidebar-nested-link href=https://doc.tryfastgpt.ai/docs/guide/dashboard/workflow/content_extract/>文本内容提取</a></li><li><a class=sidebar-nested-link href=https://doc.tryfastgpt.ai/docs/guide/dashboard/workflow/user-selection/>用户选择</a></li><li><a class=sidebar-nested-link href=https://doc.tryfastgpt.ai/docs/guide/dashboard/workflow/form_input/>表单输入</a></li><li><a class=sidebar-nested-link href=https://doc.tryfastgpt.ai/docs/guide/dashboard/workflow/text_editor/>文本拼接</a></li><li><a class=sidebar-nested-link href=https://doc.tryfastgpt.ai/docs/guide/dashboard/workflow/reply/>指定回复</a></li><li><a class=sidebar-nested-link href=https://doc.tryfastgpt.ai/docs/guide/dashboard/workflow/document_parsing/>文档解析</a></li><li><a class=sidebar-nested-link href=https://doc.tryfastgpt.ai/docs/guide/dashboard/workflow/http/>HTTP 请求</a></li><li><a class=sidebar-nested-link href=https://doc.tryfastgpt.ai/docs/guide/dashboard/workflow/tfswitch/>判断器</a></li><li><a class=sidebar-nested-link href=https://doc.tryfastgpt.ai/docs/guide/dashboard/workflow/variable_update/>变量更新</a></li><li><a class=sidebar-nested-link href=https://doc.tryfastgpt.ai/docs/guide/dashboard/workflow/sandbox/>代码运行</a></li><li><a class=sidebar-nested-link href=https://doc.tryfastgpt.ai/docs/guide/dashboard/workflow/loop/>批量运行</a></li><li><a class=sidebar-nested-link href=https://doc.tryfastgpt.ai/docs/guide/dashboard/workflow/knowledge_base_search_merge/>知识库搜索引用合并</a></li><li><a class=sidebar-nested-link href=https://doc.tryfastgpt.ai/docs/guide/dashboard/workflow/coreferenceresolution/>问题优化</a></li><li><a class=sidebar-nested-link href=https://doc.tryfastgpt.ai/docs/guide/dashboard/workflow/laf/>Laf 函数调用</a></li><li><a class=sidebar-nested-link href=https://doc.tryfastgpt.ai/docs/guide/dashboard/workflow/custom_feedback/>自定义反馈</a></li></ul></div></li><li><a class=sidebar-nested-link href=https://doc.tryfastgpt.ai/docs/guide/dashboard/mcp_server/>MCP 服务</a></li><li><a class=sidebar-nested-link href=https://doc.tryfastgpt.ai/docs/guide/dashboard/mcp_tools/>MCP 工具集</a></li><li><a class=sidebar-nested-link href=https://doc.tryfastgpt.ai/docs/guide/dashboard/gapier/>使用 Gapier 快速导入Agent工具</a></li></ul></div></li><li class="sidebar-dropdown nested"><button class=btn>
|
||
系统插件</button><div class=sidebar-submenu><ul><li><a class=sidebar-nested-link href=https://doc.tryfastgpt.ai/docs/guide/plugins/how_to_submit_system_plugin/>如何提交系统插件</a></li><li><a class=sidebar-nested-link href=https://doc.tryfastgpt.ai/docs/guide/plugins/searxng_plugin_guide/>SearXNG 搜索插件配置与使用说明</a></li><li><a class=sidebar-nested-link href=https://doc.tryfastgpt.ai/docs/guide/plugins/google_search_plugin_guide/>Google 搜索插件填写说明</a></li><li><a class=sidebar-nested-link href=https://doc.tryfastgpt.ai/docs/guide/plugins/bing_search_plugin/>Bing 搜索插件填写说明</a></li><li><a class=sidebar-nested-link href=https://doc.tryfastgpt.ai/docs/guide/plugins/doc2x_plugin_guide/>Doc2x 插件填写说明</a></li></ul></div></li><li class="sidebar-dropdown nested"><button class=btn>
|
||
知识库</button><div class=sidebar-submenu><ul><li><a class=sidebar-nested-link href=https://doc.tryfastgpt.ai/docs/guide/knowledge_base/rag/>知识库基础原理介绍</a></li><li><a class=sidebar-nested-link href=https://doc.tryfastgpt.ai/docs/guide/knowledge_base/dataset_engine/>知识库搜索方案和参数</a></li><li><a class=sidebar-nested-link href=https://doc.tryfastgpt.ai/docs/guide/knowledge_base/api_dataset/>API 文件库</a></li><li><a class=sidebar-nested-link href=https://doc.tryfastgpt.ai/docs/guide/knowledge_base/lark_dataset/>飞书知识库</a></li><li><a class=sidebar-nested-link href=https://doc.tryfastgpt.ai/docs/guide/knowledge_base/yuque_dataset/>语雀文件库</a></li><li><a class=sidebar-nested-link href=https://doc.tryfastgpt.ai/docs/guide/knowledge_base/websync/>Web 站点同步</a></li><li><a class=sidebar-nested-link href=https://doc.tryfastgpt.ai/docs/guide/knowledge_base/externalfile/>外部文件知识库</a></li></ul></div></li><li class="sidebar-dropdown nested"><button class=btn>
|
||
团队与权限</button><div class=sidebar-submenu><ul><li><a class=sidebar-nested-link href=https://doc.tryfastgpt.ai/docs/guide/team_permissions/team_roles_permissions/>团队&成员组&权限</a></li><li><a class=sidebar-nested-link href=https://doc.tryfastgpt.ai/docs/guide/team_permissions/invitation_link/>邀请链接说明文档</a></li></ul></div></li><li class="sidebar-dropdown nested"><button class=btn>
|
||
对话框</button><div class=sidebar-submenu><ul><li><a class=sidebar-nested-link href=https://doc.tryfastgpt.ai/docs/guide/dialogboxes/htmlrendering/>对话框与HTML渲染</a></li><li><a class=sidebar-nested-link href=https://doc.tryfastgpt.ai/docs/guide/dialogboxes/quotelist/>知识库引用分块阅读器</a></li></ul></div></li><li class="sidebar-dropdown nested"><button class=btn>
|
||
商业版后台</button><div class=sidebar-submenu><ul><li><a class=sidebar-nested-link href=https://doc.tryfastgpt.ai/docs/guide/admin/sso/>SSO & 外部成员同步</a></li><li><a class=sidebar-nested-link href=https://doc.tryfastgpt.ai/docs/guide/admin/teammode/>团队模式说明文档</a></li></ul></div></li></ul></div></li><li class=sidebar-dropdown><button class=btn>
|
||
<i class="material-icons me-2">cases</i>
|
||
使用案例</button><div class=sidebar-submenu><ul><li class="sidebar-dropdown nested"><button class=btn>
|
||
外部调用 FastGPT</button><div class=sidebar-submenu><ul><li><a class=sidebar-nested-link href=https://doc.tryfastgpt.ai/docs/use-cases/external-integration/openapi/>通过 API 访问应用</a></li><li><a class=sidebar-nested-link href=https://doc.tryfastgpt.ai/docs/use-cases/external-integration/feishu/>接入飞书机器人教程</a></li><li><a class=sidebar-nested-link href=https://doc.tryfastgpt.ai/docs/use-cases/external-integration/dingtalk/>接入钉钉机器人教程</a></li><li><a class=sidebar-nested-link href=https://doc.tryfastgpt.ai/docs/use-cases/external-integration/official_account/>接入微信公众号教程</a></li><li><a class=sidebar-nested-link href=https://doc.tryfastgpt.ai/docs/use-cases/external-integration/onwechat/>对接 chatgpt-on-wechat</a></li><li><a class=sidebar-nested-link href=https://doc.tryfastgpt.ai/docs/use-cases/external-integration/wechat/>接入微信和企业微信</a></li><li><a class=sidebar-nested-link href=https://doc.tryfastgpt.ai/docs/use-cases/external-integration/iframe_integration/>iframe 接入</a></li></ul></div></li><li class="sidebar-dropdown nested"><button class=btn>
|
||
应用搭建案例</button><div class=sidebar-submenu><ul><li><a class=sidebar-nested-link href=https://doc.tryfastgpt.ai/docs/use-cases/app-cases/submit_application_template/>如何提交应用模板</a></li><li><a class=sidebar-nested-link href=https://doc.tryfastgpt.ai/docs/use-cases/app-cases/translate-subtitle-using-gpt/>长字幕翻译</a></li><li><a class=sidebar-nested-link href=https://doc.tryfastgpt.ai/docs/use-cases/app-cases/multi_turn_translation_bot/>多轮翻译机器人</a></li><li><a class=sidebar-nested-link href=https://doc.tryfastgpt.ai/docs/use-cases/app-cases/english_essay_correction_bot/>英语作文纠错机器人</a></li><li><a class=sidebar-nested-link href=https://doc.tryfastgpt.ai/docs/use-cases/app-cases/fixingevidence/>固定开头和结尾内容</a></li><li><a class=sidebar-nested-link href=https://doc.tryfastgpt.ai/docs/use-cases/app-cases/lab_appointment/>实验室预约</a></li><li><a class=sidebar-nested-link href=https://doc.tryfastgpt.ai/docs/use-cases/app-cases/dalle3/>Dalle3 绘图</a></li><li><a class=sidebar-nested-link href=https://doc.tryfastgpt.ai/docs/use-cases/app-cases/google_search/>接入谷歌搜索</a></li><li><a class=sidebar-nested-link href=https://doc.tryfastgpt.ai/docs/use-cases/app-cases/feishu_webhook/>发送飞书webhook通知</a></li></ul></div></li></ul></div></li><li class="sidebar-dropdown current active"><button class=btn>
|
||
<i class="material-icons me-2">code_blocks</i>
|
||
开发与部署指南</button><div class="sidebar-submenu d-block"><ul><li><a class=sidebar-nested-link href=https://doc.tryfastgpt.ai/docs/development/intro/>快速开始本地开发</a></li><li><a class=sidebar-nested-link href=https://doc.tryfastgpt.ai/docs/development/sealos/>Sealos 一键部署</a></li><li><a class=sidebar-nested-link href=https://doc.tryfastgpt.ai/docs/development/docker/>Docker Compose 快速部署</a></li><li><a class=sidebar-nested-link href=https://doc.tryfastgpt.ai/docs/development/configuration/>配置文件介绍</a></li><li><a class=sidebar-nested-link href=https://doc.tryfastgpt.ai/docs/development/faq/>私有部署常见问题</a></li><li class="sidebar-dropdown nested"><button class=btn>
|
||
模型配置方案</button><div class=sidebar-submenu><ul><li><a class=sidebar-nested-link href=https://doc.tryfastgpt.ai/docs/development/modelconfig/intro/>FastGPT 模型配置说明</a></li><li><a class=sidebar-nested-link href=https://doc.tryfastgpt.ai/docs/development/modelconfig/ai-proxy/>通过 AI Proxy 接入模型</a></li><li><a class=sidebar-nested-link href=https://doc.tryfastgpt.ai/docs/development/modelconfig/one-api/>通过 OneAPI 接入模型</a></li><li><a class=sidebar-nested-link href=https://doc.tryfastgpt.ai/docs/development/modelconfig/siliconcloud/>通过 SiliconCloud 体验开源模型</a></li><li><a class=sidebar-nested-link href=https://doc.tryfastgpt.ai/docs/development/modelconfig/ppio/>通过 PPIO LLM API 接入模型</a></li></ul></div></li><li class="sidebar-dropdown nested"><button class=btn>
|
||
版本更新/升级操作</button><div class=sidebar-submenu><ul><li><a class=sidebar-nested-link href=https://doc.tryfastgpt.ai/docs/development/upgrading/intro/>升级说明</a></li><li><a class=sidebar-nested-link href=https://doc.tryfastgpt.ai/docs/development/upgrading/4911/>V4.9.11(进行中)</a></li><li><a class=sidebar-nested-link href=https://doc.tryfastgpt.ai/docs/development/upgrading/4910/>V4.9.10</a></li><li><a class=sidebar-nested-link href=https://doc.tryfastgpt.ai/docs/development/upgrading/499/>V4.9.9</a></li><li><a class=sidebar-nested-link href=https://doc.tryfastgpt.ai/docs/development/upgrading/498/>V4.9.8</a></li><li><a class=sidebar-nested-link href=https://doc.tryfastgpt.ai/docs/development/upgrading/497/>V4.9.7</a></li><li><a class=sidebar-nested-link href=https://doc.tryfastgpt.ai/docs/development/upgrading/496/>V4.9.6</a></li><li><a class=sidebar-nested-link href=https://doc.tryfastgpt.ai/docs/development/upgrading/495/>V4.9.5</a></li><li><a class=sidebar-nested-link href=https://doc.tryfastgpt.ai/docs/development/upgrading/494/>V4.9.4</a></li><li><a class=sidebar-nested-link href=https://doc.tryfastgpt.ai/docs/development/upgrading/493/>V4.9.3</a></li><li><a class=sidebar-nested-link href=https://doc.tryfastgpt.ai/docs/development/upgrading/492/>V4.9.2</a></li><li><a class=sidebar-nested-link href=https://doc.tryfastgpt.ai/docs/development/upgrading/491/>V4.9.1</a></li><li><a class=sidebar-nested-link href=https://doc.tryfastgpt.ai/docs/development/upgrading/490/>V4.9.0(包含升级脚本)</a></li><li><a class=sidebar-nested-link href=https://doc.tryfastgpt.ai/docs/development/upgrading/4823/>V4.8.23</a></li><li><a class=sidebar-nested-link href=https://doc.tryfastgpt.ai/docs/development/upgrading/4822/>V4.8.22(包含升级脚本)</a></li><li><a class=sidebar-nested-link href=https://doc.tryfastgpt.ai/docs/development/upgrading/4821/>V4.8.21</a></li><li><a class=sidebar-nested-link href=https://doc.tryfastgpt.ai/docs/development/upgrading/4820/>V4.8.20(包含升级脚本)</a></li><li><a class=sidebar-nested-link href=https://doc.tryfastgpt.ai/docs/development/upgrading/4819/>V4.8.19(包含升级脚本)</a></li><li><a class=sidebar-nested-link href=https://doc.tryfastgpt.ai/docs/development/upgrading/4818/>V4.8.18(包含升级脚本)</a></li><li><a class=sidebar-nested-link href=https://doc.tryfastgpt.ai/docs/development/upgrading/4817/>V4.8.17(包含升级脚本)</a></li><li><a class=sidebar-nested-link href=https://doc.tryfastgpt.ai/docs/development/upgrading/4816/>V4.8.16(更新配置文件)</a></li><li><a class=sidebar-nested-link href=https://doc.tryfastgpt.ai/docs/development/upgrading/4815/>V4.8.15(包含升级脚本)</a></li><li><a class=sidebar-nested-link href=https://doc.tryfastgpt.ai/docs/development/upgrading/4814/>V4.8.14</a></li><li><a class=sidebar-nested-link href=https://doc.tryfastgpt.ai/docs/development/upgrading/4813/>V4.8.13</a></li><li><a class=sidebar-nested-link href=https://doc.tryfastgpt.ai/docs/development/upgrading/4812/>V4.8.12(包含升级脚本)</a></li><li><a class=sidebar-nested-link href=https://doc.tryfastgpt.ai/docs/development/upgrading/4811/>V4.8.11（商业版初始化）</a></li><li><a class=sidebar-nested-link href=https://doc.tryfastgpt.ai/docs/development/upgrading/4810/>V4.8.10(包含升级脚本)</a></li><li><a class=sidebar-nested-link href=https://doc.tryfastgpt.ai/docs/development/upgrading/489/>V4.8.9（需要初始化）</a></li><li><a class=sidebar-nested-link href=https://doc.tryfastgpt.ai/docs/development/upgrading/488/>V4.8.8(包含升级脚本)</a></li><li><a class=sidebar-nested-link href=https://doc.tryfastgpt.ai/docs/development/upgrading/487/>V4.8.7</a></li><li><a class=sidebar-nested-link href=https://doc.tryfastgpt.ai/docs/development/upgrading/486/>V4.8.6(包含升级脚本)</a></li><li><a class=sidebar-nested-link href=https://doc.tryfastgpt.ai/docs/development/upgrading/485/>V4.8.5(包含升级脚本)</a></li><li><a class=sidebar-nested-link href=https://doc.tryfastgpt.ai/docs/development/upgrading/484/>V4.8.4(包含升级脚本)</a></li><li><a class=sidebar-nested-link href=https://doc.tryfastgpt.ai/docs/development/upgrading/483/>V4.8.3</a></li><li><a class=sidebar-nested-link href=https://doc.tryfastgpt.ai/docs/development/upgrading/482/>V4.8.2</a></li><li><a class=sidebar-nested-link href=https://doc.tryfastgpt.ai/docs/development/upgrading/481/>V4.8.1(包含升级脚本)</a></li><li><a class=sidebar-nested-link href=https://doc.tryfastgpt.ai/docs/development/upgrading/48/>V4.8</a></li><li><a class=sidebar-nested-link href=https://doc.tryfastgpt.ai/docs/development/upgrading/471/>V4.7.1(包含升级脚本)</a></li><li><a class=sidebar-nested-link href=https://doc.tryfastgpt.ai/docs/development/upgrading/47/>V4.7（需要初始化）</a></li><li><a class=sidebar-nested-link href=https://doc.tryfastgpt.ai/docs/development/upgrading/469/>V4.6.9(包含升级脚本)</a></li><li><a class=sidebar-nested-link href=https://doc.tryfastgpt.ai/docs/development/upgrading/468/>V4.6.8（需要初始化）</a></li><li><a class=sidebar-nested-link href=https://doc.tryfastgpt.ai/docs/development/upgrading/467/>V4.6.7（需要初始化）</a></li><li><a class=sidebar-nested-link href=https://doc.tryfastgpt.ai/docs/development/upgrading/466/>V4.6.6（需要改配置文件）</a></li><li><a class=sidebar-nested-link href=https://doc.tryfastgpt.ai/docs/development/upgrading/465/>V4.6.5（需要改配置文件）</a></li><li><a class=sidebar-nested-link href=https://doc.tryfastgpt.ai/docs/development/upgrading/464/>V4.6.4(包含升级脚本)</a></li><li><a class=sidebar-nested-link href=https://doc.tryfastgpt.ai/docs/development/upgrading/463/>V4.6.3(包含升级脚本)</a></li><li><a class=sidebar-nested-link href=https://doc.tryfastgpt.ai/docs/development/upgrading/462/>V4.6.2(包含升级脚本)</a></li><li><a class=sidebar-nested-link href=https://doc.tryfastgpt.ai/docs/development/upgrading/461/>V4.6.1</a></li><li><a class=sidebar-nested-link href=https://doc.tryfastgpt.ai/docs/development/upgrading/46/>V4.6(包含升级脚本)</a></li><li><a class=sidebar-nested-link href=https://doc.tryfastgpt.ai/docs/development/upgrading/452/>V4.5.2</a></li><li><a class=sidebar-nested-link href=https://doc.tryfastgpt.ai/docs/development/upgrading/451/>V4.5.1(需进行初始化)</a></li><li><a class=sidebar-nested-link href=https://doc.tryfastgpt.ai/docs/development/upgrading/45/>V4.5(需进行较为复杂更新)</a></li><li><a class=sidebar-nested-link href=https://doc.tryfastgpt.ai/docs/development/upgrading/447/>V4.4.7（需执行升级脚本）</a></li><li><a class=sidebar-nested-link href=https://doc.tryfastgpt.ai/docs/development/upgrading/446/>V4.4.6</a></li><li><a class=sidebar-nested-link href=https://doc.tryfastgpt.ai/docs/development/upgrading/445/>V4.4.5(包含升级脚本)</a></li><li><a class=sidebar-nested-link href=https://doc.tryfastgpt.ai/docs/development/upgrading/442/>升级到 V4.4.2(包含升级脚本)</a></li><li><a class=sidebar-nested-link href=https://doc.tryfastgpt.ai/docs/development/upgrading/441/>升级到 V4.4.1(包含升级脚本)</a></li><li><a class=sidebar-nested-link href=https://doc.tryfastgpt.ai/docs/development/upgrading/44/>升级到 V4.4(包含升级脚本)</a></li><li><a class=sidebar-nested-link href=https://doc.tryfastgpt.ai/docs/development/upgrading/43/>升级到 V4.3(包含升级脚本)</a></li><li><a class=sidebar-nested-link href=https://doc.tryfastgpt.ai/docs/development/upgrading/421/>升级到 V4.2.1</a></li><li><a class=sidebar-nested-link href=https://doc.tryfastgpt.ai/docs/development/upgrading/42/>升级到 V4.2</a></li><li><a class=sidebar-nested-link href=https://doc.tryfastgpt.ai/docs/development/upgrading/41/>升级到 V4.1</a></li><li><a class=sidebar-nested-link href=https://doc.tryfastgpt.ai/docs/development/upgrading/40/>升级到 V4.0</a></li></ul></div></li><li class="sidebar-dropdown nested"><button class=btn>
|
||
OpenAPI 接口文档</button><div class=sidebar-submenu><ul><li><a class=sidebar-nested-link href=https://doc.tryfastgpt.ai/docs/development/openapi/intro/>OpenAPI 介绍</a></li><li><a class=sidebar-nested-link href=https://doc.tryfastgpt.ai/docs/development/openapi/chat/>对话接口</a></li><li><a class=sidebar-nested-link href=https://doc.tryfastgpt.ai/docs/development/openapi/dataset/>知识库接口</a></li><li><a class=sidebar-nested-link href=https://doc.tryfastgpt.ai/docs/development/openapi/share/>分享链接身份鉴权</a></li></ul></div></li><li class="sidebar-dropdown nested current active"><button class=btn>
|
||
本地模型使用</button><div class="sidebar-submenu d-block"><ul><li><a class=sidebar-nested-link href=https://doc.tryfastgpt.ai/docs/development/custom-models/marker/>接入 Marker PDF 文档解析</a></li><li class=current><a class=sidebar-nested-link href=https://doc.tryfastgpt.ai/docs/development/custom-models/xinference/>使用 Xinference 接入本地模型</a></li><li><a class=sidebar-nested-link href=https://doc.tryfastgpt.ai/docs/development/custom-models/bge-rerank/>接入 bge-rerank 重排模型</a></li><li><a class=sidebar-nested-link href=https://doc.tryfastgpt.ai/docs/development/custom-models/chatglm2/>接入 ChatGLM2-6B</a></li><li><a class=sidebar-nested-link href=https://doc.tryfastgpt.ai/docs/development/custom-models/m3e/>接入 M3E 向量模型</a></li><li><a class=sidebar-nested-link href=https://doc.tryfastgpt.ai/docs/development/custom-models/chatglm2-m3e/>接入 ChatGLM2-m3e 模型</a></li><li><a class=sidebar-nested-link href=https://doc.tryfastgpt.ai/docs/development/custom-models/ollama/>使用 Ollama 接入本地模型</a></li></ul></div></li><li class="sidebar-dropdown nested"><button class=btn>
|
||
代理方案</button><div class=sidebar-submenu><ul><li><a class=sidebar-nested-link href=https://doc.tryfastgpt.ai/docs/development/proxy/nginx/>Nginx 中转</a></li><li><a class=sidebar-nested-link href=https://doc.tryfastgpt.ai/docs/development/proxy/http_proxy/>HTTP 代理中转</a></li><li><a class=sidebar-nested-link href=https://doc.tryfastgpt.ai/docs/development/proxy/cloudflare/>Cloudflare Worker 中转</a></li></ul></div></li><li class="sidebar-dropdown nested"><button class=btn>
|
||
迁移&备份</button><div class=sidebar-submenu><ul><li><a class=sidebar-nested-link href=https://doc.tryfastgpt.ai/docs/development/migration/docker_mongo/>Docker Mongo迁移(dump模式)</a></li><li><a class=sidebar-nested-link href=https://doc.tryfastgpt.ai/docs/development/migration/docker_db/>Docker 数据库迁移(无脑操作)</a></li></ul></div></li><li class="sidebar-dropdown nested"><button class=btn>
|
||
设计方案</button><div class=sidebar-submenu><ul><li><a class=sidebar-nested-link href=https://doc.tryfastgpt.ai/docs/development/design/dataset/>数据集</a></li></ul></div></li></ul></div></li><li class=sidebar-dropdown><button class=btn>
|
||
<i class="material-icons me-2">quiz</i>
|
||
FAQ</button><div class=sidebar-submenu><ul><li><a class=sidebar-nested-link href=https://doc.tryfastgpt.ai/docs/faq/docker/>Docker 部署问题</a></li><li><a class=sidebar-nested-link href=https://doc.tryfastgpt.ai/docs/faq/privatedeploy/>私有部署常见问题</a></li><li><a class=sidebar-nested-link href=https://doc.tryfastgpt.ai/docs/faq/chat/>聊天框问题</a></li><li><a class=sidebar-nested-link href=https://doc.tryfastgpt.ai/docs/faq/app/>应用使用问题</a></li><li><a class=sidebar-nested-link href=https://doc.tryfastgpt.ai/docs/faq/dataset/>知识库使用问题</a></li><li><a class=sidebar-nested-link href=https://doc.tryfastgpt.ai/docs/faq/external_channel_integration/>接入外部渠道</a></li><li><a class=sidebar-nested-link href=https://doc.tryfastgpt.ai/docs/faq/error/>报错</a></li><li><a class=sidebar-nested-link href=https://doc.tryfastgpt.ai/docs/faq/points_consumption/>积分消耗</a></li><li><a class=sidebar-nested-link href=https://doc.tryfastgpt.ai/docs/faq/other/>其他问题</a></li></ul></div></li><li class=sidebar-dropdown><button class=btn>
|
||
<i class="material-icons me-2">shopping_cart</i>
|
||
收费说明</button><div class=sidebar-submenu><ul><li><a class=sidebar-nested-link href=https://doc.tryfastgpt.ai/docs/shopping_cart/saas/>线上版定价</a></li><li><a class=sidebar-nested-link href=https://doc.tryfastgpt.ai/docs/shopping_cart/intro/>商业版</a></li></ul></div></li><li class=sidebar-dropdown><button class=btn>
|
||
<i class="material-icons me-2">handshake</i>
|
||
协议</button><div class=sidebar-submenu><ul><li><a class=sidebar-nested-link href=https://doc.tryfastgpt.ai/docs/agreement/open-source/>开源协议</a></li><li><a class=sidebar-nested-link href=https://doc.tryfastgpt.ai/docs/agreement/terms/>服务协议</a></li><li><a class=sidebar-nested-link href=https://doc.tryfastgpt.ai/docs/agreement/privacy/>隐私政策</a></li></ul></div></li><li><a class=sidebar-root-link href=https://doc.tryfastgpt.ai/docs/community/><i class="material-icons me-2">forum</i>
|
||
加入社区</a></li></ul></div><ul class="sidebar-footer list-unstyled mb-0"></ul></nav><main class="page-content bg-transparent"><div id=top-header class="top-header d-print-none"><div class="header-bar d-flex justify-content-between"><div class="d-flex align-items-center"><a href=https://tryfastgpt.ai/ class="logo-icon me-3" aria-label=HomePage alt=HomePage><div class=small><svg width="26" height="26" viewBox="0 0 1041 1348" fill="none" xmlns="http://www.w3.org/2000/svg"><path d="M340.837.33933 681.068.338989V.455643C684.032.378397 686.999.339702 689.967.339702 735.961.3397 781.504 9.62899 823.997 27.6772 866.49 45.7254 905.099 72.1791 937.622 105.528 970.144 138.877 995.942 178.467 1013.54 222.04 1031.14 265.612 1040.2 312.312 1040.2 359.474H340.836V1347.84C296.157 1347.84 251.914 1338.55 210.636 1320.49 169.357 1302.43 131.85 1275.95 100.257 1242.58 68.6636 1209.21 43.6023 1169.59 26.5041 1125.99 11.3834 1087.43 2.75216 1046.42.957956 1004.81H.605869L.605897 368.098H.70363c-.597878-26.267 1.53378-52.655 6.43943-78.687C20.2709 219.745 52.6748 155.754 100.257 105.528 147.839 55.3017 208.462 21.0975 274.461 7.24017 296.426 2.62833 318.657.339101 340.837.33933z" fill="url(#paint0_linear_1172_228)"/><path d="M633.639 904.645H513.029V576.37H635.422V576.377C678.161 576.607 720.454 585.093 759.951 601.37 799.997 617.874 836.384 642.064 867.033 672.559c30.65 30.495 54.963 66.698 71.55 106.542C955.171 818.944 963.709 861.648 963.709 904.775H633.639V904.645z" fill="url(#paint1_linear_1172_228)"/><defs><linearGradient id="paint0_linear_1172_228" x1="520.404" y1=".338989" x2="520.404" y2="1347.84" gradientUnits="userSpaceOnUse"><stop stop-color="#326dff"/><stop offset="1" stop-color="#8eaeff"/></linearGradient><linearGradient id="paint1_linear_1172_228" x1="738.369" y1="576.37" x2="738.369" y2="904.775" gradientUnits="userSpaceOnUse"><stop stop-color="#326dff"/><stop offset="1" stop-color="#8eaeff"/></linearGradient></defs></svg></div><div class=big><svg width="26" height="26" viewBox="0 0 1041 1348" fill="none" xmlns="http://www.w3.org/2000/svg"><path d="M340.837.33933 681.068.338989V.455643C684.032.378397 686.999.339702 689.967.339702 735.961.3397 781.504 9.62899 823.997 27.6772 866.49 45.7254 905.099 72.1791 937.622 105.528 970.144 138.877 995.942 178.467 1013.54 222.04 1031.14 265.612 1040.2 312.312 1040.2 359.474H340.836V1347.84C296.157 1347.84 251.914 1338.55 210.636 1320.49 169.357 1302.43 131.85 1275.95 100.257 1242.58 68.6636 1209.21 43.6023 1169.59 26.5041 1125.99 11.3834 1087.43 2.75216 1046.42.957956 1004.81H.605869L.605897 368.098H.70363c-.597878-26.267 1.53378-52.655 6.43943-78.687C20.2709 219.745 52.6748 155.754 100.257 105.528 147.839 55.3017 208.462 21.0975 274.461 7.24017 296.426 2.62833 318.657.339101 340.837.33933z" fill="url(#paint0_linear_1172_228)"/><path d="M633.639 904.645H513.029V576.37H635.422V576.377C678.161 576.607 720.454 585.093 759.951 601.37 799.997 617.874 836.384 642.064 867.033 672.559c30.65 30.495 54.963 66.698 71.55 106.542C955.171 818.944 963.709 861.648 963.709 904.775H633.639V904.645z" fill="url(#paint1_linear_1172_228)"/><defs><linearGradient id="paint0_linear_1172_228" x1="520.404" y1=".338989" x2="520.404" y2="1347.84" gradientUnits="userSpaceOnUse"><stop stop-color="#326dff"/><stop offset="1" stop-color="#8eaeff"/></linearGradient><linearGradient id="paint1_linear_1172_228" x1="738.369" y1="576.37" x2="738.369" y2="904.775" gradientUnits="userSpaceOnUse"><stop stop-color="#326dff"/><stop offset="1" stop-color="#8eaeff"/></linearGradient></defs></svg></div></a><button id=close-sidebar class="btn btn-icon btn-soft">
|
||
<span class="material-icons size-20 menu-icon align-middle">menu</span></button>
|
||
<span><div id=docsearch></div></span></div><div class="d-flex align-items-center"><ul class="list-unstyled mb-0"><li class="list-inline-item mb-0"><a href=https://github.com/labring/FastGPT alt=github rel="noopener noreferrer" target=_blank><div class="btn btn-icon btn-default border-0"><svg xmlns="http://www.w3.org/2000/svg" width="24" height="24" viewBox="0 0 24 24" fill="none" stroke="currentcolor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"><title>GitHub</title><path d="M9 19c-5 1.5-5-2.5-7-3m14 6v-3.87a3.37 3.37.0 00-.94-2.61c3.14-.35 6.44-1.54 6.44-7A5.44 5.44.0 0020 4.77 5.07 5.07.0 0019.91 1S18.73.65 16 2.48a13.38 13.38.0 00-7 0C6.27.65 5.09 1 5.09 1A5.07 5.07.0 005 4.77 5.44 5.44.0 003.5 8.55c0 5.42 3.3 6.61 6.44 7A3.37 3.37.0 009 18.13V22"/></svg></div></a></li><li class="list-inline-item mb-0"><a href=https://oss.laf.run/htr4n1-images/fastgpt-qr-code.jpg alt=wechat rel="noopener noreferrer" target=_blank><div class="btn btn-icon btn-default border-0"><svg xmlns="http://www.w3.org/2000/svg" width="28" height="28" viewBox="0 0 28 28" fill="none" stroke="currentcolor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"><title>微信群</title><path d="M21.6696 15.9962C21.9464 15.9925 22.2107 15.881 22.4064 15.6853 22.6021 15.4896 22.7137 15.2252 22.7173 14.9485c0-.581-.465500000000002-1.0477-1.0477-1.0477C21.0875 13.9008 20.622 14.3675 20.622 14.9485c0 .583300000000001.4666 1.0477 1.0476 1.0477zm-5.1625.0C16.7839 15.9925 17.0482 15.881 17.2439 15.6853S17.5512 15.2252 17.5548 14.9485c0-.581-.466699999999999-1.0477-1.0477-1.0477C15.9238 13.9008 15.4595 14.3675 15.4595 14.9485c0 .583300000000001.4655 1.0477 1.0476 1.0477zm7.6615 5.88C24.0939 21.9177 24.0342 21.9818 23.9981 22.0594 23.9621 22.1369 23.9515 22.2239 23.968 22.3078 23.968 22.3638 23.968 22.421 23.9971 22.4793c.1144.486500000000003.343 1.2612.343 1.2904C24.3401 23.8548 24.3693 23.912 24.3693 23.9703 24.3693 24.0041 24.3626 24.0376 24.3496 24.0688 24.3367 24.1 24.3177 24.1284 24.2937 24.1522 24.2697 24.1761 24.2413 24.195 24.21 24.2078 24.1788 24.2206 24.1453 24.2272 24.1115 24.227 24.0531 24.227 24.0251 24.199 23.968 24.171L22.2775 23.1957C22.154 23.1277 22.0168 23.0886 21.8761 23.0813 21.791 23.0813 21.7046 23.0813 21.6475 23.1093 20.8448 23.3392 20.0141 23.4535 19.1263 23.4535c-4.27.0-7.7082-2.8665-7.7082-6.4225.0-3.5548 3.4382-6.4225 7.7082-6.4225 4.2688.0 7.707 2.8677 7.707 6.4225.0 1.9215-1.0313 3.6703-2.6647 4.8463V21.8762zM19.4518 9.44883C19.3429 9.44521 19.234 9.44327 19.1251 9.443c-4.8696.0-8.8736 3.3367-8.8736 7.5892.0.6463.0932999999999993 1.2716.266 1.8666H10.4136C9.39426 18.8885 8.38067 18.7442 7.39896 18.4695 7.31263 18.4403 7.22629 18.4403 7.13996 18.4403 6.96754 18.444 6.79921 18.4935 6.65229 18.5838L4.61296 19.7563C4.55463 19.7855 4.49746 19.8147 4.44029 19.8147 4.35703 19.8138 4.27744 19.7803 4.21856 19.7214 4.15968 19.6625 4.1262 19.5829 4.12529 19.4997 4.12529 19.4133 4.15329 19.3562 4.18246 19.2698 4.21046 19.2418 4.46946 18.2968 4.61296 17.7252 4.61296 17.6668 4.64096 17.5817 4.64096 17.5245 4.64065 17.4245 4.61721 17.3259 4.57248 17.2364 4.52775 17.147 4.46295 17.0691 4.38313 17.0088c-1.98217-1.4035-3.2165-3.4918-3.2165-5.8111.0-4.26187 4.165-7.6977 9.27497-7.6977 4.3925.0 8.085 2.5305 9.0102 5.94767V9.44883zM13.4388 9.9365c.6685.0 1.197-.5565 1.197-1.197.0-.6685-.528499999999999-1.197-1.197-1.197S12.2418 8.071 12.2418 8.7395 12.7703 9.9365 13.4388 9.9365zm-6.13667.0c.6685.0 1.19816-.5565 1.19816-1.197.0-.6685-.52966-1.197-1.19816-1.197-.66734.0-1.197.528499999999999-1.197 1.197s.52966 1.197 1.197 1.197z"/></svg></div></a></li><li class="list-inline-item mb-0"><a href=https://oss.laf.run/otnvvf-imgs/fastgpt-feishu1.png alt=lark rel="noopener noreferrer" target=_blank><div class="btn btn-icon btn-default border-0"><svg t="1728884315234" class="icon" viewBox="0 0 1224 1024" xmlns="http://www.w3.org/2000/svg" p-id="4847" width="28" height="28" fill="currentcolor"><path d="M1224.146926 401.768509a50.444385 50.444385.0 00-23.813954-38.631991c-6.095363-3.741292-61.752335-36.782364-141.475481-43.949671A317.253146 317.253146.0 00922.972928 336.16979L746.964061 25.579507A50.444385 50.444385.0 00703.077446.0H284.809419A50.444385 50.444385.0 00248.027055 84.97777c3.236848 3.447033 296.360763 315.739814 426.969683 459.653442-59.734559 55.762064-103.558119 83.800735-127.666331 96.832201L346.435643 500.639504a50.045034 50.045034.0 00-6.97814-4.098606L79.416697 314.205464A50.444385 50.444385.0 00.744475 364.124387c.210185 1.177036 20.619142 118.607361 42.036988 237.635091 44.033744 245.538045 48.994109 257.896919 52.272994 266.11515 5.065457 12.611096 14.334613 24.549601 44.895503 44.538188a595.916337 595.916337.0 0069.361029 38.337733c49.519571 23.603769 128.212812 54.437899 221.59798 67.25918a623.009175 623.009175.0 0085.061845 5.948234c131.491697.0 290.055215-44.138837 418.373119-211.404011 73.564728-96.054517 118.250046-163.944252 154.086578-218.592335 44.033745-67.070014 70.622139-107.551633 118.838564-150.177139a50.444385 50.444385.0 0016.877851-42.015969zM673.693591 100.88877 834.443032 384.638437a413.097477 413.097477.0 00-63.055481 59.356226c-8.743693 10.04684-17.256183 19.568218-25.579507 28.711263-89.559802-98.744884-248.774893-269.373017-344.619225-371.817156zM305.491617 882.125167c-59.86067-22.594881-102.065806-47.85911-118.523287-59.692523-10.299062-45.610132-39.935138-209.638457-65.829922-355.780044l391.238243 274.270325a48.132351 48.132351.0 006.725918 3.951477l189.166445 132.689752A398.300458 398.300458.0 01552.85827 921.702991c-97.336645 7.713787-188.262649-17.277202-247.366653-39.577824zm698.654734-343.442189c-34.932737 53.197808-78.398982 119.385045-149.819824 212.496972a503.371908 503.371908.0 01-58.641598 64.33761L637.499745 704.687042c35.31107-23.813953 81.152405-60.070855 135.905581-114.803013a48.342536 48.342536.0 0014.944149-15.154334c18.790533-19.379051 38.568936-40.859952 59.272153-64.694924 57.086229-65.745849 124.009113-96.243683 198.540692-90.673782a247.639894 247.639894.0 0138.589955 6.011289c-28.290893 33.62959-51.936698 69.63427-80.605924 113.3107z" p-id="4848"/></svg></div></a></li></ul><button id=mode class="btn btn-icon btn-default ms-2" type=button aria-label="Toggle user interface mode">
|
||
<span class=toggle-dark><svg xmlns="http://www.w3.org/2000/svg" height="30" width="30" viewBox="0 0 48 48" fill="currentcolor"><path d="M24 42q-7.5.0-12.75-5.25T6 24t5.25-12.75T24 6q.4.0.85.025.45.025 1.15.075-1.8 1.6-2.8 3.95t-1 4.95q0 4.5 3.15 7.65Q28.5 25.8 33 25.8q2.6.0 4.95-.925T41.9 22.3q.05.6.075.975Q42 23.65 42 24q0 7.5-5.25 12.75T24 42zm0-3q5.45.0 9.5-3.375t5.05-7.925q-1.25.55-2.675.825Q34.45 28.8 33 28.8q-5.75.0-9.775-4.025T19.2 15q0-1.2.25-2.575t.9-3.125q-4.9 1.35-8.125 5.475Q9 18.9 9 24q0 6.25 4.375 10.625T24 39zm-.2-14.85z"/></svg></span><span class=toggle-light><svg xmlns="http://www.w3.org/2000/svg" height="30" width="30" viewBox="0 0 48 48" fill="currentcolor"><path d="M24 31q2.9.0 4.95-2.05T31 24t-2.05-4.95T24 17t-4.95 2.05T17 24t2.05 4.95T24 31zm0 3q-4.15.0-7.075-2.925T14 24t2.925-7.075T24 14t7.075 2.925T34 24t-2.925 7.075T24 34zM3.5 25.5q-.65.0-1.075-.425Q2 24.65 2 24t.425-1.075Q2.85 22.5 3.5 22.5h5q.65.0 1.075.425Q10 23.35 10 24t-.425 1.075T8.5 25.5zm36 0q-.65.0-1.075-.425Q38 24.65 38 24t.425-1.075T39.5 22.5h5q.65.0 1.075.425Q46 23.35 46 24t-.425 1.075-1.075.425zM24 10q-.65.0-1.075-.425Q22.5 9.15 22.5 8.5v-5q0-.65.425-1.075Q23.35 2 24 2t1.075.425T25.5 3.5v5q0 .65-.425 1.075Q24.65 10 24 10zm0 36q-.65.0-1.075-.425T22.5 44.5v-5q0-.65.425-1.075Q23.35 38 24 38t1.075.425.425 1.075v5q0 .65-.425 1.075Q24.65 46 24 46zM12 14.1l-2.85-2.8q-.45-.45-.425-1.075.025-.625.425-1.075.45-.45 1.075-.45t1.075.45L14.1 12q.4.45.4 1.05.0.6-.4 1-.4.45-1.025.45T12 14.1zm24.7 24.75L33.9 36q-.4-.45-.4-1.075t.45-1.025q.4-.45 1-.45t1.05.45l2.85 2.8q.45.45.425 1.075-.025.625-.425 1.075-.45.45-1.075.45t-1.075-.45zM33.9 14.1q-.45-.45-.45-1.05.0-.6.45-1.05l2.8-2.85q.45-.45 1.075-.425.625.025 1.075.425.45.45.45 1.075t-.45 1.075L36 14.1q-.4.4-1.025.4t-1.075-.4zM9.15 38.85q-.45-.45-.45-1.075t.45-1.075L12 33.9q.45-.45 1.05-.45.6.0 1.05.45.45.45.45 1.05.0.6-.45 1.05l-2.8 2.85q-.45.45-1.075.425-.625-.025-1.075-.425zM24 24z"/></svg></span></button></div></div></div><div class=container-fluid><div class=layout-spacing><div class="d-md-flex justify-content-between align-items-center"><nav aria-label=breadcrumb class="d-inline-block pb-2 mt-1 mt-sm-0"><ul id=breadcrumbs class="breadcrumb bg-transparent mb-0" itemscope itemtype=https://schema.org/BreadcrumbList><li class="breadcrumb-item text-capitalize active" aria-current=page itemprop=itemListElement itemscope itemtype=https://schema.org/ListItem><a itemprop=item href=/docs/><i class="material-icons size-20 align-text-bottom" itemprop=name>Home</i></a><meta itemprop=position content="1"></li><li class="breadcrumb-item text-capitalize" itemprop=itemListElement itemscope itemtype=https://schema.org/ListItem><a itemprop=item href=/docs/development/><span itemprop=name>开发与部署指南</span></a><meta itemprop=position content="2"></li><li class="breadcrumb-item text-capitalize" itemprop=itemListElement itemscope itemtype=https://schema.org/ListItem><a itemprop=item href=/docs/development/custom-models/><span itemprop=name>本地模型使用</span></a><meta itemprop=position content="3"></li><li class="breadcrumb-item text-capitalize active" itemprop=itemListElement itemscope itemtype=https://schema.org/ListItem><span itemprop=name>使用 Xinference 接入本地模型</span><meta itemprop=position content="4"></li></ul></nav></div><div class="row flex-xl-nowrap"><div class="docs-toc col-xl-3 d-xl-block"><toc><div class="fw-bold text-uppercase mb-2">使用 Xinference 接入本地模型</div><nav id=toc><ul><li><ul><li><a href=#安装-xinference>安装 Xinference</a><ul><li><a href=#1-服务器>1. 服务器</a></li><li><a href=#2-个人设备>2. 个人设备</a></li></ul></li><li><a href=#创建并部署模型以-qwen-14b-模型为例>创建并部署模型（以 Qwen-14B 模型为例）</a><ul><li><a href=#1-webui-方式启动模型>1. WebUI 方式启动模型</a></li><li><a href=#2-命令行方式启动模型>2. 命令行方式启动模型</a></li></ul></li><li><a href=#将本地模型接入-one-api>将本地模型接入 One API</a></li><li><a href=#将本地模型接入-fastgpt>将本地模型接入 FastGPT</a></li></ul></li></ul></nav></toc></div><div class="docs-toc-mobile d-print-none d-xl-none"><button id=toc-dropdown-btn class="btn-secondary dropdown-toggle" type=button data-bs-toggle=dropdown data-bs-offset=0,0 aria-expanded=false>
|
||
Table of Contents</button><nav id=toc-mobile><ul class=dropdown-menu><li><ul><li><a href=#安装-xinference>安装 Xinference</a><ul><li><a href=#1-服务器>1. 服务器</a></li><li><a href=#2-个人设备>2. 个人设备</a></li></ul></li><li><a href=#创建并部署模型以-qwen-14b-模型为例>创建并部署模型（以 Qwen-14B 模型为例）</a><ul><li><a href=#1-webui-方式启动模型>1. WebUI 方式启动模型</a></li><li><a href=#2-命令行方式启动模型>2. 命令行方式启动模型</a></li></ul></li><li><a href=#将本地模型接入-one-api>将本地模型接入 One API</a></li><li><a href=#将本地模型接入-fastgpt>将本地模型接入 FastGPT</a></li></ul></li></ul></nav></div><div class="docs-content col-12 col-xl-9 mt-0"><div class="mb-0 d-flex"><i class="material-icons title-icon me-2">api</i><h1 class="content-title mb-0">使用 Xinference 接入本地模型</h1></div><p class="lead mb-3">一站式本地 LLM 私有化部署</p><div id=content class=main-content data-bs-spy=scroll data-bs-root-margin="0px 0px -65%" data-bs-target=#toc-mobile><div data-prismjs-copy data-prismjs-copy-success data-prismjs-copy-error><p><a href=https://github.com/xorbitsai/inference rel=external target=_blank>Xinference<svg width="16" height="16" viewBox="0 0 24 24" xmlns="http://www.w3.org/2000/svg"><path fill="currentcolor" d="M14 5c-.552.0-1-.448-1-1s.448-1 1-1h6c.552.0 1 .448 1 1v6c0 .552-.448 1-1 1s-1-.448-1-1V6.414l-7.293 7.293c-.391.39-1.024.39-1.414.0-.391-.391-.391-1.024.0-1.414L17.586 5H14zM5 7c-.552.0-1 .448-1 1v11c0 .552.448 1 1 1h11c.552.0 1-.448 1-1v-4.563c0-.552.448-1 1-1s1 .448 1 1V19c0 1.657-1.343 3-3 3H5c-1.657.0-3-1.343-3-3V8c0-1.657 1.343-3 3-3h4.563c.552.0 1 .448 1 1s-.448 1-1 1H5z"/></svg></a> 是一款开源模型推理平台，除了支持 LLM，它还可以部署 Embedding 和 ReRank 模型，这在企业级 RAG 构建中非常关键。同时，Xinference 还提供 Function Calling 等高级功能。还支持分布式部署，也就是说，随着未来应用调用量的增长，它可以进行水平扩展。</p><h2 id=安装-xinference>安装 Xinference <a href=#%e5%ae%89%e8%a3%85-xinference class=anchor aria-hidden=true><i class="material-icons align-middle">link</i></a></h2><p>Xinference 支持多种推理引擎作为后端，以满足不同场景下部署大模型的需要，下面会分使用场景来介绍一下这三种推理后端，以及他们的使用方法。</p><h3 id=1-服务器>1. 服务器 <a href=#1-%e6%9c%8d%e5%8a%a1%e5%99%a8 class=anchor aria-hidden=true><i class="material-icons align-middle">link</i></a></h3><p>如果你的目标是在一台 Linux 或者 Window 服务器上部署大模型，可以选择 Transformers 或 vLLM 作为 Xinference 的推理后端：</p><ul><li><a href=https://huggingface.co/docs/transformers/index rel=external target=_blank>Transformers<svg width="16" height="16" viewBox="0 0 24 24" xmlns="http://www.w3.org/2000/svg"><path fill="currentcolor" d="M14 5c-.552.0-1-.448-1-1s.448-1 1-1h6c.552.0 1 .448 1 1v6c0 .552-.448 1-1 1s-1-.448-1-1V6.414l-7.293 7.293c-.391.39-1.024.39-1.414.0-.391-.391-.391-1.024.0-1.414L17.586 5H14zM5 7c-.552.0-1 .448-1 1v11c0 .552.448 1 1 1h11c.552.0 1-.448 1-1v-4.563c0-.552.448-1 1-1s1 .448 1 1V19c0 1.657-1.343 3-3 3H5c-1.657.0-3-1.343-3-3V8c0-1.657 1.343-3 3-3h4.563c.552.0 1 .448 1 1s-.448 1-1 1H5z"/></svg></a>：通过集成 Huggingface 的 Transformers 库作为后端，Xinference 可以最快地 集成当今自然语言处理（NLP）领域的最前沿模型（自然也包括 LLM）。</li><li><a href=https://vllm.ai/ rel=external target=_blank>vLLM<svg width="16" height="16" viewBox="0 0 24 24" xmlns="http://www.w3.org/2000/svg"><path fill="currentcolor" d="M14 5c-.552.0-1-.448-1-1s.448-1 1-1h6c.552.0 1 .448 1 1v6c0 .552-.448 1-1 1s-1-.448-1-1V6.414l-7.293 7.293c-.391.39-1.024.39-1.414.0-.391-.391-.391-1.024.0-1.414L17.586 5H14zM5 7c-.552.0-1 .448-1 1v11c0 .552.448 1 1 1h11c.552.0 1-.448 1-1v-4.563c0-.552.448-1 1-1s1 .448 1 1V19c0 1.657-1.343 3-3 3H5c-1.657.0-3-1.343-3-3V8c0-1.657 1.343-3 3-3h4.563c.552.0 1 .448 1 1s-.448 1-1 1H5z"/></svg></a>: vLLM 是由加州大学伯克利分校开发的一个开源库，专为高效服务大型语言模型（LLM）而设计。它引入了 PagedAttention 算法， 通过有效管理注意力键和值来改善内存管理，吞吐量能够达到 Transformers 的 24 倍，因此 vLLM 适合在生产环境中使用，应对高并发的用户访问。</li></ul><p>假设你服务器配备 NVIDIA 显卡，可以参考<a href=https://xorbits.cn/blogs/langchain-streamlit-doc-chat rel=external target=_blank>这篇文章中的指令来安装 CUDA<svg width="16" height="16" viewBox="0 0 24 24" xmlns="http://www.w3.org/2000/svg"><path fill="currentcolor" d="M14 5c-.552.0-1-.448-1-1s.448-1 1-1h6c.552.0 1 .448 1 1v6c0 .552-.448 1-1 1s-1-.448-1-1V6.414l-7.293 7.293c-.391.39-1.024.39-1.414.0-.391-.391-.391-1.024.0-1.414L17.586 5H14zM5 7c-.552.0-1 .448-1 1v11c0 .552.448 1 1 1h11c.552.0 1-.448 1-1v-4.563c0-.552.448-1 1-1s1 .448 1 1V19c0 1.657-1.343 3-3 3H5c-1.657.0-3-1.343-3-3V8c0-1.657 1.343-3 3-3h4.563c.552.0 1 .448 1 1s-.448 1-1 1H5z"/></svg></a>，从而让 Xinference 最大限度地利用显卡的加速功能。</p><h4 id=docker-部署>Docker 部署 <a href=#docker-%e9%83%a8%e7%bd%b2 class=anchor aria-hidden=true><i class="material-icons align-middle">link</i></a></h4><p>你可以使用 Xinference 官方的 Docker 镜像来一键安装和启动 Xinference 服务（确保你的机器上已经安装了 Docker），命令如下：</p><div class=prism-codeblock><pre id=1d46bd6 class=language-bash>
|
||
<code>docker run -p 9997:9997 --gpus all xprobe/xinference:latest xinference-local -H 0.0.0.0</code>
|
||
</pre></div><h4 id=直接部署>直接部署 <a href=#%e7%9b%b4%e6%8e%a5%e9%83%a8%e7%bd%b2 class=anchor aria-hidden=true><i class="material-icons align-middle">link</i></a></h4><p>首先我们需要准备一个 3.9 以上的 Python 环境运行来 Xinference，建议先根据 conda 官网文档安装 conda。 然后使用以下命令来创建 3.11 的 Python 环境：</p><div class=prism-codeblock><pre id=bd80a84 class=language-bash>
|
||
<code>conda create --name py311 python=3.11
|
||
conda activate py311</code>
|
||
</pre></div><p>以下两条命令在安装 Xinference 时，将安装 Transformers 和 vLLM 作为 Xinference 的推理引擎后端：</p><div class=prism-codeblock><pre id=9d06985 class=language-bash>
|
||
<code>pip install "xinference[transformers]"
|
||
pip install "xinference[vllm]"
|
||
pip install "xinference[transformers,vllm]" # 同时安装</code>
|
||
</pre></div><p>PyPi 在 安装 Transformers 和 vLLM 时会自动安装 PyTorch，但自动安装的 CUDA 版本可能与你的环境不匹配，此时你可以根据 PyTorch 官网中的<a href=https://pytorch.org/get-started/locally/ rel=external target=_blank>安装指南<svg width="16" height="16" viewBox="0 0 24 24" xmlns="http://www.w3.org/2000/svg"><path fill="currentcolor" d="M14 5c-.552.0-1-.448-1-1s.448-1 1-1h6c.552.0 1 .448 1 1v6c0 .552-.448 1-1 1s-1-.448-1-1V6.414l-7.293 7.293c-.391.39-1.024.39-1.414.0-.391-.391-.391-1.024.0-1.414L17.586 5H14zM5 7c-.552.0-1 .448-1 1v11c0 .552.448 1 1 1h11c.552.0 1-.448 1-1v-4.563c0-.552.448-1 1-1s1 .448 1 1V19c0 1.657-1.343 3-3 3H5c-1.657.0-3-1.343-3-3V8c0-1.657 1.343-3 3-3h4.563c.552.0 1 .448 1 1s-.448 1-1 1H5z"/></svg></a>来手动安装。</p><p>只需要输入如下命令，就可以在服务上启动 Xinference 服务：</p><div class=prism-codeblock><pre id=6260678 class=language-bash>
|
||
<code>xinference-local -H 0.0.0.0</code>
|
||
</pre></div><p>Xinference 默认会在本地启动服务，端口默认为 9997。因为这里配置了-H 0.0.0.0参数，非本地客户端也可以通过机器的 IP 地址来访问 Xinference 服务。</p><h3 id=2-个人设备>2. 个人设备 <a href=#2-%e4%b8%aa%e4%ba%ba%e8%ae%be%e5%a4%87 class=anchor aria-hidden=true><i class="material-icons align-middle">link</i></a></h3><p>如果你想在自己的 Macbook 或者个人电脑上部署大模型，推荐安装 CTransformers 作为 Xinference 的推理后端。CTransformers 是用 GGML 实现的 C++ 版本 Transformers。</p><p><a href=https://ggml.ai/ rel=external target=_blank>GGML<svg width="16" height="16" viewBox="0 0 24 24" xmlns="http://www.w3.org/2000/svg"><path fill="currentcolor" d="M14 5c-.552.0-1-.448-1-1s.448-1 1-1h6c.552.0 1 .448 1 1v6c0 .552-.448 1-1 1s-1-.448-1-1V6.414l-7.293 7.293c-.391.39-1.024.39-1.414.0-.391-.391-.391-1.024.0-1.414L17.586 5H14zM5 7c-.552.0-1 .448-1 1v11c0 .552.448 1 1 1h11c.552.0 1-.448 1-1v-4.563c0-.552.448-1 1-1s1 .448 1 1V19c0 1.657-1.343 3-3 3H5c-1.657.0-3-1.343-3-3V8c0-1.657 1.343-3 3-3h4.563c.552.0 1 .448 1 1s-.448 1-1 1H5z"/></svg></a> 是一个能让大语言模型在<a href=https://github.com/ggerganov/llama.cpp/discussions/205 rel=external target=_blank>消费级硬件上运行<svg width="16" height="16" viewBox="0 0 24 24" xmlns="http://www.w3.org/2000/svg"><path fill="currentcolor" d="M14 5c-.552.0-1-.448-1-1s.448-1 1-1h6c.552.0 1 .448 1 1v6c0 .552-.448 1-1 1s-1-.448-1-1V6.414l-7.293 7.293c-.391.39-1.024.39-1.414.0-.391-.391-.391-1.024.0-1.414L17.586 5H14zM5 7c-.552.0-1 .448-1 1v11c0 .552.448 1 1 1h11c.552.0 1-.448 1-1v-4.563c0-.552.448-1 1-1s1 .448 1 1V19c0 1.657-1.343 3-3 3H5c-1.657.0-3-1.343-3-3V8c0-1.657 1.343-3 3-3h4.563c.552.0 1 .448 1 1s-.448 1-1 1H5z"/></svg></a>的 C++ 库。 GGML 最大的特色在于模型量化。量化一个大语言模型其实就是降低权重表示精度的过程，从而减少使用模型所需的资源。 例如，表示一个高精度浮点数（例如 0.0001）比表示一个低精度浮点数（例如 0.1）需要更多空间。由于 LLM 在推理时需要加载到内存中的，因此你需要花费硬盘空间来存储它们，并且在执行期间有足够大的 RAM 来加载它们，GGML 支持许多不同的量化策略，每种策略在效率和性能之间提供不同的权衡。</p><p>通过以下命令来安装 CTransformers 作为 Xinference 的推理后端：</p><div class=prism-codeblock><pre id=46feced class=language-bash>
|
||
<code>pip install xinference
|
||
pip install ctransformers</code>
|
||
</pre></div><p>因为 GGML 是一个 C++ 库，Xinference 通过 <code>llama-cpp-python</code> 这个库来实现语言绑定。对于不同的硬件平台，我们需要使用不同的编译参数来安装：</p><ul><li>Apple Metal（MPS）：<code>CMAKE_ARGS="-DLLAMA_METAL=on" pip install llama-cpp-python</code></li><li>Nvidia GPU：<code>CMAKE_ARGS="-DLLAMA_CUBLAS=on" pip install llama-cpp-python</code></li><li>AMD GPU：<code>CMAKE_ARGS="-DLLAMA_HIPBLAS=on" pip install llama-cpp-python</code></li></ul><p>安装后只需要输入 <code>xinference-local</code>，就可以在你的 Mac 上启动 Xinference 服务。</p><h2 id=创建并部署模型以-qwen-14b-模型为例>创建并部署模型（以 Qwen-14B 模型为例） <a href=#%e5%88%9b%e5%bb%ba%e5%b9%b6%e9%83%a8%e7%bd%b2%e6%a8%a1%e5%9e%8b%e4%bb%a5-qwen-14b-%e6%a8%a1%e5%9e%8b%e4%b8%ba%e4%be%8b class=anchor aria-hidden=true><i class="material-icons align-middle">link</i></a></h2><h3 id=1-webui-方式启动模型>1. WebUI 方式启动模型 <a href=#1-webui-%e6%96%b9%e5%bc%8f%e5%90%af%e5%8a%a8%e6%a8%a1%e5%9e%8b class=anchor aria-hidden=true><i class="material-icons align-middle">link</i></a></h3><p>Xinference 启动之后，在浏览器中输入: <code>http://127.0.0.1:9997</code>，我们可以访问到本地 Xinference 的 Web UI。</p><p>打开“Launch Model”标签，搜索到 qwen-chat，选择模型启动的相关参数，然后点击模型卡片左下方的小火箭🚀按钮，就可以部署该模型到 Xinference。 默认 Model UID 是 qwen-chat（后续通过将通过这个 ID 来访问模型）。</p><p><img src=https://cdn.jsdelivr.net/gh/yangchuansheng/fastgpt-imgs@main/imgs/xinference-launch-model.png alt loading=lazy class=medium-zoom-image></p><p>当你第一次启动 Qwen 模型时，Xinference 会从 HuggingFace 下载模型参数，大概需要几分钟的时间。Xinference 将模型文件缓存在本地，这样之后启动时就不需要重新下载了。 Xinference 还支持从其他模型站点下载模型文件，例如 <a href=https://inference.readthedocs.io/en/latest/models/sources/sources.html rel=external target=_blank>modelscope<svg width="16" height="16" viewBox="0 0 24 24" xmlns="http://www.w3.org/2000/svg"><path fill="currentcolor" d="M14 5c-.552.0-1-.448-1-1s.448-1 1-1h6c.552.0 1 .448 1 1v6c0 .552-.448 1-1 1s-1-.448-1-1V6.414l-7.293 7.293c-.391.39-1.024.39-1.414.0-.391-.391-.391-1.024.0-1.414L17.586 5H14zM5 7c-.552.0-1 .448-1 1v11c0 .552.448 1 1 1h11c.552.0 1-.448 1-1v-4.563c0-.552.448-1 1-1s1 .448 1 1V19c0 1.657-1.343 3-3 3H5c-1.657.0-3-1.343-3-3V8c0-1.657 1.343-3 3-3h4.563c.552.0 1 .448 1 1s-.448 1-1 1H5z"/></svg></a>。</p><h3 id=2-命令行方式启动模型>2. 命令行方式启动模型 <a href=#2-%e5%91%bd%e4%bb%a4%e8%a1%8c%e6%96%b9%e5%bc%8f%e5%90%af%e5%8a%a8%e6%a8%a1%e5%9e%8b class=anchor aria-hidden=true><i class="material-icons align-middle">link</i></a></h3><p>我们也可以使用 Xinference 的命令行工具来启动模型，默认 Model UID 是 qwen-chat（后续通过将通过这个 ID 来访问模型）。</p><div class=prism-codeblock><pre id=297ab18 class=language-bash>
|
||
<code>xinference launch -n qwen-chat -s 14 -f pytorch</code>
|
||
</pre></div><p>除了 WebUI 和命令行工具， Xinference 还提供了 Python SDK 和 RESTful API 等多种交互方式， 更多用法可以参考 <a href=https://inference.readthedocs.io/en/latest/getting_started/index.html rel=external target=_blank>Xinference 官方文档<svg width="16" height="16" viewBox="0 0 24 24" xmlns="http://www.w3.org/2000/svg"><path fill="currentcolor" d="M14 5c-.552.0-1-.448-1-1s.448-1 1-1h6c.552.0 1 .448 1 1v6c0 .552-.448 1-1 1s-1-.448-1-1V6.414l-7.293 7.293c-.391.39-1.024.39-1.414.0-.391-.391-.391-1.024.0-1.414L17.586 5H14zM5 7c-.552.0-1 .448-1 1v11c0 .552.448 1 1 1h11c.552.0 1-.448 1-1v-4.563c0-.552.448-1 1-1s1 .448 1 1V19c0 1.657-1.343 3-3 3H5c-1.657.0-3-1.343-3-3V8c0-1.657 1.343-3 3-3h4.563c.552.0 1 .448 1 1s-.448 1-1 1H5z"/></svg></a>。</p><h2 id=将本地模型接入-one-api>将本地模型接入 One API <a href=#%e5%b0%86%e6%9c%ac%e5%9c%b0%e6%a8%a1%e5%9e%8b%e6%8e%a5%e5%85%a5-one-api class=anchor aria-hidden=true><i class="material-icons align-middle">link</i></a></h2><p>One API 的部署和接入请参考<a data-bs-delay='{"hide":300,"show":550}' data-bs-html=true data-bs-title="<a href='/docs/development/modelconfig/one-api/'><p>模型配置方案</p><strong>通过 OneAPI 接入模型</strong><br>通过 OneAPI 接入模型</a>" data-bs-toggle=tooltip href=/docs/development/modelconfig/one-api/>这里</a>。</p><p>为 qwen1.5-chat 添加一个渠道，这里的 Base URL 需要填 Xinference 服务的端点，并且注册 qwen-chat (模型的 UID) 。</p><p><img src=https://cdn.jsdelivr.net/gh/yangchuansheng/fastgpt-imgs@main/imgs/one-api-add-xinference-models.jpg alt loading=lazy class=medium-zoom-image></p><p>可以使用以下命令进行测试：</p><div class=prism-codeblock><pre id=0297458 class=language-bash>
|
||
<code>curl --location --request POST 'https://<oneapi_url>/v1/chat/completions' \
|
||
--header 'Authorization: Bearer <oneapi_token>' \
|
||
--header 'Content-Type: application/json' \
|
||
--data-raw '{
|
||
"model": "qwen-chat",
|
||
"messages": [{"role": "user", "content": "Hello!"}]
|
||
}'</code>
|
||
</pre></div><p>将 <oneapi_url> 替换为你的 One API 地址，<oneapi_token> 替换为你的 One API 令牌。model 为刚刚在 One API 填写的自定义模型。</p><h2 id=将本地模型接入-fastgpt>将本地模型接入 FastGPT <a href=#%e5%b0%86%e6%9c%ac%e5%9c%b0%e6%a8%a1%e5%9e%8b%e6%8e%a5%e5%85%a5-fastgpt class=anchor aria-hidden=true><i class="material-icons align-middle">link</i></a></h2><p>修改 FastGPT 的 <code>config.json</code> 配置文件的 llmModels 部分加入 qwen-chat 模型：</p><div class=prism-codeblock><pre id=5ef6fc9 class=language-json>
|
||
<code>...
|
||
"llmModels": [
|
||
{
|
||
"model": "qwen-chat", // 模型名(对应OneAPI中渠道的模型名)
|
||
"name": "Qwen", // 模型别名
|
||
"avatar": "/imgs/model/Qwen.svg", // 模型的logo
|
||
"maxContext": 125000, // 最大上下文
|
||
"maxResponse": 4000, // 最大回复
|
||
"quoteMaxToken": 120000, // 最大引用内容
|
||
"maxTemperature": 1.2, // 最大温度
|
||
"charsPointsPrice": 0, // n积分/1k token（商业版）
|
||
"censor": false, // 是否开启敏感校验（商业版）
|
||
"vision": true, // 是否支持图片输入
|
||
"datasetProcess": true, // 是否设置为知识库处理模型（QA），务必保证至少有一个为true，否则知识库会报错
|
||
"usedInClassify": true, // 是否用于问题分类（务必保证至少有一个为true）
|
||
"usedInExtractFields": true, // 是否用于内容提取（务必保证至少有一个为true）
|
||
"usedInToolCall": true, // 是否用于工具调用（务必保证至少有一个为true）
|
||
"toolChoice": true, // 是否支持工具选择（分类，内容提取，工具调用会用到。）
|
||
"functionCall": false, // 是否支持函数调用（分类，内容提取，工具调用会用到。会优先使用 toolChoice，如果为false，则使用 functionCall，如果仍为 false，则使用提示词模式）
|
||
"customCQPrompt": "", // 自定义文本分类提示词（不支持工具和函数调用的模型
|
||
"customExtractPrompt": "", // 自定义内容提取提示词
|
||
"defaultSystemChatPrompt": "", // 对话默认携带的系统提示词
|
||
"defaultConfig": {} // 请求API时，挟带一些默认配置（比如 GLM4 的 top_p）
|
||
}
|
||
],
|
||
...</code>
|
||
</pre></div><p>然后重启 FastGPT 就可以在应用配置中选择 Qwen 模型进行对话：</p><p><img src=https://cdn.jsdelivr.net/gh/yangchuansheng/fastgpt-imgs@main/imgs/fastgpt-list-models.png alt loading=lazy class=medium-zoom-image></p><hr><ul><li>参考：<a href=https://xorbits.cn/blogs/fastgpt-weather-chat rel=external target=_blank>FastGPT + Xinference：一站式本地 LLM 私有化部署和应用开发<svg width="16" height="16" viewBox="0 0 24 24" xmlns="http://www.w3.org/2000/svg"><path fill="currentcolor" d="M14 5c-.552.0-1-.448-1-1s.448-1 1-1h6c.552.0 1 .448 1 1v6c0 .552-.448 1-1 1s-1-.448-1-1V6.414l-7.293 7.293c-.391.39-1.024.39-1.414.0-.391-.391-.391-1.024.0-1.414L17.586 5H14zM5 7c-.552.0-1 .448-1 1v11c0 .552.448 1 1 1h11c.552.0 1-.448 1-1v-4.563c0-.552.448-1 1-1s1 .448 1 1V19c0 1.657-1.343 3-3 3H5c-1.657.0-3-1.343-3-3V8c0-1.657 1.343-3 3-3h4.563c.552.0 1 .448 1 1s-.448 1-1 1H5z"/></svg></a></li></ul></div><div class="gitinfo d-flex flex-wrap justify-content-between align-items-center opacity-85 pt-3"><div id=edit-this-page class=mt-1><a href=https://github.com/labring/FastGPT/blob/main/docSite/content/zh-cn/docs/development/custom-models/xinference.md alt="使用 Xinference 接入本地模型" rel="noopener noreferrer" target=_blank><span class="me-1 align-text-bottom"><svg width="20" height="20" viewBox="0 0 32 32" xmlns="http://www.w3.org/2000/svg" fill="currentcolor"><path d="M16 .396c-8.839.0-16 7.167-16 16 0 7.073 4.584 13.068 10.937 15.183.803.151 1.093-.344 1.093-.772.0-.38-.009-1.385-.015-2.719-4.453.964-5.391-2.151-5.391-2.151-.729-1.844-1.781-2.339-1.781-2.339-1.448-.989.115-.968.115-.968 1.604.109 2.448 1.645 2.448 1.645 1.427 2.448 3.744 1.74 4.661 1.328.14-1.031.557-1.74 1.011-2.135-3.552-.401-7.287-1.776-7.287-7.907.0-1.751.62-3.177 1.645-4.297-.177-.401-.719-2.031.141-4.235.0.0 1.339-.427 4.4 1.641 1.281-.355 2.641-.532 4-.541 1.36.009 2.719.187 4 .541 3.043-2.068 4.381-1.641 4.381-1.641.859 2.204.317 3.833.161 4.235 1.015 1.12 1.635 2.547 1.635 4.297.0 6.145-3.74 7.5-7.296 7.891.556.479 1.077 1.464 1.077 2.959.0 2.14-.02 3.864-.02 4.385.0.416.28.916 1.104.755 6.4-2.093 10.979-8.093 10.979-15.156.0-8.833-7.161-16-16-16z"/></svg></span>编辑此页面</a></div></div></div><div><hr class=doc-hr><div id=doc-nav class=d-print-none><div class="row flex-xl-nowrap"><div class="col-sm-6 pt-2 doc-next"><a href=/docs/development/custom-models/marker/><div class="card h-100 my-1"><div class="card-body py-2"><p class="card-title fs-5 fw-semibold lh-base mb-0"><i class="material-icons align-middle">navigate_before</i> 接入 Marker PDF 文档解析</p><p class="card-text ms-2">使用 Marker 解析 PDF 文档，可实现图片提取和布局 …</p></div></div></a></div><div class="col-sm-6 pt-2 doc-prev"><a class=ms-auto href=/docs/faq/dataset/><div class="card h-100 my-1 text-end"><div class="card-body py-2"><p class="card-title fs-5 fw-semibold lh-base mb-0">知识库使用问题 <i class="material-icons align-middle">navigate_next</i></p><p class="card-text me-2">常见知识库使用问题</p></div></div></a></div></div></div></div></div></div></div></div><footer class="shadow py-3 d-print-none"><div class="row align-items-center" style=height:90px><div class=col><div class="text-sm-start text-center mx-md-2"><p class=mb-0>© 2025 the FastGPT Authors.</p><p class=github-badge><span class=badge-subject>云操作系统</span><span class="badge-value bg-blue"><a style=color:#fff href=https://sealos.io/ target=_blank>Sealos</a></span>
|
||
<span class=badge-subject>云开发</span><span class="badge-value bg-brightgreen"><a style=color:#fff href=https://laf.run target=_blank>Laf</a></span>
|
||
<span class=badge-subject>云原生存储</span><span class="badge-value bg-orange"><a style=color:#fff href=https://github.com/labring/sealfs target=_blank>Sealfs</a></span></p><div id=fixed-box><div class=feedback-btn-wrapper><a href=https://fael3z0zfze.feishu.cn/share/base/form/shrcnRxj3utrzjywsom96Px4sud style=text-decoration:none target=_blank><button id=feedback-btn title="Give feedback"><svg class="inline w-5 h-5" viewBox="0 0 20 20" fill="currentcolor"><path fill-rule="evenodd" d="M18 13V5a2 2 0 00-2-2H4A2 2 0 002 5v8a2 2 0 002 2h3l3 3 3-3h3a2 2 0 002-2zM5 7a1 1 0 011-1h8a1 1 0 110 2H6A1 1 0 015 7zm1 3a1 1 0 100 2h3a1 1 0 100-2H6z" clip-rule="evenodd"/></svg>
|
||
<span _msttexthash=6039839 _msthash=295>咨询合作</span></button></a></div></div></div></div></div></footer></main></div></div><button onclick=topFunction() id=back-to-top aria-label="Back to Top Button" class="back-to-top fs-5"><svg width="24" height="24"><path d="M12 10.224l-6.3 6.3-1.38-1.372L12 7.472l7.68 7.68-1.38 1.376z" style="fill:#fff"/></svg></button>
|
||
<script>(()=>{var e=document.getElementById("mode");e!==null&&(window.matchMedia("(prefers-color-scheme: dark)").addEventListener("change",e=>{e.matches?(localStorage.setItem("theme","dark"),document.documentElement.setAttribute("data-dark-mode","")):(localStorage.setItem("theme","light"),document.documentElement.removeAttribute("data-dark-mode"))}),e.addEventListener("click",()=>{document.documentElement.toggleAttribute("data-dark-mode"),localStorage.setItem("theme",document.documentElement.hasAttribute("data-dark-mode")?"dark":"light")}),localStorage.getItem("theme")==="dark"?document.documentElement.setAttribute("data-dark-mode",""):document.documentElement.removeAttribute("data-dark-mode"))})()</script><script src=/docs/js/bootstrap.58faa4dc6cac13820c8a72b1fdd5d716641b3129ef981b1956fa23d69ad59f180fed6d42cb92030ac272ba85e7f8247d.js integrity=sha384-WPqk3GysE4IMinKx/dXXFmQbMSnvmBsZVvoj1prVnxgP7W1Cy5IDCsJyuoXn+CR9 defer></script>
|
||
<script type=text/javascript src=https://doc.tryfastgpt.ai/docs/js/bundle.min.68e80f383e0615acc689eae489a8c1d728bea3861d874ec1044cc85566fd9ff1b10385a1feadd324701c587f66e425bc.js integrity=sha384-aOgPOD4GFazGierkiajB1yi+o4Ydh07BBEzIVWb9n/GxA4Wh/q3TJHAcWH9m5CW8 crossorigin=anonymous defer></script>
|
||
<script src=https://cdn.jsdelivr.net/npm/medium-zoom/dist/medium-zoom.min.js crossorigin=anonymous referrerpolicy=no-referrer></script>
|
||
<script>const images=Array.from(document.querySelectorAll(".medium-zoom-image"));images.forEach(e=>{mediumZoom(e,{margin:0,scrollOffset:40,container:null,template:null,background:"rgba(0, 0, 0, 0.8)"})})</script><script>window.addEventListener("DOMContentLoaded",function(){docsearch({container:"#docsearch",appId:"5BEWEMH0YA",apiKey:"14834e919a87217d919d6d881fcacac3",indexName:"fastgpt",placeholder:"搜索",translations:{button:{buttonText:"搜索",buttonAriaLabel:"搜索"},modal:{startScreen:{noRecentSearchesText:"没有最近搜索"},footer:{selectText:"选中",selectKeyAriaLabel:"Enter key",navigateText:"导航",navigateUpKeyAriaLabel:"Arrow up",navigateDownKeyAriaLabel:"Arrow down",closeText:"关闭",closeKeyAriaLabel:"Escape key",searchByText:"搜索提供"},noResultsScreen:{noResultsText:"没有结果：",suggestedQueryText:"试试搜索",reportMissingResultsText:"Believe this query should return results?",reportMissingResultsLinkText:"Let us know."}}}})})</script></body></html> |