Article(id=1327010539991052840, tenantId=1146029695717560320, journalId=1324759914146529478, issueId=1327010443811468006, articleNumber=null, orderNo=null, doi=10.7544/issn1000-1239.202440016, pmid=null, cstr=null, oa=null, hot=null, price=null, onlineType=0, articleFormat=0, articleType=null, articleTypeStr=research-article, receivedDate=1704988800000, receivedDateStr=2024-01-12, revisedDate=1726588800000, revisedDateStr=2024-09-18, acceptedDate=1728921600000, acceptedDateStr=2024-10-15, onlineDate=1794313567396, onlineDateStr=2026-11-10, pubDate=1746028800000, pubDateStr=2025-05-01, doiRegisterDate=null, doiRegisterDateStr=null, onlineIssueDate=1794313567396, onlineIssueDateStr=2026-11-10, onlineJustAcceptDate=null, onlineJustAcceptDateStr=null, onlineFirstDate=null, onlineFirstDateStr=null, sourceXml=null, magXml=null, createTime=1794313567396, creator=13701087609, updateTime=1794313567396, updator=13701087609, issue=Issue{id=1327010443811468006, tenantId=1146029695717560320, journalId=1324759914146529478, year='2025', volume='62', issue='5', pageStart='1091', pageEnd='1329', issueExtLink='null', onlineDate='null', pubDate='1746028800000', pubDateStr='2025-05-01', beforeIssueId=null, nextIssueId=null, price=null, status=1, issueComplete=1, articleOrder=1, issueType=-1, specialIssue=null, createTime=1794313544465, creator='13701087609', updateTime=1794376781779, updator='13701087609', preIssue=null, nextIssue=null, articleTotal=null, ext={EN=IssueExt(id=1327275680737374365, tenantId=1146029695717560320, journalId=1324759914146529478, issueId=1327010443811468006, language=EN, specialIssueTitle=, coverIllustrator=null, specialIssueEditor=, specialIssueAbout=), CN=IssueExt(id=1327275680737374366, tenantId=1146029695717560320, journalId=1324759914146529478, issueId=1327010443811468006, language=CN, specialIssueTitle=, coverIllustrator=null, specialIssueEditor=, specialIssueAbout=)}, issueFiles=null, downloadFileDto=null}, startPage=1164, endPage=1189, ext={EN=ArticleExt(id=1327010540213350953, articleId=1327010539991052840, tenantId=1146029695717560320, journalId=1324759914146529478, language=EN, title=Survey on System Optimization for Mixture of Experts in the Era of Large Models, columnId=1325489891789910943, journalTitle=Journal of Computer Research and Development, columnName=Artificial Intelligence, runingTitle=null, highlight=null, articleAbstract=
In recent years, large models have made unprecedented progresses in variety of domains, such as natural language processing and machine vision. Mixture of experts (MoE) has emerged as one of the most popular architectures for large models due to its distinct advantages in model parameter scalability, computational cost control and complex task processing. However, with the continuous increase of the parameter scale, the execution efficiency and scalability of the system are becoming increasingly challenging to meet the demand, and must be addressed urgently. The system optimization approach is an effective solution to solve this problem, which has become a hot research area. In light of this, we review the present research status of MoE system optimization techniques in the era of large model in this paper. To begin, we describe the present development state of work for MoE large model, and analyze the performance bottlenecks it faces on the system side. Then, we comprehensively sort out and deeply analyze the most recent research progress from four system core dimensions, ranging from memory occupation, communication latency, computational efficiency to parallel scaling, and compare and elaborate on the key technologies, application scenarios and optimization directions; finally, we summarize the current research state of MoE system optimization and outline some future research directions as well.
, authors=Hongzhi Shi, Jian Zhao*, Yaqian Zhao, Ruyang Li, Hui Wei, Kekun Hu, Dongchao Wen, Liang Jin, authorsList=Hongzhi Shi, Jian Zhao, Yaqian Zhao, Ruyang Li, Hui Wei, Kekun Hu, Dongchao Wen, Liang Jin, authorCompany=null, correspAuthors=Jian Zhao, authorNote=
born in 1988. Master. Member of CCF. His main research interests include computer architecture and deep learning
born in 1987. Master. His main research interests include computer architecture, and deep learning
born in 1981. PhD, senior engineer. Senior member of CCF. Her main research interests include computer architecture and artificial intelligence
born in 1990. PhD, senior engineer. Senior member of CCF. Her main research interests include deep reinforcement learning, autonomous driving, and scenario-oriented AI acceleration
born in 1987. PhD, senior engineer. Member of CCF. His main research interests include virtual reality and 3D vision
born in 1987. PhD, senior engineer. Member of CCF. His main research interests include graph computing and graph deep learning
born in 1979. Master, professor. Member of IEEE and CCF. His main research interests include computer vision and trustworthy deep learning
born in 1986. Master. His main research interests include computer vision and multimodalities
, correspAuthorsNote=null, copyrightStatement=Copyright © 2025 Journal of Computer Research and Development. All rights reserved., copyrightOwner=null, extLink=null, articleAbsUrl=null, sourceXml=null, magXml=null, pdfUrl=null, pdf=null, pdfFileSize=null, pdfExtLink=null, richHtmlUrl=null, mobilePdfUrl=null, reviewReport=null, pdfFirstPage=null, abstractGraph=null, abstractGraphContent=null, abstractVideo=null, citation=null, cebUrl=null, magXmlContent=null, mapNumber=null, fund=null), CN=ArticleExt(id=1327010546395755100, articleId=1327010539991052840, tenantId=1146029695717560320, journalId=1324759914146529478, language=CN, title=大模型时代的混合专家系统优化综述, columnId=1325489892104483745, journalTitle=计算机研究与发展, columnName=人工智能, runingTitle=null, highlight=null, articleAbstract=
近年来,大模型推动自然语言处理、机器视觉等众多领域取得前所未有的进展. 混合专家(mixture of experts,MoE)凭借在模型参数扩展、计算成本控制和复杂任务处理等方面的独特优势成为大模型的主流架构之一. 然而,随着参数规模的持续增长,系统的执行效率和可扩展能力愈发难以满足需求,亟待解决. 系统优化方法是解决这一挑战的有效途径,日益成为研究热点. 故综述大模型时代MoE系统优化技术的研究现状,首先介绍MoE大模型的发展现状,并分析其在系统端面临的性能瓶颈;然后从内存占用、通信延迟、计算效率和并行扩展4个系统核心维度对最新的研究进展进行全面梳理和深入分析,并对其中涉及的关键技术、适用场景和待优化方向进行详细对比阐述;最后总结MoE系统优化的研究现状,并展望未来研究方向.
, authors=史宏志, 赵健*, 赵雅倩, 李茹杨, 魏辉, 胡克坤, 温东超, 金良, authorsList=史宏志, 赵健, 赵雅倩, 李茹杨, 魏辉, 胡克坤, 温东超, 金良, authorCompany=null, correspAuthors=赵健, authorNote=
 |
1988年生. 硕士. CCF会员. 主要研究方向为计算机体系结构、深度学习 |
 |
1987年生. 硕士. 主要研究方向为计算机体系结构、深度学习 |
 |
1981年生. 博士,高级工程师. CCF高级会员. 主要研究方向为计算机体系结构、人工智能 |
 |
1990年生. 博士,高级工程师. CCF高级会员. 主要研究方向为深度强化学习、自动驾驶、面向场景的AI加速 |
 |
1987年生. 博士,高级工程师. CCF会员. 主要研究方向为虚拟现实、3维视觉 |
 |
1987年生. 博士,高级工程师. CCF会员. 主要研究方向为图计算、图深度学习 |
 |
1979年生. 硕士,正高级工程师. IEEE会员,CCF会员. 主要研究方向为计算机视觉、可信深度学习 |
 |
1986年生. 硕士. 主要研究方向为计算机视觉、多模态 |
, correspAuthorsNote=
, copyrightStatement=版权所有 © 2025 计算机研究与发展编辑部, copyrightOwner=null, extLink=null, articleAbsUrl=null, sourceXml=uuLLROSUd1Sz6jz+veDj0g==, magXml=jiHGDykWYLqBwsGHmGUA8w==, pdfUrl=null, pdf=krXEZRVyFHqs5FEG02VFEQ==, pdfFileSize=2721696, pdfExtLink=null, richHtmlUrl=null, mobilePdfUrl=null, reviewReport=null, pdfFirstPage=null, abstractGraph=EAQ9/caiByqNpefnqCEO5w==, abstractGraphContent=null, abstractVideo=null, citation=null, cebUrl=null, magXmlContent=AiJkugtliSu90IbWMU9iGw==, mapNumber=null, fund=null)}, authors=[Author(id=1327010548778119780, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327010539991052840, orderNo=0, firstName=null, middleName=null, lastName=null, nameCn=null, orcid=null, stid=null, country=null, authorPic=null, dead=0, email=shihzh@ieisystem.com, emailSecond=null, emailThird=null, correspondingAuthor=0, authorType=1, ext={EN=AuthorExt(id=1327010548899754597, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327010539991052840, authorId=1327010548778119780, language=EN, stringName=Hongzhi Shi, firstName=Hongzhi, middleName=null, lastName=Shi, prefix=null, suffix=null, authorComment=null, nameInitials=null, affiliation=null, department=null, xref=null, address=null, bio={"content":"
born in 1988. Master. Member of CCF. His main research interests include computer architecture and deep learning
"}, bioImg=null, bioContent=
born in 1988. Master. Member of CCF. His main research interests include computer architecture and deep learning
, aboutCorrespAuthor=null), CN=AuthorExt(id=1327010548966863462, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327010539991052840, authorId=1327010548778119780, language=CN, stringName=史宏志, firstName=null, middleName=null, lastName=null, prefix=null, suffix=null, authorComment=null, nameInitials=null, affiliation=null, department=null, xref=null, address=null, bio={"img":"nWx+ff72I/pc7KdXsWfLjQ==","content":"
1988年生. 硕士. CCF会员. 主要研究方向为计算机体系结构、深度学习
"}, bioImg=nWx+ff72I/pc7KdXsWfLjQ==, bioContent=
1988年生. 硕士. CCF会员. 主要研究方向为计算机体系结构、深度学习
, aboutCorrespAuthor=null)}, companyList=null), Author(id=1327010549050749544, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327010539991052840, orderNo=1, firstName=null, middleName=null, lastName=null, nameCn=null, orcid=null, stid=null, country=null, authorPic=null, dead=0, email=zhao_jian@ieisystem.com, emailSecond=null, emailThird=null, correspondingAuthor=1, authorType=1, ext={EN=AuthorExt(id=1327010549130441321, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327010539991052840, authorId=1327010549050749544, language=EN, stringName=Jian Zhao, firstName=Jian, middleName=null, lastName=Zhao, prefix=null, suffix=null, authorComment=null, nameInitials=null, affiliation=null, department=null, xref=*, address=null, bio={"content":"
born in 1987. Master. His main research interests include computer architecture, and deep learning
"}, bioImg=null, bioContent=
born in 1987. Master. His main research interests include computer architecture, and deep learning
, aboutCorrespAuthor=null), CN=AuthorExt(id=1327010549205938794, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327010539991052840, authorId=1327010549050749544, language=CN, stringName=赵健, firstName=null, middleName=null, lastName=null, prefix=null, suffix=null, authorComment=null, nameInitials=null, affiliation=null, department=null, xref=*, address=null, bio={"img":"vMb3hI6kB79vZFerVla88w==","content":"
1987年生. 硕士. 主要研究方向为计算机体系结构、深度学习
"}, bioImg=vMb3hI6kB79vZFerVla88w==, bioContent=
1987年生. 硕士. 主要研究方向为计算机体系结构、深度学习
, aboutCorrespAuthor=null)}, companyList=null), Author(id=1327010549264659052, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327010539991052840, orderNo=2, firstName=null, middleName=null, lastName=null, nameCn=null, orcid=null, stid=null, country=null, authorPic=null, dead=0, email=null, emailSecond=null, emailThird=null, correspondingAuthor=0, authorType=1, ext={EN=AuthorExt(id=1327010549377905261, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327010539991052840, authorId=1327010549264659052, language=EN, stringName=Yaqian Zhao, firstName=Yaqian, middleName=null, lastName=Zhao, prefix=null, suffix=null, authorComment=null, nameInitials=null, affiliation=null, department=null, xref=null, address=null, bio={"content":"
born in 1981. PhD, senior engineer. Senior member of CCF. Her main research interests include computer architecture and artificial intelligence
"}, bioImg=null, bioContent=
born in 1981. PhD, senior engineer. Senior member of CCF. Her main research interests include computer architecture and artificial intelligence
, aboutCorrespAuthor=null), CN=AuthorExt(id=1327010549436625518, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327010539991052840, authorId=1327010549264659052, language=CN, stringName=赵雅倩, firstName=null, middleName=null, lastName=null, prefix=null, suffix=null, authorComment=null, nameInitials=null, affiliation=null, department=null, xref=null, address=null, bio={"img":"i+SVzAcHgstHJ3MffaeoSg==","content":"
1981年生. 博士,高级工程师. CCF高级会员. 主要研究方向为计算机体系结构、人工智能
"}, bioImg=i+SVzAcHgstHJ3MffaeoSg==, bioContent=
1981年生. 博士,高级工程师. CCF高级会员. 主要研究方向为计算机体系结构、人工智能
, aboutCorrespAuthor=null)}, companyList=null), Author(id=1327010549495345776, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327010539991052840, orderNo=3, firstName=null, middleName=null, lastName=null, nameCn=null, orcid=null, stid=null, country=null, authorPic=null, dead=0, email=null, emailSecond=null, emailThird=null, correspondingAuthor=0, authorType=1, ext={EN=AuthorExt(id=1327010549558260337, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327010539991052840, authorId=1327010549495345776, language=EN, stringName=Ruyang Li, firstName=Ruyang, middleName=null, lastName=Li, prefix=null, suffix=null, authorComment=null, nameInitials=null, affiliation=null, department=null, xref=null, address=null, bio={"content":"
born in 1990. PhD, senior engineer. Senior member of CCF. Her main research interests include deep reinforcement learning, autonomous driving, and scenario-oriented AI acceleration
"}, bioImg=null, bioContent=
born in 1990. PhD, senior engineer. Senior member of CCF. Her main research interests include deep reinforcement learning, autonomous driving, and scenario-oriented AI acceleration
, aboutCorrespAuthor=null), CN=AuthorExt(id=1327010549633757810, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327010539991052840, authorId=1327010549495345776, language=CN, stringName=李茹杨, firstName=null, middleName=null, lastName=null, prefix=null, suffix=null, authorComment=null, nameInitials=null, affiliation=null, department=null, xref=null, address=null, bio={"img":"nG69HOHi8zves7YNCY6BoA==","content":"
1990年生. 博士,高级工程师. CCF高级会员. 主要研究方向为深度强化学习、自动驾驶、面向场景的AI加速
"}, bioImg=nG69HOHi8zves7YNCY6BoA==, bioContent=
1990年生. 博士,高级工程师. CCF高级会员. 主要研究方向为深度强化学习、自动驾驶、面向场景的AI加速
, aboutCorrespAuthor=null)}, companyList=null), Author(id=1327010549709255284, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327010539991052840, orderNo=4, firstName=null, middleName=null, lastName=null, nameCn=null, orcid=null, stid=null, country=null, authorPic=null, dead=0, email=null, emailSecond=null, emailThird=null, correspondingAuthor=0, authorType=1, ext={EN=AuthorExt(id=1327010549788947061, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327010539991052840, authorId=1327010549709255284, language=EN, stringName=Hui Wei, firstName=Hui, middleName=null, lastName=Wei, prefix=null, suffix=null, authorComment=null, nameInitials=null, affiliation=null, department=null, xref=null, address=null, bio={"content":"
born in 1987. PhD, senior engineer. Member of CCF. His main research interests include virtual reality and 3D vision
"}, bioImg=null, bioContent=
born in 1987. PhD, senior engineer. Member of CCF. His main research interests include virtual reality and 3D vision
, aboutCorrespAuthor=null), CN=AuthorExt(id=1327010549889610358, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327010539991052840, authorId=1327010549709255284, language=CN, stringName=魏辉, firstName=null, middleName=null, lastName=null, prefix=null, suffix=null, authorComment=null, nameInitials=null, affiliation=null, department=null, xref=null, address=null, bio={"img":"tc3jMFBJ1yjw/bmQlqZJPQ==","content":"
1987年生. 博士,高级工程师. CCF会员. 主要研究方向为虚拟现实、3维视觉
"}, bioImg=tc3jMFBJ1yjw/bmQlqZJPQ==, bioContent=
1987年生. 博士,高级工程师. CCF会员. 主要研究方向为虚拟现实、3维视觉
, aboutCorrespAuthor=null)}, companyList=null), Author(id=1327010549981885048, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327010539991052840, orderNo=5, firstName=null, middleName=null, lastName=null, nameCn=null, orcid=null, stid=null, country=null, authorPic=null, dead=0, email=null, emailSecond=null, emailThird=null, correspondingAuthor=0, authorType=1, ext={EN=AuthorExt(id=1327010550065771129, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327010539991052840, authorId=1327010549981885048, language=EN, stringName=Kekun Hu, firstName=Kekun, middleName=null, lastName=Hu, prefix=null, suffix=null, authorComment=null, nameInitials=null, affiliation=null, department=null, xref=null, address=null, bio={"content":"
born in 1987. PhD, senior engineer. Member of CCF. His main research interests include graph computing and graph deep learning
"}, bioImg=null, bioContent=
born in 1987. PhD, senior engineer. Member of CCF. His main research interests include graph computing and graph deep learning
, aboutCorrespAuthor=null), CN=AuthorExt(id=1327010550149657210, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327010539991052840, authorId=1327010549981885048, language=CN, stringName=胡克坤, firstName=null, middleName=null, lastName=null, prefix=null, suffix=null, authorComment=null, nameInitials=null, affiliation=null, department=null, xref=null, address=null, bio={"img":"O1tiUn+MkrmZXvDFyFwffA==","content":"
1987年生. 博士,高级工程师. CCF会员. 主要研究方向为图计算、图深度学习
"}, bioImg=O1tiUn+MkrmZXvDFyFwffA==, bioContent=
1987年生. 博士,高级工程师. CCF会员. 主要研究方向为图计算、图深度学习
, aboutCorrespAuthor=null)}, companyList=null), Author(id=1327010550225154684, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327010539991052840, orderNo=6, firstName=null, middleName=null, lastName=null, nameCn=null, orcid=null, stid=null, country=null, authorPic=null, dead=0, email=null, emailSecond=null, emailThird=null, correspondingAuthor=0, authorType=1, ext={EN=AuthorExt(id=1327010550292263549, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327010539991052840, authorId=1327010550225154684, language=EN, stringName=Dongchao Wen, firstName=Dongchao, middleName=null, lastName=Wen, prefix=null, suffix=null, authorComment=null, nameInitials=null, affiliation=null, department=null, xref=null, address=null, bio={"content":"
born in 1979. Master, professor. Member of IEEE and CCF. His main research interests include computer vision and trustworthy deep learning
"}, bioImg=null, bioContent=
born in 1979. Master, professor. Member of IEEE and CCF. His main research interests include computer vision and trustworthy deep learning
, aboutCorrespAuthor=null), CN=AuthorExt(id=1327010550363566718, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327010539991052840, authorId=1327010550225154684, language=CN, stringName=温东超, firstName=null, middleName=null, lastName=null, prefix=null, suffix=null, authorComment=null, nameInitials=null, affiliation=null, department=null, xref=null, address=null, bio={"img":"zsZ6qfMPNi0+ehVeY+tiZA==","content":"
1979年生. 硕士,正高级工程师. IEEE会员,CCF会员. 主要研究方向为计算机视觉、可信深度学习
"}, bioImg=zsZ6qfMPNi0+ehVeY+tiZA==, bioContent=
1979年生. 硕士,正高级工程师. IEEE会员,CCF会员. 主要研究方向为计算机视觉、可信深度学习
, aboutCorrespAuthor=null)}, companyList=null), Author(id=1327010550439064192, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327010539991052840, orderNo=7, firstName=null, middleName=null, lastName=null, nameCn=null, orcid=null, stid=null, country=null, authorPic=null, dead=0, email=null, emailSecond=null, emailThird=null, correspondingAuthor=0, authorType=1, ext={EN=AuthorExt(id=1327010550527144577, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327010539991052840, authorId=1327010550439064192, language=EN, stringName=Liang Jin, firstName=Liang, middleName=null, lastName=Jin, prefix=null, suffix=null, authorComment=null, nameInitials=null, affiliation=null, department=null, xref=null, address=null, bio={"content":"
born in 1986. Master. His main research interests include computer vision and multimodalities
"}, bioImg=null, bioContent=
born in 1986. Master. His main research interests include computer vision and multimodalities
, aboutCorrespAuthor=null), CN=AuthorExt(id=1327010550590059138, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327010539991052840, authorId=1327010550439064192, language=CN, stringName=金良, firstName=null, middleName=null, lastName=null, prefix=null, suffix=null, authorComment=null, nameInitials=null, affiliation=null, department=null, xref=null, address=null, bio={"img":"9GcQ/uDERH4j/MCodAcd1Q==","content":"
1986年生. 硕士. 主要研究方向为计算机视觉、多模态
"}, bioImg=9GcQ/uDERH4j/MCodAcd1Q==, bioContent=
1986年生. 硕士. 主要研究方向为计算机视觉、多模态
, aboutCorrespAuthor=null)}, companyList=null)], keywords=[Keyword(id=1327010550711693955, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327010539991052840, language=EN, orderNo=1, keyword=large model), Keyword(id=1327010550782997124, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327010539991052840, language=EN, orderNo=2, keyword=mixture of experts), Keyword(id=1327010550845911685, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327010539991052840, language=EN, orderNo=3, keyword=memory offloading), Keyword(id=1327010550913020550, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327010539991052840, language=EN, orderNo=4, keyword=hierarchical communication), Keyword(id=1327010552565576327, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327010539991052840, language=EN, orderNo=5, keyword=expert placement), Keyword(id=1327010552641073800, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327010539991052840, language=EN, orderNo=6, keyword=expert activation prediction), Keyword(id=1327010552703988362, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327010539991052840, language=EN, orderNo=7, keyword=adaptive parallelism), Keyword(id=1327010552762708620, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327010539991052840, language=CN, orderNo=1, keyword=大模型), Keyword(id=1327010552875954829, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327010539991052840, language=CN, orderNo=2, keyword=混合专家), Keyword(id=1327010552972423822, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327010539991052840, language=CN, orderNo=3, keyword=内存卸载), Keyword(id=1327010553039532687, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327010539991052840, language=CN, orderNo=4, keyword=分层通信), Keyword(id=1327010553102447248, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327010539991052840, language=CN, orderNo=5, keyword=专家放置), Keyword(id=1327010553165361810, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327010539991052840, language=CN, orderNo=6, keyword=专家激活预测), Keyword(id=1327010553245053587, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327010539991052840, language=CN, orderNo=7, keyword=自适应并行)], refs=[Reference(id=1327010559523926716, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327010539991052840, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=1, rfOrder=0, authorNames=null, journalName=null, refType=null, unstructuredReference=Kaplan J, McCandlish S, Henighan T, et al. Scaling laws for neural language models[J]. arXiv preprint, arXiv: 2001.08361, 2020, articleTitle=null, refAbstract=null), Reference(id=1327010559683310269, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327010539991052840, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=2, rfOrder=1, authorNames=null, journalName=null, refType=null, unstructuredReference=Raffel C, Shazeer N, Roberts A, et al. Exploring the limits of transfer learning with a unified text-to-text transformer[J]. Journal of Machine Learning Research, 2020, 21(140): 1−67, articleTitle=null, refAbstract=null), Reference(id=1327010559825916606, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327010539991052840, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=3, rfOrder=2, authorNames=null, journalName=null, refType=null, unstructuredReference=Brown T, Mann B, Ryder N, et al. Language models are few-shot learners[C] //Proc of the 34th Int Conf on Neural Information Processing Systems. New York: Curran Associates, 2020: 1877−1901, articleTitle=null, refAbstract=null), Reference(id=1327010559947551423, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327010539991052840, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=4, rfOrder=3, authorNames=null, journalName=null, refType=null, unstructuredReference=Rae J W, Borgeaud S, Cai T, et al. Scaling language models: Methods, analysis & insights from training Gopher[J]. arXiv preprint, arXiv: 2112.11446, 2021, articleTitle=null, refAbstract=null), Reference(id=1327010561537192640, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327010539991052840, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=5, rfOrder=4, authorNames=null, journalName=null, refType=null, unstructuredReference=Chowdhery A, Narang S, Devlin J, et al. PaLM: Scaling language modeling with pathways[J]. arXiv preprint, arXiv:2204.02311, 2022, articleTitle=null, refAbstract=null), Reference(id=1327010561608495810, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327010539991052840, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=6, rfOrder=5, authorNames=null, journalName=null, refType=null, unstructuredReference=Jacobs R A, Jordan M I, Nowlan S J, et al. Adaptive mixtures of local experts[J]. Neural Computation, 1991, 3(1): 79−87, articleTitle=null, refAbstract=null), Reference(id=1327010561675604675, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327010539991052840, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=7, rfOrder=6, authorNames=null, journalName=null, refType=null, unstructuredReference=Riquelme C, Puigcerver J, Mustafa B, et al. Scaling vision with sparse mixture of experts[C] //Proc of the 35th Int Conf on Neural Information Processing Systems. New York: Curran Associates, 2021: 8583−8595, articleTitle=null, refAbstract=null), Reference(id=1327010561751102148, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327010539991052840, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=8, rfOrder=7, authorNames=null, journalName=null, refType=null, unstructuredReference=Fan Zhiwen, Sarkar R, Jiang Ziyu, et al. M3ViT: Mixture-of-experts vision transformer for efficient multi-task learning with model-accelerator co-design[C] //Proc of the 36th Int Conf on Neural Information Processing Systems. New York: Curran Associates, 2022: 28441−28457, articleTitle=null, refAbstract=null), Reference(id=1327010561860154053, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327010539991052840, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=9, rfOrder=8, authorNames=null, journalName=null, refType=null, unstructuredReference=Li Bo, Shen Yifei, Yang Jingkang, et al. Sparse mixture-of-experts are domain generalizable learners[J]. arXiv preprint, arXiv: 2206.04046, 2023, articleTitle=null, refAbstract=null), Reference(id=1327010561927262918, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327010539991052840, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=10, rfOrder=9, authorNames=null, journalName=null, refType=null, unstructuredReference=Xue Fuzhao, Shi Ziji, Wei Futao, et al. Go wider instead of deeper[C] //Proc of the 36th AAAI Conf on Artificial Intelligence. Palo Alto, CA: AAAI, 2022: 8779−8787, articleTitle=null, refAbstract=null), Reference(id=1327010562002760391, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327010539991052840, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=11, rfOrder=10, authorNames=null, journalName=null, refType=null, unstructuredReference=Shen Sheng, Yao Zhewei, Li Chunyuan, et al. Scaling vision-language models with sparse mixture of experts[J]. arXiv preprint, arXiv: 2303.07226, 2023, articleTitle=null, refAbstract=null), Reference(id=1327010562061480648, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327010539991052840, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=12, rfOrder=11, authorNames=null, journalName=null, refType=null, unstructuredReference=Dai Yong, Tang Duyu, Liu Liangxin, et al. One model, multiple modalities: A sparsely activated approach for text, sound, image, video and code[J]. arXiv preprint, arXiv: 2205.06126, 2022, articleTitle=null, refAbstract=null), Reference(id=1327010562128589513, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327010539991052840, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=13, rfOrder=12, authorNames=null, journalName=null, refType=null, unstructuredReference=Mustafa B, Riquelme C, Puigcerver J, et al. Multimodal contrastive learning with LIMoE: The language-image mixture of experts[C] //Proc of the 36th Int Conf on Neural Information Processing Systems. New York: Curran Associates, 2022: 9564−9576, articleTitle=null, refAbstract=null), Reference(id=1327010562216669898, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327010539991052840, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=14, rfOrder=13, authorNames=null, journalName=null, refType=null, unstructuredReference=Kumatani K, Gmyr R, Salinas F C, et al. Building a great multi-lingual teacher with sparsely-gated mixture of experts for speech recognition[J]. arXiv preprint, arXiv: 2112.05820, 2022, articleTitle=null, refAbstract=null), Reference(id=1327010562317333195, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327010539991052840, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=15, rfOrder=14, authorNames=null, journalName=null, refType=null, unstructuredReference=You Zhao, Feng Shulin, Su Dan, et al. SpeechMoE: Scaling to large acoustic models with dynamic routing mixture of experts[J]. arXiv preprint, arXiv: 2105.03036, 2021, articleTitle=null, refAbstract=null), Reference(id=1327010562401219276, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327010539991052840, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=16, rfOrder=15, authorNames=null, journalName=null, refType=null, unstructuredReference=You Zhao, Feng Shulin, Su Dan, et al. Speechmoe2: Mixture-of-experts model with improved routing[C] //Proc of the 2022 IEEE Int Conf on Acoustics, Speech and Signal Processing. Piscataway, NJ: IEEE, 2022: 7217−7221, articleTitle=null, refAbstract=null), Reference(id=1327010562464133837, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327010539991052840, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=17, rfOrder=16, authorNames=null, journalName=null, refType=null, unstructuredReference=Li Dingcheng, Li Xu, Wang Jun, et al. Video recommendation with multi-gate mixture of experts soft actor critic[C] //Proc of the 43rd Int ACM SIGIR Conf on Research and Development in Information Retrieval. New York: ACM, 2020: 1553−1556, articleTitle=null, refAbstract=null), Reference(id=1327010562548019918, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327010539991052840, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=18, rfOrder=17, authorNames=null, journalName=null, refType=null, unstructuredReference=曹泽麟,徐君,董振华,等. 基于多任务学习的位置倾向性得分预测算法[J]. 计算机研究与 发展,2023,60(1):85−94, articleTitle=null, refAbstract=null), Reference(id=1327010562610934479, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327010539991052840, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=18, rfOrder=18, authorNames=null, journalName=null, refType=null, unstructuredReference=Cao Zelin, Xu Jun, Dong Zhenhua, et al. Prediction of the positional propensity scores based on multi task learning[J]. Journal of Computer Research and Development, 2023, 60(1): 85−94 (in Chinese), articleTitle=null, refAbstract=null), Reference(id=1327010562678043344, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327010539991052840, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=19, rfOrder=19, authorNames=null, journalName=null, refType=null, unstructuredReference=Fedus W, Zoph B, Shazeer N. Switch Transformers: Scaling to trillion parameter models with simple and efficient sparsity[J]. arXiv preprint, arXiv: 2101.03961, 2021, articleTitle=null, refAbstract=null), Reference(id=1327010562745152209, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327010539991052840, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=20, rfOrder=20, authorNames=null, journalName=null, refType=null, unstructuredReference=Shazeer N, Mirhoseini A, Maziarz K, et al. Outrageously large neural networks: The sparsely-gated mixture-of-experts layer[J]. arXiv preprint, arXiv: 1701.06538, 2017, articleTitle=null, refAbstract=null), Reference(id=1327010562896147155, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327010539991052840, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=21, rfOrder=21, authorNames=null, journalName=null, refType=null, unstructuredReference=Zoph B, Bello I, Kumar S, et al. ST-MoE: Designing stable and transferable sparse expert models[J]. arXiv preprint, arXiv: 2202.08906, 2022, articleTitle=null, refAbstract=null), Reference(id=1327010562967450324, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327010539991052840, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=22, rfOrder=22, authorNames=null, journalName=null, refType=null, unstructuredReference=Lee-Thorp J, Ainslie J. Sparse mixers: Combining MoE and mixing to build a more efficient bert[J]. arXiv preprint, arXiv: 2205.12399, 2022, articleTitle=null, refAbstract=null), Reference(id=1327010563051336405, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327010539991052840, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=23, rfOrder=23, authorNames=null, journalName=null, refType=null, unstructuredReference=Kudugunta S, Huang Y, Bapna A, et al. Beyond distillation: Task-level mixture-of-experts for efficient inference[J]. arXiv preprint, arXiv: 2110.03742, 2021, articleTitle=null, refAbstract=null), Reference(id=1327010563135222486, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327010539991052840, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=24, rfOrder=24, authorNames=null, journalName=null, refType=null, unstructuredReference=Du Nan, Huang Yanping, Dai A M, et al. GLaM: Efficient scaling of language models with mixture-of-experts[C] //Proc of the 39th Int Conf on Machine Learning. New York: PMLR, 2022: 5547−5569, articleTitle=null, refAbstract=null), Reference(id=1327010563244274391, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327010539991052840, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=25, rfOrder=25, authorNames=null, journalName=null, refType=null, unstructuredReference=Lou Yuxuan, Xue Fuzhao, Zheng Zangwei, et al. Cross-token modeling with conditional computation[J]. arXiv preprint, arXiv: 2109.02008, 2022, articleTitle=null, refAbstract=null), Reference(id=1327010563319771864, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327010539991052840, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=26, rfOrder=26, authorNames=null, journalName=null, refType=null, unstructuredReference=Lin Junyang, Men Rui, Yang An, et al. M6: A Chinese multimodal pretrainer[J]. arXiv preprint, arXiv: 2103.00823, 2021, articleTitle=null, refAbstract=null), Reference(id=1327010563403657945, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327010539991052840, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=27, rfOrder=27, authorNames=null, journalName=null, refType=null, unstructuredReference=Lin Junyang, Yang An, Bai Jinze, et al. M6-10T: A sharing-delinking paradigm for efficient multi-trillion parameter pretraining[J]. arXiv preprint, arXiv: 2110.03888, 2021, articleTitle=null, refAbstract=null), Reference(id=1327010563755979482, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327010539991052840, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=28, rfOrder=28, authorNames=null, journalName=null, refType=null, unstructuredReference=Ren Xiaozhe, Zhou Pingyi, Meng Xinfan, et al. PANGU-Σ: Towards trillion parameter language model with sparse heterogeneous computing[J]. arXiv preprint, arXiv: 2303.10845, 2023, articleTitle=null, refAbstract=null), Reference(id=1327010563827282651, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327010539991052840, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=29, rfOrder=29, authorNames=null, journalName=null, refType=null, unstructuredReference=Jiang A Q, Sablayrolles A, Roux A, et al. Mixtral of experts[J]. arXiv preprint, arXiv: 2401.04088, 2024, articleTitle=null, refAbstract=null), Reference(id=1327010563902780124, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327010539991052840, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=30, rfOrder=30, authorNames=null, journalName=null, refType=null, unstructuredReference=Nguyen H D, Chamroukhi F. Practical and theoretical aspects of mixture‐of‐experts modeling: An overview[J]. Wiley Interdisciplinary Reviews: Data Mining and Knowledge Discovery, 2018, 8(4): e1246, articleTitle=null, refAbstract=null), Reference(id=1327010563965694685, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327010539991052840, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=31, rfOrder=31, authorNames=null, journalName=null, refType=null, unstructuredReference=Masoudnia S, Ebrahimpour R. Mixture of experts: A literature survey[J]. Artificial Intelligence Review, 2014, 42(2): 275−293, articleTitle=null, refAbstract=null), Reference(id=1327010564028609246, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327010539991052840, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=32, rfOrder=32, authorNames=null, journalName=null, refType=null, unstructuredReference=Yuksel S E, Wilson J N, Gader P D. Twenty years of mixture of experts[J]. IEEE Transactions on Neural Networks and Learning Systems, 2012, 23(8): 1177−1193, articleTitle=null, refAbstract=null), Reference(id=1327010564116689631, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327010539991052840, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=33, rfOrder=33, authorNames=null, journalName=null, refType=null, unstructuredReference=Fedus W, Dean J, Zoph B. A review of sparse expert models in deep learning[J]. arXiv preprint, arXiv: 2209.01667, 2022, articleTitle=null, refAbstract=null), Reference(id=1327010564200575712, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327010539991052840, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=34, rfOrder=34, authorNames=null, journalName=null, refType=null, unstructuredReference=Liu Tianlin, Blondel M, Riquelme C, et al. Routers in vision mixture of experts: An empirical study[J]. arXiv preprint, arXiv: 2401.15969, 2024, articleTitle=null, refAbstract=null), Reference(id=1327010564267684577, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327010539991052840, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=35, rfOrder=35, authorNames=null, journalName=null, refType=null, unstructuredReference=Cai Weilin, Jiang Juyong, Wang Fan, et al. A survey on mixture of experts[J]. arXiv preprint, arXiv: 2407.06204, 2024, articleTitle=null, refAbstract=null), Reference(id=1327010564330599138, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327010539991052840, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=36, rfOrder=36, authorNames=null, journalName=null, refType=null, unstructuredReference=Lepikhin D, Lee H, Xu Yuandong, et al. GShard: Scaling giant models with conditional computation and automatic sharding [J]. arXiv preprint, arXiv: 2006.16668, 2020, articleTitle=null, refAbstract=null), Reference(id=1327010565987349220, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327010539991052840, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=37, rfOrder=37, authorNames=null, journalName=null, refType=null, unstructuredReference=Zhang Zhengyan, Lin Yankai, Liu Zhiyuan, et al. MoEfication: Transformer feed-forward layers are mixtures of experts[J]. arXiv preprint, arXiv: 2110.01786, 2021, articleTitle=null, refAbstract=null), Reference(id=1327010566067040997, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327010539991052840, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=38, rfOrder=38, authorNames=null, journalName=null, refType=null, unstructuredReference=Zuo Simiao, Zhang Qingru, Liang Chen, et al. MoEBERT: From BERT to mixture-of-experts via importance-guided adaptation[J]. arXiv preprint, arXiv: 2204.07675, 2022, articleTitle=null, refAbstract=null), Reference(id=1327010566129955558, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327010539991052840, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=39, rfOrder=39, authorNames=null, journalName=null, refType=null, unstructuredReference=Zhu Tong, Qu Xiaoye, Dong Daize, et al. LLaMA-MoE: Building mixture-of-experts from LLaMA with continual pre-training[J]. arXiv preprint, arXiv: 2406.16554, 2024, articleTitle=null, refAbstract=null), Reference(id=1327010566209647335, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327010539991052840, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=40, rfOrder=40, authorNames=null, journalName=null, refType=null, unstructuredReference=Dai Damai, Deng Chenqi, Zhao Chenggang, et al. DeepSeekMoE: Towards ultimate expert specialization in mixture-of-experts language models[J]. arXiv preprint, arXiv: 2401.06066, 2024, articleTitle=null, refAbstract=null), Reference(id=1327010566280950504, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327010539991052840, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=41, rfOrder=41, authorNames=null, journalName=null, refType=null, unstructuredReference=Xue Fuzhao, Zheng Zian, Fu Yao, et al. OpenMoE: An early effort on open mixture-of-experts language models[J]. arXiv preprint, arXiv: 2402.01739, 2024, articleTitle=null, refAbstract=null), Reference(id=1327010566360642281, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327010539991052840, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=42, rfOrder=42, authorNames=null, journalName=null, refType=null, unstructuredReference=xAI. Open release of Grok-1[EB/OL]. [2024-08-02]. https://x.ai/blog/ grok-os, articleTitle=null, refAbstract=null), Reference(id=1327010566444528363, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327010539991052840, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=43, rfOrder=43, authorNames=null, journalName=null, refType=null, unstructuredReference=Reid M, Savinov N, Teplyashin D, et al. Gemini 1.5: Unlocking multimodal understanding across millions of tokens of context[J]. arXiv preprint, arXiv: 2403.05530, 2024, articleTitle=null, refAbstract=null), Reference(id=1327010566528414444, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327010539991052840, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=44, rfOrder=44, authorNames=null, journalName=null, refType=null, unstructuredReference=Snowflake AI Research Team. Snowflake arctic: The best LLM for enterprise AI — Efficiently intelligent, truly open[EB/OL]. [2024-08-02]. https://www.snowflake.com/en/blog/arctic-open-efficient-foundation-language-models-snowflake/, articleTitle=null, refAbstract=null), Reference(id=1327010566603911917, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327010539991052840, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=45, rfOrder=45, authorNames=null, journalName=null, refType=null, unstructuredReference=The Mosaic Research Team. Introducin DBRX: A new state-of-the-art open LLM[EB/OL]. 2024 [2024-08-02]. https://www.databricks.com/blog/introducing-dbrx-new-state-art-open-llm, articleTitle=null, refAbstract=null), Reference(id=1327010566679409390, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327010539991052840, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=46, rfOrder=46, authorNames=null, journalName=null, refType=null, unstructuredReference=Choquette J, Gandhi W, Giroux O, et al. Nvidia A100 tensor core GPU: Performance and innovation[J]. IEEE Micro, 2021, 41(2): 29−35, articleTitle=null, refAbstract=null), Reference(id=1327010566733935343, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327010539991052840, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=47, rfOrder=47, authorNames=null, journalName=null, refType=null, unstructuredReference=Choquette J. Nvidia Hopper H100 GPU: Scaling performance[J]. IEEE Micro, 2023, 43(3): 9−17, articleTitle=null, refAbstract=null), Reference(id=1327010566801044208, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327010539991052840, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=48, rfOrder=48, authorNames=null, journalName=null, refType=null, unstructuredReference=Ren Jie, Rajbhandari S, Aminabadi R Y, et al. ZeRO-Offload: Democratizing billion-scale model training[C] //Proc of the 2021 USENIX Annual Technical Conf. Berkeley, CA: USENIX Association, 2021: 551−564, articleTitle=null, refAbstract=null), Reference(id=1327010566905901809, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327010539991052840, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=49, rfOrder=49, authorNames=null, journalName=null, refType=null, unstructuredReference=Chen Xiaoming, Chen D Z, Hu X S. MoDNN: Memory optimal DNN training on GPUs[C] //Proc of the 21st Conf & Exhibition on Design, Automation & Test in Europe. Piscataway, NJ: IEEE, 2018: 13−18, articleTitle=null, refAbstract=null), Reference(id=1327010567031730930, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327010539991052840, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=50, rfOrder=50, authorNames=null, journalName=null, refType=null, unstructuredReference=Shriram S B, Garg A, Kulkarni P. Dynamic memory management for GPU-based training of deep neural networks[C] //Proc of the 33rd IEEE Int Parallel and Distributed Processing Symp. Piscataway, NJ: IEEE, 2019: 200−209, articleTitle=null, refAbstract=null), Reference(id=1327010567107228403, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327010539991052840, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=51, rfOrder=51, authorNames=null, journalName=null, refType=null, unstructuredReference=Ren Jie, Luo Jiaolin, Wu Kai, et al. Sentinel: Efficient tensor migration and allocation on heterogeneous memory systems for deep learning[C] //Proc of the 27th IEEE Transactions Symp on High Performance Computer Architecture. Piscataway, NJ: IEEE, 2021: 598−611, articleTitle=null, refAbstract=null), Reference(id=1327010567207891700, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327010539991052840, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=52, rfOrder=52, authorNames=null, journalName=null, refType=null, unstructuredReference=Huang C C, Jin Gu, Li Jinyang. SwapAdvisor: Pushing deep learning beyond the GPU memory limit via smart swapping[C] //Proc of the 25th Int Conf on Architectural Support for Programming Languages and Operating Systems. New York: ACM, 2020: 1341−1355, articleTitle=null, refAbstract=null), Reference(id=1327010567337915125, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327010539991052840, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=53, rfOrder=53, authorNames=null, journalName=null, refType=null, unstructuredReference=Huang Haiyang, Ardalani N, Sun Anna, et al. Towards MoE deployment: Mitigating inefficiencies in mixture-of-expert (MoE) inference[J]. arXiv preprint, arXiv: 2303.06182, 2023, articleTitle=null, refAbstract=null), Reference(id=1327010567425995510, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327010539991052840, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=54, rfOrder=54, authorNames=null, journalName=null, refType=null, unstructuredReference=Eliseev A, Mazur D. Fast inference of mixture-of-experts language models with offloading[J]. arXiv preprint, arXiv: 2312.17238, 2023, articleTitle=null, refAbstract=null), Reference(id=1327010567518270199, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327010539991052840, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=55, rfOrder=55, authorNames=null, journalName=null, refType=null, unstructuredReference=Kong Rui, Li Yuanchun, Feng Qingtian, et al. Serving MoE models on resource-constrained edge devices via dynamic expert swapping[J]. arXiv preprint, arXiv: 2308.15030, 2023, articleTitle=null, refAbstract=null), Reference(id=1327010567631516408, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327010539991052840, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=56, rfOrder=56, authorNames=null, journalName=null, refType=null, unstructuredReference=Hwang R, Wei Jianyu, Cao Shijie, et al. Pre-gated MoE: An algorithm-system co-design for fast and scalable mixture-of-expert inference[J]. arXiv preprint, arXiv: 2308.12066, 2023, articleTitle=null, refAbstract=null), Reference(id=1327010567774122745, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327010539991052840, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=57, rfOrder=57, authorNames=null, journalName=null, refType=null, unstructuredReference=Shen Liang, Wu Zhihua, Gong Weibao, et al. SE-MoE: A scalable and efficient mixture-of-experts distributed training and inference system[J]. arXiv preprint, arXiv: 2205.10034, 2023, articleTitle=null, refAbstract=null), Reference(id=1327010567895757562, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327010539991052840, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=58, rfOrder=58, authorNames=null, journalName=null, refType=null, unstructuredReference=Liu Juncai, Wang J H, Jiang Yimin. Janus: A unified distributed training framework for sparse mixture-of-experts models[C] //Proc of the 37th ACM SIGCOMM 2023 Conf. New York: ACM, 2023: 486−498, articleTitle=null, refAbstract=null), Reference(id=1327010567988032251, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327010539991052840, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=59, rfOrder=59, authorNames=null, journalName=null, refType=null, unstructuredReference=Kim Y, Lim H, Han D. Scaling beyond the GPU memory limit for large mixture-of-experts model training[C] //Proc of the 41st Int Conf on Machine Learning. New York: PMLR, 2024: 24342−24353, articleTitle=null, refAbstract=null), Reference(id=1327010568046752508, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327010539991052840, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=60, rfOrder=60, authorNames=null, journalName=null, refType=null, unstructuredReference=Yi Rongjie, Guo Liwei, Wei Shiyun, et al. EdgeMoE: Fast on-device inference of MoE-based large language models[J]. arXiv preprint, arXiv: 2308.14352, 2023, articleTitle=null, refAbstract=null), Reference(id=1327010568122249981, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327010539991052840, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=61, rfOrder=61, authorNames=null, journalName=null, refType=null, unstructuredReference=Xue Leyang, Fu Yao, Lu Zhan, et al. MoE-Infinity: Activation-aware expert offloading for efficient MoE serving[J]. arXiv preprint, arXiv: 2401.14361, 2024, articleTitle=null, refAbstract=null), Reference(id=1327010568248079102, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327010539991052840, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=62, rfOrder=62, authorNames=null, journalName=null, refType=null, unstructuredReference=Kamahori K, Gu Yile, Zhu Kan, et al. Fiddler: CPU-GPU orchestration for fast inference of mixture-of-experts models[J]. arXiv preprint, arXiv: 2402.07033, 2024, articleTitle=null, refAbstract=null), Reference(id=1327010568386491135, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327010539991052840, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=63, rfOrder=63, authorNames=null, journalName=null, refType=null, unstructuredReference=Zhang Zheng, Xia Yaqi, Wang Hulin, et al. MPipeMoE: Memory efficient MoE for pre-trained models with adaptive pipeline parallelism[C] //Proc of the 37th IEEE Parallel and Distributed Processing Symp. Piscataway, NJ: IEEE, 2023: 167−177, articleTitle=null, refAbstract=null), Reference(id=1327010568537486080, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327010539991052840, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=64, rfOrder=64, authorNames=null, journalName=null, refType=null, unstructuredReference=Jain P, Jain A, Nrusimha A, et al. Checkmate: Breaking the memory wall with optimal tensor rematerialization[C/OL] //Proc of the 3rd Machine Learning and Systems. 2020 [2024-08-02]. https://proceedings.mlsys.org/paper_files/paper/2020/file/0b816ae8f06f8dd3543dc3d9ef196cab-Paper.pdf, articleTitle=null, refAbstract=null), Reference(id=1327010568621372161, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327010539991052840, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=65, rfOrder=65, authorNames=null, journalName=null, refType=null, unstructuredReference=Chen Tianqi, Xu Bing, Zhang Chiyuan, et al. Training deep nets with sublinear memory cost[J]. arXiv preprint, arXiv: 1604.06174, 2016, articleTitle=null, refAbstract=null), Reference(id=1327010568717841154, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327010539991052840, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=66, rfOrder=66, authorNames=null, journalName=null, refType=null, unstructuredReference=Peng Xuan, Shi Xuanhua, Dai Hulin, et al. Capuchin: Tensor-based GPU memory management for deep learning[C] //Proc of the 25th Int Conf on Architectural Support for Programming Languages and Operating Systems. New York: ACM, 2020: 891−905, articleTitle=null, refAbstract=null), Reference(id=1327010568780755715, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327010539991052840, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=67, rfOrder=67, authorNames=null, journalName=null, refType=null, unstructuredReference=Wang Linnan, Ye Jinmian, Zhao Yiyang, et al. SuperNeurons: Dynamic GPU memory management for training deep neural networks[J]. ACM SIGPLAN Notices, 2018, 53(1): 41−53, articleTitle=null, refAbstract=null), Reference(id=1327010568852058884, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327010539991052840, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=68, rfOrder=68, authorNames=null, journalName=null, refType=null, unstructuredReference=Kim Y J, Awan A A, Muzio A, et al. Scalable and efficient MoE training for multitask multilingual models[J]. arXiv preprint, arXiv: 2109.10465, 2021, articleTitle=null, refAbstract=null), Reference(id=1327010570596889349, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327010539991052840, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=69, rfOrder=69, authorNames=null, journalName=null, refType=null, unstructuredReference=Singh S, Ruwase O, Awan A A, et al. A hybrid tensor-expert-data parallelism approach to optimize mixture-of-experts training[C] //Proc of the 37th Int Conf on Supercomputing. New York: ACM, 2023: 203−214, articleTitle=null, refAbstract=null), Reference(id=1327010570726912774, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327010539991052840, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=70, rfOrder=70, authorNames=null, journalName=null, refType=null, unstructuredReference=Heo T, Rashidi S, Man C, et al. Exploring memory expansion designs for training mixture-of-experts models[C/OL] //Proc of the 1st Workshop on Hot Topics in System Infrastructure. 2024 [2024-08-02]. https:// hotinfra23.github.io/papers/hotinfra23-paper4.pdf, articleTitle=null, refAbstract=null), Reference(id=1327010570802410247, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327010539991052840, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=71, rfOrder=71, authorNames=null, journalName=null, refType=null, unstructuredReference=Hinton G, Vinyals O, Dean J. Distilling the knowledge in a neural network[J]. arXiv preprint, arXiv: 1503.02531, 2015, articleTitle=null, refAbstract=null), Reference(id=1327010570869519112, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327010539991052840, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=72, rfOrder=72, authorNames=null, journalName=null, refType=null, unstructuredReference=Molchanov P, Tyree S, Karras T, et al. Pruning convolutional neural networks for resource efficient inference[J]. arXiv preprint, arXiv: 1611.06440, 2016, articleTitle=null, refAbstract=null), Reference(id=1327010570940822281, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327010539991052840, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=73, rfOrder=73, authorNames=null, journalName=null, refType=null, unstructuredReference=Han Song, Mao Huizi, Dally W J. Deep compression: Compressing deep neural networks with pruning, trained quantization and Huffman coding[J]. arXiv preprint, arXiv: 1510.00149, 2016, articleTitle=null, refAbstract=null), Reference(id=1327010571012125450, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327010539991052840, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=74, rfOrder=74, authorNames=null, journalName=null, refType=null, unstructuredReference=Dong Zhen, Yao Zhewei, Arfeen D, et al. HAWQ-V2: Hessian aware trace-weighted quantization of neural networks[C] //Proc of the 34th Int Conf on Neural Information Processing Systems. New York: Curran Associates, 2020: 18518−18529, articleTitle=null, refAbstract=null), Reference(id=1327010571087622923, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327010539991052840, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=75, rfOrder=75, authorNames=null, journalName=null, refType=null, unstructuredReference=Micikevicius P, Narang S, Alben J, et al. Mixed precision training[J]. arXiv preprint, arXiv: 1710.03740, 2018, articleTitle=null, refAbstract=null), Reference(id=1327010571158926092, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327010539991052840, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=76, rfOrder=76, authorNames=null, journalName=null, refType=null, unstructuredReference=Dabre R, Fujita A. Recurrent stacking of layers in neural networks: An application to neural machine translation[J]. arXiv preprint, arXiv: 2106.10002, 2021, articleTitle=null, refAbstract=null), Reference(id=1327010571238617869, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327010539991052840, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=77, rfOrder=77, authorNames=null, journalName=null, refType=null, unstructuredReference=Lan Zhenzhong, Chen Mingda, Goodman S, et al. ALBERT: A lite BERT for self-supervised learning of language representations[J]. arXiv preprint, arXiv: 1909.11942, 2020, articleTitle=null, refAbstract=null), Reference(id=1327010571301532430, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327010539991052840, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=78, rfOrder=78, authorNames=null, journalName=null, refType=null, unstructuredReference=Rajbhandari S, Li Conglong, Yao Zhewei, et al. DeepSpeed-MoE: Advancing mixture-of-experts inference and training to power next-generation AI scale[C] //Proc of the 39th Int Conf on Machine Learning. New York: PMLR, 2022: 18332−18346, articleTitle=null, refAbstract=null), Reference(id=1327010571414778639, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327010539991052840, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=79, rfOrder=79, authorNames=null, journalName=null, refType=null, unstructuredReference=Koishekenov Y, Berard A, Nikoulina V. Memory-efficient NLLB−200: Language-specific expert pruning of a massively multilingual machine translation model[J]. arXiv preprint, arXiv: 2212.09811, 2022, articleTitle=null, refAbstract=null), Reference(id=1327010571481887504, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327010539991052840, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=80, rfOrder=80, authorNames=null, journalName=null, refType=null, unstructuredReference=Lu Xudong, Liu Qi, Xu Yuhui, et al. Not all experts are equal: Efficient expert pruning and skipping for mixture-of-experts large language models[J]. arXiv preprint, arXiv: 2402.14800, 2024, articleTitle=null, refAbstract=null), Reference(id=1327010571557384977, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327010539991052840, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=81, rfOrder=81, authorNames=null, journalName=null, refType=null, unstructuredReference=Muzio A, Sun A, He C. SEER-MoE: Sparse expert efficiency through regularization for mixture-of-experts[J]. arXiv preprint, arXiv: 2404.05089, 2024, articleTitle=null, refAbstract=null), Reference(id=1327010571616105234, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327010539991052840, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=82, rfOrder=82, authorNames=null, journalName=null, refType=null, unstructuredReference=Chowdhury M N R, Wang Meng, Maghraoui K E, et al. A provably effective method for pruning experts in fine-tuned sparse mixture-of-experts[J]. arXiv preprint, arXiv: 2405.16646, 2024, articleTitle=null, refAbstract=null), Reference(id=1327010571695797011, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327010539991052840, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=83, rfOrder=83, authorNames=null, journalName=null, refType=null, unstructuredReference=Liu Enshu, Zhu Junyi, Lin Zinan, et al. Efficient expert pruning for sparse mixture-of-experts language models: Enhancing performance and reducing inference costs[J]. arXiv preprint, arXiv: 2407.00945, 2024, articleTitle=null, refAbstract=null), Reference(id=1327010571754517268, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327010539991052840, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=84, rfOrder=84, authorNames=null, journalName=null, refType=null, unstructuredReference=Kim Y J, Fahim R, Awadalla H H. Mixture of quantized experts (MoQE): Complementary effect of low-bit quantization and robustness[J]. arXiv preprint, arXiv: 2310.02410, 2023, articleTitle=null, refAbstract=null), Reference(id=1327010571825820437, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327010539991052840, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=85, rfOrder=85, authorNames=null, journalName=null, refType=null, unstructuredReference=Frantar E, Alistarh D. QMoE: Practical sub−1-bit compression of trillion-parameter models[J]. arXiv preprint, arXiv: 2310.16795, 2023, articleTitle=null, refAbstract=null), Reference(id=1327010571888734998, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327010539991052840, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=86, rfOrder=86, authorNames=null, journalName=null, refType=null, unstructuredReference=Kim Y J, Henry R, Fahim R, et al. Who says elephants can’t run: Bringing large scale MoE models into cloud scale production[J]. arXiv preprint, arXiv: 2211.10017, 2022, articleTitle=null, refAbstract=null), Reference(id=1327010571964232471, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327010539991052840, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=87, rfOrder=87, authorNames=null, journalName=null, refType=null, unstructuredReference=Kim Y J, Henry R, Fahim R, et al. FineQuant: Unlocking efficiency with fine-grained weight-only quantization for LLMs[J]. arXiv preprint, arXiv: 2308.09723, 2023, articleTitle=null, refAbstract=null), Reference(id=1327010572031341336, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327010539991052840, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=88, rfOrder=88, authorNames=null, journalName=null, refType=null, unstructuredReference=Imani H R, Amirany A, El-Ghazawi T. Mixture of experts with mixture of precisions for tuning quality of service[J]. arXiv preprint, arXiv: 2407.14417, 2024, articleTitle=null, refAbstract=null), Reference(id=1327010572094255897, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327010539991052840, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=89, rfOrder=89, authorNames=null, journalName=null, refType=null, unstructuredReference=Gao Zefeng, Liu Peiyu, Zhao Xin, et al. Parameter-efficient mixture-of-experts architecture for pre-trained language models[J]. arXiv preprint, arXiv: 2203.01104, 2022, articleTitle=null, refAbstract=null), Reference(id=1327010572157170458, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327010539991052840, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=90, rfOrder=90, authorNames=null, journalName=null, refType=null, unstructuredReference=He S, Fan R Z, Ding Liang, et al. Merging experts into one: Improving computational efficiency of mixture of experts[J]. arXiv preprint, arXiv: 2310.09832, 2023, articleTitle=null, refAbstract=null), Reference(id=1327010572257833755, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327010539991052840, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=91, rfOrder=91, authorNames=null, journalName=null, refType=null, unstructuredReference=Zhang Rongyu, Luo Yulin, Liu Jiaming, et al. Efficient deweahter mixture-of-experts with uncertainty-aware feature-wise linear modulation[C] //Proc of the 38th AAAI Conf on Artificial Intelligence. Palo Alto, CA: AAAI, 2024: 16812−16820, articleTitle=null, refAbstract=null), Reference(id=1327010572366885660, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327010539991052840, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=92, rfOrder=92, authorNames=null, journalName=null, refType=null, unstructuredReference=Chen Tianyu, Huang Shaohan, Xie Yuan, et al. Task-specific expert pruning for sparse mixture-of-experts[J]. arXiv preprint, arXiv: 2206.00277, 2022, articleTitle=null, refAbstract=null), Reference(id=1327010572433994525, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327010539991052840, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=93, rfOrder=93, authorNames=null, journalName=null, refType=null, unstructuredReference=Xue Fuzhao, He Xiaoxin, Ren Xiaozhe, et al. One student knows all experts know: From sparse to dense[J]. arXiv preprint, arXiv: 2201.10890, 2022, articleTitle=null, refAbstract=null), Reference(id=1327010572501103390, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327010539991052840, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=94, rfOrder=94, authorNames=null, journalName=null, refType=null, unstructuredReference=Li Jiamin, Jiang Yimin, Zhu Yibo, et al. Accelerating distributed MoE training and inference with Lina[C] //Proc of the 2023 USENIX Annual Technical Conf. Berkeley, CA: USENIX Association, 2023: 945−959, articleTitle=null, refAbstract=null), Reference(id=1327010572568212255, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327010539991052840, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=95, rfOrder=95, authorNames=null, journalName=null, refType=null, unstructuredReference=Nie Xiaonan, Zhao Pinxue, Miao Xupeng, et al. HetuMoE: An efficient trillion-scale mixture-of-expert distributed training system[J]. arXiv preprint, arXiv: 2203.14685, 2022, articleTitle=null, refAbstract=null), Reference(id=1327010572631126816, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327010539991052840, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=96, rfOrder=96, authorNames=null, journalName=null, refType=null, unstructuredReference=Hwang C, Cui Wei, Xiong Yifan, et al. Tutel: Adaptive mixture-of-experts at scale[C/OL] //Proc of the 6th Machine Learning and Systems. 2023 [2024-08-02]. https://proceedings.mlsys.org/paper_files/paper/2023/file/5616d34cf8ff73942cfd5aa922842556-Paper-mlsys2023.pdf, articleTitle=null, refAbstract=null), Reference(id=1327010572706624289, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327010539991052840, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=97, rfOrder=97, authorNames=null, journalName=null, refType=null, unstructuredReference=He Chaoyang, Zheng Shuai, Zhang A, et al. SMILE: Scaling mixture-of-experts with efficient bi-level routing[J]. arXiv preprint, arXiv: 2212.05191, 2022, articleTitle=null, refAbstract=null), Reference(id=1327010572773733154, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327010539991052840, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=98, rfOrder=98, authorNames=null, journalName=null, refType=null, unstructuredReference=Shoeybi M, Patwary M, Puri R, et al. Megatron-LM: Training multi-billion parameter language models using model parallelism[J]. arXiv preprint, arXiv: 1909.08053, 2020, articleTitle=null, refAbstract=null), Reference(id=1327010572840842019, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327010539991052840, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=99, rfOrder=99, authorNames=null, journalName=null, refType=null, unstructuredReference=Yao Jinghan, Anthony Q, Shafi A, et al. Exploiting inter-layer expert affinity for accelerating mixture-of-experts model inference[C] //Proc of the 38th IEEE Int Parallel and Distributed Processing Symp. Piscataway, NJ: IEEE, 2024: 915−925, articleTitle=null, refAbstract=null), Reference(id=1327010572907950884, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327010539991052840, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=100, rfOrder=100, authorNames=null, journalName=null, refType=null, unstructuredReference=Liu Rui, Kim Y J, Muzio A, et al. Gating Dropout: Communication-efficient regularization for sparsely activated transformers[C] //Proc of the 39th Int Conf on Machine Learning. New York: PMLR, 2022: 13782−13792, articleTitle=null, refAbstract=null), Reference(id=1327010572991836966, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327010539991052840, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=101, rfOrder=101, authorNames=null, journalName=null, refType=null, unstructuredReference=He Jiaao, Zhai Jidong, Antunes T, et al. FasterMoE: Modeling and optimizing training of large-scale dynamic pre-trained models[C] //Proc of the 27th ACM SIGPLAN Symp on Principles and Practice of Parallel Programming. New York: ACM, 2022: 120−134, articleTitle=null, refAbstract=null), Reference(id=1327010573075723047, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327010539991052840, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=102, rfOrder=102, authorNames=null, journalName=null, refType=null, unstructuredReference=Chen Chang, Li Min, Wu Zhihua, et al. TA-MoE: Topology-aware large scale mixture-of-expert training[C] //Proc of the 36th Int Conf on Neural Information Processing Systems. New York: Curran Associates, 2022: 22173−22186, articleTitle=null, refAbstract=null), Reference(id=1327010573159609128, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327010539991052840, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=103, rfOrder=103, authorNames=null, journalName=null, refType=null, unstructuredReference=Zeng Zhiyuan, Xiong Deyi. SCoMoE: Efficient mixtures of experts with structured communication[C] //Proc of the 11th Int Conf on Learning Representations. Amherst, MA: OpenReview. net, 2023: 1−23, articleTitle=null, refAbstract=null), Reference(id=1327010573230912297, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327010539991052840, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=104, rfOrder=104, authorNames=null, journalName=null, refType=null, unstructuredReference=Kossmann F, Jia Zhihao, Aiken A. Optimizing mixture of experts using dynamic recompilations[J]. arXiv preprint, arXiv: 2205.01848, 2022, articleTitle=null, refAbstract=null), Reference(id=1327010573285438250, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327010539991052840, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=105, rfOrder=105, authorNames=null, journalName=null, refType=null, unstructuredReference=Zheng Bojian, Jiang Ziheng, Yu C H, et al. DietCode: Automatic optimization for dynamic tensor programs[C/OL] //Proc of the 5th Machine Learning and Systems. 2022 [2024-08-02]. https://proceedings.mlsys.org/paper_files/paper/2022/file/f89b79c9a28d4cae22ef9e557d9fa191-Paper.pdf, articleTitle=null, refAbstract=null), Reference(id=1327010573344158507, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327010539991052840, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=106, rfOrder=106, authorNames=null, journalName=null, refType=null, unstructuredReference=Zheng Zhen, Pan Zaifeng, Wang Dalin, et al. BladeDISC: Optimizing dynamic shape machine learning workloads via compiler approach[J]. Proceedings of ACM on Management of Data, 2023, 1(3): 1−29, articleTitle=null, refAbstract=null), Reference(id=1327010575009297196, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327010539991052840, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=107, rfOrder=107, authorNames=null, journalName=null, refType=null, unstructuredReference=Chen Simin, Wei Shiyi, Liu Cong, et al. DyCL: Dynamic neural network compilation via program rewriting and graph optimization[C] //Proc of the 32nd ACM SIGSOFT Int Symp on Software Testing and Analysis. New York: ACM, 2023: 614−626, articleTitle=null, refAbstract=null), Reference(id=1327010575093183278, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327010539991052840, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=108, rfOrder=108, authorNames=null, journalName=null, refType=null, unstructuredReference=Yu Feng, Li Guanglin, Zhao Jiacheng, et al. Optimizing dynamic-shape neural networks on accelerators via on-the-fly micro-kernel polymerization[C] //Proc of the 29th ACM Int Conf on Architectural Support for Programming Languages and Operating Systems. New York: ACM, 2024: 797−812, articleTitle=null, refAbstract=null), Reference(id=1327010575156097839, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327010539991052840, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=109, rfOrder=109, authorNames=null, journalName=null, refType=null, unstructuredReference=He Jiaao, Qiu Jiezhong, Zeng Aohan, et al. FastMoE: A fast mixture-of-expert training system[J]. arXiv preprint, arXiv: 2103.13262, 2021, articleTitle=null, refAbstract=null), Reference(id=1327010575260955440, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327010539991052840, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=110, rfOrder=110, authorNames=null, journalName=null, refType=null, unstructuredReference=Gale T, Narayanan D, Young C, et al. MegaBlocks: Efficient sparse training with mixture-of-experts[C/OL] //Proc of the 6th Machine Learning and Systems. 2023 [2024-08-02]. https://proceedings.mlsys.org/paper_files/paper/2023/file/5a54f79333768effe7e8927bcccffe40-Paper-mlsys2023.pdf, articleTitle=null, refAbstract=null), Reference(id=1327010575332258609, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327010539991052840, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=111, rfOrder=111, authorNames=null, journalName=null, refType=null, unstructuredReference=Zheng Ningxin, Jiang Huiqiang, Zhang Quanlu, et al. PIT: Optimization of dynamic sparse deep learning models via permutation invariant transformation[C] //Proc of the 29th Symp on Operating Systems Principles. New York: ACM, 2023: 331−347, articleTitle=null, refAbstract=null), Reference(id=1327010575411950386, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327010539991052840, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=112, rfOrder=112, authorNames=null, journalName=null, refType=null, unstructuredReference=Tan S, Shen Yikang, Panda R, et al. Scattered mixture-of-experts implementation[J]. arXiv preprint, arXiv: 2403.08245, 2024, articleTitle=null, refAbstract=null), Reference(id=1327010575491642163, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327010539991052840, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=113, rfOrder=113, authorNames=null, journalName=null, refType=null, unstructuredReference=Nie Xiaonan, Miao Xupeng, Cao Shijie, et al. EvoMoE: An evolutional mixture-of-experts training framework via dense-to-sparse gate[J]. arXiv preprint, arXiv: 2112.14397, 2022, articleTitle=null, refAbstract=null), Reference(id=1327010575554556724, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327010539991052840, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=114, rfOrder=114, authorNames=null, journalName=null, refType=null, unstructuredReference=Zhou Yanqi, Lei Tao, Liu Hanxiao, et al. Mixture-of-experts with expert choice routing[C] //Proc of the 36th Int Conf on Neural Information Processing Systems. New York: Curran Associates, 2022: 7103−7114, articleTitle=null, refAbstract=null), Reference(id=1327010575646831413, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327010539991052840, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=115, rfOrder=115, authorNames=null, journalName=null, refType=null, unstructuredReference=Zeng Zhiyuan, Guo Qipeng, Fei Zhaoye, et al. Turn waste into worth: Rectifying top-k router of MoE[J]. arXiv preprint, arXiv: 2402.12399, 2024, articleTitle=null, refAbstract=null), Reference(id=1327010575726523190, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327010539991052840, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=116, rfOrder=116, authorNames=null, journalName=null, refType=null, unstructuredReference=Lewis M, Bhosale S, Dettmers T, et al. Base layers: Simplifying training of large, sparse models[C] //Proc of the 38th Int Conf on Machine Learning. New York: PMLR, 2021: 6265−6274, articleTitle=null, refAbstract=null), Reference(id=1327010575843963703, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327010539991052840, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=117, rfOrder=117, authorNames=null, journalName=null, refType=null, unstructuredReference=Clark A, Casas D D L, Guy A, et al. Unified scaling laws for routed language models[C] //Proc of the 39th Int Conf on Machine Learning. New York: PMLR, 2022: 4057−4086, articleTitle=null, refAbstract=null), Reference(id=1327010575927849784, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327010539991052840, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=118, rfOrder=118, authorNames=null, journalName=null, refType=null, unstructuredReference=Liu Tianlin, Puigcerver J, Blondel M. Sparsity-constrained optimal transport[J]. arXiv preprint, arXiv: 2209.15466, 2023, articleTitle=null, refAbstract=null), Reference(id=1327010575999152953, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327010539991052840, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=119, rfOrder=119, authorNames=null, journalName=null, refType=null, unstructuredReference=Roller S, Sukhbaatar S, Weston J, et al. Hash layers for large sparse models[C] //Proc of the 35th Int Conf on Neural Information Processing Systems. New York: Curran Associates, 2021: 17555−17566, articleTitle=null, refAbstract=null), Reference(id=1327010576078844730, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327010539991052840, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=120, rfOrder=120, authorNames=null, journalName=null, refType=null, unstructuredReference=Zuo Simiao, Liu Xiaodong, Jiao Jian, et al. Taming sparsely activated transformer with stochastic experts[J]. arXiv preprint, arXiv: 2110.04260, 2022, articleTitle=null, refAbstract=null), Reference(id=1327010576145953595, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327010539991052840, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=121, rfOrder=121, authorNames=null, journalName=null, refType=null, unstructuredReference=Puigcerver J, Riquelme C, Mustafa B, et al. From sparse to soft mixtures of experts[J]. arXiv preprint, arXiv: 2308.00951, 2023, articleTitle=null, refAbstract=null), Reference(id=1327010576217256764, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327010539991052840, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=122, rfOrder=122, authorNames=null, journalName=null, refType=null, unstructuredReference=Yu Ping, Artetxe M, Ott M, et al. Efficient language modeling with sparse all-MLP[J]. arXiv preprint, arXiv: 2203.06850, 2022, articleTitle=null, refAbstract=null), Reference(id=1327010576280171325, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327010539991052840, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=123, rfOrder=123, authorNames=null, journalName=null, refType=null, unstructuredReference=Muqeeth M, Liu Haokun, Raffel C. Soft merging of experts with adaptive routing[J]. arXiv preprint, arXiv: 2306.03745, 2023, articleTitle=null, refAbstract=null), Reference(id=1327010576338891582, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327010539991052840, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=124, rfOrder=124, authorNames=null, journalName=null, refType=null, unstructuredReference=Hazimeh H, Zhao Zhe, Chowdhery A, et al. DSelect-k: Differentiable selection in the mixture of experts with applications to multi-task learning[C] //Proc of the 35th Int Conf on Neural Information Processing Systems. New York: Curran Associates, 2021: 29335−29347, articleTitle=null, refAbstract=null), Reference(id=1327010576418583359, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327010539991052840, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=125, rfOrder=125, authorNames=null, journalName=null, refType=null, unstructuredReference=Ibrahim S, Chen W, Hazimeh H, et al. COMET: Learning cardinality constrained mixture of experts with trees and local search[C] //Proc of the 29th ACM SIGKDD Conf on Knowledge Discovery and Data Mining. New York: ACM, 2023: 832−844, articleTitle=null, refAbstract=null), Reference(id=1327010576531829568, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327010539991052840, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=126, rfOrder=126, authorNames=null, journalName=null, refType=null, unstructuredReference=Zhai Mingshu, He Jiaao, Ma Zixuan, et al. SmartMoE: Efficiently training sparsely-activated models through combining offline and online parallelization[C] //Proc of the 2023 USENIX Annual Technical Conf. Berkeley, CA: USENIX Association, 2023: 961−975, articleTitle=null, refAbstract=null), Reference(id=1327010576661852993, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327010539991052840, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=127, rfOrder=127, authorNames=null, journalName=null, refType=null, unstructuredReference=Nie Xiaonan, Miao Xupeng, Wang Zilong, et al. FlexMoE: Scaling large-scale sparse pre-trained model training via dynamic device placement[J]. Proceedings of ACM on Management of Data, 2023, 1(1): 1−19, articleTitle=null, refAbstract=null), Reference(id=1327010576745739074, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327010539991052840, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=128, rfOrder=128, authorNames=null, journalName=null, refType=null, unstructuredReference=Du Zhixu, Li Shiyu, Wu Yuhao, et al. SiDA: Sparsity-inspired data-aware serving for efficient and scalable large mixture-of-experts models[C/OL] //Proc of the 7th Machine Learning and Systems. 2024 [2024-08-02]. https://proceedings.mlsys.org/paper_files/paper/2024/file/698cfaf72a208aef2e78bcac55b74328-Paper-Conference.pdf, articleTitle=null, refAbstract=null), Reference(id=1327010576951259971, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327010539991052840, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=129, rfOrder=129, authorNames=null, journalName=null, refType=null, unstructuredReference=Shazeer N, Cheng Youlong, Parmar N, et al. Mesh-Tensorflow: Deep learning for supercomputers[C] //Proc of the 32nd Int Conf on Neural Information Processing Systems. New York: Curran Associates, 2018: 10435−10444, articleTitle=null, refAbstract=null), Reference(id=1327010577056117572, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327010539991052840, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=130, rfOrder=130, authorNames=null, journalName=null, refType=null, unstructuredReference=Rajbhandari S, Rasley J, Ruwase O, et al. ZeRO: Memory optimizations toward training trillion parameter models[C/OL] //Proc of Int Conf for High Performance Computing, Networking, Storage and Analysis. Piscataway, NJ: IEEE, 2020 [2024-09-10]. https://dl.acm.org/ doi/abs/10.5555/3433701.3433727, articleTitle=null, refAbstract=null), Reference(id=1327010577190335302, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327010539991052840, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=131, rfOrder=131, authorNames=null, journalName=null, refType=null, unstructuredReference=Kosson A, Chiley V, Venigalla A, et al. Pipelined backpropagation at scale: Training large models without batches[C/OL] //Proc of the 4th Machine Learning and Systems. 2021 [2024-08-02]. https://proceedings.mlsys.org/paper_files/paper/2021/file/0c8abcf158ed12d0dd94480681186fda-Paper.pdf, articleTitle=null, refAbstract=null), Reference(id=1327010577274221383, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327010539991052840, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=132, rfOrder=132, authorNames=null, journalName=null, refType=null, unstructuredReference=Huang Yanping, Cheng Youlong, Bapna A, et al. GPipe: Efficient training of giant neural networks using pipeline parallelism[C] //Proc of the 33rd Int Conf on Neural Information Processing Systems. New York: Curran Associates, 2019: 103−112, articleTitle=null, refAbstract=null), Reference(id=1327010577362301769, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327010539991052840, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=133, rfOrder=133, authorNames=null, journalName=null, refType=null, unstructuredReference=Narayanan D, Harlap A, Phanishayee A, et al. PipeDream: Generalized pipeline parallelism for dnn training[C/OL] //Proc of the 27th ACM Symp on Operating Systems Principles. New York: ACM, 2019 [2024-09-10]. https://dl.acm.org/doi/abs/10.1145/3341301.3359646, articleTitle=null, refAbstract=null), Reference(id=1327010577433604938, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327010539991052840, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=134, rfOrder=134, authorNames=null, journalName=null, refType=null, unstructuredReference=Narayanan D, Phanishayee A, Shi Kaiyu, et al. Memory-efficient pipeline-parallel DNN training[C] //Proc of the 38th Int Conf on Machine Learning. New York: PMLR, 2021: 7937−7947, articleTitle=null, refAbstract=null), Reference(id=1327010577530073931, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327010539991052840, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=135, rfOrder=135, authorNames=null, journalName=null, refType=null, unstructuredReference=Fu Yichao, Yuhao Qing, Zhao Shixiong, et al. AMPipe: Accelerating MoE model training with intra-block pipelining[EB/OL]. 2024 [2024-08-02]. https://openreview.net/pdf?id=yLgr02IsXY, articleTitle=null, refAbstract=null), Reference(id=1327010577613960012, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327010539991052840, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=136, rfOrder=136, authorNames=null, journalName=null, refType=null, unstructuredReference=Jiang Chenyu, Tian Ye, Jia Zhen, et al. Lancet: Accelerating mixture-of-experts training via whole graph computation-communication overlapping[J]. arXiv preprint, arXiv: 2404.19429, 2024, articleTitle=null, refAbstract=null), Reference(id=1327010577681068877, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327010539991052840, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=137, rfOrder=137, authorNames=null, journalName=null, refType=null, unstructuredReference=Shi Shaohuai, Pan Xinglin, Chu Xiaowen, et al. PipeMoE: Accelerating mixture-of-experts through adaptive pipelining[C/OL] //Proc of the 2023 IEEE Conf on Computer Communications. Piscataway, NJ: IEEE, 2023 [2024-09-10]. https://ieeexplore.ieee.org/abstract/document/10228874, articleTitle=null, refAbstract=null), Reference(id=1327010577743983438, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327010539991052840, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=138, rfOrder=138, authorNames=null, journalName=null, refType=null, unstructuredReference=Aminabadi R Y, Rajbhandari S, Awan A A, et al. DeepSpeed-Inference: Enabling efficient inference of transformer models at unprecedented scale[C/OL] //Proc of Int Conf for High Performance Computing, Networking, Storage and Analysis. New York: ACM, 2022 [2024-09-10]. https://dl.acm.org/doi/abs/10.5555/3571885.3571946, articleTitle=null, refAbstract=null), Reference(id=1327010577819480911, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327010539991052840, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=139, rfOrder=139, authorNames=null, journalName=null, refType=null, unstructuredReference=Valiant L G. A bridging model for parallel computation[J]. Communications of the ACM, 1990, 33(8): 103−111, articleTitle=null, refAbstract=null), Reference(id=1327010579493008209, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327010539991052840, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=140, rfOrder=140, authorNames=null, journalName=null, refType=null, unstructuredReference=Narayanan D, Shoeybi M, Casper J, et al. Efficient large-scale language model training on GPU clusters using Megatron-LM[C/OL] //Proc of Int Conf for High Performance Computing, Networking, Storage and Analysis. New York: ACM, 2021 [2024-09-10]. https://dl.acm.org/doi/ abs/10.1145/3458817.3476209, articleTitle=null, refAbstract=null), Reference(id=1327010579560117075, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327010539991052840, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=141, rfOrder=141, authorNames=null, journalName=null, refType=null, unstructuredReference=Wang Guanhua, Qin Heyang, Jacobs S A, et al. ZeRO++: Extremely efficient collective communication for giant model training[J]. arXiv preprint, arXiv: 2306.10209, 2023, articleTitle=null, refAbstract=null), Reference(id=1327010579631420244, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327010539991052840, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=142, rfOrder=142, authorNames=null, journalName=null, refType=null, unstructuredReference=Rajbhandari S, Ruwase O, Rasley J, et al. ZeRO-infinity: Breaking the GPU memory wall for extreme scale deep learning[C/OL] //Proc of Int Conf for High Performance Computing, Networking, Storage and Analysis. New York: ACM, 2021 [2024-09-10]. https://dl.acm.org/doi/ abs/10.1145/3458817.3476205, articleTitle=null, refAbstract=null), Reference(id=1327010579757249365, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327010539991052840, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=143, rfOrder=143, authorNames=null, journalName=null, refType=null, unstructuredReference=Jia Zhihao, Lin S, Qi C R, et al. Exploring hidden dimensions in parallelizing convolutional neural networks[C] // Proc of the 35th Int Conf on Machine Learning. New York: PMLR, 2018: 2279−2288, articleTitle=null, refAbstract=null), Reference(id=1327010579853718358, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327010539991052840, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=144, rfOrder=144, authorNames=null, journalName=null, refType=null, unstructuredReference=Zheng Lianmin, Li Zhuohan, Zhang Hao, et al. Alpa: Automating inter-and intra-operator parallelism for distributed deep learning[C] //Proc of the 16th USENIX Symp on Operating Systems Design and Implementation. Berkeley, CA: USENIX Association, 2022: 559−578, articleTitle=null, refAbstract=null), Reference(id=1327010579916632919, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327010539991052840, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=145, rfOrder=145, authorNames=null, journalName=null, refType=null, unstructuredReference=Li Zhuohan, Zheng Lianmin, Zhong Yinmin, et al. AlpaServe: Statistical multiplexing with model parallelism for deep learning serving[C] //Proc of the 17th USENIX Symp on Operating Systems Design and Implementation. Berkeley, CA: USENIX Association, 2023: 663−679, articleTitle=null, refAbstract=null), Reference(id=1327010580000519000, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327010539991052840, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=146, rfOrder=146, authorNames=null, journalName=null, refType=null, unstructuredReference=Jia Zhihao, Zaharia M, Aiken A. Beyond data and model parallelism for deep neural networks[C/OL] //Proc of the 2nd Machine Learning and Systems. 2019 [2024-08-02]. https://proceedings.mlsys.org/paper_files/paper/2019/file/b422680f3db0986ddd7f8f126baaf0fa-Paper.pdf, articleTitle=null, refAbstract=null), Reference(id=1327010580071822169, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327010539991052840, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=147, rfOrder=147, authorNames=null, journalName=null, refType=null, unstructuredReference=Artetxe M, Bhosale S, Goyal N, et al. Efficient large scale language modeling with mixtures of experts[J]. arXiv preprint, arXiv: 2112.10684, 2022, articleTitle=null, refAbstract=null), Reference(id=1327010580210234202, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327010539991052840, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=148, rfOrder=148, authorNames=null, journalName=null, refType=null, unstructuredReference=Zhao Yanli, Gu A, Varma R, et al. Pytorch FSDP: Experiences on scaling fully sharded data parallel[J]. arXiv preprint, arXiv: 2304.11277, 2023, articleTitle=null, refAbstract=null)], funds=null, companyList=[AuthorCompany(id=1327010548488712797, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327010539991052840, xref=null, ext=[AuthorCompanyExt(id=1327010548497101406, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327010539991052840, companyId=1327010548488712797, language=EN, country=null, province=null, city=null, postcode=null, companyName=null, departmentName=null, remark=IEIT SYSTEMS Co., Ltd., Jinan 250101), AuthorCompanyExt(id=1327010548505490015, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327010539991052840, companyId=1327010548488712797, language=CN, country=null, province=null, city=null, postcode=null, companyName=null, departmentName=null, remark=浪潮电子信息产业股份有限公司 济南 250101)]), AuthorCompany(id=1327010548648096352, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327010539991052840, xref=null, ext=[AuthorCompanyExt(id=1327010548690039393, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327010539991052840, companyId=1327010548648096352, language=EN, country=null, province=null, city=null, postcode=null, companyName=null, departmentName=null, remark=Inspur (Beijing) Electronic Information Industry Co., Ltd., Beijing 100095), AuthorCompanyExt(id=1327010548711010914, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327010539991052840, companyId=1327010548648096352, language=CN, country=null, province=null, city=null, postcode=null, companyName=null, departmentName=null, remark=浪潮(北京)电子信息产业有限公司 北京 100095)])], figs=[ArticleFig(id=1327010553391854228, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327010539991052840, language=EN, label=Figure 1, caption=
Taxonomy of MoE systems optimization techniques, figureFileSmall=NYhqjxCAm8w8PLfLxUa5tg==, figureFileBig=EAQ9/caiByqNpefnqCEO5w==, tableContent=null), ArticleFig(id=1327010553538654869, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327010539991052840, language=CN, label=图1, caption=
MoE系统优化技术分类, figureFileSmall=NYhqjxCAm8w8PLfLxUa5tg==, figureFileBig=EAQ9/caiByqNpefnqCEO5w==, tableContent=null), ArticleFig(id=1327010553811284631, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327010539991052840, language=EN, label=Figure 2, caption=
The architecture of MoE model, figureFileSmall=NkkFbdGW4EeK1Z8qs1U0zA==, figureFileBig=ytmOOu5yXqdwUZf0Ti5zEA==, tableContent=null), ArticleFig(id=1327010553886782104, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327010539991052840, language=CN, label=图2, caption=
MoE 模型架构, figureFileSmall=NkkFbdGW4EeK1Z8qs1U0zA==, figureFileBig=ytmOOu5yXqdwUZf0Ti5zEA==, tableContent=null), ArticleFig(id=1327010553953890969, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327010539991052840, language=EN, label=Figure 3, caption=
An example of expert parallel execution flow of MoE, figureFileSmall=EtuSnxzkkm8HqfW54y0bQw==, figureFileBig=cKfEGOOcg2rIRU5vPIjctQ==, tableContent=null), ArticleFig(id=1327010554029388442, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327010539991052840, language=CN, label=图3, caption=
MoE专家并行执行流程示例, figureFileSmall=EtuSnxzkkm8HqfW54y0bQw==, figureFileBig=cKfEGOOcg2rIRU5vPIjctQ==, tableContent=null), ArticleFig(id=1327010554092303003, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327010539991052840, language=EN, label=Figure 4, caption=
MoE-architectured Transformer model, figureFileSmall=a3M11dMM+G+BT2V1G3ZRzQ==, figureFileBig=rsvgPFm4lQ/4rX1ZrdabhA==, tableContent=null), ArticleFig(id=1327010554176189084, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327010539991052840, language=CN, label=图4, caption=
MoE架构Transformer模型, figureFileSmall=a3M11dMM+G+BT2V1G3ZRzQ==, figureFileBig=rsvgPFm4lQ/4rX1ZrdabhA==, tableContent=null), ArticleFig(id=1327010554255880861, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327010539991052840, language=EN, label=Figure 5, caption=
Corresponding relationship between core research directions and challenges, figureFileSmall=mctyZNE688m30fBPt73LBg==, figureFileBig=KL0afb6GJ79xpfLHA9GUJA==, tableContent=null), ArticleFig(id=1327010554331378334, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327010539991052840, language=CN, label=图5, caption=
核心研究方向与挑战对应关系, figureFileSmall=mctyZNE688m30fBPt73LBg==, figureFileBig=KL0afb6GJ79xpfLHA9GUJA==, tableContent=null), ArticleFig(id=1327010554415264415, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327010539991052840, language=EN, label=Figure 6, caption=
Illustration of parameter offloading in MoE models, figureFileSmall=QcmXsTnM+d6678XMkznTJw==, figureFileBig=cR5k475YinKo5Or0pNq+rg==, tableContent=null), ArticleFig(id=1327010554490761888, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327010539991052840, language=CN, label=图6, caption=
MoE模型参数卸载示意图, figureFileSmall=QcmXsTnM+d6678XMkznTJw==, figureFileBig=cR5k475YinKo5Or0pNq+rg==, tableContent=null), ArticleFig(id=1327010554578842273, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327010539991052840, language=EN, label=Figure 7, caption=
A typical hierarchical All-to-All communication approach, figureFileSmall=XzjuxtoyMlt99C3p4ZbpCg==, figureFileBig=hInyGswGrLBmMngoKeFWAg==, tableContent=null), ArticleFig(id=1327010554658534050, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327010539991052840, language=CN, label=图7, caption=
一种典型的分层All-to-All通信方法, figureFileSmall=XzjuxtoyMlt99C3p4ZbpCg==, figureFileBig=hInyGswGrLBmMngoKeFWAg==, tableContent=null), ArticleFig(id=1327010554755003043, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327010539991052840, language=EN, label=Figure 8, caption=
Elimination of redundant communication data in tensor parallel group, figureFileSmall=G3ZX75OARmlA8ETLiemTnw==, figureFileBig=atfebDUsMSbqXigdsszgVg==, tableContent=null), ArticleFig(id=1327010554847277732, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327010539991052840, language=CN, label=图8, caption=
张量并行组冗余通信数据消除, figureFileSmall=G3ZX75OARmlA8ETLiemTnw==, figureFileBig=atfebDUsMSbqXigdsszgVg==, tableContent=null), ArticleFig(id=1327010554989884069, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327010539991052840, language=EN, label=Figure 9, caption=
Method for forcibly limiting the amount of data communication between nodes, figureFileSmall=VtnW1XjdI8B38eAwCN8kPw==, figureFileBig=p5V4lsK1dT2PJ8nalQQIYg==, tableContent=null), ArticleFig(id=1327010555065381542, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327010539991052840, language=CN, label=图9, caption=
强制限制节点间通信的数据量方法, figureFileSmall=VtnW1XjdI8B38eAwCN8kPw==, figureFileBig=p5V4lsK1dT2PJ8nalQQIYg==, tableContent=null), ArticleFig(id=1327010555157656231, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327010539991052840, language=EN, label=Figure 10, caption=
Dynamic capacity of experts, figureFileSmall=06XsxHlPzuIWbzaOq968Qg==, figureFileBig=Da7WeiOHFyQzr6YAAMTOmQ==, tableContent=null), ArticleFig(id=1327010555237348008, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327010539991052840, language=CN, label=图10, caption=
专家动态容量, figureFileSmall=06XsxHlPzuIWbzaOq968Qg==, figureFileBig=Da7WeiOHFyQzr6YAAMTOmQ==, tableContent=null), ArticleFig(id=1327010555350594217, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327010539991052840, language=EN, label=Figure 11, caption=
Illustration of expert placement strategy, figureFileSmall=aWRA4YmQKDjEtEYIrTt4/Q==, figureFileBig=cw5y0n7oWtmkV5yakBjgHw==, tableContent=null), ArticleFig(id=1327010557007344298, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327010539991052840, language=CN, label=图11, caption=
专家放置策略示意图, figureFileSmall=aWRA4YmQKDjEtEYIrTt4/Q==, figureFileBig=cw5y0n7oWtmkV5yakBjgHw==, tableContent=null), ArticleFig(id=1327010557078647467, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327010539991052840, language=EN, label=Figure 12, caption=
Resources allocation utilizing statistical-based activation prediction method, figureFileSmall=lQlp30SqLgvQLbaX+ELLZA==, figureFileBig=hMqcxEcf9SBGbvKLRDp3qA==, tableContent=null), ArticleFig(id=1327010557179310764, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327010539991052840, language=CN, label=图12, caption=
利用基于统计的激活预测方法分配资源, figureFileSmall=lQlp30SqLgvQLbaX+ELLZA==, figureFileBig=hMqcxEcf9SBGbvKLRDp3qA==, tableContent=null), ArticleFig(id=1327010557300945581, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327010539991052840, language=EN, label=Figure 13, caption=
Illustration of machine learning-based expert activation prediction method, figureFileSmall=OCxqn4GVdxOW+aex5ss88Q==, figureFileBig=ITst6xmysVnVvFz3Lfa1vw==, tableContent=null), ArticleFig(id=1327010557485494958, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327010539991052840, language=CN, label=图13, caption=
基于机器学习的专家激活预测方法示意图, figureFileSmall=OCxqn4GVdxOW+aex5ss88Q==, figureFileBig=ITst6xmysVnVvFz3Lfa1vw==, tableContent=null), ArticleFig(id=1327010557594546863, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327010539991052840, language=EN, label=Figure 14, caption=
An example timeline of forward propagation pipeline parallelism in an MoE layer, figureFileSmall=Oc1CE3zcKWUYeUzmpfc4+A==, figureFileBig=e+yRqwJeJLEFxT6ZhGPEhQ==, tableContent=null), ArticleFig(id=1327010557665850032, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327010539991052840, language=CN, label=图14, caption=
MoE层前向流水并行的时间线示例, figureFileSmall=Oc1CE3zcKWUYeUzmpfc4+A==, figureFileBig=e+yRqwJeJLEFxT6ZhGPEhQ==, tableContent=null), ArticleFig(id=1327010557770707633, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327010539991052840, language=EN, label=Figure 15, caption=
Data, tensor and expert parallelisms, figureFileSmall=5uTnKfzPBBzdn5z/FiUC5g==, figureFileBig=3SK3myjqE2HXdDjS/zs4aw==, tableContent=null), ArticleFig(id=1327010557858788018, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327010539991052840, language=CN, label=图15, caption=
数据、张量和专家并行, figureFileSmall=5uTnKfzPBBzdn5z/FiUC5g==, figureFileBig=3SK3myjqE2HXdDjS/zs4aw==, tableContent=null), ArticleFig(id=1327010557984617139, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327010539991052840, language=EN, label=Figure 16, caption=
Overview of adaptive parallel strategy, figureFileSmall=HhAHSZ8+5KyhaDSgXnNQOg==, figureFileBig=BS14yZDoVTrJLHCskDWcAA==, tableContent=null), ArticleFig(id=1327010558144000692, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327010539991052840, language=CN, label=图16, caption=
自适应并行策略概览, figureFileSmall=HhAHSZ8+5KyhaDSgXnNQOg==, figureFileBig=BS14yZDoVTrJLHCskDWcAA==, tableContent=null), ArticleFig(id=1327010558265635509, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327010539991052840, language=EN, label=Table 1, caption=
Comparison of MoE and Ensemble Learning, figureFileSmall=null, figureFileBig=null, tableContent=
| 类别 | MoE | 集成学习 |
|---|
| 模型集成 | 都涉及整合多个模型结果以提高预测精度,都利用不同模型的优势来解决复杂问题. |
| 应用目的 | 不仅提高模型预测精度和泛化能力,还提高模型收敛速度和执行效率. | 提高模型预测精度和泛化能力. |
| 模型结构 | 由多个专家和门控网络组成,每个专家负责处理模型的一部分子任务,门控网络决定专家和子任务的映射关系. | 通常由多个独立训练的相同或不同算法模型构成. |
| 任务分解 | 将复杂任务分解为相对简单的子任务,专家网络专注于处理特定子任务. | 通常不涉及任务分解,多个学习器同时处理整个任务,通过投票、加权或平均等方法整合结果. |
| 训练方法 | 门控网络学习将任务分配给相应专家网络,专家网络专注于特定任务学习. | 各学习器独立训练,可使用不同的数据或算法,不涉及可学习的任务分配网络. |
| 应用场景 | 适用于复杂且任务可以分解为不同子任务的场景. | 适用于提升模型泛化能力和鲁棒性,尤其在数据或者特征有限的情况. |
| 稀疏性 | 只有少量专家在给定的时间被激活,提高计算效率. | 通常不具有稀疏性,所有模型都参与最终的预测. |
| 动态性 | 根据输入数据动态激活最适合的专家网络. | 通常不具有动态性,模型的集成是静态的,且在训练过程就已确定. |
), ArticleFig(id=1327010558353715894, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327010539991052840, language=CN, label=表1, caption=
MoE与集成学习对比
, figureFileSmall=null, figureFileBig=null, tableContent=
| 类别 | MoE | 集成学习 |
|---|
| 模型集成 | 都涉及整合多个模型结果以提高预测精度,都利用不同模型的优势来解决复杂问题. |
| 应用目的 | 不仅提高模型预测精度和泛化能力,还提高模型收敛速度和执行效率. | 提高模型预测精度和泛化能力. |
| 模型结构 | 由多个专家和门控网络组成,每个专家负责处理模型的一部分子任务,门控网络决定专家和子任务的映射关系. | 通常由多个独立训练的相同或不同算法模型构成. |
| 任务分解 | 将复杂任务分解为相对简单的子任务,专家网络专注于处理特定子任务. | 通常不涉及任务分解,多个学习器同时处理整个任务,通过投票、加权或平均等方法整合结果. |
| 训练方法 | 门控网络学习将任务分配给相应专家网络,专家网络专注于特定任务学习. | 各学习器独立训练,可使用不同的数据或算法,不涉及可学习的任务分配网络. |
| 应用场景 | 适用于复杂且任务可以分解为不同子任务的场景. | 适用于提升模型泛化能力和鲁棒性,尤其在数据或者特征有限的情况. |
| 稀疏性 | 只有少量专家在给定的时间被激活,提高计算效率. | 通常不具有稀疏性,所有模型都参与最终的预测. |
| 动态性 | 根据输入数据动态激活最适合的专家网络. | 通常不具有动态性,模型的集成是静态的,且在训练过程就已确定. |
), ArticleFig(id=1327010558605374135, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327010539991052840, language=EN, label=Table 2, caption=
Description of Symbols, figureFileSmall=null, figureFileBig=null, tableContent=
| 符号 | 描述 |
|---|
| G | 每个节点内GPU的数量 |
| N | 节点总数 |
| P | GPU设备总数(P=GN) |
), ArticleFig(id=1327010558689260216, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327010539991052840, language=CN, label=表2, caption=
符号描述
, figureFileSmall=null, figureFileBig=null, tableContent=
| 符号 | 描述 |
|---|
| G | 每个节点内GPU的数量 |
| N | 节点总数 |
| P | GPU设备总数(P=GN) |
), ArticleFig(id=1327010558953501369, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327010539991052840, language=EN, label=Table 3, caption=
Summary of Representative Works on MoE System Optimization, figureFileSmall=null, figureFileBig=null, tableContent=
| 类型 | 子类 | 适用场景 | 文献 |
|---|
| [69] | [56] | [57] | [58] | [63] | [78] | [109] | [96] | [126] | [94] | [95] | [101] |
|---|
| 注:“其他*”代表路由无关通信迟优化中的其他优化. |
| 内存占用 | 内存卸载 | 设备内存有限、模型参数规模大、模型推理 | √ | | √ | | √ | | | | | | | |
| 参数压缩 | | | √ | | | √ | | | | | | |
| 通信延迟 | 分层通信 | 通用,特别适合小批量数据通信 | | | √ | √ | | √ | | √ | | | √ | |
| 冗余消除 | 模型推理或通过张量并行训练、推理 | √ | | | | | √ | | | | | | |
| 其他* | 通用,Janus除外 | | | √ | √ | √ | | | √ | | | | |
| 拓扑敏感路由 | 模型训练,特别是网络拓扑复杂的集群 | | | | | | | | | | | | √ |
| 计算效率 | 动态容量 | 通用,特别是专家数量多且偏好差异大的模型 | | | | | | | √ | √ | | | | |
| 专家负载 | 负载均衡损失用于训练,路由均衡门控均可 | | | | | | | | | | | | |
| 设备负载 | 专家负载差异明显、变化显著,设备数量多 | | | √ | | | | | | √ | √ | | √ |
| 激活预测 | 模型推理,但通常需要训练过程的统计数据 | | √ | | | | | | | √ | √ | | |
| 静态流水 | 非极致性能需求 | | | | | | | | | | √ | | √ |
| 自适应流水 | 模型规模大、执行开销大、极致性能需求 | | | | | √ | | | √ | | | | |
| 内核计算 | 通用 | | | | | | √ | √ | √ | | | √ | |
| 并行扩展 | 静态并行 | 同一模型频繁使用,训练、推理开销不大 | √ | | | | | √ | √ | | | | | |
| 自适应并行 | 极致性能需求、模型频繁变化、大规模训练推理 | | | | | | | | √ | √ | | | |
), ArticleFig(id=1327010559242908346, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327010539991052840, language=CN, label=表3, caption=
MoE系统优化的代表工作总结
, figureFileSmall=null, figureFileBig=null, tableContent=
| 类型 | 子类 | 适用场景 | 文献 |
|---|
| [69] | [56] | [57] | [58] | [63] | [78] | [109] | [96] | [126] | [94] | [95] | [101] |
|---|
| 注:“其他*”代表路由无关通信迟优化中的其他优化. |
| 内存占用 | 内存卸载 | 设备内存有限、模型参数规模大、模型推理 | √ | | √ | | √ | | | | | | | |
| 参数压缩 | | | √ | | | √ | | | | | | |
| 通信延迟 | 分层通信 | 通用,特别适合小批量数据通信 | | | √ | √ | | √ | | √ | | | √ | |
| 冗余消除 | 模型推理或通过张量并行训练、推理 | √ | | | | | √ | | | | | | |
| 其他* | 通用,Janus除外 | | | √ | √ | √ | | | √ | | | | |
| 拓扑敏感路由 | 模型训练,特别是网络拓扑复杂的集群 | | | | | | | | | | | | √ |
| 计算效率 | 动态容量 | 通用,特别是专家数量多且偏好差异大的模型 | | | | | | | √ | √ | | | | |
| 专家负载 | 负载均衡损失用于训练,路由均衡门控均可 | | | | | | | | | | | | |
| 设备负载 | 专家负载差异明显、变化显著,设备数量多 | | | √ | | | | | | √ | √ | | √ |
| 激活预测 | 模型推理,但通常需要训练过程的统计数据 | | √ | | | | | | | √ | √ | | |
| 静态流水 | 非极致性能需求 | | | | | | | | | | √ | | √ |
| 自适应流水 | 模型规模大、执行开销大、极致性能需求 | | | | | √ | | | √ | | | | |
| 内核计算 | 通用 | | | | | | √ | √ | √ | | | √ | |
| 并行扩展 | 静态并行 | 同一模型频繁使用,训练、推理开销不大 | √ | | | | | √ | √ | | | | | |
| 自适应并行 | 极致性能需求、模型频繁变化、大规模训练推理 | | | | | | | | √ | √ | | | |
)], attaches=null, journal=Journal(id=1324759606947315908, delFlag=0, nameCn=计算机研究与发展, nameEn=Journal of Computer Research and Development, nameHistory1=null, nameHistory2=null, issn=1000-1239, eissn=null, cn=11-1777/TP, coden=null, periodic=0, language=CN, oaType=null, ccby=null, superviseOffice=null, ownerOffice=null, pubOffice=null, editorOffice=null, officeType=null, aims=null, clcCode=null, officeProv=null, officeCity=null, officeAddr=null, officeZip=null, officeEmail=null, officePhone=null, editDirector=null, officeDirector=null, officeDirectorPhone=null, officeStaffNum=null, officeEmpNum=null, coverPicUrl=k2l8zgqwstuxxevFX2qxVA==, journalPrice=null, startedYear=null, abbrevIsoEn=Journal of Computer Research and Development, journalRemark=null, publicationField=null, createdTime=1793776903141, updatedTime=1793850698612, createdBy=18614031015, updatedBy=13701087609, firstLetterCn=J, firstLetterEn=J, subjectCode=Electronic Information, subjectName=null, subjectCodeEn=Electronic Information, subjectNameEn=null, picCn=k2l8zgqwstuxxevFX2qxVA==, picEn=r6k0Ry+RD+c/VXj5QarmDw==, jcr=null, cjcr=null, exts=[JournalExt(id=1325069127710286688, language=CN, name=计算机研究与发展, nameHistory1=null, nameHistory2=null, managedBy=, sponsoredBy=, publishedBy=, editorOffice=, officeProv=null, officeCity=null, officeAddr=, officeZip=, editDirector=, officeDirector=null, officePhone=null, coverPicUrl=null, journalRemark=, submitArticleUrl=null, websiteUrl=, createdTime=1793850698640, updatedTime=1793850698640, createdBy=13701087609, updatedBy=13701087609, submissionGuidelinesUrl=, submissionAuthorUrl=https://crad.ict.ac.cn/journalx/authorLogOn.action, submissionEditorUrl=https://crad.ict.ac.cn/journalx/editorLogOn.action, submissionReviewUrl=https://crad.ict.ac.cn/journalx/expertLogOn.action, submissionCeEditorUrl=https://crad.ict.ac.cn/journalx/editorInChiefLogOn.action, submissionAeEditorUrl=, option={"copyright":""}), JournalExt(id=1325069127777395553, language=EN, name=Journal of Computer Research and Development, nameHistory1=null, nameHistory2=null, managedBy=, sponsoredBy=, publishedBy=, editorOffice=, officeProv=null, officeCity=null, officeAddr=, officeZip=, editDirector=, officeDirector=null, officePhone=null, coverPicUrl=null, journalRemark=, submitArticleUrl=null, websiteUrl=, createdTime=1793850698656, updatedTime=1793850698656, createdBy=13701087609, updatedBy=13701087609, submissionGuidelinesUrl=, submissionAuthorUrl=https://crad.ict.ac.cn/journalx/authorLogOn.action, submissionEditorUrl=https://crad.ict.ac.cn/journalx/editorLogOn.action, submissionReviewUrl=https://crad.ict.ac.cn/journalx/expertLogOn.action, submissionCeEditorUrl=https://crad.ict.ac.cn/journalx/editorInChiefLogOn.action, submissionAeEditorUrl=, option={"copyright":""})], databaseList=null, tenantJournalId=1324759914146529478, websiteList=[Website(id=1325070300139892771, webName=null, webTitle=null, webDomain=null, webCopyrigh=null, webIpcNo=null, seoTitle=null, seoKeywords=null, seoDescription=null, tenantJournalId=null, journalId=1324759914146529478, journalNameCn=null, journalNameEn=null, grayFlag=null, tenantId=1146029695717560320, platformId=null, journalGroupId=null, journalGroupNameCn=null, journalGroupNameEn=null, type=1, domain=https://castjournals.cast.org.cn/joweb/jsjyjyfz/CN, language=CN, createTime=1793850978169, createBy=18614031015, updateTime=1793851040259, updateBy=18614031015, name=计算机研究与发展-中文, tplId=1146099689490845704, title=计算机研究与发展, delFlag=0, indexPage=/home, props=[WebsiteProps(id=1325070893751349360, tenantId=1146029695717560320, journalId=null, journalGroupId=null, siteId=1325070300139892771, code=articleTextType, value=kx, createTime=1793851119697, updateTime=1793851119697, creator=18614031015, updator=18614031015), WebsiteProps(id=1325070893730377837, tenantId=1146029695717560320, journalId=null, journalGroupId=null, siteId=1325070300139892771, code=banner, value=null, createTime=1793851119692, updateTime=1793851119692, creator=18614031015, updator=18614031015), WebsiteProps(id=1325070893772320883, tenantId=1146029695717560320, journalId=null, journalGroupId=null, siteId=1325070300139892771, code=grayFlag, value=0, createTime=1793851119702, updateTime=1793851119702, creator=18614031015, updator=18614031015), WebsiteProps(id=1325070893721989228, tenantId=1146029695717560320, journalId=null, journalGroupId=null, siteId=1325070300139892771, code=logo, value=https://castjournals.cast.org.cn/joweb/jsjyjyfz/CN/file/pic?fileId=U27tDsy717YF7XSPA+Vwdw==, createTime=1793851119690, updateTime=1793851119690, creator=18614031015, updator=18614031015), WebsiteProps(id=1325070893784903797, tenantId=1146029695717560320, journalId=null, journalGroupId=null, siteId=1325070300139892771, code=minRunFlag, value=0, createTime=1793851119705, updateTime=1793851119705, creator=18614031015, updator=18614031015), WebsiteProps(id=1325070893742960751, tenantId=1146029695717560320, journalId=null, journalGroupId=null, siteId=1325070300139892771, code=picServerUrl, value=https://castjournals.cast.org.cn/joweb/jsjyjyfz/CN/file/pic, createTime=1793851119695, updateTime=1793851119695, creator=18614031015, updator=18614031015), WebsiteProps(id=1325070893780709492, tenantId=1146029695717560320, journalId=null, journalGroupId=null, siteId=1325070300139892771, code=silenceFlag, value=0, createTime=1793851119704, updateTime=1793851119704, creator=18614031015, updator=18614031015), WebsiteProps(id=1325070893738766446, tenantId=1146029695717560320, journalId=null, journalGroupId=null, siteId=1325070300139892771, code=staticResourcePath, value=https://castjournals.cast.org.cn/joweb/cast_kjdb_cn_619/, createTime=1793851119694, updateTime=1793851119694, creator=18614031015, updator=18614031015), WebsiteProps(id=1325070893755543665, tenantId=1146029695717560320, journalId=null, journalGroupId=null, siteId=1325070300139892771, code=themeColor, value=null, createTime=1793851119698, updateTime=1793851119698, creator=18614031015, updator=18614031015), WebsiteProps(id=1325070893763932274, tenantId=1146029695717560320, journalId=null, journalGroupId=null, siteId=1325070300139892771, code=themeStyle, value=null, createTime=1793851119700, updateTime=1793851119700, creator=18614031015, updator=18614031015)]), Website(id=1325070300580294698, webName=null, webTitle=null, webDomain=null, webCopyrigh=null, webIpcNo=null, seoTitle=null, seoKeywords=null, seoDescription=null, tenantJournalId=null, journalId=1324759914146529478, journalNameCn=null, journalNameEn=null, grayFlag=null, tenantId=1146029695717560320, platformId=null, journalGroupId=null, journalGroupNameCn=null, journalGroupNameEn=null, type=1, domain=https://castjournals.cast.org.cn/joweb/jsjyjyfz/EN, language=EN, createTime=1793850978274, createBy=18614031015, updateTime=1793851036638, updateBy=18614031015, name=计算机研究与发展-英文, tplId=1146101810881728533, title=Journal of Computer Research and Development, delFlag=0, indexPage=/home, props=[WebsiteProps(id=1325070925288321148, tenantId=1146029695717560320, journalId=null, journalGroupId=null, siteId=1325070300580294698, code=articleTextType, value=kx, createTime=1793851127216, updateTime=1793851127216, creator=18614031015, updator=18614031015), WebsiteProps(id=1325070925263155321, tenantId=1146029695717560320, journalId=null, journalGroupId=null, siteId=1325070300580294698, code=banner, value=null, createTime=1793851127210, updateTime=1793851127210, creator=18614031015, updator=18614031015), WebsiteProps(id=1325070925309292671, tenantId=1146029695717560320, journalId=null, journalGroupId=null, siteId=1325070300580294698, code=grayFlag, value=0, createTime=1793851127221, updateTime=1793851127221, creator=18614031015, updator=18614031015), WebsiteProps(id=1325070925254766712, tenantId=1146029695717560320, journalId=null, journalGroupId=null, siteId=1325070300580294698, code=logo, value=https://castjournals.cast.org.cn/joweb/jsjyjyfz/EN/file/pic?fileId=U27tDsy717YF7XSPA+Vwdw==, createTime=1793851127208, updateTime=1793851127208, creator=18614031015, updator=18614031015), WebsiteProps(id=1325070925326069889, tenantId=1146029695717560320, journalId=null, journalGroupId=null, siteId=1325070300580294698, code=minRunFlag, value=0, createTime=1793851127225, updateTime=1793851127225, creator=18614031015, updator=18614031015), WebsiteProps(id=1325070925279932539, tenantId=1146029695717560320, journalId=null, journalGroupId=null, siteId=1325070300580294698, code=picServerUrl, value=https://castjournals.cast.org.cn/joweb/jsjyjyfz/EN/file/pic, createTime=1793851127214, updateTime=1793851127214, creator=18614031015, updator=18614031015), WebsiteProps(id=1325070925317681280, tenantId=1146029695717560320, journalId=null, journalGroupId=null, siteId=1325070300580294698, code=silenceFlag, value=0, createTime=1793851127223, updateTime=1793851127223, creator=18614031015, updator=18614031015), WebsiteProps(id=1325070925271543930, tenantId=1146029695717560320, journalId=null, journalGroupId=null, siteId=1325070300580294698, code=staticResourcePath, value=https://castjournals.cast.org.cn/joweb/cast_kjdb_en_623/, createTime=1793851127212, updateTime=1793851127212, creator=18614031015, updator=18614031015), WebsiteProps(id=1325070925292515453, tenantId=1146029695717560320, journalId=null, journalGroupId=null, siteId=1325070300580294698, code=themeColor, value=null, createTime=1793851127217, updateTime=1793851127217, creator=18614031015, updator=18614031015), WebsiteProps(id=1325070925300904062, tenantId=1146029695717560320, journalId=null, journalGroupId=null, siteId=1325070300580294698, code=themeStyle, value=null, createTime=1793851127219, updateTime=1793851127219, creator=18614031015, updator=18614031015)])], journalTitle=计算机研究与发展, weixinUrl=null, journalUrl=https://crad.ict.ac.cn/, iacademicId=null, status=1, seqNo=null, journalTitleEn=Journal of Computer Research and Development, journalPhotoCn=k2l8zgqwstuxxevFX2qxVA==, journalPhotoEn=r6k0Ry+RD+c/VXj5QarmDw==, journalFirstLetter=J, journalRecommend=null, journalNew=null, journalCollection=null, jcrJf=null, cjcrJf=null, jcrJfStr=null, cjcrJfStr=null, submissionFirstDecision=null, sciSubjectClassification=null, casSubjectClassification=null, citeScore=null, totalCitationFrequency=null, icpCode=null, psCode=null, advertisingLicenseCode=null, copyrightInformation=null, country=null, option=, provinceCode=null, provinceName=null, collectFlag=false, interPubPlatform=, interPubPlatformUrl=null), detailUrlCn=https://castjournals.cast.org.cn/joweb/jsjyjyfz/CN/10.7544/issn1000-1239.202440016, detailUrlEn=https://castjournals.cast.org.cn/joweb/jsjyjyfz/EN/10.7544/issn1000-1239.202440016, pdfUrlCn=https://castjournals.cast.org.cn/joweb/jsjyjyfz/CN/PDF/10.7544/issn1000-1239.202440016, pdfUrlEn=https://castjournals.cast.org.cn/joweb/jsjyjyfz/EN/PDF/10.7544/issn1000-1239.202440016, aliStartDate=0, aliEndDate=0, collectionFlag=false, citedCount=null, citedUrl=null, previewStatus=0, delFlag=0, hasFullText=1, orderTime=1746028800000, fullTextJson=null, articleText=null, reference=null)