Article(id=1327017398063969045, tenantId=1146029695717560320, journalId=1324759914146529478, issueId=1327017337032651028, articleNumber=null, orderNo=null, doi=10.7544/issn1000-1239.202440560, pmid=null, cstr=null, oa=null, hot=null, price=null, onlineType=0, articleFormat=0, articleType=null, articleTypeStr=research-article, receivedDate=1718899200000, receivedDateStr=2024-06-21, revisedDate=1746720000000, revisedDateStr=2025-05-09, acceptedDate=1749571200000, acceptedDateStr=2025-06-11, onlineDate=1794315202489, onlineDateStr=2026-11-10, pubDate=1764518400000, pubDateStr=2025-12-01, doiRegisterDate=null, doiRegisterDateStr=null, onlineIssueDate=1794315202489, onlineIssueDateStr=2026-11-10, onlineJustAcceptDate=null, onlineJustAcceptDateStr=null, onlineFirstDate=null, onlineFirstDateStr=null, sourceXml=null, magXml=null, createTime=1794315202489, creator=13701087609, updateTime=1794315202489, updator=13701087609, issue=Issue{id=1327017337032651028, tenantId=1146029695717560320, journalId=1324759914146529478, year='2025', volume='62', issue='12', pageStart='2889', pageEnd='3146', issueExtLink='null', onlineDate='null', pubDate='1764518400000', pubDateStr='2025-12-01', beforeIssueId=null, nextIssueId=null, price=null, status=1, issueComplete=1, articleOrder=1, issueType=-1, specialIssue=null, createTime=1794315187938, creator='13701087609', updateTime=1794377025235, updator='13701087609', preIssue=null, nextIssue=null, articleTotal=null, ext={EN=IssueExt(id=1327276701995872766, tenantId=1146029695717560320, journalId=1324759914146529478, issueId=1327017337032651028, language=EN, specialIssueTitle=, coverIllustrator=null, specialIssueEditor=, specialIssueAbout=), CN=IssueExt(id=1327276701995872767, tenantId=1146029695717560320, journalId=1324759914146529478, issueId=1327017337032651028, language=CN, specialIssueTitle=, coverIllustrator=null, specialIssueEditor=, specialIssueAbout=)}, issueFiles=null, downloadFileDto=null}, startPage=3060, endPage=3076, ext={EN=ArticleExt(id=1327017398303044375, articleId=1327017398063969045, tenantId=1146029695717560320, journalId=1324759914146529478, language=EN, title=AdaptDNN: An Adaptive and Scalable LLM Distributed Training System, columnId=1327017398223352598, journalTitle=Journal of Computer Research and Development, columnName=Ditributed Computing, runingTitle=null, highlight=null, articleAbstract=
As the number of parameters in deep learning models continues to increase, the cost of training also keeps rising. To reduce training costs, using spot instances provided by cloud service providers has become a viable solution. Spot instances are priced at only 30% of normal instances, which can significantly lower training costs. However, despite the low cost of elastic instances, there is a risk of them being reclaimed at any time, posing new challenges to the stability of the model training system. To address the fault tolerance issue in the context of elastic instances, existing work mainly falls into two categories: checkpoint-based fault tolerance and redundancy-based fault tolerance. Checkpoint-based solutions incur substantial overhead, while redundancy-based solutions impose certain limitations on the model’s parallelism strategy, leading to suboptimal training efficiency. We propose a solution for training in the context of spot instances that leverages the grace period of spot instances to back up training progress, thereby reducing fault tolerance overhead. It also employs the bottleneck alleviation approach to adjust the parallelism strategy, maximizing the use of available cluster resources and enhancing training efficiency. Experimental results show that this solution not only achieves low-cost fault tolerance but also ensures training efficiency, allowing for efficient completion of model training tasks in the context of spot instances and reducing overall training costs.
, authors=Guodong Liu
1, 2, Jiaqi Zhu
1, 2, Ziyuan Gao
1, 2, Yungang Bao
1, 2, 3, Sa Wang
1, 2, *, authorsList=Guodong Liu, Jiaqi Zhu, Ziyuan Gao, Yungang Bao, Sa Wang, authorCompany=null, correspAuthors=Sa Wang, authorNote=
born in 1997. PhD candidate. His main research interests include distributed systems and machine learning systems
born in 1999. Master candidate. His main research interests include operating systems and machine learning systems. (zhujiaqi22s@ict.ac.cn)
born in 2001. Master candidate. His main research interests include machine learning system, virtualization, and cloud computing. (gaoziyuan23s@ict.ac.cn)
born in 1980. PhD, professor. Member of CCF. His main research interests include data center architecture, agile design methodology of processor chips, and ecosystem of open-source processor chips. (baoyg@ict.ac.cn)
born in 1986. PhD, associate professor. Member of CCF. His main research interests include cloud computing, operating systems, and system modeling and performance analysis
, correspAuthorsNote=null, copyrightStatement=Copyright © 2025 Journal of Computer Research and Development. All rights reserved., copyrightOwner=null, extLink=null, articleAbsUrl=null, sourceXml=null, magXml=null, pdfUrl=null, pdf=null, pdfFileSize=null, pdfExtLink=null, richHtmlUrl=null, mobilePdfUrl=null, reviewReport=null, pdfFirstPage=null, abstractGraph=null, abstractGraphContent=null, abstractVideo=null, citation=null, cebUrl=null, magXmlContent=null, mapNumber=null, fund=null), CN=ArticleExt(id=1327017405081039681, articleId=1327017398063969045, tenantId=1146029695717560320, journalId=1324759914146529478, language=CN, title=AdaptDNN:一个自适应可伸缩的大模型分布式训练系统, columnId=1327017398395319064, journalTitle=计算机研究与发展, columnName=分布式计算, runingTitle=null, highlight=null, articleAbstract=
随着深度学习模型参数量的不断增加,训练成本也在不断上升. 为了减少训练成本,使用云服务厂商提供的弹性实例训练模型成为了一个可行的解决方案. 弹性实例的价格仅为正常实例的30%,可以有效降低训练成本. 虽然弹性实例价格低廉,但随时都有被回收的风险,对模型训练系统的稳定性提出了新的挑战. 为了解决弹性实例场景下的容错问题,现有的工作主要有2类解决方案,分别是基于存盘点的容错和基于冗余性的容错. 基于存盘点的方案开销较大,而基于冗余性的方案则对模型的并行策略有一定的限制,导致训练效率并非最优. AdaptDNN是一种自适应可伸缩的大模型分布式训练系统. 在弹性实例训练场景,AdaptDNN利用弹性实例宽限期完成训练进度的备份,降低容错开销;并利用“瓶颈消除”思想调整模型并行策略,最大化利用集群可用资源,提升训练效率. 实验结果表明,AdaptDNN既能实现低成本容错,又能保证模型训练效率,从而可以在弹性实例场景高效完成模型训练任务,降低模型训练成本.
, authors=刘国栋
1, 2, 朱家祺
1, 2, 高梓源
1, 2, 包云岗
1, 2, 3, 王卅
1, 2, *, authorsList=刘国栋, 朱家祺, 高梓源, 包云岗, 王卅, authorCompany=null, correspAuthors=王卅, authorNote=
 |
1997年生. 博士研究生. 主要研究方向为分布式系统、机器学习系统 |
 |
1999年生. 硕士研究生. 主要研究方向为操作系统、机器学习系统 |
 |
2001年生. 硕士研究生. 主要研究方向为机器学习系统、虚拟化、云计算 |
 |
1980年生. 博士,研究员. CCF会员. 主要研究方向为数据中心体系结构、处理器芯片敏捷设计方法论、开源芯片生态 |
 |
1986年生. 博士,副研究员. CCF会员. 主要研究方向为云计算、操作系统、系统建模与性能分析 |
, correspAuthorsNote=
, copyrightStatement=版权所有 © 2025 计算机研究与发展编辑部, copyrightOwner=null, extLink=null, articleAbsUrl=null, sourceXml=gkHQZ2JKXzc1z1ZefH2iNA==, magXml=7b6BAQm8L46UYOzzLnzY1Q==, pdfUrl=null, pdf=7mP6AAkVYsp37ptnVaMe1A==, pdfFileSize=1717681, pdfExtLink=null, richHtmlUrl=null, mobilePdfUrl=null, reviewReport=null, pdfFirstPage=null, abstractGraph=jLHYlTcMToAbKo/OM3DfTg==, abstractGraphContent=null, abstractVideo=null, citation=null, cebUrl=null, magXmlContent=2nKMC+1/PqQLVW+JEy2zCg==, mapNumber=null, fund=null)}, authors=[Author(id=1327017405626299212, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327017398063969045, orderNo=0, firstName=null, middleName=null, lastName=null, nameCn=null, orcid=null, stid=null, country=null, authorPic=null, dead=0, email=liuguodong19z@ict.ac.cn, emailSecond=null, emailThird=null, correspondingAuthor=0, authorType=1, ext={EN=AuthorExt(id=1327017405701796687, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327017398063969045, authorId=1327017405626299212, language=EN, stringName=Guodong Liu, firstName=Guodong, middleName=null, lastName=Liu, prefix=null, suffix=null, authorComment=null, nameInitials=null, affiliation=null, department=null, xref=
1, 2, address=
1State Key Lab of Processors (Institute of Computing Technology, Chinese Academy of Sciences), Beijing 100190
2School of Computer Science and Technology, University of Chinese Academy of Sciences, Beijing 100049, bio={"content":"
born in 1997. PhD candidate. His main research interests include distributed systems and machine learning systems
"}, bioImg=null, bioContent=
born in 1997. PhD candidate. His main research interests include distributed systems and machine learning systems
, aboutCorrespAuthor=null), CN=AuthorExt(id=1327017405785682768, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327017398063969045, authorId=1327017405626299212, language=CN, stringName=刘国栋, firstName=null, middleName=null, lastName=null, prefix=null, suffix=null, authorComment=null, nameInitials=null, affiliation=null, department=null, xref=
1, 2, address=
1处理器芯片全国重点实验室(中国科学院计算技术研究所) 北京 100190
2中国科学院大学计算机科学与技术学院 北京 100049, bio={"img":"kqpEJ9QjvXTxp0ddc18OCA==","content":"
1997年生. 博士研究生. 主要研究方向为分布式系统、机器学习系统
"}, bioImg=kqpEJ9QjvXTxp0ddc18OCA==, bioContent=
1997年生. 博士研究生. 主要研究方向为分布式系统、机器学习系统
, aboutCorrespAuthor=null)}, companyList=[AuthorCompany(id=1327017405328503618, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327017398063969045, xref=1, ext=[AuthorCompanyExt(id=1327017405336892227, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327017398063969045, companyId=1327017405328503618, language=EN, country=null, province=null, city=null, postcode=null, companyName=null, departmentName=null, remark=
1State Key Lab of Processors (Institute of Computing Technology, Chinese Academy of Sciences), Beijing 100190), AuthorCompanyExt(id=1327017405345280836, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327017398063969045, companyId=1327017405328503618, language=CN, country=null, province=null, city=null, postcode=null, companyName=null, departmentName=null, remark=
1处理器芯片全国重点实验室(中国科学院计算技术研究所) 北京 100190)]), AuthorCompany(id=1327017405404001093, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327017398063969045, xref=2, ext=[AuthorCompanyExt(id=1327017405412389702, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327017398063969045, companyId=1327017405404001093, language=EN, country=null, province=null, city=null, postcode=null, companyName=null, departmentName=null, remark=
2School of Computer Science and Technology, University of Chinese Academy of Sciences, Beijing 100049), AuthorCompanyExt(id=1327017405420778311, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327017398063969045, companyId=1327017405404001093, language=CN, country=null, province=null, city=null, postcode=null, companyName=null, departmentName=null, remark=
2中国科学院大学计算机科学与技术学院 北京 100049)])]), Author(id=1327017405848597330, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327017398063969045, orderNo=1, firstName=null, middleName=null, lastName=null, nameCn=null, orcid=null, stid=null, country=null, authorPic=null, dead=0, email=zhujiaqi22s@ict.ac.cn, emailSecond=null, emailThird=null, correspondingAuthor=0, authorType=1, ext={EN=AuthorExt(id=1327017405953454933, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327017398063969045, authorId=1327017405848597330, language=EN, stringName=Jiaqi Zhu, firstName=Jiaqi, middleName=null, lastName=Zhu, prefix=null, suffix=null, authorComment=null, nameInitials=null, affiliation=null, department=null, xref=
1, 2, address=
1State Key Lab of Processors (Institute of Computing Technology, Chinese Academy of Sciences), Beijing 100190
2School of Computer Science and Technology, University of Chinese Academy of Sciences, Beijing 100049, bio={"content":"
born in 1999. Master candidate. His main research interests include operating systems and machine learning systems. (zhujiaqi22s@ict.ac.cn)
"}, bioImg=null, bioContent=
born in 1999. Master candidate. His main research interests include operating systems and machine learning systems. (zhujiaqi22s@ict.ac.cn)
, aboutCorrespAuthor=null), CN=AuthorExt(id=1327017406033146710, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327017398063969045, authorId=1327017405848597330, language=CN, stringName=朱家祺, firstName=null, middleName=null, lastName=null, prefix=null, suffix=null, authorComment=null, nameInitials=null, affiliation=null, department=null, xref=
1, 2, address=
1处理器芯片全国重点实验室(中国科学院计算技术研究所) 北京 100190
2中国科学院大学计算机科学与技术学院 北京 100049, bio={"img":"Dybo/kwYC4LzvNaihwOjgw==","content":"
1999年生. 硕士研究生. 主要研究方向为操作系统、机器学习系统
"}, bioImg=Dybo/kwYC4LzvNaihwOjgw==, bioContent=
1999年生. 硕士研究生. 主要研究方向为操作系统、机器学习系统
, aboutCorrespAuthor=null)}, companyList=[AuthorCompany(id=1327017405328503618, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327017398063969045, xref=1, ext=[AuthorCompanyExt(id=1327017405336892227, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327017398063969045, companyId=1327017405328503618, language=EN, country=null, province=null, city=null, postcode=null, companyName=null, departmentName=null, remark=
1State Key Lab of Processors (Institute of Computing Technology, Chinese Academy of Sciences), Beijing 100190), AuthorCompanyExt(id=1327017405345280836, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327017398063969045, companyId=1327017405328503618, language=CN, country=null, province=null, city=null, postcode=null, companyName=null, departmentName=null, remark=
1处理器芯片全国重点实验室(中国科学院计算技术研究所) 北京 100190)]), AuthorCompany(id=1327017405404001093, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327017398063969045, xref=2, ext=[AuthorCompanyExt(id=1327017405412389702, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327017398063969045, companyId=1327017405404001093, language=EN, country=null, province=null, city=null, postcode=null, companyName=null, departmentName=null, remark=
2School of Computer Science and Technology, University of Chinese Academy of Sciences, Beijing 100049), AuthorCompanyExt(id=1327017405420778311, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327017398063969045, companyId=1327017405404001093, language=CN, country=null, province=null, city=null, postcode=null, companyName=null, departmentName=null, remark=
2中国科学院大学计算机科学与技术学院 北京 100049)])]), Author(id=1327017406133810008, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327017398063969045, orderNo=2, firstName=null, middleName=null, lastName=null, nameCn=null, orcid=null, stid=null, country=null, authorPic=null, dead=0, email=gaoziyuan23s@ict.ac.cn, emailSecond=null, emailThird=null, correspondingAuthor=0, authorType=1, ext={EN=AuthorExt(id=1327017406221890395, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327017398063969045, authorId=1327017406133810008, language=EN, stringName=Ziyuan Gao, firstName=Ziyuan, middleName=null, lastName=Gao, prefix=null, suffix=null, authorComment=null, nameInitials=null, affiliation=null, department=null, xref=
1, 2, address=
1State Key Lab of Processors (Institute of Computing Technology, Chinese Academy of Sciences), Beijing 100190
2School of Computer Science and Technology, University of Chinese Academy of Sciences, Beijing 100049, bio={"content":"
born in 2001. Master candidate. His main research interests include machine learning system, virtualization, and cloud computing. (gaoziyuan23s@ict.ac.cn)
"}, bioImg=null, bioContent=
born in 2001. Master candidate. His main research interests include machine learning system, virtualization, and cloud computing. (gaoziyuan23s@ict.ac.cn)
, aboutCorrespAuthor=null), CN=AuthorExt(id=1327017406288999260, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327017398063969045, authorId=1327017406133810008, language=CN, stringName=高梓源, firstName=null, middleName=null, lastName=null, prefix=null, suffix=null, authorComment=null, nameInitials=null, affiliation=null, department=null, xref=
1, 2, address=
1处理器芯片全国重点实验室(中国科学院计算技术研究所) 北京 100190
2中国科学院大学计算机科学与技术学院 北京 100049, bio={"img":"aFfixB295d9HZRPRXvmtnA==","content":"
2001年生. 硕士研究生. 主要研究方向为机器学习系统、虚拟化、云计算
"}, bioImg=aFfixB295d9HZRPRXvmtnA==, bioContent=
2001年生. 硕士研究生. 主要研究方向为机器学习系统、虚拟化、云计算
, aboutCorrespAuthor=null)}, companyList=[AuthorCompany(id=1327017405328503618, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327017398063969045, xref=1, ext=[AuthorCompanyExt(id=1327017405336892227, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327017398063969045, companyId=1327017405328503618, language=EN, country=null, province=null, city=null, postcode=null, companyName=null, departmentName=null, remark=
1State Key Lab of Processors (Institute of Computing Technology, Chinese Academy of Sciences), Beijing 100190), AuthorCompanyExt(id=1327017405345280836, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327017398063969045, companyId=1327017405328503618, language=CN, country=null, province=null, city=null, postcode=null, companyName=null, departmentName=null, remark=
1处理器芯片全国重点实验室(中国科学院计算技术研究所) 北京 100190)]), AuthorCompany(id=1327017405404001093, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327017398063969045, xref=2, ext=[AuthorCompanyExt(id=1327017405412389702, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327017398063969045, companyId=1327017405404001093, language=EN, country=null, province=null, city=null, postcode=null, companyName=null, departmentName=null, remark=
2School of Computer Science and Technology, University of Chinese Academy of Sciences, Beijing 100049), AuthorCompanyExt(id=1327017405420778311, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327017398063969045, companyId=1327017405404001093, language=CN, country=null, province=null, city=null, postcode=null, companyName=null, departmentName=null, remark=
2中国科学院大学计算机科学与技术学院 北京 100049)])]), Author(id=1327017406389662558, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327017398063969045, orderNo=3, firstName=null, middleName=null, lastName=null, nameCn=null, orcid=null, stid=null, country=null, authorPic=null, dead=0, email=baoyg@ict.ac.cn, emailSecond=null, emailThird=null, correspondingAuthor=0, authorType=1, ext={EN=AuthorExt(id=1327017406557434722, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327017398063969045, authorId=1327017406389662558, language=EN, stringName=Yungang Bao, firstName=Yungang, middleName=null, lastName=Bao, prefix=null, suffix=null, authorComment=null, nameInitials=null, affiliation=null, department=null, xref=
1, 2, 3, address=
1State Key Lab of Processors (Institute of Computing Technology, Chinese Academy of Sciences), Beijing 100190
2School of Computer Science and Technology, University of Chinese Academy of Sciences, Beijing 100049
3Beijing Institute of Open Source Chip, Beijing 100080, bio={"content":"
born in 1980. PhD, professor. Member of CCF. His main research interests include data center architecture, agile design methodology of processor chips, and ecosystem of open-source processor chips. (baoyg@ict.ac.cn)
"}, bioImg=null, bioContent=
born in 1980. PhD, professor. Member of CCF. His main research interests include data center architecture, agile design methodology of processor chips, and ecosystem of open-source processor chips. (baoyg@ict.ac.cn)
, aboutCorrespAuthor=null), CN=AuthorExt(id=1327017406620349283, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327017398063969045, authorId=1327017406389662558, language=CN, stringName=包云岗, firstName=null, middleName=null, lastName=null, prefix=null, suffix=null, authorComment=null, nameInitials=null, affiliation=null, department=null, xref=
1, 2, 3, address=
1处理器芯片全国重点实验室(中国科学院计算技术研究所) 北京 100190
2中国科学院大学计算机科学与技术学院 北京 100049
3北京开源芯片研究院 北京 100080, bio={"img":"7P6RQM1rfHT6kQbd8KJsVQ==","content":"
1980年生. 博士,研究员. CCF会员. 主要研究方向为数据中心体系结构、处理器芯片敏捷设计方法论、开源芯片生态
"}, bioImg=7P6RQM1rfHT6kQbd8KJsVQ==, bioContent=
1980年生. 博士,研究员. CCF会员. 主要研究方向为数据中心体系结构、处理器芯片敏捷设计方法论、开源芯片生态
, aboutCorrespAuthor=null)}, companyList=[AuthorCompany(id=1327017405328503618, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327017398063969045, xref=1, ext=[AuthorCompanyExt(id=1327017405336892227, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327017398063969045, companyId=1327017405328503618, language=EN, country=null, province=null, city=null, postcode=null, companyName=null, departmentName=null, remark=
1State Key Lab of Processors (Institute of Computing Technology, Chinese Academy of Sciences), Beijing 100190), AuthorCompanyExt(id=1327017405345280836, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327017398063969045, companyId=1327017405328503618, language=CN, country=null, province=null, city=null, postcode=null, companyName=null, departmentName=null, remark=
1处理器芯片全国重点实验室(中国科学院计算技术研究所) 北京 100190)]), AuthorCompany(id=1327017405404001093, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327017398063969045, xref=2, ext=[AuthorCompanyExt(id=1327017405412389702, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327017398063969045, companyId=1327017405404001093, language=EN, country=null, province=null, city=null, postcode=null, companyName=null, departmentName=null, remark=
2School of Computer Science and Technology, University of Chinese Academy of Sciences, Beijing 100049), AuthorCompanyExt(id=1327017405420778311, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327017398063969045, companyId=1327017405404001093, language=CN, country=null, province=null, city=null, postcode=null, companyName=null, departmentName=null, remark=
2中国科学院大学计算机科学与技术学院 北京 100049)]), AuthorCompany(id=1327017405504664392, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327017398063969045, xref=3, ext=[AuthorCompanyExt(id=1327017405517247305, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327017398063969045, companyId=1327017405504664392, language=EN, country=null, province=null, city=null, postcode=null, companyName=null, departmentName=null, remark=
3Beijing Institute of Open Source Chip, Beijing 100080), AuthorCompanyExt(id=1327017405525635914, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327017398063969045, companyId=1327017405504664392, language=CN, country=null, province=null, city=null, postcode=null, companyName=null, departmentName=null, remark=
3北京开源芯片研究院 北京 100080)])]), Author(id=1327017406683263845, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327017398063969045, orderNo=4, firstName=null, middleName=null, lastName=null, nameCn=null, orcid=null, stid=null, country=null, authorPic=null, dead=0, email=wangsa@ict.ac.cn, emailSecond=null, emailThird=null, correspondingAuthor=1, authorType=1, ext={EN=AuthorExt(id=1327017406771344232, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327017398063969045, authorId=1327017406683263845, language=EN, stringName=Sa Wang, firstName=Sa, middleName=null, lastName=Wang, prefix=null, suffix=null, authorComment=null, nameInitials=null, affiliation=null, department=null, xref=
1, 2, *, address=
1State Key Lab of Processors (Institute of Computing Technology, Chinese Academy of Sciences), Beijing 100190
2School of Computer Science and Technology, University of Chinese Academy of Sciences, Beijing 100049, bio={"content":"
born in 1986. PhD, associate professor. Member of CCF. His main research interests include cloud computing, operating systems, and system modeling and performance analysis
"}, bioImg=null, bioContent=
born in 1986. PhD, associate professor. Member of CCF. His main research interests include cloud computing, operating systems, and system modeling and performance analysis
, aboutCorrespAuthor=null), CN=AuthorExt(id=1327017406846841705, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327017398063969045, authorId=1327017406683263845, language=CN, stringName=王卅, firstName=null, middleName=null, lastName=null, prefix=null, suffix=null, authorComment=null, nameInitials=null, affiliation=null, department=null, xref=
1, 2, *, address=
1处理器芯片全国重点实验室(中国科学院计算技术研究所) 北京 100190
2中国科学院大学计算机科学与技术学院 北京 100049, bio={"img":"zanVPMzIJcEekagYZAutzw==","content":"
1986年生. 博士,副研究员. CCF会员. 主要研究方向为云计算、操作系统、系统建模与性能分析
"}, bioImg=zanVPMzIJcEekagYZAutzw==, bioContent=
1986年生. 博士,副研究员. CCF会员. 主要研究方向为云计算、操作系统、系统建模与性能分析
, aboutCorrespAuthor=null)}, companyList=[AuthorCompany(id=1327017405328503618, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327017398063969045, xref=1, ext=[AuthorCompanyExt(id=1327017405336892227, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327017398063969045, companyId=1327017405328503618, language=EN, country=null, province=null, city=null, postcode=null, companyName=null, departmentName=null, remark=
1State Key Lab of Processors (Institute of Computing Technology, Chinese Academy of Sciences), Beijing 100190), AuthorCompanyExt(id=1327017405345280836, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327017398063969045, companyId=1327017405328503618, language=CN, country=null, province=null, city=null, postcode=null, companyName=null, departmentName=null, remark=
1处理器芯片全国重点实验室(中国科学院计算技术研究所) 北京 100190)]), AuthorCompany(id=1327017405404001093, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327017398063969045, xref=2, ext=[AuthorCompanyExt(id=1327017405412389702, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327017398063969045, companyId=1327017405404001093, language=EN, country=null, province=null, city=null, postcode=null, companyName=null, departmentName=null, remark=
2School of Computer Science and Technology, University of Chinese Academy of Sciences, Beijing 100049), AuthorCompanyExt(id=1327017405420778311, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327017398063969045, companyId=1327017405404001093, language=CN, country=null, province=null, city=null, postcode=null, companyName=null, departmentName=null, remark=
2中国科学院大学计算机科学与技术学院 北京 100049)])])], keywords=[Keyword(id=1327017406943310698, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327017398063969045, language=EN, orderNo=1, keyword=deep learning), Keyword(id=1327017407006225259, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327017398063969045, language=EN, orderNo=2, keyword=neural network), Keyword(id=1327017407060751212, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327017398063969045, language=EN, orderNo=3, keyword=distributed training), Keyword(id=1327017407115277165, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327017398063969045, language=EN, orderNo=4, keyword=spot instance), Keyword(id=1327017407169803118, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327017398063969045, language=EN, orderNo=5, keyword=fault tolerance), Keyword(id=1327017407224329071, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327017398063969045, language=CN, orderNo=1, keyword=深度学习), Keyword(id=1327017407308215152, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327017398063969045, language=CN, orderNo=2, keyword=神经网络), Keyword(id=1327017407396295537, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327017398063969045, language=CN, orderNo=3, keyword=分布式训练), Keyword(id=1327017409057239922, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327017398063969045, language=CN, orderNo=4, keyword=弹性实例), Keyword(id=1327017409124348787, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327017398063969045, language=CN, orderNo=5, keyword=容错)], refs=[Reference(id=1327017411825480602, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327017398063969045, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=1, rfOrder=0, authorNames=null, journalName=null, refType=null, unstructuredReference=OpenAI. AI and compute[EB/OL]. (2018-05-16)[2024-05-30]. https://openai.com/index/ai-and-compute/, articleTitle=null, refAbstract=null), Reference(id=1327017411892589467, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327017398063969045, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=2, rfOrder=1, authorNames=null, journalName=null, refType=null, unstructuredReference=Li Mu, Andersen D, Park J, et al. Scaling distributed machine learning with the parameter server[C]//Proc of the 11th USENIX Symp on Operating Systems Design and Implementation. Berkeley, CA: USENIX Association, 2014: 583−598, articleTitle=null, refAbstract=null), Reference(id=1327017413591282588, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327017398063969045, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=3, rfOrder=2, authorNames=null, journalName=null, refType=null, unstructuredReference=Jia Zhihao, Zaharia M, Aiken A. Beyond data and model parallelism for deep neural networks[C/OL]//Proc of the 2nd Machine Learning and Systems Conf. 2019[2024-10-21]. https://proceedings.mlsys.org/paper_files/paper/2019/file/b422680f3db0986ddd7f8f126baaf0fa-Paper.pdf, articleTitle=null, refAbstract=null), Reference(id=1327017413666780061, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327017398063969045, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=4, rfOrder=3, authorNames=null, journalName=null, refType=null, unstructuredReference=Narayanan D, Harlap A, Phanishayee A, et al. PipeDream: Generalized pipeline parallelism for DNN training[C]//Proc of the 27th ACM Symp on Operating Systems Principles. New York: ACM, 2019: 1−15, articleTitle=null, refAbstract=null), Reference(id=1327017413733888926, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327017398063969045, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=5, rfOrder=4, authorNames=null, journalName=null, refType=null, unstructuredReference=Brown T, Mann B, Ryder N, et al. Language models are few-shot learners[C/OL]//Proc of the 34th Conf on Neural Information Processing Systems. Cambridge, MA: MIT, 2020[2024-10-21]. https://papers.nips.cc/paper_files/paper/2020/file/1457c0d6bfcb4967418bfb8ac142f64a-Paper.pdf, articleTitle=null, refAbstract=null), Reference(id=1327017413821969311, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327017398063969045, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=6, rfOrder=5, authorNames=null, journalName=null, refType=null, unstructuredReference=Duan Jiangfei, Song Ziang, Miao Xupeng, et al. Parcae: Proactive, liveput-optimized DNN training on preemptible instances[C]//Proc of the 21st USENIX Symp on Networked Systems Design and Implementation. Berkeley, CA: USENIX Association, 2024: 1121−1139, articleTitle=null, refAbstract=null), Reference(id=1327017413897466784, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327017398063969045, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=7, rfOrder=6, authorNames=null, journalName=null, refType=null, unstructuredReference=Thorpe J, Zhao Pengzhan, Eyolfson J, et al. Bamboo: Making preemptible instances resilient for affordable training of large DNNs[C]//Proc of the 20th USENIX Symp on Networked Systems Design and Implementation. Berkeley, CA: USENIX Association, 2023: 497−513, articleTitle=null, refAbstract=null), Reference(id=1327017413989741473, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327017398063969045, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=8, rfOrder=7, authorNames=null, journalName=null, refType=null, unstructuredReference=Li Jiamin, Xu Hong, Zhu Yibo, et al. Lyra: Elastic scheduling for deep learning clusters[C]//Proc of the 18th European Conf on Computer Systems. New York: ACM, 2023: 835−850, articleTitle=null, refAbstract=null), Reference(id=1327017414061044642, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327017398063969045, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=9, rfOrder=8, authorNames=null, journalName=null, refType=null, unstructuredReference=Narayanan D, Shoeybi M, Casper J, et al. Efficient large-scale language model training on GPU clusters using megatron-LM[C]//Proc of the 34th Int Conf for High Performance Computing, Networking, Storage and Analysis. New York: ACM, 2021: 1−15, articleTitle=null, refAbstract=null), Reference(id=1327017414132347811, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327017398063969045, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=10, rfOrder=9, authorNames=null, journalName=null, refType=null, unstructuredReference=Mohan J, Phanishayee A, Chidambaram V. CheckFreq: Frequent, fine-grained DNN checkpointing[C]//Proc of the 19th USENIX Conf on File and Storage Technologies. Berkeley, CA: USENIX Association, 2021: 203−216, articleTitle=null, refAbstract=null), Reference(id=1327017414195262372, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327017398063969045, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=11, rfOrder=10, authorNames=null, journalName=null, refType=null, unstructuredReference=Athlur S, Saran N, Sivathanu M, et al. Varuna: Scalable, low-cost training of massive deep learning models[C]//Proc of the 17th European Conf on Computer Systems. New York: ACM, 2022: 472−487, articleTitle=null, refAbstract=null), Reference(id=1327017414258176933, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327017398063969045, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=12, rfOrder=11, authorNames=null, journalName=null, refType=null, unstructuredReference=Jang I, Yang Zhenning, Zhang Zhen, et al. Oobleck: Resilient distributed training of large models using pipeline templates[C]//Proc of the 29th Symp on Operating Systems Principles. New York: ACM, 2023: 382−395, articleTitle=null, refAbstract=null), Reference(id=1327017414329480102, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327017398063969045, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=13, rfOrder=12, authorNames=null, journalName=null, refType=null, unstructuredReference=Zheng Lianmin, Li Zhuohan, Zhang Hao, et al. Alpa: Automating inter-and intra-operator parallelism for distributed deep learning[C]//Proc of the 16th USENIX Symp on Operating Systems Design and Implementation. Berkeley, CA: USENIX Association, 2022: 559−578, articleTitle=null, refAbstract=null), Reference(id=1327017414384006055, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327017398063969045, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=14, rfOrder=13, authorNames=null, journalName=null, refType=null, unstructuredReference=Liu Guodong, Miao Youshan, Lin Zhiqi, et al. Aceso: Efficient parallel DNN training through iterative bottleneck alleviation[C]//Proc of the 19th European Conf on Computer Systems. New York: ACM, 2024: 163−181, articleTitle=null, refAbstract=null), Reference(id=1327017414476280744, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327017398063969045, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=15, rfOrder=14, authorNames=null, journalName=null, refType=null, unstructuredReference=Dean J, Corrado G, Monga R, et al. Large scale distributed deep networks[C/OL]//Proc of the 26th Conf on Neural Information Processing Systems. Cambridge, MA: MIT, 2012[2024-10-21]. https://papers.nips.cc/paper_files/paper/2012/file/6aca97005c68f1206823815f66102863-Paper.pdf, articleTitle=null, refAbstract=null), Reference(id=1327017414560166825, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327017398063969045, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=16, rfOrder=15, authorNames=null, journalName=null, refType=null, unstructuredReference=Wang Minjie, Huang C, Li Jinyang. Supporting very large models using automatic dataflow graph partitioning[C]//Proc of the 14th European Conf on Computer Systems. New York: ACM, 2019: 1−17, articleTitle=null, refAbstract=null), Reference(id=1327017414618887082, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327017398063969045, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=17, rfOrder=16, authorNames=null, journalName=null, refType=null, unstructuredReference=Mirhoseini A, Pham H, Le Q, et al. Device placement optimization with reinforcement learning[C]//Proc of the 34th Int Conf on Machine Learning. New York: PMLR, 2017: 2430-−2439, articleTitle=null, refAbstract=null), Reference(id=1327017414685995947, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327017398063969045, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=18, rfOrder=17, authorNames=null, journalName=null, refType=null, unstructuredReference=Gao Yuanxiang, Chen Li, Li Baochun. Spotlight: Optimizing device placement for training deep neural networks[C]//Proc of the 35th Int Conf on Machine Learning. New York: PMLR, 2018: 1676−1684, articleTitle=null, refAbstract=null), Reference(id=1327017414753104812, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327017398063969045, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=19, rfOrder=18, authorNames=null, journalName=null, refType=null, unstructuredReference=Xu Yuanzhong, HyoukJoong L, Chen Dehao, et al. GSPMD: General and scalable parallelization for ML computation graphs[J]. arXiv preprint, arXiv: 2105.04663, articleTitle=null, refAbstract=null), Reference(id=1327017414820213677, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327017398063969045, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=20, rfOrder=19, authorNames=null, journalName=null, refType=null, unstructuredReference=Huang Yanping, Cheng Youlong, Bapna A, et al. Gpipe: Efficient training of giant neural networks using pipeline parallelism[C/OL]//Proc of the 33rd Conf on Neural Information Processing Systems. Cambridge, MA: MIT, 2019[2024-10-21]. https://proceedings.neurips.cc/paper_files/paper/2019/file/093f65e080a295f8076b1c5722a46aa2-Paper.pdf, articleTitle=null, refAbstract=null), Reference(id=1327017414874739630, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327017398063969045, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=21, rfOrder=20, authorNames=null, journalName=null, refType=null, unstructuredReference=Narayanan D, Phanishayee A, Shi K, et al. Memory-efficient pipeline-parallel DNN training[C]//Proc of the 38th Int Conf on Machine Learning. New York: PMLR, 2021: 7937−7947, articleTitle=null, refAbstract=null), Reference(id=1327017414975402927, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327017398063969045, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=22, rfOrder=21, authorNames=null, journalName=null, refType=null, unstructuredReference=Li Shigang, Torsten H. Chimera: Efficiently training large-scale neural networks with bidirectional pipelines[C]//Proc of the 34th Int Conf for High Performance Computing, Networking, Storage and Analysis. New York: ACM, 2021: 1−14, articleTitle=null, refAbstract=null), Reference(id=1327017415042511792, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327017398063969045, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=23, rfOrder=22, authorNames=null, journalName=null, refType=null, unstructuredReference=Fan Shiqing, Rong Yi, Meng Chen, et al. DAPPLE: A pipelined data parallel approach for training large models[C]//Proc of the 26th ACM SIGPLAN Symp on Principles and Practice of Parallel Programming. New York: ACM, 2021: 431−445, articleTitle=null, refAbstract=null), Reference(id=1327017415109620657, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327017398063969045, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=24, rfOrder=23, authorNames=null, journalName=null, refType=null, unstructuredReference=PyTorch. Torch Distributed Elastic[EB/OL]. [2024-05-30]. https://pytorch.org/docs/stable/distributed.elastic.html, articleTitle=null, refAbstract=null), Reference(id=1327017415180923826, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327017398063969045, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=25, rfOrder=24, authorNames=null, journalName=null, refType=null, unstructuredReference=The Horovod Authors. Elastic Hovorod[EB/OL]. [2024-05-30]. https://horovod.readthedocs.io/en/stable/elastic_include.html, articleTitle=null, refAbstract=null), Reference(id=1327017415252226995, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327017398063969045, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=26, rfOrder=25, authorNames=null, journalName=null, refType=null, unstructuredReference=Wu Yidi, Ma Kaihao, Yan Xiao, et al. Elastic deep learning in multi-tenant GPU clusters[J]. IEEE Transactions on Parallel and Distributed Systems, 2021, 33(1): 144−158, articleTitle=null, refAbstract=null), Reference(id=1327017415323530164, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327017398063969045, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=27, rfOrder=26, authorNames=null, journalName=null, refType=null, unstructuredReference=Xiao Wencong, Bhardwaj R, Ramjee R, et al. Gandiva: Introspective cluster scheduling for deep learning[C]//Proc of the 13th USENIX Symp on Operating Systems Design and Implementation. Berkeley, CA: USENIX Association, 2018: 595−610, articleTitle=null, refAbstract=null), Reference(id=1327017415403221941, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327017398063969045, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=28, rfOrder=27, authorNames=null, journalName=null, refType=null, unstructuredReference=Qiao A, Choe S, Subramanya S, et al. Pollux: Co-adaptive cluster scheduling for goodput-optimized deep learning[C]//Proc of the 15th USENIX Symp on Operating Systems Design and Implementation. Berkeley, CA: USENIX Association, 2021: 1−18, articleTitle=null, refAbstract=null), Reference(id=1327017415478719414, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327017398063969045, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=29, rfOrder=28, authorNames=null, journalName=null, refType=null, unstructuredReference=Gu Diandian, Zhao Yihao, Zhong Yinmin, et al. ElasticFlow: An elastic serverless training platform for distributed deep learning[C]//Proc of the 28th ACM Int Conf on Architectural Support for Programming Languages and Operating Systems. New York: ACM, 2023: 266−280, articleTitle=null, refAbstract=null), Reference(id=1327017415604548535, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327017398063969045, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=30, rfOrder=29, authorNames=null, journalName=null, refType=null, unstructuredReference=Wang Zhuang, Jia Zhen, Zheng Shuai, et al. Gemini: Fast failure recovery in distributed training with in-memory checkpoints[C]//Proc of the 29th Symp on Operating Systems Principles. New York: ACM, 2023: 364−381, articleTitle=null, refAbstract=null), Reference(id=1327017415680046008, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327017398063969045, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=31, rfOrder=30, authorNames=null, journalName=null, refType=null, unstructuredReference=He Tao, Li Xue, Wang Zhibin, et al. Unicron: Economizing self-healing LLM training at scale[J]. arXiv preprint, arXiv: 2401.00134, 2023, articleTitle=null, refAbstract=null), Reference(id=1327017415751349177, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327017398063969045, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=32, rfOrder=31, authorNames=null, journalName=null, refType=null, unstructuredReference=Jiang Ziheng, Lin Haibin, Zhong Yinmin, et al. MegaScale: Scaling large language model training to more than <styled-content style-type="number"> 10000 </styled-content> GPUs[C]//Proc of the 21st USENIX Symp on Networked Systems Design and Implementation. Berkeley, CA: USENIX Association, 2024: 745−760, articleTitle=null, refAbstract=null), Reference(id=1327017415822652346, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327017398063969045, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=33, rfOrder=32, authorNames=null, journalName=null, refType=null, unstructuredReference=Wang Yuxin, Kang Xueze, Shi Shaohuai, et al. Reliable and efficient in-memory fault tolerance of large language model pretraining[EB/OL]. (2024-08-19)[2024-09-06]. https://arxiv.org/abs/2310.12670, articleTitle=null, refAbstract=null), Reference(id=1327017415906538427, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327017398063969045, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=34, rfOrder=33, authorNames=null, journalName=null, refType=null, unstructuredReference=Lian Xinyu, Jacobs S, Kurilenko L, et al. Universal checkpointing: Efficient and flexible checkpointing for large scale distributed training[J]. arXiv preprint, arXiv: 2406.18820, 2024, articleTitle=null, refAbstract=null), Reference(id=1327017415990424508, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327017398063969045, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=35, rfOrder=34, authorNames=null, journalName=null, refType=null, unstructuredReference=Wan Borui, Han Mingji, Sheng Yiyao, et al. ByteCheckpoint: A unified checkpointing system for LLM development[J]. arXiv preprint, arXiv: 2407.20143, 2024, articleTitle=null, refAbstract=null), Reference(id=1327017416086893501, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327017398063969045, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=36, rfOrder=35, authorNames=null, journalName=null, refType=null, unstructuredReference=Gandhi S, Zhao M, Skiadopoulos A, et al. SlipStream: Adapting pipelines for distributed training of large DNNs amid failures[J]. arXiv preprint, arXiv: 2405.14009, 2024, articleTitle=null, refAbstract=null), Reference(id=1327017416183362494, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327017398063969045, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=37, rfOrder=36, authorNames=null, journalName=null, refType=null, unstructuredReference=Vaswani A, Shazeer N, Parmar N, et al. Attention is all you need[C/OL]//Proc of the 31st Conf on Neural Information Processing Systems. Cambridge, MA: MIT, 2017[2024-10-21]. https://user.phil.hhu.de/~cwurm/wp-content/uploads/2020/01/7181-attention-is-all-you-need.pdf, articleTitle=null, refAbstract=null), Reference(id=1327017416254665663, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327017398063969045, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=38, rfOrder=37, authorNames=null, journalName=null, refType=null, unstructuredReference=Navarro A, Asenjo R, Tabik S, et al. Analytical modeling of pipeline parallelism[C]//Proc of the 18th Int Conf on Parallel Architectures and Compilation Techniques. Piscataway, NJ: IEEE, 2009: 291−290, articleTitle=null, refAbstract=null), Reference(id=1327017416325968832, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327017398063969045, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=39, rfOrder=38, authorNames=null, journalName=null, refType=null, unstructuredReference=Perez T, Chen Wei, Ji R, et al. PETS: Bottleneck-aware spark tuning with parameter ensembles[C]//Proc of the 27th Int Conf on Computer Communication and Networks. Piscataway, NJ: IEEE, 2018: 1−9, articleTitle=null, refAbstract=null), Reference(id=1327017416401466305, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327017398063969045, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=40, rfOrder=39, authorNames=null, journalName=null, refType=null, unstructuredReference=Kahveci B, Gedik B. Joker: Elastic stream processing with organic adaptation[J]. Journal of Parallel and Distributed Computing, 2020, 137: 205-223, articleTitle=null, refAbstract=null), Reference(id=1327017418125325250, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327017398063969045, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=40, rfOrder=40, authorNames=null, journalName=null, refType=null, unstructuredReference=Kahveci B, Gedik B. Joker: Elastic stream processing with organic adaptation[J]. Journal of Parallel and Distributed Computing, 2020, 137: 205-223, articleTitle=null, refAbstract=null), Reference(id=1327017418238571459, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327017398063969045, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=41, rfOrder=41, authorNames=null, journalName=null, refType=null, unstructuredReference=NVIDIA. ConnectX-5: Advanced offload capabilities for the most demanding applications[EB/OL]. [2024-05-30]. https://www.nvidia.com/ en-us/networking/ethernet/connectx-5/, articleTitle=null, refAbstract=null)], funds=null, companyList=[AuthorCompany(id=1327017405328503618, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327017398063969045, xref=1, ext=[AuthorCompanyExt(id=1327017405336892227, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327017398063969045, companyId=1327017405328503618, language=EN, country=null, province=null, city=null, postcode=null, companyName=null, departmentName=null, remark=
1State Key Lab of Processors (Institute of Computing Technology, Chinese Academy of Sciences), Beijing 100190), AuthorCompanyExt(id=1327017405345280836, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327017398063969045, companyId=1327017405328503618, language=CN, country=null, province=null, city=null, postcode=null, companyName=null, departmentName=null, remark=
1处理器芯片全国重点实验室(中国科学院计算技术研究所) 北京 100190)]), AuthorCompany(id=1327017405404001093, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327017398063969045, xref=2, ext=[AuthorCompanyExt(id=1327017405412389702, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327017398063969045, companyId=1327017405404001093, language=EN, country=null, province=null, city=null, postcode=null, companyName=null, departmentName=null, remark=
2School of Computer Science and Technology, University of Chinese Academy of Sciences, Beijing 100049), AuthorCompanyExt(id=1327017405420778311, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327017398063969045, companyId=1327017405404001093, language=CN, country=null, province=null, city=null, postcode=null, companyName=null, departmentName=null, remark=
2中国科学院大学计算机科学与技术学院 北京 100049)]), AuthorCompany(id=1327017405504664392, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327017398063969045, xref=3, ext=[AuthorCompanyExt(id=1327017405517247305, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327017398063969045, companyId=1327017405504664392, language=EN, country=null, province=null, city=null, postcode=null, companyName=null, departmentName=null, remark=
3Beijing Institute of Open Source Chip, Beijing 100080), AuthorCompanyExt(id=1327017405525635914, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327017398063969045, companyId=1327017405504664392, language=CN, country=null, province=null, city=null, postcode=null, companyName=null, departmentName=null, remark=
3北京开源芯片研究院 北京 100080)])], figs=[ArticleFig(id=1327017409266955124, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327017398063969045, language=EN, label=Figure 1, caption=
System overview, figureFileSmall=edI/EJ3q3g4GHyi/3F9t1A==, figureFileBig=jLHYlTcMToAbKo/OM3DfTg==, tableContent=null), ArticleFig(id=1327017409338258293, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327017398063969045, language=CN, label=图1, caption=
系统概览, figureFileSmall=edI/EJ3q3g4GHyi/3F9t1A==, figureFileBig=jLHYlTcMToAbKo/OM3DfTg==, tableContent=null), ArticleFig(id=1327017409501836150, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327017398063969045, language=EN, label=Figure 2, caption=
3D parallelism, figureFileSmall=ErDFHO+cbn2YyZFLmh490A==, figureFileBig=e0GQv5Vp6Hfvid+FUbGoLQ==, tableContent=null), ArticleFig(id=1327017409564750712, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327017398063969045, language=CN, label=图2, caption=
3D并行, figureFileSmall=ErDFHO+cbn2YyZFLmh490A==, figureFileBig=e0GQv5Vp6Hfvid+FUbGoLQ==, tableContent=null), ArticleFig(id=1327017409627665273, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327017398063969045, language=EN, label=Figure 3, caption=
Heterogenous pipeline, figureFileSmall=NzPxAokLSTYGpoah/qs6eQ==, figureFileBig=xCWAF5JlL4KwrX3VZXYi4w==, tableContent=null), ArticleFig(id=1327017409698968442, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327017398063969045, language=CN, label=图3, caption=
异构流水线, figureFileSmall=NzPxAokLSTYGpoah/qs6eQ==, figureFileBig=xCWAF5JlL4KwrX3VZXYi4w==, tableContent=null), ArticleFig(id=1327017409757688699, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327017398063969045, language=EN, label=Figure 4, caption=
Comparison of model reconstruction overhead of different fault-tolerant methods, figureFileSmall=xriYJyEkQK4zl/UGbX+1FQ==, figureFileBig=gaTr9VdIVX67L7Kr78ks5A==, tableContent=null), ArticleFig(id=1327017409812214652, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327017398063969045, language=CN, label=图4, caption=
不同容错方式的模型重构开销对比, figureFileSmall=xriYJyEkQK4zl/UGbX+1FQ==, figureFileBig=gaTr9VdIVX67L7Kr78ks5A==, tableContent=null), ArticleFig(id=1327017409866740605, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327017398063969045, language=EN, label=Figure 5, caption=
Searching for parallel schemes by using bottleneck alleviation idea, figureFileSmall=2pqxotJYB3tprKg0Uxa4Sw==, figureFileBig=+alhFog6S6ybr3v/n8HBBg==, tableContent=null), ArticleFig(id=1327017409929655166, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327017398063969045, language=CN, label=图5, caption=
使用“瓶颈消除”思想搜索并行方案, figureFileSmall=2pqxotJYB3tprKg0Uxa4Sw==, figureFileBig=+alhFog6S6ybr3v/n8HBBg==, tableContent=null), ArticleFig(id=1327017410021929855, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327017398063969045, language=EN, label=Figure 6, caption=
The first performance bottleneck in fault tolerance scenario, figureFileSmall=zVOjR5SK3O4WkbR+rV78fg==, figureFileBig=QIWRgzR03++qc8tN7NxNVA==, tableContent=null), ArticleFig(id=1327017410089038720, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327017398063969045, language=CN, label=图6, caption=
容错场景下的第1种性能瓶颈, figureFileSmall=zVOjR5SK3O4WkbR+rV78fg==, figureFileBig=QIWRgzR03++qc8tN7NxNVA==, tableContent=null), ArticleFig(id=1327017410147758977, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327017398063969045, language=EN, label=Figure 7, caption=
The second performance bottleneck in fault tolerance scenario, figureFileSmall=nYEkaerjnhjkVilZ9MNYUw==, figureFileBig=bmUY62du1UyvlTMQCN1MlA==, tableContent=null), ArticleFig(id=1327017410206479234, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327017398063969045, language=CN, label=图7, caption=
容错场景下的第2种性能瓶颈, figureFileSmall=nYEkaerjnhjkVilZ9MNYUw==, figureFileBig=bmUY62du1UyvlTMQCN1MlA==, tableContent=null), ArticleFig(id=1327017410261005187, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327017398063969045, language=EN, label=Figure 8, caption=
Workflow of bottleneck alleviation algorithm, figureFileSmall=Sn3YLyibGhe0WBk4eWCSPQ==, figureFileBig=sZNCM4NhlilL+SuQAMeMPQ==, tableContent=null), ArticleFig(id=1327017410323919748, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327017398063969045, language=CN, label=图8, caption=
“瓶颈消除”算法工作流程, figureFileSmall=Sn3YLyibGhe0WBk4eWCSPQ==, figureFileBig=sZNCM4NhlilL+SuQAMeMPQ==, tableContent=null), ArticleFig(id=1327017410399417221, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327017398063969045, language=EN, label=Figure 9, caption=
Overall throughput comparison of training system, figureFileSmall=W+DK5riCG1GCkkSnDzwFaQ==, figureFileBig=fJh0lnbkdMBebfF0Lh0NxQ==, tableContent=null), ArticleFig(id=1327017410458137478, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327017398063969045, language=CN, label=图9, caption=
训练系统总体吞吐量对比, figureFileSmall=W+DK5riCG1GCkkSnDzwFaQ==, figureFileBig=fJh0lnbkdMBebfF0Lh0NxQ==, tableContent=null), ArticleFig(id=1327017410516857735, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327017398063969045, language=EN, label=Figure 10, caption=
Comparison of training time per round, figureFileSmall=4y1aXFl/CO2YEO0Q4zc8BQ==, figureFileBig=5srYOjjpE1zeIWvrWKII1g==, tableContent=null), ArticleFig(id=1327017410567189384, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327017398063969045, language=CN, label=图10, caption=
每轮次训练时间对比, figureFileSmall=4y1aXFl/CO2YEO0Q4zc8BQ==, figureFileBig=5srYOjjpE1zeIWvrWKII1g==, tableContent=null), ArticleFig(id=1327017410621715337, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327017398063969045, language=EN, label=Figure 11, caption=
Comparison of time for fault recovery, figureFileSmall=lWzQZ+UXWOA0B5XR8bnsGw==, figureFileBig=ifWvUNGETAUx8dCQrOH4zw==, tableContent=null), ArticleFig(id=1327017410676241290, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327017398063969045, language=CN, label=图11, caption=
故障恢复时间对比, figureFileSmall=lWzQZ+UXWOA0B5XR8bnsGw==, figureFileBig=ifWvUNGETAUx8dCQrOH4zw==, tableContent=null), ArticleFig(id=1327017410734961547, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327017398063969045, language=EN, label=Figure 12, caption=
Comparison of training throughput under real spot instance trace, figureFileSmall=WXDFfSB08TWHclnCLqRrIA==, figureFileBig=CzgPOBUiMwWywA2XT1ykOQ==, tableContent=null), ArticleFig(id=1327017410785293196, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327017398063969045, language=CN, label=图12, caption=
真实弹性实例trace下的训练吞吐量对比, figureFileSmall=WXDFfSB08TWHclnCLqRrIA==, figureFileBig=CzgPOBUiMwWywA2XT1ykOQ==, tableContent=null), ArticleFig(id=1327017410848207757, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327017398063969045, language=EN, label=Figure 13, caption=
Influence of MTTF on iters, figureFileSmall=5Jc34oVkgpWfgYQ2qddClw==, figureFileBig=CRvAmRNczgm3V3iuS8iobA==, tableContent=null), ArticleFig(id=1327017410915316622, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327017398063969045, language=CN, label=图13, caption=
平均故障时间对iters的影响, figureFileSmall=5Jc34oVkgpWfgYQ2qddClw==, figureFileBig=CRvAmRNczgm3V3iuS8iobA==, tableContent=null), ArticleFig(id=1327017410990814095, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327017398063969045, language=EN, label=Figure 14, caption=
Influence of bandwidth on iters, figureFileSmall=T0PFkiiYg8ldbUo1JVQCmg==, figureFileBig=wmpnG6Y/dRBsO3Xpr+2qWg==, tableContent=null), ArticleFig(id=1327017411129226128, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327017398063969045, language=CN, label=图14, caption=
网络带宽对iters的影响, figureFileSmall=T0PFkiiYg8ldbUo1JVQCmg==, figureFileBig=wmpnG6Y/dRBsO3Xpr+2qWg==, tableContent=null), ArticleFig(id=1327017411200529297, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327017398063969045, language=EN, label=Figure 15, caption=
Influence of grace period on iters, figureFileSmall=tUOaDiineVNBwp+jRyWE2w==, figureFileBig=L9LFCtuDfJUn+OCSjxW5iA==, tableContent=null), ArticleFig(id=1327017411263443858, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327017398063969045, language=CN, label=图15, caption=
宽限期对iters的影响, figureFileSmall=tUOaDiineVNBwp+jRyWE2w==, figureFileBig=L9LFCtuDfJUn+OCSjxW5iA==, tableContent=null), ArticleFig(id=1327017411322164115, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327017398063969045, language=EN, label=Table 1, caption=
Experimental Platform Configuration, figureFileSmall=null, figureFileBig=null, tableContent=
| 配置 | 参数 |
|---|
| 处理器 | Intel® Xeon® Gold 5218R CPU |
| GPU | NVIDIA V100-PCIE(16 GB) |
| 操作系统 | Ubuntu 18.04.5 LTS |
| CUDA版本 | 11.6 |
| PyTorch版本 | 2.0.0 |
), ArticleFig(id=1327017411397661588, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327017398063969045, language=CN, label=表1, caption=
实验平台配置
, figureFileSmall=null, figureFileBig=null, tableContent=
| 配置 | 参数 |
|---|
| 处理器 | Intel® Xeon® Gold 5218R CPU |
| GPU | NVIDIA V100-PCIE(16 GB) |
| 操作系统 | Ubuntu 18.04.5 LTS |
| CUDA版本 | 11.6 |
| PyTorch版本 | 2.0.0 |
), ArticleFig(id=1327017411464770453, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327017398063969045, language=EN, label=Table 2, caption=
DNN Model Parameters, figureFileSmall=null, figureFileBig=null, tableContent=
| 配置 | 参数 |
|---|
| 参数量 | 350×106, 1.3×109, 2.7×109, 6.7×109 |
| 精度 | FP16 |
| 数据批次大小 | 1 024 |
| 序列长度 | 2 048 |
), ArticleFig(id=1327017411544462230, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327017398063969045, language=CN, label=表2, caption=
DNN模型参数
, figureFileSmall=null, figureFileBig=null, tableContent=
| 配置 | 参数 |
|---|
| 参数量 | 350×106, 1.3×109, 2.7×109, 6.7×109 |
| 精度 | FP16 |
| 数据批次大小 | 1 024 |
| 序列长度 | 2 048 |
), ArticleFig(id=1327017411628348311, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327017398063969045, language=EN, label=Table 3, caption=
Average Throughput Comparison, figureFileSmall=null, figureFileBig=null, tableContent=
| 模型 | GPT-3 1.3B | | GPT-3 2.7B |
|---|
| trace-1 | trace-2 | | trace-1 | trace-2 |
|---|
| Checkpoint | 17.13 | 16.38 | | 8.21 | 7.90 |
| Oobleck | 22.83 | 21.15 | | 12.07 | 11.79 |
| AdaptDNN(本文) | 27.89 | 26.92 | | 14.22 | 14.15 |
), ArticleFig(id=1327017411708040089, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327017398063969045, language=CN, label=表3, caption=
平均吞吐量对比
, figureFileSmall=null, figureFileBig=null, tableContent=
| 模型 | GPT-3 1.3B | | GPT-3 2.7B |
|---|
| trace-1 | trace-2 | | trace-1 | trace-2 |
|---|
| Checkpoint | 17.13 | 16.38 | | 8.21 | 7.90 |
| Oobleck | 22.83 | 21.15 | | 12.07 | 11.79 |
| AdaptDNN(本文) | 27.89 | 26.92 | | 14.22 | 14.15 |
)], attaches=null, journal=Journal(id=1324759606947315908, delFlag=0, nameCn=计算机研究与发展, nameEn=Journal of Computer Research and Development, nameHistory1=null, nameHistory2=null, issn=1000-1239, eissn=null, cn=11-1777/TP, coden=null, periodic=0, language=CN, oaType=null, ccby=null, superviseOffice=null, ownerOffice=null, pubOffice=null, editorOffice=null, officeType=null, aims=null, clcCode=null, officeProv=null, officeCity=null, officeAddr=null, officeZip=null, officeEmail=null, officePhone=null, editDirector=null, officeDirector=null, officeDirectorPhone=null, officeStaffNum=null, officeEmpNum=null, coverPicUrl=k2l8zgqwstuxxevFX2qxVA==, journalPrice=null, startedYear=null, abbrevIsoEn=Journal of Computer Research and Development, journalRemark=null, publicationField=null, createdTime=1793776903141, updatedTime=1793850698612, createdBy=18614031015, updatedBy=13701087609, firstLetterCn=J, firstLetterEn=J, subjectCode=Electronic Information, subjectName=null, subjectCodeEn=Electronic Information, subjectNameEn=null, picCn=k2l8zgqwstuxxevFX2qxVA==, picEn=r6k0Ry+RD+c/VXj5QarmDw==, jcr=null, cjcr=null, exts=[JournalExt(id=1325069127710286688, language=CN, name=计算机研究与发展, nameHistory1=null, nameHistory2=null, managedBy=, sponsoredBy=, publishedBy=, editorOffice=, officeProv=null, officeCity=null, officeAddr=, officeZip=, editDirector=, officeDirector=null, officePhone=null, coverPicUrl=null, journalRemark=, submitArticleUrl=null, websiteUrl=, createdTime=1793850698640, updatedTime=1793850698640, createdBy=13701087609, updatedBy=13701087609, submissionGuidelinesUrl=, submissionAuthorUrl=https://crad.ict.ac.cn/journalx/authorLogOn.action, submissionEditorUrl=https://crad.ict.ac.cn/journalx/editorLogOn.action, submissionReviewUrl=https://crad.ict.ac.cn/journalx/expertLogOn.action, submissionCeEditorUrl=https://crad.ict.ac.cn/journalx/editorInChiefLogOn.action, submissionAeEditorUrl=, option={"copyright":""}), JournalExt(id=1325069127777395553, language=EN, name=Journal of Computer Research and Development, nameHistory1=null, nameHistory2=null, managedBy=, sponsoredBy=, publishedBy=, editorOffice=, officeProv=null, officeCity=null, officeAddr=, officeZip=, editDirector=, officeDirector=null, officePhone=null, coverPicUrl=null, journalRemark=, submitArticleUrl=null, websiteUrl=, createdTime=1793850698656, updatedTime=1793850698656, createdBy=13701087609, updatedBy=13701087609, submissionGuidelinesUrl=, submissionAuthorUrl=https://crad.ict.ac.cn/journalx/authorLogOn.action, submissionEditorUrl=https://crad.ict.ac.cn/journalx/editorLogOn.action, submissionReviewUrl=https://crad.ict.ac.cn/journalx/expertLogOn.action, submissionCeEditorUrl=https://crad.ict.ac.cn/journalx/editorInChiefLogOn.action, submissionAeEditorUrl=, option={"copyright":""})], databaseList=null, tenantJournalId=1324759914146529478, websiteList=[Website(id=1325070300139892771, webName=null, webTitle=null, webDomain=null, webCopyrigh=null, webIpcNo=null, seoTitle=null, seoKeywords=null, seoDescription=null, tenantJournalId=null, journalId=1324759914146529478, journalNameCn=null, journalNameEn=null, grayFlag=null, tenantId=1146029695717560320, platformId=null, journalGroupId=null, journalGroupNameCn=null, journalGroupNameEn=null, type=1, domain=https://castjournals.cast.org.cn/joweb/jsjyjyfz/CN, language=CN, createTime=1793850978169, createBy=18614031015, updateTime=1793851040259, updateBy=18614031015, name=计算机研究与发展-中文, tplId=1146099689490845704, title=计算机研究与发展, delFlag=0, indexPage=/home, props=[WebsiteProps(id=1325070893751349360, tenantId=1146029695717560320, journalId=null, journalGroupId=null, siteId=1325070300139892771, code=articleTextType, value=kx, createTime=1793851119697, updateTime=1793851119697, creator=18614031015, updator=18614031015), WebsiteProps(id=1325070893730377837, tenantId=1146029695717560320, journalId=null, journalGroupId=null, siteId=1325070300139892771, code=banner, value=null, createTime=1793851119692, updateTime=1793851119692, creator=18614031015, updator=18614031015), WebsiteProps(id=1325070893772320883, tenantId=1146029695717560320, journalId=null, journalGroupId=null, siteId=1325070300139892771, code=grayFlag, value=0, createTime=1793851119702, updateTime=1793851119702, creator=18614031015, updator=18614031015), WebsiteProps(id=1325070893721989228, tenantId=1146029695717560320, journalId=null, journalGroupId=null, siteId=1325070300139892771, code=logo, value=https://castjournals.cast.org.cn/joweb/jsjyjyfz/CN/file/pic?fileId=U27tDsy717YF7XSPA+Vwdw==, createTime=1793851119690, updateTime=1793851119690, creator=18614031015, updator=18614031015), WebsiteProps(id=1325070893784903797, tenantId=1146029695717560320, journalId=null, journalGroupId=null, siteId=1325070300139892771, code=minRunFlag, value=0, createTime=1793851119705, updateTime=1793851119705, creator=18614031015, updator=18614031015), WebsiteProps(id=1325070893742960751, tenantId=1146029695717560320, journalId=null, journalGroupId=null, siteId=1325070300139892771, code=picServerUrl, value=https://castjournals.cast.org.cn/joweb/jsjyjyfz/CN/file/pic, createTime=1793851119695, updateTime=1793851119695, creator=18614031015, updator=18614031015), WebsiteProps(id=1325070893780709492, tenantId=1146029695717560320, journalId=null, journalGroupId=null, siteId=1325070300139892771, code=silenceFlag, value=0, createTime=1793851119704, updateTime=1793851119704, creator=18614031015, updator=18614031015), WebsiteProps(id=1325070893738766446, tenantId=1146029695717560320, journalId=null, journalGroupId=null, siteId=1325070300139892771, code=staticResourcePath, value=https://castjournals.cast.org.cn/joweb/cast_kjdb_cn_619/, createTime=1793851119694, updateTime=1793851119694, creator=18614031015, updator=18614031015), WebsiteProps(id=1325070893755543665, tenantId=1146029695717560320, journalId=null, journalGroupId=null, siteId=1325070300139892771, code=themeColor, value=null, createTime=1793851119698, updateTime=1793851119698, creator=18614031015, updator=18614031015), WebsiteProps(id=1325070893763932274, tenantId=1146029695717560320, journalId=null, journalGroupId=null, siteId=1325070300139892771, code=themeStyle, value=null, createTime=1793851119700, updateTime=1793851119700, creator=18614031015, updator=18614031015)]), Website(id=1325070300580294698, webName=null, webTitle=null, webDomain=null, webCopyrigh=null, webIpcNo=null, seoTitle=null, seoKeywords=null, seoDescription=null, tenantJournalId=null, journalId=1324759914146529478, journalNameCn=null, journalNameEn=null, grayFlag=null, tenantId=1146029695717560320, platformId=null, journalGroupId=null, journalGroupNameCn=null, journalGroupNameEn=null, type=1, domain=https://castjournals.cast.org.cn/joweb/jsjyjyfz/EN, language=EN, createTime=1793850978274, createBy=18614031015, updateTime=1793851036638, updateBy=18614031015, name=计算机研究与发展-英文, tplId=1146101810881728533, title=Journal of Computer Research and Development, delFlag=0, indexPage=/home, props=[WebsiteProps(id=1325070925288321148, tenantId=1146029695717560320, journalId=null, journalGroupId=null, siteId=1325070300580294698, code=articleTextType, value=kx, createTime=1793851127216, updateTime=1793851127216, creator=18614031015, updator=18614031015), WebsiteProps(id=1325070925263155321, tenantId=1146029695717560320, journalId=null, journalGroupId=null, siteId=1325070300580294698, code=banner, value=null, createTime=1793851127210, updateTime=1793851127210, creator=18614031015, updator=18614031015), WebsiteProps(id=1325070925309292671, tenantId=1146029695717560320, journalId=null, journalGroupId=null, siteId=1325070300580294698, code=grayFlag, value=0, createTime=1793851127221, updateTime=1793851127221, creator=18614031015, updator=18614031015), WebsiteProps(id=1325070925254766712, tenantId=1146029695717560320, journalId=null, journalGroupId=null, siteId=1325070300580294698, code=logo, value=https://castjournals.cast.org.cn/joweb/jsjyjyfz/EN/file/pic?fileId=U27tDsy717YF7XSPA+Vwdw==, createTime=1793851127208, updateTime=1793851127208, creator=18614031015, updator=18614031015), WebsiteProps(id=1325070925326069889, tenantId=1146029695717560320, journalId=null, journalGroupId=null, siteId=1325070300580294698, code=minRunFlag, value=0, createTime=1793851127225, updateTime=1793851127225, creator=18614031015, updator=18614031015), WebsiteProps(id=1325070925279932539, tenantId=1146029695717560320, journalId=null, journalGroupId=null, siteId=1325070300580294698, code=picServerUrl, value=https://castjournals.cast.org.cn/joweb/jsjyjyfz/EN/file/pic, createTime=1793851127214, updateTime=1793851127214, creator=18614031015, updator=18614031015), WebsiteProps(id=1325070925317681280, tenantId=1146029695717560320, journalId=null, journalGroupId=null, siteId=1325070300580294698, code=silenceFlag, value=0, createTime=1793851127223, updateTime=1793851127223, creator=18614031015, updator=18614031015), WebsiteProps(id=1325070925271543930, tenantId=1146029695717560320, journalId=null, journalGroupId=null, siteId=1325070300580294698, code=staticResourcePath, value=https://castjournals.cast.org.cn/joweb/cast_kjdb_en_623/, createTime=1793851127212, updateTime=1793851127212, creator=18614031015, updator=18614031015), WebsiteProps(id=1325070925292515453, tenantId=1146029695717560320, journalId=null, journalGroupId=null, siteId=1325070300580294698, code=themeColor, value=null, createTime=1793851127217, updateTime=1793851127217, creator=18614031015, updator=18614031015), WebsiteProps(id=1325070925300904062, tenantId=1146029695717560320, journalId=null, journalGroupId=null, siteId=1325070300580294698, code=themeStyle, value=null, createTime=1793851127219, updateTime=1793851127219, creator=18614031015, updator=18614031015)])], journalTitle=计算机研究与发展, weixinUrl=null, journalUrl=https://crad.ict.ac.cn/, iacademicId=null, status=1, seqNo=null, journalTitleEn=Journal of Computer Research and Development, journalPhotoCn=k2l8zgqwstuxxevFX2qxVA==, journalPhotoEn=r6k0Ry+RD+c/VXj5QarmDw==, journalFirstLetter=J, journalRecommend=null, journalNew=null, journalCollection=null, jcrJf=null, cjcrJf=null, jcrJfStr=null, cjcrJfStr=null, submissionFirstDecision=null, sciSubjectClassification=null, casSubjectClassification=null, citeScore=null, totalCitationFrequency=null, icpCode=null, psCode=null, advertisingLicenseCode=null, copyrightInformation=null, country=null, option=, provinceCode=null, provinceName=null, collectFlag=false, interPubPlatform=, interPubPlatformUrl=null), detailUrlCn=https://castjournals.cast.org.cn/joweb/jsjyjyfz/CN/10.7544/issn1000-1239.202440560, detailUrlEn=https://castjournals.cast.org.cn/joweb/jsjyjyfz/EN/10.7544/issn1000-1239.202440560, pdfUrlCn=https://castjournals.cast.org.cn/joweb/jsjyjyfz/CN/PDF/10.7544/issn1000-1239.202440560, pdfUrlEn=https://castjournals.cast.org.cn/joweb/jsjyjyfz/EN/PDF/10.7544/issn1000-1239.202440560, aliStartDate=0, aliEndDate=0, collectionFlag=false, citedCount=null, citedUrl=null, previewStatus=0, delFlag=0, hasFullText=1, orderTime=1764518400000, fullTextJson=null, articleText=null, reference=null)