Article(id=1327012989124849767, tenantId=1146029695717560320, journalId=1324759914146529478, issueId=1327012646391493336, articleNumber=null, orderNo=null, doi=10.7544/issn1000-1239.202550147, pmid=null, cstr=null, oa=null, hot=null, price=null, onlineType=0, articleFormat=0, articleType=null, articleTypeStr=research-article, receivedDate=1740758400000, receivedDateStr=2025-03-01, revisedDate=1744214400000, revisedDateStr=2025-04-10, acceptedDate=null, acceptedDateStr=null, onlineDate=1794314151315, onlineDateStr=2026-11-10, pubDate=1748793600000, pubDateStr=2025-06-02, doiRegisterDate=null, doiRegisterDateStr=null, onlineIssueDate=1794314151315, onlineIssueDateStr=2026-11-10, onlineJustAcceptDate=null, onlineJustAcceptDateStr=null, onlineFirstDate=null, onlineFirstDateStr=null, sourceXml=null, magXml=null, createTime=1794314151315, creator=13701087609, updateTime=1794314151315, updator=13701087609, issue=Issue{id=1327012646391493336, tenantId=1146029695717560320, journalId=1324759914146529478, year='2025', volume='62', issue='6', pageStart='1331', pageEnd='1593', issueExtLink='null', onlineDate='null', pubDate='1748793600000', pubDateStr='2025-06-02', beforeIssueId=null, nextIssueId=null, price=null, status=1, issueComplete=1, articleOrder=1, issueType=-1, specialIssue=null, createTime=1794314069601, creator='13701087609', updateTime=1794376817187, updator='13701087609', preIssue=null, nextIssue=null, articleTotal=null, ext={EN=IssueExt(id=1327275829270262152, tenantId=1146029695717560320, journalId=1324759914146529478, issueId=1327012646391493336, language=EN, specialIssueTitle=, coverIllustrator=null, specialIssueEditor=, specialIssueAbout=), CN=IssueExt(id=1327275829270262153, tenantId=1146029695717560320, journalId=1324759914146529478, issueId=1327012646391493336, language=CN, specialIssueTitle=, coverIllustrator=null, specialIssueEditor=, specialIssueAbout=)}, issueFiles=null, downloadFileDto=null}, startPage=1380, endPage=1395, ext={EN=ArticleExt(id=1327012990920011881, articleId=1327012989124849767, tenantId=1146029695717560320, journalId=1324759914146529478, language=EN, title=Resilio: An Elastic Fault-tolerant Training System for Large Language Models, columnId=1327012740067078573, journalTitle=Journal of Computer Research and Development, columnName=High Performance Computing System, runingTitle=null, highlight=null, articleAbstract=
Large language models with hundreds of billions of parameters are driving rapid technology innovations and business model transformations in artificial intelligence and heterogeneous computing. However, training such models requires prolonged occupation of extensive hardware resources, thus often incurring diverse high-frequency software/hardware failures. These failures not only are challenging to diagnose, but also lead to much longer training time due to unwanted computation waste and slow training convergence. Resilio, an elastic fault-tolerant system for training large language models is proposed, to provide an efficient automated fault recovery mechanism. It is designed to target multiple typical failure scenarios during training processes, such as network interruptions, node crashes, and process failures. Leveraging the characteristics of the parallel model training strategies and underlying hierarchical storage architectures, Resilio implements multi-layer optimizations on checkpoint read/write operations and Just-In-Time (JIT) recovery mechanisms. For models with 100 billion scale parameters, Resilio reduces the end-to-end recovery time under 10 minutes, while reducing the re-started computation after interruptions to the cost of a single training iteration. Upon variations of the computation resources, Resilio can quickly identify the cluster configurations to enable optimal parallel training strategies. Combined with the fault-tolerant scheduling capability, the system ensures adaptive and elastic resource allocations to greatly improve training efficiency and boost GPU utilization across large-scale computing clusters.
, authors=Yan Li
1, Sile Yang
1, Chengchun Liu
1, Linmei Wang
1, Yaolin Tian
1, Xinhang Zhang
1, Yu Zhu
2, Chunpu Li
1, Lei Sun
1, Shengen Yan
2, Limin Xiao
1, *, Weifeng Zhang
1, authorsList=Yan Li, Sile Yang, Chengchun Liu, Linmei Wang, Yaolin Tian, Xinhang Zhang, Yu Zhu, Chunpu Li, Lei Sun, Shengen Yan, Limin Xiao, Weifeng Zhang, authorCompany=null, correspAuthors=Limin Xiao, authorNote=
born in 1985. PhD, senior engineer. Member of CCF. His main research interests include high performance computing, heterogeneous computing, and deep learning
born in 1994. Master. His main research interests include heterogenous computing and network communication
born in 1992. Master. His main research interests include high performance computing and heterogeneous computing
born in 1985. Master. Member of CCF. Her main research interests include high performance computing and heterogeneous computing
born in 1999. Master. Her main research interests include optimization of LLM, heterogeneous computing, and visual SLAM
born in 2000. Master. His main research interest includes distributed training of deep learning
born in 1998. PhD candidate. His main research interests include LLM training architecture design and multi-core architecture
born in 1978. Bachelor. Her main research interests include heterogeneous computing and cloud computing
born in 1980. Bachelor. His main research interests include heterogeneous computing and edge computing
born in 1986. PhD, associate professor. His main research interests include heterogeneous computing, intelligent computing, and machine learning systems
born in 1970. PhD. Distinguished member of CCF. Distinguished researcher at Lenovo Research. His main research interests include computer architecture, heterogeneous intelligent computing, high performance computing, and intelligent computing chip
born in 1968. PhD. Corporate VP at Lenovo Group. His main research interests include computer architecture, heterogeneous intelligent computing, wireless communication, and intelligent computing chip
, correspAuthorsNote=null, copyrightStatement=Copyright © 2025 Journal of Computer Research and Development. All rights reserved., copyrightOwner=null, extLink=null, articleAbsUrl=null, sourceXml=null, magXml=null, pdfUrl=null, pdf=null, pdfFileSize=null, pdfExtLink=null, richHtmlUrl=null, mobilePdfUrl=null, reviewReport=null, pdfFirstPage=null, abstractGraph=null, abstractGraphContent=null, abstractVideo=null, citation=null, cebUrl=null, magXmlContent=null, mapNumber=null, fund=null), CN=ArticleExt(id=1327012996720734364, articleId=1327012989124849767, tenantId=1146029695717560320, journalId=1324759914146529478, language=CN, title=Resilio:一种大模型弹性训练容错系统, columnId=1327012740230656431, journalTitle=计算机研究与发展, columnName=高性能计算系统, runingTitle=null, highlight=null, articleAbstract=
具备千亿级参数的大型语言模型正在引领当今人工智能与异构计算的技术革新及商业模式的深刻转变. 然而,大模型训练任务需要长时间占用大量的硬件资源,软硬件故障发生的频率高且类型较多,并且故障原因难定位导致训练中断时间较长. 针对大模型训练过程中面临的网络中断、节点宕机、进程崩溃等多种典型故障, 提出一种大模型弹性容错系统Resilio来提供高效自动的恢复机制. 基于模型训练的并行策略与硬件的存储层次特点,Resilio通过多层次优化检查点读写操作和即时检查点保存机制,对于千亿规模参数模型,可以将端到端故障恢复时间缩短至10 min以内,模型中断后的重新训练时间缩短至单次训练迭代时间. 当集群资源弹性变化时,Resilio能够快速准确地获取大模型训练最优并行策略配置,与容错调度组件共同确保系统的自适应能力,弹性调度训练资源用以提升作业的训练效率和集群GPU资源利用率.
, authors=李焱
1, 杨偲乐
1, 刘成春
1, 王林梅
1, 田瑶琳
1, 张信航
1, 朱昱
2, 李莼蒲
1, 孙磊
1, 颜深根
2, 肖利民
1, *, 张伟丰
1, authorsList=李焱, 杨偲乐, 刘成春, 王林梅, 田瑶琳, 张信航, 朱昱, 李莼蒲, 孙磊, 颜深根, 肖利民, 张伟丰, authorCompany=null, correspAuthors=肖利民, authorNote=
 |
1985年生. 博士,高级工程师. CCF会员. 主要研究方向为高性能计算、异构计算、深度学习 |
 |
1994年生. 硕士. 主要研究方向为异构计算、网络通信 |
 |
1992年生. 硕士. 主要研究方向为高性能计算、异构计算 |
 |
1985年生. 硕士. CCF会员. 主要研究方向为高性能计算、异构计算 |
 |
1999年生. 硕士. 主要研究方向为大语言模型优化、异构计算、视觉SLAM |
 |
2000年生. 硕士. 主要研究方向为深度学习分布式训练 |
 |
1998年生. 博士研究生. 主要研究方向为大模型训练架构设计、多芯粒架构 |
 |
1978年生. 学士. 主要研究方向为异构计算、云计算 |
 |
1980年生. 学士. 主要研究方向为异构计算、边缘计算 |
 |
1986年生. 博士,副教授. 主要研究方向为异构计算、智能计算、机器学习系统 |
 |
1970年生. 博士. CCF杰出会员. 联想研究院首席研究员. 主要研究方向为计算机体系结构、异构智能计算、高性能计算、智能计算芯片 |
 |
1968年生. 博士. 联想集团副总裁. 主要研究方向为计算机体系结构、异构智能计算、无线通信、智能计算芯片 |
, correspAuthorsNote=
, copyrightStatement=版权所有 © 2025 计算机研究与发展编辑部, copyrightOwner=null, extLink=null, articleAbsUrl=null, sourceXml=aGfkEPo1tZWurjyt3McbNA==, magXml=OQsF8Q/rZ6AJ7RuxPcKwsA==, pdfUrl=null, pdf=tyJPb71zdCsQRX/Enf4ZaA==, pdfFileSize=2795779, pdfExtLink=null, richHtmlUrl=null, mobilePdfUrl=null, reviewReport=null, pdfFirstPage=null, abstractGraph=8lp5NmwOz01/TjlFyejXZQ==, abstractGraphContent=null, abstractVideo=null, citation=null, cebUrl=null, magXmlContent=RgM3Bj6BbawRhSXnH405aQ==, mapNumber=null, fund=null)}, authors=[Author(id=1327012997450543268, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012989124849767, orderNo=0, firstName=null, middleName=null, lastName=null, nameCn=null, orcid=null, stid=null, country=null, authorPic=null, dead=0, email=liyan665@gmail.com, emailSecond=null, emailThird=null, correspondingAuthor=0, authorType=1, ext={EN=AuthorExt(id=1327012997567983782, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012989124849767, authorId=1327012997450543268, language=EN, stringName=Yan Li, firstName=Yan, middleName=null, lastName=Li, prefix=null, suffix=null, authorComment=null, nameInitials=null, affiliation=null, department=null, xref=
1, address=
1Lenovo Research, Beijing 100094, bio={"content":"
born in 1985. PhD, senior engineer. Member of CCF. His main research interests include high performance computing, heterogeneous computing, and deep learning
"}, bioImg=null, bioContent=
born in 1985. PhD, senior engineer. Member of CCF. His main research interests include high performance computing, heterogeneous computing, and deep learning
, aboutCorrespAuthor=null), CN=AuthorExt(id=1327012997664452775, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012989124849767, authorId=1327012997450543268, language=CN, stringName=李焱, firstName=null, middleName=null, lastName=null, prefix=null, suffix=null, authorComment=null, nameInitials=null, affiliation=null, department=null, xref=
1, address=
1联想研究院 北京 100094, bio={"img":"hH0EXDQqNlklVlMhScnXuA==","content":"
1985年生. 博士,高级工程师. CCF会员. 主要研究方向为高性能计算、异构计算、深度学习
"}, bioImg=hH0EXDQqNlklVlMhScnXuA==, bioContent=
1985年生. 博士,高级工程师. CCF会员. 主要研究方向为高性能计算、异构计算、深度学习
, aboutCorrespAuthor=null)}, companyList=[AuthorCompany(id=1327012997068861597, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012989124849767, xref=1, ext=[AuthorCompanyExt(id=1327012997081444510, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012989124849767, companyId=1327012997068861597, language=EN, country=null, province=null, city=null, postcode=null, companyName=null, departmentName=null, remark=
1Lenovo Research, Beijing 100094), AuthorCompanyExt(id=1327012997094027423, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012989124849767, companyId=1327012997068861597, language=CN, country=null, province=null, city=null, postcode=null, companyName=null, departmentName=null, remark=
1联想研究院 北京 100094)])]), Author(id=1327012997735755945, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012989124849767, orderNo=1, firstName=null, middleName=null, lastName=null, nameCn=null, orcid=null, stid=null, country=null, authorPic=null, dead=0, email=null, emailSecond=null, emailThird=null, correspondingAuthor=0, authorType=1, ext={EN=AuthorExt(id=1327012997828030635, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012989124849767, authorId=1327012997735755945, language=EN, stringName=Sile Yang, firstName=Sile, middleName=null, lastName=Yang, prefix=null, suffix=null, authorComment=null, nameInitials=null, affiliation=null, department=null, xref=
1, address=
1Lenovo Research, Beijing 100094, bio={"content":"
born in 1994. Master. His main research interests include heterogenous computing and network communication
"}, bioImg=null, bioContent=
born in 1994. Master. His main research interests include heterogenous computing and network communication
, aboutCorrespAuthor=null), CN=AuthorExt(id=1327012997916111020, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012989124849767, authorId=1327012997735755945, language=CN, stringName=杨偲乐, firstName=null, middleName=null, lastName=null, prefix=null, suffix=null, authorComment=null, nameInitials=null, affiliation=null, department=null, xref=
1, address=
1联想研究院 北京 100094, bio={"img":"p+8MheCcYXAvG6xM1TbNlQ==","content":"
1994年生. 硕士. 主要研究方向为异构计算、网络通信
"}, bioImg=p+8MheCcYXAvG6xM1TbNlQ==, bioContent=
1994年生. 硕士. 主要研究方向为异构计算、网络通信
, aboutCorrespAuthor=null)}, companyList=[AuthorCompany(id=1327012997068861597, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012989124849767, xref=1, ext=[AuthorCompanyExt(id=1327012997081444510, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012989124849767, companyId=1327012997068861597, language=EN, country=null, province=null, city=null, postcode=null, companyName=null, departmentName=null, remark=
1Lenovo Research, Beijing 100094), AuthorCompanyExt(id=1327012997094027423, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012989124849767, companyId=1327012997068861597, language=CN, country=null, province=null, city=null, postcode=null, companyName=null, departmentName=null, remark=
1联想研究院 北京 100094)])]), Author(id=1327012997995802798, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012989124849767, orderNo=2, firstName=null, middleName=null, lastName=null, nameCn=null, orcid=null, stid=null, country=null, authorPic=null, dead=0, email=null, emailSecond=null, emailThird=null, correspondingAuthor=0, authorType=1, ext={EN=AuthorExt(id=1327012999694495920, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012989124849767, authorId=1327012997995802798, language=EN, stringName=Chengchun Liu, firstName=Chengchun, middleName=null, lastName=Liu, prefix=null, suffix=null, authorComment=null, nameInitials=null, affiliation=null, department=null, xref=
1, address=
1Lenovo Research, Beijing 100094, bio={"content":"
born in 1992. Master. His main research interests include high performance computing and heterogeneous computing
"}, bioImg=null, bioContent=
born in 1992. Master. His main research interests include high performance computing and heterogeneous computing
, aboutCorrespAuthor=null), CN=AuthorExt(id=1327012999778382001, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012989124849767, authorId=1327012997995802798, language=CN, stringName=刘成春, firstName=null, middleName=null, lastName=null, prefix=null, suffix=null, authorComment=null, nameInitials=null, affiliation=null, department=null, xref=
1, address=
1联想研究院 北京 100094, bio={"img":"jds2vAITuY0k3Bt9Jkyjaw==","content":"
1992年生. 硕士. 主要研究方向为高性能计算、异构计算
"}, bioImg=jds2vAITuY0k3Bt9Jkyjaw==, bioContent=
1992年生. 硕士. 主要研究方向为高性能计算、异构计算
, aboutCorrespAuthor=null)}, companyList=[AuthorCompany(id=1327012997068861597, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012989124849767, xref=1, ext=[AuthorCompanyExt(id=1327012997081444510, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012989124849767, companyId=1327012997068861597, language=EN, country=null, province=null, city=null, postcode=null, companyName=null, departmentName=null, remark=
1Lenovo Research, Beijing 100094), AuthorCompanyExt(id=1327012997094027423, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012989124849767, companyId=1327012997068861597, language=CN, country=null, province=null, city=null, postcode=null, companyName=null, departmentName=null, remark=
1联想研究院 北京 100094)])]), Author(id=1327012999883239603, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012989124849767, orderNo=3, firstName=null, middleName=null, lastName=null, nameCn=null, orcid=null, stid=null, country=null, authorPic=null, dead=0, email=null, emailSecond=null, emailThird=null, correspondingAuthor=0, authorType=1, ext={EN=AuthorExt(id=1327012999988097205, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012989124849767, authorId=1327012999883239603, language=EN, stringName=Linmei Wang, firstName=Linmei, middleName=null, lastName=Wang, prefix=null, suffix=null, authorComment=null, nameInitials=null, affiliation=null, department=null, xref=
1, address=
1Lenovo Research, Beijing 100094, bio={"content":"
born in 1985. Master. Member of CCF. Her main research interests include high performance computing and heterogeneous computing
"}, bioImg=null, bioContent=
born in 1985. Master. Member of CCF. Her main research interests include high performance computing and heterogeneous computing
, aboutCorrespAuthor=null), CN=AuthorExt(id=1327013000076177590, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012989124849767, authorId=1327012999883239603, language=CN, stringName=王林梅, firstName=null, middleName=null, lastName=null, prefix=null, suffix=null, authorComment=null, nameInitials=null, affiliation=null, department=null, xref=
1, address=
1联想研究院 北京 100094, bio={"img":"qTtr16jtjmiBGgyoFNKkow==","content":"
1985年生. 硕士. CCF会员. 主要研究方向为高性能计算、异构计算
"}, bioImg=qTtr16jtjmiBGgyoFNKkow==, bioContent=
1985年生. 硕士. CCF会员. 主要研究方向为高性能计算、异构计算
, aboutCorrespAuthor=null)}, companyList=[AuthorCompany(id=1327012997068861597, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012989124849767, xref=1, ext=[AuthorCompanyExt(id=1327012997081444510, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012989124849767, companyId=1327012997068861597, language=EN, country=null, province=null, city=null, postcode=null, companyName=null, departmentName=null, remark=
1Lenovo Research, Beijing 100094), AuthorCompanyExt(id=1327012997094027423, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012989124849767, companyId=1327012997068861597, language=CN, country=null, province=null, city=null, postcode=null, companyName=null, departmentName=null, remark=
1联想研究院 北京 100094)])]), Author(id=1327013000160063672, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012989124849767, orderNo=4, firstName=null, middleName=null, lastName=null, nameCn=null, orcid=null, stid=null, country=null, authorPic=null, dead=0, email=null, emailSecond=null, emailThird=null, correspondingAuthor=0, authorType=1, ext={EN=AuthorExt(id=1327013000252338362, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012989124849767, authorId=1327013000160063672, language=EN, stringName=Yaolin Tian, firstName=Yaolin, middleName=null, lastName=Tian, prefix=null, suffix=null, authorComment=null, nameInitials=null, affiliation=null, department=null, xref=
1, address=
1Lenovo Research, Beijing 100094, bio={"content":"
born in 1999. Master. Her main research interests include optimization of LLM, heterogeneous computing, and visual SLAM
"}, bioImg=null, bioContent=
born in 1999. Master. Her main research interests include optimization of LLM, heterogeneous computing, and visual SLAM
, aboutCorrespAuthor=null), CN=AuthorExt(id=1327013000336224443, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012989124849767, authorId=1327013000160063672, language=CN, stringName=田瑶琳, firstName=null, middleName=null, lastName=null, prefix=null, suffix=null, authorComment=null, nameInitials=null, affiliation=null, department=null, xref=
1, address=
1联想研究院 北京 100094, bio={"img":"S0DI6T5eaZ1/NIHgIjTrSw==","content":"
1999年生. 硕士. 主要研究方向为大语言模型优化、异构计算、视觉SLAM
"}, bioImg=S0DI6T5eaZ1/NIHgIjTrSw==, bioContent=
1999年生. 硕士. 主要研究方向为大语言模型优化、异构计算、视觉SLAM
, aboutCorrespAuthor=null)}, companyList=[AuthorCompany(id=1327012997068861597, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012989124849767, xref=1, ext=[AuthorCompanyExt(id=1327012997081444510, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012989124849767, companyId=1327012997068861597, language=EN, country=null, province=null, city=null, postcode=null, companyName=null, departmentName=null, remark=
1Lenovo Research, Beijing 100094), AuthorCompanyExt(id=1327012997094027423, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012989124849767, companyId=1327012997068861597, language=CN, country=null, province=null, city=null, postcode=null, companyName=null, departmentName=null, remark=
1联想研究院 北京 100094)])]), Author(id=1327013000411721917, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012989124849767, orderNo=5, firstName=null, middleName=null, lastName=null, nameCn=null, orcid=null, stid=null, country=null, authorPic=null, dead=0, email=null, emailSecond=null, emailThird=null, correspondingAuthor=0, authorType=1, ext={EN=AuthorExt(id=1327013000516579519, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012989124849767, authorId=1327013000411721917, language=EN, stringName=Xinhang Zhang, firstName=Xinhang, middleName=null, lastName=Zhang, prefix=null, suffix=null, authorComment=null, nameInitials=null, affiliation=null, department=null, xref=
1, address=
1Lenovo Research, Beijing 100094, bio={"content":"
born in 2000. Master. His main research interest includes distributed training of deep learning
"}, bioImg=null, bioContent=
born in 2000. Master. His main research interest includes distributed training of deep learning
, aboutCorrespAuthor=null), CN=AuthorExt(id=1327013000613048512, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012989124849767, authorId=1327013000411721917, language=CN, stringName=张信航, firstName=null, middleName=null, lastName=null, prefix=null, suffix=null, authorComment=null, nameInitials=null, affiliation=null, department=null, xref=
1, address=
1联想研究院 北京 100094, bio={"img":"j8gu/ly1AKib3A4RBZjkuQ==","content":"
2000年生. 硕士. 主要研究方向为深度学习分布式训练
"}, bioImg=j8gu/ly1AKib3A4RBZjkuQ==, bioContent=
2000年生. 硕士. 主要研究方向为深度学习分布式训练
, aboutCorrespAuthor=null)}, companyList=[AuthorCompany(id=1327012997068861597, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012989124849767, xref=1, ext=[AuthorCompanyExt(id=1327012997081444510, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012989124849767, companyId=1327012997068861597, language=EN, country=null, province=null, city=null, postcode=null, companyName=null, departmentName=null, remark=
1Lenovo Research, Beijing 100094), AuthorCompanyExt(id=1327012997094027423, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012989124849767, companyId=1327012997068861597, language=CN, country=null, province=null, city=null, postcode=null, companyName=null, departmentName=null, remark=
1联想研究院 北京 100094)])]), Author(id=1327013000705323202, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012989124849767, orderNo=6, firstName=null, middleName=null, lastName=null, nameCn=null, orcid=null, stid=null, country=null, authorPic=null, dead=0, email=null, emailSecond=null, emailThird=null, correspondingAuthor=0, authorType=1, ext={EN=AuthorExt(id=1327013000805986500, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012989124849767, authorId=1327013000705323202, language=EN, stringName=Yu Zhu, firstName=Yu, middleName=null, lastName=Zhu, prefix=null, suffix=null, authorComment=null, nameInitials=null, affiliation=null, department=null, xref=
2, address=
2Nanoscale Integrated Circuits and Systems Lab (Tsinghua University), Beijing 100084, bio={"content":"
born in 1998. PhD candidate. His main research interests include LLM training architecture design and multi-core architecture
"}, bioImg=null, bioContent=
born in 1998. PhD candidate. His main research interests include LLM training architecture design and multi-core architecture
, aboutCorrespAuthor=null), CN=AuthorExt(id=1327013000898261189, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012989124849767, authorId=1327013000705323202, language=CN, stringName=朱昱, firstName=null, middleName=null, lastName=null, prefix=null, suffix=null, authorComment=null, nameInitials=null, affiliation=null, department=null, xref=
2, address=
2纳米集成电路与系统实验室(清华大学) 北京 100084, bio={"img":"4DFW+LBwukVPKOyPcOShLg==","content":"
1998年生. 博士研究生. 主要研究方向为大模型训练架构设计、多芯粒架构
"}, bioImg=4DFW+LBwukVPKOyPcOShLg==, bioContent=
1998年生. 博士研究生. 主要研究方向为大模型训练架构设计、多芯粒架构
, aboutCorrespAuthor=null)}, companyList=[AuthorCompany(id=1327012997215662240, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012989124849767, xref=2, ext=[AuthorCompanyExt(id=1327012997228245153, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012989124849767, companyId=1327012997215662240, language=EN, country=null, province=null, city=null, postcode=null, companyName=null, departmentName=null, remark=
2Nanoscale Integrated Circuits and Systems Lab (Tsinghua University), Beijing 100084), AuthorCompanyExt(id=1327012997240828066, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012989124849767, companyId=1327012997215662240, language=CN, country=null, province=null, city=null, postcode=null, companyName=null, departmentName=null, remark=
2纳米集成电路与系统实验室(清华大学) 北京 100084)])]), Author(id=1327013000986341575, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012989124849767, orderNo=7, firstName=null, middleName=null, lastName=null, nameCn=null, orcid=null, stid=null, country=null, authorPic=null, dead=0, email=null, emailSecond=null, emailThird=null, correspondingAuthor=0, authorType=1, ext={EN=AuthorExt(id=1327013001070227657, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012989124849767, authorId=1327013000986341575, language=EN, stringName=Chunpu Li, firstName=Chunpu, middleName=null, lastName=Li, prefix=null, suffix=null, authorComment=null, nameInitials=null, affiliation=null, department=null, xref=
1, address=
1Lenovo Research, Beijing 100094, bio={"content":"
born in 1978. Bachelor. Her main research interests include heterogeneous computing and cloud computing
"}, bioImg=null, bioContent=
born in 1978. Bachelor. Her main research interests include heterogeneous computing and cloud computing
, aboutCorrespAuthor=null), CN=AuthorExt(id=1327013001141530826, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012989124849767, authorId=1327013000986341575, language=CN, stringName=李莼蒲, firstName=null, middleName=null, lastName=null, prefix=null, suffix=null, authorComment=null, nameInitials=null, affiliation=null, department=null, xref=
1, address=
1联想研究院 北京 100094, bio={"img":"BU+JfCFCHJiNWjYbLwkUBg==","content":"
1978年生. 学士. 主要研究方向为异构计算、云计算
"}, bioImg=BU+JfCFCHJiNWjYbLwkUBg==, bioContent=
1978年生. 学士. 主要研究方向为异构计算、云计算
, aboutCorrespAuthor=null)}, companyList=[AuthorCompany(id=1327012997068861597, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012989124849767, xref=1, ext=[AuthorCompanyExt(id=1327012997081444510, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012989124849767, companyId=1327012997068861597, language=EN, country=null, province=null, city=null, postcode=null, companyName=null, departmentName=null, remark=
1Lenovo Research, Beijing 100094), AuthorCompanyExt(id=1327012997094027423, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012989124849767, companyId=1327012997068861597, language=CN, country=null, province=null, city=null, postcode=null, companyName=null, departmentName=null, remark=
1联想研究院 北京 100094)])]), Author(id=1327013001246388428, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012989124849767, orderNo=8, firstName=null, middleName=null, lastName=null, nameCn=null, orcid=null, stid=null, country=null, authorPic=null, dead=0, email=null, emailSecond=null, emailThird=null, correspondingAuthor=0, authorType=1, ext={EN=AuthorExt(id=1327013001414160590, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012989124849767, authorId=1327013001246388428, language=EN, stringName=Lei Sun, firstName=Lei, middleName=null, lastName=Sun, prefix=null, suffix=null, authorComment=null, nameInitials=null, affiliation=null, department=null, xref=
1, address=
1Lenovo Research, Beijing 100094, bio={"content":"
born in 1980. Bachelor. His main research interests include heterogeneous computing and edge computing
"}, bioImg=null, bioContent=
born in 1980. Bachelor. His main research interests include heterogeneous computing and edge computing
, aboutCorrespAuthor=null), CN=AuthorExt(id=1327013001607098575, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012989124849767, authorId=1327013001246388428, language=CN, stringName=孙磊, firstName=null, middleName=null, lastName=null, prefix=null, suffix=null, authorComment=null, nameInitials=null, affiliation=null, department=null, xref=
1, address=
1联想研究院 北京 100094, bio={"img":"JTzoMzG5XbjgvmgSSyKCWA==","content":"
1980年生. 学士. 主要研究方向为异构计算、边缘计算
"}, bioImg=JTzoMzG5XbjgvmgSSyKCWA==, bioContent=
1980年生. 学士. 主要研究方向为异构计算、边缘计算
, aboutCorrespAuthor=null)}, companyList=[AuthorCompany(id=1327012997068861597, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012989124849767, xref=1, ext=[AuthorCompanyExt(id=1327012997081444510, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012989124849767, companyId=1327012997068861597, language=EN, country=null, province=null, city=null, postcode=null, companyName=null, departmentName=null, remark=
1Lenovo Research, Beijing 100094), AuthorCompanyExt(id=1327012997094027423, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012989124849767, companyId=1327012997068861597, language=CN, country=null, province=null, city=null, postcode=null, companyName=null, departmentName=null, remark=
1联想研究院 北京 100094)])]), Author(id=1327013001791647953, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012989124849767, orderNo=9, firstName=null, middleName=null, lastName=null, nameCn=null, orcid=null, stid=null, country=null, authorPic=null, dead=0, email=null, emailSecond=null, emailThird=null, correspondingAuthor=0, authorType=1, ext={EN=AuthorExt(id=1327013001984585940, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012989124849767, authorId=1327013001791647953, language=EN, stringName=Shengen Yan, firstName=Shengen, middleName=null, lastName=Yan, prefix=null, suffix=null, authorComment=null, nameInitials=null, affiliation=null, department=null, xref=
2, address=
2Nanoscale Integrated Circuits and Systems Lab (Tsinghua University), Beijing 100084, bio={"content":"
born in 1986. PhD, associate professor. His main research interests include heterogeneous computing, intelligent computing, and machine learning systems
"}, bioImg=null, bioContent=
born in 1986. PhD, associate professor. His main research interests include heterogeneous computing, intelligent computing, and machine learning systems
, aboutCorrespAuthor=null), CN=AuthorExt(id=1327013002102026453, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012989124849767, authorId=1327013001791647953, language=CN, stringName=颜深根, firstName=null, middleName=null, lastName=null, prefix=null, suffix=null, authorComment=null, nameInitials=null, affiliation=null, department=null, xref=
2, address=
2纳米集成电路与系统实验室(清华大学) 北京 100084, bio={"img":"Mkw6alFd1Z+yfblnG4mC2A==","content":"
1986年生. 博士,副教授. 主要研究方向为异构计算、智能计算、机器学习系统
"}, bioImg=Mkw6alFd1Z+yfblnG4mC2A==, bioContent=
1986年生. 博士,副教授. 主要研究方向为异构计算、智能计算、机器学习系统
, aboutCorrespAuthor=null)}, companyList=[AuthorCompany(id=1327012997215662240, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012989124849767, xref=2, ext=[AuthorCompanyExt(id=1327012997228245153, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012989124849767, companyId=1327012997215662240, language=EN, country=null, province=null, city=null, postcode=null, companyName=null, departmentName=null, remark=
2Nanoscale Integrated Circuits and Systems Lab (Tsinghua University), Beijing 100084), AuthorCompanyExt(id=1327012997240828066, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012989124849767, companyId=1327012997215662240, language=CN, country=null, province=null, city=null, postcode=null, companyName=null, departmentName=null, remark=
2纳米集成电路与系统实验室(清华大学) 北京 100084)])]), Author(id=1327013002240438487, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012989124849767, orderNo=10, firstName=null, middleName=null, lastName=null, nameCn=null, orcid=null, stid=null, country=null, authorPic=null, dead=0, email=xiaolm@lenovo.com, emailSecond=null, emailThird=null, correspondingAuthor=1, authorType=1, ext={EN=AuthorExt(id=1327013002349490393, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012989124849767, authorId=1327013002240438487, language=EN, stringName=Limin Xiao, firstName=Limin, middleName=null, lastName=Xiao, prefix=null, suffix=null, authorComment=null, nameInitials=null, affiliation=null, department=null, xref=
1, *, address=
1Lenovo Research, Beijing 100094, bio={"content":"
born in 1970. PhD. Distinguished member of CCF. Distinguished researcher at Lenovo Research. His main research interests include computer architecture, heterogeneous intelligent computing, high performance computing, and intelligent computing chip
"}, bioImg=null, bioContent=
born in 1970. PhD. Distinguished member of CCF. Distinguished researcher at Lenovo Research. His main research interests include computer architecture, heterogeneous intelligent computing, high performance computing, and intelligent computing chip
, aboutCorrespAuthor=null), CN=AuthorExt(id=1327013002441765082, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012989124849767, authorId=1327013002240438487, language=CN, stringName=肖利民, firstName=null, middleName=null, lastName=null, prefix=null, suffix=null, authorComment=null, nameInitials=null, affiliation=null, department=null, xref=
1, *, address=
1联想研究院 北京 100094, bio={"img":"N5xUo7UOO7ejVDX0z9jdUQ==","content":"
1970年生. 博士. CCF杰出会员. 联想研究院首席研究员. 主要研究方向为计算机体系结构、异构智能计算、高性能计算、智能计算芯片
"}, bioImg=N5xUo7UOO7ejVDX0z9jdUQ==, bioContent=
1970年生. 博士. CCF杰出会员. 联想研究院首席研究员. 主要研究方向为计算机体系结构、异构智能计算、高性能计算、智能计算芯片
, aboutCorrespAuthor=null)}, companyList=[AuthorCompany(id=1327012997068861597, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012989124849767, xref=1, ext=[AuthorCompanyExt(id=1327012997081444510, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012989124849767, companyId=1327012997068861597, language=EN, country=null, province=null, city=null, postcode=null, companyName=null, departmentName=null, remark=
1Lenovo Research, Beijing 100094), AuthorCompanyExt(id=1327012997094027423, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012989124849767, companyId=1327012997068861597, language=CN, country=null, province=null, city=null, postcode=null, companyName=null, departmentName=null, remark=
1联想研究院 北京 100094)])]), Author(id=1327013004144652510, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012989124849767, orderNo=11, firstName=null, middleName=null, lastName=null, nameCn=null, orcid=null, stid=null, country=null, authorPic=null, dead=0, email=null, emailSecond=null, emailThird=null, correspondingAuthor=0, authorType=1, ext={EN=AuthorExt(id=1327013004257898721, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012989124849767, authorId=1327013004144652510, language=EN, stringName=Weifeng Zhang, firstName=Weifeng, middleName=null, lastName=Zhang, prefix=null, suffix=null, authorComment=null, nameInitials=null, affiliation=null, department=null, xref=
1, address=
1Lenovo Research, Beijing 100094, bio={"content":"
born in 1968. PhD. Corporate VP at Lenovo Group. His main research interests include computer architecture, heterogeneous intelligent computing, wireless communication, and intelligent computing chip
"}, bioImg=null, bioContent=
born in 1968. PhD. Corporate VP at Lenovo Group. His main research interests include computer architecture, heterogeneous intelligent computing, wireless communication, and intelligent computing chip
, aboutCorrespAuthor=null), CN=AuthorExt(id=1327013004396310754, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012989124849767, authorId=1327013004144652510, language=CN, stringName=张伟丰, firstName=null, middleName=null, lastName=null, prefix=null, suffix=null, authorComment=null, nameInitials=null, affiliation=null, department=null, xref=
1, address=
1联想研究院 北京 100094, bio={"img":"ZF7N3WPQTE/udhAaJucbpg==","content":"
1968年生. 博士. 联想集团副总裁. 主要研究方向为计算机体系结构、异构智能计算、无线通信、智能计算芯片
"}, bioImg=ZF7N3WPQTE/udhAaJucbpg==, bioContent=
1968年生. 博士. 联想集团副总裁. 主要研究方向为计算机体系结构、异构智能计算、无线通信、智能计算芯片
, aboutCorrespAuthor=null)}, companyList=[AuthorCompany(id=1327012997068861597, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012989124849767, xref=1, ext=[AuthorCompanyExt(id=1327012997081444510, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012989124849767, companyId=1327012997068861597, language=EN, country=null, province=null, city=null, postcode=null, companyName=null, departmentName=null, remark=
1Lenovo Research, Beijing 100094), AuthorCompanyExt(id=1327012997094027423, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012989124849767, companyId=1327012997068861597, language=CN, country=null, province=null, city=null, postcode=null, companyName=null, departmentName=null, remark=
1联想研究院 北京 100094)])])], keywords=[Keyword(id=1327013004576665827, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012989124849767, language=EN, orderNo=1, keyword=large-scale model training), Keyword(id=1327013004660551908, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012989124849767, language=EN, orderNo=2, keyword=deep learning), Keyword(id=1327013004748632293, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012989124849767, language=EN, orderNo=3, keyword=fault tolerance), Keyword(id=1327013004853489894, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012989124849767, language=EN, orderNo=4, keyword=failure detection), Keyword(id=1327013004945764583, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012989124849767, language=EN, orderNo=5, keyword=elastic training), Keyword(id=1327013005029650664, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012989124849767, language=EN, orderNo=6, keyword=automatic parallelization), Keyword(id=1327013005126119657, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012989124849767, language=CN, orderNo=1, keyword=大模型训练), Keyword(id=1327013005239365866, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012989124849767, language=CN, orderNo=2, keyword=深度学习), Keyword(id=1327013005319057643, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012989124849767, language=CN, orderNo=3, keyword=容错), Keyword(id=1327013005411332332, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012989124849767, language=CN, orderNo=4, keyword=故障检测), Keyword(id=1327013005520384237, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012989124849767, language=CN, orderNo=5, keyword=弹性训练), Keyword(id=1327013005654601966, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012989124849767, language=CN, orderNo=6, keyword=自动并行)], refs=[Reference(id=1327013013703471386, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012989124849767, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=1, rfOrder=0, authorNames=null, journalName=null, refType=null, unstructuredReference=Vaswani A, Shazeer N, Parmar N, et al. Attention is all you need[C]//Proc of the 31st Int Conf on Neural Information Processing Systems (NIPS). Red Hook, NY: Curran Associates, 2017: 6000–6010, articleTitle=null, refAbstract=null), Reference(id=1327013013812523291, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012989124849767, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=2, rfOrder=1, authorNames=null, journalName=null, refType=null, unstructuredReference=翟恩南,操佳敏,钱坤,等. 面向大模型时代的网络基础设施研究:挑战、阶段成果与展望[J]. 计算机研究与发展,2024,61(11):2664−2677, articleTitle=null, refAbstract=null), Reference(id=1327013013917380892, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012989124849767, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=2, rfOrder=2, authorNames=null, journalName=null, refType=null, unstructuredReference=Zhai Ennan, Cao Jiamin, Qiankun, et al. Research on network infrastructure in the era of large models: Challenges, stage achievements, and prospects[J]. Journal of Computer Research and Development, 2024, 61(11): 2664−2677 (in Chinese), articleTitle=null, refAbstract=null), Reference(id=1327013014005461277, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012989124849767, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=3, rfOrder=3, authorNames=null, journalName=null, refType=null, unstructuredReference=Zhang Susan, Roller S, Goyal N, et al. Opt: Open pre-trained transformer language models[J]. arXiv preprint, arXiv: 2205.01068, 2022, articleTitle=null, refAbstract=null), Reference(id=1327013014089347358, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012989124849767, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=4, rfOrder=4, authorNames=null, journalName=null, refType=null, unstructuredReference=Dubey A, Jauhri A, Pandey A, et al. The Llama 3 herd of models[J]. arXiv preprint, arXiv: 2407.21783, 2024, articleTitle=null, refAbstract=null), Reference(id=1327013014210982175, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012989124849767, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=5, rfOrder=5, authorNames=null, journalName=null, refType=null, unstructuredReference=Floridi L, Chiriatti M. GPT-3: Its nature, scope, limits, and consequences[J]. Minds and Machines, 2020, 30: 681−694, articleTitle=null, refAbstract=null), Reference(id=1327013014307451169, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012989124849767, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=6, rfOrder=6, authorNames=null, journalName=null, refType=null, unstructuredReference=Meta. Pytorch: Tensors and dynamic neural networks in Python with strong GPU acceleration[CP/OL]. 2016[2024-02-19]. https://github.com/pytorch/pytorch, articleTitle=null, refAbstract=null), Reference(id=1327013014391337250, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012989124849767, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=7, rfOrder=7, authorNames=null, journalName=null, refType=null, unstructuredReference=Google. Tensorflow: An open source machine learning framework for everyone[CP/OL]. 2015[2024-02-19]. https://github.com/tensorflow/te-nsorflow, articleTitle=null, refAbstract=null), Reference(id=1327013014475223332, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012989124849767, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=8, rfOrder=8, authorNames=null, journalName=null, refType=null, unstructuredReference=Microsoft. DeepSpeed[EB/OL]. 2023[2024-02-19]. https://github.com/microsoft/DeepSpeed, articleTitle=null, refAbstract=null), Reference(id=1327013014550720805, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012989124849767, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=9, rfOrder=9, authorNames=null, journalName=null, refType=null, unstructuredReference=Shoeybi M, Patwary M, Puri R, et al. Megatron-LM: Training multi-billion parameter language models using model parallelism[J]. arXiv preprint, arXiv: 1909.08053, 2019, articleTitle=null, refAbstract=null), Reference(id=1327013014668161318, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012989124849767, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=10, rfOrder=10, authorNames=null, journalName=null, refType=null, unstructuredReference=Wu Baodong, Xia Lei, Li Qingping, et al. TRANSOM: An efficient fault-tolerant system for training LLMs[J]. arXiv preprint, arXiv: 2310.10046, 2023, articleTitle=null, refAbstract=null), Reference(id=1327013014747853095, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012989124849767, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=11, rfOrder=11, authorNames=null, journalName=null, refType=null, unstructuredReference=Ant Group. DLRover: An automatic distributed deep learning system[CP/OL]. 2025[2025-02-19]. https://github.com/intelligent-machine-learning/dlrover, articleTitle=null, refAbstract=null), Reference(id=1327013014831739176, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012989124849767, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=12, rfOrder=12, authorNames=null, journalName=null, refType=null, unstructuredReference=Ant Group. DLRover’s technical practice of training stability assurance for thousands of calorie-level large models on Kubernetes[EB/OL]. 2025[2025-02-19]. https://github.com/intelligent-machine-learning/dlr- over/blob/master/docs/blogs/stabilize_llm_training_cn.md, articleTitle=null, refAbstract=null), Reference(id=1327013014936596777, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012989124849767, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=13, rfOrder=13, authorNames=null, journalName=null, refType=null, unstructuredReference=Mohan J, Phanishayee A, Chidambaram V. CheckFreq: Frequent, fine-grained DNN checkpointing[C]//Proc of the 19th USENIX Conf on File and Storage Technologies (FAST). Berkeley, CA: USENIX Association, 2021: 203−216, articleTitle=null, refAbstract=null), Reference(id=1327013015024677162, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012989124849767, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=14, rfOrder=14, authorNames=null, journalName=null, refType=null, unstructuredReference=Wang Guanhua, Ruwase O, Xie Bing, et al. FastPersist: Accelerating model checkpointing in deep learning[J]. arXiv preprint, arXiv: 2406.13768, 2024, articleTitle=null, refAbstract=null), Reference(id=1327013015104368939, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012989124849767, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=15, rfOrder=15, authorNames=null, journalName=null, refType=null, unstructuredReference=Gupta T, Krishnan S, Kumar R, et al. Just-in-time checkpointing: Low cost error recovery from deep learning training failures[C]//Proc of the 19th European Conf on Computer Systems (EuroSys). New York: ACM, 2024: 1110−1125, articleTitle=null, refAbstract=null), Reference(id=1327013015200837932, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012989124849767, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=16, rfOrder=16, authorNames=null, journalName=null, refType=null, unstructuredReference=Wang Zhuang, Jia Zhen, Zheng Shuai, et al. GEMINI: Fast failure recovery in distributed training with in-memory checkpoints[C]//Proc of the 29th Symp on Operating Systems Principles (SOSP). New York: ACM, 2023: 364−381, articleTitle=null, refAbstract=null), Reference(id=1327013015280529709, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012989124849767, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=17, rfOrder=17, authorNames=null, journalName=null, refType=null, unstructuredReference=He Tao, Li Xue, Wang Zhibin, et al. Unicron: Economizing self-healing LLM training at scale[J]. arXiv preprint, arXiv: 2401.00134, 2023, articleTitle=null, refAbstract=null), Reference(id=1327013015351832878, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012989124849767, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=18, rfOrder=18, authorNames=null, journalName=null, refType=null, unstructuredReference=Xiang Wu, Li Yakun, Ren Yuquan, et al. Gödel: Unified large-scale resource management and scheduling at ByteDance[C]//Proc of the 14th ACM Symp on Cloud Computing (SoCC). New York: ACM, 2023: 308−323, articleTitle=null, refAbstract=null), Reference(id=1327013015469273391, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012989124849767, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=19, rfOrder=19, authorNames=null, journalName=null, refType=null, unstructuredReference=Liu K, Kosaian J, Rashmi K V. ECRM: Efficient fault tolerancefor recommendation model training via erasure coding[J]. arXiv preprint, arXiv: 2104.01981, 2021, articleTitle=null, refAbstract=null), Reference(id=1327013015544770864, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012989124849767, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=20, rfOrder=20, authorNames=null, journalName=null, refType=null, unstructuredReference=Hu Qinghao, Ye Zhisheng, Wang Zerui, et al. Characterization of large language model development in the datacenter[C]//Proc of the 21st USENIX Symp on Networked Systems Design and Implementation (NSDI). Berkeley, CA: USENIX Association, 2024: 709−729, articleTitle=null, refAbstract=null), Reference(id=1327013015632851249, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012989124849767, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=21, rfOrder=21, authorNames=null, journalName=null, refType=null, unstructuredReference=Wu Tianyuan, Wang Wei, Yu Yinghao, et al. FALCON: Pinpointing and mitigating stragglers for large-scale hybrid-parallel training[J]. arXiv preprint, arXiv: 2410.12588, 2024, articleTitle=null, refAbstract=null), Reference(id=1327013015699960114, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012989124849767, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=22, rfOrder=22, authorNames=null, journalName=null, refType=null, unstructuredReference=Lao C L, Yu Minlan, Akella A, et al. TrainMover: Efficient ML training live migration with no memory overhead[J]. arXiv preprint, arXiv: 2412.12636, 2024, articleTitle=null, refAbstract=null), Reference(id=1327013015779651891, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012989124849767, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=23, rfOrder=23, authorNames=null, journalName=null, refType=null, unstructuredReference=Jiang Ziheng, Lin Haibin, Zhong Yinmin, et al. MegaScale: Scaling large language model training to more than 10, 000 GPUs[C]//Proc of the 21st USENIX Symp on Networked Systems Design and Implementation (NSDI). Berkeley, CA: USENIX Association, 2024: 745−760, articleTitle=null, refAbstract=null), Reference(id=1327013015867732276, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012989124849767, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=24, rfOrder=24, authorNames=null, journalName=null, refType=null, unstructuredReference=Eisenman A, Matam K K, Ingram S, et al. Check-N-Run: A checkpointing system for training deep learning recommendation models[C]//Proc of the 19th USENIX Symp on Networked Systems Design and Implementation (NSDI). Berkeley, CA: USENIX Association, 2022: 929−943, articleTitle=null, refAbstract=null), Reference(id=1327013015955812661, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012989124849767, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=25, rfOrder=25, authorNames=null, journalName=null, refType=null, unstructuredReference=Li Mingzhen, Xiao Wencong, Yang Hailong, et al. EasyScale: Elastic training with consistent accuracy and improved utilization on GPUs[C]//Proc of the Int Conf for High Performance Computing, Networking, Storage and Analysis (SC). New York: ACM, 2023: 1−14, articleTitle=null, refAbstract=null), Reference(id=1327013017683865910, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012989124849767, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=26, rfOrder=26, authorNames=null, journalName=null, refType=null, unstructuredReference=Subramanya S J, Arfeen D, Lin Shouxu, et al. Sia: Heterogeneity-aware, goodput-optimized ML-cluster scheduling[C]//Proc of the 29th Symp on Operating Systems Principles (SOSP). New York: ACM, 2023: 642−657, articleTitle=null, refAbstract=null), Reference(id=1327013017759363383, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012989124849767, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=27, rfOrder=27, authorNames=null, journalName=null, refType=null, unstructuredReference=Wagenländer M, Li Guo, Zhao Bo, et al. Tenplex: Dynamic parallelism for deep learning using parallelizable tensor collections[C]//Proc of the 30th Symp on Operating Systems Principles (SOSP). New York: ACM, 2024: 195−210, articleTitle=null, refAbstract=null), Reference(id=1327013017826472248, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012989124849767, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=28, rfOrder=28, authorNames=null, journalName=null, refType=null, unstructuredReference=NVIDIA. NCCL Tests: Check both the performance and the correctness of NCCL operations[CP/OL]. 2017[2025-02-19]. https://github.com/N-VIDIA/nccl-tests, articleTitle=null, refAbstract=null), Reference(id=1327013017931329849, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012989124849767, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=29, rfOrder=29, authorNames=null, journalName=null, refType=null, unstructuredReference=Zheng Lianmin, Li Zhuohan, Zhang Hao, et al. Alpa: Automating inter- and intra-operator parallelism for distributed deep learning[C]//Proc of the 16th USENIX Symp on Operating Systems Design and Implementation (OSDI). Berkeley, CA: USENIX Association, 2022: 559−578, articleTitle=null, refAbstract=null), Reference(id=1327013018011021626, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012989124849767, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=30, rfOrder=30, authorNames=null, journalName=null, refType=null, unstructuredReference=Google. XLA: An open-source machine learning (ML) compiler for GPUs, CPUs, and ML accelerators[CP/OL]. 2022[2025-02-19]. https://github.com/openxla/xla, articleTitle=null, refAbstract=null), Reference(id=1327013018078130491, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012989124849767, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=31, rfOrder=31, authorNames=null, journalName=null, refType=null, unstructuredReference=Chowdhery A, Narang S, Devlin J, et al. Palm: Scaling language modeling with pathways[J]. Journal of Machine Learning Research, 2023, 24(240): 1−113, articleTitle=null, refAbstract=null)], funds=null, companyList=[AuthorCompany(id=1327012997068861597, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012989124849767, xref=1, ext=[AuthorCompanyExt(id=1327012997081444510, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012989124849767, companyId=1327012997068861597, language=EN, country=null, province=null, city=null, postcode=null, companyName=null, departmentName=null, remark=
1Lenovo Research, Beijing 100094), AuthorCompanyExt(id=1327012997094027423, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012989124849767, companyId=1327012997068861597, language=CN, country=null, province=null, city=null, postcode=null, companyName=null, departmentName=null, remark=
1联想研究院 北京 100094)]), AuthorCompany(id=1327012997215662240, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012989124849767, xref=2, ext=[AuthorCompanyExt(id=1327012997228245153, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012989124849767, companyId=1327012997215662240, language=EN, country=null, province=null, city=null, postcode=null, companyName=null, departmentName=null, remark=
2Nanoscale Integrated Circuits and Systems Lab (Tsinghua University), Beijing 100084), AuthorCompanyExt(id=1327012997240828066, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012989124849767, companyId=1327012997215662240, language=CN, country=null, province=null, city=null, postcode=null, companyName=null, departmentName=null, remark=
2纳米集成电路与系统实验室(清华大学) 北京 100084)])], figs=[ArticleFig(id=1327013005797208303, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012989124849767, language=EN, label=Figure 1, caption=
Performance optimization decomposition of model fault recovery time for 109 billion parameters, figureFileSmall=NKMlV4l2ug+3XktAUI2y2g==, figureFileBig=8lp5NmwOz01/TjlFyejXZQ==, tableContent=null), ArticleFig(id=1327013005939814640, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012989124849767, language=CN, label=图1, caption=
1 090 亿参数模型故障恢复时间性能优化分解, figureFileSmall=NKMlV4l2ug+3XktAUI2y2g==, figureFileBig=8lp5NmwOz01/TjlFyejXZQ==, tableContent=null), ArticleFig(id=1327013006417965297, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012989124849767, language=EN, label=Figure 2, caption=
Overall architecture of Resilio, figureFileSmall=jvmjQ1NhSQCwMlkzPDY8eA==, figureFileBig=MDVy9Vfok9L9fcnZLE0Q/A==, tableContent=null), ArticleFig(id=1327013006527017202, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012989124849767, language=CN, label=图2, caption=
Resilio整体架构, figureFileSmall=jvmjQ1NhSQCwMlkzPDY8eA==, figureFileBig=MDVy9Vfok9L9fcnZLE0Q/A==, tableContent=null), ArticleFig(id=1327013006598320371, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012989124849767, language=EN, label=Figure 3, caption=
Multi-layer asynchronous CKPT save and load, figureFileSmall=Og20YibB1jFzkYaISmAg/Q==, figureFileBig=rnJZD+yaYEm8Hfe8a/QVQg==, tableContent=null), ArticleFig(id=1327013006728343796, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012989124849767, language=CN, label=图3, caption=
多层次异步CKPT保存和加载, figureFileSmall=Og20YibB1jFzkYaISmAg/Q==, figureFileBig=rnJZD+yaYEm8Hfe8a/QVQg==, tableContent=null), ArticleFig(id=1327013006808035573, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012989124849767, language=EN, label=Figure 4, caption=
The workflow of JIT CKPT, figureFileSmall=pQf8mmUiQjGuoPmyhmeUmQ==, figureFileBig=5GUNKdQ4OmkTQB29iSC6EA==, tableContent=null), ArticleFig(id=1327013006933864694, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012989124849767, language=CN, label=图4, caption=
JIT CKPT工作流程, figureFileSmall=pQf8mmUiQjGuoPmyhmeUmQ==, figureFileBig=5GUNKdQ4OmkTQB29iSC6EA==, tableContent=null), ArticleFig(id=1327013008594809079, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012989124849767, language=EN, label=Figure 5, caption=
Workflow diagram of auto-parallel submodule, figureFileSmall=B7ZlcSBgQfx27WZnqCToiQ==, figureFileBig=7upuZM5+Wzg0P93TUzFG4Q==, tableContent=null), ArticleFig(id=1327013008695472376, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012989124849767, language=CN, label=图5, caption=
自动并行子模块工作流程图, figureFileSmall=B7ZlcSBgQfx27WZnqCToiQ==, figureFileBig=7upuZM5+Wzg0P93TUzFG4Q==, tableContent=null), ArticleFig(id=1327013008787747065, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012989124849767, language=EN, label=Figure 6, caption=
End-to-end recovery time-consuming for GPT-109B model, figureFileSmall=LT0UM+PphoSz2D1WC6rB5Q==, figureFileBig=cvv25tktiaFata3YVw364w==, tableContent=null), ArticleFig(id=1327013008892604666, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012989124849767, language=CN, label=图6, caption=
GPT-109B模型端到端恢复耗时, figureFileSmall=LT0UM+PphoSz2D1WC6rB5Q==, figureFileBig=cvv25tktiaFata3YVw364w==, tableContent=null), ArticleFig(id=1327013008972296443, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012989124849767, language=EN, label=Figure 7, caption=
End-to-end recovery time-consuming for GPT-13B model, figureFileSmall=N41OYGcNIfHKvyVK4I/Xww==, figureFileBig=IHdLTqYlWBk4lnZY7ToTiA==, tableContent=null), ArticleFig(id=1327013009064571132, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012989124849767, language=CN, label=图7, caption=
GPT-13B模型端到端恢复耗时, figureFileSmall=N41OYGcNIfHKvyVK4I/Xww==, figureFileBig=IHdLTqYlWBk4lnZY7ToTiA==, tableContent=null), ArticleFig(id=1327013009169428734, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012989124849767, language=EN, label=Figure 8, caption=
End-to-end recovery time-consuming for GPT-70B model, figureFileSmall=YL4tL4qJl8TPCT8r5w5gsQ==, figureFileBig=y5XDJtesK/cxvKZUT0k6Uw==, tableContent=null), ArticleFig(id=1327013009299452159, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012989124849767, language=CN, label=图8, caption=
GPT-70B模型端到端恢复耗时, figureFileSmall=YL4tL4qJl8TPCT8r5w5gsQ==, figureFileBig=y5XDJtesK/cxvKZUT0k6Uw==, tableContent=null), ArticleFig(id=1327013009433669888, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012989124849767, language=EN, label=Figure 9, caption=
Comparison of time consumption for HPPS simulator, figureFileSmall=ihWv4YmI2ko0Kd2eUwQTTw==, figureFileBig=2gSrLLWiOT3xLQDdx+R9+A==, tableContent=null), ArticleFig(id=1327013009542721793, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012989124849767, language=CN, label=图9, caption=
HPPS模拟器耗时对比, figureFileSmall=ihWv4YmI2ko0Kd2eUwQTTw==, figureFileBig=2gSrLLWiOT3xLQDdx+R9+A==, tableContent=null), ArticleFig(id=1327013009630802178, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012989124849767, language=EN, label=Table 1, caption=
Configuration Parameters of Server Cluster-A, figureFileSmall=null, figureFileBig=null, tableContent=
| 参数 | 配置环境 |
|---|
| CPU | Intel Xeon Platinum 8378A@3.0 GHz |
| CPU核数 | 128 |
| 操作系统 | Ubuntu 22.04 LTS |
| GPU | 8 × NVIDIA A100 |
| 内存/TB | 1 |
| 网络带宽/Gbps | 25 |
), ArticleFig(id=1327013009714688259, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012989124849767, language=CN, label=表1, caption=
Cluster-A服务器配置参数
, figureFileSmall=null, figureFileBig=null, tableContent=
| 参数 | 配置环境 |
|---|
| CPU | Intel Xeon Platinum 8378A@3.0 GHz |
| CPU核数 | 128 |
| 操作系统 | Ubuntu 22.04 LTS |
| GPU | 8 × NVIDIA A100 |
| 内存/TB | 1 |
| 网络带宽/Gbps | 25 |
), ArticleFig(id=1327013009802768644, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012989124849767, language=EN, label=Table 2, caption=
Configuration Parameters of Server Cluster-B, figureFileSmall=null, figureFileBig=null, tableContent=
| 参数 | 配置环境 |
|---|
| CPU | Intel Xeon Gold 6526Y@3.9 GHz |
| CPU核数 | 32 |
| 操作系统 | Ubuntu 22.04 LTS |
| GPU | 8 × NVIDIA L20 |
| 内存/GB | 540 |
| 网络带宽/Gbps | 200 |
), ArticleFig(id=1327013009882460421, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012989124849767, language=CN, label=表2, caption=
Cluster-B服务器配置参数
, figureFileSmall=null, figureFileBig=null, tableContent=
| 参数 | 配置环境 |
|---|
| CPU | Intel Xeon Gold 6526Y@3.9 GHz |
| CPU核数 | 32 |
| 操作系统 | Ubuntu 22.04 LTS |
| GPU | 8 × NVIDIA L20 |
| 内存/GB | 540 |
| 网络带宽/Gbps | 200 |
), ArticleFig(id=1327013009978929414, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012989124849767, language=EN, label=Table 3, caption=
Configuration Parameters of Server Cluster-C, figureFileSmall=null, figureFileBig=null, tableContent=
| 参数 | 配置环境 |
|---|
| CPU | Intel Xeon Gold 5317 CPU@3.00 GHz |
| CPU核数 | 48 |
| 操作系统 | Ubuntu 22.04 LTS |
| GPU | 4 × NVIDIA A100 |
| 内存/TB | 2 |
), ArticleFig(id=1327013010067009799, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012989124849767, language=CN, label=表3, caption=
Cluster-C服务器配置参数
, figureFileSmall=null, figureFileBig=null, tableContent=
| 参数 | 配置环境 |
|---|
| CPU | Intel Xeon Gold 5317 CPU@3.00 GHz |
| CPU核数 | 48 |
| 操作系统 | Ubuntu 22.04 LTS |
| GPU | 4 × NVIDIA A100 |
| 内存/TB | 2 |
), ArticleFig(id=1327013010142507272, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012989124849767, language=EN, label=Table 4, caption=
Explanation of Fault Types, figureFileSmall=null, figureFileBig=null, tableContent=
| 故障类型 | 释义 | |
|---|
| 致命异常 | 训练进程抛出的不可自动修复异常,比如GPU OOM、NCCL异常等 | |
| pod崩溃 | Kubernetes控制层不合理的迁移操作、节点故障、OOM等情况导致的pod直接中断退出 | |
| pod误杀 | 研发或运维人员误操作导致pod被删除 | |
| 网络断开 | 计算节点网络断开 | |
| 节点宕机 | 计算节点由于断电、故障导致的宕机 | |
| 组件故障 | 容错机制核心组件自身遭遇的故障 | |
), ArticleFig(id=1327013010213810441, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012989124849767, language=CN, label=表4, caption=
故障类型释义
, figureFileSmall=null, figureFileBig=null, tableContent=
| 故障类型 | 释义 | |
|---|
| 致命异常 | 训练进程抛出的不可自动修复异常,比如GPU OOM、NCCL异常等 | |
| pod崩溃 | Kubernetes控制层不合理的迁移操作、节点故障、OOM等情况导致的pod直接中断退出 | |
| pod误杀 | 研发或运维人员误操作导致pod被删除 | |
| 网络断开 | 计算节点网络断开 | |
| 节点宕机 | 计算节点由于断电、故障导致的宕机 | |
| 组件故障 | 容错机制核心组件自身遭遇的故障 | |
), ArticleFig(id=1327013010301890826, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012989124849767, language=EN, label=Table 5, caption=
Comparison of CKPT Loading Time, figureFileSmall=null, figureFileBig=null, tableContent=
| 模型 | CKPT 大 小/GB | 加载时间/s |
|---|
| 原生NFS | DLRover Memory | Resilio NVME | Resilio Memor |
|---|
| GPT-109B | 1536 | 546.6 | 3.8 | 44.3 | 1.5 |
| GPT-70B | 904 | 322.0 | 2.3 | 25.1 | 1.1 |
| GPT-13B | 176 | 60.5 | 2.2 | 17.8 | 0.9 |
), ArticleFig(id=1327013010398359819, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012989124849767, language=CN, label=表5, caption=
CKPT加载时间对比
, figureFileSmall=null, figureFileBig=null, tableContent=
| 模型 | CKPT 大 小/GB | 加载时间/s |
|---|
| 原生NFS | DLRover Memory | Resilio NVME | Resilio Memor |
|---|
| GPT-109B | 1536 | 546.6 | 3.8 | 44.3 | 1.5 |
| GPT-70B | 904 | 322.0 | 2.3 | 25.1 | 1.1 |
| GPT-13B | 176 | 60.5 | 2.2 | 17.8 | 0.9 |
), ArticleFig(id=1327013010473857292, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012989124849767, language=EN, label=Table 6, caption=
Comparison of CKPT Saving Time s, figureFileSmall=null, figureFileBig=null, tableContent=
| 模型 | 原生NFS | DLRover Memory | Resilio NVME | Resilio Memory | Resilio Block |
|---|
| GPT-109B | 579.0 | 2.2 | 69.4 | 1.63 | 0.2 |
| GPT-70B | 335.0 | 1.3 | 53.8 | 1.18 | 0.2 |
| GPT-13B | 66.2 | 1.2 | 27.1 | 0.90 | 0.2 |
), ArticleFig(id=1327013010553549069, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012989124849767, language=CN, label=表6, caption=
CKPT保存时间对比
, figureFileSmall=null, figureFileBig=null, tableContent=
| 模型 | 原生NFS | DLRover Memory | Resilio NVME | Resilio Memory | Resilio Block |
|---|
| GPT-109B | 579.0 | 2.2 | 69.4 | 1.63 | 0.2 |
| GPT-70B | 335.0 | 1.3 | 53.8 | 1.18 | 0.2 |
| GPT-13B | 66.2 | 1.2 | 27.1 | 0.90 | 0.2 |
), ArticleFig(id=1327013010624852238, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012989124849767, language=EN, label=Table 7, caption=
Performance Overhead of Our Proposed Scheme, figureFileSmall=null, figureFileBig=null, tableContent=
| 模型 | CKPT大小/GB | 迭代耗时/s | 迭代耗时(开启JIT)/s | 开销/% |
|---|
| GPT-70B | 904 | 2.91 | 2.96 | 1.65 |
| GPT-13B | 176 | 2.71 | 2.75 | 1.30 |
), ArticleFig(id=1327013010717126927, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012989124849767, language=CN, label=表7, caption=
本文方案性能开销
, figureFileSmall=null, figureFileBig=null, tableContent=
| 模型 | CKPT大小/GB | 迭代耗时/s | 迭代耗时(开启JIT)/s | 开销/% |
|---|
| GPT-70B | 904 | 2.91 | 2.96 | 1.65 |
| GPT-13B | 176 | 2.71 | 2.75 | 1.30 |
), ArticleFig(id=1327013010792624400, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012989124849767, language=EN, label=Table 8, caption=
Failure Recovery Time-Consuming of Our Proposed Scheme, figureFileSmall=null, figureFileBig=null, tableContent=
| 模型 | CKPT 大小/GB | 故障诊断 耗时/s | CKPT内存 恢复耗时/s | CKPT RDMA 恢复耗时/s |
|---|
| GPT-70B | 904 | 6.0 | 1.22 | 7.5 |
| GPT-13B | 176 | 5.4 | 1.02 | 4.4 |
), ArticleFig(id=1327013010872316177, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012989124849767, language=CN, label=表8, caption=
本文方案故障恢复耗时
, figureFileSmall=null, figureFileBig=null, tableContent=
| 模型 | CKPT 大小/GB | 故障诊断 耗时/s | CKPT内存 恢复耗时/s | CKPT RDMA 恢复耗时/s |
|---|
| GPT-70B | 904 | 6.0 | 1.22 | 7.5 |
| GPT-13B | 176 | 5.4 | 1.02 | 4.4 |
), ArticleFig(id=1327013010947813650, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012989124849767, language=EN, label=Table 9, caption=
Effect of Fault Frequency on CKPT Overhead, figureFileSmall=null, figureFileBig=null, tableContent=
| 故障频率(次/天) | 故障检测耗时/s | 保存耗时(本文)/s | 开销占比/% |
|---|
| 10 | 6.0 | 1.22 | 0.08 |
| 100 | 6.0 | 1.22 | 0.84 |
), ArticleFig(id=1327013011027505427, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012989124849767, language=CN, label=表9, caption=
故障频率对CKPT开销的影响
, figureFileSmall=null, figureFileBig=null, tableContent=
| 故障频率(次/天) | 故障检测耗时/s | 保存耗时(本文)/s | 开销占比/% |
|---|
| 10 | 6.0 | 1.22 | 0.08 |
| 100 | 6.0 | 1.22 | 0.84 |
), ArticleFig(id=1327013011111391508, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012989124849767, language=EN, label=Table 10, caption=
Testing List of HPPS Model, figureFileSmall=null, figureFileBig=null, tableContent=
| 模型 | Billion | 批次大小 | 流水线并行度 |
|---|
| OPT | [1.3, 2.7, 6.7] | 256 | {4,2} |
| GPT-2 | 1.5 | 256 | {4,2} |
), ArticleFig(id=1327013011392409877, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012989124849767, language=CN, label=表10, caption=
HPPS模型测试列表
, figureFileSmall=null, figureFileBig=null, tableContent=
| 模型 | Billion | 批次大小 | 流水线并行度 |
|---|
| OPT | [1.3, 2.7, 6.7] | 256 | {4,2} |
| GPT-2 | 1.5 | 256 | {4,2} |
), ArticleFig(id=1327013013145628950, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012989124849767, language=EN, label=Table 11, caption=
Accuracy Comparison of HPPS Simulator, figureFileSmall=null, figureFileBig=null, tableContent=
| 模型 | Billion | HPPS单次迭代 时间/s | Alpa单次迭代 时间/s | 精度误差/% |
|---|
| OPT | 1.3 | 5.23 | 4.95 | 5.59 |
| OPT | 2.7 | 8.17 | 8.62 | 5.18 |
| OPT | 6.7 | 16.61 | 19.05 | 12.81 |
| GPT-2 | 1.5 | 8.26 | 7.68 | 7.51 |
), ArticleFig(id=1327013013317595415, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012989124849767, language=CN, label=表11, caption=
HPPS模拟器精度对比
, figureFileSmall=null, figureFileBig=null, tableContent=
| 模型 | Billion | HPPS单次迭代 时间/s | Alpa单次迭代 时间/s | 精度误差/% |
|---|
| OPT | 1.3 | 5.23 | 4.95 | 5.59 |
| OPT | 2.7 | 8.17 | 8.62 | 5.18 |
| OPT | 6.7 | 16.61 | 19.05 | 12.81 |
| GPT-2 | 1.5 | 8.26 | 7.68 | 7.51 |
), ArticleFig(id=1327013013464396056, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012989124849767, language=EN, label=Table 12, caption=
Improvement of MFU Based on HPPS Test Model %, figureFileSmall=null, figureFileBig=null, tableContent=
| 模型 | HPPS | Alpa | MFU提升 |
|---|
| OPT-1.3B | 35.93 | 33.88 | 2.05 |
| OPT-2.7B | 50.72 | 38.60 | 12.12 |
| OPT-6.7B | 64.64 | 48.48 | 16.16 |
| GPT-2-1.5B | 31.40 | 28.27 | 3.13 |
), ArticleFig(id=1327013013581836569, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012989124849767, language=CN, label=表12, caption=
基于HPPS测试模型的MFU提升
, figureFileSmall=null, figureFileBig=null, tableContent=
| 模型 | HPPS | Alpa | MFU提升 |
|---|
| OPT-1.3B | 35.93 | 33.88 | 2.05 |
| OPT-2.7B | 50.72 | 38.60 | 12.12 |
| OPT-6.7B | 64.64 | 48.48 | 16.16 |
| GPT-2-1.5B | 31.40 | 28.27 | 3.13 |
)], attaches=null, journal=Journal(id=1324759606947315908, delFlag=0, nameCn=计算机研究与发展, nameEn=Journal of Computer Research and Development, nameHistory1=null, nameHistory2=null, issn=1000-1239, eissn=null, cn=11-1777/TP, coden=null, periodic=0, language=CN, oaType=null, ccby=null, superviseOffice=null, ownerOffice=null, pubOffice=null, editorOffice=null, officeType=null, aims=null, clcCode=null, officeProv=null, officeCity=null, officeAddr=null, officeZip=null, officeEmail=null, officePhone=null, editDirector=null, officeDirector=null, officeDirectorPhone=null, officeStaffNum=null, officeEmpNum=null, coverPicUrl=k2l8zgqwstuxxevFX2qxVA==, journalPrice=null, startedYear=null, abbrevIsoEn=Journal of Computer Research and Development, journalRemark=null, publicationField=null, createdTime=1793776903141, updatedTime=1793850698612, createdBy=18614031015, updatedBy=13701087609, firstLetterCn=J, firstLetterEn=J, subjectCode=Electronic Information, subjectName=null, subjectCodeEn=Electronic Information, subjectNameEn=null, picCn=k2l8zgqwstuxxevFX2qxVA==, picEn=r6k0Ry+RD+c/VXj5QarmDw==, jcr=null, cjcr=null, exts=[JournalExt(id=1325069127710286688, language=CN, name=计算机研究与发展, nameHistory1=null, nameHistory2=null, managedBy=, sponsoredBy=, publishedBy=, editorOffice=, officeProv=null, officeCity=null, officeAddr=, officeZip=, editDirector=, officeDirector=null, officePhone=null, coverPicUrl=null, journalRemark=, submitArticleUrl=null, websiteUrl=, createdTime=1793850698640, updatedTime=1793850698640, createdBy=13701087609, updatedBy=13701087609, submissionGuidelinesUrl=, submissionAuthorUrl=https://crad.ict.ac.cn/journalx/authorLogOn.action, submissionEditorUrl=https://crad.ict.ac.cn/journalx/editorLogOn.action, submissionReviewUrl=https://crad.ict.ac.cn/journalx/expertLogOn.action, submissionCeEditorUrl=https://crad.ict.ac.cn/journalx/editorInChiefLogOn.action, submissionAeEditorUrl=, option={"copyright":""}), JournalExt(id=1325069127777395553, language=EN, name=Journal of Computer Research and Development, nameHistory1=null, nameHistory2=null, managedBy=, sponsoredBy=, publishedBy=, editorOffice=, officeProv=null, officeCity=null, officeAddr=, officeZip=, editDirector=, officeDirector=null, officePhone=null, coverPicUrl=null, journalRemark=, submitArticleUrl=null, websiteUrl=, createdTime=1793850698656, updatedTime=1793850698656, createdBy=13701087609, updatedBy=13701087609, submissionGuidelinesUrl=, submissionAuthorUrl=https://crad.ict.ac.cn/journalx/authorLogOn.action, submissionEditorUrl=https://crad.ict.ac.cn/journalx/editorLogOn.action, submissionReviewUrl=https://crad.ict.ac.cn/journalx/expertLogOn.action, submissionCeEditorUrl=https://crad.ict.ac.cn/journalx/editorInChiefLogOn.action, submissionAeEditorUrl=, option={"copyright":""})], databaseList=null, tenantJournalId=1324759914146529478, websiteList=[Website(id=1325070300139892771, webName=null, webTitle=null, webDomain=null, webCopyrigh=null, webIpcNo=null, seoTitle=null, seoKeywords=null, seoDescription=null, tenantJournalId=null, journalId=1324759914146529478, journalNameCn=null, journalNameEn=null, grayFlag=null, tenantId=1146029695717560320, platformId=null, journalGroupId=null, journalGroupNameCn=null, journalGroupNameEn=null, type=1, domain=https://castjournals.cast.org.cn/joweb/jsjyjyfz/CN, language=CN, createTime=1793850978169, createBy=18614031015, updateTime=1793851040259, updateBy=18614031015, name=计算机研究与发展-中文, tplId=1146099689490845704, title=计算机研究与发展, delFlag=0, indexPage=/home, props=[WebsiteProps(id=1325070893751349360, tenantId=1146029695717560320, journalId=null, journalGroupId=null, siteId=1325070300139892771, code=articleTextType, value=kx, createTime=1793851119697, updateTime=1793851119697, creator=18614031015, updator=18614031015), WebsiteProps(id=1325070893730377837, tenantId=1146029695717560320, journalId=null, journalGroupId=null, siteId=1325070300139892771, code=banner, value=null, createTime=1793851119692, updateTime=1793851119692, creator=18614031015, updator=18614031015), WebsiteProps(id=1325070893772320883, tenantId=1146029695717560320, journalId=null, journalGroupId=null, siteId=1325070300139892771, code=grayFlag, value=0, createTime=1793851119702, updateTime=1793851119702, creator=18614031015, updator=18614031015), WebsiteProps(id=1325070893721989228, tenantId=1146029695717560320, journalId=null, journalGroupId=null, siteId=1325070300139892771, code=logo, value=https://castjournals.cast.org.cn/joweb/jsjyjyfz/CN/file/pic?fileId=U27tDsy717YF7XSPA+Vwdw==, createTime=1793851119690, updateTime=1793851119690, creator=18614031015, updator=18614031015), WebsiteProps(id=1325070893784903797, tenantId=1146029695717560320, journalId=null, journalGroupId=null, siteId=1325070300139892771, code=minRunFlag, value=0, createTime=1793851119705, updateTime=1793851119705, creator=18614031015, updator=18614031015), WebsiteProps(id=1325070893742960751, tenantId=1146029695717560320, journalId=null, journalGroupId=null, siteId=1325070300139892771, code=picServerUrl, value=https://castjournals.cast.org.cn/joweb/jsjyjyfz/CN/file/pic, createTime=1793851119695, updateTime=1793851119695, creator=18614031015, updator=18614031015), WebsiteProps(id=1325070893780709492, tenantId=1146029695717560320, journalId=null, journalGroupId=null, siteId=1325070300139892771, code=silenceFlag, value=0, createTime=1793851119704, updateTime=1793851119704, creator=18614031015, updator=18614031015), WebsiteProps(id=1325070893738766446, tenantId=1146029695717560320, journalId=null, journalGroupId=null, siteId=1325070300139892771, code=staticResourcePath, value=https://castjournals.cast.org.cn/joweb/cast_kjdb_cn_619/, createTime=1793851119694, updateTime=1793851119694, creator=18614031015, updator=18614031015), WebsiteProps(id=1325070893755543665, tenantId=1146029695717560320, journalId=null, journalGroupId=null, siteId=1325070300139892771, code=themeColor, value=null, createTime=1793851119698, updateTime=1793851119698, creator=18614031015, updator=18614031015), WebsiteProps(id=1325070893763932274, tenantId=1146029695717560320, journalId=null, journalGroupId=null, siteId=1325070300139892771, code=themeStyle, value=null, createTime=1793851119700, updateTime=1793851119700, creator=18614031015, updator=18614031015)]), Website(id=1325070300580294698, webName=null, webTitle=null, webDomain=null, webCopyrigh=null, webIpcNo=null, seoTitle=null, seoKeywords=null, seoDescription=null, tenantJournalId=null, journalId=1324759914146529478, journalNameCn=null, journalNameEn=null, grayFlag=null, tenantId=1146029695717560320, platformId=null, journalGroupId=null, journalGroupNameCn=null, journalGroupNameEn=null, type=1, domain=https://castjournals.cast.org.cn/joweb/jsjyjyfz/EN, language=EN, createTime=1793850978274, createBy=18614031015, updateTime=1793851036638, updateBy=18614031015, name=计算机研究与发展-英文, tplId=1146101810881728533, title=Journal of Computer Research and Development, delFlag=0, indexPage=/home, props=[WebsiteProps(id=1325070925288321148, tenantId=1146029695717560320, journalId=null, journalGroupId=null, siteId=1325070300580294698, code=articleTextType, value=kx, createTime=1793851127216, updateTime=1793851127216, creator=18614031015, updator=18614031015), WebsiteProps(id=1325070925263155321, tenantId=1146029695717560320, journalId=null, journalGroupId=null, siteId=1325070300580294698, code=banner, value=null, createTime=1793851127210, updateTime=1793851127210, creator=18614031015, updator=18614031015), WebsiteProps(id=1325070925309292671, tenantId=1146029695717560320, journalId=null, journalGroupId=null, siteId=1325070300580294698, code=grayFlag, value=0, createTime=1793851127221, updateTime=1793851127221, creator=18614031015, updator=18614031015), WebsiteProps(id=1325070925254766712, tenantId=1146029695717560320, journalId=null, journalGroupId=null, siteId=1325070300580294698, code=logo, value=https://castjournals.cast.org.cn/joweb/jsjyjyfz/EN/file/pic?fileId=U27tDsy717YF7XSPA+Vwdw==, createTime=1793851127208, updateTime=1793851127208, creator=18614031015, updator=18614031015), WebsiteProps(id=1325070925326069889, tenantId=1146029695717560320, journalId=null, journalGroupId=null, siteId=1325070300580294698, code=minRunFlag, value=0, createTime=1793851127225, updateTime=1793851127225, creator=18614031015, updator=18614031015), WebsiteProps(id=1325070925279932539, tenantId=1146029695717560320, journalId=null, journalGroupId=null, siteId=1325070300580294698, code=picServerUrl, value=https://castjournals.cast.org.cn/joweb/jsjyjyfz/EN/file/pic, createTime=1793851127214, updateTime=1793851127214, creator=18614031015, updator=18614031015), WebsiteProps(id=1325070925317681280, tenantId=1146029695717560320, journalId=null, journalGroupId=null, siteId=1325070300580294698, code=silenceFlag, value=0, createTime=1793851127223, updateTime=1793851127223, creator=18614031015, updator=18614031015), WebsiteProps(id=1325070925271543930, tenantId=1146029695717560320, journalId=null, journalGroupId=null, siteId=1325070300580294698, code=staticResourcePath, value=https://castjournals.cast.org.cn/joweb/cast_kjdb_en_623/, createTime=1793851127212, updateTime=1793851127212, creator=18614031015, updator=18614031015), WebsiteProps(id=1325070925292515453, tenantId=1146029695717560320, journalId=null, journalGroupId=null, siteId=1325070300580294698, code=themeColor, value=null, createTime=1793851127217, updateTime=1793851127217, creator=18614031015, updator=18614031015), WebsiteProps(id=1325070925300904062, tenantId=1146029695717560320, journalId=null, journalGroupId=null, siteId=1325070300580294698, code=themeStyle, value=null, createTime=1793851127219, updateTime=1793851127219, creator=18614031015, updator=18614031015)])], journalTitle=计算机研究与发展, weixinUrl=null, journalUrl=https://crad.ict.ac.cn/, iacademicId=null, status=1, seqNo=null, journalTitleEn=Journal of Computer Research and Development, journalPhotoCn=k2l8zgqwstuxxevFX2qxVA==, journalPhotoEn=r6k0Ry+RD+c/VXj5QarmDw==, journalFirstLetter=J, journalRecommend=null, journalNew=null, journalCollection=null, jcrJf=null, cjcrJf=null, jcrJfStr=null, cjcrJfStr=null, submissionFirstDecision=null, sciSubjectClassification=null, casSubjectClassification=null, citeScore=null, totalCitationFrequency=null, icpCode=null, psCode=null, advertisingLicenseCode=null, copyrightInformation=null, country=null, option=, provinceCode=null, provinceName=null, collectFlag=false, interPubPlatform=, interPubPlatformUrl=null), detailUrlCn=https://castjournals.cast.org.cn/joweb/jsjyjyfz/CN/10.7544/issn1000-1239.202550147, detailUrlEn=https://castjournals.cast.org.cn/joweb/jsjyjyfz/EN/10.7544/issn1000-1239.202550147, pdfUrlCn=https://castjournals.cast.org.cn/joweb/jsjyjyfz/CN/PDF/10.7544/issn1000-1239.202550147, pdfUrlEn=https://castjournals.cast.org.cn/joweb/jsjyjyfz/EN/PDF/10.7544/issn1000-1239.202550147, aliStartDate=0, aliEndDate=0, collectionFlag=false, citedCount=null, citedUrl=null, previewStatus=0, delFlag=0, hasFullText=1, orderTime=1748793600000, fullTextJson=null, articleText=null, reference=null)