Article(id=1329456295243002829, tenantId=1146029695717560320, journalId=1324759914146529478, issueId=1329456279791186761, articleNumber=null, orderNo=null, doi=10.7544/issn1000-1239.202330554, pmid=null, cstr=null, oa=null, hot=null, price=null, onlineType=0, articleFormat=0, articleType=null, articleTypeStr=null, receivedDate=1688313600000, receivedDateStr=2023-07-03, revisedDate=1701014400000, revisedDateStr=2023-11-27, acceptedDate=null, acceptedDateStr=null, onlineDate=1794896680888, onlineDateStr=2026-11-17, pubDate=1711900800000, pubDateStr=2024-04-01, doiRegisterDate=null, doiRegisterDateStr=null, onlineIssueDate=1794896680888, onlineIssueDateStr=2026-11-17, onlineJustAcceptDate=null, onlineJustAcceptDateStr=null, onlineFirstDate=null, onlineFirstDateStr=null, sourceXml=null, magXml=null, createTime=1794896680888, creator=13701087609, updateTime=1794896680888, updator=13701087609, issue=Issue{id=1329456279791186761, tenantId=1146029695717560320, journalId=1324759914146529478, year='2024', volume='61', issue='4', pageStart='809', pageEnd='1084', issueExtLink='null', onlineDate='null', pubDate='1711900800000', pubDateStr='2024-04-01', beforeIssueId=null, nextIssueId=null, price=null, status=1, issueComplete=1, articleOrder=1, issueType=-1, specialIssue=null, createTime=1794896677204, creator='13701087609', updateTime=1794899232988, updator='13701087609', preIssue=null, nextIssue=null, articleTotal=null, ext={EN=IssueExt(id=1329466999882756515, tenantId=1146029695717560320, journalId=1324759914146529478, issueId=1329456279791186761, language=EN, specialIssueTitle=, coverIllustrator=null, specialIssueEditor=, specialIssueAbout=), CN=IssueExt(id=1329466999882756516, tenantId=1146029695717560320, journalId=1324759914146529478, issueId=1329456279791186761, language=CN, specialIssueTitle=, coverIllustrator=null, specialIssueEditor=, specialIssueAbout=)}, issueFiles=null, downloadFileDto=null}, startPage=809, endPage=823, ext={EN=ArticleExt(id=1329456295482078158, articleId=1329456295243002829, tenantId=1146029695717560320, journalId=1324759914146529478, language=EN, title=From BERT to ChatGPT: Challenges and Technical Development of Storage Systems for Large Model Training, columnId=1327010913338634647, journalTitle=Journal of Computer Research and Development, columnName=Architecture, runingTitle=null, highlight=null, articleAbstract=
The large models represented by ChatGPT have attracted a lot of attention from industry and academia for their excellent performance on text generation and semantic understanding tasks. The number of large model parameters has increased tens of thousands of times in three years and is still growing, which brings new challenges to storage systems. First, we analyze the storage challenges of large model training, pointing out that large model training has unique computation patterns, storage access patterns, and data characteristics, which makes traditional storage techniques inefficient in handling large model training tasks. Then, we describe three types of storage acceleration techniques and two types of fault-tolerant techniques. The storage acceleration techniques for large model training include: 1) distributed storage technique based on large model computation patterns designs the partitioning, storage, and transferring strategies of model data in distributed clusters based on the partitioning of large model computation tasks and the dependencies between computation tasks; 2) heterogeneous storage access pattern-aware technique for large model training develops data prefetching and transferring strategies among heterogeneous devices with the predictability of storage access patterns in large model training; 3) large model data reduction technique reduces the data size in the model training process according to the characteristics of large model data. The storage fault-tolerant techniques for large model training include: 1) parameter checkpointing technique stores the large model parameters to persistent storage devices; 2) redundant computation technique computes the same version of parameters repeatedly in multiple GPUs. Finally, we give the summary and suggestions for future research.
, authors=Yangyang Feng, Qing Wang, Minhui Xie, Jiwu Shu*, authorsList=Yangyang Feng, Qing Wang, Minhui Xie, Jiwu Shu, authorCompany=null, correspAuthors=Jiwu Shu, authorNote=
born in 1998. PhD candidate. His main research interests include storage systems and machine learning systems
born in 1997. PhD. His main research interests include storage systems and memory systems
born in 1997. PhD candidate. Student member of CCF. His main research interests include storage systems and machine learning systems
born in 1968. PhD, professor, PhD supervisor. Fellow of CCF. His main research interests include intelligent storage systems, non-volatile memory storage systems and technologies, storage security and reliability, and parallel and distributed computing
, correspAuthorsNote=null, copyrightStatement=Copyright © 2024 Journal of Computer Research and Development. All rights reserved., copyrightOwner=null, extLink=null, articleAbsUrl=null, sourceXml=null, magXml=null, pdfUrl=null, pdf=null, pdfFileSize=null, pdfExtLink=null, richHtmlUrl=null, mobilePdfUrl=null, reviewReport=null, pdfFirstPage=null, abstractGraph=null, abstractGraphContent=null, abstractVideo=null, citation=null, cebUrl=null, magXmlContent=null, mapNumber=null, fund=null), CN=ArticleExt(id=1329456299923846129, articleId=1329456295243002829, tenantId=1146029695717560320, journalId=1324759914146529478, language=CN, title=从BERT到ChatGPT:大模型训练中的存储系统挑战与技术发展, columnId=1325489904427344387, journalTitle=计算机研究与发展, columnName=体系结构, runingTitle=null, highlight=null, articleAbstract=
以ChatGPT为代表的大模型在文字生成、语义理解等任务上表现卓越,引起了工业界和学术界的广泛关注. 大模型的参数量在3年内增长数万倍,且仍呈现增长的趋势. 首先分析了大模型训练的存储挑战,指出大模型训练的存储需求大,且具有独特的计算模式、访存模式、数据特征,这使得针对互联网、大数据等应用的传统存储技术在处理大模型训练任务时效率低下,且容错开销大. 然后分别阐述了针对大模型训练的3类存储加速技术与2类存储容错技术. 针对大模型训练的存储加速技术包括:1)基于大模型计算模式的分布式显存管理技术,依据大模型计算任务的划分模式和计算任务间的依赖关系,设计模型数据在分布式集群中的划分、存储和传输策略;2)大模型训练访存感知的异构存储技术,借助大模型训练中的访存模式可预测的特性,设计异构设备中的数据预取和传输策略;3)大模型数据缩减技术,针对大模型数据的特征,对模型训练过程中的数据进行缩减. 针对大模型训练的存储容错技术包括:1)参数检查点技术,将大模型参数存储至持久化存储介质;2)冗余计算技术,在多张GPU中重复计算相同版本的参数. 最后给出了总结和展望.
, authors=冯杨洋, 汪庆, 谢旻晖, 舒继武*, authorsList=冯杨洋, 汪庆, 谢旻晖, 舒继武, authorCompany=null, correspAuthors=舒继武, authorNote=
 |
1998年生. 博士研究生. 主要研究方向为存储系统、机器学习系统 |
 |
1997年生. 博士. 主要研究方向为存储系统、内存系统 |
 |
1997年生. 博士研究生. CCF学生会员. 主要研究方向为存储系统、机器学习系统 |
 |
1968年生. 博士,教授,博士生导师. CCF会士. 主要研究方向为智能存储系统、非易失内存存储系统与技术、存储安全与可靠性、并行与分布式计算 |
, correspAuthorsNote=
, copyrightStatement=版权所有 © 2024 计算机研究与发展编辑部, copyrightOwner=null, extLink=null, articleAbsUrl=null, sourceXml=lnVdLucUYhLhE4sJxSp5JA==, magXml=VYqgZ40YLqBRKumycmCevQ==, pdfUrl=null, pdf=E1ZDSU7Gk3OnDPTbZy0nvA==, pdfFileSize=1784686, pdfExtLink=null, richHtmlUrl=null, mobilePdfUrl=null, reviewReport=null, pdfFirstPage=null, abstractGraph=J19neWKpt7ihI4y44eBBiw==, abstractGraphContent=null, abstractVideo=null, citation=null, cebUrl=null, magXmlContent=HcRtpClSqa3qK33R+kyd5A==, mapNumber=null, fund=null)}, authors=[Author(id=1329456300221641718, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1329456295243002829, orderNo=0, firstName=null, middleName=null, lastName=null, nameCn=null, orcid=null, stid=null, country=null, authorPic=null, dead=0, email=fyy21@mails.tsinghua.edu.cn, emailSecond=null, emailThird=null, correspondingAuthor=0, authorType=1, ext={EN=AuthorExt(id=1329456300313916408, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1329456295243002829, authorId=1329456300221641718, language=EN, stringName=Yangyang Feng, firstName=Yangyang, middleName=null, lastName=Feng, prefix=null, suffix=null, authorComment=null, nameInitials=null, affiliation=null, department=null, xref=null, address=
1Department of Computer Science and Technology, Tsinghua University, Beijing 100084, bio={"content":"
born in 1998. PhD candidate. His main research interests include storage systems and machine learning systems
"}, bioImg=null, bioContent=
born in 1998. PhD candidate. His main research interests include storage systems and machine learning systems
, aboutCorrespAuthor=null), CN=AuthorExt(id=1329456300406191097, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1329456295243002829, authorId=1329456300221641718, language=CN, stringName=冯杨洋, firstName=null, middleName=null, lastName=null, prefix=null, suffix=null, authorComment=null, nameInitials=null, affiliation=null, department=null, xref=null, address=
1清华大学计算机科学与技术系 北京 100084, bio={"img":"UWsS/qLfWEVid7CshxXjbA==","content":"
1998年生. 博士研究生. 主要研究方向为存储系统、机器学习系统
"}, bioImg=UWsS/qLfWEVid7CshxXjbA==, bioContent=
1998年生. 博士研究生. 主要研究方向为存储系统、机器学习系统
, aboutCorrespAuthor=null)}, companyList=[AuthorCompany(id=1329456300125172722, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1329456295243002829, xref=1, ext=[AuthorCompanyExt(id=1329456300133561331, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1329456295243002829, companyId=1329456300125172722, language=EN, country=null, province=null, city=null, postcode=null, companyName=null, departmentName=null, remark=
1Department of Computer Science and Technology, Tsinghua University, Beijing 100084), AuthorCompanyExt(id=1329456300150338548, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1329456295243002829, companyId=1329456300125172722, language=CN, country=null, province=null, city=null, postcode=null, companyName=null, departmentName=null, remark=
1清华大学计算机科学与技术系 北京 100084)])]), Author(id=1329456300460717051, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1329456295243002829, orderNo=1, firstName=null, middleName=null, lastName=null, nameCn=null, orcid=null, stid=null, country=null, authorPic=null, dead=0, email=null, emailSecond=null, emailThird=null, correspondingAuthor=0, authorType=1, ext={EN=AuthorExt(id=1329456302062941181, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1329456295243002829, authorId=1329456300460717051, language=EN, stringName=Qing Wang, firstName=Qing, middleName=null, lastName=Wang, prefix=null, suffix=null, authorComment=null, nameInitials=null, affiliation=null, department=null, xref=null, address=
1Department of Computer Science and Technology, Tsinghua University, Beijing 100084, bio={"content":"
born in 1997. PhD. His main research interests include storage systems and memory systems
"}, bioImg=null, bioContent=
born in 1997. PhD. His main research interests include storage systems and memory systems
, aboutCorrespAuthor=null), CN=AuthorExt(id=1329456302134244350, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1329456295243002829, authorId=1329456300460717051, language=CN, stringName=汪庆, firstName=null, middleName=null, lastName=null, prefix=null, suffix=null, authorComment=null, nameInitials=null, affiliation=null, department=null, xref=null, address=
1清华大学计算机科学与技术系 北京 100084, bio={"img":"m4Xrw6gtc8ni0ZbPBk3lfg==","content":"
1997年生. 博士. 主要研究方向为存储系统、内存系统
"}, bioImg=m4Xrw6gtc8ni0ZbPBk3lfg==, bioContent=
1997年生. 博士. 主要研究方向为存储系统、内存系统
, aboutCorrespAuthor=null)}, companyList=[AuthorCompany(id=1329456300125172722, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1329456295243002829, xref=1, ext=[AuthorCompanyExt(id=1329456300133561331, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1329456295243002829, companyId=1329456300125172722, language=EN, country=null, province=null, city=null, postcode=null, companyName=null, departmentName=null, remark=
1Department of Computer Science and Technology, Tsinghua University, Beijing 100084), AuthorCompanyExt(id=1329456300150338548, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1329456295243002829, companyId=1329456300125172722, language=CN, country=null, province=null, city=null, postcode=null, companyName=null, departmentName=null, remark=
1清华大学计算机科学与技术系 北京 100084)])]), Author(id=1329456302201353216, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1329456295243002829, orderNo=2, firstName=null, middleName=null, lastName=null, nameCn=null, orcid=null, stid=null, country=null, authorPic=null, dead=0, email=null, emailSecond=null, emailThird=null, correspondingAuthor=0, authorType=1, ext={EN=AuthorExt(id=1329456302306209793, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1329456295243002829, authorId=1329456302201353216, language=EN, stringName=Minhui Xie, firstName=Minhui, middleName=null, lastName=Xie, prefix=null, suffix=null, authorComment=null, nameInitials=null, affiliation=null, department=null, xref=null, address=
1Department of Computer Science and Technology, Tsinghua University, Beijing 100084, bio={"content":"
born in 1997. PhD candidate. Student member of CCF. His main research interests include storage systems and machine learning systems
"}, bioImg=null, bioContent=
born in 1997. PhD candidate. Student member of CCF. His main research interests include storage systems and machine learning systems
, aboutCorrespAuthor=null), CN=AuthorExt(id=1329456302364930050, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1329456295243002829, authorId=1329456302201353216, language=CN, stringName=谢旻晖, firstName=null, middleName=null, lastName=null, prefix=null, suffix=null, authorComment=null, nameInitials=null, affiliation=null, department=null, xref=null, address=
1清华大学计算机科学与技术系 北京 100084, bio={"img":"DEoyQdQMUUCiASTquF0rKQ==","content":"
1997年生. 博士研究生. CCF学生会员. 主要研究方向为存储系统、机器学习系统
"}, bioImg=DEoyQdQMUUCiASTquF0rKQ==, bioContent=
1997年生. 博士研究生. CCF学生会员. 主要研究方向为存储系统、机器学习系统
, aboutCorrespAuthor=null)}, companyList=[AuthorCompany(id=1329456300125172722, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1329456295243002829, xref=1, ext=[AuthorCompanyExt(id=1329456300133561331, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1329456295243002829, companyId=1329456300125172722, language=EN, country=null, province=null, city=null, postcode=null, companyName=null, departmentName=null, remark=
1Department of Computer Science and Technology, Tsinghua University, Beijing 100084), AuthorCompanyExt(id=1329456300150338548, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1329456295243002829, companyId=1329456300125172722, language=CN, country=null, province=null, city=null, postcode=null, companyName=null, departmentName=null, remark=
1清华大学计算机科学与技术系 北京 100084)])]), Author(id=1329456302440427524, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1329456295243002829, orderNo=3, firstName=null, middleName=null, lastName=null, nameCn=null, orcid=null, stid=null, country=null, authorPic=null, dead=0, email=shujw@tsinghua.edu.cn, emailSecond=null, emailThird=null, correspondingAuthor=1, authorType=1, ext={EN=AuthorExt(id=1329456302494953478, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1329456295243002829, authorId=1329456302440427524, language=EN, stringName=Jiwu Shu, firstName=Jiwu, middleName=null, lastName=Shu, prefix=null, suffix=null, authorComment=null, nameInitials=null, affiliation=null, department=null, xref=*, address=
1Department of Computer Science and Technology, Tsinghua University, Beijing 100084, bio={"content":"
born in 1968. PhD, professor, PhD supervisor. Fellow of CCF. His main research interests include intelligent storage systems, non-volatile memory storage systems and technologies, storage security and reliability, and parallel and distributed computing
"}, bioImg=null, bioContent=
born in 1968. PhD, professor, PhD supervisor. Fellow of CCF. His main research interests include intelligent storage systems, non-volatile memory storage systems and technologies, storage security and reliability, and parallel and distributed computing
, aboutCorrespAuthor=null), CN=AuthorExt(id=1329456302566256647, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1329456295243002829, authorId=1329456302440427524, language=CN, stringName=舒继武, firstName=null, middleName=null, lastName=null, prefix=null, suffix=null, authorComment=null, nameInitials=null, affiliation=null, department=null, xref=*, address=
1清华大学计算机科学与技术系 北京 100084, bio={"img":"QTyjUK6BiGcMFGNKW1sCGw==","content":"
1968年生. 博士,教授,博士生导师. CCF会士. 主要研究方向为智能存储系统、非易失内存存储系统与技术、存储安全与可靠性、并行与分布式计算
"}, bioImg=QTyjUK6BiGcMFGNKW1sCGw==, bioContent=
1968年生. 博士,教授,博士生导师. CCF会士. 主要研究方向为智能存储系统、非易失内存存储系统与技术、存储安全与可靠性、并行与分布式计算
, aboutCorrespAuthor=null)}, companyList=[AuthorCompany(id=1329456300125172722, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1329456295243002829, xref=1, ext=[AuthorCompanyExt(id=1329456300133561331, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1329456295243002829, companyId=1329456300125172722, language=EN, country=null, province=null, city=null, postcode=null, companyName=null, departmentName=null, remark=
1Department of Computer Science and Technology, Tsinghua University, Beijing 100084), AuthorCompanyExt(id=1329456300150338548, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1329456295243002829, companyId=1329456300125172722, language=CN, country=null, province=null, city=null, postcode=null, companyName=null, departmentName=null, remark=
1清华大学计算机科学与技术系 北京 100084)])])], keywords=[Keyword(id=1329456302666919944, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1329456295243002829, language=EN, orderNo=1, keyword=ChatGPT), Keyword(id=1329456302738223113, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1329456295243002829, language=EN, orderNo=2, keyword=large model), Keyword(id=1329456302830497802, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1329456295243002829, language=EN, orderNo=3, keyword=storage system), Keyword(id=1329456302885023755, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1329456295243002829, language=EN, orderNo=4, keyword=fault-tolerant), Keyword(id=1329456302947938316, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1329456295243002829, language=EN, orderNo=5, keyword=large model training system), Keyword(id=1329456303010852877, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1329456295243002829, language=CN, orderNo=1, keyword=ChatGPT), Keyword(id=1329456303073767438, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1329456295243002829, language=CN, orderNo=2, keyword=大模型), Keyword(id=1329456303149264911, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1329456295243002829, language=CN, orderNo=3, keyword=存储系统), Keyword(id=1329456303237345296, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1329456295243002829, language=CN, orderNo=4, keyword=容错), Keyword(id=1329456303300259857, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1329456295243002829, language=CN, orderNo=5, keyword=大模型训练系统)], refs=[Reference(id=1329456307989491764, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1329456295243002829, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=1, rfOrder=0, authorNames=null, journalName=null, refType=null, unstructuredReference=Vaswani A, Shazeer N, Parmar N, et al. Attention is all you need [C/OL] //Proc of the 30th Conf on Neural Information Processing Systems. Cambridge, MA: MIT, 2017[2023-05-30].https://proceedings.neurips.cc/paper/2017/file/3f5ee243547dee91fbd053c1c4a845aa-Paper.pdf, articleTitle=null, refAbstract=null), Reference(id=1329456308069183541, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1329456295243002829, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=2, rfOrder=1, authorNames=null, journalName=null, refType=null, unstructuredReference=Devlin J, Chang M, Lee K, et al. BERT: Pre-training of deep bidirectional transformers for language understanding [J]. arXiv preprint, arXiv: 1810.04805, 2018, articleTitle=null, refAbstract=null), Reference(id=1329456308136292406, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1329456295243002829, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=3, rfOrder=2, authorNames=null, journalName=null, refType=null, unstructuredReference=OpenAI. GPT-4 technical report [J]. arXiv preprint, arXiv: 2303.08774, 2023, articleTitle=null, refAbstract=null), Reference(id=1329456308203401271, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1329456295243002829, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=4, rfOrder=3, authorNames=null, journalName=null, refType=null, unstructuredReference=Bae J, Lee J, Jin Y, et al. FlashNeuron: SSD-enabled large-batch training of very deep neural networks[C]//Proc of the 19th USENIX Conf on File and Storage Technologies. Berkeley, CA: USENIX Association, 2021: 387-401, articleTitle=null, refAbstract=null), Reference(id=1329456308266315832, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1329456295243002829, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=5, rfOrder=4, authorNames=null, journalName=null, refType=null, unstructuredReference=Ruder S. An overview of gradient descent optimization algorithms[J]. arXiv preprint, arXiv: 1609.04747, 2016, articleTitle=null, refAbstract=null), Reference(id=1329456308350201913, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1329456295243002829, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=6, rfOrder=5, authorNames=null, journalName=null, refType=null, unstructuredReference=Kingma D, Ba J. Adam: A method for stochastic optimization[J]. arXiv preprint, arXiv: 1412.6980, 2014, articleTitle=null, refAbstract=null), Reference(id=1329456308408922170, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1329456295243002829, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=7, rfOrder=6, authorNames=null, journalName=null, refType=null, unstructuredReference=Thorpe J, Zhao Pengzhan, Eyolfson J, et al. Bamboo: Making preemptible instances resilient for affordable training of large DNNs[C]//Proc of the 20th USENIX Symp on Networked Systems Design and Implementation. Berkeley, CA: USENIX Association, 2023: 497-513, articleTitle=null, refAbstract=null), Reference(id=1329456308476031035, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1329456295243002829, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=8, rfOrder=7, authorNames=null, journalName=null, refType=null, unstructuredReference=Zhang Susan, Roller S, Goyal N, et al. OPT: Open pre-trained transformer language models[J]. arXiv preprint, arXiv: 2205.01068, 2022, articleTitle=null, refAbstract=null), Reference(id=1329456308547334204, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1329456295243002829, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=9, rfOrder=8, authorNames=null, journalName=null, refType=null, unstructuredReference=Jeon M, Venkataraman S, Phanishayee A, et al. Analysis of large-scale multi-tenant GPU clusters for DNN training workloads[C]// Proc of USENIX Annual Technical Conf. Berkeley, CA: USENIX Association, 2019: 947-960, articleTitle=null, refAbstract=null), Reference(id=1329456308606054461, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1329456295243002829, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=10, rfOrder=9, authorNames=null, journalName=null, refType=null, unstructuredReference=Shvachko K, Kuang H, Radia S, et al. The Hadoop distributed file system[C/OL]//Proc of the 26th IEEE Symp on Mass Storage Systems and Technologies. Piscataway, NJ: IEEE, 2010[2023-05-21].https://www.computer.org/csdl/proceedings-article/msst/2010/05496972/12OmNwxlrhU, articleTitle=null, refAbstract=null), Reference(id=1329456308673163326, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1329456295243002829, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=11, rfOrder=10, authorNames=null, journalName=null, refType=null, unstructuredReference=Zaharia M, Chowdhury M, Franklin M J, et al. Spark: Cluster computing with working sets[C/OL]//Proc of the 2nd USENIX Workshop on Hot Topics in Cloud Computing. Berkeley, CA: USENIX Association, 2010[2023-05-21].https://www.usenix.org/legacy/event/hotcloud10/tech/full_papers/Zaharia.pdf, articleTitle=null, refAbstract=null), Reference(id=1329456308769632319, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1329456295243002829, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=12, rfOrder=11, authorNames=null, journalName=null, refType=null, unstructuredReference=Weil S, Brandt S, Miller E, et al. Ceph: A scalable, high-performance distributed file system[C]//Proc of the 7th Symp on Operating Systems Design and Implementation. Berkeley, CA: USENIX Association, 2006: 307−320, articleTitle=null, refAbstract=null), Reference(id=1329456308840935488, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1329456295243002829, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=13, rfOrder=12, authorNames=null, journalName=null, refType=null, unstructuredReference=Rajbhandari S, Rasley J, Ruwase O, et al. ZeRO: Memory optimizations toward training trillion parameter models[C/OL]//Proc of the Int Conf for High Performance Computing, Networking, Storage and Analysis. Piscataway, NJ: IEEE, 2020[2023-05-21].https://dl.acm.org/doi/pdf/10.5555/3433701.3433727, articleTitle=null, refAbstract=null), Reference(id=1329456308903850049, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1329456295243002829, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=14, rfOrder=13, authorNames=null, journalName=null, refType=null, unstructuredReference=Huang Yanping, Cheng Youlong, Bapna A, et al. GPipe: Efficient training of giant neural networks using pipeline parallelism[C/OL]// Proc of the 33rd Conf on Neural Information Processing Systems. Cambridge, MA: MIT, 2019[2023-05-30].https://proceedings.neurips.cc/paper_files/paper/2019/file/093f65e080a295f8076b1c5722a46aa2-Paper.pdf, articleTitle=null, refAbstract=null), Reference(id=1329456308970958914, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1329456295243002829, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=15, rfOrder=14, authorNames=null, journalName=null, refType=null, unstructuredReference=Narayanan D, Harlap A, Phanishayee A, et al. PipeDream: Generalized pipeline parallelism for DNN training[C/OL]//Proc of the 27th ACM Symp on Operating Systems Principles. New York: ACM, 2019[2023-05-20].https://dl.acm.org/doi/pdf/10.1145/3341301.3359646, articleTitle=null, refAbstract=null), Reference(id=1329456309042262083, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1329456295243002829, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=16, rfOrder=15, authorNames=null, journalName=null, refType=null, unstructuredReference=Jain A, Awan A, Aljuhani A, et al. GEMS: GPU-enabled memory-aware model-parallelism system for distributed DNN training[C/OL]// Proc of the Int Conf for High Performance Computing, Networking, Storage and Analysis. Piscataway, NJ: IEEE, 2020[2023-05-21].https://ieeexplore.ieee.org/stamp/stamp.jsp?tp=&arnumber=9355254, articleTitle=null, refAbstract=null), Reference(id=1329456309105176644, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1329456295243002829, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=17, rfOrder=16, authorNames=null, journalName=null, refType=null, unstructuredReference=Narayanan D, Phanishayee A, Shi Kaiyu, et al. Memory-efficient pipeline-parallel DNN training[C]//Proc of the 38th Int Conf on Machine Learning Research. New York: PMLR, 2021: 7937−7947, articleTitle=null, refAbstract=null), Reference(id=1329456309168091205, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1329456295243002829, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=18, rfOrder=17, authorNames=null, journalName=null, refType=null, unstructuredReference=Fan Shiqing, Rong Yi, Meng Chen, et al. DAPPLE: A pipelined data parallel approach for training large models[C]//Proc of the 26th ACM SIGPLAN Symp on Principles and Practice of Parallel Programming. New York: ACM, 2021: 431−445, articleTitle=null, refAbstract=null), Reference(id=1329456309243588678, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1329456295243002829, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=19, rfOrder=18, authorNames=null, journalName=null, refType=null, unstructuredReference=Li Shigang, Hoefler T. Chimera: Efficiently training large-scale neural networks with bidirectional pipelines[C/OL]// Proc of the Int Conf for High Performance Computing, Networking, Storage and Analysis. Piscataway, NJ: IEEE, 2020[2023-05-21]. https://dl.acm.org/doi/abs/10.1145/3458817.3476145, articleTitle=null, refAbstract=null), Reference(id=1329456309306503239, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1329456295243002829, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=20, rfOrder=19, authorNames=null, journalName=null, refType=null, unstructuredReference=Shoeybi M, Patwary M, Puri R, et al. Megatron-LM: Training multi-billion parameter language models using model parallelism[J]. arXiv preprint, arXiv: 1909.08053, 2019, articleTitle=null, refAbstract=null), Reference(id=1329456310912921672, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1329456295243002829, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=21, rfOrder=20, authorNames=null, journalName=null, refType=null, unstructuredReference=Xu Qifan, You Yang. An efficient 2D method for training super-large deep learning models[C]//Proc of Int Symp on Parallel and Distributed Processing. Piscataway, NJ: IEEE, 2021: 222−232, articleTitle=null, refAbstract=null), Reference(id=1329456310984224841, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1329456295243002829, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=22, rfOrder=21, authorNames=null, journalName=null, refType=null, unstructuredReference=Wang Boxiang, Xu Qifan, Bian Zhengda, et al. Tesseract: Parallelize the tensor parallelism efficiently[C/OL]//Proc of the 51st Int Conf on Parallel Processing. New York: ACM, 2022[2023-05-21]. https://dl.acm.org/doi/abs/10.1145/3545008.3545087, articleTitle=null, refAbstract=null), Reference(id=1329456311047139402, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1329456295243002829, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=23, rfOrder=22, authorNames=null, journalName=null, refType=null, unstructuredReference=Bian Zhengda, Xu Qifan, Wang Boxiang, et al. Maximizing parallelism in distributed training for huge neural networks[J]. arXiv preprint, arXiv: 2105.14450, 2021, articleTitle=null, refAbstract=null), Reference(id=1329456311110053963, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1329456295243002829, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=24, rfOrder=23, authorNames=null, journalName=null, refType=null, unstructuredReference=Narayanan D, Shoeybi M, Casper J, et al. Efficient large-scale language model training on GPU clusters using Megatron-LM[C/OL]// Proc of the Int Conf for High Performance Computing, Networking, Storage and Analysis. Piscataway, NJ: IEEE, 2021[2023-05-21].https://dl.acm.org/doi/abs/10.1145/3458817.3476209, articleTitle=null, refAbstract=null), Reference(id=1329456311177162828, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1329456295243002829, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=25, rfOrder=24, authorNames=null, journalName=null, refType=null, unstructuredReference=Fang Jiarui, Zhu Zilin, Li Shenggui, et al. Parallel training of pre-trained models via chunk-based dynamic memory management[J]. IEEE Transactions on Parallel and Distributed Systems, 2022, 34(1): 304−315, articleTitle=null, refAbstract=null), Reference(id=1329456311273631821, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1329456295243002829, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=26, rfOrder=25, authorNames=null, journalName=null, refType=null, unstructuredReference=Rhu M, Gimelshein N, Clemons J, et al. vDNN: Virtualized deep neural networks for scalable, memory-efficient neural network design[C/OL]//Proc of the 49th Annual IEEE/ACM Int Symp on Microarchitecture. Piscataway, NJ: IEEE, 2016[2023-05-21]. https://ieeexplore.ieee.org/abstract/document/7783721, articleTitle=null, refAbstract=null), Reference(id=1329456311344934990, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1329456295243002829, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=27, rfOrder=26, authorNames=null, journalName=null, refType=null, unstructuredReference=Wang Linna, Ye Jinmian, Zhao Yiyang, et al. SuperNeurons: Dynamic GPU memory management for training deep neural networks[C]//Proc of the 23rd ACM SIGPLAN Symp on Principles and Practice of Parallel Programming. New York: ACM, 2018: 41−53, articleTitle=null, refAbstract=null), Reference(id=1329456311424626767, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1329456295243002829, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=28, rfOrder=27, authorNames=null, journalName=null, refType=null, unstructuredReference=Huang C, Jin Gu, Li Jinyang. SwapAdvisor: Pushing deep learning beyond the GPU memory limit via smart swapping[C]//Proc of the 25th Int Conf on Architectural Support for Programming Languages and Operating Systems. New York: ACM, 2020: 1341−1355, articleTitle=null, refAbstract=null), Reference(id=1329456311508512848, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1329456295243002829, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=29, rfOrder=28, authorNames=null, journalName=null, refType=null, unstructuredReference=Peng Xuan, Shi Xuanhua, Dai Hulin, et al. Capuchin: Tensor-based GPU memory management for deep learning[C]//Proc of the 25th Int Conf on Architectural Support for Programming Languages and Operating Systems. New York: ACM, 2020: 891−905, articleTitle=null, refAbstract=null), Reference(id=1329456311592398929, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1329456295243002829, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=30, rfOrder=29, authorNames=null, journalName=null, refType=null, unstructuredReference=Ren Jie, Rajbhandari S, Aminabadi R Y, et al. ZeRO-Offload: Democratizing billion-scale model training[C]//Proc of USENIX Annual Technical Conf. Berkeley, CA: USENIX Association, 2021: 551−564, articleTitle=null, refAbstract=null), Reference(id=1329456311667896402, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1329456295243002829, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=31, rfOrder=30, authorNames=null, journalName=null, refType=null, unstructuredReference=Li Youjie, Phanishayee A, Murray D, et al. Harmony: Overcoming the hurdles of GPU memory capacity to train massive DNN models on commodity servers[J]. arXiv preprint, arXiv: 2202.01306, 2022, articleTitle=null, refAbstract=null), Reference(id=1329456311735005267, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1329456295243002829, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=32, rfOrder=31, authorNames=null, journalName=null, refType=null, unstructuredReference=Feng Yangyang, Xie Minhui, Tian Zijie, et al. Mobius: Fine tuning large-scale models on commodity GPU servers[C]//Proc of the 28th ACM Int Conf on Architectural Support for Programming Languages and Operating Systems. New York: ACM, 2023: 489−501, articleTitle=null, refAbstract=null), Reference(id=1329456311810502740, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1329456295243002829, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=33, rfOrder=32, authorNames=null, journalName=null, refType=null, unstructuredReference=Rajbhandari S, Ruwase O, Rasley J, et al. ZeRO-Infinity: Breaking the GPU memory wall for extreme scale deep learning[C/OL]// Proc of the Int Conf for High Performance Computing, Networking, Storage and Analysis. Piscataway, NJ: IEEE, 2021[2023-05-21].https://dl.acm.org/doi/abs/10.1145/3458817.3476205, articleTitle=null, refAbstract=null), Reference(id=1329456311877611605, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1329456295243002829, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=34, rfOrder=33, authorNames=null, journalName=null, refType=null, unstructuredReference=Buluç A, Fineman J T, Frigo M, et al. Parallel sparse matrix-vector and matrix-transpose-vector multiplication using compressed sparse blocks[C]//Proc of the 21st Annual Symp on Parallelism in Algorithms and Architectures. New York: ACM, 2009: 233−244, articleTitle=null, refAbstract=null), Reference(id=1329456311953109078, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1329456295243002829, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=35, rfOrder=34, authorNames=null, journalName=null, refType=null, unstructuredReference=Scipy. scipy. sparse. coo_matrix[EB/OL]. 2023 [2023-06-29].https://docs.scipy.org/doc/scipy/reference/generated/scipy.sparse.coo_matrix.html, articleTitle=null, refAbstract=null), Reference(id=1329456312011829335, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1329456295243002829, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=36, rfOrder=35, authorNames=null, journalName=null, refType=null, unstructuredReference=Scipy. scipy. sparse. lil_matrix[EB/OL]. 2023 [2023-06-29].https://docs.scipy.org/doc/scipy/reference/generated/scipy.sparse.lil_matrix.html, articleTitle=null, refAbstract=null), Reference(id=1329456312070549592, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1329456295243002829, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=37, rfOrder=36, authorNames=null, journalName=null, refType=null, unstructuredReference=Fedus W, Zoph B, Shazeer N. Switch transformers: Scaling to trillion parameter models with simple and efficient sparsity[J]. The Journal of Machine Learning Research, 2022, 23(1): 5232−5270, articleTitle=null, refAbstract=null), Reference(id=1329456312129269849, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1329456295243002829, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=38, rfOrder=37, authorNames=null, journalName=null, refType=null, unstructuredReference=Chen Tianqi, Xu Bing, Zhang Chiyuan, et al. Training deep nets with sublinear memory cost[J]. arXiv preprint, arXiv: 1604.06174, 2016, articleTitle=null, refAbstract=null), Reference(id=1329456312192184410, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1329456295243002829, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=39, rfOrder=38, authorNames=null, journalName=null, refType=null, unstructuredReference=Jain P, Jain A, Nrusimha A, et al. Checkmate: Breaking the memory wall with optimal tensor rematerialization[C/OL]// Proc of the 3rd Machine Learning and Systems. 2020[2023-05-20].https://proceedings.mlsys.org/paper_files/paper/2020/file/0b816ae8f06f8dd3543dc3d9ef196cab-Paper.pdf, articleTitle=null, refAbstract=null), Reference(id=1329456312263487579, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1329456295243002829, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=40, rfOrder=39, authorNames=null, journalName=null, refType=null, unstructuredReference=Micikevicius P, Narang S, Alben J, et al. Mixed precision training[J]. arXiv preprint, arXiv: 1710.03740, 2017, articleTitle=null, refAbstract=null), Reference(id=1329456312355762268, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1329456295243002829, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=41, rfOrder=40, authorNames=null, journalName=null, refType=null, unstructuredReference=Chen Jianfei, Zheng Lianmin, Yao Zhewei, et al. ActNN: Reducing training memory footprint via 2-bit activation compressed training[C]//Proc of the 38th Int Conf on Machine Learning Research. New York: PMLR, 2021: 1803−1813, articleTitle=null, refAbstract=null), Reference(id=1329456312414482525, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1329456295243002829, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=42, rfOrder=41, authorNames=null, journalName=null, refType=null, unstructuredReference=Mohan J, Phanishayee A, Chidambaram V. CheckFreq: Frequent, fine-frained DNN checkpointing[C]//Proc of the 19th USENIX Conf on File and Storage Technologies. Berkeley, CA: USENIX Association, 2021: 203−216, articleTitle=null, refAbstract=null), Reference(id=1329456312481591390, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1329456295243002829, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=43, rfOrder=42, authorNames=null, journalName=null, refType=null, unstructuredReference=NVIDIA. Megatron-LM [EB/OL]. 2023 [2023-06-01].https://github.com/NVIDIA/Megatron-LM, articleTitle=null, refAbstract=null), Reference(id=1329456312548700255, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1329456295243002829, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=44, rfOrder=43, authorNames=null, journalName=null, refType=null, unstructuredReference=Microsoft. DeepSpeed [EB/OL]. 2023 [2023-06-10].https://github.com/microsoft/DeepSpeed, articleTitle=null, refAbstract=null), Reference(id=1329456312624197728, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1329456295243002829, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=45, rfOrder=44, authorNames=null, journalName=null, refType=null, unstructuredReference=HPCAITech. Colossal-AI [EB/OL]. 2023 [2023-06-10].https://github.com/hpcaitech/ColossalAI, articleTitle=null, refAbstract=null), Reference(id=1329456312695500897, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1329456295243002829, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=46, rfOrder=45, authorNames=null, journalName=null, refType=null, unstructuredReference=OneFlow Inc. OneFlow [EB/OL]. 2023 [2023-06-10].https://github.com/Oneflow-Inc/oneflow, articleTitle=null, refAbstract=null)], funds=null, companyList=[AuthorCompany(id=1329456300125172722, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1329456295243002829, xref=1, ext=[AuthorCompanyExt(id=1329456300133561331, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1329456295243002829, companyId=1329456300125172722, language=EN, country=null, province=null, city=null, postcode=null, companyName=null, departmentName=null, remark=
1Department of Computer Science and Technology, Tsinghua University, Beijing 100084), AuthorCompanyExt(id=1329456300150338548, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1329456295243002829, companyId=1329456300125172722, language=CN, country=null, province=null, city=null, postcode=null, companyName=null, departmentName=null, remark=
1清华大学计算机科学与技术系 北京 100084)])], figs=[ArticleFig(id=1329456303405117458, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1329456295243002829, language=EN, label=Figure 1, caption=
Increase of model size, figureFileSmall=OztYgH7iHe1bdW3ojicpXQ==, figureFileBig=J19neWKpt7ihI4y44eBBiw==, tableContent=null), ArticleFig(id=1329456303468032019, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1329456295243002829, language=CN, label=图1, caption=
模型规模增长, figureFileSmall=OztYgH7iHe1bdW3ojicpXQ==, figureFileBig=J19neWKpt7ihI4y44eBBiw==, tableContent=null), ArticleFig(id=1329456303686135828, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1329456295243002829, language=EN, label=Figure 2, caption=
Illustration of model partition in ZeRO-DP [13], figureFileSmall=brA2RXgWfEYVsmsOJoEWww==, figureFileBig=q3dXZDV1yx6ZuqllbNPF4w==, tableContent=null), ArticleFig(id=1329456303749050389, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1329456295243002829, language=CN, label=图2, caption=
ZeRO-DP模型划分示意图[13], figureFileSmall=brA2RXgWfEYVsmsOJoEWww==, figureFileBig=q3dXZDV1yx6ZuqllbNPF4w==, tableContent=null), ArticleFig(id=1329456303820353558, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1329456295243002829, language=EN, label=Figure 3, caption=
Illustration of GPipe[14] and PipeDream[15], figureFileSmall=dmHClbNUeE5GHSb61pSb2A==, figureFileBig=lYrF79VZglXm1+tKvY51uw==, tableContent=null), ArticleFig(id=1329456303874879511, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1329456295243002829, language=CN, label=图3, caption=
GPipe[14]和PipeDream[15]示意图, figureFileSmall=dmHClbNUeE5GHSb61pSb2A==, figureFileBig=lYrF79VZglXm1+tKvY51uw==, tableContent=null), ArticleFig(id=1329456303941988376, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1329456295243002829, language=EN, label=Figure 4, caption=
Tensor splitting[20], figureFileSmall=5s3yNfghRNAyS40dhJ56bg==, figureFileBig=1A+RIBL8WxTuPwQeqxq2Xw==, tableContent=null), ArticleFig(id=1329456304017485849, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1329456295243002829, language=CN, label=图4, caption=
张量切分[20], figureFileSmall=5s3yNfghRNAyS40dhJ56bg==, figureFileBig=1A+RIBL8WxTuPwQeqxq2Xw==, tableContent=null), ArticleFig(id=1329456304097177626, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1329456295243002829, language=EN, label=Figure 5, caption=
Illustration of 1D tensor parallelism in Megaton[24], figureFileSmall=2WlNgaO1gVxtpWt3z76l0Q==, figureFileBig=utqLPHv0PPjay8seXTb2Ig==, tableContent=null), ArticleFig(id=1329456304155897883, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1329456295243002829, language=CN, label=图5, caption=
Megatron的1D张量并行示意图[24], figureFileSmall=2WlNgaO1gVxtpWt3z76l0Q==, figureFileBig=utqLPHv0PPjay8seXTb2Ig==, tableContent=null), ArticleFig(id=1329456304214618140, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1329456295243002829, language=EN, label=Figure 6, caption=
Illustration of ZeRO-Offload[30], figureFileSmall=JLcoT9qK8422acOlubkYuw==, figureFileBig=bjQTqrRT42n7PV4q9caUTg==, tableContent=null), ArticleFig(id=1329456304277532701, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1329456295243002829, language=CN, label=图6, caption=
ZeRO-Offload 示意图[30], figureFileSmall=JLcoT9qK8422acOlubkYuw==, figureFileBig=bjQTqrRT42n7PV4q9caUTg==, tableContent=null), ArticleFig(id=1329456304348835870, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1329456295243002829, language=EN, label=Figure 7, caption=
Data center GPU server and commodity GPU server[32], figureFileSmall=Ajh9YLPhIY9C0S0wnhXjYA==, figureFileBig=M0PKVjSe12i9O1jZ2BAc4w==, tableContent=null), ArticleFig(id=1329456304424333343, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1329456295243002829, language=CN, label=图7, caption=
数据中心GPU服务器和消费级GPU服务器[32], figureFileSmall=Ajh9YLPhIY9C0S0wnhXjYA==, figureFileBig=M0PKVjSe12i9O1jZ2BAc4w==, tableContent=null), ArticleFig(id=1329456304516608032, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1329456295243002829, language=EN, label=Figure 8, caption=
Illustration of offloading strategy in FlashNeuron[4], figureFileSmall=e76ABuUJqv9o+bGTWNre/w==, figureFileBig=ws26kDWbN6Fz1QMyvuz9hg==, tableContent=null), ArticleFig(id=1329456304579522593, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1329456295243002829, language=CN, label=图8, caption=
FlashNeuron 卸载策略示意图[4], figureFileSmall=e76ABuUJqv9o+bGTWNre/w==, figureFileBig=ws26kDWbN6Fz1QMyvuz9hg==, tableContent=null), ArticleFig(id=1329456304646631458, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1329456295243002829, language=EN, label=Figure 9, caption=
Illustration of ZeRO-Infinity[33], figureFileSmall=PTYXhIARJMJqU4LQCn0S1A==, figureFileBig=IcGQ1HJlWFXMizDZ3gW4hQ==, tableContent=null), ArticleFig(id=1329456304793432099, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1329456295243002829, language=CN, label=图9, caption=
ZeRO-Infinity示意图[33], figureFileSmall=PTYXhIARJMJqU4LQCn0S1A==, figureFileBig=IcGQ1HJlWFXMizDZ3gW4hQ==, tableContent=null), ArticleFig(id=1329456304852152356, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1329456295243002829, language=EN, label=Figure 10, caption=
Illustration of mixed precision training[40], figureFileSmall=WrjKUZNnlqRk5Fs1JSfrmA==, figureFileBig=HFzU/fwKV4d5rlARdMpZ4Q==, tableContent=null), ArticleFig(id=1329456306471153701, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1329456295243002829, language=CN, label=图10, caption=
混合精度训练示意图[40], figureFileSmall=WrjKUZNnlqRk5Fs1JSfrmA==, figureFileBig=HFzU/fwKV4d5rlARdMpZ4Q==, tableContent=null), ArticleFig(id=1329456306588594214, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1329456295243002829, language=EN, label=Figure 11, caption=
Usage scenarios of various large model training storage technologies, figureFileSmall=ize2ojiuDzEWmfr08Lw+sA==, figureFileBig=W/L5gaFrDzIG/7toI4YiHA==, tableContent=null), ArticleFig(id=1329456306668285991, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1329456295243002829, language=CN, label=图11, caption=
各类大模型训练存储技术使用场景, figureFileSmall=ize2ojiuDzEWmfr08Lw+sA==, figureFileBig=W/L5gaFrDzIG/7toI4YiHA==, tableContent=null), ArticleFig(id=1329456306739589160, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1329456295243002829, language=EN, label=Figure 12, caption=
Illustration of CheckFreq[42], figureFileSmall=vqG/r7wqTQ2As71J3lpbPQ==, figureFileBig=ekGvd1CJsiw/AkpP49gHGg==, tableContent=null), ArticleFig(id=1329456306840252457, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1329456295243002829, language=CN, label=图12, caption=
CheckFreq 示意图[42], figureFileSmall=vqG/r7wqTQ2As71J3lpbPQ==, figureFileBig=ekGvd1CJsiw/AkpP49gHGg==, tableContent=null), ArticleFig(id=1329456306940915754, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1329456295243002829, language=EN, label=Figure 13, caption=
Bubbles in pipeline parallelism[7], figureFileSmall=wjpGKAbSdfM9yqKfFCEALg==, figureFileBig=LMR7c6uMz5HiifncmSxz9g==, tableContent=null), ArticleFig(id=1329456307058356267, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1329456295243002829, language=CN, label=图13, caption=
流水线并行的气泡[7], figureFileSmall=wjpGKAbSdfM9yqKfFCEALg==, figureFileBig=LMR7c6uMz5HiifncmSxz9g==, tableContent=null), ArticleFig(id=1329456307138048044, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1329456295243002829, language=EN, label=Table 1, caption=
Comparison of Large Model Training Schemes Based on Different Pipeline Modes
, figureFileSmall=null, figureFileBig=null, tableContent=
| 系统 | 显存开销 | 气泡 | 收敛性 |
|---|
| GEMS[16] | 中 | 差 | 好 |
| GPipe[14] | 低 | 差 | 好 |
| PipeDream[15] | 高 | 好 | 差 |
| PipeDream2BW[17] | 中 | 好 | 差 |
| DAPPLE[18] | 低 | 差 | 好 |
| Chimera[19] | 中 | 中 | 好 |
), ArticleFig(id=1329456307276460077, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1329456295243002829, language=CN, label=表1, caption=
基于不同流水线模式的大模型训练方案对比
, figureFileSmall=null, figureFileBig=null, tableContent=
| 系统 | 显存开销 | 气泡 | 收敛性 |
|---|
| GEMS[16] | 中 | 差 | 好 |
| GPipe[14] | 低 | 差 | 好 |
| PipeDream[15] | 高 | 好 | 差 |
| PipeDream2BW[17] | 中 | 好 | 差 |
| DAPPLE[18] | 低 | 差 | 好 |
| Chimera[19] | 中 | 中 | 好 |
), ArticleFig(id=1329456307393900590, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1329456295243002829, language=EN, label=Table 2, caption=
Comparison of Four Types of Tensor Parallelism
, figureFileSmall=null, figureFileBig=null, tableContent=
张量并行 维度 | 参数张量 的存储开销 | 输入张量 的存储开销 | 传输参数张量 的通信开销 | 传输输入张量 的通信开销 |
|---|
| 1D | 低 | 高 | 无 | 中 |
| 2D | 低 | 低 | 中 | 中 |
| 2.5D | 中 | 低 | 中 | 低 |
| 3D | 低 | 低 | 低 | 低 |
), ArticleFig(id=1329456307544895535, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1329456295243002829, language=CN, label=表2, caption=
4种张量并行的对比
, figureFileSmall=null, figureFileBig=null, tableContent=
张量并行 维度 | 参数张量 的存储开销 | 输入张量 的存储开销 | 传输参数张量 的通信开销 | 传输输入张量 的通信开销 |
|---|
| 1D | 低 | 高 | 无 | 中 |
| 2D | 低 | 低 | 中 | 中 |
| 2.5D | 中 | 低 | 中 | 低 |
| 3D | 低 | 低 | 低 | 低 |
), ArticleFig(id=1329456307628781616, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1329456295243002829, language=EN, label=Table 3, caption=
Comparison of Different Large Model Training System Using Heterogeneous Storage Resources
, figureFileSmall=null, figureFileBig=null, tableContent=
| 系统 | 存储介质 | 并行策略 | 卸载模型参数 | 卸载优化器 | 卸载中间变量 |
|---|
| vDNN[26] | 显存+DRAM | | 否 | 否 | 是 |
| SuperNeurons[27] | 显存+DRAM | | 否 | 否 | 是(仅卸载卷积层) |
| SwapAdvisor[28] | 显存+DRAM | | 是 | 否 | 是 |
| Capuchin[29] | 显存+DRAM | | 否 | 否 | 是 |
| ZeRO-Offload[30] | 显存+DRAM | ZeRO-DP | 否 | 是 | 是 |
| Harmony[31] | 显存+DRAM | 流水线 | 是 | 是 | 是 |
| Mobius[32] | 显存+DRAM | 流水线 | 是 | 是 | 是 |
| FlashNeuron[4] | 显存+SSD | | 是 | 否 | 是 |
| ZeRO-Infinity[33] | 显存+DRAM+SSD | ZeRO-DP | 是 | 是 | 是 |
), ArticleFig(id=1329456307700084785, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1329456295243002829, language=CN, label=表3, caption=
不同使用异构存储资源的大模型训练系统对比
, figureFileSmall=null, figureFileBig=null, tableContent=
| 系统 | 存储介质 | 并行策略 | 卸载模型参数 | 卸载优化器 | 卸载中间变量 |
|---|
| vDNN[26] | 显存+DRAM | | 否 | 否 | 是 |
| SuperNeurons[27] | 显存+DRAM | | 否 | 否 | 是(仅卸载卷积层) |
| SwapAdvisor[28] | 显存+DRAM | | 是 | 否 | 是 |
| Capuchin[29] | 显存+DRAM | | 否 | 否 | 是 |
| ZeRO-Offload[30] | 显存+DRAM | ZeRO-DP | 否 | 是 | 是 |
| Harmony[31] | 显存+DRAM | 流水线 | 是 | 是 | 是 |
| Mobius[32] | 显存+DRAM | 流水线 | 是 | 是 | 是 |
| FlashNeuron[4] | 显存+SSD | | 是 | 否 | 是 |
| ZeRO-Infinity[33] | 显存+DRAM+SSD | ZeRO-DP | 是 | 是 | 是 |
), ArticleFig(id=1329456307792359474, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1329456295243002829, language=EN, label=Table 4, caption=
Comparison of Different Large Model Training Storage Technologies
, figureFileSmall=null, figureFileBig=null, tableContent=
| 存储技术 | 显存开销 | 通信开销 | 额外计算开销 |
|---|
基于模型层间并行的分布式显存管理技术 (以ZeRO-DP中Pos+g+p为例) | $ P+O+A+T $ | $ 3P\left(N-1\right) $ | |
基于模型层间并行的分布式显存管理技术 (以GPipe为例) | $ P+O+A+T $ | $ 2a(N-1) $ | |
基于模型层内并行的分布式显存管理技术 (以1D张量并行为例) | $ P+O+A+T $ | $ 2A(N-1) $ | |
| 基于DRAM的异构存储技术 | $ (1-\alpha )P+(1-\beta )O+(1-\gamma )A+T $ | $ 3\alpha P+\beta O+2\gamma A $ | |
| 基于SSD的异构存储技术 | $ (1-\alpha )P+(1-\beta )O+(1-\gamma )A+T $ | $ 3\alpha P+\beta O+2\gamma A $ | |
| 激活量检查点与重算技术 | $ P+O+\theta A+T $ | | 重算开销 |
| 混合精度技术 | $ pP+qO+kA+T $ | | 精度转化开销 |
), ArticleFig(id=1329456307876245555, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1329456295243002829, language=CN, label=表4, caption=
各种不同大模型训练存储技术的比较
, figureFileSmall=null, figureFileBig=null, tableContent=
| 存储技术 | 显存开销 | 通信开销 | 额外计算开销 |
|---|
基于模型层间并行的分布式显存管理技术 (以ZeRO-DP中Pos+g+p为例) | $ P+O+A+T $ | $ 3P\left(N-1\right) $ | |
基于模型层间并行的分布式显存管理技术 (以GPipe为例) | $ P+O+A+T $ | $ 2a(N-1) $ | |
基于模型层内并行的分布式显存管理技术 (以1D张量并行为例) | $ P+O+A+T $ | $ 2A(N-1) $ | |
| 基于DRAM的异构存储技术 | $ (1-\alpha )P+(1-\beta )O+(1-\gamma )A+T $ | $ 3\alpha P+\beta O+2\gamma A $ | |
| 基于SSD的异构存储技术 | $ (1-\alpha )P+(1-\beta )O+(1-\gamma )A+T $ | $ 3\alpha P+\beta O+2\gamma A $ | |
| 激活量检查点与重算技术 | $ P+O+\theta A+T $ | | 重算开销 |
| 混合精度技术 | $ pP+qO+kA+T $ | | 精度转化开销 |
)], attaches=null, journal=Journal(id=1324759606947315908, delFlag=0, nameCn=计算机研究与发展, nameEn=Journal of Computer Research and Development, nameHistory1=null, nameHistory2=null, issn=1000-1239, eissn=null, cn=11-1777/TP, coden=null, periodic=0, language=CN, oaType=null, ccby=null, superviseOffice=null, ownerOffice=null, pubOffice=null, editorOffice=null, officeType=null, aims=null, clcCode=null, officeProv=null, officeCity=null, officeAddr=null, officeZip=null, officeEmail=null, officePhone=null, editDirector=null, officeDirector=null, officeDirectorPhone=null, officeStaffNum=null, officeEmpNum=null, coverPicUrl=k2l8zgqwstuxxevFX2qxVA==, journalPrice=null, startedYear=null, abbrevIsoEn=Journal of Computer Research and Development, journalRemark=null, publicationField=null, createdTime=1793776903141, updatedTime=1793850698612, createdBy=18614031015, updatedBy=13701087609, firstLetterCn=J, firstLetterEn=J, subjectCode=Electronic Information, subjectName=null, subjectCodeEn=Electronic Information, subjectNameEn=null, picCn=k2l8zgqwstuxxevFX2qxVA==, picEn=r6k0Ry+RD+c/VXj5QarmDw==, jcr=null, cjcr=null, exts=[JournalExt(id=1325069127710286688, language=CN, name=计算机研究与发展, nameHistory1=null, nameHistory2=null, managedBy=, sponsoredBy=, publishedBy=, editorOffice=, officeProv=null, officeCity=null, officeAddr=, officeZip=, editDirector=, officeDirector=null, officePhone=null, coverPicUrl=null, journalRemark=, submitArticleUrl=null, websiteUrl=, createdTime=1793850698640, updatedTime=1793850698640, createdBy=13701087609, updatedBy=13701087609, submissionGuidelinesUrl=, submissionAuthorUrl=https://crad.ict.ac.cn/journalx/authorLogOn.action, submissionEditorUrl=https://crad.ict.ac.cn/journalx/editorLogOn.action, submissionReviewUrl=https://crad.ict.ac.cn/journalx/expertLogOn.action, submissionCeEditorUrl=https://crad.ict.ac.cn/journalx/editorInChiefLogOn.action, submissionAeEditorUrl=, option={"copyright":""}), JournalExt(id=1325069127777395553, language=EN, name=Journal of Computer Research and Development, nameHistory1=null, nameHistory2=null, managedBy=, sponsoredBy=, publishedBy=, editorOffice=, officeProv=null, officeCity=null, officeAddr=, officeZip=, editDirector=, officeDirector=null, officePhone=null, coverPicUrl=null, journalRemark=, submitArticleUrl=null, websiteUrl=, createdTime=1793850698656, updatedTime=1793850698656, createdBy=13701087609, updatedBy=13701087609, submissionGuidelinesUrl=, submissionAuthorUrl=https://crad.ict.ac.cn/journalx/authorLogOn.action, submissionEditorUrl=https://crad.ict.ac.cn/journalx/editorLogOn.action, submissionReviewUrl=https://crad.ict.ac.cn/journalx/expertLogOn.action, submissionCeEditorUrl=https://crad.ict.ac.cn/journalx/editorInChiefLogOn.action, submissionAeEditorUrl=, option={"copyright":""})], databaseList=null, tenantJournalId=1324759914146529478, websiteList=[Website(id=1325070300139892771, webName=null, webTitle=null, webDomain=null, webCopyrigh=null, webIpcNo=null, seoTitle=null, seoKeywords=null, seoDescription=null, tenantJournalId=null, journalId=1324759914146529478, journalNameCn=null, journalNameEn=null, grayFlag=null, tenantId=1146029695717560320, platformId=null, journalGroupId=null, journalGroupNameCn=null, journalGroupNameEn=null, type=1, domain=https://castjournals.cast.org.cn/joweb/jsjyjyfz/CN, language=CN, createTime=1793850978169, createBy=18614031015, updateTime=1793851040259, updateBy=18614031015, name=计算机研究与发展-中文, tplId=1146099689490845704, title=计算机研究与发展, delFlag=0, indexPage=/home, props=[WebsiteProps(id=1325070893751349360, tenantId=1146029695717560320, journalId=null, journalGroupId=null, siteId=1325070300139892771, code=articleTextType, value=kx, createTime=1793851119697, updateTime=1793851119697, creator=18614031015, updator=18614031015), WebsiteProps(id=1325070893730377837, tenantId=1146029695717560320, journalId=null, journalGroupId=null, siteId=1325070300139892771, code=banner, value=null, createTime=1793851119692, updateTime=1793851119692, creator=18614031015, updator=18614031015), WebsiteProps(id=1325070893772320883, tenantId=1146029695717560320, journalId=null, journalGroupId=null, siteId=1325070300139892771, code=grayFlag, value=0, createTime=1793851119702, updateTime=1793851119702, creator=18614031015, updator=18614031015), WebsiteProps(id=1325070893721989228, tenantId=1146029695717560320, journalId=null, journalGroupId=null, siteId=1325070300139892771, code=logo, value=https://castjournals.cast.org.cn/joweb/jsjyjyfz/CN/file/pic?fileId=U27tDsy717YF7XSPA+Vwdw==, createTime=1793851119690, updateTime=1793851119690, creator=18614031015, updator=18614031015), WebsiteProps(id=1325070893784903797, tenantId=1146029695717560320, journalId=null, journalGroupId=null, siteId=1325070300139892771, code=minRunFlag, value=0, createTime=1793851119705, updateTime=1793851119705, creator=18614031015, updator=18614031015), WebsiteProps(id=1325070893742960751, tenantId=1146029695717560320, journalId=null, journalGroupId=null, siteId=1325070300139892771, code=picServerUrl, value=https://castjournals.cast.org.cn/joweb/jsjyjyfz/CN/file/pic, createTime=1793851119695, updateTime=1793851119695, creator=18614031015, updator=18614031015), WebsiteProps(id=1325070893780709492, tenantId=1146029695717560320, journalId=null, journalGroupId=null, siteId=1325070300139892771, code=silenceFlag, value=0, createTime=1793851119704, updateTime=1793851119704, creator=18614031015, updator=18614031015), WebsiteProps(id=1325070893738766446, tenantId=1146029695717560320, journalId=null, journalGroupId=null, siteId=1325070300139892771, code=staticResourcePath, value=https://castjournals.cast.org.cn/joweb/cast_kjdb_cn_619/, createTime=1793851119694, updateTime=1793851119694, creator=18614031015, updator=18614031015), WebsiteProps(id=1325070893755543665, tenantId=1146029695717560320, journalId=null, journalGroupId=null, siteId=1325070300139892771, code=themeColor, value=null, createTime=1793851119698, updateTime=1793851119698, creator=18614031015, updator=18614031015), WebsiteProps(id=1325070893763932274, tenantId=1146029695717560320, journalId=null, journalGroupId=null, siteId=1325070300139892771, code=themeStyle, value=null, createTime=1793851119700, updateTime=1793851119700, creator=18614031015, updator=18614031015)]), Website(id=1325070300580294698, webName=null, webTitle=null, webDomain=null, webCopyrigh=null, webIpcNo=null, seoTitle=null, seoKeywords=null, seoDescription=null, tenantJournalId=null, journalId=1324759914146529478, journalNameCn=null, journalNameEn=null, grayFlag=null, tenantId=1146029695717560320, platformId=null, journalGroupId=null, journalGroupNameCn=null, journalGroupNameEn=null, type=1, domain=https://castjournals.cast.org.cn/joweb/jsjyjyfz/EN, language=EN, createTime=1793850978274, createBy=18614031015, updateTime=1793851036638, updateBy=18614031015, name=计算机研究与发展-英文, tplId=1146101810881728533, title=Journal of Computer Research and Development, delFlag=0, indexPage=/home, props=[WebsiteProps(id=1325070925288321148, tenantId=1146029695717560320, journalId=null, journalGroupId=null, siteId=1325070300580294698, code=articleTextType, value=kx, createTime=1793851127216, updateTime=1793851127216, creator=18614031015, updator=18614031015), WebsiteProps(id=1325070925263155321, tenantId=1146029695717560320, journalId=null, journalGroupId=null, siteId=1325070300580294698, code=banner, value=null, createTime=1793851127210, updateTime=1793851127210, creator=18614031015, updator=18614031015), WebsiteProps(id=1325070925309292671, tenantId=1146029695717560320, journalId=null, journalGroupId=null, siteId=1325070300580294698, code=grayFlag, value=0, createTime=1793851127221, updateTime=1793851127221, creator=18614031015, updator=18614031015), WebsiteProps(id=1325070925254766712, tenantId=1146029695717560320, journalId=null, journalGroupId=null, siteId=1325070300580294698, code=logo, value=https://castjournals.cast.org.cn/joweb/jsjyjyfz/EN/file/pic?fileId=U27tDsy717YF7XSPA+Vwdw==, createTime=1793851127208, updateTime=1793851127208, creator=18614031015, updator=18614031015), WebsiteProps(id=1325070925326069889, tenantId=1146029695717560320, journalId=null, journalGroupId=null, siteId=1325070300580294698, code=minRunFlag, value=0, createTime=1793851127225, updateTime=1793851127225, creator=18614031015, updator=18614031015), WebsiteProps(id=1325070925279932539, tenantId=1146029695717560320, journalId=null, journalGroupId=null, siteId=1325070300580294698, code=picServerUrl, value=https://castjournals.cast.org.cn/joweb/jsjyjyfz/EN/file/pic, createTime=1793851127214, updateTime=1793851127214, creator=18614031015, updator=18614031015), WebsiteProps(id=1325070925317681280, tenantId=1146029695717560320, journalId=null, journalGroupId=null, siteId=1325070300580294698, code=silenceFlag, value=0, createTime=1793851127223, updateTime=1793851127223, creator=18614031015, updator=18614031015), WebsiteProps(id=1325070925271543930, tenantId=1146029695717560320, journalId=null, journalGroupId=null, siteId=1325070300580294698, code=staticResourcePath, value=https://castjournals.cast.org.cn/joweb/cast_kjdb_en_623/, createTime=1793851127212, updateTime=1793851127212, creator=18614031015, updator=18614031015), WebsiteProps(id=1325070925292515453, tenantId=1146029695717560320, journalId=null, journalGroupId=null, siteId=1325070300580294698, code=themeColor, value=null, createTime=1793851127217, updateTime=1793851127217, creator=18614031015, updator=18614031015), WebsiteProps(id=1325070925300904062, tenantId=1146029695717560320, journalId=null, journalGroupId=null, siteId=1325070300580294698, code=themeStyle, value=null, createTime=1793851127219, updateTime=1793851127219, creator=18614031015, updator=18614031015)])], journalTitle=计算机研究与发展, weixinUrl=null, journalUrl=https://crad.ict.ac.cn/, iacademicId=null, status=1, seqNo=null, journalTitleEn=Journal of Computer Research and Development, journalPhotoCn=k2l8zgqwstuxxevFX2qxVA==, journalPhotoEn=r6k0Ry+RD+c/VXj5QarmDw==, journalFirstLetter=J, journalRecommend=null, journalNew=null, journalCollection=null, jcrJf=null, cjcrJf=null, jcrJfStr=null, cjcrJfStr=null, submissionFirstDecision=null, sciSubjectClassification=null, casSubjectClassification=null, citeScore=null, totalCitationFrequency=null, icpCode=null, psCode=null, advertisingLicenseCode=null, copyrightInformation=null, country=null, option=, provinceCode=null, provinceName=null, collectFlag=false, interPubPlatform=, interPubPlatformUrl=null), detailUrlCn=https://castjournals.cast.org.cn/joweb/jsjyjyfz/CN/10.7544/issn1000-1239.202330554, detailUrlEn=https://castjournals.cast.org.cn/joweb/jsjyjyfz/EN/10.7544/issn1000-1239.202330554, pdfUrlCn=https://castjournals.cast.org.cn/joweb/jsjyjyfz/CN/PDF/10.7544/issn1000-1239.202330554, pdfUrlEn=https://castjournals.cast.org.cn/joweb/jsjyjyfz/EN/PDF/10.7544/issn1000-1239.202330554, aliStartDate=0, aliEndDate=0, collectionFlag=false, citedCount=null, citedUrl=null, previewStatus=0, delFlag=0, hasFullText=1, orderTime=1711900800000, fullTextJson=null, articleText=null, reference=null)