Article(id=1325495741652803708, tenantId=1146029695717560320, journalId=1324759914146529478, issueId=1325495515563041259, articleNumber=null, orderNo=null, doi=10.7544/issn1000-1239.202550478, pmid=null, cstr=null, oa=null, hot=null, price=null, onlineType=0, articleFormat=0, articleType=null, articleTypeStr=research-article, receivedDate=1750089600000, receivedDateStr=2025-06-17, revisedDate=1760544000000, revisedDateStr=2025-10-16, acceptedDate=1761840000000, acceptedDateStr=2025-10-31, onlineDate=1793952411329, onlineDateStr=2026-11-06, pubDate=1772294400000, pubDateStr=2026-03-01, doiRegisterDate=null, doiRegisterDateStr=null, onlineIssueDate=1793952411329, onlineIssueDateStr=2026-11-06, onlineJustAcceptDate=null, onlineJustAcceptDateStr=null, onlineFirstDate=null, onlineFirstDateStr=null, sourceXml=null, magXml=null, createTime=1793952411329, creator=13701087609, updateTime=1793952411329, updator=13701087609, issue=Issue{id=1325495515563041259, tenantId=1146029695717560320, journalId=1324759914146529478, year='2026', volume='63', issue='3', pageStart='539', pageEnd='823', issueExtLink='null', onlineDate='null', pubDate='1772294400000', pubDateStr='2026-03-01', beforeIssueId=null, nextIssueId=null, price=null, status=1, issueComplete=1, articleOrder=1, issueType=-1, specialIssue=null, createTime=1793952357425, creator='13701087609', updateTime=1793952549954, updator='13701087609', preIssue=null, nextIssue=null, articleTotal=null, ext={EN=IssueExt(id=1325496323193057998, tenantId=1146029695717560320, journalId=1324759914146529478, issueId=1325495515563041259, language=EN, specialIssueTitle=, coverIllustrator=null, specialIssueEditor=, specialIssueAbout=), CN=IssueExt(id=1325496323193057999, tenantId=1146029695717560320, journalId=1324759914146529478, issueId=1325495515563041259, language=CN, specialIssueTitle=, coverIllustrator=null, specialIssueEditor=, specialIssueAbout=)}, issueFiles=null, downloadFileDto=null}, startPage=627, endPage=639, ext={EN=ArticleExt(id=1325495742101594237, articleId=1325495741652803708, tenantId=1146029695717560320, journalId=1324759914146529478, language=EN, title=HiTrain: Heterogeneous Memory Offloading and I/O Optimization for Large Language Model Training, columnId=1325495518645859037, journalTitle=Journal of Computer Research and Development, columnName=Special Issue on Data Storage and Processing for New Quality Productive Forces, runingTitle=null, highlight=null, articleAbstract=
With the continuous growth of the parameter scale of large language models (LLMs), fine-tuning models with hundreds of billions of parameters pose extremely high requirements for computing and storage resources. Traditional distributed training schemes often rely on a large number of high-end graphics processing units (GPUs) and high-speed interconnect networks, resulting in extremely high training costs. Although the existing single-GPU training schemes relieve the pressure on GPU memory through tensor offloading, they still face problems such as low I/O transmission efficiency and insufficient device utilization. Traditional kernel-space I/O operations introduce frequent system calls and context switching during large-scale tensor transfers, becoming a key bottleneck that restricts performance. Meanwhile, the optimizer computations fail to fully leverage the parallel capabilities of multi-core CPUs, making it difficult to effectively overlap with GPU computations, further limiting system performance. In response to the above problems, we propose HiTrain, a heterogeneous memory offloading and I/O optimization scheme for LLM training, with a focus on the design and implementation of two key technologies: First, it constructs a high-performance tensor storage module based on the storage performance development kit (SPDK), which manages tensor data in user space, thereby avoiding the overhead of the kernel I/O stack and improving the concurrency and throughput of tensor offloading. Second, it designs and implements an asynchronous-optimizer based storage-compute pipeline scheduling module, which reorders optimizer execution to reduce GPU idle time and thus enhance overall training efficiency. The experimental results show that on the server equipped with a single GPU and a non-volatile memory express solid state drive (NVMe SSD), the scheme proposed can make full use of the computing and storage resources in the system, improve the efficiency of tensor unloading and loading in the process of model training by 32.7%, and increase the overall training throughput to 1.49 times of the existing scheme, offering a practical and cost-effective path for LLM training.
, authors=Zhihang Tang
1, 2, Hao Lan
3, *, Zhengguo Liu
4, Qing Wang
3, Laiping Zhao
1, Keqiu Li
1, authorsList=Zhihang Tang, Hao Lan, Zhengguo Liu, Qing Wang, Laiping Zhao, Keqiu Li, authorCompany=null, correspAuthors=Hao Lan, authorNote=
born in 1993. PhD candidate, assistant researcher. Member of CCF. His main research interests include distributed computing and large model system
born in 1993. PhD, assistant researcher. Member of CCF. His main research interest includes AI systems and algorithms
born in 1999. Master candidate. Student member of CCF. His main research interests include AI systems and storage systems
born in 1997. PhD, assistant researcher. Member of CCF. His main research interests include storage systems and memory systems
born in 1985. PhD, professor, PhD supervisor. Member of CCF. His main research interest includes cloud computing system
born in 1971. PhD, professor, PhD supervisor. Fellow of CCF. His main research interests include computer networks and cloud computing
, correspAuthorsNote=null, copyrightStatement=Copyright © 2026 Journal of Computer Research and Development. All rights reserved., copyrightOwner=null, extLink=null, articleAbsUrl=null, sourceXml=null, magXml=null, pdfUrl=null, pdf=null, pdfFileSize=null, pdfExtLink=null, richHtmlUrl=null, mobilePdfUrl=null, reviewReport=null, pdfFirstPage=null, abstractGraph=null, abstractGraphContent=null, abstractVideo=null, citation=null, cebUrl=null, magXmlContent=null, mapNumber=null, fund=null), CN=ArticleExt(id=1325495754604814498, articleId=1325495741652803708, tenantId=1146029695717560320, journalId=1324759914146529478, language=CN, title=HiTrain:面向大模型训练的异构内存卸载与I/O优化, columnId=1325495520386495199, journalTitle=计算机研究与发展, columnName=面向新质生产力的数据存储与处理专题, runingTitle=null, highlight=null, articleAbstract=
随着大语言模型(large language models,LLMs)(以下简称“大模型”)参数规模的持续增长,微调百亿级参数大模型对计算和存储资源提出了极高要求。传统分布式训练方案通常依赖大量高端GPU和高速互联网络,训练成本极为昂贵。现有单GPU训练方案虽通过张量卸载缓解显存压力,但仍然面临I/O传输效率低和设备利用率不足等问题。传统内核态I/O操作在大规模张量迁移中引入频繁的系统调用和上下文切换,成为制约性能的关键瓶颈;同时,优化器计算无法充分发挥多核CPU的并行能力,难以实现与GPU计算的有效重叠,进一步限制了系统性能。针对上述问题,提出了一种面向大模型训练的异构内存卸载与I/O优化方案HiTrain。首先构建了基于存储性能开发工具包(storage performance development kit,SPDK)的高性能张量存储模块,通过在用户态管理张量数据,避免了内核I/O栈开销,从而提高张量卸载的并发性与吞吐率;其次,设计并实现了基于异步优化器的存储-计算流水线调度模块,通过对优化器的执行进行优化重排来减少GPU等待时间,提高整体训练效率。实验结果表明,在配备单张GPU和非易失性存储器快速固态硬盘(non-volatile memory express solid state drive,NVMe SSD)的服务器上,所提出的方案能够充分利用系统中的存算资源,使得模型训练过程中张量卸载与加载效率提升32.7%,整体训练吞吐提升至现有方案的1.49倍,为低成本大模型训练提供了切实可行的技术路径。
, authors=汤志航
1, 2, 兰颢
3, *, 刘政国
4, 汪庆
3, 赵来平
1, 李克秋
1, authorsList=汤志航, 兰颢, 刘政国, 汪庆, 赵来平, 李克秋, authorCompany=null, correspAuthors=兰颢, authorNote=
 |
1993年生。博士研究生,助理研究员。CCF会员。主要研究方向为分布式计算、大模型系统 |
 |
1993 年生。博士,助理研究员。CCF会员。主要研究方向为人工智能系统与算法 |
 |
1999年生。硕士研究生。CCF学生会员。主要研究方向为人工智能系统、存储系统 |
 |
1997年生。博士,助理研究员。CCF会员。主要研究方向为存储系统、内存系统 |
 |
1985年生。博士,教授,博士生导师。CCF会员。主要研究方向为云计算系统 |
 |
1971年生。博士,教授,博士生导师。CCF会士。主要研究方向为计算机网络、云计算 |
, correspAuthorsNote=
, copyrightStatement=版权所有 © 2026 计算机研究与发展编辑部, copyrightOwner=null, extLink=null, articleAbsUrl=null, sourceXml=DmH8uIBFGeb+vgfhaVnMBQ==, magXml=86HMWRGtSmmdlCi+6DdRSQ==, pdfUrl=null, pdf=qazF+XKcg9de/WhBG7V2vg==, pdfFileSize=1281068, pdfExtLink=null, richHtmlUrl=null, mobilePdfUrl=null, reviewReport=null, pdfFirstPage=null, abstractGraph=1PTJGNIezXpY7Uy3p/QHGg==, abstractGraphContent=null, abstractVideo=null, citation=null, cebUrl=null, magXmlContent=Ec2AA13Gp7gNU/TZYW4NZQ==, mapNumber=null, fund=null)}, authors=[Author(id=1325495758371299505, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1325495741652803708, orderNo=0, firstName=null, middleName=null, lastName=null, nameCn=null, orcid=null, stid=null, country=null, authorPic=null, dead=0, email=tangzhihang@tju.edu.cn, emailSecond=null, emailThird=null, correspondingAuthor=0, authorType=1, ext={EN=AuthorExt(id=1325495758782341300, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1325495741652803708, authorId=1325495758371299505, language=EN, stringName=Zhihang Tang, firstName=Zhihang, middleName=null, lastName=Tang, prefix=null, suffix=null, authorComment=null, nameInitials=null, affiliation=null, department=null, xref=
1, 2, address=
1College of Intelligence and Computing, Tianjin University, Tianjin 300350
2Research Center for High Efficiency Computing Infrastructure, Zhejiang Lab, Hangzhou 311121, bio={"content":"
born in 1993. PhD candidate, assistant researcher. Member of CCF. His main research interests include distributed computing and large model system
"}, bioImg=null, bioContent=
born in 1993. PhD candidate, assistant researcher. Member of CCF. His main research interests include distributed computing and large model system
, aboutCorrespAuthor=null), CN=AuthorExt(id=1325495758853644469, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1325495741652803708, authorId=1325495758371299505, language=CN, stringName=汤志航, firstName=null, middleName=null, lastName=null, prefix=null, suffix=null, authorComment=null, nameInitials=null, affiliation=null, department=null, xref=
1, 2, address=
1天津大学智能与计算学部 天津 300350
2之江实验室高效能计算设施研究中心 杭州 311121, bio={"img":"3U13d5Msp1ImwlbXzsETkQ==","content":"
1993年生。博士研究生,助理研究员。CCF会员。主要研究方向为分布式计算、大模型系统
"}, bioImg=3U13d5Msp1ImwlbXzsETkQ==, bioContent=
1993年生。博士研究生,助理研究员。CCF会员。主要研究方向为分布式计算、大模型系统
, aboutCorrespAuthor=null)}, companyList=[AuthorCompany(id=1325495755447869603, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1325495741652803708, xref=1, ext=[AuthorCompanyExt(id=1325495755464646820, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1325495741652803708, companyId=1325495755447869603, language=EN, country=null, province=null, city=null, postcode=null, companyName=null, departmentName=null, remark=
1College of Intelligence and Computing, Tianjin University, Tianjin 300350), AuthorCompanyExt(id=1325495755485618341, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1325495741652803708, companyId=1325495755447869603, language=CN, country=null, province=null, city=null, postcode=null, companyName=null, departmentName=null, remark=
1天津大学智能与计算学部 天津 300350)]), AuthorCompany(id=1325495755879882918, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1325495741652803708, xref=2, ext=[AuthorCompanyExt(id=1325495755905048743, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1325495741652803708, companyId=1325495755879882918, language=EN, country=null, province=null, city=null, postcode=null, companyName=null, departmentName=null, remark=
2Research Center for High Efficiency Computing Infrastructure, Zhejiang Lab, Hangzhou 311121), AuthorCompanyExt(id=1325495755926020264, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1325495741652803708, companyId=1325495755879882918, language=CN, country=null, province=null, city=null, postcode=null, companyName=null, departmentName=null, remark=
2之江实验室高效能计算设施研究中心 杭州 311121)])]), Author(id=1325495759222743224, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1325495741652803708, orderNo=1, firstName=null, middleName=null, lastName=null, nameCn=null, orcid=null, stid=null, country=null, authorPic=null, dead=0, email=lanhao@tsinghua.edu.cn, emailSecond=null, emailThird=null, correspondingAuthor=1, authorType=1, ext={EN=AuthorExt(id=1325495759306629306, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1325495741652803708, authorId=1325495759222743224, language=EN, stringName=Hao Lan, firstName=Hao, middleName=null, lastName=Lan, prefix=null, suffix=null, authorComment=null, nameInitials=null, affiliation=null, department=null, xref=
3, *, address=
3Department of Computer Science and Technology, Tsinghua University, Beijing 100084, bio={"content":"
born in 1993. PhD, assistant researcher. Member of CCF. His main research interest includes AI systems and algorithms
"}, bioImg=null, bioContent=
born in 1993. PhD, assistant researcher. Member of CCF. His main research interest includes AI systems and algorithms
, aboutCorrespAuthor=null), CN=AuthorExt(id=1325495759654756539, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1325495741652803708, authorId=1325495759222743224, language=CN, stringName=兰颢, firstName=null, middleName=null, lastName=null, prefix=null, suffix=null, authorComment=null, nameInitials=null, affiliation=null, department=null, xref=
3, *, address=
3清华大学计算机科学与技术系 北京 100084, bio={"img":"c1usmDoRxvwGsVVFK+oBOg==","content":"
1993 年生。博士,助理研究员。CCF会员。主要研究方向为人工智能系统与算法
"}, bioImg=c1usmDoRxvwGsVVFK+oBOg==, bioContent=
1993 年生。博士,助理研究员。CCF会员。主要研究方向为人工智能系统与算法
, aboutCorrespAuthor=null)}, companyList=[AuthorCompany(id=1325495757599547562, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1325495741652803708, xref=3, ext=[AuthorCompanyExt(id=1325495757612130475, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1325495741652803708, companyId=1325495757599547562, language=EN, country=null, province=null, city=null, postcode=null, companyName=null, departmentName=null, remark=
3Department of Computer Science and Technology, Tsinghua University, Beijing 100084), AuthorCompanyExt(id=1325495757620519084, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1325495741652803708, companyId=1325495757599547562, language=CN, country=null, province=null, city=null, postcode=null, companyName=null, departmentName=null, remark=
3清华大学计算机科学与技术系 北京 100084)])]), Author(id=1325495759747031229, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1325495741652803708, orderNo=2, firstName=null, middleName=null, lastName=null, nameCn=null, orcid=null, stid=null, country=null, authorPic=null, dead=0, email=null, emailSecond=null, emailThird=null, correspondingAuthor=0, authorType=1, ext={EN=AuthorExt(id=1325495760153878720, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1325495741652803708, authorId=1325495759747031229, language=EN, stringName=Zhengguo Liu, firstName=Zhengguo, middleName=null, lastName=Liu, prefix=null, suffix=null, authorComment=null, nameInitials=null, affiliation=null, department=null, xref=
4, address=
4Department of Information and Communication Engineering, Xiamen University, Xiamen, Fujian 361005, bio={"content":"
born in 1999. Master candidate. Student member of CCF. His main research interests include AI systems and storage systems
"}, bioImg=null, bioContent=
born in 1999. Master candidate. Student member of CCF. His main research interests include AI systems and storage systems
, aboutCorrespAuthor=null), CN=AuthorExt(id=1325495760476840129, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1325495741652803708, authorId=1325495759747031229, language=CN, stringName=刘政国, firstName=null, middleName=null, lastName=null, prefix=null, suffix=null, authorComment=null, nameInitials=null, affiliation=null, department=null, xref=
4, address=
4厦门大学信息与通信工程系 福建厦门 361005, bio={"img":"IwoPdwG1IcO6nFWxH/mySQ==","content":"
1999年生。硕士研究生。CCF学生会员。主要研究方向为人工智能系统、存储系统
"}, bioImg=IwoPdwG1IcO6nFWxH/mySQ==, bioContent=
1999年生。硕士研究生。CCF学生会员。主要研究方向为人工智能系统、存储系统
, aboutCorrespAuthor=null)}, companyList=[AuthorCompany(id=1325495757964452013, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1325495741652803708, xref=4, ext=[AuthorCompanyExt(id=1325495757977034926, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1325495741652803708, companyId=1325495757964452013, language=EN, country=null, province=null, city=null, postcode=null, companyName=null, departmentName=null, remark=
4Department of Information and Communication Engineering, Xiamen University, Xiamen, Fujian 361005), AuthorCompanyExt(id=1325495758014783663, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1325495741652803708, companyId=1325495757964452013, language=CN, country=null, province=null, city=null, postcode=null, companyName=null, departmentName=null, remark=
4厦门大学信息与通信工程系 福建厦门 361005)])]), Author(id=1325495762154561733, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1325495741652803708, orderNo=3, firstName=null, middleName=null, lastName=null, nameCn=null, orcid=null, stid=null, country=null, authorPic=null, dead=0, email=null, emailSecond=null, emailThird=null, correspondingAuthor=0, authorType=1, ext={EN=AuthorExt(id=1325495762263613639, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1325495741652803708, authorId=1325495762154561733, language=EN, stringName=Qing Wang, firstName=Qing, middleName=null, lastName=Wang, prefix=null, suffix=null, authorComment=null, nameInitials=null, affiliation=null, department=null, xref=
3, address=
3Department of Computer Science and Technology, Tsinghua University, Beijing 100084, bio={"content":"
born in 1997. PhD, assistant researcher. Member of CCF. His main research interests include storage systems and memory systems
"}, bioImg=null, bioContent=
born in 1997. PhD, assistant researcher. Member of CCF. His main research interests include storage systems and memory systems
, aboutCorrespAuthor=null), CN=AuthorExt(id=1325495762330722504, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1325495741652803708, authorId=1325495762154561733, language=CN, stringName=汪庆, firstName=null, middleName=null, lastName=null, prefix=null, suffix=null, authorComment=null, nameInitials=null, affiliation=null, department=null, xref=
3, address=
3清华大学计算机科学与技术系 北京 100084, bio={"img":"tIiMcO/KsRlcuc9trwGi7w==","content":"
1997年生。博士,助理研究员。CCF会员。主要研究方向为存储系统、内存系统
"}, bioImg=tIiMcO/KsRlcuc9trwGi7w==, bioContent=
1997年生。博士,助理研究员。CCF会员。主要研究方向为存储系统、内存系统
, aboutCorrespAuthor=null)}, companyList=[AuthorCompany(id=1325495757599547562, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1325495741652803708, xref=3, ext=[AuthorCompanyExt(id=1325495757612130475, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1325495741652803708, companyId=1325495757599547562, language=EN, country=null, province=null, city=null, postcode=null, companyName=null, departmentName=null, remark=
3Department of Computer Science and Technology, Tsinghua University, Beijing 100084), AuthorCompanyExt(id=1325495757620519084, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1325495741652803708, companyId=1325495757599547562, language=CN, country=null, province=null, city=null, postcode=null, companyName=null, departmentName=null, remark=
3清华大学计算机科学与技术系 北京 100084)])]), Author(id=1325495762565603530, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1325495741652803708, orderNo=4, firstName=null, middleName=null, lastName=null, nameCn=null, orcid=null, stid=null, country=null, authorPic=null, dead=0, email=null, emailSecond=null, emailThird=null, correspondingAuthor=0, authorType=1, ext={EN=AuthorExt(id=1325495762641101004, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1325495741652803708, authorId=1325495762565603530, language=EN, stringName=Laiping Zhao, firstName=Laiping, middleName=null, lastName=Zhao, prefix=null, suffix=null, authorComment=null, nameInitials=null, affiliation=null, department=null, xref=
1, address=
1College of Intelligence and Computing, Tianjin University, Tianjin 300350, bio={"content":"
born in 1985. PhD, professor, PhD supervisor. Member of CCF. His main research interest includes cloud computing system
"}, bioImg=null, bioContent=
born in 1985. PhD, professor, PhD supervisor. Member of CCF. His main research interest includes cloud computing system
, aboutCorrespAuthor=null), CN=AuthorExt(id=1325495762720792781, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1325495741652803708, authorId=1325495762565603530, language=CN, stringName=赵来平, firstName=null, middleName=null, lastName=null, prefix=null, suffix=null, authorComment=null, nameInitials=null, affiliation=null, department=null, xref=
1, address=
1天津大学智能与计算学部 天津 300350, bio={"img":"13eaOUYaadV3VB6squB3aA==","content":"
1985年生。博士,教授,博士生导师。CCF会员。主要研究方向为云计算系统
"}, bioImg=13eaOUYaadV3VB6squB3aA==, bioContent=
1985年生。博士,教授,博士生导师。CCF会员。主要研究方向为云计算系统
, aboutCorrespAuthor=null)}, companyList=[AuthorCompany(id=1325495755447869603, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1325495741652803708, xref=1, ext=[AuthorCompanyExt(id=1325495755464646820, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1325495741652803708, companyId=1325495755447869603, language=EN, country=null, province=null, city=null, postcode=null, companyName=null, departmentName=null, remark=
1College of Intelligence and Computing, Tianjin University, Tianjin 300350), AuthorCompanyExt(id=1325495755485618341, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1325495741652803708, companyId=1325495755447869603, language=CN, country=null, province=null, city=null, postcode=null, companyName=null, departmentName=null, remark=
1天津大学智能与计算学部 天津 300350)])]), Author(id=1325495763039559887, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1325495741652803708, orderNo=5, firstName=null, middleName=null, lastName=null, nameCn=null, orcid=null, stid=null, country=null, authorPic=null, dead=0, email=null, emailSecond=null, emailThird=null, correspondingAuthor=0, authorType=1, ext={EN=AuthorExt(id=1325495763119251665, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1325495741652803708, authorId=1325495763039559887, language=EN, stringName=Keqiu Li, firstName=Keqiu, middleName=null, lastName=Li, prefix=null, suffix=null, authorComment=null, nameInitials=null, affiliation=null, department=null, xref=
1, address=
1College of Intelligence and Computing, Tianjin University, Tianjin 300350, bio={"content":"
born in 1971. PhD, professor, PhD supervisor. Fellow of CCF. His main research interests include computer networks and cloud computing
"}, bioImg=null, bioContent=
born in 1971. PhD, professor, PhD supervisor. Fellow of CCF. His main research interests include computer networks and cloud computing
, aboutCorrespAuthor=null), CN=AuthorExt(id=1325495763412852946, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1325495741652803708, authorId=1325495763039559887, language=CN, stringName=李克秋, firstName=null, middleName=null, lastName=null, prefix=null, suffix=null, authorComment=null, nameInitials=null, affiliation=null, department=null, xref=
1, address=
1天津大学智能与计算学部 天津 300350, bio={"img":"Oq6IeV+sSzfRXhVQgDzPFA==","content":"
1971年生。博士,教授,博士生导师。CCF会士。主要研究方向为计算机网络、云计算
"}, bioImg=Oq6IeV+sSzfRXhVQgDzPFA==, bioContent=
1971年生。博士,教授,博士生导师。CCF会士。主要研究方向为计算机网络、云计算
, aboutCorrespAuthor=null)}, companyList=[AuthorCompany(id=1325495755447869603, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1325495741652803708, xref=1, ext=[AuthorCompanyExt(id=1325495755464646820, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1325495741652803708, companyId=1325495755447869603, language=EN, country=null, province=null, city=null, postcode=null, companyName=null, departmentName=null, remark=
1College of Intelligence and Computing, Tianjin University, Tianjin 300350), AuthorCompanyExt(id=1325495755485618341, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1325495741652803708, companyId=1325495755447869603, language=CN, country=null, province=null, city=null, postcode=null, companyName=null, departmentName=null, remark=
1天津大学智能与计算学部 天津 300350)])])], keywords=[Keyword(id=1325495763505127635, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1325495741652803708, language=EN, orderNo=1, keyword=large model training), Keyword(id=1325495763568042196, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1325495741652803708, language=EN, orderNo=2, keyword=heterogeneous memory), Keyword(id=1325495763811311829, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1325495741652803708, language=EN, orderNo=3, keyword=tensor offload), Keyword(id=1325495763878420694, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1325495741652803708, language=EN, orderNo=4, keyword=user-space high-performance I/O), Keyword(id=1325495763945529559, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1325495741652803708, language=EN, orderNo=5, keyword=storage-computing pipeline), Keyword(id=1325495764021027032, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1325495741652803708, language=CN, orderNo=1, keyword=大模型训练), Keyword(id=1325495764318822617, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1325495741652803708, language=CN, orderNo=2, keyword=异构内存), Keyword(id=1325495764390125786, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1325495741652803708, language=CN, orderNo=3, keyword=张量卸载), Keyword(id=1325495764717281499, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1325495741652803708, language=CN, orderNo=4, keyword=用户态高性能I/O), Keyword(id=1325495764796973276, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1325495741652803708, language=CN, orderNo=5, keyword=存储-计算流水线)], refs=[Reference(id=1325495773936361723, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1325495741652803708, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=1, rfOrder=0, authorNames=null, journalName=null, refType=null, unstructuredReference=Vaswani A, Shazeer N, Parmar N, et al. Attention is all you need[J/OL]. Advances in Neural Information Processing Systems, 2017[2025-06-02]. https://proceedings.neurips.cc/paper/2017/hash/3f5ee243547dee91fbd053c1c4a845aa-Abstract.html, articleTitle=null, refAbstract=null), Reference(id=1325495774733279484, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1325495741652803708, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=2, rfOrder=1, authorNames=null, journalName=null, refType=null, unstructuredReference=Achiam J, Adler S, Agarwal S, et al. GPT−4 technical report[J]. arXiv preprint, arXiv: 2303.08774, 2023, articleTitle=null, refAbstract=null), Reference(id=1325495776004153597, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1325495741652803708, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=3, rfOrder=2, authorNames=null, journalName=null, refType=null, unstructuredReference=Jiang Juyong, Wang Fan, Shen Jiasi, et al. A survey on large language models for code generation[J]. arXiv preprint, arXiv: 2406.00515, 2024, articleTitle=null, refAbstract=null), Reference(id=1325495776431972606, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1325495741652803708, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=4, rfOrder=3, authorNames=null, journalName=null, refType=null, unstructuredReference=Wu Jiayu, Gan Wensheng, Chen Zefeng, et al. Multimodal large language models: A survey[C]//Proc of the 2023 IEEE Int Conf on Big Data (BigData). Piscataway, NJ: IEEE, 2023: 2247−2256, articleTitle=null, refAbstract=null), Reference(id=1325495776834625791, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1325495741652803708, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=5, rfOrder=4, authorNames=null, journalName=null, refType=null, unstructuredReference=冯杨洋, 汪庆, 谢旻晖, 等. 从BERT 到ChatGPT: 大模型训练中的存储系统挑战与技术发展[J]. 计算机研究与发展, 2024, 61(4): 809−823, articleTitle=null, refAbstract=null), Reference(id=1325495777237278976, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1325495741652803708, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=5, rfOrder=5, authorNames=null, journalName=null, refType=null, unstructuredReference=Feng Yangyang, Wang Qing, Xie Minhui, et al. From BERT to ChatGPT: Challenges and technical development of storage systems for large model training[J]. Journal of Computer Research and Development, 2024, 61(4): 809−823 (in Chinese), articleTitle=null, refAbstract=null), Reference(id=1325495777694458113, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1325495741652803708, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=6, rfOrder=6, authorNames=null, journalName=null, refType=null, unstructuredReference=NVIDIA. NVIDIA H200 tensor core GPU[EB/OL]. Santa Clara, CA: NVIDIA Corporation, 2023 (2024-05-26)[2025-06-03]. https://www.nvidia.com/en-us/data-center/h200/, articleTitle=null, refAbstract=null), Reference(id=1325495778189385986, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1325495741652803708, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=7, rfOrder=7, authorNames=null, journalName=null, refType=null, unstructuredReference=Touvron H, Martin L, Stone K, et al. LlaMA 2: Open foundation and fine-tuned chat models[J]. arXiv preprint, arXiv: 2307.09288, 2023, articleTitle=null, refAbstract=null), Reference(id=1325495780194263299, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1325495741652803708, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=8, rfOrder=8, authorNames=null, journalName=null, refType=null, unstructuredReference=Luo Xudong, Deng Zhiqi, Yang Bingxia, et al. Pre-trained language models in medicine: A survey[J]. Artificial Intelligence in Medicine, 2024, 154: 102904, articleTitle=null, refAbstract=null), Reference(id=1325495780592722180, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1325495741652803708, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=9, rfOrder=9, authorNames=null, journalName=null, refType=null, unstructuredReference=Li Junyi, Tang Tianyi, Wayne X Z, et al. Pre-trained language models for text generation: A survey[J]. ACM Computing Surveys, 2024, 56(9): 1−39, articleTitle=null, refAbstract=null), Reference(id=1325495780672413957, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1325495741652803708, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=10, rfOrder=10, authorNames=null, journalName=null, refType=null, unstructuredReference=Rasley J, Rajbhandari S, Ruwase O, et al. DeepSpeed: System optimizations enable training deep learning models with over 100 billion parameters[C]//Proc of the 26th ACM SIGKDD Int Conf on Knowledge Discovery & Data Mining. New York: ACM, 2020: 3505−3506, articleTitle=null, refAbstract=null), Reference(id=1325495781112815878, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1325495741652803708, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=11, rfOrder=11, authorNames=null, journalName=null, refType=null, unstructuredReference=Shoeybi M, Patwary M, Puri R, et al. Megatron-LM: Training multi-billion parameter language models using model parallelism[J]. arXiv preprint, arXiv: 1909.08053, 2019, articleTitle=null, refAbstract=null), Reference(id=1325495781494497543, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1325495741652803708, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=12, rfOrder=12, authorNames=null, journalName=null, refType=null, unstructuredReference=Park Y M, Ahn S Y, Lim E J, et al. Deep learning model parallelism[J]. Electronics and Telecommunications Trends, 2018, 33(4): 1−13, articleTitle=null, refAbstract=null), Reference(id=1325495781968453897, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1325495741652803708, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=13, rfOrder=13, authorNames=null, journalName=null, refType=null, unstructuredReference=Fan Shiqing, Rong Yi, Meng Chen, et al. DAPPLE: A pipelined data parallel approach for training large models[C]//Proc of the 26th ACM SIGPLAN Symp on Principles and Practice of Parallel Programming. New York: ACM, 2021: 431−445, articleTitle=null, refAbstract=null), Reference(id=1325495783193190666, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1325495741652803708, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=14, rfOrder=14, authorNames=null, journalName=null, refType=null, unstructuredReference=Liu Aixin, Feng Bei, Xue Bing, et al. DeepSeek-v3 technical report[J]. arXiv preprint, arXiv: 2412.19437, 2024, articleTitle=null, refAbstract=null), Reference(id=1325495784866717963, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1325495741652803708, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=15, rfOrder=15, authorNames=null, journalName=null, refType=null, unstructuredReference=Li Shenggui, Liu Hongxin, Bian Zhengda, et al. Colossal-AI: A unified deep learning system for large-scale parallel training[C]//Proc of the 52nd Int Conf on Parallel Processing. New York: ACM, 2023: 766−775, articleTitle=null, refAbstract=null), Reference(id=1325495785336480013, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1325495741652803708, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=16, rfOrder=16, authorNames=null, journalName=null, refType=null, unstructuredReference=Pfister G. An introduction to the InfiniBand architecture[J]. High Performance Mass Storage and Parallel I/O, 2001, 42(102): 617−632, articleTitle=null, refAbstract=null), Reference(id=1325495785634275598, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1325495741652803708, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=17, rfOrder=17, authorNames=null, journalName=null, refType=null, unstructuredReference=Foley D, Danskin J. Ultra-performance Pascal GPU and NVLink interconnect[J]. IEEE Micro, 2017, 37(2): 7−17, articleTitle=null, refAbstract=null), Reference(id=1325495785864962321, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1325495741652803708, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=18, rfOrder=18, authorNames=null, journalName=null, refType=null, unstructuredReference=Bae J, Lee J, Jin Y, et al. FlashNeuron: SSD-enabled large-batch training of very deep neural networks[C]//Proc of the 19th USENIX Conf on File and Storage Technologies (FAST’21). Berkeley: CA, USENIX Association, 2021: 387−401, articleTitle=null, refAbstract=null), Reference(id=1325495786087260434, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1325495741652803708, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=19, rfOrder=19, authorNames=null, journalName=null, refType=null, unstructuredReference=Rajbhandari S, Rasley J, Ruwase O, et al. ZeRO: Memory optimizations toward training trillion parameter models[C]//Proc of the 2020 Int Conf for High Performance Computing, Networking, Storage and Analysis(SC’20). Piscataway, NJ: IEEE, 2020: 1−16, articleTitle=null, refAbstract=null), Reference(id=1325495786175340819, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1325495741652803708, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=20, rfOrder=20, authorNames=null, journalName=null, refType=null, unstructuredReference=Ren Jie, Rajbhandari S, Aminabadi R Y, et al. ZeRO-Offload: Democratizing billion-scale model training[C]//Proc of the 2021 USENIX Annual Technical Conf (USENIX ATC’21). Berkeley: CA USENIX Association, 2021: 551−564, articleTitle=null, refAbstract=null), Reference(id=1325495786577994004, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1325495741652803708, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=21, rfOrder=21, authorNames=null, journalName=null, refType=null, unstructuredReference=Rajbhandari S, Ruwase O, Rasley J, et al. ZeRO-Infinity: Breaking the gpu memory wall for extreme scale deep learning[C]//Proc of the 2021 Int Conf for High Performance Computing, Networking, Storage and Analysis (SC’21). Piscataway, NJ: IEEE, 2021: 1−14, articleTitle=null, refAbstract=null), Reference(id=1325495786951287061, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1325495741652803708, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=22, rfOrder=22, authorNames=null, journalName=null, refType=null, unstructuredReference=Liao Changyue, Sun Mo, Yang Zihan, et al. LoHan: Low-cost high-performance framework to fine-tune 100 B model on a consumer GPU[J]. arXiv Preprint, arXiv: 2403.06504, 2024, articleTitle=null, refAbstract=null), Reference(id=1325495787316191510, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1325495741652803708, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=23, rfOrder=23, authorNames=null, journalName=null, refType=null, unstructuredReference=Yang Ziye, Harris J R, Walker B, et al. SPDK: A development kit to build high performance storage applications[C]//Proc of the 2017 IEEE Int Conf on Cloud Computing Technology and Science (CloudCom2017). Piscataway, NJ: IEEE, 2017: 154−161, articleTitle=null, refAbstract=null), Reference(id=1325495787391688983, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1325495741652803708, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=24, rfOrder=24, authorNames=null, journalName=null, refType=null, unstructuredReference=Thompson A, Newburn C J. GPUDirect storage: A direct path between storage and GPU memory[EB/OL]. NVIDIA Developer Whitepapers, 2019[2025-06-05]. https://developer.nvidia.com/blog/gpudirect-storage, articleTitle=null, refAbstract=null), Reference(id=1325495789107159321, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1325495741652803708, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=25, rfOrder=25, authorNames=null, journalName=null, refType=null, unstructuredReference=Wu Kun, Park J B, Zhang X, et al. SSDTrain: An activation offloading framework to SSDs for faster large language model training[J]. arXiv preprint, arXiv: 2408.10013, 2024, articleTitle=null, refAbstract=null), Reference(id=1325495789421732122, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1325495741652803708, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=26, rfOrder=26, authorNames=null, journalName=null, refType=null, unstructuredReference=Zhang Haoyang, Zhou Yirui, Xue Yuqi, et al. G10: Enabling an efficient unified GPU memory and storage architecture with smart tensor migrations[C]//Proc of the 56th Annual IEEE/ACM Int Symp on Microarchitecture. Piscataway, NJ: IEEE, 2023: 395−410, articleTitle=null, refAbstract=null), Reference(id=1325495789484646683, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1325495741652803708, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=27, rfOrder=27, authorNames=null, journalName=null, refType=null, unstructuredReference=Xu Dong, Feng Yuan, Shin K, et al. Efficient tensor offloading for large deep-learning model training based on compute express link[C]//Proc of the 2024 Int Conf for High Performance Computing, Networking, Storage and Analysis (SC’24). Piscataway, NJ: IEEE, 2024: 1−18, articleTitle=null, refAbstract=null), Reference(id=1325495789551755548, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1325495741652803708, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=28, rfOrder=28, authorNames=null, journalName=null, refType=null, unstructuredReference=Didona D, Pfefferle J, Ioannou N, et al. Understanding modern storage APIs: A systematic study of libaio, SPDK, and io_uring[C]//Proc of the 15th ACM Int Systems and Storage Conf. New York: ACM, 2022: 120−127, articleTitle=null, refAbstract=null), Reference(id=1325495789891494173, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1325495741652803708, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=29, rfOrder=29, authorNames=null, journalName=null, refType=null, unstructuredReference=Ravi J, Byna S, Koziol Q, et al. Evaluating asynchronous parallel I/O on HPC systems[C]//Proc of the 37th IEEE Int Parallel and Distributed Processing Symp (IPDPS’23). Piscataway, NJ: IEEE, 2023: 211−221, articleTitle=null, refAbstract=null), Reference(id=1325495790264787230, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1325495741652803708, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=30, rfOrder=30, authorNames=null, journalName=null, refType=null, unstructuredReference=郝栋栋, 高聪明, 舒继武. 面向SCSI子系统的用户空间存储架构设计[J]. 计算机研究与发展, 2025, 62(3): 633−647, articleTitle=null, refAbstract=null), Reference(id=1325495790696800543, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1325495741652803708, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=30, rfOrder=31, authorNames=null, journalName=null, refType=null, unstructuredReference=Hao Dongdong, Gao Congming, Shu Jiwu. Design of user space storage architecture for SCSI subsystem[J]. Journal of Computer Research and Development, 2025, 62(3): 633−647 (in Chinese), articleTitle=null, refAbstract=null), Reference(id=1325495790814241056, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1325495741652803708, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=31, rfOrder=32, authorNames=null, journalName=null, refType=null, unstructuredReference=Li Junzhen, Pan Xiurui, Yi Shushu, et al. A survey on user-space storage and its implementations[J]. arXiv preprint, arXiv: 2306.10503, 2023, articleTitle=null, refAbstract=null), Reference(id=1325495791183339809, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1325495741652803708, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=32, rfOrder=33, authorNames=null, journalName=null, refType=null, unstructuredReference=Kim H, Lee Y, Kim J S. NVMeDirect: A user-space I/O framework for application-specific optimization on NVMe SSDs[C]//Proc of the 8th USENIX Workshop on Hot Topics in Storage and File Systems (HotStorage’16). Berkeley, CA: USENIX Association, 2016: 1−5, articleTitle=null, refAbstract=null), Reference(id=1325495791531467042, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1325495741652803708, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=33, rfOrder=34, authorNames=null, journalName=null, refType=null, unstructuredReference=The Linux Foundation. I/O latency optimization with polling [R]. San Francisco, CA: The Linux Foundation, 2017, articleTitle=null, refAbstract=null), Reference(id=1325495791976063268, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1325495741652803708, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=34, rfOrder=35, authorNames=null, journalName=null, refType=null, unstructuredReference=Huang Jian, Badam A, Qureshi M K, et al. Unified address translation for memory-mapped SSDs with FlashMap [C] //Proc of the 42nd Annual Int Symp on Computer Architecture. New York: ACM, 2015: 580−591, articleTitle=null, refAbstract=null), Reference(id=1325495793666367782, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1325495741652803708, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=35, rfOrder=36, authorNames=null, journalName=null, refType=null, unstructuredReference=Gerhorst L. Analysis of interrupt handling overhead in the Linux kernel [D]. Erlangen: Friedrich-Alexander-Universität Erlangen-Nürnberg (FAU), 2018, articleTitle=null, refAbstract=null), Reference(id=1325495794064826663, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1325495741652803708, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=36, rfOrder=37, authorNames=null, journalName=null, refType=null, unstructuredReference=Ren Zebin, Trivedi A. Performance characterization of modern storage stacks: posix I/O, libaio, spdk, and io_uring[C]//Proc of the 3rd Workshop on Challenges and Opportunities of Efficient and Performant Storage Systems. New York: ACM, 2023: 35−45, articleTitle=null, refAbstract=null)], funds=null, companyList=[AuthorCompany(id=1325495755447869603, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1325495741652803708, xref=1, ext=[AuthorCompanyExt(id=1325495755464646820, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1325495741652803708, companyId=1325495755447869603, language=EN, country=null, province=null, city=null, postcode=null, companyName=null, departmentName=null, remark=
1College of Intelligence and Computing, Tianjin University, Tianjin 300350), AuthorCompanyExt(id=1325495755485618341, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1325495741652803708, companyId=1325495755447869603, language=CN, country=null, province=null, city=null, postcode=null, companyName=null, departmentName=null, remark=
1天津大学智能与计算学部 天津 300350)]), AuthorCompany(id=1325495755879882918, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1325495741652803708, xref=2, ext=[AuthorCompanyExt(id=1325495755905048743, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1325495741652803708, companyId=1325495755879882918, language=EN, country=null, province=null, city=null, postcode=null, companyName=null, departmentName=null, remark=
2Research Center for High Efficiency Computing Infrastructure, Zhejiang Lab, Hangzhou 311121), AuthorCompanyExt(id=1325495755926020264, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1325495741652803708, companyId=1325495755879882918, language=CN, country=null, province=null, city=null, postcode=null, companyName=null, departmentName=null, remark=
2之江实验室高效能计算设施研究中心 杭州 311121)]), AuthorCompany(id=1325495757599547562, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1325495741652803708, xref=3, ext=[AuthorCompanyExt(id=1325495757612130475, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1325495741652803708, companyId=1325495757599547562, language=EN, country=null, province=null, city=null, postcode=null, companyName=null, departmentName=null, remark=
3Department of Computer Science and Technology, Tsinghua University, Beijing 100084), AuthorCompanyExt(id=1325495757620519084, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1325495741652803708, companyId=1325495757599547562, language=CN, country=null, province=null, city=null, postcode=null, companyName=null, departmentName=null, remark=
3清华大学计算机科学与技术系 北京 100084)]), AuthorCompany(id=1325495757964452013, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1325495741652803708, xref=4, ext=[AuthorCompanyExt(id=1325495757977034926, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1325495741652803708, companyId=1325495757964452013, language=EN, country=null, province=null, city=null, postcode=null, companyName=null, departmentName=null, remark=
4Department of Information and Communication Engineering, Xiamen University, Xiamen, Fujian 361005), AuthorCompanyExt(id=1325495758014783663, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1325495741652803708, companyId=1325495757964452013, language=CN, country=null, province=null, city=null, postcode=null, companyName=null, departmentName=null, remark=
4厦门大学信息与通信工程系 福建厦门 361005)])], figs=[ArticleFig(id=1325495764910219486, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1325495741652803708, language=EN, label=Figure 1, caption=
Comparison of model size and growth speed of accelerator memory, figureFileSmall=N9nPMLCczZYipvT+6erqaQ==, figureFileBig=1PTJGNIezXpY7Uy3p/QHGg==, tableContent=null), ArticleFig(id=1325495766332088543, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1325495741652803708, language=CN, label=图1, caption=
模型规模和加速器内存增长速度对比, figureFileSmall=N9nPMLCczZYipvT+6erqaQ==, figureFileBig=1PTJGNIezXpY7Uy3p/QHGg==, tableContent=null), ArticleFig(id=1325495766751518944, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1325495741652803708, language=EN, label=Figure 2, caption=
Update steps of optimizer, figureFileSmall=HySCnMLKl/FvFGslUK15KQ==, figureFileBig=JBHqoBezHycQhedrLSnYnQ==, tableContent=null), ArticleFig(id=1325495766818627809, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1325495741652803708, language=CN, label=图2, caption=
优化器更新步骤, figureFileSmall=HySCnMLKl/FvFGslUK15KQ==, figureFileBig=JBHqoBezHycQhedrLSnYnQ==, tableContent=null), ArticleFig(id=1325495766889930978, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1325495741652803708, language=EN, label=Figure 3, caption=
Comparison of kernel-space and user-space I/O paths, figureFileSmall=hf1dRVWrqORhtlbfRbzURg==, figureFileBig=EsyBPuHYWzZsXmqpe5//IQ==, tableContent=null), ArticleFig(id=1325495767187726563, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1325495741652803708, language=CN, label=图3, caption=
内核态与用户态I/O路径对比, figureFileSmall=hf1dRVWrqORhtlbfRbzURg==, figureFileBig=EsyBPuHYWzZsXmqpe5//IQ==, tableContent=null), ArticleFig(id=1325495767259029732, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1325495741652803708, language=EN, label=Figure 4, caption=
Storage-computing pipeline in training process, figureFileSmall=N6ggm/j2GEWVsdf7OYNy3A==, figureFileBig=TK8R1Lc2rnDdQz4CGrCAGA==, tableContent=null), ArticleFig(id=1325495767330332901, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1325495741652803708, language=CN, label=图4, caption=
训练过程中的存储-计算流水线, figureFileSmall=N6ggm/j2GEWVsdf7OYNy3A==, figureFileBig=TK8R1Lc2rnDdQz4CGrCAGA==, tableContent=null), ArticleFig(id=1325495767607156966, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1325495741652803708, language=EN, label=Figure 5, caption=
Overall architecture diagram of HiTrain, figureFileSmall=bkVRCsSonF1o8NpWlpsXHw==, figureFileBig=WmMh6JkYkckXvZgTok+fHg==, tableContent=null), ArticleFig(id=1325495768034975975, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1325495741652803708, language=CN, label=图5, caption=
HiTrain整体架构图, figureFileSmall=bkVRCsSonF1o8NpWlpsXHw==, figureFileBig=WmMh6JkYkckXvZgTok+fHg==, tableContent=null), ArticleFig(id=1325495768144027880, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1325495741652803708, language=EN, label=Figure 6, caption=
Comparison of pipeline optimization, figureFileSmall=oLu6Uyn+VVek/El9d9YQeQ==, figureFileBig=qo39PLaZOOn79yiXwj6HzQ==, tableContent=null), ArticleFig(id=1325495768454406377, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1325495741652803708, language=CN, label=图6, caption=
流水线优化对比, figureFileSmall=oLu6Uyn+VVek/El9d9YQeQ==, figureFileBig=qo39PLaZOOn79yiXwj6HzQ==, tableContent=null), ArticleFig(id=1325495768538292459, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1325495741652803708, language=EN, label=Figure 7, caption=
Performance comparison of sequential read/write with different block sizes, figureFileSmall=xtKdqVxWH3r12rgHIER3gA==, figureFileBig=XS9nlHmi5EdGgvtb9+NeyQ==, tableContent=null), ArticleFig(id=1325495768865448172, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1325495741652803708, language=CN, label=图7, caption=
不同块大小下的顺序读/写性能对比, figureFileSmall=xtKdqVxWH3r12rgHIER3gA==, figureFileBig=XS9nlHmi5EdGgvtb9+NeyQ==, tableContent=null), ArticleFig(id=1325495768924168429, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1325495741652803708, language=EN, label=Figure 8, caption=
Performance comparison of random read/write with different core numbers, figureFileSmall=X4BA94ZmCzT0rI31z4e/XA==, figureFileBig=ynO3genc/rA4itLtwHki0w==, tableContent=null), ArticleFig(id=1325495768991277294, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1325495741652803708, language=CN, label=图8, caption=
不同核心数下的随机读/写性能对比, figureFileSmall=X4BA94ZmCzT0rI31z4e/XA==, figureFileBig=ynO3genc/rA4itLtwHki0w==, tableContent=null), ArticleFig(id=1325495769280684271, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1325495741652803708, language=EN, label=Figure 9, caption=
Performance comparison of I/O module for different file blocks in actual training, figureFileSmall=6iOPB/AvhK8WxrNVX+8K1w==, figureFileBig=iWmWI1tFnH4Y1G3LRFFsog==, tableContent=null), ArticleFig(id=1325495769368764656, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1325495741652803708, language=CN, label=图9, caption=
实际训练中不同文件块I/O模块性能对比, figureFileSmall=6iOPB/AvhK8WxrNVX+8K1w==, figureFileBig=iWmWI1tFnH4Y1G3LRFFsog==, tableContent=null), ArticleFig(id=1325495769435873521, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1325495741652803708, language=EN, label=Figure 10, caption=
Performance comparison under different model scales, figureFileSmall=CZdImoKsWKGkZXto2SFexA==, figureFileBig=FUbnnIXnnYG5m8sbmjhKWQ==, tableContent=null), ArticleFig(id=1325495769502982386, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1325495741652803708, language=CN, label=图10, caption=
不同模型规模下的训练性能对比, figureFileSmall=CZdImoKsWKGkZXto2SFexA==, figureFileBig=FUbnnIXnnYG5m8sbmjhKWQ==, tableContent=null), ArticleFig(id=1325495769691726067, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1325495741652803708, language=EN, label=Figure 11, caption=
Performance comparison of training with different batch sizes, figureFileSmall=kQUa2Iyzgwd5LXDMzmVNtA==, figureFileBig=+dVtAwO6FbqJnzTi1qpZnQ==, tableContent=null), ArticleFig(id=1325495771423973620, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1325495741652803708, language=CN, label=图11, caption=
不同批大小下的训练性能对比, figureFileSmall=kQUa2Iyzgwd5LXDMzmVNtA==, figureFileBig=+dVtAwO6FbqJnzTi1qpZnQ==, tableContent=null), ArticleFig(id=1325495771847598325, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1325495741652803708, language=EN, label=Table 1, caption=
Comparison of Different Schemes on Memory Offloading, figureFileSmall=null, figureFileBig=null, tableContent=
| 方法 | 内存优化机制 | 优化器位置 | 重计算 | 性能瓶颈 |
|---|
| FlashNeuron | 模型状态→Flash 部分激活值→Flash | GPU | 需要 | Flash存储带宽有限。 |
| ZeRO-Infinity | 模型状态→SSD 激活值→CPU内存 | CPU | 需要 | CPU-GPU计算非重叠导致GPU空闲;大量重计算导致PCIe带宽闲置; 激活值卸载到CPU内存产生额外卸载开销。 |
| SSDTrain | 激活值→SSD | GPU | 不需要 | 性能高度依赖于底层存储设备的带宽。 |
| G10 | 模型状态→SSD 激活值→SSD | GPU | 不需要 | 反向传播时GPU需等待SSD传输优化器状态;与大量激活值导致GPU闲置。 |
| LoHan | 模型状态→SSD 部分激活值→CPU内存 | CPU | 需要 | 激活值卸载到CPU内存产生额外卸载开销; I/O操作的上下文切换开销大。 |
), ArticleFig(id=1325495772225085686, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1325495741652803708, language=CN, label=表1, caption=
不同内存卸载方案对比
, figureFileSmall=null, figureFileBig=null, tableContent=
| 方法 | 内存优化机制 | 优化器位置 | 重计算 | 性能瓶颈 |
|---|
| FlashNeuron | 模型状态→Flash 部分激活值→Flash | GPU | 需要 | Flash存储带宽有限。 |
| ZeRO-Infinity | 模型状态→SSD 激活值→CPU内存 | CPU | 需要 | CPU-GPU计算非重叠导致GPU空闲;大量重计算导致PCIe带宽闲置; 激活值卸载到CPU内存产生额外卸载开销。 |
| SSDTrain | 激活值→SSD | GPU | 不需要 | 性能高度依赖于底层存储设备的带宽。 |
| G10 | 模型状态→SSD 激活值→SSD | GPU | 不需要 | 反向传播时GPU需等待SSD传输优化器状态;与大量激活值导致GPU闲置。 |
| LoHan | 模型状态→SSD 部分激活值→CPU内存 | CPU | 需要 | 激活值卸载到CPU内存产生额外卸载开销; I/O操作的上下文切换开销大。 |
), ArticleFig(id=1325495772636127479, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1325495741652803708, language=EN, label=Table 2, caption=
Tensor Storage Overhead in LLM Training, figureFileSmall=null, figureFileBig=null, tableContent=
| 张量类型 | 符号 | 精度 | 存储开销/B | 产生阶段 |
|---|
| 模型参数 | P16 | FP16 | 2N | 反向传播 |
| 优化器状态 | OS32 | FP32 | 8N | 反向传播 |
| 梯度 | G16 | FP16 | 2N | 反向传播 |
| 激活值 | A16 | FP16 | 2×Bs×L×D×Ln | 前向传播 |
), ArticleFig(id=1325495772732596472, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1325495741652803708, language=CN, label=表2, caption=
大模型训练过程中的张量存储开销
, figureFileSmall=null, figureFileBig=null, tableContent=
| 张量类型 | 符号 | 精度 | 存储开销/B | 产生阶段 |
|---|
| 模型参数 | P16 | FP16 | 2N | 反向传播 |
| 优化器状态 | OS32 | FP32 | 8N | 反向传播 |
| 梯度 | G16 | FP16 | 2N | 反向传播 |
| 激活值 | A16 | FP16 | 2×Bs×L×D×Ln | 前向传播 |
), ArticleFig(id=1325495773135249657, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1325495741652803708, language=EN, label=Table 3, caption=
Experimental Environment Configuration, figureFileSmall=null, figureFileBig=null, tableContent=
| 配置 | 配置环境 |
|---|
| CPU | Intel® Xeon® Silver 4314 (2.40 GHz) |
| 内存 | 2×256 GB DDR5 4800 MTps |
| SSD | Dell NVMe PM1743 RI E3.S+ Intel NVMe Datacenter SSD [Optane] |
| GPU | NVIDIA RTX3090/RTX4090 |
| 操作系统 | Ubuntu 22.04.5 LTS (Kernel 5.15.0) |
| SPDK版本 | v24.09 |
| CUDA | CUDA11.8 |
| PyTorch | 2.0.0 |
), ArticleFig(id=1325495773495959802, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1325495741652803708, language=CN, label=表3, caption=
实验环境配置
, figureFileSmall=null, figureFileBig=null, tableContent=
| 配置 | 配置环境 |
|---|
| CPU | Intel® Xeon® Silver 4314 (2.40 GHz) |
| 内存 | 2×256 GB DDR5 4800 MTps |
| SSD | Dell NVMe PM1743 RI E3.S+ Intel NVMe Datacenter SSD [Optane] |
| GPU | NVIDIA RTX3090/RTX4090 |
| 操作系统 | Ubuntu 22.04.5 LTS (Kernel 5.15.0) |
| SPDK版本 | v24.09 |
| CUDA | CUDA11.8 |
| PyTorch | 2.0.0 |
)], attaches=null, journal=Journal(id=1324759606947315908, delFlag=0, nameCn=计算机研究与发展, nameEn=Journal of Computer Research and Development, nameHistory1=null, nameHistory2=null, issn=1000-1239, eissn=null, cn=11-1777/TP, coden=null, periodic=0, language=CN, oaType=null, ccby=null, superviseOffice=null, ownerOffice=null, pubOffice=null, editorOffice=null, officeType=null, aims=null, clcCode=null, officeProv=null, officeCity=null, officeAddr=null, officeZip=null, officeEmail=null, officePhone=null, editDirector=null, officeDirector=null, officeDirectorPhone=null, officeStaffNum=null, officeEmpNum=null, coverPicUrl=k2l8zgqwstuxxevFX2qxVA==, journalPrice=null, startedYear=null, abbrevIsoEn=Journal of Computer Research and Development, journalRemark=null, publicationField=null, createdTime=1793776903141, updatedTime=1793850698612, createdBy=18614031015, updatedBy=13701087609, firstLetterCn=J, firstLetterEn=J, subjectCode=Electronic Information, subjectName=null, subjectCodeEn=Electronic Information, subjectNameEn=null, picCn=k2l8zgqwstuxxevFX2qxVA==, picEn=r6k0Ry+RD+c/VXj5QarmDw==, jcr=null, cjcr=null, exts=[JournalExt(id=1325069127710286688, language=CN, name=计算机研究与发展, nameHistory1=null, nameHistory2=null, managedBy=, sponsoredBy=, publishedBy=, editorOffice=, officeProv=null, officeCity=null, officeAddr=, officeZip=, editDirector=, officeDirector=null, officePhone=null, coverPicUrl=null, journalRemark=, submitArticleUrl=null, websiteUrl=, createdTime=1793850698640, updatedTime=1793850698640, createdBy=13701087609, updatedBy=13701087609, submissionGuidelinesUrl=, submissionAuthorUrl=https://crad.ict.ac.cn/journalx/authorLogOn.action, submissionEditorUrl=https://crad.ict.ac.cn/journalx/editorLogOn.action, submissionReviewUrl=https://crad.ict.ac.cn/journalx/expertLogOn.action, submissionCeEditorUrl=https://crad.ict.ac.cn/journalx/editorInChiefLogOn.action, submissionAeEditorUrl=, option={"copyright":""}), JournalExt(id=1325069127777395553, language=EN, name=Journal of Computer Research and Development, nameHistory1=null, nameHistory2=null, managedBy=, sponsoredBy=, publishedBy=, editorOffice=, officeProv=null, officeCity=null, officeAddr=, officeZip=, editDirector=, officeDirector=null, officePhone=null, coverPicUrl=null, journalRemark=, submitArticleUrl=null, websiteUrl=, createdTime=1793850698656, updatedTime=1793850698656, createdBy=13701087609, updatedBy=13701087609, submissionGuidelinesUrl=, submissionAuthorUrl=https://crad.ict.ac.cn/journalx/authorLogOn.action, submissionEditorUrl=https://crad.ict.ac.cn/journalx/editorLogOn.action, submissionReviewUrl=https://crad.ict.ac.cn/journalx/expertLogOn.action, submissionCeEditorUrl=https://crad.ict.ac.cn/journalx/editorInChiefLogOn.action, submissionAeEditorUrl=, option={"copyright":""})], databaseList=null, tenantJournalId=1324759914146529478, websiteList=[Website(id=1325070300139892771, webName=null, webTitle=null, webDomain=null, webCopyrigh=null, webIpcNo=null, seoTitle=null, seoKeywords=null, seoDescription=null, tenantJournalId=null, journalId=1324759914146529478, journalNameCn=null, journalNameEn=null, grayFlag=null, tenantId=1146029695717560320, platformId=null, journalGroupId=null, journalGroupNameCn=null, journalGroupNameEn=null, type=1, domain=https://castjournals.cast.org.cn/joweb/jsjyjyfz/CN, language=CN, createTime=1793850978169, createBy=18614031015, updateTime=1793851040259, updateBy=18614031015, name=计算机研究与发展-中文, tplId=1146099689490845704, title=计算机研究与发展, delFlag=0, indexPage=/home, props=[WebsiteProps(id=1325070893751349360, tenantId=1146029695717560320, journalId=null, journalGroupId=null, siteId=1325070300139892771, code=articleTextType, value=kx, createTime=1793851119697, updateTime=1793851119697, creator=18614031015, updator=18614031015), WebsiteProps(id=1325070893730377837, tenantId=1146029695717560320, journalId=null, journalGroupId=null, siteId=1325070300139892771, code=banner, value=null, createTime=1793851119692, updateTime=1793851119692, creator=18614031015, updator=18614031015), WebsiteProps(id=1325070893772320883, tenantId=1146029695717560320, journalId=null, journalGroupId=null, siteId=1325070300139892771, code=grayFlag, value=0, createTime=1793851119702, updateTime=1793851119702, creator=18614031015, updator=18614031015), WebsiteProps(id=1325070893721989228, tenantId=1146029695717560320, journalId=null, journalGroupId=null, siteId=1325070300139892771, code=logo, value=https://castjournals.cast.org.cn/joweb/jsjyjyfz/CN/file/pic?fileId=U27tDsy717YF7XSPA+Vwdw==, createTime=1793851119690, updateTime=1793851119690, creator=18614031015, updator=18614031015), WebsiteProps(id=1325070893784903797, tenantId=1146029695717560320, journalId=null, journalGroupId=null, siteId=1325070300139892771, code=minRunFlag, value=0, createTime=1793851119705, updateTime=1793851119705, creator=18614031015, updator=18614031015), WebsiteProps(id=1325070893742960751, tenantId=1146029695717560320, journalId=null, journalGroupId=null, siteId=1325070300139892771, code=picServerUrl, value=https://castjournals.cast.org.cn/joweb/jsjyjyfz/CN/file/pic, createTime=1793851119695, updateTime=1793851119695, creator=18614031015, updator=18614031015), WebsiteProps(id=1325070893780709492, tenantId=1146029695717560320, journalId=null, journalGroupId=null, siteId=1325070300139892771, code=silenceFlag, value=0, createTime=1793851119704, updateTime=1793851119704, creator=18614031015, updator=18614031015), WebsiteProps(id=1325070893738766446, tenantId=1146029695717560320, journalId=null, journalGroupId=null, siteId=1325070300139892771, code=staticResourcePath, value=https://castjournals.cast.org.cn/joweb/cast_kjdb_cn_619/, createTime=1793851119694, updateTime=1793851119694, creator=18614031015, updator=18614031015), WebsiteProps(id=1325070893755543665, tenantId=1146029695717560320, journalId=null, journalGroupId=null, siteId=1325070300139892771, code=themeColor, value=null, createTime=1793851119698, updateTime=1793851119698, creator=18614031015, updator=18614031015), WebsiteProps(id=1325070893763932274, tenantId=1146029695717560320, journalId=null, journalGroupId=null, siteId=1325070300139892771, code=themeStyle, value=null, createTime=1793851119700, updateTime=1793851119700, creator=18614031015, updator=18614031015)]), Website(id=1325070300580294698, webName=null, webTitle=null, webDomain=null, webCopyrigh=null, webIpcNo=null, seoTitle=null, seoKeywords=null, seoDescription=null, tenantJournalId=null, journalId=1324759914146529478, journalNameCn=null, journalNameEn=null, grayFlag=null, tenantId=1146029695717560320, platformId=null, journalGroupId=null, journalGroupNameCn=null, journalGroupNameEn=null, type=1, domain=https://castjournals.cast.org.cn/joweb/jsjyjyfz/EN, language=EN, createTime=1793850978274, createBy=18614031015, updateTime=1793851036638, updateBy=18614031015, name=计算机研究与发展-英文, tplId=1146101810881728533, title=Journal of Computer Research and Development, delFlag=0, indexPage=/home, props=[WebsiteProps(id=1325070925288321148, tenantId=1146029695717560320, journalId=null, journalGroupId=null, siteId=1325070300580294698, code=articleTextType, value=kx, createTime=1793851127216, updateTime=1793851127216, creator=18614031015, updator=18614031015), WebsiteProps(id=1325070925263155321, tenantId=1146029695717560320, journalId=null, journalGroupId=null, siteId=1325070300580294698, code=banner, value=null, createTime=1793851127210, updateTime=1793851127210, creator=18614031015, updator=18614031015), WebsiteProps(id=1325070925309292671, tenantId=1146029695717560320, journalId=null, journalGroupId=null, siteId=1325070300580294698, code=grayFlag, value=0, createTime=1793851127221, updateTime=1793851127221, creator=18614031015, updator=18614031015), WebsiteProps(id=1325070925254766712, tenantId=1146029695717560320, journalId=null, journalGroupId=null, siteId=1325070300580294698, code=logo, value=https://castjournals.cast.org.cn/joweb/jsjyjyfz/EN/file/pic?fileId=U27tDsy717YF7XSPA+Vwdw==, createTime=1793851127208, updateTime=1793851127208, creator=18614031015, updator=18614031015), WebsiteProps(id=1325070925326069889, tenantId=1146029695717560320, journalId=null, journalGroupId=null, siteId=1325070300580294698, code=minRunFlag, value=0, createTime=1793851127225, updateTime=1793851127225, creator=18614031015, updator=18614031015), WebsiteProps(id=1325070925279932539, tenantId=1146029695717560320, journalId=null, journalGroupId=null, siteId=1325070300580294698, code=picServerUrl, value=https://castjournals.cast.org.cn/joweb/jsjyjyfz/EN/file/pic, createTime=1793851127214, updateTime=1793851127214, creator=18614031015, updator=18614031015), WebsiteProps(id=1325070925317681280, tenantId=1146029695717560320, journalId=null, journalGroupId=null, siteId=1325070300580294698, code=silenceFlag, value=0, createTime=1793851127223, updateTime=1793851127223, creator=18614031015, updator=18614031015), WebsiteProps(id=1325070925271543930, tenantId=1146029695717560320, journalId=null, journalGroupId=null, siteId=1325070300580294698, code=staticResourcePath, value=https://castjournals.cast.org.cn/joweb/cast_kjdb_en_623/, createTime=1793851127212, updateTime=1793851127212, creator=18614031015, updator=18614031015), WebsiteProps(id=1325070925292515453, tenantId=1146029695717560320, journalId=null, journalGroupId=null, siteId=1325070300580294698, code=themeColor, value=null, createTime=1793851127217, updateTime=1793851127217, creator=18614031015, updator=18614031015), WebsiteProps(id=1325070925300904062, tenantId=1146029695717560320, journalId=null, journalGroupId=null, siteId=1325070300580294698, code=themeStyle, value=null, createTime=1793851127219, updateTime=1793851127219, creator=18614031015, updator=18614031015)])], journalTitle=计算机研究与发展, weixinUrl=null, journalUrl=https://crad.ict.ac.cn/, iacademicId=null, status=1, seqNo=null, journalTitleEn=Journal of Computer Research and Development, journalPhotoCn=k2l8zgqwstuxxevFX2qxVA==, journalPhotoEn=r6k0Ry+RD+c/VXj5QarmDw==, journalFirstLetter=J, journalRecommend=null, journalNew=null, journalCollection=null, jcrJf=null, cjcrJf=null, jcrJfStr=null, cjcrJfStr=null, submissionFirstDecision=null, sciSubjectClassification=null, casSubjectClassification=null, citeScore=null, totalCitationFrequency=null, icpCode=null, psCode=null, advertisingLicenseCode=null, copyrightInformation=null, country=null, option=, provinceCode=null, provinceName=null, collectFlag=false, interPubPlatform=, interPubPlatformUrl=null), detailUrlCn=https://castjournals.cast.org.cn/joweb/jsjyjyfz/CN/10.7544/issn1000-1239.202550478, detailUrlEn=https://castjournals.cast.org.cn/joweb/jsjyjyfz/EN/10.7544/issn1000-1239.202550478, pdfUrlCn=https://castjournals.cast.org.cn/joweb/jsjyjyfz/CN/PDF/10.7544/issn1000-1239.202550478, pdfUrlEn=https://castjournals.cast.org.cn/joweb/jsjyjyfz/EN/PDF/10.7544/issn1000-1239.202550478, aliStartDate=0, aliEndDate=0, collectionFlag=false, citedCount=null, citedUrl=null, previewStatus=0, delFlag=0, hasFullText=1, orderTime=1772294400000, fullTextJson=null, articleText=null, reference=null)