Article(id=1327012676691145592, tenantId=1146029695717560320, journalId=1324759914146529478, issueId=1327012646391493336, articleNumber=null, orderNo=null, doi=10.7544/issn1000-1239.202550127, pmid=null, cstr=null, oa=null, hot=null, price=null, onlineType=0, articleFormat=0, articleType=null, articleTypeStr=research-article, receivedDate=1740758400000, receivedDateStr=2025-03-01, revisedDate=1744473600000, revisedDateStr=2025-04-13, acceptedDate=null, acceptedDateStr=null, onlineDate=1794314076825, onlineDateStr=2026-11-10, pubDate=1748793600000, pubDateStr=2025-06-02, doiRegisterDate=null, doiRegisterDateStr=null, onlineIssueDate=1794314076825, onlineIssueDateStr=2026-11-10, onlineJustAcceptDate=null, onlineJustAcceptDateStr=null, onlineFirstDate=null, onlineFirstDateStr=null, sourceXml=null, magXml=null, createTime=1794314076825, creator=13701087609, updateTime=1794314076825, updator=13701087609, issue=Issue{id=1327012646391493336, tenantId=1146029695717560320, journalId=1324759914146529478, year='2025', volume='62', issue='6', pageStart='1331', pageEnd='1593', issueExtLink='null', onlineDate='null', pubDate='1748793600000', pubDateStr='2025-06-02', beforeIssueId=null, nextIssueId=null, price=null, status=1, issueComplete=1, articleOrder=1, issueType=-1, specialIssue=null, createTime=1794314069601, creator='13701087609', updateTime=1794376817187, updator='13701087609', preIssue=null, nextIssue=null, articleTotal=null, ext={EN=IssueExt(id=1327275829270262152, tenantId=1146029695717560320, journalId=1324759914146529478, issueId=1327012646391493336, language=EN, specialIssueTitle=, coverIllustrator=null, specialIssueEditor=, specialIssueAbout=), CN=IssueExt(id=1327275829270262153, tenantId=1146029695717560320, journalId=1324759914146529478, issueId=1327012646391493336, language=CN, specialIssueTitle=, coverIllustrator=null, specialIssueEditor=, specialIssueAbout=)}, issueFiles=null, downloadFileDto=null}, startPage=1513, endPage=1529, ext={EN=ArticleExt(id=1327012677051855738, articleId=1327012676691145592, tenantId=1146029695717560320, journalId=1324759914146529478, language=EN, title=Pipe-RLHF: A Computation Mode-Aware Parallel Framework for RLHF, columnId=1327012676955386745, journalTitle=Journal of Computer Research and Development, columnName=Chip Architecture Design, runingTitle=null, highlight=null, articleAbstract=
Reinforcement learning with human feedback (RLHF) has been proven effective in aligning large language models (LLMs) with human preferences. The most costly part of RLHF is proximal policy optimization (PPO), which consists of three dependent steps. Different PPO steps in RLHF exhibit different computation modes, simply employing the same parallelization strategy to accelerate all steps that involve multiple model variants, as done in existing frameworks, will lead to poor performance in the PPO generation step due to insufficient utilization of computational resources. Thus, we introduce Pipe-RLHF, a parallelism framework for RLHF fine-tuning, which adaptively employs distinct parallelization strategies for different steps according to the computation mode. Specifically, we first investigate the characteristics of various computation modes to explore their best-fit parallelization approach. And then, we present a novel delayed inter-batch pipeline parallelization approach specifically designed for the PPO generation step, enabling the sufficient utilization of computational resources. Subsequently, based on the proposed inter-batch pipeline parallelization approach, we define a hierarchical parallel plan space for distributed RLHF fine-tuning. Finally, we present optimization algorithms to find the optimal parallelization plan from the defined hierarchical parallel plan space to minimize the overall time consumption. Implementation and evaluation across multiple LLMs demonstrates that the proposed Pipe-RLHF achieves 3.7 times speedup compared with existing methods while achieving near-linear scalability.
, authors=Ying Xu
1, 2, Mengdi Wang
1, Long Cheng
3, Lian Liu
1, 2, Shixin Zhao
1, 2, Lei Zhang
1, Ying Wang
1, authorsList=Ying Xu, Mengdi Wang, Long Cheng, Lian Liu, Shixin Zhao, Lei Zhang, Ying Wang, authorCompany=null, correspAuthors=null, authorNote=
born in 1996. PhD candidate. Her main research interests include machine learning system and parallel deep learning framework
born in 1995. Assistant professor. Her main research interests include accelerator design and multi-core system
born in 1985. PhD, professor, PhD supervisor. His main research interest includes distributed computing and high-performance systems
born in 1999. PhD candidate. His main research interest includes LLM infrastructure、hardware-software co-design
born in 1999. PhD candidate. Her main research interest includes computer architecture
born in 1981. PhD, professor. Member of CCF. His main research interests include chip and computing system
born in 1985. PhD, professor. Senior member of CCF. His main research interests include novel EDA, and processor and memory system architecture
, correspAuthorsNote=null, copyrightStatement=Copyright © 2025 Journal of Computer Research and Development. All rights reserved., copyrightOwner=null, extLink=null, articleAbsUrl=null, sourceXml=null, magXml=null, pdfUrl=null, pdf=null, pdfFileSize=null, pdfExtLink=null, richHtmlUrl=null, mobilePdfUrl=null, reviewReport=null, pdfFirstPage=null, abstractGraph=null, abstractGraphContent=null, abstractVideo=null, citation=null, cebUrl=null, magXmlContent=null, mapNumber=null, fund=null), CN=ArticleExt(id=1327012682869355437, articleId=1327012676691145592, tenantId=1146029695717560320, journalId=1324759914146529478, language=CN, title=Pipe-RLHF: 计算模式感知的RLHF并行加速框架, columnId=1327012677139936123, journalTitle=计算机研究与发展, columnName=芯片架构设计, runingTitle=null, highlight=null, articleAbstract=
基于人类反馈的强化学习(reinforcement learning with human feedback,RLHF)作为当前大语言模型(large language models,LLMs)对齐的主流方法,其核心优化算法——近端策略优化(proximal policy optimization,PPO)却面临着显著的效率问题. PPO由生成、推理、训练3个相互关联的阶段组成,各个阶段有着不同的计算特性. 然而,现有的RLHF并行框架采用相同并行策略顺序执行PPO的所有阶段,这导致以下2个问题:其一,生成阶段不能充分利用计算资源,进而影响整体效率;其二,阶段间严格串行执行,未能充分利用潜在并行性. 针对上述问题,提出了一个新型RLHF并行框架——Pipe-RLHF.该框架能够自适应地根据各阶段的计算特征确定最优并行策略,突破现有阶段串行范式,采用异步PPO算法发掘阶段间的并行性. 具体而言,创新性地提出了适用于PPO生成阶段的延迟批间流水线并行方法,显著提升了该阶段的计算资源利用率;再次,使用异步PPO解放阶段间的依赖关系,将阶段间并行应用到PPO的加速上;最后,针对PPO算法的整体优化,构建了分层并行策略空间,并提出了一套优化算法以实现该空间中的最优解搜索. 通过在多个大语言模型上的性能评估实验表明,相较于现有方法,Pipe-RLHF最高可实现3.7倍的加速比,充分验证了该框架的有效性和优越性.
, authors=徐颖
1, 2, 王梦迪
1, 程龙
3, 刘炼
1, 2, 赵世新
1, 2, 张磊
1, 王颖
1, authorsList=徐颖, 王梦迪, 程龙, 刘炼, 赵世新, 张磊, 王颖, authorCompany=null, correspAuthors=null, authorNote=
 |
1996年生. 博士研究生. 主要研究方向为机器学习系统、深度学习并行框架 |
 |
1995年生. 助理研究员. 主要研究方向为加速器设计、多核系统 |
 |
1985年生. 博士,教授,博士生导师. 主要研究方向为分布式计算、高性能系统 |
 |
1999年生. 博士研究生. 主要研究方向为大语言模型系统架构设计、软硬件协同设计 |
 |
1999年生. 博士研究生. 主要研究方向为计算机体系结构 |
 |
1981年生. 博士,正高级工程师. CCF会员. 主要研究方向为芯片、计算系统 |
 |
1985年生. 博士,研究员. CCF高级会员. 主要研究方向为新型EDA、处理器与存储系统体系结构 |
, correspAuthorsNote=null, copyrightStatement=版权所有 © 2025 计算机研究与发展编辑部, copyrightOwner=null, extLink=null, articleAbsUrl=null, sourceXml=eyqMF2eC9JdoOFRO5wUBMw==, magXml=caREVS5/O8+/AKaZPaXaMA==, pdfUrl=null, pdf=gE07r3uX3SD9rYcu5dgDHw==, pdfFileSize=1541416, pdfExtLink=null, richHtmlUrl=null, mobilePdfUrl=null, reviewReport=null, pdfFirstPage=null, abstractGraph=TFUkHvRlMl9graXj4BiZNQ==, abstractGraphContent=null, abstractVideo=null, citation=null, cebUrl=null, magXmlContent=WUc34kbl3O/mT8LREHmWaA==, mapNumber=null, fund=null)}, authors=[Author(id=1327012685042004921, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012676691145592, orderNo=0, firstName=null, middleName=null, lastName=null, nameCn=null, orcid=null, stid=null, country=null, authorPic=null, dead=0, email=xuying18z@ict.ac.cn, emailSecond=null, emailThird=null, correspondingAuthor=0, authorType=1, ext={EN=AuthorExt(id=1327012685125891004, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012676691145592, authorId=1327012685042004921, language=EN, stringName=Ying Xu, firstName=Ying, middleName=null, lastName=Xu, prefix=null, suffix=null, authorComment=null, nameInitials=null, affiliation=null, department=null, xref=
1, 2, address=
1Institute of Computing Technology, Chinese Academy of Sciences, Beijing 100190
2University of Chinese Academy of Sciences, Beijing 100190, bio={"content":"
born in 1996. PhD candidate. Her main research interests include machine learning system and parallel deep learning framework
"}, bioImg=null, bioContent=
born in 1996. PhD candidate. Her main research interests include machine learning system and parallel deep learning framework
, aboutCorrespAuthor=null), CN=AuthorExt(id=1327012685201388477, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012676691145592, authorId=1327012685042004921, language=CN, stringName=徐颖, firstName=null, middleName=null, lastName=null, prefix=null, suffix=null, authorComment=null, nameInitials=null, affiliation=null, department=null, xref=
1, 2, address=
1中国科学院计算技术研究所 北京 100190
2中国科学院大学 北京 100190, bio={"img":"DENf+2BfcH2ih411a8jwcg==","content":"
1996年生. 博士研究生. 主要研究方向为机器学习系统、深度学习并行框架
"}, bioImg=DENf+2BfcH2ih411a8jwcg==, bioContent=
1996年生. 博士研究生. 主要研究方向为机器学习系统、深度学习并行框架
, aboutCorrespAuthor=null)}, companyList=[AuthorCompany(id=1327012683150373806, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012676691145592, xref=1, ext=[AuthorCompanyExt(id=1327012683158762415, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012676691145592, companyId=1327012683150373806, language=EN, country=null, province=null, city=null, postcode=null, companyName=null, departmentName=null, remark=
1Institute of Computing Technology, Chinese Academy of Sciences, Beijing 100190), AuthorCompanyExt(id=1327012683167151024, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012676691145592, companyId=1327012683150373806, language=CN, country=null, province=null, city=null, postcode=null, companyName=null, departmentName=null, remark=
1中国科学院计算技术研究所 北京 100190)]), AuthorCompany(id=1327012683255231409, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012676691145592, xref=2, ext=[AuthorCompanyExt(id=1327012683284591538, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012676691145592, companyId=1327012683255231409, language=EN, country=null, province=null, city=null, postcode=null, companyName=null, departmentName=null, remark=
2University of Chinese Academy of Sciences, Beijing 100190), AuthorCompanyExt(id=1327012683297174451, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012676691145592, companyId=1327012683255231409, language=CN, country=null, province=null, city=null, postcode=null, companyName=null, departmentName=null, remark=
2中国科学院大学 北京 100190)])]), Author(id=1327012685276885951, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012676691145592, orderNo=1, firstName=null, middleName=null, lastName=null, nameCn=null, orcid=null, stid=null, country=null, authorPic=null, dead=0, email=null, emailSecond=null, emailThird=null, correspondingAuthor=0, authorType=1, ext={EN=AuthorExt(id=1327012685356577729, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012676691145592, authorId=1327012685276885951, language=EN, stringName=Mengdi Wang, firstName=Mengdi, middleName=null, lastName=Wang, prefix=null, suffix=null, authorComment=null, nameInitials=null, affiliation=null, department=null, xref=
1, address=
1Institute of Computing Technology, Chinese Academy of Sciences, Beijing 100190, bio={"content":"
born in 1995. Assistant professor. Her main research interests include accelerator design and multi-core system
"}, bioImg=null, bioContent=
born in 1995. Assistant professor. Her main research interests include accelerator design and multi-core system
, aboutCorrespAuthor=null), CN=AuthorExt(id=1327012685415297986, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012676691145592, authorId=1327012685276885951, language=CN, stringName=王梦迪, firstName=null, middleName=null, lastName=null, prefix=null, suffix=null, authorComment=null, nameInitials=null, affiliation=null, department=null, xref=
1, address=
1中国科学院计算技术研究所 北京 100190, bio={"img":"XdV6mzDHK9VkDs2B56DDUA==","content":"
1995年生. 助理研究员. 主要研究方向为加速器设计、多核系统
"}, bioImg=XdV6mzDHK9VkDs2B56DDUA==, bioContent=
1995年生. 助理研究员. 主要研究方向为加速器设计、多核系统
, aboutCorrespAuthor=null)}, companyList=[AuthorCompany(id=1327012683150373806, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012676691145592, xref=1, ext=[AuthorCompanyExt(id=1327012683158762415, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012676691145592, companyId=1327012683150373806, language=EN, country=null, province=null, city=null, postcode=null, companyName=null, departmentName=null, remark=
1Institute of Computing Technology, Chinese Academy of Sciences, Beijing 100190), AuthorCompanyExt(id=1327012683167151024, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012676691145592, companyId=1327012683150373806, language=CN, country=null, province=null, city=null, postcode=null, companyName=null, departmentName=null, remark=
1中国科学院计算技术研究所 北京 100190)])]), Author(id=1327012685482406852, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012676691145592, orderNo=2, firstName=null, middleName=null, lastName=null, nameCn=null, orcid=null, stid=null, country=null, authorPic=null, dead=0, email=null, emailSecond=null, emailThird=null, correspondingAuthor=0, authorType=1, ext={EN=AuthorExt(id=1327012685570487238, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012676691145592, authorId=1327012685482406852, language=EN, stringName=Long Cheng, firstName=Long, middleName=null, lastName=Cheng, prefix=null, suffix=null, authorComment=null, nameInitials=null, affiliation=null, department=null, xref=
3, address=
3School of Control and Computer Engineering, North China Electric Power University, Beijing 102206, bio={"content":"
born in 1985. PhD, professor, PhD supervisor. His main research interest includes distributed computing and high-performance systems
"}, bioImg=null, bioContent=
born in 1985. PhD, professor, PhD supervisor. His main research interest includes distributed computing and high-performance systems
, aboutCorrespAuthor=null), CN=AuthorExt(id=1327012685683733447, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012676691145592, authorId=1327012685482406852, language=CN, stringName=程龙, firstName=null, middleName=null, lastName=null, prefix=null, suffix=null, authorComment=null, nameInitials=null, affiliation=null, department=null, xref=
3, address=
3华北电力大学控制与计算机工程学院 北京 102206, bio={"img":"Gqw18szoYQR2/e8+Sby8Jw==","content":"
1985年生. 博士,教授,博士生导师. 主要研究方向为分布式计算、高性能系统
"}, bioImg=Gqw18szoYQR2/e8+Sby8Jw==, bioContent=
1985年生. 博士,教授,博士生导师. 主要研究方向为分布式计算、高性能系统
, aboutCorrespAuthor=null)}, companyList=[AuthorCompany(id=1327012684958118836, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012676691145592, xref=3, ext=[AuthorCompanyExt(id=1327012684962313141, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012676691145592, companyId=1327012684958118836, language=EN, country=null, province=null, city=null, postcode=null, companyName=null, departmentName=null, remark=
3School of Control and Computer Engineering, North China Electric Power University, Beijing 102206), AuthorCompanyExt(id=1327012684970701750, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012676691145592, companyId=1327012684958118836, language=CN, country=null, province=null, city=null, postcode=null, companyName=null, departmentName=null, remark=
3华北电力大学控制与计算机工程学院 北京 102206)])]), Author(id=1327012685755036617, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012676691145592, orderNo=3, firstName=null, middleName=null, lastName=null, nameCn=null, orcid=null, stid=null, country=null, authorPic=null, dead=0, email=null, emailSecond=null, emailThird=null, correspondingAuthor=0, authorType=1, ext={EN=AuthorExt(id=1327012685834728396, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012676691145592, authorId=1327012685755036617, language=EN, stringName=Lian Liu, firstName=Lian, middleName=null, lastName=Liu, prefix=null, suffix=null, authorComment=null, nameInitials=null, affiliation=null, department=null, xref=
1, 2, address=
1Institute of Computing Technology, Chinese Academy of Sciences, Beijing 100190
2University of Chinese Academy of Sciences, Beijing 100190, bio={"content":"
born in 1999. PhD candidate. His main research interest includes LLM infrastructure、hardware-software co-design
"}, bioImg=null, bioContent=
born in 1999. PhD candidate. His main research interest includes LLM infrastructure、hardware-software co-design
, aboutCorrespAuthor=null), CN=AuthorExt(id=1327012685897642957, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012676691145592, authorId=1327012685755036617, language=CN, stringName=刘炼, firstName=null, middleName=null, lastName=null, prefix=null, suffix=null, authorComment=null, nameInitials=null, affiliation=null, department=null, xref=
1, 2, address=
1中国科学院计算技术研究所 北京 100190
2中国科学院大学 北京 100190, bio={"img":"AkYDGV2VDDR9H0rj0XN4Lg==","content":"
1999年生. 博士研究生. 主要研究方向为大语言模型系统架构设计、软硬件协同设计
"}, bioImg=AkYDGV2VDDR9H0rj0XN4Lg==, bioContent=
1999年生. 博士研究生. 主要研究方向为大语言模型系统架构设计、软硬件协同设计
, aboutCorrespAuthor=null)}, companyList=[AuthorCompany(id=1327012683150373806, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012676691145592, xref=1, ext=[AuthorCompanyExt(id=1327012683158762415, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012676691145592, companyId=1327012683150373806, language=EN, country=null, province=null, city=null, postcode=null, companyName=null, departmentName=null, remark=
1Institute of Computing Technology, Chinese Academy of Sciences, Beijing 100190), AuthorCompanyExt(id=1327012683167151024, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012676691145592, companyId=1327012683150373806, language=CN, country=null, province=null, city=null, postcode=null, companyName=null, departmentName=null, remark=
1中国科学院计算技术研究所 北京 100190)]), AuthorCompany(id=1327012683255231409, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012676691145592, xref=2, ext=[AuthorCompanyExt(id=1327012683284591538, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012676691145592, companyId=1327012683255231409, language=EN, country=null, province=null, city=null, postcode=null, companyName=null, departmentName=null, remark=
2University of Chinese Academy of Sciences, Beijing 100190), AuthorCompanyExt(id=1327012683297174451, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012676691145592, companyId=1327012683255231409, language=CN, country=null, province=null, city=null, postcode=null, companyName=null, departmentName=null, remark=
2中国科学院大学 北京 100190)])]), Author(id=1327012685964751823, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012676691145592, orderNo=4, firstName=null, middleName=null, lastName=null, nameCn=null, orcid=null, stid=null, country=null, authorPic=null, dead=0, email=null, emailSecond=null, emailThird=null, correspondingAuthor=0, authorType=1, ext={EN=AuthorExt(id=1327012686069609426, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012676691145592, authorId=1327012685964751823, language=EN, stringName=Shixin Zhao, firstName=Shixin, middleName=null, lastName=Zhao, prefix=null, suffix=null, authorComment=null, nameInitials=null, affiliation=null, department=null, xref=
1, 2, address=
1Institute of Computing Technology, Chinese Academy of Sciences, Beijing 100190
2University of Chinese Academy of Sciences, Beijing 100190, bio={"content":"
born in 1999. PhD candidate. Her main research interest includes computer architecture
"}, bioImg=null, bioContent=
born in 1999. PhD candidate. Her main research interest includes computer architecture
, aboutCorrespAuthor=null), CN=AuthorExt(id=1327012686161884115, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012676691145592, authorId=1327012685964751823, language=CN, stringName=赵世新, firstName=null, middleName=null, lastName=null, prefix=null, suffix=null, authorComment=null, nameInitials=null, affiliation=null, department=null, xref=
1, 2, address=
1中国科学院计算技术研究所 北京 100190
2中国科学院大学 北京 100190, bio={"img":"JQwhJoDWwZGRTG1do7/HbQ==","content":"
1999年生. 博士研究生. 主要研究方向为计算机体系结构
"}, bioImg=JQwhJoDWwZGRTG1do7/HbQ==, bioContent=
1999年生. 博士研究生. 主要研究方向为计算机体系结构
, aboutCorrespAuthor=null)}, companyList=[AuthorCompany(id=1327012683150373806, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012676691145592, xref=1, ext=[AuthorCompanyExt(id=1327012683158762415, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012676691145592, companyId=1327012683150373806, language=EN, country=null, province=null, city=null, postcode=null, companyName=null, departmentName=null, remark=
1Institute of Computing Technology, Chinese Academy of Sciences, Beijing 100190), AuthorCompanyExt(id=1327012683167151024, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012676691145592, companyId=1327012683150373806, language=CN, country=null, province=null, city=null, postcode=null, companyName=null, departmentName=null, remark=
1中国科学院计算技术研究所 北京 100190)]), AuthorCompany(id=1327012683255231409, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012676691145592, xref=2, ext=[AuthorCompanyExt(id=1327012683284591538, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012676691145592, companyId=1327012683255231409, language=EN, country=null, province=null, city=null, postcode=null, companyName=null, departmentName=null, remark=
2University of Chinese Academy of Sciences, Beijing 100190), AuthorCompanyExt(id=1327012683297174451, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012676691145592, companyId=1327012683255231409, language=CN, country=null, province=null, city=null, postcode=null, companyName=null, departmentName=null, remark=
2中国科学院大学 北京 100190)])]), Author(id=1327012686224798677, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012676691145592, orderNo=5, firstName=null, middleName=null, lastName=null, nameCn=null, orcid=null, stid=null, country=null, authorPic=null, dead=0, email=null, emailSecond=null, emailThird=null, correspondingAuthor=0, authorType=1, ext={EN=AuthorExt(id=1327012686312879063, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012676691145592, authorId=1327012686224798677, language=EN, stringName=Lei Zhang, firstName=Lei, middleName=null, lastName=Zhang, prefix=null, suffix=null, authorComment=null, nameInitials=null, affiliation=null, department=null, xref=
1, address=
1Institute of Computing Technology, Chinese Academy of Sciences, Beijing 100190, bio={"content":"
born in 1981. PhD, professor. Member of CCF. His main research interests include chip and computing system
"}, bioImg=null, bioContent=
born in 1981. PhD, professor. Member of CCF. His main research interests include chip and computing system
, aboutCorrespAuthor=null), CN=AuthorExt(id=1327012686379987928, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012676691145592, authorId=1327012686224798677, language=CN, stringName=张磊, firstName=null, middleName=null, lastName=null, prefix=null, suffix=null, authorComment=null, nameInitials=null, affiliation=null, department=null, xref=
1, address=
1中国科学院计算技术研究所 北京 100190, bio={"img":"SG3c/MaOAOCQlehMUFcuog==","content":"
1981年生. 博士,正高级工程师. CCF会员. 主要研究方向为芯片、计算系统
"}, bioImg=SG3c/MaOAOCQlehMUFcuog==, bioContent=
1981年生. 博士,正高级工程师. CCF会员. 主要研究方向为芯片、计算系统
, aboutCorrespAuthor=null)}, companyList=[AuthorCompany(id=1327012683150373806, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012676691145592, xref=1, ext=[AuthorCompanyExt(id=1327012683158762415, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012676691145592, companyId=1327012683150373806, language=EN, country=null, province=null, city=null, postcode=null, companyName=null, departmentName=null, remark=
1Institute of Computing Technology, Chinese Academy of Sciences, Beijing 100190), AuthorCompanyExt(id=1327012683167151024, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012676691145592, companyId=1327012683150373806, language=CN, country=null, province=null, city=null, postcode=null, companyName=null, departmentName=null, remark=
1中国科学院计算技术研究所 北京 100190)])]), Author(id=1327012686438708186, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012676691145592, orderNo=6, firstName=null, middleName=null, lastName=null, nameCn=null, orcid=null, stid=null, country=null, authorPic=null, dead=0, email=null, emailSecond=null, emailThird=null, correspondingAuthor=0, authorType=1, ext={EN=AuthorExt(id=1327012686530982876, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012676691145592, authorId=1327012686438708186, language=EN, stringName=Ying Wang, firstName=Ying, middleName=null, lastName=Wang, prefix=null, suffix=null, authorComment=null, nameInitials=null, affiliation=null, department=null, xref=
1, address=
1Institute of Computing Technology, Chinese Academy of Sciences, Beijing 100190, bio={"content":"
born in 1985. PhD, professor. Senior member of CCF. His main research interests include novel EDA, and processor and memory system architecture
"}, bioImg=null, bioContent=
born in 1985. PhD, professor. Senior member of CCF. His main research interests include novel EDA, and processor and memory system architecture
, aboutCorrespAuthor=null), CN=AuthorExt(id=1327012686627451869, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012676691145592, authorId=1327012686438708186, language=CN, stringName=王颖, firstName=null, middleName=null, lastName=null, prefix=null, suffix=null, authorComment=null, nameInitials=null, affiliation=null, department=null, xref=
1, address=
1中国科学院计算技术研究所 北京 100190, bio={"img":"oq44dw3DfE6YPKNN0iX3VQ==","content":"
1985年生. 博士,研究员. CCF高级会员. 主要研究方向为新型EDA、处理器与存储系统体系结构
"}, bioImg=oq44dw3DfE6YPKNN0iX3VQ==, bioContent=
1985年生. 博士,研究员. CCF高级会员. 主要研究方向为新型EDA、处理器与存储系统体系结构
, aboutCorrespAuthor=null)}, companyList=[AuthorCompany(id=1327012683150373806, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012676691145592, xref=1, ext=[AuthorCompanyExt(id=1327012683158762415, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012676691145592, companyId=1327012683150373806, language=EN, country=null, province=null, city=null, postcode=null, companyName=null, departmentName=null, remark=
1Institute of Computing Technology, Chinese Academy of Sciences, Beijing 100190), AuthorCompanyExt(id=1327012683167151024, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012676691145592, companyId=1327012683150373806, language=CN, country=null, province=null, city=null, postcode=null, companyName=null, departmentName=null, remark=
1中国科学院计算技术研究所 北京 100190)])])], keywords=[Keyword(id=1327012686786835422, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012676691145592, language=EN, orderNo=1, keyword=reinforcement learning with human feedback (RLHF)), Keyword(id=1327012686853944287, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012676691145592, language=EN, orderNo=2, keyword=proximal policy optimization (PPO)), Keyword(id=1327012686925247456, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012676691145592, language=EN, orderNo=3, keyword=large language models fine-tuning), Keyword(id=1327012687042687969, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012676691145592, language=EN, orderNo=4, keyword=distributed systems), Keyword(id=1327012687143351266, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012676691145592, language=EN, orderNo=5, keyword=parallel computing), Keyword(id=1327012687332094948, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012676691145592, language=CN, orderNo=1, keyword=基于人类反馈的强化学习), Keyword(id=1327012687411786725, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012676691145592, language=CN, orderNo=2, keyword=近端策略优化), Keyword(id=1327012687483089894, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012676691145592, language=CN, orderNo=3, keyword=大模型微调), Keyword(id=1327012687566975975, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012676691145592, language=CN, orderNo=4, keyword=分布式系统), Keyword(id=1327012687663444968, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012676691145592, language=CN, orderNo=5, keyword=并行计算)], refs=[Reference(id=1327012694005231635, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012676691145592, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=1, rfOrder=0, authorNames=null, journalName=null, refType=null, unstructuredReference=Christiano P F, Leike J, Brown T B, et al. Deep reinforcement learning from human preferences[C]//Proc of the 31st Int Conf on Neural Information Processing Systems. Red Hook, NY: Curran Associates Inc., 2017: 4302–4310, articleTitle=null, refAbstract=null), Reference(id=1327012694068146197, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012676691145592, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=2, rfOrder=1, authorNames=null, journalName=null, refType=null, unstructuredReference=Macglashan J, Ho M K, Loftin R, et al. Interactive learning from policy-dependent human feedback[C]//Proce of the 34th Int Conf on Machine Learning - Volume 70. New York: ACM, 2017: 2285–2294, articleTitle=null, refAbstract=null), Reference(id=1327012694131060758, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012676691145592, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=3, rfOrder=2, authorNames=null, journalName=null, refType=null, unstructuredReference=Warnell G, Waytowich N, Lawhern V, et al. Deep TAMER: Interactive agent shaping in high-dimensional state spaces[C]//Proc of the 32nd AAAI Conf on Artificial Intelligence and 30th Innovative Applications of Artificial Intelligence Conf and 8th AAAI Symp on Educational Advances in Artificial Intelligence. Palo Alto, CA: AAAI, 2018: 1545−1554, articleTitle=null, refAbstract=null), Reference(id=1327012694235918359, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012676691145592, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=4, rfOrder=3, authorNames=null, journalName=null, refType=null, unstructuredReference=Brown D, Goo W, Nagarajan P, et al. Extrapolating beyond suboptimal demonstrations via inverse reinforcement learning from observations[C]//Proc of Int Conf on Machine Learning. Cambridge, MA: JMLR, 2019: 783−792, articleTitle=null, refAbstract=null), Reference(id=1327012694370136088, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012676691145592, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=5, rfOrder=4, authorNames=null, journalName=null, refType=null, unstructuredReference=Shin D, Dragan A D, Brown D S. Benchmarks and algorithms for offline preference-based reward learning[J]. arXiv preprint, arXiv: 2301.01392, 2023, articleTitle=null, refAbstract=null), Reference(id=1327012694516936729, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012676691145592, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=6, rfOrder=5, authorNames=null, journalName=null, refType=null, unstructuredReference=Sadigh D, Dragan A, Sastry S, et al. Active preference-based learning of reward functions[C]//Proc of Robotics: Science and Systems. Cambridge, Massachusetts: Escholarship. https://www.roboticsproceedings.org/rss13/p53.html, articleTitle=null, refAbstract=null), Reference(id=1327012694600822810, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012676691145592, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=7, rfOrder=6, authorNames=null, journalName=null, refType=null, unstructuredReference=Maghakian J, Mineiro P, Panaganti K, et al. Personalized reward learning with interaction-grounded learning (IGL)[C]//Proc of the 11th Int Conf on Learning Representations. Washington, DC: OpenReview. net, 2022, articleTitle=null, refAbstract=null), Reference(id=1327012694676320283, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012676691145592, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=8, rfOrder=7, authorNames=null, journalName=null, refType=null, unstructuredReference=矣晓沅,谢幸. 大模型道德价值观对齐问题剖析[J]. 计算机研究与发展,2023,60(9):1926−1945, articleTitle=null, refAbstract=null), Reference(id=1327012694747623452, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012676691145592, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=8, rfOrder=8, authorNames=null, journalName=null, refType=null, unstructuredReference=Yi Xiaoyuan, Xie Xing. Unpacking the ethical value alignment in big models[J]. Journal of Computer Research and Development, 2023, 60(9): 1926−1945 (in Chinese), articleTitle=null, refAbstract=null), Reference(id=1327012694810538013, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012676691145592, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=9, rfOrder=9, authorNames=null, journalName=null, refType=null, unstructuredReference=Ouyang Long, Wu J, Jiang Xu, et al. Training language models to follow instructions with human feedback[C]//Proc of Conf on Neural Information Processing Systems 2022, Red Hook, NY: Curran Associates Inc., 2022: 27730−27744, articleTitle=null, refAbstract=null), Reference(id=1327012694940561438, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012676691145592, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=10, rfOrder=10, authorNames=null, journalName=null, refType=null, unstructuredReference=Bai Y, Jones A, Ndousse K, et al. Training a helpful and harmless assistant with reinforcement learning from human feedback[J]. arXiv preprint, arXiv: 2204.05862, 2022, articleTitle=null, refAbstract=null), Reference(id=1327012695066390559, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012676691145592, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=11, rfOrder=11, authorNames=null, journalName=null, refType=null, unstructuredReference=Achiam J, Adler S, Agarwal S, et al. GPT−4 technical report[R/OL]. OpenAI, 2023[2025-04-08]. https://arxiv.org/pdf/2303.08774, articleTitle=null, refAbstract=null), Reference(id=1327012695150276640, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012676691145592, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=12, rfOrder=12, authorNames=null, journalName=null, refType=null, unstructuredReference=Anthropic. Introducing claude[EB/OL]. 2023[2025-04-09]. https://www.anthropic.com/news/introducing-claude, articleTitle=null, refAbstract=null), Reference(id=1327012695225774113, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012676691145592, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=13, rfOrder=13, authorNames=null, journalName=null, refType=null, unstructuredReference=Glaese A, Mcaleese N, Trębacz M, et al. Improving alignment of dialogue agents via targeted human judgements[J]. arXiv preprint, arXiv: 2209.14375, 2022, articleTitle=null, refAbstract=null), Reference(id=1327012695280300066, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012676691145592, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=14, rfOrder=14, authorNames=null, journalName=null, refType=null, unstructuredReference=Pichai S, Hassabis D. Introducing Gemini: Our largest and most capable AI model[EB/OL]. 2023[2025-04-09]. https://blog.google/technology/ai/google-gemini-ai/, articleTitle=null, refAbstract=null), Reference(id=1327012695372574755, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012676691145592, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=15, rfOrder=15, authorNames=null, journalName=null, refType=null, unstructuredReference=Bi Xiao, Chen Deli, Chen Guanting, et al. DeepSeek LLM: Scaling open-source language models with longtermism[J]. arXiv preprint, arXiv: 2401.02954, 2024, articleTitle=null, refAbstract=null), Reference(id=1327012695439683620, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012676691145592, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=16, rfOrder=16, authorNames=null, journalName=null, refType=null, unstructuredReference=Stiennon N, Ouyang Long, Wu J, et al. Learning to summarize with human feedback[C]//Proc of Conf on Neural Information Processing Systems 2020. Red Hook, NY: Curran Associates Inc., 2020: 3008−3021, articleTitle=null, refAbstract=null), Reference(id=1327012695498403877, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012676691145592, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=17, rfOrder=17, authorNames=null, journalName=null, refType=null, unstructuredReference=Schulman J, Wolski F, Dhariwal P, et al. 2017. Proximal policy optimization algorithms[J]. arXiv preprint, arXiv: 1707.06347, 2017, articleTitle=null, refAbstract=null), Reference(id=1327012695569707046, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012676691145592, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=18, rfOrder=18, authorNames=null, journalName=null, refType=null, unstructuredReference=Zheng Rui, Dou Shihan, Gao Songyang, et al. Secrets of RLHF in large language models part I: PPO[J]. arXiv preprint, arXiv: 2307.04964, 2023, articleTitle=null, refAbstract=null), Reference(id=1327012695645204519, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012676691145592, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=19, rfOrder=19, authorNames=null, journalName=null, refType=null, unstructuredReference=Yao Zhewei, Aminabadi R Y, Ruwase O, et al. 2023. DeepSpeed-Chat: Easy, fast and affordable RLHF training of ChatGPT-like models at all scales[J]. arXiv preprint, arXiv: 2308.01320, 2023, articleTitle=null, refAbstract=null), Reference(id=1327012695724896296, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012676691145592, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=20, rfOrder=20, authorNames=null, journalName=null, refType=null, unstructuredReference=Werra L, Belkada Y, Tunstall L, et al. trl: Transformer reinforcement learning[EB/OL]. 2020[2025-04-09]. https://github.com/huggingface/trl, articleTitle=null, refAbstract=null), Reference(id=1327012695804588073, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012676691145592, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=21, rfOrder=21, authorNames=null, journalName=null, refType=null, unstructuredReference=Havrilla A, Zhuravinskyi M, Phung D, et al. Trlx: A framework for large scale reinforcement learning from human feedback[C]//Proc of the 2023 Conf on Empirical Methods in Natural Language Processing. Stroudsburg, PA: ACL, 2023: 8578−8595, articleTitle=null, refAbstract=null), Reference(id=1327012695884279850, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012676691145592, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=22, rfOrder=22, authorNames=null, journalName=null, refType=null, unstructuredReference=Lai Z, Li S, Tang X, et al. Merak: An efficient distributed DNN training framework with automated 3D parallelism for giant foundation models[J]. IEEE Transactions on Parallel and Distributed Systems, 2023, 34(5): 1466−1478, articleTitle=null, refAbstract=null), Reference(id=1327012695955583019, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012676691145592, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=23, rfOrder=23, authorNames=null, journalName=null, refType=null, unstructuredReference=Song J, Yim J, Jung J, et al. Optimus-CC: Efficient large NLP model training with 3D parallelism aware communication compression[C]//Proc of the 28th ACM Int Conf on Architectural Support for Programming Languages and Operating Systems. New York: ACM, 2023: 560–573, articleTitle=null, refAbstract=null), Reference(id=1327012696018497580, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012676691145592, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=24, rfOrder=24, authorNames=null, journalName=null, refType=null, unstructuredReference=Knox W B, Stone P. TAMER: Training an agent manually via evaluative reinforcement[C]//Proc of the 7th IEEE Int Conf on Development and Learning. Piscataway, NJ: IEEE, 2008: 292−297, articleTitle=null, refAbstract=null), Reference(id=1327012696085606445, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012676691145592, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=25, rfOrder=25, authorNames=null, journalName=null, refType=null, unstructuredReference=Ziegler D M, Stiennon N, Wu J, et al. 2019. Fine-tuning language models from human preferences[J]. arXiv preprint, arXiv: 1909.08593, 2019, articleTitle=null, refAbstract=null), Reference(id=1327012696198852654, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012676691145592, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=26, rfOrder=26, authorNames=null, journalName=null, refType=null, unstructuredReference=Wu J, Ouyang Long, Ziegler D M, et al. 2021. Recursively summarizing books with human feedback[J]. arXiv preprint, arXiv: 2109.10862, 2021, articleTitle=null, refAbstract=null), Reference(id=1327012696261767216, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012676691145592, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=27, rfOrder=27, authorNames=null, journalName=null, refType=null, unstructuredReference=Santacroce M, Lu Yadong, Yu Han, et al. 2023. Efficient RLHF: Reducing the memory usage of PPO[J]. arXiv preprint, arXiv: 2309.00754, 2023, articleTitle=null, refAbstract=null), Reference(id=1327012696328876081, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012676691145592, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=28, rfOrder=28, authorNames=null, journalName=null, refType=null, unstructuredReference=Zheng Lianmin, Li Zhouhan, Zhang Hao, et al. Alpa: Automating inter-and intra-operator parallelism for distributed deep learning[C]//Proc of the 16th USENIX Symp on Operating Systems Design and Implementation (OSDI 22). Berkeley, CA: USENIX Association, 2022: 559−578, articleTitle=null, refAbstract=null), Reference(id=1327012696400179250, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012676691145592, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=29, rfOrder=29, authorNames=null, journalName=null, refType=null, unstructuredReference=Xiao W, Bhardwaj R, Ramjee R, et al. Gandiva: Introspective cluster scheduling for deep learning[C]//Proc of the 13th USENIX Conf on Operating Systems Design and Implementation. Berkeley, CA: USENIX Association, 2018: 595–610, articleTitle=null, refAbstract=null), Reference(id=1327012696622477363, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012676691145592, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=30, rfOrder=30, authorNames=null, journalName=null, refType=null, unstructuredReference=Shoeybi M, Patwary M, Puri R, et al. 2019. Metatron-LM: Training multi-billion parameter language models using model parallelism[J]. arXiv preprint, arXiv: 1909.08053, 2019, articleTitle=null, refAbstract=null), Reference(id=1327012696706363444, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012676691145592, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=31, rfOrder=31, authorNames=null, journalName=null, refType=null, unstructuredReference=Huang Yanping, Cheng Youlong, Bapna A, et al. GPipe: Efficient training of giant neural networks using pipeline parallelism[C]//Proc of the 33rd Int Conf on Neural Information Processing Systems. Red Hook, NY: Curran Associates Inc., 2019: 103−112, articleTitle=null, refAbstract=null), Reference(id=1327012696790249525, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012676691145592, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=32, rfOrder=32, authorNames=null, journalName=null, refType=null, unstructuredReference=Narayanan D, Harlap A, Phanishayee A, et al. PipeDream: Generalized pipeline parallelism for DNN training[C]//Proc of the 27th ACM Symposium on Operating Systems Principles. New York: ACM, 2019: 1–15, articleTitle=null, refAbstract=null), Reference(id=1327012698514108470, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012676691145592, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=33, rfOrder=33, authorNames=null, journalName=null, refType=null, unstructuredReference=Rajbhandari S, Rasley J, Ruwase O, et al. ZeRO: Memory optimizations toward training trillion parameter models[C]//Proc of SC20: Int Conf for High Performance Computing, Networking, Storage and Analysis. Piscataway, NJ: IEEE, 2020: 1−16, articleTitle=null, refAbstract=null), Reference(id=1327012698614771768, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012676691145592, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=34, rfOrder=34, authorNames=null, journalName=null, refType=null, unstructuredReference=Hu E J, Shen Y, Wallis P, et al. Lora: Low-rank adaptation of large language models[C]//Proc of Int Conf on Learning Representations. openreview. net, 2022, articleTitle=null, refAbstract=null), Reference(id=1327012698677686329, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012676691145592, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=35, rfOrder=35, authorNames=null, journalName=null, refType=null, unstructuredReference=Hu Jian, Wu Xibin, Zhu Zilin, et al. OpenRLHF: An easy-to-use, scalable and high-performance RLHF framework[J]. arXiv preprint, arXiv: 2405.11143, 2024, articleTitle=null, refAbstract=null), Reference(id=1327012698748989498, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012676691145592, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=36, rfOrder=36, authorNames=null, journalName=null, refType=null, unstructuredReference=Sheng Guangming, Zhang Chi, Ye Zilingfeng, et al. HybridFlow: A flexible and efficient RLHF framework[J]. arXiv preprint, arXiv: 2409.19256, 2024, articleTitle=null, refAbstract=null), Reference(id=1327012698832875579, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012676691145592, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=37, rfOrder=37, authorNames=null, journalName=null, refType=null, unstructuredReference=Li Shenggui, Liu Hongxin, Bian Zhengda, et al. Colossal-AI: A unified deep learning system for large-scale parallel training[C]//Proc of the 52nd Int Conf on Parallel Processing. New York: ACM, 2023: 766–775, articleTitle=null, refAbstract=null), Reference(id=1327012698904178748, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012676691145592, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=38, rfOrder=38, authorNames=null, journalName=null, refType=null, unstructuredReference=Zheng Yaowei, Zhang Richong, Zhang Junhao, et al. LLaMA-Factory: Unified efficient fine-tuning of 100+ language models[C]//Proc of the 62nd Annual Meeting of the Association for Computational Linguistics. Stroudsburg, PA: ACL, 2024: 400−410, articleTitle=null, refAbstract=null), Reference(id=1327012698967093309, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012676691145592, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=39, rfOrder=39, authorNames=null, journalName=null, refType=null, unstructuredReference=Volcengine. veRL: Volcano engine reinforcement learning for LLMs[EB/OL]. 2025[2025-04-08]. https://github.com/vocengine/verl, articleTitle=null, refAbstract=null), Reference(id=1327012699025813566, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012676691145592, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=40, rfOrder=40, authorNames=null, journalName=null, refType=null, unstructuredReference=Qin Haotong, Ma Xudong, Zheng Xingyu, et al. Accurate LoRA-finetuning quantization of LLMs via information retention[J]. arXiv preprint, arXiv: 2402.05445, 2024, articleTitle=null, refAbstract=null), Reference(id=1327012699147448383, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012676691145592, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=41, rfOrder=41, authorNames=null, journalName=null, refType=null, unstructuredReference=Dao T, Fu D Y, Ermon S, et al. 2022. FlashAttention: Fast and memory-efficient exact attention with IO-awareness[C]//Proc of the 36th Int Conf on Neural Information Processing Systems. New Orleans, LA: Curran Associates Inc. 1189: 16344−16359, articleTitle=null, refAbstract=null), Reference(id=1327012699231334464, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012676691145592, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=42, rfOrder=42, authorNames=null, journalName=null, refType=null, unstructuredReference=Dao T, Haziza D, Massa F, et al. Flash-Decoding for long-context inference[EB/OL]. 2024[2025-03-01]. https://crfm.stanford.edu/2023/10/12/flashdecoding.html, articleTitle=null, refAbstract=null), Reference(id=1327012699294249025, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012676691145592, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=43, rfOrder=43, authorNames=null, journalName=null, refType=null, unstructuredReference=葛旭冉,欧洋,王博,等. 大语言模型推理中的存储优化技术综述[J]. 计算机研究与发展,2025,62(3):545−562, articleTitle=null, refAbstract=null), Reference(id=1327012699361357890, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012676691145592, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=43, rfOrder=44, authorNames=null, journalName=null, refType=null, unstructuredReference=Ge Xuran, Ou Yang, Wang Bo, et al. Survey of storage optimization techniques in large language model inference[J]. Journal of Computer Research and Development, 2025, 62(3): 545−562 (in Chinese), articleTitle=null, refAbstract=null), Reference(id=1327012699441049667, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012676691145592, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=44, rfOrder=45, authorNames=null, journalName=null, refType=null, unstructuredReference=DeepSpeed. ZeRO-Inference: Democratizing massive model inference[EB/OL]. 2022[2025-04-08]. https://www.deepspeed.ai/2022/09/09/zero-inference.html, articleTitle=null, refAbstract=null), Reference(id=1327012699508158532, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012676691145592, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=45, rfOrder=46, authorNames=null, journalName=null, refType=null, unstructuredReference=Sheng Ying, Zheng Lianmin, Yuan Binhang, et al. FlexGen: High-throughput generative inference of large language models with a single GPU[C]//Proc of Int Conf on Machine Learning. Cambridge, MA: PMLR, 2023: 31094−31116, articleTitle=null, refAbstract=null), Reference(id=1327012699587850309, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012676691145592, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=46, rfOrder=47, authorNames=null, journalName=null, refType=null, unstructuredReference=Kwon W, Li Zhouhan, Zhuang Siyan, et al. Efficient memory management for large language model serving with pagedattention[C]//Proc of the 29th Symp on Operating Systems Principles. New York: ACM, 2023: 611−626, articleTitle=null, refAbstract=null), Reference(id=1327012699650764870, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012676691145592, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=47, rfOrder=48, authorNames=null, journalName=null, refType=null, unstructuredReference=Nvidia. Triton inference server[EB/OL]. 2025[2025-04-09]. https://github.com/triton-inference-server/server, articleTitle=null, refAbstract=null), Reference(id=1327012699722068039, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012676691145592, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=48, rfOrder=49, authorNames=null, journalName=null, refType=null, unstructuredReference=Yu G I, Jeong J S, Kim G W, et al. Orca: A distributed serving system for Transformer-Based generative models[C]//Proc of the 16th USENIX Symp on Operating Systems Design and Implementation (OSDI 22). Berkeley, CA: USENIX Association, 2022: 521−538, articleTitle=null, refAbstract=null)], funds=null, companyList=[AuthorCompany(id=1327012683150373806, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012676691145592, xref=1, ext=[AuthorCompanyExt(id=1327012683158762415, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012676691145592, companyId=1327012683150373806, language=EN, country=null, province=null, city=null, postcode=null, companyName=null, departmentName=null, remark=
1Institute of Computing Technology, Chinese Academy of Sciences, Beijing 100190), AuthorCompanyExt(id=1327012683167151024, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012676691145592, companyId=1327012683150373806, language=CN, country=null, province=null, city=null, postcode=null, companyName=null, departmentName=null, remark=
1中国科学院计算技术研究所 北京 100190)]), AuthorCompany(id=1327012683255231409, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012676691145592, xref=2, ext=[AuthorCompanyExt(id=1327012683284591538, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012676691145592, companyId=1327012683255231409, language=EN, country=null, province=null, city=null, postcode=null, companyName=null, departmentName=null, remark=
2University of Chinese Academy of Sciences, Beijing 100190), AuthorCompanyExt(id=1327012683297174451, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012676691145592, companyId=1327012683255231409, language=CN, country=null, province=null, city=null, postcode=null, companyName=null, departmentName=null, remark=
2中国科学院大学 北京 100190)]), AuthorCompany(id=1327012684958118836, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012676691145592, xref=3, ext=[AuthorCompanyExt(id=1327012684962313141, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012676691145592, companyId=1327012684958118836, language=EN, country=null, province=null, city=null, postcode=null, companyName=null, departmentName=null, remark=
3School of Control and Computer Engineering, North China Electric Power University, Beijing 102206), AuthorCompanyExt(id=1327012684970701750, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012676691145592, companyId=1327012684958118836, language=CN, country=null, province=null, city=null, postcode=null, companyName=null, departmentName=null, remark=
3华北电力大学控制与计算机工程学院 北京 102206)])], figs=[ArticleFig(id=1327012687785079785, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012676691145592, language=EN, label=Figure 1, caption=
Process of fine-tuning LLM with PPO, figureFileSmall=lZ026fZFWNGOLXiXPcq5nA==, figureFileBig=TFUkHvRlMl9graXj4BiZNQ==, tableContent=null), ArticleFig(id=1327012689450218475, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012676691145592, language=CN, label=图1, caption=
使用PPO微调大语言模型的流程, figureFileSmall=lZ026fZFWNGOLXiXPcq5nA==, figureFileBig=TFUkHvRlMl9graXj4BiZNQ==, tableContent=null), ArticleFig(id=1327012689651545069, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012676691145592, language=EN, label=Figure 2, caption=
Three computation modes for LLMs, figureFileSmall=Co81khKO5EWAKkTlJZjGpQ==, figureFileBig=zliUO0gE9KB1aIiofxw2SQ==, tableContent=null), ArticleFig(id=1327012689718653934, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012676691145592, language=CN, label=图2, caption=
大语言模型的3种计算模式, figureFileSmall=Co81khKO5EWAKkTlJZjGpQ==, figureFileBig=zliUO0gE9KB1aIiofxw2SQ==, tableContent=null), ArticleFig(id=1327012689831900143, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012676691145592, language=EN, label=Figure 3, caption=
Time-consuming analysis of speeding up all PPO steps using 3D parallelism strategy, figureFileSmall=TUBu93FxxysKClYC32L0LQ==, figureFileBig=ROU6oj5/5pRMdasrlfhRcg==, tableContent=null), ArticleFig(id=1327012689919980528, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012676691145592, language=CN, label=图3, caption=
使用三维并行策略加速所有PPO阶段的耗时分析, figureFileSmall=TUBu93FxxysKClYC32L0LQ==, figureFileBig=ROU6oj5/5pRMdasrlfhRcg==, tableContent=null), ArticleFig(id=1327012690003866609, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012676691145592, language=EN, label=Figure 4, caption=
Acceleration generation step using 3D parallelism strategy, figureFileSmall=1E58PsRixR516B+GOO6VOA==, figureFileBig=wrrl8+xcoF2QXEjqWbknJA==, tableContent=null), ArticleFig(id=1327012690070975474, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012676691145592, language=CN, label=图4, caption=
使用三维并行策略加速生成阶段, figureFileSmall=1E58PsRixR516B+GOO6VOA==, figureFileBig=wrrl8+xcoF2QXEjqWbknJA==, tableContent=null), ArticleFig(id=1327012690159055859, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012676691145592, language=EN, label=Figure 5, caption=
Inter-stage dependent relationship analysis in different PPO algorithms, figureFileSmall=p2AjCGSHWNI7C2M69UbcWQ==, figureFileBig=V3028UY2SXOgVI0/bnm8nA==, tableContent=null), ArticleFig(id=1327012690230359028, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012676691145592, language=CN, label=图5, caption=
不同PPO算法中阶段间依赖关系分析, figureFileSmall=p2AjCGSHWNI7C2M69UbcWQ==, figureFileBig=V3028UY2SXOgVI0/bnm8nA==, tableContent=null), ArticleFig(id=1327012690297467893, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012676691145592, language=EN, label=Figure 6, caption=
Timing diagram of operation of generation stage under diverse bubble filling approaches, figureFileSmall=5F/Rgt7/q/jk37WbMfpeHA==, figureFileBig=GDF22XWIgF3EAuurm/Hx+w==, tableContent=null), ArticleFig(id=1327012690351993846, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012676691145592, language=CN, label=图6, caption=
不同气泡填充方法下生成阶段的运算时序图, figureFileSmall=5F/Rgt7/q/jk37WbMfpeHA==, figureFileBig=GDF22XWIgF3EAuurm/Hx+w==, tableContent=null), ArticleFig(id=1327012690414908407, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012676691145592, language=EN, label=Figure 7, caption=
An example of delayed inter-batch pipeline, figureFileSmall=uGzLXk0emHQtW5SSmWDLDg==, figureFileBig=qM0crw+IKaOD/eSIbYoh8Q==, tableContent=null), ArticleFig(id=1327012690482017272, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012676691145592, language=CN, label=图7, caption=
延迟批间流水线实例, figureFileSmall=uGzLXk0emHQtW5SSmWDLDg==, figureFileBig=qM0crw+IKaOD/eSIbYoh8Q==, tableContent=null), ArticleFig(id=1327012690544931833, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012676691145592, language=EN, label=Figure 8, caption=
Version delay of delayed inter-batch pipeline parallelism, figureFileSmall=MG+mdSwUNe3vbWlzAeBsVg==, figureFileBig=wqRqfSQfJZabIQNZQkYZxQ==, tableContent=null), ArticleFig(id=1327012690607846394, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012676691145592, language=CN, label=图8, caption=
延迟批间流水线并行的版本延迟, figureFileSmall=MG+mdSwUNe3vbWlzAeBsVg==, figureFileBig=wqRqfSQfJZabIQNZQkYZxQ==, tableContent=null), ArticleFig(id=1327012690674955259, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012676691145592, language=EN, label=Figure 9, caption=
Comparison of the model evaluation reward after using PPO and asynchronous PPO for fine-tuning, figureFileSmall=Y7xTAWon6xMhatXEBPecSw==, figureFileBig=VXKVOOdjM+kAPhU3E6rn3Q==, tableContent=null), ArticleFig(id=1327012690737869820, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012676691145592, language=CN, label=图9, caption=
使用PPO和异步PPO进行微调后的模型评估奖励对比, figureFileSmall=Y7xTAWon6xMhatXEBPecSw==, figureFileBig=VXKVOOdjM+kAPhU3E6rn3Q==, tableContent=null), ArticleFig(id=1327012690834338813, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012676691145592, language=EN, label=Figure 10, caption=
Comparision of throughput, figureFileSmall=Tnl3PYO87MqtwwiyiyeaBA==, figureFileBig=iNw1VXfO/06zsMBIhV5pZg==, tableContent=null), ArticleFig(id=1327012690905641982, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012676691145592, language=CN, label=图10, caption=
吞吐量对比, figureFileSmall=Tnl3PYO87MqtwwiyiyeaBA==, figureFileBig=iNw1VXfO/06zsMBIhV5pZg==, tableContent=null), ArticleFig(id=1327012690976945151, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012676691145592, language=EN, label=Figure 11, caption=
Comparison of scalability and time cost analysis, figureFileSmall=/T2pTg1Put00HKTGdqVRvA==, figureFileBig=dRKFWyKLXVt4KQJYjw23zA==, tableContent=null), ArticleFig(id=1327012691052441600, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012676691145592, language=CN, label=图11, caption=
可扩展性对比和耗时分析, figureFileSmall=/T2pTg1Put00HKTGdqVRvA==, figureFileBig=dRKFWyKLXVt4KQJYjw23zA==, tableContent=null), ArticleFig(id=1327012691136327682, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012676691145592, language=EN, label=Figure 12, caption=
Comparison between 3D parallelism and delayed inter-batch pipeline parallelism in generation efficiency, figureFileSmall=Rw1l9PSpsSKTvWYYAJfOhg==, figureFileBig=h8gUm3ySWqfU9LXpOlP5EQ==, tableContent=null), ArticleFig(id=1327012691211825155, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012676691145592, language=CN, label=图12, caption=
三维并行与延迟批间流水线并行的生成效率对比, figureFileSmall=Rw1l9PSpsSKTvWYYAJfOhg==, figureFileBig=h8gUm3ySWqfU9LXpOlP5EQ==, tableContent=null), ArticleFig(id=1327012691278934020, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012676691145592, language=EN, label=Figure 13, caption=
Time-consuming of GPT-6B for the low computational density task group and the high computational density task group, figureFileSmall=JGgFG/JQwYFYaMT+3BRK0Q==, figureFileBig=Ihm+yc95Wq0PD7BdwaduDg==, tableContent=null), ArticleFig(id=1327012691354431493, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012676691145592, language=CN, label=图13, caption=
GPT-6B的低计算密度任务组耗时和高计算密度任务组耗时, figureFileSmall=JGgFG/JQwYFYaMT+3BRK0Q==, figureFileBig=Ihm+yc95Wq0PD7BdwaduDg==, tableContent=null), ArticleFig(id=1327012691429928966, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012676691145592, language=EN, label=Figure 14, caption=
Fine-tuning time cost of three models obtained by combining Pipe-RLHF with DeepSpeed inference engine, figureFileSmall=KVAVeTmibdAmrU3iiK+Ofw==, figureFileBig=mM7l1u29X6hpnXbMV4HWMA==, tableContent=null), ArticleFig(id=1327012691492843527, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012676691145592, language=CN, label=图14, caption=
对3种模型结合Pipe-RLHF和DeepSpeed推理引擎取得的微调耗时, figureFileSmall=KVAVeTmibdAmrU3iiK+Ofw==, figureFileBig=mM7l1u29X6hpnXbMV4HWMA==, tableContent=null), ArticleFig(id=1327012691559952392, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012676691145592, language=EN, label=Figure 15, caption=
Training curves of synchronous PPO and asynchronous PPO, figureFileSmall=cj66X+0wxR5WYdbubkShIw==, figureFileBig=wnCtEJbQXU8bQp4/LhcF4A==, tableContent=null), ArticleFig(id=1327012691627061257, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012676691145592, language=CN, label=图15, caption=
同步PPO和异步PPO的训练曲线, figureFileSmall=cj66X+0wxR5WYdbubkShIw==, figureFileBig=wnCtEJbQXU8bQp4/LhcF4A==, tableContent=null), ArticleFig(id=1327012691706753034, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012676691145592, language=EN, label=Figure 16, caption=
Throughput of models for fine-tuning using Pipe-RLHF with different Pmax, figureFileSmall=o4IhuesSblnAqbYrcf+YJw==, figureFileBig=fWWqy/FAC0Vyx2KtjNxD/Q==, tableContent=null), ArticleFig(id=1327012691782250507, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012676691145592, language=CN, label=图16, caption=
不同Pmax下使用Pipe-RLHF进行模型微调的吞吐量, figureFileSmall=o4IhuesSblnAqbYrcf+YJw==, figureFileBig=fWWqy/FAC0Vyx2KtjNxD/Q==, tableContent=null), ArticleFig(id=1327012691874525196, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012676691145592, language=EN, label=Table 1, caption=
Computation Mode of Models in Each PPO Stage, figureFileSmall=null, figureFileBig=null, tableContent=
| 阶段名称 | 模型种类 | 计算模式 |
|---|
| 生成 | 策略模型 | 生成 |
| 推理 | 策略模型、参考模型、奖励模型 | 推理 |
| 训练 | 策略模型 | 训练 |
), ArticleFig(id=1327012691958411277, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012676691145592, language=CN, label=表1, caption=
PPO各阶段中模型的计算模式
, figureFileSmall=null, figureFileBig=null, tableContent=
| 阶段名称 | 模型种类 | 计算模式 |
|---|
| 生成 | 策略模型 | 生成 |
| 推理 | 策略模型、参考模型、奖励模型 | 推理 |
| 训练 | 策略模型 | 训练 |
), ArticleFig(id=1327012692046491662, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012676691145592, language=EN, label=Table 2, caption=
Model Architectures in Experiments, figureFileSmall=null, figureFileBig=null, tableContent=
| 模型 | 隐藏层维度 | 注意力头数目 | 模型层数 |
|---|
| GPT-6B | 4 096 | 32 | 32 |
| GPT-20B | 6 144 | 64 | 44 |
| GPT-66B | 9 216 | 72 | 64 |
| GPT-175B | 12 288 | 96 | 96 |
), ArticleFig(id=1327012692147154959, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012676691145592, language=CN, label=表2, caption=
实验模型结构
, figureFileSmall=null, figureFileBig=null, tableContent=
| 模型 | 隐藏层维度 | 注意力头数目 | 模型层数 |
|---|
| GPT-6B | 4 096 | 32 | 32 |
| GPT-20B | 6 144 | 64 | 44 |
| GPT-66B | 9 216 | 72 | 64 |
| GPT-175B | 12 288 | 96 | 96 |
), ArticleFig(id=1327012692226846737, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012676691145592, language=EN, label=Table 3, caption=
Comparison of Computing Resource Utilization %, figureFileSmall=null, figureFileBig=null, tableContent=
| 模型 | 三维并行 计算资源利用率 | Pipe-RLHF 计算资源利用率 |
|---|
| GPT-6B | 23.7 | 68.7 |
| GPT-20B | 38.3 | 74.4 |
| GPT-66B | 55.6 | 76.3 |
| GPT-175B | 64.2 | 77.5 |
), ArticleFig(id=1327012692310732818, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012676691145592, language=CN, label=表3, caption=
计算资源利用率对比
, figureFileSmall=null, figureFileBig=null, tableContent=
| 模型 | 三维并行 计算资源利用率 | Pipe-RLHF 计算资源利用率 |
|---|
| GPT-6B | 23.7 | 68.7 |
| GPT-20B | 38.3 | 74.4 |
| GPT-66B | 55.6 | 76.3 |
| GPT-175B | 64.2 | 77.5 |
)], attaches=null, journal=Journal(id=1324759606947315908, delFlag=0, nameCn=计算机研究与发展, nameEn=Journal of Computer Research and Development, nameHistory1=null, nameHistory2=null, issn=1000-1239, eissn=null, cn=11-1777/TP, coden=null, periodic=0, language=CN, oaType=null, ccby=null, superviseOffice=null, ownerOffice=null, pubOffice=null, editorOffice=null, officeType=null, aims=null, clcCode=null, officeProv=null, officeCity=null, officeAddr=null, officeZip=null, officeEmail=null, officePhone=null, editDirector=null, officeDirector=null, officeDirectorPhone=null, officeStaffNum=null, officeEmpNum=null, coverPicUrl=k2l8zgqwstuxxevFX2qxVA==, journalPrice=null, startedYear=null, abbrevIsoEn=Journal of Computer Research and Development, journalRemark=null, publicationField=null, createdTime=1793776903141, updatedTime=1793850698612, createdBy=18614031015, updatedBy=13701087609, firstLetterCn=J, firstLetterEn=J, subjectCode=Electronic Information, subjectName=null, subjectCodeEn=Electronic Information, subjectNameEn=null, picCn=k2l8zgqwstuxxevFX2qxVA==, picEn=r6k0Ry+RD+c/VXj5QarmDw==, jcr=null, cjcr=null, exts=[JournalExt(id=1325069127710286688, language=CN, name=计算机研究与发展, nameHistory1=null, nameHistory2=null, managedBy=, sponsoredBy=, publishedBy=, editorOffice=, officeProv=null, officeCity=null, officeAddr=, officeZip=, editDirector=, officeDirector=null, officePhone=null, coverPicUrl=null, journalRemark=, submitArticleUrl=null, websiteUrl=, createdTime=1793850698640, updatedTime=1793850698640, createdBy=13701087609, updatedBy=13701087609, submissionGuidelinesUrl=, submissionAuthorUrl=https://crad.ict.ac.cn/journalx/authorLogOn.action, submissionEditorUrl=https://crad.ict.ac.cn/journalx/editorLogOn.action, submissionReviewUrl=https://crad.ict.ac.cn/journalx/expertLogOn.action, submissionCeEditorUrl=https://crad.ict.ac.cn/journalx/editorInChiefLogOn.action, submissionAeEditorUrl=, option={"copyright":""}), JournalExt(id=1325069127777395553, language=EN, name=Journal of Computer Research and Development, nameHistory1=null, nameHistory2=null, managedBy=, sponsoredBy=, publishedBy=, editorOffice=, officeProv=null, officeCity=null, officeAddr=, officeZip=, editDirector=, officeDirector=null, officePhone=null, coverPicUrl=null, journalRemark=, submitArticleUrl=null, websiteUrl=, createdTime=1793850698656, updatedTime=1793850698656, createdBy=13701087609, updatedBy=13701087609, submissionGuidelinesUrl=, submissionAuthorUrl=https://crad.ict.ac.cn/journalx/authorLogOn.action, submissionEditorUrl=https://crad.ict.ac.cn/journalx/editorLogOn.action, submissionReviewUrl=https://crad.ict.ac.cn/journalx/expertLogOn.action, submissionCeEditorUrl=https://crad.ict.ac.cn/journalx/editorInChiefLogOn.action, submissionAeEditorUrl=, option={"copyright":""})], databaseList=null, tenantJournalId=1324759914146529478, websiteList=[Website(id=1325070300139892771, webName=null, webTitle=null, webDomain=null, webCopyrigh=null, webIpcNo=null, seoTitle=null, seoKeywords=null, seoDescription=null, tenantJournalId=null, journalId=1324759914146529478, journalNameCn=null, journalNameEn=null, grayFlag=null, tenantId=1146029695717560320, platformId=null, journalGroupId=null, journalGroupNameCn=null, journalGroupNameEn=null, type=1, domain=https://castjournals.cast.org.cn/joweb/jsjyjyfz/CN, language=CN, createTime=1793850978169, createBy=18614031015, updateTime=1793851040259, updateBy=18614031015, name=计算机研究与发展-中文, tplId=1146099689490845704, title=计算机研究与发展, delFlag=0, indexPage=/home, props=[WebsiteProps(id=1325070893751349360, tenantId=1146029695717560320, journalId=null, journalGroupId=null, siteId=1325070300139892771, code=articleTextType, value=kx, createTime=1793851119697, updateTime=1793851119697, creator=18614031015, updator=18614031015), WebsiteProps(id=1325070893730377837, tenantId=1146029695717560320, journalId=null, journalGroupId=null, siteId=1325070300139892771, code=banner, value=null, createTime=1793851119692, updateTime=1793851119692, creator=18614031015, updator=18614031015), WebsiteProps(id=1325070893772320883, tenantId=1146029695717560320, journalId=null, journalGroupId=null, siteId=1325070300139892771, code=grayFlag, value=0, createTime=1793851119702, updateTime=1793851119702, creator=18614031015, updator=18614031015), WebsiteProps(id=1325070893721989228, tenantId=1146029695717560320, journalId=null, journalGroupId=null, siteId=1325070300139892771, code=logo, value=https://castjournals.cast.org.cn/joweb/jsjyjyfz/CN/file/pic?fileId=U27tDsy717YF7XSPA+Vwdw==, createTime=1793851119690, updateTime=1793851119690, creator=18614031015, updator=18614031015), WebsiteProps(id=1325070893784903797, tenantId=1146029695717560320, journalId=null, journalGroupId=null, siteId=1325070300139892771, code=minRunFlag, value=0, createTime=1793851119705, updateTime=1793851119705, creator=18614031015, updator=18614031015), WebsiteProps(id=1325070893742960751, tenantId=1146029695717560320, journalId=null, journalGroupId=null, siteId=1325070300139892771, code=picServerUrl, value=https://castjournals.cast.org.cn/joweb/jsjyjyfz/CN/file/pic, createTime=1793851119695, updateTime=1793851119695, creator=18614031015, updator=18614031015), WebsiteProps(id=1325070893780709492, tenantId=1146029695717560320, journalId=null, journalGroupId=null, siteId=1325070300139892771, code=silenceFlag, value=0, createTime=1793851119704, updateTime=1793851119704, creator=18614031015, updator=18614031015), WebsiteProps(id=1325070893738766446, tenantId=1146029695717560320, journalId=null, journalGroupId=null, siteId=1325070300139892771, code=staticResourcePath, value=https://castjournals.cast.org.cn/joweb/cast_kjdb_cn_619/, createTime=1793851119694, updateTime=1793851119694, creator=18614031015, updator=18614031015), WebsiteProps(id=1325070893755543665, tenantId=1146029695717560320, journalId=null, journalGroupId=null, siteId=1325070300139892771, code=themeColor, value=null, createTime=1793851119698, updateTime=1793851119698, creator=18614031015, updator=18614031015), WebsiteProps(id=1325070893763932274, tenantId=1146029695717560320, journalId=null, journalGroupId=null, siteId=1325070300139892771, code=themeStyle, value=null, createTime=1793851119700, updateTime=1793851119700, creator=18614031015, updator=18614031015)]), Website(id=1325070300580294698, webName=null, webTitle=null, webDomain=null, webCopyrigh=null, webIpcNo=null, seoTitle=null, seoKeywords=null, seoDescription=null, tenantJournalId=null, journalId=1324759914146529478, journalNameCn=null, journalNameEn=null, grayFlag=null, tenantId=1146029695717560320, platformId=null, journalGroupId=null, journalGroupNameCn=null, journalGroupNameEn=null, type=1, domain=https://castjournals.cast.org.cn/joweb/jsjyjyfz/EN, language=EN, createTime=1793850978274, createBy=18614031015, updateTime=1793851036638, updateBy=18614031015, name=计算机研究与发展-英文, tplId=1146101810881728533, title=Journal of Computer Research and Development, delFlag=0, indexPage=/home, props=[WebsiteProps(id=1325070925288321148, tenantId=1146029695717560320, journalId=null, journalGroupId=null, siteId=1325070300580294698, code=articleTextType, value=kx, createTime=1793851127216, updateTime=1793851127216, creator=18614031015, updator=18614031015), WebsiteProps(id=1325070925263155321, tenantId=1146029695717560320, journalId=null, journalGroupId=null, siteId=1325070300580294698, code=banner, value=null, createTime=1793851127210, updateTime=1793851127210, creator=18614031015, updator=18614031015), WebsiteProps(id=1325070925309292671, tenantId=1146029695717560320, journalId=null, journalGroupId=null, siteId=1325070300580294698, code=grayFlag, value=0, createTime=1793851127221, updateTime=1793851127221, creator=18614031015, updator=18614031015), WebsiteProps(id=1325070925254766712, tenantId=1146029695717560320, journalId=null, journalGroupId=null, siteId=1325070300580294698, code=logo, value=https://castjournals.cast.org.cn/joweb/jsjyjyfz/EN/file/pic?fileId=U27tDsy717YF7XSPA+Vwdw==, createTime=1793851127208, updateTime=1793851127208, creator=18614031015, updator=18614031015), WebsiteProps(id=1325070925326069889, tenantId=1146029695717560320, journalId=null, journalGroupId=null, siteId=1325070300580294698, code=minRunFlag, value=0, createTime=1793851127225, updateTime=1793851127225, creator=18614031015, updator=18614031015), WebsiteProps(id=1325070925279932539, tenantId=1146029695717560320, journalId=null, journalGroupId=null, siteId=1325070300580294698, code=picServerUrl, value=https://castjournals.cast.org.cn/joweb/jsjyjyfz/EN/file/pic, createTime=1793851127214, updateTime=1793851127214, creator=18614031015, updator=18614031015), WebsiteProps(id=1325070925317681280, tenantId=1146029695717560320, journalId=null, journalGroupId=null, siteId=1325070300580294698, code=silenceFlag, value=0, createTime=1793851127223, updateTime=1793851127223, creator=18614031015, updator=18614031015), WebsiteProps(id=1325070925271543930, tenantId=1146029695717560320, journalId=null, journalGroupId=null, siteId=1325070300580294698, code=staticResourcePath, value=https://castjournals.cast.org.cn/joweb/cast_kjdb_en_623/, createTime=1793851127212, updateTime=1793851127212, creator=18614031015, updator=18614031015), WebsiteProps(id=1325070925292515453, tenantId=1146029695717560320, journalId=null, journalGroupId=null, siteId=1325070300580294698, code=themeColor, value=null, createTime=1793851127217, updateTime=1793851127217, creator=18614031015, updator=18614031015), WebsiteProps(id=1325070925300904062, tenantId=1146029695717560320, journalId=null, journalGroupId=null, siteId=1325070300580294698, code=themeStyle, value=null, createTime=1793851127219, updateTime=1793851127219, creator=18614031015, updator=18614031015)])], journalTitle=计算机研究与发展, weixinUrl=null, journalUrl=https://crad.ict.ac.cn/, iacademicId=null, status=1, seqNo=null, journalTitleEn=Journal of Computer Research and Development, journalPhotoCn=k2l8zgqwstuxxevFX2qxVA==, journalPhotoEn=r6k0Ry+RD+c/VXj5QarmDw==, journalFirstLetter=J, journalRecommend=null, journalNew=null, journalCollection=null, jcrJf=null, cjcrJf=null, jcrJfStr=null, cjcrJfStr=null, submissionFirstDecision=null, sciSubjectClassification=null, casSubjectClassification=null, citeScore=null, totalCitationFrequency=null, icpCode=null, psCode=null, advertisingLicenseCode=null, copyrightInformation=null, country=null, option=, provinceCode=null, provinceName=null, collectFlag=false, interPubPlatform=, interPubPlatformUrl=null), detailUrlCn=https://castjournals.cast.org.cn/joweb/jsjyjyfz/CN/10.7544/issn1000-1239.202550127, detailUrlEn=https://castjournals.cast.org.cn/joweb/jsjyjyfz/EN/10.7544/issn1000-1239.202550127, pdfUrlCn=https://castjournals.cast.org.cn/joweb/jsjyjyfz/CN/PDF/10.7544/issn1000-1239.202550127, pdfUrlEn=https://castjournals.cast.org.cn/joweb/jsjyjyfz/EN/PDF/10.7544/issn1000-1239.202550127, aliStartDate=0, aliEndDate=0, collectionFlag=false, citedCount=null, citedUrl=null, previewStatus=0, delFlag=0, hasFullText=1, orderTime=1748793600000, fullTextJson=null, articleText=null, reference=null)