Article(id=1327012750812885554, tenantId=1146029695717560320, journalId=1324759914146529478, issueId=1327012646391493336, articleNumber=null, orderNo=null, doi=10.7544/issn1000-1239.202550120, pmid=null, cstr=null, oa=null, hot=null, price=null, onlineType=0, articleFormat=0, articleType=null, articleTypeStr=research-article, receivedDate=1740758400000, receivedDateStr=2025-03-01, revisedDate=1744041600000, revisedDateStr=2025-04-08, acceptedDate=null, acceptedDateStr=null, onlineDate=1794314094498, onlineDateStr=2026-11-10, pubDate=1748793600000, pubDateStr=2025-06-02, doiRegisterDate=null, doiRegisterDateStr=null, onlineIssueDate=1794314094498, onlineIssueDateStr=2026-11-10, onlineJustAcceptDate=null, onlineJustAcceptDateStr=null, onlineFirstDate=null, onlineFirstDateStr=null, sourceXml=null, magXml=null, createTime=1794314094498, creator=13701087609, updateTime=1794314094498, updator=13701087609, issue=Issue{id=1327012646391493336, tenantId=1146029695717560320, journalId=1324759914146529478, year='2025', volume='62', issue='6', pageStart='1331', pageEnd='1593', issueExtLink='null', onlineDate='null', pubDate='1748793600000', pubDateStr='2025-06-02', beforeIssueId=null, nextIssueId=null, price=null, status=1, issueComplete=1, articleOrder=1, issueType=-1, specialIssue=null, createTime=1794314069601, creator='13701087609', updateTime=1794376817187, updator='13701087609', preIssue=null, nextIssue=null, articleTotal=null, ext={EN=IssueExt(id=1327275829270262152, tenantId=1146029695717560320, journalId=1324759914146529478, issueId=1327012646391493336, language=EN, specialIssueTitle=, coverIllustrator=null, specialIssueEditor=, specialIssueAbout=), CN=IssueExt(id=1327275829270262153, tenantId=1146029695717560320, journalId=1324759914146529478, issueId=1327012646391493336, language=CN, specialIssueTitle=, coverIllustrator=null, specialIssueEditor=, specialIssueAbout=)}, issueFiles=null, downloadFileDto=null}, startPage=1363, endPage=1379, ext={EN=ArticleExt(id=1327012751010017843, articleId=1327012750812885554, tenantId=1146029695717560320, journalId=1324759914146529478, language=EN, title=Performance Modeling and Optimization for Large-Scale Heterogeneous Consistency Integrated Computing System, columnId=1327012740067078573, journalTitle=Journal of Computer Research and Development, columnName=High Performance Computing System, runingTitle=null, highlight=null, articleAbstract=
With the widespread adoption and development of large-scale artificial intelligence applications, the demand for computing power in artificial intelligence from both industry and academia is increasing. Heterogeneous consistency integrated computing systems, which combine heterogeneous computing technology with cache consistency technology, are gradually becoming an important solution for building intelligent computing centers in the future. However, due to the immaturity of heterogeneous computing and consistency interconnect technologies, it is hard for existing research to model the performance of such systems, making it difficult for researchers to evaluate construction schemes, predict computing performance, and assess system optimization methods at a low cost. We propose HCSim, a performance modeling tool for heterogeneous consistency integrated computing systems, addressing challenges in modeling system topology and inaccuracies in workload modeling within consistency systems. HCSim provides researchers with a flexible, low-cost, and efficient modeling and simulation tool for evaluating interconnect topologies and AI computing tasks. Using HCSim, we model a heterogeneous consistency integrated computing system with thousands of accelerators, and simulate the data-parallel distributed training task of the LLAMA2-13B large language model (LLM) on this system, exploring the impact of variables such as heterogeneous computing power distribution, bandwidth, latency, and task scale on system performance and task execution efficiency. Furthermore, we also design optimization strategies for the communication issues in heterogeneous consistency integrated computing systems and validate the effectiveness of these strategies using HCSim. The simulation results show that HCSim not only meets the performance modeling needs of heterogeneous consistency integrated computing systems, but can also be applied to evaluate and verify optimization strategies for such systems.
, authors=Rengang Li
1, Yinan Tang
2, *, Zhenhua Guo
2, Li Wang
2, Zan Zong
1, Guangwen Yang
1, authorsList=Rengang Li, Yinan Tang, Zhenhua Guo, Li Wang, Zan Zong, Guangwen Yang, authorCompany=null, correspAuthors=Yinan Tang, authorNote=
born in 1980. PhD candidate, senior engineer. His main research interest includes heterogeneous computing
born in 1993. PhD. His main research interests include heterogeneous computing and simulation system
born in 1988. PhD. His main research interests include computer system architecture and heterogeneous computing
born in 1989. Master. Her main research interests include heterogeneous computing and artificial intelligence
born in 1994. PhD. His research interest includes performance acceleration of distributed deep learning systems and large-scale data processing systems
born in 1963. PhD, professor, PhD supervisor. His main research interest includes high performance computing
, correspAuthorsNote=null, copyrightStatement=Copyright © 2025 Journal of Computer Research and Development. All rights reserved., copyrightOwner=null, extLink=null, articleAbsUrl=null, sourceXml=null, magXml=null, pdfUrl=null, pdf=null, pdfFileSize=null, pdfExtLink=null, richHtmlUrl=null, mobilePdfUrl=null, reviewReport=null, pdfFirstPage=null, abstractGraph=null, abstractGraphContent=null, abstractVideo=null, citation=null, cebUrl=null, magXmlContent=null, mapNumber=null, fund=null), CN=ArticleExt(id=1327012754529038937, articleId=1327012750812885554, tenantId=1146029695717560320, journalId=1324759914146529478, language=CN, title=大规模异构一致性融合计算系统的性能建模与优化, columnId=1327012740230656431, journalTitle=计算机研究与发展, columnName=高性能计算系统, runingTitle=null, highlight=null, articleAbstract=
随着大规模人工智能应用的普及与发展,工业界和学术界对于人工智能算力的需求逐渐提升,结合了异构计算技术与缓存一致性技术的异构一致性融合计算系统逐渐成为未来构建智算中心的重要解决方案. 然而,由于异构计算和一致性互连技术尚不成熟,现有工作难以实现对该系统进行性能建模,导致研究者无法以低成本完成异构一致性融合计算系统的建设方案评估、计算性能预测以及系统优化方法评测等工作. 提出了一种面向异构一致性融合计算系统的性能建模工具HCSim,解决了现有建模仿真研究中对该系统拓扑架构建模困难、对一致性系统中工作负载建模不准确等问题,为研究者提供了一个可灵活建模、评估互连拓扑与AI计算任务的低成本、高效建模仿真工具. 利用HCSim,建模了千卡互连的异构一致性融合计算系统,并在该系统上模拟了LLAMA2-13B大语言模型(large language model,LLM)的数据并行分布式训练任务,探究了异构算力分布、带宽、时延和任务规模等变量对系统性能与任务执行效率的影响. 进一步地,针对异构一致性融合计算系统的通信问题,设计了相应的优化方案,并利用HCSim进行了效果验证. 仿真结果说明HCSim不仅能够满足异构一致性融合计算系统的性能建模需求,同时也可以被应用于评估、验证异构一致性融合计算系统的优化方案.
, authors=李仁刚
1, 唐轶男
2, *, 郭振华
2, 王丽
2, 宗瓒
1, 杨广文
1, authorsList=李仁刚, 唐轶男, 郭振华, 王丽, 宗瓒, 杨广文, authorCompany=null, correspAuthors=唐轶男, authorNote=
 |
1980年生. 博士研究生,正高级工程师. 主要研究方向为异构计算 |
 |
1993年生. 博士. 主要研究方向为异构计算、仿真系统 |
 |
1988年生. 博士. 主要研究方向为计算机系统结构、异构计算 |
 |
1989年生. 硕士. 主要研究方向异构计算、人工智能 |
 |
1994年生. 博士. 主要研究方向为分布式深度学习系统和大规模数据处理系统的性能加速 |
 |
1963年生. 博士,教授,博士生导师. 主要研究方向为高性能计算 |
, correspAuthorsNote=
, copyrightStatement=版权所有 © 2025 计算机研究与发展编辑部, copyrightOwner=null, extLink=null, articleAbsUrl=null, sourceXml=xllTY01MpOG60Q2cyN6+vQ==, magXml=3DQHptBWVauz0gexwBCtDw==, pdfUrl=null, pdf=Yp4oj6gYwgPIygi+4XYayg==, pdfFileSize=3154508, pdfExtLink=null, richHtmlUrl=null, mobilePdfUrl=null, reviewReport=null, pdfFirstPage=null, abstractGraph=ykLUI5ZET8cBVykPzJoHqA==, abstractGraphContent=null, abstractVideo=null, citation=null, cebUrl=null, magXmlContent=IDiZmj1J5O2qBqf6RQJMKQ==, mapNumber=null, fund=null)}, authors=[Author(id=1327012756617802338, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012750812885554, orderNo=0, firstName=null, middleName=null, lastName=null, nameCn=null, orcid=null, stid=null, country=null, authorPic=null, dead=0, email=lirg@ieisystem.com, emailSecond=null, emailThird=null, correspondingAuthor=0, authorType=1, ext={EN=AuthorExt(id=1327012756693299812, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012750812885554, authorId=1327012756617802338, language=EN, stringName=Rengang Li, firstName=Rengang, middleName=null, lastName=Li, prefix=null, suffix=null, authorComment=null, nameInitials=null, affiliation=null, department=null, xref=
1, address=
1Department of Computer Science and Technology, Tsinghua University, Beijing 100084, bio={"content":"
born in 1980. PhD candidate, senior engineer. His main research interest includes heterogeneous computing
"}, bioImg=null, bioContent=
born in 1980. PhD candidate, senior engineer. His main research interest includes heterogeneous computing
, aboutCorrespAuthor=null), CN=AuthorExt(id=1327012756772991589, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012750812885554, authorId=1327012756617802338, language=CN, stringName=李仁刚, firstName=null, middleName=null, lastName=null, prefix=null, suffix=null, authorComment=null, nameInitials=null, affiliation=null, department=null, xref=
1, address=
1清华大学计算机科学与技术系 北京 100084, bio={"img":"tEvQV32eskEDBdNG4LKLxQ==","content":"
1980年生. 博士研究生,正高级工程师. 主要研究方向为异构计算
"}, bioImg=tEvQV32eskEDBdNG4LKLxQ==, bioContent=
1980年生. 博士研究生,正高级工程师. 主要研究方向为异构计算
, aboutCorrespAuthor=null)}, companyList=[AuthorCompany(id=1327012755070104155, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012750812885554, xref=1, ext=[AuthorCompanyExt(id=1327012755078492764, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012750812885554, companyId=1327012755070104155, language=EN, country=null, province=null, city=null, postcode=null, companyName=null, departmentName=null, remark=
1Department of Computer Science and Technology, Tsinghua University, Beijing 100084), AuthorCompanyExt(id=1327012755086881373, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012750812885554, companyId=1327012755070104155, language=CN, country=null, province=null, city=null, postcode=null, companyName=null, departmentName=null, remark=
1清华大学计算机科学与技术系 北京 100084)])]), Author(id=1327012756831711847, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012750812885554, orderNo=1, firstName=null, middleName=null, lastName=null, nameCn=null, orcid=null, stid=null, country=null, authorPic=null, dead=0, email=tangyinan@ieisystem.com, emailSecond=null, emailThird=null, correspondingAuthor=1, authorType=1, ext={EN=AuthorExt(id=1327012756911403625, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012750812885554, authorId=1327012756831711847, language=EN, stringName=Yinan Tang, firstName=Yinan, middleName=null, lastName=Tang, prefix=null, suffix=null, authorComment=null, nameInitials=null, affiliation=null, department=null, xref=
2, *, address=
2IEIT SYSTEMS Co., Ltd., Jinan 250101, bio={"content":"
born in 1993. PhD. His main research interests include heterogeneous computing and simulation system
"}, bioImg=null, bioContent=
born in 1993. PhD. His main research interests include heterogeneous computing and simulation system
, aboutCorrespAuthor=null), CN=AuthorExt(id=1327012756974318186, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012750812885554, authorId=1327012756831711847, language=CN, stringName=唐轶男, firstName=null, middleName=null, lastName=null, prefix=null, suffix=null, authorComment=null, nameInitials=null, affiliation=null, department=null, xref=
2, *, address=
2浪潮电子信息产业股份有限公司 济南 250101, bio={"img":"St77VfQtANYt36edz+VlIQ==","content":"
1993年生. 博士. 主要研究方向为异构计算、仿真系统
"}, bioImg=St77VfQtANYt36edz+VlIQ==, bioContent=
1993年生. 博士. 主要研究方向为异构计算、仿真系统
, aboutCorrespAuthor=null)}, companyList=[AuthorCompany(id=1327012756525527646, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012750812885554, xref=2, ext=[AuthorCompanyExt(id=1327012756538110559, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012750812885554, companyId=1327012756525527646, language=EN, country=null, province=null, city=null, postcode=null, companyName=null, departmentName=null, remark=
2IEIT SYSTEMS Co., Ltd., Jinan 250101), AuthorCompanyExt(id=1327012756542304864, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012750812885554, companyId=1327012756525527646, language=CN, country=null, province=null, city=null, postcode=null, companyName=null, departmentName=null, remark=
2浪潮电子信息产业股份有限公司 济南 250101)])]), Author(id=1327012757045621356, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012750812885554, orderNo=2, firstName=null, middleName=null, lastName=null, nameCn=null, orcid=null, stid=null, country=null, authorPic=null, dead=0, email=null, emailSecond=null, emailThird=null, correspondingAuthor=0, authorType=1, ext={EN=AuthorExt(id=1327012757146284654, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012750812885554, authorId=1327012757045621356, language=EN, stringName=Zhenhua Guo, firstName=Zhenhua, middleName=null, lastName=Guo, prefix=null, suffix=null, authorComment=null, nameInitials=null, affiliation=null, department=null, xref=
2, address=
2IEIT SYSTEMS Co., Ltd., Jinan 250101, bio={"content":"
born in 1988. PhD. His main research interests include computer system architecture and heterogeneous computing
"}, bioImg=null, bioContent=
born in 1988. PhD. His main research interests include computer system architecture and heterogeneous computing
, aboutCorrespAuthor=null), CN=AuthorExt(id=1327012757213393519, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012750812885554, authorId=1327012757045621356, language=CN, stringName=郭振华, firstName=null, middleName=null, lastName=null, prefix=null, suffix=null, authorComment=null, nameInitials=null, affiliation=null, department=null, xref=
2, address=
2浪潮电子信息产业股份有限公司 济南 250101, bio={"img":"Oz/1quKJdSzHjnbSDM9u+w==","content":"
1988年生. 博士. 主要研究方向为计算机系统结构、异构计算
"}, bioImg=Oz/1quKJdSzHjnbSDM9u+w==, bioContent=
1988年生. 博士. 主要研究方向为计算机系统结构、异构计算
, aboutCorrespAuthor=null)}, companyList=[AuthorCompany(id=1327012756525527646, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012750812885554, xref=2, ext=[AuthorCompanyExt(id=1327012756538110559, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012750812885554, companyId=1327012756525527646, language=EN, country=null, province=null, city=null, postcode=null, companyName=null, departmentName=null, remark=
2IEIT SYSTEMS Co., Ltd., Jinan 250101), AuthorCompanyExt(id=1327012756542304864, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012750812885554, companyId=1327012756525527646, language=CN, country=null, province=null, city=null, postcode=null, companyName=null, departmentName=null, remark=
2浪潮电子信息产业股份有限公司 济南 250101)])]), Author(id=1327012757351805553, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012750812885554, orderNo=3, firstName=null, middleName=null, lastName=null, nameCn=null, orcid=null, stid=null, country=null, authorPic=null, dead=0, email=null, emailSecond=null, emailThird=null, correspondingAuthor=0, authorType=1, ext={EN=AuthorExt(id=1327012757465051763, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012750812885554, authorId=1327012757351805553, language=EN, stringName=Li Wang, firstName=Li, middleName=null, lastName=Wang, prefix=null, suffix=null, authorComment=null, nameInitials=null, affiliation=null, department=null, xref=
2, address=
2IEIT SYSTEMS Co., Ltd., Jinan 250101, bio={"content":"
born in 1989. Master. Her main research interests include heterogeneous computing and artificial intelligence
"}, bioImg=null, bioContent=
born in 1989. Master. Her main research interests include heterogeneous computing and artificial intelligence
, aboutCorrespAuthor=null), CN=AuthorExt(id=1327012757532160628, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012750812885554, authorId=1327012757351805553, language=CN, stringName=王丽, firstName=null, middleName=null, lastName=null, prefix=null, suffix=null, authorComment=null, nameInitials=null, affiliation=null, department=null, xref=
2, address=
2浪潮电子信息产业股份有限公司 济南 250101, bio={"img":"u6ZNjX0N15qXfLPDFeGRpg==","content":"
1989年生. 硕士. 主要研究方向异构计算、人工智能
"}, bioImg=u6ZNjX0N15qXfLPDFeGRpg==, bioContent=
1989年生. 硕士. 主要研究方向异构计算、人工智能
, aboutCorrespAuthor=null)}, companyList=[AuthorCompany(id=1327012756525527646, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012750812885554, xref=2, ext=[AuthorCompanyExt(id=1327012756538110559, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012750812885554, companyId=1327012756525527646, language=EN, country=null, province=null, city=null, postcode=null, companyName=null, departmentName=null, remark=
2IEIT SYSTEMS Co., Ltd., Jinan 250101), AuthorCompanyExt(id=1327012756542304864, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012750812885554, companyId=1327012756525527646, language=CN, country=null, province=null, city=null, postcode=null, companyName=null, departmentName=null, remark=
2浪潮电子信息产业股份有限公司 济南 250101)])]), Author(id=1327012757590880886, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012750812885554, orderNo=4, firstName=null, middleName=null, lastName=null, nameCn=null, orcid=null, stid=null, country=null, authorPic=null, dead=0, email=null, emailSecond=null, emailThird=null, correspondingAuthor=0, authorType=1, ext={EN=AuthorExt(id=1327012757653795448, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012750812885554, authorId=1327012757590880886, language=EN, stringName=Zan Zong, firstName=Zan, middleName=null, lastName=Zong, prefix=null, suffix=null, authorComment=null, nameInitials=null, affiliation=null, department=null, xref=
1, address=
1Department of Computer Science and Technology, Tsinghua University, Beijing 100084, bio={"content":"
born in 1994. PhD. His research interest includes performance acceleration of distributed deep learning systems and large-scale data processing systems
"}, bioImg=null, bioContent=
born in 1994. PhD. His research interest includes performance acceleration of distributed deep learning systems and large-scale data processing systems
, aboutCorrespAuthor=null), CN=AuthorExt(id=1327012757708321401, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012750812885554, authorId=1327012757590880886, language=CN, stringName=宗瓒, firstName=null, middleName=null, lastName=null, prefix=null, suffix=null, authorComment=null, nameInitials=null, affiliation=null, department=null, xref=
1, address=
1清华大学计算机科学与技术系 北京 100084, bio={"img":"2L5vT5Ihf9C34++s5c1n3w==","content":"
1994年生. 博士. 主要研究方向为分布式深度学习系统和大规模数据处理系统的性能加速
"}, bioImg=2L5vT5Ihf9C34++s5c1n3w==, bioContent=
1994年生. 博士. 主要研究方向为分布式深度学习系统和大规模数据处理系统的性能加速
, aboutCorrespAuthor=null)}, companyList=[AuthorCompany(id=1327012755070104155, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012750812885554, xref=1, ext=[AuthorCompanyExt(id=1327012755078492764, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012750812885554, companyId=1327012755070104155, language=EN, country=null, province=null, city=null, postcode=null, companyName=null, departmentName=null, remark=
1Department of Computer Science and Technology, Tsinghua University, Beijing 100084), AuthorCompanyExt(id=1327012755086881373, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012750812885554, companyId=1327012755070104155, language=CN, country=null, province=null, city=null, postcode=null, companyName=null, departmentName=null, remark=
1清华大学计算机科学与技术系 北京 100084)])]), Author(id=1327012757771235963, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012750812885554, orderNo=5, firstName=null, middleName=null, lastName=null, nameCn=null, orcid=null, stid=null, country=null, authorPic=null, dead=0, email=null, emailSecond=null, emailThird=null, correspondingAuthor=0, authorType=1, ext={EN=AuthorExt(id=1327012757859316349, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012750812885554, authorId=1327012757771235963, language=EN, stringName=Guangwen Yang, firstName=Guangwen, middleName=null, lastName=Yang, prefix=null, suffix=null, authorComment=null, nameInitials=null, affiliation=null, department=null, xref=
1, address=
1Department of Computer Science and Technology, Tsinghua University, Beijing 100084, bio={"content":"
born in 1963. PhD, professor, PhD supervisor. His main research interest includes high performance computing
"}, bioImg=null, bioContent=
born in 1963. PhD, professor, PhD supervisor. His main research interest includes high performance computing
, aboutCorrespAuthor=null), CN=AuthorExt(id=1327012757947396734, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012750812885554, authorId=1327012757771235963, language=CN, stringName=杨广文, firstName=null, middleName=null, lastName=null, prefix=null, suffix=null, authorComment=null, nameInitials=null, affiliation=null, department=null, xref=
1, address=
1清华大学计算机科学与技术系 北京 100084, bio={"img":"FyzY+hCmKKH3QcSP+VQNQQ==","content":"
1963年生. 博士,教授,博士生导师. 主要研究方向为高性能计算
"}, bioImg=FyzY+hCmKKH3QcSP+VQNQQ==, bioContent=
1963年生. 博士,教授,博士生导师. 主要研究方向为高性能计算
, aboutCorrespAuthor=null)}, companyList=[AuthorCompany(id=1327012755070104155, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012750812885554, xref=1, ext=[AuthorCompanyExt(id=1327012755078492764, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012750812885554, companyId=1327012755070104155, language=EN, country=null, province=null, city=null, postcode=null, companyName=null, departmentName=null, remark=
1Department of Computer Science and Technology, Tsinghua University, Beijing 100084), AuthorCompanyExt(id=1327012755086881373, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012750812885554, companyId=1327012755070104155, language=CN, country=null, province=null, city=null, postcode=null, companyName=null, departmentName=null, remark=
1清华大学计算机科学与技术系 北京 100084)])])], keywords=[Keyword(id=1327012758035477119, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012750812885554, language=EN, orderNo=1, keyword=artificial intelligence), Keyword(id=1327012758102585984, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012750812885554, language=EN, orderNo=2, keyword=high performance computing), Keyword(id=1327012758165500545, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012750812885554, language=EN, orderNo=3, keyword=modeling and simulation), Keyword(id=1327012758253580930, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012750812885554, language=EN, orderNo=4, keyword=distributed computing), Keyword(id=1327012758316495491, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012750812885554, language=EN, orderNo=5, keyword=cache consistency), Keyword(id=1327012758391992964, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012750812885554, language=EN, orderNo=6, keyword=heterogeneous computing), Keyword(id=1327012758450713221, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012750812885554, language=CN, orderNo=1, keyword=人工智能), Keyword(id=1327012758509433478, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012750812885554, language=CN, orderNo=2, keyword=高性能计算), Keyword(id=1327012758626873991, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012750812885554, language=CN, orderNo=3, keyword=建模仿真), Keyword(id=1327012758685594248, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012750812885554, language=CN, orderNo=4, keyword=分布式计算), Keyword(id=1327012758740120201, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012750812885554, language=CN, orderNo=5, keyword=缓存一致性), Keyword(id=1327012758811423370, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012750812885554, language=CN, orderNo=6, keyword=异构计算)], refs=[Reference(id=1327012763383214768, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012750812885554, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=1, rfOrder=0, authorNames=null, journalName=null, refType=null, unstructuredReference=Guo Daya, Yang Dejian, Zhang Haowei, et al. Deepseek-r1: Incentivizing reasoning capability in LLMs via reinforcement learning[J]. arXiv preprint, arXiv: 2501.12948, 2025, articleTitle=null, refAbstract=null), Reference(id=1327012763454517937, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012750812885554, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=2, rfOrder=1, authorNames=null, journalName=null, refType=null, unstructuredReference=Huang Dawei, Yan Chuan, Li Qing, et al. From large language models to large multimodal models: A literature review[J]. Applied Sciences, 2024, 14(12): 5068, articleTitle=null, refAbstract=null), Reference(id=1327012763530015410, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012750812885554, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=3, rfOrder=2, authorNames=null, journalName=null, refType=null, unstructuredReference=Jiang Ziheng, Lin Haibin, Zhong Yinmin, et al. MegaScale: Scaling large language model training to more than 10, 000 GPUs[C]//Proc of 21st USENIX Symp on Networked Systems Design and Implementation (NSDI 24). Berkeley, CA: USENIX Association, 2024: 745−760, articleTitle=null, refAbstract=null), Reference(id=1327012763605512884, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012750812885554, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=4, rfOrder=3, authorNames=null, journalName=null, refType=null, unstructuredReference=Yang Zhuoping, Ji Shixin , Chen Xingzhen, et al. Challenges and opportunities to enable large-scale computing via heterogeneous chiplets[C]//Proc of the 29th Asia and South Pacific Design Automation Conf (ASP-DAC) . Piscataway, NJ: IEEE, 2024: 765−770, articleTitle=null, refAbstract=null), Reference(id=1327012763681010356, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012750812885554, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=5, rfOrder=4, authorNames=null, journalName=null, refType=null, unstructuredReference=Saghiri A M, Vahidipour S M, Jabbarpour M R, et al. A survey of artificial intelligence challenges: Analyzing the definitions, relationships, and evolutions[J]. Applied Sciences, 2022, 12(8): 4054, articleTitle=null, refAbstract=null), Reference(id=1327012763790062261, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012750812885554, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=6, rfOrder=5, authorNames=null, journalName=null, refType=null, unstructuredReference=Rajbhandari S, Ruwase O, Rasley J, et al. Zero-infinity: Breaking the gpu memory wall for extreme scale deep learning[C]//Proc of the Int Conf for High Performance Computing, Networking, Storage And Analysis 2021. New York: ACM, 2021: 1−14, articleTitle=null, refAbstract=null), Reference(id=1327012765455200950, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012750812885554, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=7, rfOrder=6, authorNames=null, journalName=null, refType=null, unstructuredReference=Korthikanti V A, Casper J, Lym S, et al. Reducing activation recomputation in large transformer models[C]//Proc of Machine Learning and Systems 5, 2023: 341−353. https://proceedings.mlsys.org/paper_files/paper/2023/hash/80083951326cf5b35e5100260d64ed81-Abstract-mlsys2023.html, articleTitle=null, refAbstract=null), Reference(id=1327012765551669943, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012750812885554, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=8, rfOrder=7, authorNames=null, journalName=null, refType=null, unstructuredReference=Li Shen, Zhao Yanli, Varma R, et al. Pytorch distributed: Experiences on accelerating data parallel training[J]. arXiv preprint, arXiv: 2006.15704, 2020, articleTitle=null, refAbstract=null), Reference(id=1327012765631361720, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012750812885554, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=9, rfOrder=8, authorNames=null, journalName=null, refType=null, unstructuredReference=Ge Suyu, Zhang Yunan, Liu Liyuan, et al. Model tells you what to discard: Adaptive KV cache compression for LLMs[J]. arXiv preprint, arXiv: 2310.01801, 2023, articleTitle=null, refAbstract=null), Reference(id=1327012765748802233, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012750812885554, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=10, rfOrder=9, authorNames=null, journalName=null, refType=null, unstructuredReference=Chen Zixiang, Deng Yihe, Wu Yue, et al. Towards understanding mixture of experts in deep learning[J]. arXiv preprint, arXiv: 2208.02813, 2022, articleTitle=null, refAbstract=null), Reference(id=1327012765824299706, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012750812885554, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=11, rfOrder=10, authorNames=null, journalName=null, refType=null, unstructuredReference=Xu Yi, Mahar S, Liu Ziheng, et al. CXL shared memory programming: Barely distributed and almost persistent[J]. arXiv preprint, arXiv: 2405.19626, 2024, articleTitle=null, refAbstract=null), Reference(id=1327012765908185787, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012750812885554, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=12, rfOrder=11, authorNames=null, journalName=null, refType=null, unstructuredReference=Schieffer G, Wahlgren J, Ren Jie, et al. Harnessing integrated CPU-GPU system memory for HPC: A first look into grace hopper[C]//Proc of the 53rd Int Conf on Parallel Processing. New York: ACM, 2024: 199−209, articleTitle=null, refAbstract=null), Reference(id=1327012765975294652, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012750812885554, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=13, rfOrder=12, authorNames=null, journalName=null, refType=null, unstructuredReference=Xia Jing, Cheng Chuanning, Zhou Xiping, et al. Kunpeng 920: The first 7-nm chiplet-based 64-core arm soc for cloud services[J]. IEEE Micro, 2021, 41(5): 67−75, articleTitle=null, refAbstract=null), Reference(id=1327012766046597822, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012750812885554, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=14, rfOrder=13, authorNames=null, journalName=null, refType=null, unstructuredReference=Fusco L, Khalilov M, Chrapek M, et al. Understanding data movement in tightly coupled heterogeneous systems: A case study with the Grace Hopper superchip[J]. arXiv preprint, arXiv: 2408.11556, 2024, articleTitle=null, refAbstract=null), Reference(id=1327012766143066816, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012750812885554, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=15, rfOrder=14, authorNames=null, journalName=null, refType=null, unstructuredReference=CXL Organization. CXL® Specification[EB/OL]. 2020[2024-12-01]. https://computeexpresslink.org/cxl-specification/., articleTitle=null, refAbstract=null), Reference(id=1327012766222758593, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012750812885554, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=16, rfOrder=15, authorNames=null, journalName=null, refType=null, unstructuredReference=Gholami A, Yao Zheiwei, Kim S, et al. AI and memory wall[J]. IEEE Micro, 2024, 44(3): 33−39, articleTitle=null, refAbstract=null), Reference(id=1327012766310838978, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012750812885554, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=17, rfOrder=16, authorNames=null, journalName=null, refType=null, unstructuredReference=Casanova H. SimGrid: A toolkit for the simulation of application scheduling[C]//Proc of the 1st IEEE/ACM Int Symp on Cluster Computing and the Grid. Piscataway, NJ: IEEE, 2021: 430−437, articleTitle=null, refAbstract=null), Reference(id=1327012766419890883, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012750812885554, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=18, rfOrder=17, authorNames=null, journalName=null, refType=null, unstructuredReference=Casanova H, Giersch A, Legrand A, et al. Lowering entry barriers to developing custom simulators of distributed applications and platforms with SimGrid[J]. Parallel Computing, 2025: 103125. https://www.sciencedirect.com/science/article/pii/S0167819125000018, articleTitle=null, refAbstract=null), Reference(id=1327012766507971268, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012750812885554, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=19, rfOrder=18, authorNames=null, journalName=null, refType=null, unstructuredReference=Saleh E, Shastry C. Simulation and modelling of task migration in distributed systems using SimGrid[C]//Proc of the Int Conf on Modeling, Simulation and Optimization. Singapore: Springer Nature Singapore, 2022: 475−486, articleTitle=null, refAbstract=null), Reference(id=1327012766570885829, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012750812885554, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=20, rfOrder=19, authorNames=null, journalName=null, refType=null, unstructuredReference=Guo Zhenhua, Tang Yinan, Zhai Jidong, et al. A survey on performance modeling and prediction for distributed DNN training[J]. IEEE Transactions on Parallel and Distributed Systems, 2024, 35(12): 2463−2478, articleTitle=null, refAbstract=null), Reference(id=1327012766642188998, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012750812885554, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=21, rfOrder=20, authorNames=null, journalName=null, refType=null, unstructuredReference=Lowe-Power J, Ahmad A. M, Akram A, et al. The Gem5 simulator: Version 20.0+[J]. arXiv preprint, arXiv: 2007.03152, 2020, articleTitle=null, refAbstract=null), Reference(id=1327012766730269383, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012750812885554, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=22, rfOrder=21, authorNames=null, journalName=null, refType=null, unstructuredReference=Bellard F. QEMU, a fast and portable dynamic translator[C]//Proc of USENIX Annual Technical Conf 2005. Berkeley, CA: USENIX Association, FREENIX Track, 2005: 10−5555, articleTitle=null, refAbstract=null), Reference(id=1327012766793183944, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012750812885554, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=23, rfOrder=22, authorNames=null, journalName=null, refType=null, unstructuredReference=Bakhoda A, Yuan G L, Fung W, et al. Analyzing CUDA workloads using a detailed GPU simulator[C]//Proc of 2009 IEEE Int Symp on Performance Analysis of Systems and Software. Piscataway, NJ: IEEE, 2009: 163−174, articleTitle=null, refAbstract=null), Reference(id=1327012766851904201, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012750812885554, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=24, rfOrder=23, authorNames=null, journalName=null, refType=null, unstructuredReference=Li Shang, Yang Zhiyuan, Reddy D, et al. DRAMSim3: A cycle-accurate, thermal-capable DRAM simulator[J]. IEEE Computer Architecture Letters, 2020, 19(2): 106−109, articleTitle=null, refAbstract=null), Reference(id=1327012766914818762, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012750812885554, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=25, rfOrder=24, authorNames=null, journalName=null, refType=null, unstructuredReference=Kim Y, Yang Weikun, Mutlu O. Ramulator: A fast and extensible DRAM simulator[J]. IEEE Computer Architecture Letters, 2015, 15(1): 45−49, articleTitle=null, refAbstract=null), Reference(id=1327012766981927627, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012750812885554, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=26, rfOrder=25, authorNames=null, journalName=null, refType=null, unstructuredReference=Henderson T R, Lacage M, Riley G F, et al. Network simulations with the NS−3 simulator[C]//Proc of SIGCOMM Demonstration 2008. New York: ACM, 2008. 527, articleTitle=null, refAbstract=null), Reference(id=1327012767053230796, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012750812885554, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=27, rfOrder=26, authorNames=null, journalName=null, refType=null, unstructuredReference=Varga A. OMNeT++[J]. Modeling and Tools for Network Simulation, 2010: 35−59, articleTitle=null, refAbstract=null), Reference(id=1327012767120339661, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012750812885554, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=28, rfOrder=27, authorNames=null, journalName=null, refType=null, unstructuredReference=Khairy M, Shen Zhesheng, Aamodt T M, et al. Accel-Sim: An extensible simulation framework for validated GPU modeling[C]//Proc of 2020 ACM/IEEE 47th Annual Int Symp on Computer Architecture (ISCA). Piscataway, NJ: IEEE, 2020: 473−486, articleTitle=null, refAbstract=null), Reference(id=1327012767183254222, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012750812885554, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=29, rfOrder=28, authorNames=null, journalName=null, refType=null, unstructuredReference=Won W, Heo T, Rashidi S, et al. Astra-sim2.0: Modeling hierarchical networks and disaggregated systems for large-model training at scale[C]//Proc of 2023 IEEE Int Symp on Performance Analysis of Systems and Software (ISPASS). Piscataway, NJ: IEEE, 2023: 283−294, articleTitle=null, refAbstract=null), Reference(id=1327012767258751695, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012750812885554, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=30, rfOrder=29, authorNames=null, journalName=null, refType=null, unstructuredReference=Moolchandani D, Kundu J, Ruelens F, et al. AMPeD: An analytical model for performance in distributed training of transformers[C]//Proc of 2023 IEEE Int Symp on Performance Analysis of Systems and Software (ISPASS). Piscataway, NJ: IEEE, 2023: 306−315, articleTitle=null, refAbstract=null), Reference(id=1327012767359414992, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012750812885554, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=31, rfOrder=30, authorNames=null, journalName=null, refType=null, unstructuredReference=Isaev M, McDonald N, Dennison L, et al. Calculon: A methodology and tool for high-level co-design of systems and large language models[C]//Proc of the Int Conf for High Performance Computing, Networking, Storage and Analysis. New York: ACM, 2023: 1−14, articleTitle=null, refAbstract=null), Reference(id=1327012767426523857, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012750812885554, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=32, rfOrder=31, authorNames=null, journalName=null, refType=null, unstructuredReference=Qi Hang, Sparks E R, Talwalkar A. Paleo: A performance model for deep neural networks[C]//Proc of Int Conf on Learning Representations. France, Toulon: MIT, 2017: 1−10, articleTitle=null, refAbstract=null), Reference(id=1327012767485244114, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012750812885554, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=33, rfOrder=32, authorNames=null, journalName=null, refType=null, unstructuredReference=Lu Wenyan, Yan Guihai, Li Jiajun, et al. FlexFlow: A flexible dataflow accelerator architecture for convolutional neural networks[C]//Proc of 2017 IEEE Int Symp on High Performance Computer Architecture (HPCA). Piscataway, NJ: IEEE, 2017: 553−564, articleTitle=null, refAbstract=null), Reference(id=1327012767569130195, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012750812885554, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=34, rfOrder=33, authorNames=null, journalName=null, refType=null, unstructuredReference=Zhu Hongyu, Phanishayee A, Pekhimenko G. Daydream: Accurately estimating the efficacy of optimizations for DNN training[C]//Proc of 2020 USENIX Annual Technical Conf (USENIX ATC 20) . Berkeley, CA: USENIX Association, 2020: 337−352, articleTitle=null, refAbstract=null), Reference(id=1327012767636239060, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012750812885554, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=35, rfOrder=34, authorNames=null, journalName=null, refType=null, unstructuredReference=Hu Hanpeng, Jiang Chenyu, Zhong Yuchen, et al. Dpro: A generic performance diagnosis and optimization toolkit for expediting distributed DNN training[C]//Proc of Machine Learning and Systems. New York: ACM, 2022: 623−637, articleTitle=null, refAbstract=null), Reference(id=1327012767699153621, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012750812885554, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=36, rfOrder=35, authorNames=null, journalName=null, refType=null, unstructuredReference=Lu Guandong, Chen Runzhe, Wang Yakai, et al. DistSim: A performance model of large-scale hybrid distributed DNN training[C]//Proc of the 20th ACM Int Conf on Computing Frontiers. New York: ACM, 2023: 112−122, articleTitle=null, refAbstract=null), Reference(id=1327012767762068182, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012750812885554, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=37, rfOrder=36, authorNames=null, journalName=null, refType=null, unstructuredReference=Santhanam K, Krishna S, Tomioka R, et al. DistIR: An intermediate representation for optimizing distributed neural networks[C]//Proc of the 1st Workshop on Machine Learning and Systems. New York: ACM, 2021: 15−23, articleTitle=null, refAbstract=null), Reference(id=1327012767833371351, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012750812885554, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=38, rfOrder=37, authorNames=null, journalName=null, refType=null, unstructuredReference=Lattner C, Amini M, Bondhugula U, et al. MLIR: Scaling compiler infrastructure for domain specific computation[C]//Proc of 2021 IEEE/ACM Int Symp on Code Generation and Optimization (CGO). Piscataway, NJ: IEEE, 2021: 2−14, articleTitle=null, refAbstract=null), Reference(id=1327012767908868824, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012750812885554, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=39, rfOrder=38, authorNames=null, journalName=null, refType=null, unstructuredReference=Duan Jiangfei, Li Xiuhong, Xu Ping, et al. Proteus: Simulating the performance of distributed DNN training[J]. IEEE Transactions on Parallel and Distributed Systems, 2024, 35(10): 1867−1878, articleTitle=null, refAbstract=null), Reference(id=1327012767975977689, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012750812885554, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=40, rfOrder=39, authorNames=null, journalName=null, refType=null, unstructuredReference=Zhang Shiwei, Yi Xiaodong, Diao Lansong, et al. Expediting distributed DNN training with device topology-aware graph deployment[J]. IEEE Transactions on Parallel and Distributed Systems, 2023, 34(4): 1281−1293, articleTitle=null, refAbstract=null), Reference(id=1327012768055669466, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012750812885554, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=41, rfOrder=40, authorNames=null, journalName=null, refType=null, unstructuredReference=Wang Haoran, Tachon T, Li Chong, et al. SMSG: Profiling-free parallelism modeling for distributed training of DNN[J]. International Journal of Parallel Programming, 2023, 51(2): 109−127, articleTitle=null, refAbstract=null), Reference(id=1327012768126972635, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012750812885554, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=42, rfOrder=41, authorNames=null, journalName=null, refType=null, unstructuredReference=Rashidi S, Sridharan S, Srinivasan S, et al. Astra-sim: Enabling sw/hw co-design exploration for distributed DL training platforms[C]//Proc of 2020 IEEE Int Symp on Performance Analysis of Systems and Software (ISPASS). Piscataway, NJ: IEEE, 2020: 81−92, articleTitle=null, refAbstract=null), Reference(id=1327012768210858716, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012750812885554, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=43, rfOrder=42, authorNames=null, journalName=null, refType=null, unstructuredReference=Samajdar A, Zhu Yuhao, Whatmough P, et al. Scale-sim: Systolic CNN accelerator simulator[J]. arXiv preprint, arXiv: 1811.02883, 2018, articleTitle=null, refAbstract=null), Reference(id=1327012768282161885, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012750812885554, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=44, rfOrder=43, authorNames=null, journalName=null, refType=null, unstructuredReference=Liu ZhiGang, Whatmough P N, Mattina M. Systolic tensor array: An efficient structured-sparse GEMM accelerator for mobile CNN inference[J]. IEEE Computer Architecture Letters, 2020, 19(1): 34−37, articleTitle=null, refAbstract=null), Reference(id=1327012769976660702, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012750812885554, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=45, rfOrder=44, authorNames=null, journalName=null, refType=null, unstructuredReference=Hagberg A, Swart P J, Schult D A. Exploring network structure, dynamics, and function using NetworkX (No. LA-UR-08-05495; LA-UR-08-5495)[R]. Los Alamos, NM: Los Alamos National Laboratory (LANL), 2008, articleTitle=null, refAbstract=null), Reference(id=1327012770039575263, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012750812885554, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=46, rfOrder=45, authorNames=null, journalName=null, refType=null, unstructuredReference=The SimGrid Team. The SimGrid models[EB/OL]. 2002[2025-01-17]. https://simgrid.frama.io/simgrid/Models.html#cm02, articleTitle=null, refAbstract=null), Reference(id=1327012770119267040, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012750812885554, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=47, rfOrder=46, authorNames=null, journalName=null, refType=null, unstructuredReference=IEIT Systems. meta brain® Artificial Intelligence Servers > AI > Servers > NF5468A5[EB/OL]. [2025-03-01]. https://en.ieisystem.com/product/ai/9573.html, articleTitle=null, refAbstract=null), Reference(id=1327012770203153121, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012750812885554, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=48, rfOrder=47, authorNames=null, journalName=null, refType=null, unstructuredReference=Li Ang, Song S L, Chen Jieyang, et al. Evaluating modern GPU interconnect: PCIe, nvlink, nv-sli, NVSwitch and gpudirect[J]. IEEE Transactions on Parallel and Distributed Systems, 2019, 31(1): 94−110, articleTitle=null, refAbstract=null), Reference(id=1327012770282844898, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012750812885554, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=49, rfOrder=48, authorNames=null, journalName=null, refType=null, unstructuredReference=Das Sharma D, Blankenship R, Berger D. An introduction to the compute express link (CXL) Interconnect[J]. ACM Computing Surveys, 2024, 56(11): 1−37, articleTitle=null, refAbstract=null), Reference(id=1327012770345759459, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012750812885554, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=50, rfOrder=49, authorNames=null, journalName=null, refType=null, unstructuredReference=王彦伟,李仁刚,徐冉,等. 基于可重构架构的数据中心异构加速软硬件系统级平台[J]. 计算机研究与发展,2024,61(6):1388−1400, articleTitle=null, refAbstract=null), Reference(id=1327012770484171492, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012750812885554, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=50, rfOrder=50, authorNames=null, journalName=null, refType=null, unstructuredReference=Wang Yanwei, Li Rengang, Xu Ran, et al. Data center heterogeneous acceleration software-hardware system-level platform based on reconfigurable architecture[J]. Journal of Computer Research and Development, 2024, 61(6): 1388−1400 (in Chinese), articleTitle=null, refAbstract=null), Reference(id=1327012770576446181, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012750812885554, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=51, rfOrder=51, authorNames=null, journalName=null, refType=null, unstructuredReference=葛旭冉,欧洋,王博,等. 大模型推理中的存储优化技术研究综述[J]. 计算机研究与发展,2025,62(3):545−562, articleTitle=null, refAbstract=null), Reference(id=1327012770664526566, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012750812885554, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=51, rfOrder=52, authorNames=null, journalName=null, refType=null, unstructuredReference=Ge Xuran, Ou Yang, Wang Bo, et al A survey of storage optimization techniques in large language model inference[J]. Journal of Computer Research and Development, 2025, 62(3): 545−562 (in Chinese), articleTitle=null, refAbstract=null), Reference(id=1327012770752606951, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012750812885554, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=52, rfOrder=53, authorNames=null, journalName=null, refType=null, unstructuredReference=Touvron H, Martin L, Stone K, et al. Llama 2: Open foundation and fine-tuned chat models[J]. arXiv preprint, arXiv: 2307.09288, 2023, articleTitle=null, refAbstract=null)], funds=null, companyList=[AuthorCompany(id=1327012755070104155, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012750812885554, xref=1, ext=[AuthorCompanyExt(id=1327012755078492764, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012750812885554, companyId=1327012755070104155, language=EN, country=null, province=null, city=null, postcode=null, companyName=null, departmentName=null, remark=
1Department of Computer Science and Technology, Tsinghua University, Beijing 100084), AuthorCompanyExt(id=1327012755086881373, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012750812885554, companyId=1327012755070104155, language=CN, country=null, province=null, city=null, postcode=null, companyName=null, departmentName=null, remark=
1清华大学计算机科学与技术系 北京 100084)]), AuthorCompany(id=1327012756525527646, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012750812885554, xref=2, ext=[AuthorCompanyExt(id=1327012756538110559, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012750812885554, companyId=1327012756525527646, language=EN, country=null, province=null, city=null, postcode=null, companyName=null, departmentName=null, remark=
2IEIT SYSTEMS Co., Ltd., Jinan 250101), AuthorCompanyExt(id=1327012756542304864, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012750812885554, companyId=1327012756525527646, language=CN, country=null, province=null, city=null, postcode=null, companyName=null, departmentName=null, remark=
2浪潮电子信息产业股份有限公司 济南 250101)])], figs=[ArticleFig(id=1327012758945641099, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012750812885554, language=EN, label=Figure 1, caption=
System architecture of HCSim, figureFileSmall=yRoerJhIm/QVVXI4MXzLSw==, figureFileBig=ykLUI5ZET8cBVykPzJoHqA==, tableContent=null), ArticleFig(id=1327012759021138572, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012750812885554, language=CN, label=图1, caption=
HCSim的系统架构, figureFileSmall=yRoerJhIm/QVVXI4MXzLSw==, figureFileBig=ykLUI5ZET8cBVykPzJoHqA==, tableContent=null), ArticleFig(id=1327012759260213901, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012750812885554, language=EN, label=Figure 2, caption=
Workload graph comparison, figureFileSmall=jt1EbAJSCpmaUTAjf9LOfQ==, figureFileBig=eCzR7RPr7m6R7EPh/4w50g==, tableContent=null), ArticleFig(id=1327012760921158287, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012750812885554, language=CN, label=图2, caption=
工作负载图对比, figureFileSmall=jt1EbAJSCpmaUTAjf9LOfQ==, figureFileBig=eCzR7RPr7m6R7EPh/4w50g==, tableContent=null), ArticleFig(id=1327012760996655760, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012750812885554, language=EN, label=Figure 3, caption=
Abstract representation of the heterogeneous consistency integrated computing systems topology in HCSim, figureFileSmall=vHGoehli/NUD4ryF5L1VYQ==, figureFileBig=FwqlqguFxbe4oUZX//mYTg==, tableContent=null), ArticleFig(id=1327012761055376017, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012750812885554, language=CN, label=图3, caption=
HCSim中异构一致性融合计算系统拓扑的抽象表示, figureFileSmall=vHGoehli/NUD4ryF5L1VYQ==, figureFileBig=FwqlqguFxbe4oUZX//mYTg==, tableContent=null), ArticleFig(id=1327012761109901970, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012750812885554, language=EN, label=Figure 4, caption=
Workflow and implementation method of HCSim, figureFileSmall=2kk9nOf9cs94KuFbiU75cg==, figureFileBig=U9uMPbx279MYzIe54oMf3A==, tableContent=null), ArticleFig(id=1327012761177010835, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012750812885554, language=CN, label=图4, caption=
HCSim的工作流程与实现方式, figureFileSmall=2kk9nOf9cs94KuFbiU75cg==, figureFileBig=U9uMPbx279MYzIe54oMf3A==, tableContent=null), ArticleFig(id=1327012761239925396, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012750812885554, language=EN, label=Figure 5, caption=
Schematic diagram of load execution engine, figureFileSmall=5JAgyY7UM/0C2AchBcQKEw==, figureFileBig=UhnIqFWkVM97sDHD7RkRTA==, tableContent=null), ArticleFig(id=1327012761332200085, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012750812885554, language=CN, label=图5, caption=
负载执行引擎的示意图, figureFileSmall=5JAgyY7UM/0C2AchBcQKEw==, figureFileBig=UhnIqFWkVM97sDHD7RkRTA==, tableContent=null), ArticleFig(id=1327012761395114646, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012750812885554, language=EN, label=Figure 6, caption=
Architecture diagram of interconnect topology without and with the introduction of the consistent interconnect technology CXL, figureFileSmall=H3MRWOSmFIFAINPqo3lfWA==, figureFileBig=C8Zt11n85BWN5oANdI8hAA==, tableContent=null), ArticleFig(id=1327012761470612119, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012750812885554, language=CN, label=图6, caption=
未引入一致性互连技术与引入了一致性互连技术CXL的互连拓扑架构图, figureFileSmall=H3MRWOSmFIFAINPqo3lfWA==, figureFileBig=C8Zt11n85BWN5oANdI8hAA==, tableContent=null), ArticleFig(id=1327012761554498200, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012750812885554, language=EN, label=Figure 7, caption=
Workload graph for distributed training of LLAMA2-13B, figureFileSmall=l2aRHQ8EZeE1Wfsr04oCnQ==, figureFileBig=/JZWlGd4M8QPFhDjWRQuyg==, tableContent=null), ArticleFig(id=1327012761642578585, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012750812885554, language=CN, label=图7, caption=
LLAMA2-13B的分布式训练负载图, figureFileSmall=l2aRHQ8EZeE1Wfsr04oCnQ==, figureFileBig=/JZWlGd4M8QPFhDjWRQuyg==, tableContent=null), ArticleFig(id=1327012761722270362, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012750812885554, language=EN, label=Figure 8, caption=
Simulation result on the computing system performance with and without introducing consistency, figureFileSmall=krx71khwQp2OF+z10yAUhQ==, figureFileBig=H/+uFuAPKRH8KMp+daewJQ==, tableContent=null), ArticleFig(id=1327012761780990619, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012750812885554, language=CN, label=图8, caption=
对比引入/不引入一致性的计算系统性能的仿真结果, figureFileSmall=krx71khwQp2OF+z10yAUhQ==, figureFileBig=H/+uFuAPKRH8KMp+daewJQ==, tableContent=null), ArticleFig(id=1327012761856488092, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012750812885554, language=EN, label=Figure 9, caption=
Execution trace of HCSim for the same operator output, figureFileSmall=FAuw9jOYrlZ12lFFsETn6w==, figureFileBig=mNeIIlA7TYqCbo6HXPksRg==, tableContent=null), ArticleFig(id=1327012761948762781, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012750812885554, language=CN, label=图9, caption=
HCSim对同一个算子输出的执行轨迹, figureFileSmall=FAuw9jOYrlZ12lFFsETn6w==, figureFileBig=mNeIIlA7TYqCbo6HXPksRg==, tableContent=null), ArticleFig(id=1327012762011677342, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012750812885554, language=EN, label=Figure 10, caption=
Impact of bandwidth on the performance of heterogeneous consistency-integrated computing systems, figureFileSmall=UWoSSk3+BJ/bHVc8OuC9aQ==, figureFileBig=XpAWnQZ9jnvJ8l9juOjr/g==, tableContent=null), ArticleFig(id=1327012762082980511, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012750812885554, language=CN, label=图10, caption=
带宽对异构一致性融合计算系统性能的影响, figureFileSmall=UWoSSk3+BJ/bHVc8OuC9aQ==, figureFileBig=XpAWnQZ9jnvJ8l9juOjr/g==, tableContent=null), ArticleFig(id=1327012762166866592, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012750812885554, language=EN, label=Figure 11, caption=
Impact of latency on the performance of heterogeneous consistency-integrated computing systems, figureFileSmall=B6at1QT7fnJLtzlKOzn9Gg==, figureFileBig=FtbO1Ga93tocePt5vgEX3Q==, tableContent=null), ArticleFig(id=1327012762254946977, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012750812885554, language=CN, label=图11, caption=
时延对异构一致性融合计算系统性能的影响, figureFileSmall=B6at1QT7fnJLtzlKOzn9Gg==, figureFileBig=FtbO1Ga93tocePt5vgEX3Q==, tableContent=null), ArticleFig(id=1327012762363998882, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012750812885554, language=EN, label=Figure 12, caption=
Comparison between traditional ring-allreduce and consistent ring-allreduce, figureFileSmall=PN02U/WSuqx5frdGEI61pA==, figureFileBig=TP5wwlNXSuY7UX6z9yyx/w==, tableContent=null), ArticleFig(id=1327012762422719139, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012750812885554, language=CN, label=图12, caption=
传统ring-allreduce和一致性ring-allreduce的对比, figureFileSmall=PN02U/WSuqx5frdGEI61pA==, figureFileBig=TP5wwlNXSuY7UX6z9yyx/w==, tableContent=null), ArticleFig(id=1327012762489828004, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012750812885554, language=EN, label=Table 1, caption=
Performance Modeling Work for Distributed AI Computing Systems, figureFileSmall=null, figureFileBig=null, tableContent=
| 工作名称 | 工作简介 |
|---|
| AMPeD[30] | 针对Transformer神经网络架构的分布式训练任务进行数学建模的研究. |
| Calculon[31] | 针对LLM的分布式训练过程进行数学建模的工作. |
| Paleo[32] | 早期使用数学模型与有向无环图(DAG)模型建模数据并行、模型并行等AI分布式训练任务的研究. |
| FlexFlow[33] | 一个开源的深度学习编译器和运行时系统,其中提供了一个基于图模型的模块,用于建模与优化分布式AI计算任务. |
| Daydream[34] | 使用细粒度的基于图的模型描述分布式训练,主要针对基于all-reduce的分布式并行任务进行建模与仿真. |
| Dpro replayer[35] | 基于图模型预测和优化数据并行分布式训练的性能. |
| DistSim[36] | 用于预测每个参与分布式训练的算力节点的详细工作时间线. |
| DistIR[37] | 一种基于MLIR[38]的中间表示,用于以伪代码的形式模拟分布式训练的执行过程. |
| Proteus[39] | 使用策略树描述并行策略,并依赖执行图来建模分布式训练. 该研究考虑了分布式训练的运行时特征. |
| TAG[40] | 使用图模型建模并预测分布式训练任务的耗时,同时基于图神经网络搜索并优化分布式训练的执行策略. |
| SMSG[41] | 一种无需进行实际性能采集分析(profiling)的分布式训练数学建模方法,仅需输入AI加速器的实际计算性能和系统通信能力,即可对分布式系统中的AI训练任务进行建模. |
| Astra-sim[29,42] | 一个帮助研究人员理解并探索分布式训练中多种软硬件协同设计空间的模拟器. |
), ArticleFig(id=1327012762556936869, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012750812885554, language=CN, label=表1, caption=
面向分布式AI计算系统的性能建模工作
, figureFileSmall=null, figureFileBig=null, tableContent=
| 工作名称 | 工作简介 |
|---|
| AMPeD[30] | 针对Transformer神经网络架构的分布式训练任务进行数学建模的研究. |
| Calculon[31] | 针对LLM的分布式训练过程进行数学建模的工作. |
| Paleo[32] | 早期使用数学模型与有向无环图(DAG)模型建模数据并行、模型并行等AI分布式训练任务的研究. |
| FlexFlow[33] | 一个开源的深度学习编译器和运行时系统,其中提供了一个基于图模型的模块,用于建模与优化分布式AI计算任务. |
| Daydream[34] | 使用细粒度的基于图的模型描述分布式训练,主要针对基于all-reduce的分布式并行任务进行建模与仿真. |
| Dpro replayer[35] | 基于图模型预测和优化数据并行分布式训练的性能. |
| DistSim[36] | 用于预测每个参与分布式训练的算力节点的详细工作时间线. |
| DistIR[37] | 一种基于MLIR[38]的中间表示,用于以伪代码的形式模拟分布式训练的执行过程. |
| Proteus[39] | 使用策略树描述并行策略,并依赖执行图来建模分布式训练. 该研究考虑了分布式训练的运行时特征. |
| TAG[40] | 使用图模型建模并预测分布式训练任务的耗时,同时基于图神经网络搜索并优化分布式训练的执行策略. |
| SMSG[41] | 一种无需进行实际性能采集分析(profiling)的分布式训练数学建模方法,仅需输入AI加速器的实际计算性能和系统通信能力,即可对分布式系统中的AI训练任务进行建模. |
| Astra-sim[29,42] | 一个帮助研究人员理解并探索分布式训练中多种软硬件协同设计空间的模拟器. |
), ArticleFig(id=1327012762649211558, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012750812885554, language=EN, label=Table 2, caption=
Element Definitions of Topology Graph, figureFileSmall=null, figureFileBig=null, tableContent=
| 元素名称 | 属性名 | 属性含义 |
|---|
| 交换节点 | | 无需定义属性 |
| 异构算力节点 | fp16算力/ FLOPS | fp16计算精度下的计算性能 |
| fp32算力/ FLOPS | fp32计算精度下的计算性能 |
| 内存节点 | 内存容量 | 包含可用的内存空间有多少 |
| 边 | 时延 | 所连接节点之间的通信时延 |
| 带宽 | 所连接节点之间的通信带宽 |
), ArticleFig(id=1327012762741486247, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012750812885554, language=CN, label=表2, caption=
拓扑图的元素定义
, figureFileSmall=null, figureFileBig=null, tableContent=
| 元素名称 | 属性名 | 属性含义 |
|---|
| 交换节点 | | 无需定义属性 |
| 异构算力节点 | fp16算力/ FLOPS | fp16计算精度下的计算性能 |
| fp32算力/ FLOPS | fp32计算精度下的计算性能 |
| 内存节点 | 内存容量 | 包含可用的内存空间有多少 |
| 边 | 时延 | 所连接节点之间的通信时延 |
| 带宽 | 所连接节点之间的通信带宽 |
), ArticleFig(id=1327012762812789416, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012750812885554, language=EN, label=Table 3, caption=
Element Definitions in the Workload Graph, figureFileSmall=null, figureFileBig=null, tableContent=
元素 名称 | 属性名 | 属性含义 |
|---|
计算 顶点 | 计算量/ FLOPs | 该计算顶点所代表的计算任务的计算量 |
| 访存量 | 该计算顶点所执行的任务需要的访存数据量 |
| 执行位置 | 该计算顶点所执行的异构算力位置 |
| 是否为起始顶点 | 该计算顶点是不是计算任务的起始顶点 |
通信 顶点 | 通信量 | 该通信顶点的总通信量 |
| 通信方式 | 该通信顶点的通信方式(例如集合通信、点对点通信) |
| 通信范围 | 根据通信顶点的通信方式,定义源节点、目的节点,或通信节点范围等 |
| 是否为起始顶点 | 该通信顶点是不是计算任务的起始顶点 |
连接顶 点的边 | 是否需要在第1次执行考虑 | 在计算任务第1次执行迭代时,不需要考虑有些边到顶点执行的依赖关系中 |
), ArticleFig(id=1327012762879898281, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012750812885554, language=CN, label=表3, caption=
负载图的元素定义
, figureFileSmall=null, figureFileBig=null, tableContent=
元素 名称 | 属性名 | 属性含义 |
|---|
计算 顶点 | 计算量/ FLOPs | 该计算顶点所代表的计算任务的计算量 |
| 访存量 | 该计算顶点所执行的任务需要的访存数据量 |
| 执行位置 | 该计算顶点所执行的异构算力位置 |
| 是否为起始顶点 | 该计算顶点是不是计算任务的起始顶点 |
通信 顶点 | 通信量 | 该通信顶点的总通信量 |
| 通信方式 | 该通信顶点的通信方式(例如集合通信、点对点通信) |
| 通信范围 | 根据通信顶点的通信方式,定义源节点、目的节点,或通信节点范围等 |
| 是否为起始顶点 | 该通信顶点是不是计算任务的起始顶点 |
连接顶 点的边 | 是否需要在第1次执行考虑 | 在计算任务第1次执行迭代时,不需要考虑有些边到顶点执行的依赖关系中 |
), ArticleFig(id=1327012762951201450, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012750812885554, language=EN, label=Table 4, caption=
Computational Load and Memory Access of Each Neural Layer in LLaMA Decoder, figureFileSmall=null, figureFileBig=null, tableContent=
| 神经层 | 前传计算量/ GFLOPs | 前传访存量/ GB | 反传计算量/ GFLOPs | 反传访存量/ GB |
|---|
| layer norm | 可忽略 | 0.00002+0.039×b | 可忽略 | 0.00015 + 0.156b |
| linear (K) | b×195 | 0.0488+0.039×b | b×390 | 0.39 + 0.156b |
| linear (Q) | b×195 | 0.0488+0.039×b | b×390 | 0.39 + 0.156b |
| linear (V) | b×195 | 0.0488+0.039×b | b×390 | 0.39 + 0.156b |
| self-attention | b×312.5 | 3.242×b | b×625 | 0.234b |
| concat | 可忽略 | 可忽略 | 可忽略 | 可忽略 |
| linear | b×195 | 0.0488+0.039×b | b×390 | 0.39 + 0.156b |
| dropout | 可忽略 | 0.0195×b | 可忽略 | 0.0195 |
| add | 可忽略 | 0.078×b | 可忽略 | 0.039 |
| layer norm | 可忽略 | 0.00002+0.039×b | 可忽略 | 0.00015 + 0.156b |
| linear | b×780 | 0.195 + 0.039×b | b×1560 | 1.562GB + 0.39b |
| GeLU | 可忽略 | 0.156×b | 可忽略 | 0.078 |
| linear | b×780 | 0.195 + 0.156×b | b×1560 | 1.562GB + 0.39b |
| dropout | 可忽略 | 0.0195×b | 可忽略 | 0.0195 |
| add | 可忽略 | 0.078×b | 可忽略 | 0.156 |
), ArticleFig(id=1327012763026698923, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012750812885554, language=CN, label=表4, caption=
LLaMA Decoder中每个神经层的计算量与访存量
, figureFileSmall=null, figureFileBig=null, tableContent=
| 神经层 | 前传计算量/ GFLOPs | 前传访存量/ GB | 反传计算量/ GFLOPs | 反传访存量/ GB |
|---|
| layer norm | 可忽略 | 0.00002+0.039×b | 可忽略 | 0.00015 + 0.156b |
| linear (K) | b×195 | 0.0488+0.039×b | b×390 | 0.39 + 0.156b |
| linear (Q) | b×195 | 0.0488+0.039×b | b×390 | 0.39 + 0.156b |
| linear (V) | b×195 | 0.0488+0.039×b | b×390 | 0.39 + 0.156b |
| self-attention | b×312.5 | 3.242×b | b×625 | 0.234b |
| concat | 可忽略 | 可忽略 | 可忽略 | 可忽略 |
| linear | b×195 | 0.0488+0.039×b | b×390 | 0.39 + 0.156b |
| dropout | 可忽略 | 0.0195×b | 可忽略 | 0.0195 |
| add | 可忽略 | 0.078×b | 可忽略 | 0.039 |
| layer norm | 可忽略 | 0.00002+0.039×b | 可忽略 | 0.00015 + 0.156b |
| linear | b×780 | 0.195 + 0.039×b | b×1560 | 1.562GB + 0.39b |
| GeLU | 可忽略 | 0.156×b | 可忽略 | 0.078 |
| linear | b×780 | 0.195 + 0.156×b | b×1560 | 1.562GB + 0.39b |
| dropout | 可忽略 | 0.0195×b | 可忽略 | 0.0195 |
| add | 可忽略 | 0.078×b | 可忽略 | 0.156 |
), ArticleFig(id=1327012763093807788, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012750812885554, language=EN, label=Table 5, caption=
Simulation Parameter Settings, figureFileSmall=null, figureFileBig=null, tableContent=
| 异构一致性融合计算系统 | LLaMA2-13B任务配置 |
|---|
| 算力节点 | 带宽 | 时延 |
|---|
| ToR_to_L1 | 12.5 GB/s | 5 μs | 训练时每个XPU 输入的batch_size | 1 |
| NIC_to_ToR | 12.5 GB/s | 5 μs | seq_len | 4096 |
| UPI | 62.4 GB/s | 100 ns | 隐藏层维度 | 5120 |
| PCIE_link | 128 GB/s | 250 ns | 使用算力节点 数量(scale) | 变量 |
| XPU_link | 900 GB/s | 100 ns | | |
| CXL_link | 变量 | 变量 | | |
), ArticleFig(id=1327012763169305261, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012750812885554, language=CN, label=表5, caption=
仿真参数设定
, figureFileSmall=null, figureFileBig=null, tableContent=
| 异构一致性融合计算系统 | LLaMA2-13B任务配置 |
|---|
| 算力节点 | 带宽 | 时延 |
|---|
| ToR_to_L1 | 12.5 GB/s | 5 μs | 训练时每个XPU 输入的batch_size | 1 |
| NIC_to_ToR | 12.5 GB/s | 5 μs | seq_len | 4096 |
| UPI | 62.4 GB/s | 100 ns | 隐藏层维度 | 5120 |
| PCIE_link | 128 GB/s | 250 ns | 使用算力节点 数量(scale) | 变量 |
| XPU_link | 900 GB/s | 100 ns | | |
| CXL_link | 变量 | 变量 | | |
), ArticleFig(id=1327012763228025518, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012750812885554, language=EN, label=Table 6, caption=
Comparison Results Before and After Optimization, figureFileSmall=null, figureFileBig=null, tableContent=
| 计算系统架构 | 传统ring-allreduce | 一致性ring-allreduce |
|---|
| 64个H100 同构 | 7.98 batch/s | 8.12 batch/s |
| 256个H100 同构 | 31.97 batch/s | 32.53 batch/s |
| 128个H100+128个 A100异构 | 10.65 batch/s | 10.71 batch/s |
| 128个H100+128个 V100异构 | 12.86 batch/s | 12.93 batch/s |
), ArticleFig(id=1327012763295134383, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012750812885554, language=CN, label=表6, caption=
优化前后对比结果
, figureFileSmall=null, figureFileBig=null, tableContent=
| 计算系统架构 | 传统ring-allreduce | 一致性ring-allreduce |
|---|
| 64个H100 同构 | 7.98 batch/s | 8.12 batch/s |
| 256个H100 同构 | 31.97 batch/s | 32.53 batch/s |
| 128个H100+128个 A100异构 | 10.65 batch/s | 10.71 batch/s |
| 128个H100+128个 V100异构 | 12.86 batch/s | 12.93 batch/s |
)], attaches=null, journal=Journal(id=1324759606947315908, delFlag=0, nameCn=计算机研究与发展, nameEn=Journal of Computer Research and Development, nameHistory1=null, nameHistory2=null, issn=1000-1239, eissn=null, cn=11-1777/TP, coden=null, periodic=0, language=CN, oaType=null, ccby=null, superviseOffice=null, ownerOffice=null, pubOffice=null, editorOffice=null, officeType=null, aims=null, clcCode=null, officeProv=null, officeCity=null, officeAddr=null, officeZip=null, officeEmail=null, officePhone=null, editDirector=null, officeDirector=null, officeDirectorPhone=null, officeStaffNum=null, officeEmpNum=null, coverPicUrl=k2l8zgqwstuxxevFX2qxVA==, journalPrice=null, startedYear=null, abbrevIsoEn=Journal of Computer Research and Development, journalRemark=null, publicationField=null, createdTime=1793776903141, updatedTime=1793850698612, createdBy=18614031015, updatedBy=13701087609, firstLetterCn=J, firstLetterEn=J, subjectCode=Electronic Information, subjectName=null, subjectCodeEn=Electronic Information, subjectNameEn=null, picCn=k2l8zgqwstuxxevFX2qxVA==, picEn=r6k0Ry+RD+c/VXj5QarmDw==, jcr=null, cjcr=null, exts=[JournalExt(id=1325069127710286688, language=CN, name=计算机研究与发展, nameHistory1=null, nameHistory2=null, managedBy=, sponsoredBy=, publishedBy=, editorOffice=, officeProv=null, officeCity=null, officeAddr=, officeZip=, editDirector=, officeDirector=null, officePhone=null, coverPicUrl=null, journalRemark=, submitArticleUrl=null, websiteUrl=, createdTime=1793850698640, updatedTime=1793850698640, createdBy=13701087609, updatedBy=13701087609, submissionGuidelinesUrl=, submissionAuthorUrl=https://crad.ict.ac.cn/journalx/authorLogOn.action, submissionEditorUrl=https://crad.ict.ac.cn/journalx/editorLogOn.action, submissionReviewUrl=https://crad.ict.ac.cn/journalx/expertLogOn.action, submissionCeEditorUrl=https://crad.ict.ac.cn/journalx/editorInChiefLogOn.action, submissionAeEditorUrl=, option={"copyright":""}), JournalExt(id=1325069127777395553, language=EN, name=Journal of Computer Research and Development, nameHistory1=null, nameHistory2=null, managedBy=, sponsoredBy=, publishedBy=, editorOffice=, officeProv=null, officeCity=null, officeAddr=, officeZip=, editDirector=, officeDirector=null, officePhone=null, coverPicUrl=null, journalRemark=, submitArticleUrl=null, websiteUrl=, createdTime=1793850698656, updatedTime=1793850698656, createdBy=13701087609, updatedBy=13701087609, submissionGuidelinesUrl=, submissionAuthorUrl=https://crad.ict.ac.cn/journalx/authorLogOn.action, submissionEditorUrl=https://crad.ict.ac.cn/journalx/editorLogOn.action, submissionReviewUrl=https://crad.ict.ac.cn/journalx/expertLogOn.action, submissionCeEditorUrl=https://crad.ict.ac.cn/journalx/editorInChiefLogOn.action, submissionAeEditorUrl=, option={"copyright":""})], databaseList=null, tenantJournalId=1324759914146529478, websiteList=[Website(id=1325070300139892771, webName=null, webTitle=null, webDomain=null, webCopyrigh=null, webIpcNo=null, seoTitle=null, seoKeywords=null, seoDescription=null, tenantJournalId=null, journalId=1324759914146529478, journalNameCn=null, journalNameEn=null, grayFlag=null, tenantId=1146029695717560320, platformId=null, journalGroupId=null, journalGroupNameCn=null, journalGroupNameEn=null, type=1, domain=https://castjournals.cast.org.cn/joweb/jsjyjyfz/CN, language=CN, createTime=1793850978169, createBy=18614031015, updateTime=1793851040259, updateBy=18614031015, name=计算机研究与发展-中文, tplId=1146099689490845704, title=计算机研究与发展, delFlag=0, indexPage=/home, props=[WebsiteProps(id=1325070893751349360, tenantId=1146029695717560320, journalId=null, journalGroupId=null, siteId=1325070300139892771, code=articleTextType, value=kx, createTime=1793851119697, updateTime=1793851119697, creator=18614031015, updator=18614031015), WebsiteProps(id=1325070893730377837, tenantId=1146029695717560320, journalId=null, journalGroupId=null, siteId=1325070300139892771, code=banner, value=null, createTime=1793851119692, updateTime=1793851119692, creator=18614031015, updator=18614031015), WebsiteProps(id=1325070893772320883, tenantId=1146029695717560320, journalId=null, journalGroupId=null, siteId=1325070300139892771, code=grayFlag, value=0, createTime=1793851119702, updateTime=1793851119702, creator=18614031015, updator=18614031015), WebsiteProps(id=1325070893721989228, tenantId=1146029695717560320, journalId=null, journalGroupId=null, siteId=1325070300139892771, code=logo, value=https://castjournals.cast.org.cn/joweb/jsjyjyfz/CN/file/pic?fileId=U27tDsy717YF7XSPA+Vwdw==, createTime=1793851119690, updateTime=1793851119690, creator=18614031015, updator=18614031015), WebsiteProps(id=1325070893784903797, tenantId=1146029695717560320, journalId=null, journalGroupId=null, siteId=1325070300139892771, code=minRunFlag, value=0, createTime=1793851119705, updateTime=1793851119705, creator=18614031015, updator=18614031015), WebsiteProps(id=1325070893742960751, tenantId=1146029695717560320, journalId=null, journalGroupId=null, siteId=1325070300139892771, code=picServerUrl, value=https://castjournals.cast.org.cn/joweb/jsjyjyfz/CN/file/pic, createTime=1793851119695, updateTime=1793851119695, creator=18614031015, updator=18614031015), WebsiteProps(id=1325070893780709492, tenantId=1146029695717560320, journalId=null, journalGroupId=null, siteId=1325070300139892771, code=silenceFlag, value=0, createTime=1793851119704, updateTime=1793851119704, creator=18614031015, updator=18614031015), WebsiteProps(id=1325070893738766446, tenantId=1146029695717560320, journalId=null, journalGroupId=null, siteId=1325070300139892771, code=staticResourcePath, value=https://castjournals.cast.org.cn/joweb/cast_kjdb_cn_619/, createTime=1793851119694, updateTime=1793851119694, creator=18614031015, updator=18614031015), WebsiteProps(id=1325070893755543665, tenantId=1146029695717560320, journalId=null, journalGroupId=null, siteId=1325070300139892771, code=themeColor, value=null, createTime=1793851119698, updateTime=1793851119698, creator=18614031015, updator=18614031015), WebsiteProps(id=1325070893763932274, tenantId=1146029695717560320, journalId=null, journalGroupId=null, siteId=1325070300139892771, code=themeStyle, value=null, createTime=1793851119700, updateTime=1793851119700, creator=18614031015, updator=18614031015)]), Website(id=1325070300580294698, webName=null, webTitle=null, webDomain=null, webCopyrigh=null, webIpcNo=null, seoTitle=null, seoKeywords=null, seoDescription=null, tenantJournalId=null, journalId=1324759914146529478, journalNameCn=null, journalNameEn=null, grayFlag=null, tenantId=1146029695717560320, platformId=null, journalGroupId=null, journalGroupNameCn=null, journalGroupNameEn=null, type=1, domain=https://castjournals.cast.org.cn/joweb/jsjyjyfz/EN, language=EN, createTime=1793850978274, createBy=18614031015, updateTime=1793851036638, updateBy=18614031015, name=计算机研究与发展-英文, tplId=1146101810881728533, title=Journal of Computer Research and Development, delFlag=0, indexPage=/home, props=[WebsiteProps(id=1325070925288321148, tenantId=1146029695717560320, journalId=null, journalGroupId=null, siteId=1325070300580294698, code=articleTextType, value=kx, createTime=1793851127216, updateTime=1793851127216, creator=18614031015, updator=18614031015), WebsiteProps(id=1325070925263155321, tenantId=1146029695717560320, journalId=null, journalGroupId=null, siteId=1325070300580294698, code=banner, value=null, createTime=1793851127210, updateTime=1793851127210, creator=18614031015, updator=18614031015), WebsiteProps(id=1325070925309292671, tenantId=1146029695717560320, journalId=null, journalGroupId=null, siteId=1325070300580294698, code=grayFlag, value=0, createTime=1793851127221, updateTime=1793851127221, creator=18614031015, updator=18614031015), WebsiteProps(id=1325070925254766712, tenantId=1146029695717560320, journalId=null, journalGroupId=null, siteId=1325070300580294698, code=logo, value=https://castjournals.cast.org.cn/joweb/jsjyjyfz/EN/file/pic?fileId=U27tDsy717YF7XSPA+Vwdw==, createTime=1793851127208, updateTime=1793851127208, creator=18614031015, updator=18614031015), WebsiteProps(id=1325070925326069889, tenantId=1146029695717560320, journalId=null, journalGroupId=null, siteId=1325070300580294698, code=minRunFlag, value=0, createTime=1793851127225, updateTime=1793851127225, creator=18614031015, updator=18614031015), WebsiteProps(id=1325070925279932539, tenantId=1146029695717560320, journalId=null, journalGroupId=null, siteId=1325070300580294698, code=picServerUrl, value=https://castjournals.cast.org.cn/joweb/jsjyjyfz/EN/file/pic, createTime=1793851127214, updateTime=1793851127214, creator=18614031015, updator=18614031015), WebsiteProps(id=1325070925317681280, tenantId=1146029695717560320, journalId=null, journalGroupId=null, siteId=1325070300580294698, code=silenceFlag, value=0, createTime=1793851127223, updateTime=1793851127223, creator=18614031015, updator=18614031015), WebsiteProps(id=1325070925271543930, tenantId=1146029695717560320, journalId=null, journalGroupId=null, siteId=1325070300580294698, code=staticResourcePath, value=https://castjournals.cast.org.cn/joweb/cast_kjdb_en_623/, createTime=1793851127212, updateTime=1793851127212, creator=18614031015, updator=18614031015), WebsiteProps(id=1325070925292515453, tenantId=1146029695717560320, journalId=null, journalGroupId=null, siteId=1325070300580294698, code=themeColor, value=null, createTime=1793851127217, updateTime=1793851127217, creator=18614031015, updator=18614031015), WebsiteProps(id=1325070925300904062, tenantId=1146029695717560320, journalId=null, journalGroupId=null, siteId=1325070300580294698, code=themeStyle, value=null, createTime=1793851127219, updateTime=1793851127219, creator=18614031015, updator=18614031015)])], journalTitle=计算机研究与发展, weixinUrl=null, journalUrl=https://crad.ict.ac.cn/, iacademicId=null, status=1, seqNo=null, journalTitleEn=Journal of Computer Research and Development, journalPhotoCn=k2l8zgqwstuxxevFX2qxVA==, journalPhotoEn=r6k0Ry+RD+c/VXj5QarmDw==, journalFirstLetter=J, journalRecommend=null, journalNew=null, journalCollection=null, jcrJf=null, cjcrJf=null, jcrJfStr=null, cjcrJfStr=null, submissionFirstDecision=null, sciSubjectClassification=null, casSubjectClassification=null, citeScore=null, totalCitationFrequency=null, icpCode=null, psCode=null, advertisingLicenseCode=null, copyrightInformation=null, country=null, option=, provinceCode=null, provinceName=null, collectFlag=false, interPubPlatform=, interPubPlatformUrl=null), detailUrlCn=https://castjournals.cast.org.cn/joweb/jsjyjyfz/CN/10.7544/issn1000-1239.202550120, detailUrlEn=https://castjournals.cast.org.cn/joweb/jsjyjyfz/EN/10.7544/issn1000-1239.202550120, pdfUrlCn=https://castjournals.cast.org.cn/joweb/jsjyjyfz/CN/PDF/10.7544/issn1000-1239.202550120, pdfUrlEn=https://castjournals.cast.org.cn/joweb/jsjyjyfz/EN/PDF/10.7544/issn1000-1239.202550120, aliStartDate=0, aliEndDate=0, collectionFlag=false, citedCount=null, citedUrl=null, previewStatus=0, delFlag=0, hasFullText=1, orderTime=1748793600000, fullTextJson=null, articleText=null, reference=null)