Article(id=1329455711169393557, tenantId=1146029695717560320, journalId=1324759914146529478, issueId=1329455653363495339, articleNumber=null, orderNo=null, doi=10.7544/issn1000-1239.202330303, pmid=null, cstr=null, oa=null, hot=null, price=null, onlineType=0, articleFormat=0, articleType=null, articleTypeStr=null, receivedDate=1680796800000, receivedDateStr=2023-04-07, revisedDate=1695744000000, revisedDateStr=2023-09-27, acceptedDate=null, acceptedDateStr=null, onlineDate=1794896541634, onlineDateStr=2026-11-17, pubDate=1706803200000, pubDateStr=2024-02-02, doiRegisterDate=null, doiRegisterDateStr=null, onlineIssueDate=1794896541634, onlineIssueDateStr=2026-11-17, onlineJustAcceptDate=null, onlineJustAcceptDateStr=null, onlineFirstDate=null, onlineFirstDateStr=null, sourceXml=null, magXml=null, createTime=1794896541634, creator=13701087609, updateTime=1794896541634, updator=13701087609, issue=Issue{id=1329455653363495339, tenantId=1146029695717560320, journalId=1324759914146529478, year='2024', volume='61', issue='2', pageStart='273', pageEnd='538', issueExtLink='null', onlineDate='null', pubDate='1706803200000', pubDateStr='2024-02-02', beforeIssueId=null, nextIssueId=null, price=null, status=1, issueComplete=1, articleOrder=1, issueType=-1, specialIssue=null, createTime=1794896527853, creator='13701087609', updateTime=1794896527853, updator='13701087609', preIssue=null, nextIssue=null, articleTotal=null, ext=null, issueFiles=null, downloadFileDto=null}, startPage=351, endPage=361, ext={EN=ArticleExt(id=1329455711349748630, articleId=1329455711169393557, tenantId=1146029695717560320, journalId=1324759914146529478, language=EN, title=Large Language Models: Principles, Implementation, and Progress, columnId=1329455694815802127, journalTitle=Journal of Computer Research and Development, columnName=Artificial Intelligence and Pattern Recognition, runingTitle=null, highlight=null, articleAbstract=
In recent years, the emergence and development of large language models (LLMs) have revolutionized the field of natural language processing and even artificial intelligence. With the increasing number of model parameters and training data, the perplexity of language models decreases in a predictable manner, which implies the improvement of performance on various natural language processing tasks. Therefore, scaling up language models has been a promising way to improve the system intelligence. In this survey, we first review the definition and scope of LLMs and provide a scale standard to distinguish “large” language models from the perspectives of performance and computing. Then, we review the development and representative work of LLMs in three dimensions: data, algorithm, and model architecture, showing how up-scaling in these dimensions drives the development of LLMs at different stages. Next, we discuss the emergent abilities of LLMs and possible interpretations behind them. We highlight three key emergent abilities, i.e., chain-of-thought prompting, in-context learning, and instruction-following, introducing their related advances and applications. Finally, we outline some potential directions and challenges of LLMs.
, authors=Wentao Shu, Ruixiao Li, Tianxiang Sun, Xuanjing Huang, Xipeng Qiu*, authorsList=Wentao Shu, Ruixiao Li, Tianxiang Sun, Xuanjing Huang, Xipeng Qiu, authorCompany=null, correspAuthors=Xipeng Qiu, authorNote=
born in 2002. Undergraduate. His main research interests include deep learning, natural language processing, and large language models
born in 2001. Undergraduate. His main research interests include deep learning and natural language processing
born in 1997. PhD candidate. His main research interests include deep learning and natural language processing
born in 1972. PhD, professor, PhD supervisor. Distinguished member of CCF. Her main research interests include natural language processing and information retrieval
born in 1983. PhD, professor, PhD supervisor. Senior member of CCF. His main research interests include artifitial intelligence, natural language processing, and large language models
, correspAuthorsNote=null, copyrightStatement=Copyright © 2024 Journal of Computer Research and Development. All rights reserved., copyrightOwner=null, extLink=null, articleAbsUrl=null, sourceXml=null, magXml=null, pdfUrl=null, pdf=null, pdfFileSize=null, pdfExtLink=null, richHtmlUrl=null, mobilePdfUrl=null, reviewReport=null, pdfFirstPage=null, abstractGraph=null, abstractGraphContent=null, abstractVideo=null, citation=null, cebUrl=null, magXmlContent=null, mapNumber=null, fund=null), CN=ArticleExt(id=1329455712943584165, articleId=1329455711169393557, tenantId=1146029695717560320, journalId=1324759914146529478, language=CN, title=大型语言模型:原理、实现与发展, columnId=1329455694975185681, journalTitle=计算机研究与发展, columnName=人工智能和模式识别, runingTitle=null, highlight=null, articleAbstract=
近年来,大型语言模型的出现和发展对自然语言处理和人工智能领域产生了变革性影响. 随着不断增大模型参数量和训练数据量,语言模型的文本建模困惑度以可预测的形式降低,在各类自然语言处理任务上的表现也持续提升. 因此,增加语言模型的参数和数据规模成为提升系统智能水平富有前景的途径. 首先回顾了大型语言模型的基本定义,从模型表现和算力需求的角度给出了“大型”语言模型的界定标准. 其次,从数据、算法、模型3个维度梳理了大型语言模型的发展历程及规律,展示了不同阶段各个维度的规模化如何推动语言模型的发展. 接着,考察了大型语言模型所表现出的涌现能力,介绍了思维链、情景学习和指令遵循等关键涌现能力的相关研究和应用现状. 最后,展望了大型语言模型的未来发展和技术挑战.
, authors=舒文韬, 李睿潇, 孙天祥, 黄萱菁, 邱锡鹏*, authorsList=舒文韬, 李睿潇, 孙天祥, 黄萱菁, 邱锡鹏, authorCompany=null, correspAuthors=邱锡鹏, authorNote=
 |
2002年生. 本科生. 主要研究方向为深度学习、自然语言处理、大型语言模型 |
 |
2001年生. 本科生. 主要研究方向为深度学习、自然语言处理 |
 |
1997年生. 博士研究生. 主要研究方向为深度学习、自然语言处理 |
 |
1972年生. 博士,教授,博士生导师. CCF杰出会员. 主要研究方向为自然语言处理、信息检索 |
 |
1983年生. 博士,教授,博士生导师. CCF 高级会员. 主要研究方向为人工智能、自然语言处理、大型语言模型 |
, correspAuthorsNote=
, copyrightStatement=版权所有 © 2024 计算机研究与发展编辑部, copyrightOwner=null, extLink=null, articleAbsUrl=null, sourceXml=AzqThix3OthCGud0Yj+Mig==, magXml=Cgz4nsa5FTpr8ULbpvJLJg==, pdfUrl=null, pdf=FOgyLMVfF8nqPZYstDxoJw==, pdfFileSize=948880, pdfExtLink=null, richHtmlUrl=null, mobilePdfUrl=null, reviewReport=null, pdfFirstPage=null, abstractGraph=XoLiFpXEK42HwBYtwaf6mw==, abstractGraphContent=null, abstractVideo=null, citation=null, cebUrl=null, magXmlContent=AX/K4w/VW2hvp0b3a2Eu8Q==, mapNumber=null, fund=null)}, authors=[Author(id=1329455713228796842, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1329455711169393557, orderNo=0, firstName=null, middleName=null, lastName=null, nameCn=null, orcid=null, stid=null, country=null, authorPic=null, dead=0, email=wtshu20@fudan.edu.cn, emailSecond=null, emailThird=null, correspondingAuthor=0, authorType=1, ext={EN=AuthorExt(id=1329455713312682924, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1329455711169393557, authorId=1329455713228796842, language=EN, stringName=Wentao Shu, firstName=Wentao, middleName=null, lastName=Shu, prefix=null, suffix=null, authorComment=null, nameInitials=null, affiliation=null, department=null, xref=null, address=
1School of Computer Science, Fudan University, Shanghai 200433, bio={"content":"
born in 2002. Undergraduate. His main research interests include deep learning, natural language processing, and large language models
"}, bioImg=null, bioContent=
born in 2002. Undergraduate. His main research interests include deep learning, natural language processing, and large language models
, aboutCorrespAuthor=null), CN=AuthorExt(id=1329455714940072877, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1329455711169393557, authorId=1329455713228796842, language=CN, stringName=舒文韬, firstName=null, middleName=null, lastName=null, prefix=null, suffix=null, authorComment=null, nameInitials=null, affiliation=null, department=null, xref=null, address=
1复旦大学计算机科学技术学院 上海 200433, bio={"img":"jJRokSVkLz8xtXxGrtA2XA==","content":"
2002年生. 本科生. 主要研究方向为深度学习、自然语言处理、大型语言模型
"}, bioImg=jJRokSVkLz8xtXxGrtA2XA==, bioContent=
2002年生. 本科生. 主要研究方向为深度学习、自然语言处理、大型语言模型
, aboutCorrespAuthor=null)}, companyList=[AuthorCompany(id=1329455713153299366, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1329455711169393557, xref=1, ext=[AuthorCompanyExt(id=1329455713157493671, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1329455711169393557, companyId=1329455713153299366, language=EN, country=null, province=null, city=null, postcode=null, companyName=null, departmentName=null, remark=
1School of Computer Science, Fudan University, Shanghai 200433), AuthorCompanyExt(id=1329455713165882280, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1329455711169393557, companyId=1329455713153299366, language=CN, country=null, province=null, city=null, postcode=null, companyName=null, departmentName=null, remark=
1复旦大学计算机科学技术学院 上海 200433)])]), Author(id=1329455715015570351, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1329455711169393557, orderNo=1, firstName=null, middleName=null, lastName=null, nameCn=null, orcid=null, stid=null, country=null, authorPic=null, dead=0, email=null, emailSecond=null, emailThird=null, correspondingAuthor=0, authorType=1, ext={EN=AuthorExt(id=1329455715082679217, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1329455711169393557, authorId=1329455715015570351, language=EN, stringName=Ruixiao Li, firstName=Ruixiao, middleName=null, lastName=Li, prefix=null, suffix=null, authorComment=null, nameInitials=null, affiliation=null, department=null, xref=null, address=
1School of Computer Science, Fudan University, Shanghai 200433, bio={"content":"
born in 2001. Undergraduate. His main research interests include deep learning and natural language processing
"}, bioImg=null, bioContent=
born in 2001. Undergraduate. His main research interests include deep learning and natural language processing
, aboutCorrespAuthor=null), CN=AuthorExt(id=1329455715141399474, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1329455711169393557, authorId=1329455715015570351, language=CN, stringName=李睿潇, firstName=null, middleName=null, lastName=null, prefix=null, suffix=null, authorComment=null, nameInitials=null, affiliation=null, department=null, xref=null, address=
1复旦大学计算机科学技术学院 上海 200433, bio={"img":"EkLPavFXg5HzaFw+1lnxMA==","content":"
2001年生. 本科生. 主要研究方向为深度学习、自然语言处理
"}, bioImg=EkLPavFXg5HzaFw+1lnxMA==, bioContent=
2001年生. 本科生. 主要研究方向为深度学习、自然语言处理
, aboutCorrespAuthor=null)}, companyList=[AuthorCompany(id=1329455713153299366, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1329455711169393557, xref=1, ext=[AuthorCompanyExt(id=1329455713157493671, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1329455711169393557, companyId=1329455713153299366, language=EN, country=null, province=null, city=null, postcode=null, companyName=null, departmentName=null, remark=
1School of Computer Science, Fudan University, Shanghai 200433), AuthorCompanyExt(id=1329455713165882280, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1329455711169393557, companyId=1329455713153299366, language=CN, country=null, province=null, city=null, postcode=null, companyName=null, departmentName=null, remark=
1复旦大学计算机科学技术学院 上海 200433)])]), Author(id=1329455715216896948, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1329455711169393557, orderNo=2, firstName=null, middleName=null, lastName=null, nameCn=null, orcid=null, stid=null, country=null, authorPic=null, dead=0, email=null, emailSecond=null, emailThird=null, correspondingAuthor=0, authorType=1, ext={EN=AuthorExt(id=1329455715296588726, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1329455711169393557, authorId=1329455715216896948, language=EN, stringName=Tianxiang Sun, firstName=Tianxiang, middleName=null, lastName=Sun, prefix=null, suffix=null, authorComment=null, nameInitials=null, affiliation=null, department=null, xref=null, address=
1School of Computer Science, Fudan University, Shanghai 200433, bio={"content":"
born in 1997. PhD candidate. His main research interests include deep learning and natural language processing
"}, bioImg=null, bioContent=
born in 1997. PhD candidate. His main research interests include deep learning and natural language processing
, aboutCorrespAuthor=null), CN=AuthorExt(id=1329455715376280503, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1329455711169393557, authorId=1329455715216896948, language=CN, stringName=孙天祥, firstName=null, middleName=null, lastName=null, prefix=null, suffix=null, authorComment=null, nameInitials=null, affiliation=null, department=null, xref=null, address=
1复旦大学计算机科学技术学院 上海 200433, bio={"img":"kNYpjXmX/3w+1yvZUMAxNQ==","content":"
1997年生. 博士研究生. 主要研究方向为深度学习、自然语言处理
"}, bioImg=kNYpjXmX/3w+1yvZUMAxNQ==, bioContent=
1997年生. 博士研究生. 主要研究方向为深度学习、自然语言处理
, aboutCorrespAuthor=null)}, companyList=[AuthorCompany(id=1329455713153299366, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1329455711169393557, xref=1, ext=[AuthorCompanyExt(id=1329455713157493671, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1329455711169393557, companyId=1329455713153299366, language=EN, country=null, province=null, city=null, postcode=null, companyName=null, departmentName=null, remark=
1School of Computer Science, Fudan University, Shanghai 200433), AuthorCompanyExt(id=1329455713165882280, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1329455711169393557, companyId=1329455713153299366, language=CN, country=null, province=null, city=null, postcode=null, companyName=null, departmentName=null, remark=
1复旦大学计算机科学技术学院 上海 200433)])]), Author(id=1329455715447583673, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1329455711169393557, orderNo=3, firstName=null, middleName=null, lastName=null, nameCn=null, orcid=null, stid=null, country=null, authorPic=null, dead=0, email=null, emailSecond=null, emailThird=null, correspondingAuthor=0, authorType=1, ext={EN=AuthorExt(id=1329455715531469755, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1329455711169393557, authorId=1329455715447583673, language=EN, stringName=Xuanjing Huang, firstName=Xuanjing, middleName=null, lastName=Huang, prefix=null, suffix=null, authorComment=null, nameInitials=null, affiliation=null, department=null, xref=null, address=
1School of Computer Science, Fudan University, Shanghai 200433, bio={"content":"
born in 1972. PhD, professor, PhD supervisor. Distinguished member of CCF. Her main research interests include natural language processing and information retrieval
"}, bioImg=null, bioContent=
born in 1972. PhD, professor, PhD supervisor. Distinguished member of CCF. Her main research interests include natural language processing and information retrieval
, aboutCorrespAuthor=null), CN=AuthorExt(id=1329455715611161532, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1329455711169393557, authorId=1329455715447583673, language=CN, stringName=黄萱菁, firstName=null, middleName=null, lastName=null, prefix=null, suffix=null, authorComment=null, nameInitials=null, affiliation=null, department=null, xref=null, address=
1复旦大学计算机科学技术学院 上海 200433, bio={"img":"o0TLCl0ufVVlD5b+RGwTzA==","content":"
1972年生. 博士,教授,博士生导师. CCF杰出会员. 主要研究方向为自然语言处理、信息检索
"}, bioImg=o0TLCl0ufVVlD5b+RGwTzA==, bioContent=
1972年生. 博士,教授,博士生导师. CCF杰出会员. 主要研究方向为自然语言处理、信息检索
, aboutCorrespAuthor=null)}, companyList=[AuthorCompany(id=1329455713153299366, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1329455711169393557, xref=1, ext=[AuthorCompanyExt(id=1329455713157493671, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1329455711169393557, companyId=1329455713153299366, language=EN, country=null, province=null, city=null, postcode=null, companyName=null, departmentName=null, remark=
1School of Computer Science, Fudan University, Shanghai 200433), AuthorCompanyExt(id=1329455713165882280, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1329455711169393557, companyId=1329455713153299366, language=CN, country=null, province=null, city=null, postcode=null, companyName=null, departmentName=null, remark=
1复旦大学计算机科学技术学院 上海 200433)])]), Author(id=1329455715707630526, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1329455711169393557, orderNo=4, firstName=null, middleName=null, lastName=null, nameCn=null, orcid=null, stid=null, country=null, authorPic=null, dead=0, email=xpqiu@fudan.edu.cn, emailSecond=null, emailThird=null, correspondingAuthor=1, authorType=1, ext={EN=AuthorExt(id=1329455715799905216, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1329455711169393557, authorId=1329455715707630526, language=EN, stringName=Xipeng Qiu, firstName=Xipeng, middleName=null, lastName=Qiu, prefix=null, suffix=null, authorComment=null, nameInitials=null, affiliation=null, department=null, xref=*, address=
1School of Computer Science, Fudan University, Shanghai 200433, bio={"content":"
born in 1983. PhD, professor, PhD supervisor. Senior member of CCF. His main research interests include artifitial intelligence, natural language processing, and large language models
"}, bioImg=null, bioContent=
born in 1983. PhD, professor, PhD supervisor. Senior member of CCF. His main research interests include artifitial intelligence, natural language processing, and large language models
, aboutCorrespAuthor=null), CN=AuthorExt(id=1329455715871208385, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1329455711169393557, authorId=1329455715707630526, language=CN, stringName=邱锡鹏, firstName=null, middleName=null, lastName=null, prefix=null, suffix=null, authorComment=null, nameInitials=null, affiliation=null, department=null, xref=*, address=
1复旦大学计算机科学技术学院 上海 200433, bio={"img":"XjA79FqS/kGnCVOzBMFzxg==","content":"
1983年生. 博士,教授,博士生导师. CCF 高级会员. 主要研究方向为人工智能、自然语言处理、大型语言模型
"}, bioImg=XjA79FqS/kGnCVOzBMFzxg==, bioContent=
1983年生. 博士,教授,博士生导师. CCF 高级会员. 主要研究方向为人工智能、自然语言处理、大型语言模型
, aboutCorrespAuthor=null)}, companyList=[AuthorCompany(id=1329455713153299366, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1329455711169393557, xref=1, ext=[AuthorCompanyExt(id=1329455713157493671, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1329455711169393557, companyId=1329455713153299366, language=EN, country=null, province=null, city=null, postcode=null, companyName=null, departmentName=null, remark=
1School of Computer Science, Fudan University, Shanghai 200433), AuthorCompanyExt(id=1329455713165882280, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1329455711169393557, companyId=1329455713153299366, language=CN, country=null, province=null, city=null, postcode=null, companyName=null, departmentName=null, remark=
1复旦大学计算机科学技术学院 上海 200433)])])], keywords=[Keyword(id=1329455716034786242, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1329455711169393557, language=EN, orderNo=1, keyword=natural language processing), Keyword(id=1329455716114478019, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1329455711169393557, language=EN, orderNo=2, keyword=neural networks), Keyword(id=1329455716215141316, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1329455711169393557, language=EN, orderNo=3, keyword=large language models), Keyword(id=1329455716286444485, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1329455711169393557, language=EN, orderNo=4, keyword=pre-training), Keyword(id=1329455716345164742, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1329455711169393557, language=EN, orderNo=5, keyword=alignment), Keyword(id=1329455716408079303, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1329455711169393557, language=CN, orderNo=1, keyword=自然语言处理), Keyword(id=1329455716512936904, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1329455711169393557, language=CN, orderNo=2, keyword=神经网络), Keyword(id=1329455716567462857, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1329455711169393557, language=CN, orderNo=3, keyword=大型语言模型), Keyword(id=1329455716630377418, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1329455711169393557, language=CN, orderNo=4, keyword=预训练), Keyword(id=1329455716726846411, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1329455711169393557, language=CN, orderNo=5, keyword=对齐)], refs=[Reference(id=1329455717616038866, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1329455711169393557, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=1, rfOrder=0, authorNames=null, journalName=null, refType=null, unstructuredReference=Kaplan J, McCandlish S, Henighan T, et al. Scaling laws for neural language models[J]. arXiv preprint, arXiv: 2001. 08361, 2020, articleTitle=null, refAbstract=null), Reference(id=1329455717708313555, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1329455711169393557, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=2, rfOrder=1, authorNames=null, journalName=null, refType=null, unstructuredReference=Brown P F, Della P V J, Desouza P V, et al. Class-based n-gram models of natural language[J]. Computational Linguistics, 1992. 18(4): 467−480, articleTitle=null, refAbstract=null), Reference(id=1329455717767033812, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1329455711169393557, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=3, rfOrder=2, authorNames=null, journalName=null, refType=null, unstructuredReference=Vaswani A, Shazeer N, Parmar N, et al. Attention is all you need[C] //Proc of the 30th Annual Conf on Neural Information Processing Systems. New York: Curran Associates, 2017: 5990−6008, articleTitle=null, refAbstract=null), Reference(id=1329455719381840854, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1329455711169393557, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=4, rfOrder=3, authorNames=null, journalName=null, refType=null, unstructuredReference=Lin Tianyang, Wang Yuxin, Liu Xiangyang et al. A survey of Transformers[J]. AI Open, 2021 (3): 111−132, articleTitle=null, refAbstract=null), Reference(id=1329455719474115543, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1329455711169393557, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=5, rfOrder=4, authorNames=null, journalName=null, refType=null, unstructuredReference=Wei J, Tay Y, Bommasani R, et al. Emergent abilities of large language models[J]. arXiv preprint, arXiv: 2206. 07682, 2022, articleTitle=null, refAbstract=null), Reference(id=1329455719528641497, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1329455711169393557, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=6, rfOrder=5, authorNames=null, journalName=null, refType=null, unstructuredReference=Rajbhandari S, Rasley J, Ruwase O, et al. ZeRo: Memory optimizations toward training trillion parameter models[C]//Proc of: Int Conf for High Performance Computing, Networking, Storage and Analysis (SC20). Piscataway, NJ: IEEE, 2020: 1−16, articleTitle=null, refAbstract=null), Reference(id=1329455719591556058, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1329455711169393557, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=7, rfOrder=6, authorNames=null, journalName=null, refType=null, unstructuredReference=Kingma D P, Ba J. Adam: A method for stochastic optimization[J]. arXiv preprint, arXiv: 1412. 6980, 2014, articleTitle=null, refAbstract=null), Reference(id=1329455719658664923, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1329455711169393557, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=8, rfOrder=7, authorNames=null, journalName=null, refType=null, unstructuredReference=Hoffmann J, Borgeaud S, Mensch A, et al. Training compute-optimal large language models[J]. arXiv preprint, arXiv: 2203. 15556, 2022, articleTitle=null, refAbstract=null), Reference(id=1329455719717385180, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1329455711169393557, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=9, rfOrder=8, authorNames=null, journalName=null, refType=null, unstructuredReference=Brown T, Mann B, Ryder N, et al. Language models are few-shot learners[J]. Advances in Neural Information Processing Systems, 2020. 33, 1877−1901, articleTitle=null, refAbstract=null), Reference(id=1329455719771911133, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1329455711169393557, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=10, rfOrder=9, authorNames=null, journalName=null, refType=null, unstructuredReference=Mikolov T, Chen Kai, Corrado G, et al. Efficient estimation of word representations in vector space[J]. arXiv preprint, arXiv: 1301. 3781, 2013, articleTitle=null, refAbstract=null), Reference(id=1329455719859991518, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1329455711169393557, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=11, rfOrder=10, authorNames=null, journalName=null, refType=null, unstructuredReference=Radford A, Narasimhan K, Salimans T, et al. Improving language understanding by generative pre-training[DB/OL].https://cdn.openai.com/research-covers/language-unsupervised/language_understanding_paper.pdf, 2018, articleTitle=null, refAbstract=null), Reference(id=1329455719914517471, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1329455711169393557, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=12, rfOrder=11, authorNames=null, journalName=null, refType=null, unstructuredReference=Devlin J, Chang M W, Lee K, et al. Bert: Pre-training of deep bidirectional transformers for language understanding[J]. arXiv preprint, arXiv: 1810. 04805, 2018, articleTitle=null, refAbstract=null), Reference(id=1329455719990014944, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1329455711169393557, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=13, rfOrder=12, authorNames=null, journalName=null, refType=null, unstructuredReference=Collobert R, Weston J. A unified architecture for natural language processing: Deep neural networks with multitask learning[C]//Proc of the 25th Int Conf on Machine Learning. New York: ACM, 2008: 160−167, articleTitle=null, refAbstract=null), Reference(id=1329455720073901025, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1329455711169393557, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=14, rfOrder=13, authorNames=null, journalName=null, refType=null, unstructuredReference=OpenAI. GPT-4 technical report[J]. arXiv preprint, arXiv: 2303. 08774, 2023, articleTitle=null, refAbstract=null), Reference(id=1329455720136815586, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1329455711169393557, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=15, rfOrder=14, authorNames=null, journalName=null, refType=null, unstructuredReference=Chung H W, Hou Le, Longpre S, et al. Scaling instruction-finetuned language models[J]. arXiv preprint, arXiv: 2210. 11416, 2022, articleTitle=null, refAbstract=null), Reference(id=1329455720203924451, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1329455711169393557, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=16, rfOrder=15, authorNames=null, journalName=null, refType=null, unstructuredReference=Touvron H, Lavril T, Izacard G, et al. LLaMa: Open and efficient foundation language models[J]. arXiv preprint, arXiv: 2302. 13971, 2023, articleTitle=null, refAbstract=null), Reference(id=1329455720283616228, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1329455711169393557, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=17, rfOrder=16, authorNames=null, journalName=null, refType=null, unstructuredReference=Peters M, Neumann M, Iyyer M. Deep contextualized word representations. [C]// Proc of the 2018 Conf of the North American Chapter of the Association for Computational Linguistics: Human Language Technologies. Stroudsburg, PA: ACL, 2018: 2227–2237, articleTitle=null, refAbstract=null), Reference(id=1329455720346530789, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1329455711169393557, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=18, rfOrder=17, authorNames=null, journalName=null, refType=null, unstructuredReference=Cho K, Van Merriënboer B, Gulcehre C, et al. Learning phrase representations using RNN encoder-decoder for statistical machine translation[J]. arXiv preprint, arXiv: 1406. 1078, 2014, articleTitle=null, refAbstract=null), Reference(id=1329455720413639654, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1329455711169393557, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=19, rfOrder=18, authorNames=null, journalName=null, refType=null, unstructuredReference=Tang G, Müller M, Rios A, et al. Why self-attention? a targeted evaluation of neural machine translation architectures[J]. arXiv preprint, arXiv: 1808. 08946, 2018, articleTitle=null, refAbstract=null), Reference(id=1329455720472359911, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1329455711169393557, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=20, rfOrder=19, authorNames=null, journalName=null, refType=null, unstructuredReference=Hoffmann J, Borgeaud S, Mensch A, et al. Training compute-optimal large language models[J]. arXiv preprint, arXiv: 2203. 15556, 2022, articleTitle=null, refAbstract=null), Reference(id=1329455720531080168, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1329455711169393557, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=21, rfOrder=20, authorNames=null, journalName=null, refType=null, unstructuredReference=Michaud E J, Liu Ziming, Girit U, et al. The quantization model of neural scaling[J]. arXiv preprint, arXiv: 2303. 13506, 2023., articleTitle=null, refAbstract=null), Reference(id=1329455720598189033, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1329455711169393557, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=22, rfOrder=21, authorNames=null, journalName=null, refType=null, unstructuredReference=Sun Tianxiang, Shao Yunfan, Qian Hong, et al. Black-box tuning for language-model-as-a-service[C]//Proc of Int Conf on Machine Learning. New York: PMLR, 2022: 20841−20855, articleTitle=null, refAbstract=null), Reference(id=1329455720770155499, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1329455711169393557, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=23, rfOrder=22, authorNames=null, journalName=null, refType=null, unstructuredReference=Akyürek E, Schuurmans D, Andreas J, et al. What learning algorithm is in-context learning? investigations with linear models[J]. arXiv preprint, arXiv: 2211. 15661, 2022, articleTitle=null, refAbstract=null), Reference(id=1329455720841458668, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1329455711169393557, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=24, rfOrder=23, authorNames=null, journalName=null, refType=null, unstructuredReference=Dai Damai, Sun Yutao, Dong Li, et al. Why can GPT learn in-context? language models secretly perform gradient descent as meta optimizers[J]. arXiv preprint, arXiv: 2212. 10559, 2022, articleTitle=null, refAbstract=null), Reference(id=1329455720904373229, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1329455711169393557, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=25, rfOrder=24, authorNames=null, journalName=null, refType=null, unstructuredReference=Min S, Lyu X, Holtzman A, et al. Rethinking the role of demonstrations: What makes in-context learning work?[J]. arXiv preprint, arXiv: 2202. 12837, 2022, articleTitle=null, refAbstract=null), Reference(id=1329455720967287790, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1329455711169393557, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=26, rfOrder=25, authorNames=null, journalName=null, refType=null, unstructuredReference=Wei J, Wei J, Tay Y, et al. Larger language models do in-context learning differently[J]. arXiv preprint, arXiv: 2303. 03846, 2023, articleTitle=null, refAbstract=null), Reference(id=1329455721042785264, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1329455711169393557, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=27, rfOrder=26, authorNames=null, journalName=null, refType=null, unstructuredReference=Zhao Z, Wallace E, Feng Si, et al. Calibrate before use: Improving few-shot performance of language models[C]// Proc of Int Conf on Machine Learning. New York: PMLR 2021: 12697−12706, articleTitle=null, refAbstract=null), Reference(id=1329455721122477041, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1329455711169393557, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=28, rfOrder=27, authorNames=null, journalName=null, refType=null, unstructuredReference=Wei J, Wang Xuezhi, Schuurmans D, et al. Chain-of-thought prompting elicits reasoning in large language models[J]. Advances in Neural Information Processing Systems, 2022. 35, 24824−24837, articleTitle=null, refAbstract=null), Reference(id=1329455721206363122, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1329455711169393557, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=29, rfOrder=28, authorNames=null, journalName=null, refType=null, unstructuredReference=Kojima T, Gu S S, Reid M, et al. Large language models are zero-shot reasoners[J]. Advances in neural information processing systems, 2022. 35, 22199−22213, articleTitle=null, refAbstract=null), Reference(id=1329455721281860595, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1329455711169393557, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=30, rfOrder=29, authorNames=null, journalName=null, refType=null, unstructuredReference=Zhou D, Schärli N, Hou L, et al. Least-to-Most prompting enables complex reasoning in large language models[J]. arXiv preprint, arXiv: 2205. 10625, 2022, articleTitle=null, refAbstract=null), Reference(id=1329455721348969460, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1329455711169393557, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=31, rfOrder=30, authorNames=null, journalName=null, refType=null, unstructuredReference=Wang Xuezhi, Wei J, Schuurmans D, et al. Self-consistency improves chain of thought reasoning in language models[J]. arXiv preprint, arXiv: 2203. 11171, 2022, articleTitle=null, refAbstract=null), Reference(id=1329455721411884021, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1329455711169393557, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=32, rfOrder=31, authorNames=null, journalName=null, refType=null, unstructuredReference=Zhang Zhuosheng, Zhang A, Li Mu, et al. Automatic chain of thought prompting in large language models[J]. arXiv preprint, arXiv: 2210. 03493, 2022, articleTitle=null, refAbstract=null), Reference(id=1329455721478992886, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1329455711169393557, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=33, rfOrder=32, authorNames=null, journalName=null, refType=null, unstructuredReference=Khashabi D, Kordi Y, Hajishirzi H. UnifiedQA-v2: Stronger generalization via broader cross-format training[J]. arXiv preprint, arXiv: 2202. 12359, 2022, articleTitle=null, refAbstract=null), Reference(id=1329455721558684663, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1329455711169393557, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=34, rfOrder=33, authorNames=null, journalName=null, refType=null, unstructuredReference=Scao T L, Fan A, Akiki C, et al. BLOOM: A 176B-parameter open-access multilingual language model[J]. arXiv preprint, arXiv: 2211. 05100, 2022, articleTitle=null, refAbstract=null), Reference(id=1329455721621599224, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1329455711169393557, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=35, rfOrder=34, authorNames=null, journalName=null, refType=null, unstructuredReference=Khashabi D, Min S, Khot T, et al. UnifiedQA: Crossing format boundaries with a single QA system[J]. arXiv preprint, arXiv: 2005. 00700, 2020, articleTitle=null, refAbstract=null), Reference(id=1329455721713873913, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1329455711169393557, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=36, rfOrder=35, authorNames=null, journalName=null, refType=null, unstructuredReference=Huang Shaohan, Dong Li, Wang Wenhui, et al. Language is not all you need: Aligning perception with language models[J]. arXiv preprint, arXiv: 2302. 14045, 2023, articleTitle=null, refAbstract=null), Reference(id=1329455721785177082, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1329455711169393557, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=37, rfOrder=36, authorNames=null, journalName=null, refType=null, unstructuredReference=Peng Zhiliang, Wang Wenhui, Dong Li, et al. Kosmos-2: Grounding multimodal large language models to the world[J]. arXiv preprint, arXiv: 2306. 14824, 2023, articleTitle=null, refAbstract=null), Reference(id=1329455721869063163, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1329455711169393557, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=38, rfOrder=37, authorNames=null, journalName=null, refType=null, unstructuredReference=Ouyang Long, Wu J, Jiang Xu, et al. Training language models to follow instructions with human feedback[J]. Advances in Neural Information Processing Systems, 2022. 35, 27730−27744, articleTitle=null, refAbstract=null), Reference(id=1329455721948754940, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1329455711169393557, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=39, rfOrder=38, authorNames=null, journalName=null, refType=null, unstructuredReference=Wang Yizhong, Kordi Y, Mishra S, et al. Self-instruct: Aligning language model with self generated instructions[J]. arXiv preprint, arXiv: 2212. 10560, 2022, articleTitle=null, refAbstract=null), Reference(id=1329455722020058109, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1329455711169393557, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=40, rfOrder=39, authorNames=null, journalName=null, refType=null, unstructuredReference=Bai Yuntao, Kadavath S, Kundu S, et al. Constitutional AI: Harmlessness from AI feedback[J]. arXiv preprint, arXiv: 2212. 08073, 2022, articleTitle=null, refAbstract=null), Reference(id=1329455722120721406, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1329455711169393557, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=41, rfOrder=40, authorNames=null, journalName=null, refType=null, unstructuredReference=Zheng Lianmin, Chiang W L, Sheng Ying, et al. Judging LLM-as-a-judge with MT-Bench and Chatbot Arena[J]. arXiv preprint, arXiv: 2306. 05685, 2023, articleTitle=null, refAbstract=null), Reference(id=1329455722192024575, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1329455711169393557, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=42, rfOrder=41, authorNames=null, journalName=null, refType=null, unstructuredReference=Wang Sinong, Li B Z, Khabsa M, et al. Linformer: Self-attention with linear complexity[J]. arXiv preprint, arXiv: 2006. 04768, 2020, articleTitle=null, refAbstract=null), Reference(id=1329455723811025920, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1329455711169393557, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=43, rfOrder=42, authorNames=null, journalName=null, refType=null, unstructuredReference=Dao T, Fu D, Ermon S, et al. Flashattention: Fast and memory-efficient exact attention with io-awareness[J]. Advances in Neural Information Processing Systems, 2022. 35, 16344−16359, articleTitle=null, refAbstract=null), Reference(id=1329455723869745152, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1329455711169393557, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=44, rfOrder=43, authorNames=null, journalName=null, refType=null, unstructuredReference=Peng Bo, Alcaide E, Anthony Q, et al. RWKV: Reinventing RNNs for the Transformer Era[J]. arXiv preprint, arXiv: 2305. 13048, 2023, articleTitle=null, refAbstract=null), Reference(id=1329455723928465409, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1329455711169393557, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=45, rfOrder=44, authorNames=null, journalName=null, refType=null, unstructuredReference=Schick T, Dwivedi-Yu J, Dessì R, et al. Toolformer: Language models can teach themselves to use tools[J]. arXiv preprint, arXiv: 2302. 04761, 2023, articleTitle=null, refAbstract=null), Reference(id=1329455724033323010, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1329455711169393557, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=46, rfOrder=45, authorNames=null, journalName=null, refType=null, unstructuredReference=陈宇飞,沈超,王骞,等. 人工智能系统安全与隐私风险[J]. 计算机研究与发展,2019,56(10):2135−2150, articleTitle=null, refAbstract=null), Reference(id=1329455724150763523, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1329455711169393557, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=46, rfOrder=46, authorNames=null, journalName=null, refType=null, unstructuredReference=Chen Yufei, Shen Chao, Wang Qian, et al. Security and privacy risks in artificial intelligence system[J]. Journal of Computer Research and Development, 2019. 56(10): 2135−2150(in Chinese), articleTitle=null, refAbstract=null)], funds=null, companyList=[AuthorCompany(id=1329455713153299366, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1329455711169393557, xref=1, ext=[AuthorCompanyExt(id=1329455713157493671, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1329455711169393557, companyId=1329455713153299366, language=EN, country=null, province=null, city=null, postcode=null, companyName=null, departmentName=null, remark=
1School of Computer Science, Fudan University, Shanghai 200433), AuthorCompanyExt(id=1329455713165882280, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1329455711169393557, companyId=1329455713153299366, language=CN, country=null, province=null, city=null, postcode=null, companyName=null, departmentName=null, remark=
1复旦大学计算机科学技术学院 上海 200433)])], figs=[ArticleFig(id=1329455716844286924, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1329455711169393557, language=EN, label=Figure 1, caption=
Development path of language models, figureFileSmall=tJhad//dmyB5ZRj6H8kDNQ==, figureFileBig=XoLiFpXEK42HwBYtwaf6mw==, tableContent=null), ArticleFig(id=1329455716999476173, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1329455711169393557, language=CN, label=图1, caption=
语言模型发展路径, figureFileSmall=tJhad//dmyB5ZRj6H8kDNQ==, figureFileBig=XoLiFpXEK42HwBYtwaf6mw==, tableContent=null), ArticleFig(id=1329455717234357198, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1329455711169393557, language=EN, label=Figure 2, caption=
Number of parameters and training FLOPs of common LLMs, figureFileSmall=CED5P4DNMipUlBGoBsY9nw==, figureFileBig=E6n9lW3yJtipSTrnm4eYcA==, tableContent=null), ArticleFig(id=1329455717314048975, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1329455711169393557, language=CN, label=图2, caption=
常见大型语言模型的参数量和训练计算量, figureFileSmall=CED5P4DNMipUlBGoBsY9nw==, figureFileBig=E6n9lW3yJtipSTrnm4eYcA==, tableContent=null), ArticleFig(id=1329455717397935056, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1329455711169393557, language=EN, label=Table 1, caption=
Comparison of Existing Large Language Models
, figureFileSmall=null, figureFileBig=null, tableContent=
| 模型 | 发布机构 | 所在国家 | 模型参数量 | 模态 | 最大序列长度 | 使用方式 |
|---|
| GPT-3 | OpenAI | 美国 | 1750亿 | 语言 | 2 048 | API |
| GPT-4 | OpenAI | 美国 | | 语言、图像 | 32 000 | API |
| Codex | OpenAI | 美国 | 120亿 | 代码 | | API |
| J1-Jumbo | AI21 Labs | 美国 | 1780亿 | 语言 | 2 048 | 受限访问 |
| J1-Grande | AI21 Labs | 美国 | 170亿 | 语言 | 2 048 | 受限访问 |
| BLOOM | BigScience | 法国 | 1760亿 | 语言 | 2 048 | 开源 |
| GPT-NeoX | EleutherAI | | 200亿 | 语言 | 2 048 | 开源 |
| Anthropic-LM | Anthropic | 美国 | 520亿 | 语言 | 8 192 | |
| Claude | Anthropic | 美国 | | 语言 | 100 000 | 受限访问 |
| CodeGen | Salesforce | 美国 | 160亿 | 代码 | 2 048 | 开源 |
| Turing-NLG | Microsoft | 美国 | 170亿 | 语言 | | |
| MT-NLG | Microsoft | 美国 | 5300亿 | 语言 | 2 048 | |
| OPT | Meta | 美国 | 1750亿 | 语言 | 2 048 | 开源 |
| LLaMA | Meta | 美国 | 650亿 | 语言 | 2 048 | 开源 |
| T5 | Google | 美国 | 110亿 | 语言 | 512 | 开源 |
| UL2 | Google | 美国 | 200亿 | 语言 | 512 | 开源 |
| AlphaCode | Google | 美国 | 410亿 | 代码 | 768 | |
| PaLM | Google | 美国 | 5400亿 | 语言 | 2 048 | API |
| LaMDA | Google | 美国 | 1370亿 | 语言 | | |
| Chinchilla | Google | 美国 | 700亿 | 语言 | | |
| Gopher | Google | 美国 | 2800亿 | 语言 | 2 048 | |
| CPM-2 | 清华大学、智源 | 中国 | 1 980亿 | 语言 | | 开源 |
| GLM-130B | 清华大学、智谱 | 中国 | 1300亿 | 语言 | 2 048 | 开源 |
| MOSS | 复旦大学 | 中国 | 160亿 | 语言 | 2 048 | 开源 |
| InternLM | 上海AI LAB | 中国 | 1040亿 | 语言 | 2 048 | |
| ERNIE 3.0 Titan | 百度 | 中国 | 2600亿 | 语言 | 512 | 受限访问 |
| 源1.0 | 浪潮 | 中国 | 2450亿 | 语言 | 2 048 | 受限访问 |
| 盘古-α | 华为 | 中国 | 2 000亿 | 语言 | 1 024 | |
| 盘古-Σ | 华为 | 中国 | 10000亿 | 语言 | 1 024 | |
| WeLM | 腾讯 | 中国 | 100亿 | 语言 | | 受限访问 |
| M6 | 阿里巴巴 | 中国 | 1000亿 | 语言、图像 | | |
| M6-10T | 阿里巴巴 | 中国 | 100000亿 | 语言、图像 | 512 | |
| PLUG | 阿里巴巴 | 中国 | 270亿 | 语言 | | |
| Baichuan | 百川智能 | 中国 | 70亿 | 语言 | 4 096 | 开源 |
| YaLM | Yandex | 俄罗斯 | 1000亿 | 语言 | 2 048 | 开源 |
), ArticleFig(id=1329455717473432529, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1329455711169393557, language=CN, label=表1, caption=
已有大型语言模型对比
, figureFileSmall=null, figureFileBig=null, tableContent=
| 模型 | 发布机构 | 所在国家 | 模型参数量 | 模态 | 最大序列长度 | 使用方式 |
|---|
| GPT-3 | OpenAI | 美国 | 1750亿 | 语言 | 2 048 | API |
| GPT-4 | OpenAI | 美国 | | 语言、图像 | 32 000 | API |
| Codex | OpenAI | 美国 | 120亿 | 代码 | | API |
| J1-Jumbo | AI21 Labs | 美国 | 1780亿 | 语言 | 2 048 | 受限访问 |
| J1-Grande | AI21 Labs | 美国 | 170亿 | 语言 | 2 048 | 受限访问 |
| BLOOM | BigScience | 法国 | 1760亿 | 语言 | 2 048 | 开源 |
| GPT-NeoX | EleutherAI | | 200亿 | 语言 | 2 048 | 开源 |
| Anthropic-LM | Anthropic | 美国 | 520亿 | 语言 | 8 192 | |
| Claude | Anthropic | 美国 | | 语言 | 100 000 | 受限访问 |
| CodeGen | Salesforce | 美国 | 160亿 | 代码 | 2 048 | 开源 |
| Turing-NLG | Microsoft | 美国 | 170亿 | 语言 | | |
| MT-NLG | Microsoft | 美国 | 5300亿 | 语言 | 2 048 | |
| OPT | Meta | 美国 | 1750亿 | 语言 | 2 048 | 开源 |
| LLaMA | Meta | 美国 | 650亿 | 语言 | 2 048 | 开源 |
| T5 | Google | 美国 | 110亿 | 语言 | 512 | 开源 |
| UL2 | Google | 美国 | 200亿 | 语言 | 512 | 开源 |
| AlphaCode | Google | 美国 | 410亿 | 代码 | 768 | |
| PaLM | Google | 美国 | 5400亿 | 语言 | 2 048 | API |
| LaMDA | Google | 美国 | 1370亿 | 语言 | | |
| Chinchilla | Google | 美国 | 700亿 | 语言 | | |
| Gopher | Google | 美国 | 2800亿 | 语言 | 2 048 | |
| CPM-2 | 清华大学、智源 | 中国 | 1 980亿 | 语言 | | 开源 |
| GLM-130B | 清华大学、智谱 | 中国 | 1300亿 | 语言 | 2 048 | 开源 |
| MOSS | 复旦大学 | 中国 | 160亿 | 语言 | 2 048 | 开源 |
| InternLM | 上海AI LAB | 中国 | 1040亿 | 语言 | 2 048 | |
| ERNIE 3.0 Titan | 百度 | 中国 | 2600亿 | 语言 | 512 | 受限访问 |
| 源1.0 | 浪潮 | 中国 | 2450亿 | 语言 | 2 048 | 受限访问 |
| 盘古-α | 华为 | 中国 | 2 000亿 | 语言 | 1 024 | |
| 盘古-Σ | 华为 | 中国 | 10000亿 | 语言 | 1 024 | |
| WeLM | 腾讯 | 中国 | 100亿 | 语言 | | 受限访问 |
| M6 | 阿里巴巴 | 中国 | 1000亿 | 语言、图像 | | |
| M6-10T | 阿里巴巴 | 中国 | 100000亿 | 语言、图像 | 512 | |
| PLUG | 阿里巴巴 | 中国 | 270亿 | 语言 | | |
| Baichuan | 百川智能 | 中国 | 70亿 | 语言 | 4 096 | 开源 |
| YaLM | Yandex | 俄罗斯 | 1000亿 | 语言 | 2 048 | 开源 |
)], attaches=null, journal=Journal(id=1324759606947315908, delFlag=0, nameCn=计算机研究与发展, nameEn=Journal of Computer Research and Development, nameHistory1=null, nameHistory2=null, issn=1000-1239, eissn=null, cn=11-1777/TP, coden=null, periodic=0, language=CN, oaType=null, ccby=null, superviseOffice=null, ownerOffice=null, pubOffice=null, editorOffice=null, officeType=null, aims=null, clcCode=null, officeProv=null, officeCity=null, officeAddr=null, officeZip=null, officeEmail=null, officePhone=null, editDirector=null, officeDirector=null, officeDirectorPhone=null, officeStaffNum=null, officeEmpNum=null, coverPicUrl=k2l8zgqwstuxxevFX2qxVA==, journalPrice=null, startedYear=null, abbrevIsoEn=Journal of Computer Research and Development, journalRemark=null, publicationField=null, createdTime=1793776903141, updatedTime=1793850698612, createdBy=18614031015, updatedBy=13701087609, firstLetterCn=J, firstLetterEn=J, subjectCode=Electronic Information, subjectName=null, subjectCodeEn=Electronic Information, subjectNameEn=null, picCn=k2l8zgqwstuxxevFX2qxVA==, picEn=r6k0Ry+RD+c/VXj5QarmDw==, jcr=null, cjcr=null, exts=[JournalExt(id=1325069127710286688, language=CN, name=计算机研究与发展, nameHistory1=null, nameHistory2=null, managedBy=, sponsoredBy=, publishedBy=, editorOffice=, officeProv=null, officeCity=null, officeAddr=, officeZip=, editDirector=, officeDirector=null, officePhone=null, coverPicUrl=null, journalRemark=, submitArticleUrl=null, websiteUrl=, createdTime=1793850698640, updatedTime=1793850698640, createdBy=13701087609, updatedBy=13701087609, submissionGuidelinesUrl=, submissionAuthorUrl=https://crad.ict.ac.cn/journalx/authorLogOn.action, submissionEditorUrl=https://crad.ict.ac.cn/journalx/editorLogOn.action, submissionReviewUrl=https://crad.ict.ac.cn/journalx/expertLogOn.action, submissionCeEditorUrl=https://crad.ict.ac.cn/journalx/editorInChiefLogOn.action, submissionAeEditorUrl=, option={"copyright":""}), JournalExt(id=1325069127777395553, language=EN, name=Journal of Computer Research and Development, nameHistory1=null, nameHistory2=null, managedBy=, sponsoredBy=, publishedBy=, editorOffice=, officeProv=null, officeCity=null, officeAddr=, officeZip=, editDirector=, officeDirector=null, officePhone=null, coverPicUrl=null, journalRemark=, submitArticleUrl=null, websiteUrl=, createdTime=1793850698656, updatedTime=1793850698656, createdBy=13701087609, updatedBy=13701087609, submissionGuidelinesUrl=, submissionAuthorUrl=https://crad.ict.ac.cn/journalx/authorLogOn.action, submissionEditorUrl=https://crad.ict.ac.cn/journalx/editorLogOn.action, submissionReviewUrl=https://crad.ict.ac.cn/journalx/expertLogOn.action, submissionCeEditorUrl=https://crad.ict.ac.cn/journalx/editorInChiefLogOn.action, submissionAeEditorUrl=, option={"copyright":""})], databaseList=null, tenantJournalId=1324759914146529478, websiteList=[Website(id=1325070300139892771, webName=null, webTitle=null, webDomain=null, webCopyrigh=null, webIpcNo=null, seoTitle=null, seoKeywords=null, seoDescription=null, tenantJournalId=null, journalId=1324759914146529478, journalNameCn=null, journalNameEn=null, grayFlag=null, tenantId=1146029695717560320, platformId=null, journalGroupId=null, journalGroupNameCn=null, journalGroupNameEn=null, type=1, domain=https://castjournals.cast.org.cn/joweb/jsjyjyfz/CN, language=CN, createTime=1793850978169, createBy=18614031015, updateTime=1793851040259, updateBy=18614031015, name=计算机研究与发展-中文, tplId=1146099689490845704, title=计算机研究与发展, delFlag=0, indexPage=/home, props=[WebsiteProps(id=1325070893751349360, tenantId=1146029695717560320, journalId=null, journalGroupId=null, siteId=1325070300139892771, code=articleTextType, value=kx, createTime=1793851119697, updateTime=1793851119697, creator=18614031015, updator=18614031015), WebsiteProps(id=1325070893730377837, tenantId=1146029695717560320, journalId=null, journalGroupId=null, siteId=1325070300139892771, code=banner, value=null, createTime=1793851119692, updateTime=1793851119692, creator=18614031015, updator=18614031015), WebsiteProps(id=1325070893772320883, tenantId=1146029695717560320, journalId=null, journalGroupId=null, siteId=1325070300139892771, code=grayFlag, value=0, createTime=1793851119702, updateTime=1793851119702, creator=18614031015, updator=18614031015), WebsiteProps(id=1325070893721989228, tenantId=1146029695717560320, journalId=null, journalGroupId=null, siteId=1325070300139892771, code=logo, value=https://castjournals.cast.org.cn/joweb/jsjyjyfz/CN/file/pic?fileId=U27tDsy717YF7XSPA+Vwdw==, createTime=1793851119690, updateTime=1793851119690, creator=18614031015, updator=18614031015), WebsiteProps(id=1325070893784903797, tenantId=1146029695717560320, journalId=null, journalGroupId=null, siteId=1325070300139892771, code=minRunFlag, value=0, createTime=1793851119705, updateTime=1793851119705, creator=18614031015, updator=18614031015), WebsiteProps(id=1325070893742960751, tenantId=1146029695717560320, journalId=null, journalGroupId=null, siteId=1325070300139892771, code=picServerUrl, value=https://castjournals.cast.org.cn/joweb/jsjyjyfz/CN/file/pic, createTime=1793851119695, updateTime=1793851119695, creator=18614031015, updator=18614031015), WebsiteProps(id=1325070893780709492, tenantId=1146029695717560320, journalId=null, journalGroupId=null, siteId=1325070300139892771, code=silenceFlag, value=0, createTime=1793851119704, updateTime=1793851119704, creator=18614031015, updator=18614031015), WebsiteProps(id=1325070893738766446, tenantId=1146029695717560320, journalId=null, journalGroupId=null, siteId=1325070300139892771, code=staticResourcePath, value=https://castjournals.cast.org.cn/joweb/cast_kjdb_cn_619/, createTime=1793851119694, updateTime=1793851119694, creator=18614031015, updator=18614031015), WebsiteProps(id=1325070893755543665, tenantId=1146029695717560320, journalId=null, journalGroupId=null, siteId=1325070300139892771, code=themeColor, value=null, createTime=1793851119698, updateTime=1793851119698, creator=18614031015, updator=18614031015), WebsiteProps(id=1325070893763932274, tenantId=1146029695717560320, journalId=null, journalGroupId=null, siteId=1325070300139892771, code=themeStyle, value=null, createTime=1793851119700, updateTime=1793851119700, creator=18614031015, updator=18614031015)]), Website(id=1325070300580294698, webName=null, webTitle=null, webDomain=null, webCopyrigh=null, webIpcNo=null, seoTitle=null, seoKeywords=null, seoDescription=null, tenantJournalId=null, journalId=1324759914146529478, journalNameCn=null, journalNameEn=null, grayFlag=null, tenantId=1146029695717560320, platformId=null, journalGroupId=null, journalGroupNameCn=null, journalGroupNameEn=null, type=1, domain=https://castjournals.cast.org.cn/joweb/jsjyjyfz/EN, language=EN, createTime=1793850978274, createBy=18614031015, updateTime=1793851036638, updateBy=18614031015, name=计算机研究与发展-英文, tplId=1146101810881728533, title=Journal of Computer Research and Development, delFlag=0, indexPage=/home, props=[WebsiteProps(id=1325070925288321148, tenantId=1146029695717560320, journalId=null, journalGroupId=null, siteId=1325070300580294698, code=articleTextType, value=kx, createTime=1793851127216, updateTime=1793851127216, creator=18614031015, updator=18614031015), WebsiteProps(id=1325070925263155321, tenantId=1146029695717560320, journalId=null, journalGroupId=null, siteId=1325070300580294698, code=banner, value=null, createTime=1793851127210, updateTime=1793851127210, creator=18614031015, updator=18614031015), WebsiteProps(id=1325070925309292671, tenantId=1146029695717560320, journalId=null, journalGroupId=null, siteId=1325070300580294698, code=grayFlag, value=0, createTime=1793851127221, updateTime=1793851127221, creator=18614031015, updator=18614031015), WebsiteProps(id=1325070925254766712, tenantId=1146029695717560320, journalId=null, journalGroupId=null, siteId=1325070300580294698, code=logo, value=https://castjournals.cast.org.cn/joweb/jsjyjyfz/EN/file/pic?fileId=U27tDsy717YF7XSPA+Vwdw==, createTime=1793851127208, updateTime=1793851127208, creator=18614031015, updator=18614031015), WebsiteProps(id=1325070925326069889, tenantId=1146029695717560320, journalId=null, journalGroupId=null, siteId=1325070300580294698, code=minRunFlag, value=0, createTime=1793851127225, updateTime=1793851127225, creator=18614031015, updator=18614031015), WebsiteProps(id=1325070925279932539, tenantId=1146029695717560320, journalId=null, journalGroupId=null, siteId=1325070300580294698, code=picServerUrl, value=https://castjournals.cast.org.cn/joweb/jsjyjyfz/EN/file/pic, createTime=1793851127214, updateTime=1793851127214, creator=18614031015, updator=18614031015), WebsiteProps(id=1325070925317681280, tenantId=1146029695717560320, journalId=null, journalGroupId=null, siteId=1325070300580294698, code=silenceFlag, value=0, createTime=1793851127223, updateTime=1793851127223, creator=18614031015, updator=18614031015), WebsiteProps(id=1325070925271543930, tenantId=1146029695717560320, journalId=null, journalGroupId=null, siteId=1325070300580294698, code=staticResourcePath, value=https://castjournals.cast.org.cn/joweb/cast_kjdb_en_623/, createTime=1793851127212, updateTime=1793851127212, creator=18614031015, updator=18614031015), WebsiteProps(id=1325070925292515453, tenantId=1146029695717560320, journalId=null, journalGroupId=null, siteId=1325070300580294698, code=themeColor, value=null, createTime=1793851127217, updateTime=1793851127217, creator=18614031015, updator=18614031015), WebsiteProps(id=1325070925300904062, tenantId=1146029695717560320, journalId=null, journalGroupId=null, siteId=1325070300580294698, code=themeStyle, value=null, createTime=1793851127219, updateTime=1793851127219, creator=18614031015, updator=18614031015)])], journalTitle=计算机研究与发展, weixinUrl=null, journalUrl=https://crad.ict.ac.cn/, iacademicId=null, status=1, seqNo=null, journalTitleEn=Journal of Computer Research and Development, journalPhotoCn=k2l8zgqwstuxxevFX2qxVA==, journalPhotoEn=r6k0Ry+RD+c/VXj5QarmDw==, journalFirstLetter=J, journalRecommend=null, journalNew=null, journalCollection=null, jcrJf=null, cjcrJf=null, jcrJfStr=null, cjcrJfStr=null, submissionFirstDecision=null, sciSubjectClassification=null, casSubjectClassification=null, citeScore=null, totalCitationFrequency=null, icpCode=null, psCode=null, advertisingLicenseCode=null, copyrightInformation=null, country=null, option=, provinceCode=null, provinceName=null, collectFlag=false, interPubPlatform=, interPubPlatformUrl=null), detailUrlCn=https://castjournals.cast.org.cn/joweb/jsjyjyfz/CN/10.7544/issn1000-1239.202330303, detailUrlEn=https://castjournals.cast.org.cn/joweb/jsjyjyfz/EN/10.7544/issn1000-1239.202330303, pdfUrlCn=https://castjournals.cast.org.cn/joweb/jsjyjyfz/CN/PDF/10.7544/issn1000-1239.202330303, pdfUrlEn=https://castjournals.cast.org.cn/joweb/jsjyjyfz/EN/PDF/10.7544/issn1000-1239.202330303, aliStartDate=0, aliEndDate=0, collectionFlag=false, citedCount=null, citedUrl=null, previewStatus=0, delFlag=0, hasFullText=1, orderTime=1706803200000, fullTextJson=null, articleText=null, reference=null)