Article(id=1327011432685744600, tenantId=1146029695717560320, journalId=1324759914146529478, issueId=1327011123016085523, articleNumber=null, orderNo=null, doi=10.7544/issn1000-1239.202330844, pmid=null, cstr=null, oa=null, hot=null, price=null, onlineType=0, articleFormat=0, articleType=null, articleTypeStr=research-article, receivedDate=1698681600000, receivedDateStr=2023-10-31, revisedDate=1711036800000, revisedDateStr=2024-03-22, acceptedDate=1716998400000, acceptedDateStr=2024-05-30, onlineDate=1794313780231, onlineDateStr=2026-11-10, pubDate=1740758400000, pubDateStr=2025-03-01, doiRegisterDate=null, doiRegisterDateStr=null, onlineIssueDate=1794313780231, onlineIssueDateStr=2026-11-10, onlineJustAcceptDate=null, onlineJustAcceptDateStr=null, onlineFirstDate=null, onlineFirstDateStr=null, sourceXml=null, magXml=null, createTime=1794313780231, creator=13701087609, updateTime=1794313780231, updator=13701087609, issue=Issue{id=1327011123016085523, tenantId=1146029695717560320, journalId=1324759914146529478, year='2025', volume='62', issue='3', pageStart='543', pageEnd='819', issueExtLink='null', onlineDate='null', pubDate='1740758400000', pubDateStr='2025-03-01', beforeIssueId=null, nextIssueId=null, price=null, status=1, issueComplete=1, articleOrder=1, issueType=-1, specialIssue=null, createTime=1794313706401, creator='13701087609', updateTime=1794376726899, updator='13701087609', preIssue=null, nextIssue=null, articleTotal=null, ext={EN=IssueExt(id=1327275450650440506, tenantId=1146029695717560320, journalId=1324759914146529478, issueId=1327011123016085523, language=EN, specialIssueTitle=, coverIllustrator=null, specialIssueEditor=, specialIssueAbout=), CN=IssueExt(id=1327275450650440507, tenantId=1146029695717560320, journalId=1324759914146529478, issueId=1327011123016085523, language=CN, specialIssueTitle=, coverIllustrator=null, specialIssueEditor=, specialIssueAbout=)}, issueFiles=null, downloadFileDto=null}, startPage=682, endPage=693, ext={EN=ArticleExt(id=1327011432866099673, articleId=1327011432685744600, tenantId=1146029695717560320, journalId=1324759914146529478, language=EN, title=LingLong: A High-Quality Small-Scale Chinese Pre-trained Language Model, columnId=1325489891789910943, journalTitle=Journal of Computer Research and Development, columnName=Artificial Intelligence, runingTitle=null, highlight=null, articleAbstract=
In recent years, large-scale autoregressive Chinese pre-trained language models (PLMs) have demonstrated outstanding performance on various natural language processing (NLP) tasks. However, these models are computationally expensive, and their word-based vocabulary poses significant challenges for practical applications. In addition, most of them use only unidirectional context information, which may result in performance degradation on many tasks, especially tasks requiring a nuanced understanding of context. To address these challenges, we introduce LingLong, a high-quality small-scale Chinese pre-trained language model. LingLong stands out due to its modest scale, comprising only 317 million parameters, making it highly deployable and resource-efficient. We tokenize the training corpus with a character-based vocabulary to mitigate the negative impacts of unknown tokens and word segmentation errors. Moreover, we go beyond the conventional unidirectional context by introducing a novel backward model. This model is trained by reversing the input order of the training data. Combining LingLong and its backward version allows for the use of bidirectional information on downstream tasks. Extensive experimental results validate the effectiveness of LingLong across a diverse set of NLP tasks. LingLong outperforms similar-sized Chinese PLMs on six downstream tasks and surpasses popular large-scale Chinese PLMs on four downstream tasks. These findings underscore the versatility and efficiency of LingLong, opening up possibilities for practical applications and advancements in the Chinese NLP field.
, authors=Dongwen Li
1, Zhenyu Zhong
1, Yufei Sun
1, *, Junyu Shen
1, Zizhi Ma
1, Chuanyue Yu
1, Yuzhi Zhang
1, 2, authorsList=Dongwen Li, Zhenyu Zhong, Yufei Sun, Junyu Shen, Zizhi Ma, Chuanyue Yu, Yuzhi Zhang, authorCompany=null, correspAuthors=Yufei Sun, authorNote=
born in 1997. PhD candidate. Student member of CCF. Her main research interests include natural language processing and deep learning
born in 1997. PhD candidate. His main research interests include natural language processing, high-performance computing, and artificial intelligence operations
born in 1976. PhD, professor, master supervisor. Her main research interests include deep learning, heterogeneous computing, and artificial intelligence
born in 2001. Master candidate.His main research interest includes natural language processing
born in 2000. Master candidate. His main research interest includes natural language processing
born in 2001. Master candidate. Her main research interest includes natural language processing
born in 1964. PhD, professor, PhD supervisor. Member of CCF. His main research interests include deep learning and other artificial intelligence-related fields
, correspAuthorsNote=null, copyrightStatement=Copyright © 2025 Journal of Computer Research and Development. All rights reserved., copyrightOwner=null, extLink=null, articleAbsUrl=null, sourceXml=null, magXml=null, pdfUrl=null, pdf=null, pdfFileSize=null, pdfExtLink=null, richHtmlUrl=null, mobilePdfUrl=null, reviewReport=null, pdfFirstPage=null, abstractGraph=null, abstractGraphContent=null, abstractVideo=null, citation=null, cebUrl=null, magXmlContent=null, mapNumber=null, fund=null), CN=ArticleExt(id=1327011436020216298, articleId=1327011432685744600, tenantId=1146029695717560320, journalId=1324759914146529478, language=CN, title=玲珑:一个小规模的高质量中文预训练语言模型, columnId=1325489892104483745, journalTitle=计算机研究与发展, columnName=人工智能, runingTitle=null, highlight=null, articleAbstract=
近年来,大规模的、基于自回归的中文预训练语言模型在各种自然语言处理任务上表现出优异性能. 然而,高昂的计算成本以及基于中文词切分数据给中文预训练语言模型实际应用带来了巨大挑战. 此外,大多基于自回归的模型只能使用单向前文信息,可能会导致模型在上下文敏感任务上的性能有所下降. 为了解决以上问题,提出并训练了一个高质量的小型中文预训练语言模型——玲珑. 该模型仅有3.17亿个参数,较小的规模使得玲珑十分容易部署和应用. 使用基于汉字的策略对训练语料进行切分,可以有效减轻未知标记和分词错误带来的负面影响,增强了玲珑在下游任务上的性能. 此外,通过对每条训练数据的输入顺序进行逆序处理,训练了一个反向玲珑模型. 将玲珑与其反向版本相结合,可以实现在下游任务中使用双向信息. 多种自然语言处理下游任务的实验结果表明,玲珑具有不错的处理下游任务的能力. 在6个数据集上玲珑超越了相近规模模型的性能,在5个数据集上超越了大模型的性能.
, authors=李东闻
1, 钟震宇
1, 孙羽菲
1, *, 申峻宇
1, 马子智
1, 于川越
1, 张玉志
1, 2, authorsList=李东闻, 钟震宇, 孙羽菲, 申峻宇, 马子智, 于川越, 张玉志, authorCompany=null, correspAuthors=孙羽菲, authorNote=
 |
1997 年生. 博士研究生. CCF学生会员. 主要研究方向为自然语言处理、深度学习 |
 |
1997 年生. 博士研究生. 主要研究方向为自然语言处理、高性能计算、智能运维 |
 |
1976 年生. 博士,教授,硕士生导师. 主要研究方向为深度学习、异构计算、人工智能 |
 |
2001 年生. 硕士研究生. 主要研究方向为自然语言处理 |
 |
2000 年生. 硕士研究生. 主要研究方向为自然语言处理 |
 |
2001年生. 硕士研究生. 主要研究方向为自然语言处理 |
 |
1964 年生. 博士,教授,博士生导师. CCF 会员. 主要研究方向为深度学习、其他人工智能相关领域 |
, correspAuthorsNote=
, copyrightStatement=版权所有 © 2025 计算机研究与发展编辑部, copyrightOwner=null, extLink=null, articleAbsUrl=null, sourceXml=DVsLDee/Xx6fBeP0tKtC8A==, magXml=PwLRgzPnm7w57/OsD42ILw==, pdfUrl=null, pdf=yTyL3J24+FzJspheZ7Dv8g==, pdfFileSize=989860, pdfExtLink=null, richHtmlUrl=null, mobilePdfUrl=null, reviewReport=null, pdfFirstPage=null, abstractGraph=R99ZZU8dD3ioRspJuYe+Sg==, abstractGraphContent=null, abstractVideo=null, citation=null, cebUrl=null, magXmlContent=yNyuGPBhd06dM4kIxgWfJw==, mapNumber=null, fund=null)}, authors=[Author(id=1327011436540310002, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327011432685744600, orderNo=0, firstName=null, middleName=null, lastName=null, nameCn=null, orcid=null, stid=null, country=null, authorPic=null, dead=0, email=lidongwen@mail.nankai.edu.cn, emailSecond=null, emailThird=null, correspondingAuthor=0, authorType=1, ext={EN=AuthorExt(id=1327011436624196084, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327011432685744600, authorId=1327011436540310002, language=EN, stringName=Dongwen Li, firstName=Dongwen, middleName=null, lastName=Li, prefix=null, suffix=null, authorComment=null, nameInitials=null, affiliation=null, department=null, xref=
1, address=
1College of Software, Nankai University, Tianjin 300450, bio={"content":"
born in 1997. PhD candidate. Student member of CCF. Her main research interests include natural language processing and deep learning
"}, bioImg=null, bioContent=
born in 1997. PhD candidate. Student member of CCF. Her main research interests include natural language processing and deep learning
, aboutCorrespAuthor=null), CN=AuthorExt(id=1327011436695499253, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327011432685744600, authorId=1327011436540310002, language=CN, stringName=李东闻, firstName=null, middleName=null, lastName=null, prefix=null, suffix=null, authorComment=null, nameInitials=null, affiliation=null, department=null, xref=
1, address=
1南开大学软件学院 天津 300450, bio={"img":"YXKR2raJahd1V35RXLdOaw==","content":"
1997 年生. 博士研究生. CCF学生会员. 主要研究方向为自然语言处理、深度学习
"}, bioImg=YXKR2raJahd1V35RXLdOaw==, bioContent=
1997 年生. 博士研究生. CCF学生会员. 主要研究方向为自然语言处理、深度学习
, aboutCorrespAuthor=null)}, companyList=[AuthorCompany(id=1327011436326400491, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327011432685744600, xref=1, ext=[AuthorCompanyExt(id=1327011436338983404, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327011432685744600, companyId=1327011436326400491, language=EN, country=null, province=null, city=null, postcode=null, companyName=null, departmentName=null, remark=
1College of Software, Nankai University, Tianjin 300450), AuthorCompanyExt(id=1327011436351566317, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327011432685744600, companyId=1327011436326400491, language=CN, country=null, province=null, city=null, postcode=null, companyName=null, departmentName=null, remark=
1南开大学软件学院 天津 300450)])]), Author(id=1327011436770996727, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327011432685744600, orderNo=1, firstName=null, middleName=null, lastName=null, nameCn=null, orcid=null, stid=null, country=null, authorPic=null, dead=0, email=null, emailSecond=null, emailThird=null, correspondingAuthor=0, authorType=1, ext={EN=AuthorExt(id=1327011436838105593, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327011432685744600, authorId=1327011436770996727, language=EN, stringName=Zhenyu Zhong, firstName=Zhenyu, middleName=null, lastName=Zhong, prefix=null, suffix=null, authorComment=null, nameInitials=null, affiliation=null, department=null, xref=
1, address=
1College of Software, Nankai University, Tianjin 300450, bio={"content":"
born in 1997. PhD candidate. His main research interests include natural language processing, high-performance computing, and artificial intelligence operations
"}, bioImg=null, bioContent=
born in 1997. PhD candidate. His main research interests include natural language processing, high-performance computing, and artificial intelligence operations
, aboutCorrespAuthor=null), CN=AuthorExt(id=1327011436959740410, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327011432685744600, authorId=1327011436770996727, language=CN, stringName=钟震宇, firstName=null, middleName=null, lastName=null, prefix=null, suffix=null, authorComment=null, nameInitials=null, affiliation=null, department=null, xref=
1, address=
1南开大学软件学院 天津 300450, bio={"img":"+2CC2FS70fHwPJeu6oOO+g==","content":"
1997 年生. 博士研究生. 主要研究方向为自然语言处理、高性能计算、智能运维
"}, bioImg=+2CC2FS70fHwPJeu6oOO+g==, bioContent=
1997 年生. 博士研究生. 主要研究方向为自然语言处理、高性能计算、智能运维
, aboutCorrespAuthor=null)}, companyList=[AuthorCompany(id=1327011436326400491, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327011432685744600, xref=1, ext=[AuthorCompanyExt(id=1327011436338983404, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327011432685744600, companyId=1327011436326400491, language=EN, country=null, province=null, city=null, postcode=null, companyName=null, departmentName=null, remark=
1College of Software, Nankai University, Tianjin 300450), AuthorCompanyExt(id=1327011436351566317, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327011432685744600, companyId=1327011436326400491, language=CN, country=null, province=null, city=null, postcode=null, companyName=null, departmentName=null, remark=
1南开大学软件学院 天津 300450)])]), Author(id=1327011437043626492, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327011432685744600, orderNo=2, firstName=null, middleName=null, lastName=null, nameCn=null, orcid=null, stid=null, country=null, authorPic=null, dead=0, email=yufei_sun@sina.com, emailSecond=null, emailThird=null, correspondingAuthor=1, authorType=1, ext={EN=AuthorExt(id=1327011437135901182, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327011432685744600, authorId=1327011437043626492, language=EN, stringName=Yufei Sun, firstName=Yufei, middleName=null, lastName=Sun, prefix=null, suffix=null, authorComment=null, nameInitials=null, affiliation=null, department=null, xref=
1, *, address=
1College of Software, Nankai University, Tianjin 300450, bio={"content":"
born in 1976. PhD, professor, master supervisor. Her main research interests include deep learning, heterogeneous computing, and artificial intelligence
"}, bioImg=null, bioContent=
born in 1976. PhD, professor, master supervisor. Her main research interests include deep learning, heterogeneous computing, and artificial intelligence
, aboutCorrespAuthor=null), CN=AuthorExt(id=1327011437253341695, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327011432685744600, authorId=1327011437043626492, language=CN, stringName=孙羽菲, firstName=null, middleName=null, lastName=null, prefix=null, suffix=null, authorComment=null, nameInitials=null, affiliation=null, department=null, xref=
1, *, address=
1南开大学软件学院 天津 300450, bio={"img":"V0IDtKYQwDE3l12Pa2G4MQ==","content":"
1976 年生. 博士,教授,硕士生导师. 主要研究方向为深度学习、异构计算、人工智能
"}, bioImg=V0IDtKYQwDE3l12Pa2G4MQ==, bioContent=
1976 年生. 博士,教授,硕士生导师. 主要研究方向为深度学习、异构计算、人工智能
, aboutCorrespAuthor=null)}, companyList=[AuthorCompany(id=1327011436326400491, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327011432685744600, xref=1, ext=[AuthorCompanyExt(id=1327011436338983404, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327011432685744600, companyId=1327011436326400491, language=EN, country=null, province=null, city=null, postcode=null, companyName=null, departmentName=null, remark=
1College of Software, Nankai University, Tianjin 300450), AuthorCompanyExt(id=1327011436351566317, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327011432685744600, companyId=1327011436326400491, language=CN, country=null, province=null, city=null, postcode=null, companyName=null, departmentName=null, remark=
1南开大学软件学院 天津 300450)])]), Author(id=1327011437345616385, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327011432685744600, orderNo=3, firstName=null, middleName=null, lastName=null, nameCn=null, orcid=null, stid=null, country=null, authorPic=null, dead=0, email=null, emailSecond=null, emailThird=null, correspondingAuthor=0, authorType=1, ext={EN=AuthorExt(id=1327011437429502467, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327011432685744600, authorId=1327011437345616385, language=EN, stringName=Junyu Shen, firstName=Junyu, middleName=null, lastName=Shen, prefix=null, suffix=null, authorComment=null, nameInitials=null, affiliation=null, department=null, xref=
1, address=
1College of Software, Nankai University, Tianjin 300450, bio={"content":"
born in 2001. Master candidate.His main research interest includes natural language processing
"}, bioImg=null, bioContent=
born in 2001. Master candidate.His main research interest includes natural language processing
, aboutCorrespAuthor=null), CN=AuthorExt(id=1327011437496611332, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327011432685744600, authorId=1327011437345616385, language=CN, stringName=申峻宇, firstName=null, middleName=null, lastName=null, prefix=null, suffix=null, authorComment=null, nameInitials=null, affiliation=null, department=null, xref=
1, address=
1南开大学软件学院 天津 300450, bio={"img":"Rmy7qXDLm9WgChJNW3fOew==","content":"
2001 年生. 硕士研究生. 主要研究方向为自然语言处理
"}, bioImg=Rmy7qXDLm9WgChJNW3fOew==, bioContent=
2001 年生. 硕士研究生. 主要研究方向为自然语言处理
, aboutCorrespAuthor=null)}, companyList=[AuthorCompany(id=1327011436326400491, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327011432685744600, xref=1, ext=[AuthorCompanyExt(id=1327011436338983404, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327011432685744600, companyId=1327011436326400491, language=EN, country=null, province=null, city=null, postcode=null, companyName=null, departmentName=null, remark=
1College of Software, Nankai University, Tianjin 300450), AuthorCompanyExt(id=1327011436351566317, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327011432685744600, companyId=1327011436326400491, language=CN, country=null, province=null, city=null, postcode=null, companyName=null, departmentName=null, remark=
1南开大学软件学院 天津 300450)])]), Author(id=1327011437576303110, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327011432685744600, orderNo=4, firstName=null, middleName=null, lastName=null, nameCn=null, orcid=null, stid=null, country=null, authorPic=null, dead=0, email=null, emailSecond=null, emailThird=null, correspondingAuthor=0, authorType=1, ext={EN=AuthorExt(id=1327011437655994888, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327011432685744600, authorId=1327011437576303110, language=EN, stringName=Zizhi Ma, firstName=Zizhi, middleName=null, lastName=Ma, prefix=null, suffix=null, authorComment=null, nameInitials=null, affiliation=null, department=null, xref=
1, address=
1College of Software, Nankai University, Tianjin 300450, bio={"content":"
born in 2000. Master candidate. His main research interest includes natural language processing
"}, bioImg=null, bioContent=
born in 2000. Master candidate. His main research interest includes natural language processing
, aboutCorrespAuthor=null), CN=AuthorExt(id=1327011437714715145, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327011432685744600, authorId=1327011437576303110, language=CN, stringName=马子智, firstName=null, middleName=null, lastName=null, prefix=null, suffix=null, authorComment=null, nameInitials=null, affiliation=null, department=null, xref=
1, address=
1南开大学软件学院 天津 300450, bio={"img":"ZsNNLnYtX/fRZ6TIEhE4Qw==","content":"
2000 年生. 硕士研究生. 主要研究方向为自然语言处理
"}, bioImg=ZsNNLnYtX/fRZ6TIEhE4Qw==, bioContent=
2000 年生. 硕士研究生. 主要研究方向为自然语言处理
, aboutCorrespAuthor=null)}, companyList=[AuthorCompany(id=1327011436326400491, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327011432685744600, xref=1, ext=[AuthorCompanyExt(id=1327011436338983404, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327011432685744600, companyId=1327011436326400491, language=EN, country=null, province=null, city=null, postcode=null, companyName=null, departmentName=null, remark=
1College of Software, Nankai University, Tianjin 300450), AuthorCompanyExt(id=1327011436351566317, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327011432685744600, companyId=1327011436326400491, language=CN, country=null, province=null, city=null, postcode=null, companyName=null, departmentName=null, remark=
1南开大学软件学院 天津 300450)])]), Author(id=1327011437773435403, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327011432685744600, orderNo=5, firstName=null, middleName=null, lastName=null, nameCn=null, orcid=null, stid=null, country=null, authorPic=null, dead=0, email=null, emailSecond=null, emailThird=null, correspondingAuthor=0, authorType=1, ext={EN=AuthorExt(id=1327011437844738573, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327011432685744600, authorId=1327011437773435403, language=EN, stringName=Chuanyue Yu, firstName=Chuanyue, middleName=null, lastName=Yu, prefix=null, suffix=null, authorComment=null, nameInitials=null, affiliation=null, department=null, xref=
1, address=
1College of Software, Nankai University, Tianjin 300450, bio={"content":"
born in 2001. Master candidate. Her main research interest includes natural language processing
"}, bioImg=null, bioContent=
born in 2001. Master candidate. Her main research interest includes natural language processing
, aboutCorrespAuthor=null), CN=AuthorExt(id=1327011439501488654, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327011432685744600, authorId=1327011437773435403, language=CN, stringName=于川越, firstName=null, middleName=null, lastName=null, prefix=null, suffix=null, authorComment=null, nameInitials=null, affiliation=null, department=null, xref=
1, address=
1南开大学软件学院 天津 300450, bio={"img":"QQZHhF2Nnu/0+BlPn5QDzw==","content":"
2001年生. 硕士研究生. 主要研究方向为自然语言处理
"}, bioImg=QQZHhF2Nnu/0+BlPn5QDzw==, bioContent=
2001年生. 硕士研究生. 主要研究方向为自然语言处理
, aboutCorrespAuthor=null)}, companyList=[AuthorCompany(id=1327011436326400491, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327011432685744600, xref=1, ext=[AuthorCompanyExt(id=1327011436338983404, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327011432685744600, companyId=1327011436326400491, language=EN, country=null, province=null, city=null, postcode=null, companyName=null, departmentName=null, remark=
1College of Software, Nankai University, Tianjin 300450), AuthorCompanyExt(id=1327011436351566317, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327011432685744600, companyId=1327011436326400491, language=CN, country=null, province=null, city=null, postcode=null, companyName=null, departmentName=null, remark=
1南开大学软件学院 天津 300450)])]), Author(id=1327011439581180433, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327011432685744600, orderNo=6, firstName=null, middleName=null, lastName=null, nameCn=null, orcid=null, stid=null, country=null, authorPic=null, dead=0, email=null, emailSecond=null, emailThird=null, correspondingAuthor=0, authorType=1, ext={EN=AuthorExt(id=1327011439677649428, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327011432685744600, authorId=1327011439581180433, language=EN, stringName=Yuzhi Zhang, firstName=Yuzhi, middleName=null, lastName=Zhang, prefix=null, suffix=null, authorComment=null, nameInitials=null, affiliation=null, department=null, xref=
1, 2, address=
1College of Software, Nankai University, Tianjin 300450
2Haihe Laboratory of Information Technology Application Innovation, Tianjin 300450, bio={"content":"
born in 1964. PhD, professor, PhD supervisor. Member of CCF. His main research interests include deep learning and other artificial intelligence-related fields
"}, bioImg=null, bioContent=
born in 1964. PhD, professor, PhD supervisor. Member of CCF. His main research interests include deep learning and other artificial intelligence-related fields
, aboutCorrespAuthor=null), CN=AuthorExt(id=1327011439753146901, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327011432685744600, authorId=1327011439581180433, language=CN, stringName=张玉志, firstName=null, middleName=null, lastName=null, prefix=null, suffix=null, authorComment=null, nameInitials=null, affiliation=null, department=null, xref=
1, 2, address=
1南开大学软件学院 天津 300450
2先进计算与关键软件(信创)海河实验室 天津 300450, bio={"img":"5v1TEgadmLFdnXng+4HHRg==","content":"
1964 年生. 博士,教授,博士生导师. CCF 会员. 主要研究方向为深度学习、其他人工智能相关领域
"}, bioImg=5v1TEgadmLFdnXng+4HHRg==, bioContent=
1964 年生. 博士,教授,博士生导师. CCF 会员. 主要研究方向为深度学习、其他人工智能相关领域
, aboutCorrespAuthor=null)}, companyList=[AuthorCompany(id=1327011436326400491, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327011432685744600, xref=1, ext=[AuthorCompanyExt(id=1327011436338983404, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327011432685744600, companyId=1327011436326400491, language=EN, country=null, province=null, city=null, postcode=null, companyName=null, departmentName=null, remark=
1College of Software, Nankai University, Tianjin 300450), AuthorCompanyExt(id=1327011436351566317, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327011432685744600, companyId=1327011436326400491, language=CN, country=null, province=null, city=null, postcode=null, companyName=null, departmentName=null, remark=
1南开大学软件学院 天津 300450)]), AuthorCompany(id=1327011436418675182, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327011432685744600, xref=2, ext=[AuthorCompanyExt(id=1327011436431258095, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327011432685744600, companyId=1327011436418675182, language=EN, country=null, province=null, city=null, postcode=null, companyName=null, departmentName=null, remark=
2Haihe Laboratory of Information Technology Application Innovation, Tianjin 300450), AuthorCompanyExt(id=1327011436456423920, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327011432685744600, companyId=1327011436418675182, language=CN, country=null, province=null, city=null, postcode=null, companyName=null, departmentName=null, remark=
2先进计算与关键软件(信创)海河实验室 天津 300450)])])], keywords=[Keyword(id=1327011439912530454, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327011432685744600, language=EN, orderNo=1, keyword=Chinese pre-trained language model), Keyword(id=1327011439979639319, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327011432685744600, language=EN, orderNo=2, keyword=small-scale), Keyword(id=1327011440038359576, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327011432685744600, language=EN, orderNo=3, keyword=character-based model), Keyword(id=1327011440235491865, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327011432685744600, language=EN, orderNo=4, keyword=backward model), Keyword(id=1327011440298406426, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327011432685744600, language=EN, orderNo=5, keyword=bidirectional information), Keyword(id=1327011440361320987, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327011432685744600, language=CN, orderNo=1, keyword=中文预训练语言模型), Keyword(id=1327011440428429852, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327011432685744600, language=CN, orderNo=2, keyword=小规模), Keyword(id=1327011440499733021, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327011432685744600, language=CN, orderNo=3, keyword=基于汉字的模型), Keyword(id=1327011440562647582, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327011432685744600, language=CN, orderNo=4, keyword=反向模型), Keyword(id=1327011440629756447, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327011432685744600, language=CN, orderNo=5, keyword=双向信息)], refs=[Reference(id=1327011444316549684, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327011432685744600, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=1, rfOrder=0, authorNames=null, journalName=null, refType=null, unstructuredReference=Radford A, Narasimhan K, Salimans T, et al. Improving language understanding by generative pre-training[EB/OL]. 2018[2023-09-19]. https://cdn.openai.com/research-covers/language-unsupervised/language_understanding_paper.pdf, articleTitle=null, refAbstract=null), Reference(id=1327011444425601589, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327011432685744600, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=2, rfOrder=1, authorNames=null, journalName=null, refType=null, unstructuredReference=Radford A, Wu J, Child R, et al. Language models are unsupervised multitask learners[J/OL]. OpenAI Blog, 2019, 1(8): 9. [2023-09-20]. https://cdn.openai.com/better-language-models/language_models_are_unsupervised_multitask_learners.pdf, articleTitle=null, refAbstract=null), Reference(id=1327011444547236406, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327011432685744600, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=3, rfOrder=2, authorNames=null, journalName=null, refType=null, unstructuredReference=Devlin J, Chang M W, Lee K, et al. BERT: Pre-training of deep bidirectional transformers for language understanding[C]//Proc of the 2019 Conf of the North American Chapter of the Association for Computational Linguistics: Human Language Technologies. Stroudsburg, PA: ACL, 2019: 4171−4186, articleTitle=null, refAbstract=null), Reference(id=1327011444639511095, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327011432685744600, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=4, rfOrder=3, authorNames=null, journalName=null, refType=null, unstructuredReference=Yang Zhlin, Dai Zihang, Yang Yiming. XLNet: Generalized autoregressive pretraining for language understanding[C]//Proc of the Advances in Neural Information Processing Systems. Cambridge, MA: MIT, 2019: 5754−5764, articleTitle=null, refAbstract=null), Reference(id=1327011444723397176, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327011432685744600, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=5, rfOrder=4, authorNames=null, journalName=null, refType=null, unstructuredReference=Lewis M, Liu Yinhan, Goyal N, et al. BART: Denoising sequence-to-sequence pre-training for natural language generation, translation, and comprehension[J]. arXiv preprint, arXiv: 1910.13461, 2019, articleTitle=null, refAbstract=null), Reference(id=1327011444828254777, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327011432685744600, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=6, rfOrder=5, authorNames=null, journalName=null, refType=null, unstructuredReference=Brown T, Mann B, Ryder N, et al. Language models are few-shot learners[C]//Proc of the Advances in Neural Information Processing Systems. Cambridge, MA: MIT, 2020: 1877−1901, articleTitle=null, refAbstract=null), Reference(id=1327011444949889594, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327011432685744600, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=7, rfOrder=6, authorNames=null, journalName=null, refType=null, unstructuredReference=舒文韬,李睿潇,孙天祥,等. 大型语言模型:原理、实现与发展[J]. 计算机研究与发展,2024,61(2):351−361, articleTitle=null, refAbstract=null), Reference(id=1327011445079913019, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327011432685744600, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=7, rfOrder=7, authorNames=null, journalName=null, refType=null, unstructuredReference=Shu Wentao, Li Ruixiao, Sun Tianxiang, et al. Large language models: Principles, implementation, and progress[J]. Journal of Computer Research and Development, 2024, 61(2): 351−361 (in Chinese), articleTitle=null, refAbstract=null), Reference(id=1327011445147021884, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327011432685744600, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=8, rfOrder=8, authorNames=null, journalName=null, refType=null, unstructuredReference=Diao Shizhe, Bai Jiaxin, Song Yan, et al. ZEN: Pre-training Chinese text encoder enhanced by n-gram representations[J]. arXiv preprint, arXiv: 1911.00720, 2019, articleTitle=null, refAbstract=null), Reference(id=1327011445226713661, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327011432685744600, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=9, rfOrder=9, authorNames=null, journalName=null, refType=null, unstructuredReference=Liu Yihan, Ott M, Goyal N, et al. RoBERTa: A robustly optimized BERT pretraining approach[J]. arXiv preprint, arXiv: 1907.11692, 2019, articleTitle=null, refAbstract=null), Reference(id=1327011445318988350, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327011432685744600, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=10, rfOrder=10, authorNames=null, journalName=null, refType=null, unstructuredReference=Sun Yu, Wang Shuohuan, Li Yukun, et al. ERNIE: Enhanced representation through knowledge integration[J]. arXiv preprint, arXiv: 1904.092233, 2019, articleTitle=null, refAbstract=null), Reference(id=1327011445373514303, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327011432685744600, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=11, rfOrder=11, authorNames=null, journalName=null, refType=null, unstructuredReference=Zhang Zhengyan, Han Xu, Zhou Hao, et al. CPM: A large-scale generative Chinese pre-trained language model[J]. AI Open, 2021, 2: 93−99, articleTitle=null, refAbstract=null), Reference(id=1327011445432234560, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327011432685744600, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=12, rfOrder=12, authorNames=null, journalName=null, refType=null, unstructuredReference=Zeng Wei, Ren Xiaozhe, Su Teng, et al. PanGu-α: Large-scale autoregressive pretrained Chinese language models with auto-parallel computation[J]. arXiv preprint, arXiv: 2104.12369, 2021, articleTitle=null, refAbstract=null), Reference(id=1327011445507732033, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327011432685744600, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=13, rfOrder=13, authorNames=null, journalName=null, refType=null, unstructuredReference=Du Zhengxiao, Qian Yujie, Liu Xiao, et al. GLM: General language model pretraining with autoregressive blank infilling[C]//Proc of the 60th Annual Meeting of the Association for Computational Linguistics. Stroudsburg, PA: ACL, 2022: 320–335, articleTitle=null, refAbstract=null), Reference(id=1327011445608395330, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327011432685744600, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=14, rfOrder=14, authorNames=null, journalName=null, refType=null, unstructuredReference=Zhang Zhengyan, Gu Yuxian, Han Xu, et al. CPM-2: Large-scale cost-effective pre-trained language models[J]. AI Open, 2021, 2: 216−224, articleTitle=null, refAbstract=null), Reference(id=1327011445688087107, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327011432685744600, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=15, rfOrder=15, authorNames=null, journalName=null, refType=null, unstructuredReference=Li Xiaoya, Meng Yuxian, Sun Xiaofei, et al. Is word segmentation necessary for deep learning of Chinese representations[C]//Proc of the 57th Annual Meeting of the Association for Computational Linguistics. Stroudsburg, PA: ACL, 2019: 3242–3252, articleTitle=null, refAbstract=null), Reference(id=1327011445755195972, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327011432685744600, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=16, rfOrder=16, authorNames=null, journalName=null, refType=null, unstructuredReference=Song Kaitao, Xu Tan, Tao Qin, et al. MASS: Masked sequence to sequence pre-training for language generation[C]//Proc of the 36th Int Conf on Machine Learning. New York: ACM, 2019: 9−15, articleTitle=null, refAbstract=null), Reference(id=1327011445822304837, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327011432685744600, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=17, rfOrder=17, authorNames=null, journalName=null, refType=null, unstructuredReference=Bi Bin, Li Chenliang, Wu Chen, et al. PALM: Pre-training an autoencoding & autoregressive language model for context-conditioned generation[C]//Proc of the 2020 Conf on Empirical Methods in Natural Language Processing. Stroudsburg, PA: ACL, 2020: 8681−8691, articleTitle=null, refAbstract=null), Reference(id=1327011445906190918, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327011432685744600, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=18, rfOrder=18, authorNames=null, journalName=null, refType=null, unstructuredReference=Cui Yiming, Che Wanxiang, Liu Ting, et al. Revisiting pre-trained models for Chinese natural language processing[C]//Findings of the Association for Computational Linguistics: EMNLP. Stroudsburg, PA: ACL, 2020: 657–668, articleTitle=null, refAbstract=null), Reference(id=1327011446065574471, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327011432685744600, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=19, rfOrder=19, authorNames=null, journalName=null, refType=null, unstructuredReference=Wei Junqiu, Ren Xiaozhe, Li Xiaoguang, et al. NEZHA: Neural contextualized representation for Chinese language understanding[J]. arXiv preprint, arXiv: 1909.00204, 2019, articleTitle=null, refAbstract=null), Reference(id=1327011446145266248, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327011432685744600, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=20, rfOrder=20, authorNames=null, journalName=null, refType=null, unstructuredReference=Bai Jinze, Bai Shuai, Chu Yunfei. Qwen technical report[J]. arXiv preprint, arXiv: 2309.16609, 2023, articleTitle=null, refAbstract=null), Reference(id=1327011446258512457, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327011432685744600, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=21, rfOrder=21, authorNames=null, journalName=null, refType=null, unstructuredReference=Xu Liang, Hu Hai, Zhang Xuanwei, et al. CLUE: A Chinese language understanding evaluation benchmark[C]//Proc of the 28th Int Conf on Computational Linguistics. Stroudsburg, PA: ACL, 2020: 4762−4772, articleTitle=null, refAbstract=null), Reference(id=1327011446384341578, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327011432685744600, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=22, rfOrder=22, authorNames=null, journalName=null, refType=null, unstructuredReference=Wang Wei, Bi Bin, Yan Ming, et al. StructBERT: Incorporating language structures into pre-training for deep language understanding[C/OL]//Proc of the 8th Int Conf on Learning Representations. Washington DC: ICLR, 2020[2024-09-19]. https://openreview.net/forum?id=BJgQ4lSFPH, articleTitle=null, refAbstract=null), Reference(id=1327011446443061835, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327011432685744600, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=23, rfOrder=23, authorNames=null, journalName=null, refType=null, unstructuredReference=Lai Yuxuan, Liu Yijia, Feng Yansong, et al. Lattice-BERT: Leveraging multi-granularity representations in Chinese pre-trained language models[C]//Proc of the 2021 Conf of the North American Chapter of the Association for Computational Linguistics: Human Language Technologies. Stroudsburg, PA: ACL, 2021: 1716–1731, articleTitle=null, refAbstract=null), Reference(id=1327011446518559308, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327011432685744600, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=24, rfOrder=24, authorNames=null, journalName=null, refType=null, unstructuredReference=Xu Liang, Zhang Xuanwei, Dong Qianqian. CLUECorpus 2020: A large-scale Chinese corpus for pre-training language model[J]. arXiv preprint, arXiv: 2003.01355, 2020, articleTitle=null, refAbstract=null), Reference(id=1327011446581473869, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327011432685744600, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=25, rfOrder=25, authorNames=null, journalName=null, refType=null, unstructuredReference=李东闻,钟震宇,申峻宇,等. NKCorpus:利用海量网络数据构建大型高质量中文数据集[J]. 数据与计算发展前沿,2022,4(3):30−45, articleTitle=null, refAbstract=null), Reference(id=1327011446648582734, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327011432685744600, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=25, rfOrder=26, authorNames=null, journalName=null, refType=null, unstructuredReference=Li Dongwen, Zhong Zhenyu, Shen Junyu, et al. NKCorpus: Extracting high quality large Chinese dataset from web data[J]. Frontiers of Data and Computing, 2022, 4(3): 30−45 (in Chinese), articleTitle=null, refAbstract=null), Reference(id=1327011446711497295, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327011432685744600, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=26, rfOrder=27, authorNames=null, journalName=null, refType=null, unstructuredReference=中华人民共和国教育部,国家语言文字工作委员会. GF 0023−2020 通用规范汉字表[S]. 北京:人民出版社,2021, articleTitle=null, refAbstract=null), Reference(id=1327011446795383376, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327011432685744600, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=26, rfOrder=28, authorNames=null, journalName=null, refType=null, unstructuredReference=State Language Commission, Ministry of Education of the People's Republic of China. GF 0023-2020 List of Commonly Used Standard Chinese Characters[S]. Beijing: People's Publishing House, 2021 (in Chinese), articleTitle=null, refAbstract=null), Reference(id=1327011448506659409, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327011432685744600, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=27, rfOrder=29, authorNames=null, journalName=null, refType=null, unstructuredReference=Wu Shaohua, Zhao Xudong, Yu Tong, et al. Yuan 1.0: Large-scale pre-trained language model in zero-shot and few-shot learning[J]. arXiv preprint, arXiv: 2110.04725, 2021, articleTitle=null, refAbstract=null), Reference(id=1327011448619905618, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327011432685744600, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=28, rfOrder=30, authorNames=null, journalName=null, refType=null, unstructuredReference=南开大学软件学院智能计算实验室. LingLong-mcpt[CP/OL]. 2023[2023-09-19]. https://github.com/NKCSICLab/linglong-mcpt, articleTitle=null, refAbstract=null), Reference(id=1327011448737346131, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327011432685744600, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=28, rfOrder=31, authorNames=null, journalName=null, refType=null, unstructuredReference=Intelligent Computing Laboratory, College of Software, Nankai University. LingLong-mcpt[CP/OL]. 2023[2023-09-19]. https://github.com/NKCSlCLab/linglong-mcpt (in Chinese), articleTitle=null, refAbstract=null), Reference(id=1327011448905118292, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327011432685744600, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=29, rfOrder=32, authorNames=null, journalName=null, refType=null, unstructuredReference=Yao Yuan, Dong Qingxiu, GuaN Jian, et al. CUGE: A Chinese language understanding and generation evaluation benchmark[J]. arXiv preprint, arXiv: 2112.13610, 2021, articleTitle=null, refAbstract=null), Reference(id=1327011449043530325, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327011432685744600, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=30, rfOrder=33, authorNames=null, journalName=null, refType=null, unstructuredReference=Li Haoran, Yuan Peng, Xu Song, et al. Aspect-aware multimodal summarization for Chinese e-commerce products[C]//Proc of the AAAI Conf on Artificial Intelligence. Palo Alto, CA: AAAI, 2020: 8188−8195, articleTitle=null, refAbstract=null), Reference(id=1327011449114833494, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327011432685744600, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=31, rfOrder=34, authorNames=null, journalName=null, refType=null, unstructuredReference=Hu Baotian, Chen Qingcai, Zhu Fangze. LCSTS: A large scale Chinese short text summarization dataset[C]//Proc of the 2015 Conf on Empirical Methods in Natural Language Processing. Stroudsburg, PA: ACL, 2015: 1967–1972, articleTitle=null, refAbstract=null), Reference(id=1327011449362297431, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327011432685744600, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=32, rfOrder=35, authorNames=null, journalName=null, refType=null, unstructuredReference=Lin C Y. ROUGE: A package for automatic evaluation of summaries[C/OL]//Proc of the Text Summarization Branches Out. 2004: 74−81. [2024-09-19]. https://aclanthology.org/W04-1013.pdf, articleTitle=null, refAbstract=null), Reference(id=1327011449446183512, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327011432685744600, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=33, rfOrder=36, authorNames=null, journalName=null, refType=null, unstructuredReference=Shao Zhihong, Huang Minlie, Wen Jiangtao, et al. Long and diverse text generation with planning-based hierarchical variational model[C]//Proc of the 2019 Conf on Empirical Methods in Natural Language Processing and the 9th Int Joint Conf on Natural Language Processing (EMNLP-JCNLP). Stroudsburg, PA: ACL, 2019: 3257–3268, articleTitle=null, refAbstract=null), Reference(id=1327011449563624025, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327011432685744600, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=34, rfOrder=37, authorNames=null, journalName=null, refType=null, unstructuredReference=Papineni K, Roukos S, Ward T, et al. BLEU: A method for automatic evaluation of machine translation[C]//Proc of the 40th Annual Meeting of the Association for Computational Linguistics. Stroudsburg, PA: ACL, 2002: 311−318, articleTitle=null, refAbstract=null), Reference(id=1327011449748173402, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327011432685744600, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=35, rfOrder=38, authorNames=null, journalName=null, refType=null, unstructuredReference=Li Juntao, Yan Rui. Overview of the NLPCC 2018 shared task: Multi-turn human-computer conversations[C]//Proc of the 7th Natural Language Processing and Chinese Computing. Berlin: Springer, 2018: 446−451, articleTitle=null, refAbstract=null), Reference(id=1327011449811087963, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327011432685744600, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=36, rfOrder=39, authorNames=null, journalName=null, refType=null, unstructuredReference=Qiu Xipeng, Qian Peng, Shi Zhan. Overview of the NLPCC-ICCPOL 2016 shared task: Chinese word segmentation for micro-blog texts[C]//Proc of the Natural Language Understanding and Intelligent Applications: 5th CCF Conf on Natural Language Processing and Chinese Computing (NLPCC 2016), and 24th Int Conf on Computer Processing of Oriental Languages. Berlin: Springer, 2016: 901−906, articleTitle=null, refAbstract=null), Reference(id=1327011449865613916, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327011432685744600, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=37, rfOrder=40, authorNames=null, journalName=null, refType=null, unstructuredReference=Emerson T. The second international Chinese word segmentation bakeoff[C/OL]//Proc of the 4th SIGHAN Workshop on Chinese Language Processing. 2005[2024-09-19]. https://aclanthology.org/I05-3017/, articleTitle=null, refAbstract=null), Reference(id=1327011449945305693, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327011432685744600, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=38, rfOrder=41, authorNames=null, journalName=null, refType=null, unstructuredReference=Liu Xin, Chen Qingcai, Deng Chong, et al. LCQMC: A large-scale Chinese question matching corpus[C]//Proc of the 27th Int Conf on Computational Linguistics. Stroudsburg, PA: ACL, 2018: 1952−1962, articleTitle=null, refAbstract=null), Reference(id=1327011450054357598, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327011432685744600, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=39, rfOrder=42, authorNames=null, journalName=null, refType=null, unstructuredReference=Wang Yan, Liu Xiaojiang, Shi Shuming. Deep neural solver for math word problems[C]//Proc of the 2017 Conf on Empirical Methods in Natural Language Processing. Stroudsburg, PA: ACL, 2017: 845−854, articleTitle=null, refAbstract=null), Reference(id=1327011450142437984, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327011432685744600, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=40, rfOrder=43, authorNames=null, journalName=null, refType=null, unstructuredReference=Wu Shih-Hung, Liu Chao-Lin, Lee Lung-Hao. Chinese spelling check evaluation at SIGHAN bake-off 2013[C/OL]//Proc of the 7th SIGHAN Workshop on Chinese Language Processing. 2013: 35−42.[2024-09-19]. https://aclanthology.org/W13-4406.pdf, articleTitle=null, refAbstract=null), Reference(id=1327011450217935457, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327011432685744600, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=41, rfOrder=44, authorNames=null, journalName=null, refType=null, unstructuredReference=Song Yan, Zhang Tong, Wang Yonggang, et al. ZEN 2.0: Continue training and adaption for n-gram enhanced text encoders[J]. arXiv preprint, arXiv: 2105.01279, 2021, articleTitle=null, refAbstract=null), Reference(id=1327011450339570274, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327011432685744600, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=42, rfOrder=45, authorNames=null, journalName=null, refType=null, unstructuredReference=Sun Yu, Wang Shuohuan, Feng Shikun, et al. ERNIE 3.0: Large-scale knowledge enhanced pre-training for language understanding and generation[J]. arXiv preprint, arXiv: 2107.02137, 2021, articleTitle=null, refAbstract=null), Reference(id=1327011450444427875, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327011432685744600, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=43, rfOrder=46, authorNames=null, journalName=null, refType=null, unstructuredReference=Xue L, Constant N, Roberts A, et al. mT5: A massively multilingual pre-trained text-to-text transformer[J]. arXiv preprint, arXiv: 2010.11934, 2020, articleTitle=null, refAbstract=null), Reference(id=1327011450515731044, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327011432685744600, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=44, rfOrder=47, authorNames=null, journalName=null, refType=null, unstructuredReference=Shao Yunfan, Geng Zhichao, Liu Yitao, et al. CPT: A pre-trained unbalanced transformer for both Chinese language understanding and generation[J]. arXiv preprint, arXiv: 2109.05729, 2021, articleTitle=null, refAbstract=null), Reference(id=1327011450570256998, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327011432685744600, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=45, rfOrder=48, authorNames=null, journalName=null, refType=null, unstructuredReference=Liu Wei, Fu Xiyan, Zhang Yue, et al. Lexicon enhanced Chinese sequence labeling using BERT adapter[C]//Proc of the 59th Annual Meeting of the Association for Computational Linguistics and the 11th Int Joint Conf on Natural Language Processing (Volume 1: Long Papers). Stroudsburg, PA: ACL, 2021: 5847–5858, articleTitle=null, refAbstract=null)], funds=null, companyList=[AuthorCompany(id=1327011436326400491, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327011432685744600, xref=1, ext=[AuthorCompanyExt(id=1327011436338983404, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327011432685744600, companyId=1327011436326400491, language=EN, country=null, province=null, city=null, postcode=null, companyName=null, departmentName=null, remark=
1College of Software, Nankai University, Tianjin 300450), AuthorCompanyExt(id=1327011436351566317, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327011432685744600, companyId=1327011436326400491, language=CN, country=null, province=null, city=null, postcode=null, companyName=null, departmentName=null, remark=
1南开大学软件学院 天津 300450)]), AuthorCompany(id=1327011436418675182, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327011432685744600, xref=2, ext=[AuthorCompanyExt(id=1327011436431258095, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327011432685744600, companyId=1327011436418675182, language=EN, country=null, province=null, city=null, postcode=null, companyName=null, departmentName=null, remark=
2Haihe Laboratory of Information Technology Application Innovation, Tianjin 300450), AuthorCompanyExt(id=1327011436456423920, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327011432685744600, companyId=1327011436418675182, language=CN, country=null, province=null, city=null, postcode=null, companyName=null, departmentName=null, remark=
2先进计算与关键软件(信创)海河实验室 天津 300450)])], figs=[ArticleFig(id=1327011440772362784, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327011432685744600, language=EN, label=Figure 1, caption=
The overall structure of LingLong, figureFileSmall=j2thoqLy1WIy0z8xpoUasQ==, figureFileBig=R99ZZU8dD3ioRspJuYe+Sg==, tableContent=null), ArticleFig(id=1327011440868831777, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327011432685744600, language=CN, label=图1, caption=
玲珑模型整体结构, figureFileSmall=j2thoqLy1WIy0z8xpoUasQ==, figureFileBig=R99ZZU8dD3ioRspJuYe+Sg==, tableContent=null), ArticleFig(id=1327011441103712802, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327011432685744600, language=EN, label=Table 1, caption=
Statistics of Training Data GB, figureFileSmall=null, figureFileBig=null, tableContent=
| 类别 | 数据量 |
|---|
| 新闻 | 36.26 |
| Common Crawl网络数据 | 7.93 |
| 百科 | 5.50 |
| 网络 | 2.35 |
| 专利 | 2.68 |
| 教育 | 1.77 |
| 小说 | 0.62 |
), ArticleFig(id=1327011441200181795, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327011432685744600, language=CN, label=表1, caption=
训练数据统计信息
, figureFileSmall=null, figureFileBig=null, tableContent=
| 类别 | 数据量 |
|---|
| 新闻 | 36.26 |
| Common Crawl网络数据 | 7.93 |
| 百科 | 5.50 |
| 网络 | 2.35 |
| 专利 | 2.68 |
| 教育 | 1.77 |
| 小说 | 0.62 |
), ArticleFig(id=1327011441284067876, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327011432685744600, language=EN, label=Table 2, caption=
Hyperparameters of LingLong, figureFileSmall=null, figureFileBig=null, tableContent=
| 超参数 | 取值 |
|---|
| 字典规模 | 13 312 |
| 嵌入维度 | 1 024 |
| 隐藏层维度 | 1 024 |
| 解码模块数量 | 24 |
| 自注意头数量 | 16 |
| 稀疏自注意步长 | 128 |
| 稀疏自注意表现力 | 8 |
| 上下文窗口长度 | 1 024 |
| 可训练参数量 | 316989440 |
), ArticleFig(id=1327011441359565349, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327011432685744600, language=CN, label=表2, caption=
玲珑中使用的超参数
, figureFileSmall=null, figureFileBig=null, tableContent=
| 超参数 | 取值 |
|---|
| 字典规模 | 13 312 |
| 嵌入维度 | 1 024 |
| 隐藏层维度 | 1 024 |
| 解码模块数量 | 24 |
| 自注意头数量 | 16 |
| 稀疏自注意步长 | 128 |
| 稀疏自注意表现力 | 8 |
| 上下文窗口长度 | 1 024 |
| 可训练参数量 | 316989440 |
), ArticleFig(id=1327011441443451430, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327011432685744600, language=EN, label=Table 3, caption=
Templates for Downstream Tasks, figureFileSmall=null, figureFileBig=null, tableContent=
| 任务 | 数据集 | 正向模板 | 反向模板 |
|---|
| 注:斜体文字表示来自数据集中的数据,正体文字是提示符或特殊分隔符. |
| 文本摘要 | CEPSUM | 类别:[“家居用品”,“箱包”,“服装”];特征信息:格式化信息;商品描述:商品描述[SEP]商品简介:商品简介 | 类别:[“装服”,“包箱”,“品用居家”];特征信息:息信化式格;商品描述:述描品商[SEP]商品简介:介简品商 |
| LCSTS | 文本:原始文本[SEP]摘要:摘要 | 文本:本文始原[SEP]摘要:要摘 |
基于结构化数 据的文本生成 | AdGen | 标题信息:标题;标签信息:标签;特征信息:商品特征;[SEP]商品描述:商品描述 | 标题信息:题标;标签信息:签标;特征信息:征特品商;[SEP]商品描述:述描品商 |
| E-Reviews | 特征信息:特征[SEP]广告文案:文案 | 特征信息:征特[SEP]广告文案:案文 |
| 问答 | KBQA | 问题:问题[SEP]答案:实体[SEP2]关系 | 问题:题问[SEP]答案:系关[SEP2]体实 |
| 中文分词 | Weibo & MSR | 原始文本:原始文本[SEP]分词结果:分词结果 | 原始文本:本文始原[SEP]分词结果:果结词分 |
| 句子对分类 | LCQMC | “句子1”与“句子2”的意思是否相似?[SEP][“是”,“否”] | ?似相否是思意的”1子句“与”2子句“[SEP][“是”,“否”] |
| 数学推理 | Math23K | 问题:问题[SEP]答案:计算公式 | 问题:题问[SEP]答案:式公算计 |
| 阅读理解 | CMRC | 文本:文本;问题:问题[SEP]答案:答案 | 文本:本文;问题:题问[SEP]答案:案答 |
), ArticleFig(id=1327011441514754599, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327011432685744600, language=CN, label=表3, caption=
下游任务模板
, figureFileSmall=null, figureFileBig=null, tableContent=
| 任务 | 数据集 | 正向模板 | 反向模板 |
|---|
| 注:斜体文字表示来自数据集中的数据,正体文字是提示符或特殊分隔符. |
| 文本摘要 | CEPSUM | 类别:[“家居用品”,“箱包”,“服装”];特征信息:格式化信息;商品描述:商品描述[SEP]商品简介:商品简介 | 类别:[“装服”,“包箱”,“品用居家”];特征信息:息信化式格;商品描述:述描品商[SEP]商品简介:介简品商 |
| LCSTS | 文本:原始文本[SEP]摘要:摘要 | 文本:本文始原[SEP]摘要:要摘 |
基于结构化数 据的文本生成 | AdGen | 标题信息:标题;标签信息:标签;特征信息:商品特征;[SEP]商品描述:商品描述 | 标题信息:题标;标签信息:签标;特征信息:征特品商;[SEP]商品描述:述描品商 |
| E-Reviews | 特征信息:特征[SEP]广告文案:文案 | 特征信息:征特[SEP]广告文案:案文 |
| 问答 | KBQA | 问题:问题[SEP]答案:实体[SEP2]关系 | 问题:题问[SEP]答案:系关[SEP2]体实 |
| 中文分词 | Weibo & MSR | 原始文本:原始文本[SEP]分词结果:分词结果 | 原始文本:本文始原[SEP]分词结果:果结词分 |
| 句子对分类 | LCQMC | “句子1”与“句子2”的意思是否相似?[SEP][“是”,“否”] | ?似相否是思意的”1子句“与”2子句“[SEP][“是”,“否”] |
| 数学推理 | Math23K | 问题:问题[SEP]答案:计算公式 | 问题:题问[SEP]答案:式公算计 |
| 阅读理解 | CMRC | 文本:文本;问题:问题[SEP]答案:答案 | 文本:本文;问题:题问[SEP]答案:案答 |
), ArticleFig(id=1327011441590252072, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327011432685744600, language=EN, label=Table 4, caption=
Summary of Our Pre-training Dataset Using Different Tokenization Strategies, figureFileSmall=null, figureFileBig=null, tableContent=
| 数据量 | 基于汉字标记策略 | 基于词语标记策略 |
|---|
| 字典规模 | 13 312 | 26 240 |
| 标记数量 | 23 710 716 503 | 19 177 964 849 |
| [UNK]数量 | 34 566 039 | 80 110 810 |
| [UNK]比例/% | 0.145 8 | 0.417 7 |
), ArticleFig(id=1327011441674138153, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327011432685744600, language=CN, label=表4, caption=
使用不同标记化策略进行预训练数据集处理结果
, figureFileSmall=null, figureFileBig=null, tableContent=
| 数据量 | 基于汉字标记策略 | 基于词语标记策略 |
|---|
| 字典规模 | 13 312 | 26 240 |
| 标记数量 | 23 710 716 503 | 19 177 964 849 |
| [UNK]数量 | 34 566 039 | 80 110 810 |
| [UNK]比例/% | 0.145 8 | 0.417 7 |
), ArticleFig(id=1327011441758024234, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327011432685744600, language=EN, label=Table 5, caption=
Performance of Models Trained Using Character-Based Strategy and Word-Based Strategy on Downstream Task Datasets %, figureFileSmall=null, figureFileBig=null, tableContent=
| 策略 | 文本摘要 | | 基于结构化数据的文本生成 | | 问答 | | 中文分词 | | 句子对分类 | | 数学推理 |
|---|
| CEPSUM 2.0 | LCSTS | | AdGen | | KBQA | | Weibo | MSR | | LCQMC | | Math23K |
|---|
| 注:对于所有任务来说,数值越高越好. |
| 基于词语策略模型 | 19.21 | 23.75 | | 6.56 | | 56.90 | | 52.78 | 60.19 | | 80.63 | | 6.10 |
| 基于汉字策略模型 | 23.73 | 30.85 | | 9.28 | | 73.00 | | 93.97 | 95.37 | | 83.00 | | 54.10 |
), ArticleFig(id=1327011441837716011, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327011432685744600, language=CN, label=表5, caption=
基于汉字策略和基于词语策略训练模型在下游任务数据集上的性能
, figureFileSmall=null, figureFileBig=null, tableContent=
| 策略 | 文本摘要 | | 基于结构化数据的文本生成 | | 问答 | | 中文分词 | | 句子对分类 | | 数学推理 |
|---|
| CEPSUM 2.0 | LCSTS | | AdGen | | KBQA | | Weibo | MSR | | LCQMC | | Math23K |
|---|
| 注:对于所有任务来说,数值越高越好. |
| 基于词语策略模型 | 19.21 | 23.75 | | 6.56 | | 56.90 | | 52.78 | 60.19 | | 80.63 | | 6.10 |
| 基于汉字策略模型 | 23.73 | 30.85 | | 9.28 | | 73.00 | | 93.97 | 95.37 | | 83.00 | | 54.10 |
), ArticleFig(id=1327011441938379308, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327011432685744600, language=EN, label=Table 6, caption=
Methods for Aggregating the Outputs of LingLong and LingLongB, figureFileSmall=null, figureFileBig=null, tableContent=
| 任务 | 中文拼写检查 | 文本摘要 | 基于结构化数据的文本生成 | 问答 | 中文分词 | 句子对分类 | 数学推理 | 阅读理解 |
|---|
| 方案 | 结合2个模型输出结果,将所有查找到的错字进行输出. | 2个模型分别输出完整结果,将结果与原始输入计算Rouge-L分数,选取得分较高的1个. | 2个模型分别输出完整结果,将结果与原始输入计算Rouge-L分数,选取得分较高的1个. | 分别使用2个模型得到的结果进行查询. 若均能查询到结果,且结果不一致,则取正向模型输出结果(查询方式见3.1节). | 结合2个模型结果,给出1个细粒度分词方案和1个粗粒度分词方案. (粗粒度:仅当玲珑和玲珑B均认为应当进行分词时才进行分词;细粒度:有任意1个模型认为应该分词时即进行分词). | 选择2个模型输出中概率最高的类别. | 从模型输出结果中选取语法正确(函数eval能够正确解析则为语法正确)的1个作为最终结果. 若均语法正确,且结果不一致,则取正向模型输出结果. | 2个模型分别输出完整结果,将结果与原始输入计算Rouge-L分数,选取得分较高的1个. |
), ArticleFig(id=1327011442022265389, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327011432685744600, language=CN, label=表6, caption=
玲珑和玲珑B输出结果的结合方式
, figureFileSmall=null, figureFileBig=null, tableContent=
| 任务 | 中文拼写检查 | 文本摘要 | 基于结构化数据的文本生成 | 问答 | 中文分词 | 句子对分类 | 数学推理 | 阅读理解 |
|---|
| 方案 | 结合2个模型输出结果,将所有查找到的错字进行输出. | 2个模型分别输出完整结果,将结果与原始输入计算Rouge-L分数,选取得分较高的1个. | 2个模型分别输出完整结果,将结果与原始输入计算Rouge-L分数,选取得分较高的1个. | 分别使用2个模型得到的结果进行查询. 若均能查询到结果,且结果不一致,则取正向模型输出结果(查询方式见3.1节). | 结合2个模型结果,给出1个细粒度分词方案和1个粗粒度分词方案. (粗粒度:仅当玲珑和玲珑B均认为应当进行分词时才进行分词;细粒度:有任意1个模型认为应该分词时即进行分词). | 选择2个模型输出中概率最高的类别. | 从模型输出结果中选取语法正确(函数eval能够正确解析则为语法正确)的1个作为最终结果. 若均语法正确,且结果不一致,则取正向模型输出结果. | 2个模型分别输出完整结果,将结果与原始输入计算Rouge-L分数,选取得分较高的1个. |
), ArticleFig(id=1327011442101957166, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327011432685744600, language=EN, label=Table 7, caption=
Performance of Bidirectional Models Versus Unidirectional Models on Downstream Tasks %, figureFileSmall=null, figureFileBig=null, tableContent=
| 模型 | 中文拼写检查 | | 文本摘要 | | 基于结构化数据的文本生成 | | 问答 | | 中文分词 | | 句子对分类 | | 数学推理 | | 阅读理解 |
|---|
| SIGHAN | | CEPSUM 2.0 | LCSTS | | AdGen | | KBQA | | Weibo | MSR | | LCQMC | | Math23K | | CMRC |
|---|
| 玲珑 | 49.80 | | 25.23 | 41.07 | | 19.24 | | 84.80 | | 95.41 | 96.03 | | 84.46 | | 68.60 | | 70.09 |
| 玲珑B | 43.88 | | 23.93 | 33.92 | | 9.46 | | 69.20 | | 95.04 | 95.84 | | 85.91 | | 59.90 | | 46.63 |
| 玲珑F+B | 63.10 | | 26.04 | 45.98 | | 20.57 | | 84.90 | | 95.20 | 96.14 | | 91.60 | | 75.60 | | 78.20 |
), ArticleFig(id=1327011442173260335, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327011432685744600, language=CN, label=表7, caption=
双向模型与单向模型在下游任务上的性能
, figureFileSmall=null, figureFileBig=null, tableContent=
| 模型 | 中文拼写检查 | | 文本摘要 | | 基于结构化数据的文本生成 | | 问答 | | 中文分词 | | 句子对分类 | | 数学推理 | | 阅读理解 |
|---|
| SIGHAN | | CEPSUM 2.0 | LCSTS | | AdGen | | KBQA | | Weibo | MSR | | LCQMC | | Math23K | | CMRC |
|---|
| 玲珑 | 49.80 | | 25.23 | 41.07 | | 19.24 | | 84.80 | | 95.41 | 96.03 | | 84.46 | | 68.60 | | 70.09 |
| 玲珑B | 43.88 | | 23.93 | 33.92 | | 9.46 | | 69.20 | | 95.04 | 95.84 | | 85.91 | | 59.90 | | 46.63 |
| 玲珑F+B | 63.10 | | 26.04 | 45.98 | | 20.57 | | 84.90 | | 95.20 | 96.14 | | 91.60 | | 75.60 | | 78.20 |
), ArticleFig(id=1327011442269729328, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327011432685744600, language=EN, label=Table 8, caption=
Examples of Text Generated by LingLong, figureFileSmall=null, figureFileBig=null, tableContent=
| 类别 | 提示文本 | 生成文本 |
|---|
人物 介绍 | 齐小明,知名学者 | 文化产业研究者. 2013年,出版《文化产业的新趋势》《中国文化产业与资本市场》. |
| 对联 | 上联:一帆风顺年年好;下联: | 风平浪静日日新 |
| 故事 | 有一天,一只小蝴蝶正在森林里飞, | 突然,它看见一只老鹰正从空中俯瞰着她. |
), ArticleFig(id=1327011442332643889, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327011432685744600, language=CN, label=表8, caption=
玲珑文本生成示例
, figureFileSmall=null, figureFileBig=null, tableContent=
| 类别 | 提示文本 | 生成文本 |
|---|
人物 介绍 | 齐小明,知名学者 | 文化产业研究者. 2013年,出版《文化产业的新趋势》《中国文化产业与资本市场》. |
| 对联 | 上联:一帆风顺年年好;下联: | 风平浪静日日新 |
| 故事 | 有一天,一只小蝴蝶正在森林里飞, | 突然,它看见一只老鹰正从空中俯瞰着她. |
), ArticleFig(id=1327011443997782578, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327011432685744600, language=EN, label=Table 9, caption=
Performance of Different Language Models on Six Downstream Tasks %, figureFileSmall=null, figureFileBig=null, tableContent=
| 模型 | 规模 | 文本摘要 | | 基于结构化数据的文本生成 | | 问答 | | 中文分词 | | 句子对分类 | | 数学推理 | | 阅读理解 |
|---|
| CEPSUM 2.0 | LCSTS | | AdGen | | KBQA | | Weibo | MSR | | LCQMC | | Math23K | | CMRC2018 |
|---|
| 注:在“规模”列中,“50M”表示该模型有5亿(50 million)个参数,“10B”表示该模型有100亿(10 billion)个参数. “∗”表示该模型规模由估算得到;“†”表示该结果由本文使用与玲珑相同策略进行微调和评估得到;“‡”表示该结果在零样本模式(未微调)下评估得到;“-”表示由于模型无法在对应任务/基准上评估或者由于模型权重不公开,因此无法得到结果. 黑体数值表示最优结果. |
| ZEN 2.0 [41] | 233M* | - | - | | - | | - | | - | 98.35 | | 88.81 | | - | | 70.77 |
| ERNIE 3.0 XBase[42] | 280M | 2.35† | 18.86† | | 7.80† | | 0† | | 50.55† | 70.37† | | 89.06† | | 0.70† | | 75.99 |
| mT5-small[43] | 300M | 9.02† | 33.10 | | 10.20 | | 0† | | 49.18† | 49.32† | | 82.10 | | 18.40 | | 0.90 |
| CPTg[44] | 393M | 26.26† | 42.80 | | 10.70 | | 84.38† | | - | - | | 90.68† | | 36.90† | | - |
| CPTu[44] | 393M | - | - | | - | | - | | 42.00† | 98.51† | | 91.29† | | - | | 68.80 |
| BART[5] | 406M | 22.16† | 40.90 | | 12.68† | | 84.68† | | 46.14† | 44.42† | | 90.93† | | 49.10† | | 61.32 |
| GLM | 335M | 17.27† | 34.25† | | 1.30† | | 99.10† | | 92.70† | 89.18† | | 84.61† | | 51.70† | | 70.74 |
| LEBERT[45] | 7.5B* | - | - | | - | | - | | - | 98.69 | | - | | - | | - |
| ERNIE 3.0[42] | 10B | - | 48.46 | | 30.16 | | - | | - | - | | 90.38 | | 75.00 | | 75.30 |
| CPM-2[14] | 11B | 0.91‡ | 35.90 | | 10.60 | | 0‡ | | 34.72‡ | 33.22‡ | | 89.16 | | 69.37 | | 15.66 |
| mT5-XXL[43] | 13B | 0.17‡ | 34.80 | | 9.80 | | 0‡ | | 42.07‡ | 43.51‡ | | 88.30 | | 61.60 | | 25.20 |
| Yuan 1.0[27] | 245B | - | - | | - | | - | | - | - | | - | | 76.90 | | 5.58‡ |
| 玲珑F+B | 317M | 26.04 | 45.98 | | 20.57 | | 84.90 | | 95.20 | 96.14 | | 91.60 | | 75.60 | | 78.20 |
), ArticleFig(id=1327011444106834483, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327011432685744600, language=CN, label=表9, caption=
不同语言模型在6个下游任务的性能
, figureFileSmall=null, figureFileBig=null, tableContent=
| 模型 | 规模 | 文本摘要 | | 基于结构化数据的文本生成 | | 问答 | | 中文分词 | | 句子对分类 | | 数学推理 | | 阅读理解 |
|---|
| CEPSUM 2.0 | LCSTS | | AdGen | | KBQA | | Weibo | MSR | | LCQMC | | Math23K | | CMRC2018 |
|---|
| 注:在“规模”列中,“50M”表示该模型有5亿(50 million)个参数,“10B”表示该模型有100亿(10 billion)个参数. “∗”表示该模型规模由估算得到;“†”表示该结果由本文使用与玲珑相同策略进行微调和评估得到;“‡”表示该结果在零样本模式(未微调)下评估得到;“-”表示由于模型无法在对应任务/基准上评估或者由于模型权重不公开,因此无法得到结果. 黑体数值表示最优结果. |
| ZEN 2.0 [41] | 233M* | - | - | | - | | - | | - | 98.35 | | 88.81 | | - | | 70.77 |
| ERNIE 3.0 XBase[42] | 280M | 2.35† | 18.86† | | 7.80† | | 0† | | 50.55† | 70.37† | | 89.06† | | 0.70† | | 75.99 |
| mT5-small[43] | 300M | 9.02† | 33.10 | | 10.20 | | 0† | | 49.18† | 49.32† | | 82.10 | | 18.40 | | 0.90 |
| CPTg[44] | 393M | 26.26† | 42.80 | | 10.70 | | 84.38† | | - | - | | 90.68† | | 36.90† | | - |
| CPTu[44] | 393M | - | - | | - | | - | | 42.00† | 98.51† | | 91.29† | | - | | 68.80 |
| BART[5] | 406M | 22.16† | 40.90 | | 12.68† | | 84.68† | | 46.14† | 44.42† | | 90.93† | | 49.10† | | 61.32 |
| GLM | 335M | 17.27† | 34.25† | | 1.30† | | 99.10† | | 92.70† | 89.18† | | 84.61† | | 51.70† | | 70.74 |
| LEBERT[45] | 7.5B* | - | - | | - | | - | | - | 98.69 | | - | | - | | - |
| ERNIE 3.0[42] | 10B | - | 48.46 | | 30.16 | | - | | - | - | | 90.38 | | 75.00 | | 75.30 |
| CPM-2[14] | 11B | 0.91‡ | 35.90 | | 10.60 | | 0‡ | | 34.72‡ | 33.22‡ | | 89.16 | | 69.37 | | 15.66 |
| mT5-XXL[43] | 13B | 0.17‡ | 34.80 | | 9.80 | | 0‡ | | 42.07‡ | 43.51‡ | | 88.30 | | 61.60 | | 25.20 |
| Yuan 1.0[27] | 245B | - | - | | - | | - | | - | - | | - | | 76.90 | | 5.58‡ |
| 玲珑F+B | 317M | 26.04 | 45.98 | | 20.57 | | 84.90 | | 95.20 | 96.14 | | 91.60 | | 75.60 | | 78.20 |
)], attaches=null, journal=Journal(id=1324759606947315908, delFlag=0, nameCn=计算机研究与发展, nameEn=Journal of Computer Research and Development, nameHistory1=null, nameHistory2=null, issn=1000-1239, eissn=null, cn=11-1777/TP, coden=null, periodic=0, language=CN, oaType=null, ccby=null, superviseOffice=null, ownerOffice=null, pubOffice=null, editorOffice=null, officeType=null, aims=null, clcCode=null, officeProv=null, officeCity=null, officeAddr=null, officeZip=null, officeEmail=null, officePhone=null, editDirector=null, officeDirector=null, officeDirectorPhone=null, officeStaffNum=null, officeEmpNum=null, coverPicUrl=k2l8zgqwstuxxevFX2qxVA==, journalPrice=null, startedYear=null, abbrevIsoEn=Journal of Computer Research and Development, journalRemark=null, publicationField=null, createdTime=1793776903141, updatedTime=1793850698612, createdBy=18614031015, updatedBy=13701087609, firstLetterCn=J, firstLetterEn=J, subjectCode=Electronic Information, subjectName=null, subjectCodeEn=Electronic Information, subjectNameEn=null, picCn=k2l8zgqwstuxxevFX2qxVA==, picEn=r6k0Ry+RD+c/VXj5QarmDw==, jcr=null, cjcr=null, exts=[JournalExt(id=1325069127710286688, language=CN, name=计算机研究与发展, nameHistory1=null, nameHistory2=null, managedBy=, sponsoredBy=, publishedBy=, editorOffice=, officeProv=null, officeCity=null, officeAddr=, officeZip=, editDirector=, officeDirector=null, officePhone=null, coverPicUrl=null, journalRemark=, submitArticleUrl=null, websiteUrl=, createdTime=1793850698640, updatedTime=1793850698640, createdBy=13701087609, updatedBy=13701087609, submissionGuidelinesUrl=, submissionAuthorUrl=https://crad.ict.ac.cn/journalx/authorLogOn.action, submissionEditorUrl=https://crad.ict.ac.cn/journalx/editorLogOn.action, submissionReviewUrl=https://crad.ict.ac.cn/journalx/expertLogOn.action, submissionCeEditorUrl=https://crad.ict.ac.cn/journalx/editorInChiefLogOn.action, submissionAeEditorUrl=, option={"copyright":""}), JournalExt(id=1325069127777395553, language=EN, name=Journal of Computer Research and Development, nameHistory1=null, nameHistory2=null, managedBy=, sponsoredBy=, publishedBy=, editorOffice=, officeProv=null, officeCity=null, officeAddr=, officeZip=, editDirector=, officeDirector=null, officePhone=null, coverPicUrl=null, journalRemark=, submitArticleUrl=null, websiteUrl=, createdTime=1793850698656, updatedTime=1793850698656, createdBy=13701087609, updatedBy=13701087609, submissionGuidelinesUrl=, submissionAuthorUrl=https://crad.ict.ac.cn/journalx/authorLogOn.action, submissionEditorUrl=https://crad.ict.ac.cn/journalx/editorLogOn.action, submissionReviewUrl=https://crad.ict.ac.cn/journalx/expertLogOn.action, submissionCeEditorUrl=https://crad.ict.ac.cn/journalx/editorInChiefLogOn.action, submissionAeEditorUrl=, option={"copyright":""})], databaseList=null, tenantJournalId=1324759914146529478, websiteList=[Website(id=1325070300139892771, webName=null, webTitle=null, webDomain=null, webCopyrigh=null, webIpcNo=null, seoTitle=null, seoKeywords=null, seoDescription=null, tenantJournalId=null, journalId=1324759914146529478, journalNameCn=null, journalNameEn=null, grayFlag=null, tenantId=1146029695717560320, platformId=null, journalGroupId=null, journalGroupNameCn=null, journalGroupNameEn=null, type=1, domain=https://castjournals.cast.org.cn/joweb/jsjyjyfz/CN, language=CN, createTime=1793850978169, createBy=18614031015, updateTime=1793851040259, updateBy=18614031015, name=计算机研究与发展-中文, tplId=1146099689490845704, title=计算机研究与发展, delFlag=0, indexPage=/home, props=[WebsiteProps(id=1325070893751349360, tenantId=1146029695717560320, journalId=null, journalGroupId=null, siteId=1325070300139892771, code=articleTextType, value=kx, createTime=1793851119697, updateTime=1793851119697, creator=18614031015, updator=18614031015), WebsiteProps(id=1325070893730377837, tenantId=1146029695717560320, journalId=null, journalGroupId=null, siteId=1325070300139892771, code=banner, value=null, createTime=1793851119692, updateTime=1793851119692, creator=18614031015, updator=18614031015), WebsiteProps(id=1325070893772320883, tenantId=1146029695717560320, journalId=null, journalGroupId=null, siteId=1325070300139892771, code=grayFlag, value=0, createTime=1793851119702, updateTime=1793851119702, creator=18614031015, updator=18614031015), WebsiteProps(id=1325070893721989228, tenantId=1146029695717560320, journalId=null, journalGroupId=null, siteId=1325070300139892771, code=logo, value=https://castjournals.cast.org.cn/joweb/jsjyjyfz/CN/file/pic?fileId=U27tDsy717YF7XSPA+Vwdw==, createTime=1793851119690, updateTime=1793851119690, creator=18614031015, updator=18614031015), WebsiteProps(id=1325070893784903797, tenantId=1146029695717560320, journalId=null, journalGroupId=null, siteId=1325070300139892771, code=minRunFlag, value=0, createTime=1793851119705, updateTime=1793851119705, creator=18614031015, updator=18614031015), WebsiteProps(id=1325070893742960751, tenantId=1146029695717560320, journalId=null, journalGroupId=null, siteId=1325070300139892771, code=picServerUrl, value=https://castjournals.cast.org.cn/joweb/jsjyjyfz/CN/file/pic, createTime=1793851119695, updateTime=1793851119695, creator=18614031015, updator=18614031015), WebsiteProps(id=1325070893780709492, tenantId=1146029695717560320, journalId=null, journalGroupId=null, siteId=1325070300139892771, code=silenceFlag, value=0, createTime=1793851119704, updateTime=1793851119704, creator=18614031015, updator=18614031015), WebsiteProps(id=1325070893738766446, tenantId=1146029695717560320, journalId=null, journalGroupId=null, siteId=1325070300139892771, code=staticResourcePath, value=https://castjournals.cast.org.cn/joweb/cast_kjdb_cn_619/, createTime=1793851119694, updateTime=1793851119694, creator=18614031015, updator=18614031015), WebsiteProps(id=1325070893755543665, tenantId=1146029695717560320, journalId=null, journalGroupId=null, siteId=1325070300139892771, code=themeColor, value=null, createTime=1793851119698, updateTime=1793851119698, creator=18614031015, updator=18614031015), WebsiteProps(id=1325070893763932274, tenantId=1146029695717560320, journalId=null, journalGroupId=null, siteId=1325070300139892771, code=themeStyle, value=null, createTime=1793851119700, updateTime=1793851119700, creator=18614031015, updator=18614031015)]), Website(id=1325070300580294698, webName=null, webTitle=null, webDomain=null, webCopyrigh=null, webIpcNo=null, seoTitle=null, seoKeywords=null, seoDescription=null, tenantJournalId=null, journalId=1324759914146529478, journalNameCn=null, journalNameEn=null, grayFlag=null, tenantId=1146029695717560320, platformId=null, journalGroupId=null, journalGroupNameCn=null, journalGroupNameEn=null, type=1, domain=https://castjournals.cast.org.cn/joweb/jsjyjyfz/EN, language=EN, createTime=1793850978274, createBy=18614031015, updateTime=1793851036638, updateBy=18614031015, name=计算机研究与发展-英文, tplId=1146101810881728533, title=Journal of Computer Research and Development, delFlag=0, indexPage=/home, props=[WebsiteProps(id=1325070925288321148, tenantId=1146029695717560320, journalId=null, journalGroupId=null, siteId=1325070300580294698, code=articleTextType, value=kx, createTime=1793851127216, updateTime=1793851127216, creator=18614031015, updator=18614031015), WebsiteProps(id=1325070925263155321, tenantId=1146029695717560320, journalId=null, journalGroupId=null, siteId=1325070300580294698, code=banner, value=null, createTime=1793851127210, updateTime=1793851127210, creator=18614031015, updator=18614031015), WebsiteProps(id=1325070925309292671, tenantId=1146029695717560320, journalId=null, journalGroupId=null, siteId=1325070300580294698, code=grayFlag, value=0, createTime=1793851127221, updateTime=1793851127221, creator=18614031015, updator=18614031015), WebsiteProps(id=1325070925254766712, tenantId=1146029695717560320, journalId=null, journalGroupId=null, siteId=1325070300580294698, code=logo, value=https://castjournals.cast.org.cn/joweb/jsjyjyfz/EN/file/pic?fileId=U27tDsy717YF7XSPA+Vwdw==, createTime=1793851127208, updateTime=1793851127208, creator=18614031015, updator=18614031015), WebsiteProps(id=1325070925326069889, tenantId=1146029695717560320, journalId=null, journalGroupId=null, siteId=1325070300580294698, code=minRunFlag, value=0, createTime=1793851127225, updateTime=1793851127225, creator=18614031015, updator=18614031015), WebsiteProps(id=1325070925279932539, tenantId=1146029695717560320, journalId=null, journalGroupId=null, siteId=1325070300580294698, code=picServerUrl, value=https://castjournals.cast.org.cn/joweb/jsjyjyfz/EN/file/pic, createTime=1793851127214, updateTime=1793851127214, creator=18614031015, updator=18614031015), WebsiteProps(id=1325070925317681280, tenantId=1146029695717560320, journalId=null, journalGroupId=null, siteId=1325070300580294698, code=silenceFlag, value=0, createTime=1793851127223, updateTime=1793851127223, creator=18614031015, updator=18614031015), WebsiteProps(id=1325070925271543930, tenantId=1146029695717560320, journalId=null, journalGroupId=null, siteId=1325070300580294698, code=staticResourcePath, value=https://castjournals.cast.org.cn/joweb/cast_kjdb_en_623/, createTime=1793851127212, updateTime=1793851127212, creator=18614031015, updator=18614031015), WebsiteProps(id=1325070925292515453, tenantId=1146029695717560320, journalId=null, journalGroupId=null, siteId=1325070300580294698, code=themeColor, value=null, createTime=1793851127217, updateTime=1793851127217, creator=18614031015, updator=18614031015), WebsiteProps(id=1325070925300904062, tenantId=1146029695717560320, journalId=null, journalGroupId=null, siteId=1325070300580294698, code=themeStyle, value=null, createTime=1793851127219, updateTime=1793851127219, creator=18614031015, updator=18614031015)])], journalTitle=计算机研究与发展, weixinUrl=null, journalUrl=https://crad.ict.ac.cn/, iacademicId=null, status=1, seqNo=null, journalTitleEn=Journal of Computer Research and Development, journalPhotoCn=k2l8zgqwstuxxevFX2qxVA==, journalPhotoEn=r6k0Ry+RD+c/VXj5QarmDw==, journalFirstLetter=J, journalRecommend=null, journalNew=null, journalCollection=null, jcrJf=null, cjcrJf=null, jcrJfStr=null, cjcrJfStr=null, submissionFirstDecision=null, sciSubjectClassification=null, casSubjectClassification=null, citeScore=null, totalCitationFrequency=null, icpCode=null, psCode=null, advertisingLicenseCode=null, copyrightInformation=null, country=null, option=, provinceCode=null, provinceName=null, collectFlag=false, interPubPlatform=, interPubPlatformUrl=null), detailUrlCn=https://castjournals.cast.org.cn/joweb/jsjyjyfz/CN/10.7544/issn1000-1239.202330844, detailUrlEn=https://castjournals.cast.org.cn/joweb/jsjyjyfz/EN/10.7544/issn1000-1239.202330844, pdfUrlCn=https://castjournals.cast.org.cn/joweb/jsjyjyfz/CN/PDF/10.7544/issn1000-1239.202330844, pdfUrlEn=https://castjournals.cast.org.cn/joweb/jsjyjyfz/EN/PDF/10.7544/issn1000-1239.202330844, aliStartDate=0, aliEndDate=0, collectionFlag=false, citedCount=null, citedUrl=null, previewStatus=0, delFlag=0, hasFullText=1, orderTime=1740758400000, fullTextJson=null, articleText=null, reference=null)