Article(id=1327017598799163887, tenantId=1146029695717560320, journalId=1324759914146529478, issueId=1327017337032651028, articleNumber=null, orderNo=null, doi=10.7544/issn1000-1239.202440780, pmid=null, cstr=null, oa=null, hot=null, price=null, onlineType=0, articleFormat=0, articleType=null, articleTypeStr=research-article, receivedDate=1728489600000, receivedDateStr=2024-10-10, revisedDate=1743955200000, revisedDateStr=2025-04-07, acceptedDate=null, acceptedDateStr=null, onlineDate=1794315250348, onlineDateStr=2026-11-10, pubDate=1764518400000, pubDateStr=2025-12-01, doiRegisterDate=null, doiRegisterDateStr=null, onlineIssueDate=1794315250348, onlineIssueDateStr=2026-11-10, onlineJustAcceptDate=null, onlineJustAcceptDateStr=null, onlineFirstDate=null, onlineFirstDateStr=null, sourceXml=null, magXml=null, createTime=1794315250347, creator=13701087609, updateTime=1794315250347, updator=13701087609, issue=Issue{id=1327017337032651028, tenantId=1146029695717560320, journalId=1324759914146529478, year='2025', volume='62', issue='12', pageStart='2889', pageEnd='3146', issueExtLink='null', onlineDate='null', pubDate='1764518400000', pubDateStr='2025-12-01', beforeIssueId=null, nextIssueId=null, price=null, status=1, issueComplete=1, articleOrder=1, issueType=-1, specialIssue=null, createTime=1794315187938, creator='13701087609', updateTime=1794377025235, updator='13701087609', preIssue=null, nextIssue=null, articleTotal=null, ext={EN=IssueExt(id=1327276701995872766, tenantId=1146029695717560320, journalId=1324759914146529478, issueId=1327017337032651028, language=EN, specialIssueTitle=, coverIllustrator=null, specialIssueEditor=, specialIssueAbout=), CN=IssueExt(id=1327276701995872767, tenantId=1146029695717560320, journalId=1324759914146529478, issueId=1327017337032651028, language=CN, specialIssueTitle=, coverIllustrator=null, specialIssueEditor=, specialIssueAbout=)}, issueFiles=null, downloadFileDto=null}, startPage=2993, endPage=3001, ext={EN=ArticleExt(id=1327017599013073392, articleId=1327017598799163887, tenantId=1146029695717560320, journalId=1324759914146529478, language=EN, title=Detecting Pretraining Data for Large Language Models via Long-Range Context, columnId=1325489891789910943, journalTitle=Journal of Computer Research and Development, columnName=Artificial Intelligence, runingTitle=null, highlight=null, articleAbstract=
Pretraining data detection aims to determine whether a piece of text belongs to the pretraining data of a large language model (LLM) when the model’s pretraining data are not publicly disclosed, which can be used to audit whether the pretraining data usage complies with legal regulations. Existing methods generally assume that an LLM tends to assign higher token probabilities to training texts compared with non-training texts, and thus identify texts with high probabilities as training texts. However, due to the significant overlap in fragments between training texts and non-training texts, an LLM may also assign relatively high token probabilities to non-training texts, which makes existing methods prone to misclassifying non-training texts as training texts. Inspired by research on the memorization capabilities of LLMs, we propose a novel method to mitigate this issue by comparing the token probabilities given the full context with those given the short-range context, and the contribution of long-range context to token probability increase is then computed, with a higher contribution indicating a greater likelihood of the text being part of the pretraining data. The key idea is that when an LLM predicts the token probabilities for training texts, the contribution of long-range context to token probability increase is greater than that for non-training texts. Experimental results on multiple datasets demonstrate the effectiveness of the proposed method. The code is available at https://github.com/zhang-wei-chao/Long-Range-Context-for-PDD.
, authors=Weichao Zhang
1, 2, 3, Ruqing Zhang
1, 2, *, Jiafeng Guo
1, 2, Yixing Fan
1, 2, authorsList=Weichao Zhang, Ruqing Zhang, Jiafeng Guo, Yixing Fan, authorCompany=null, correspAuthors=Ruqing Zhang, authorNote=
born in 2000. PhD candidate. His main research interest includes security of large language models
born in 1994. PhD, associate professor. Her main research interests include natural language processing and information retrieval
born in 1980. PhD, professor. His main research interests include representation learning, and neural models for information retrieval and filtering
born in 1990. PhD, associate professor. His main research interests include data mining and information retrieval
, correspAuthorsNote=null, copyrightStatement=Copyright © 2025 Journal of Computer Research and Development. All rights reserved., copyrightOwner=null, extLink=null, articleAbsUrl=null, sourceXml=null, magXml=null, pdfUrl=null, pdf=null, pdfFileSize=null, pdfExtLink=null, richHtmlUrl=null, mobilePdfUrl=null, reviewReport=null, pdfFirstPage=null, abstractGraph=null, abstractGraphContent=null, abstractVideo=null, citation=null, cebUrl=null, magXmlContent=null, mapNumber=null, fund=null), CN=ArticleExt(id=1327017602196550143, articleId=1327017598799163887, tenantId=1146029695717560320, journalId=1324759914146529478, language=CN, title=基于长距离上下文的大语言模型预训练数据检测方法, columnId=1325489892104483745, journalTitle=计算机研究与发展, columnName=人工智能, runingTitle=null, highlight=null, articleAbstract=
预训练数据检测方法旨在大语言模型的预训练数据未公开时,检测某段给定的文本是否属于该模型的预训练数据,可用于审查大语言模型的预训练数据的使用过程是否符合法律法规. 现有方法通常认为大语言模型对训练文本的词元概率在整体上比非训练文本的高,并基于此判定具有高预测概率的文本为训练文本. 然而,由于训练文本和非训练文本之间存在着大量的短片段重叠现象,导致模型对非训练文本的词元概率也可能比较高,使得现有方法容易将非训练文本误检为训练文本. 受大语言模型的记忆能力研究启发,通过对比给定全部上下文时的词元概率与给定短距离上下文时的词元概率之间的差异,计算得到长距离上下文对词元概率提升的贡献度,并认为贡献度越大的文本更可能是训练文本,进而缓解短片段重叠现象对检测的不利影响. 其核心思想在于,大语言模型在预测训练文本中词元的概率时,距离当前词元较远的上下文对词元概率提升的贡献度,会比在预测非训练文本中词元的概率时的贡献度更大. 在多个公开数据集上的实验结果表明该方法的有效性.
, authors=张伟超
1, 2, 3, 张儒清
1, 2, *, 郭嘉丰
1, 2, 范意兴
1, 2, authorsList=张伟超, 张儒清, 郭嘉丰, 范意兴, authorCompany=null, correspAuthors=张儒清, authorNote=
 |
2000年生. 博士研究生. 主要研究方向为大语言模型安全 |
 |
1994年生. 博士,副研究员. 主要研究方向为自然语言处理、信息检索 |
 |
1980年生. 博士,研究员. 主要研究方向为表示学习、针对信息检索与过滤的神经模型 |
 |
1990年生. 博士. 副研究员. 主要研究方向为数据挖掘、信息检索 |
, correspAuthorsNote=
, copyrightStatement=版权所有 © 2025 计算机研究与发展编辑部, copyrightOwner=null, extLink=null, articleAbsUrl=null, sourceXml=yLr7L01onsQGnPrFWBqRVQ==, magXml=sBMC2aiNOXupACfWNzQluw==, pdfUrl=null, pdf=TCGxwt7JCGhlnr/CvoOOzA==, pdfFileSize=1063279, pdfExtLink=null, richHtmlUrl=null, mobilePdfUrl=null, reviewReport=null, pdfFirstPage=null, abstractGraph=nWwEVo/aKytnReucNIpXLQ==, abstractGraphContent=null, abstractVideo=null, citation=null, cebUrl=null, magXmlContent=dpV0fGy1Xtwv3NaRZ8nUtQ==, mapNumber=null, fund=null)}, authors=[Author(id=1327017602741809675, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327017598799163887, orderNo=0, firstName=null, middleName=null, lastName=null, nameCn=null, orcid=null, stid=null, country=null, authorPic=null, dead=0, email=zhangweichao22z@ict.ac.cn, emailSecond=null, emailThird=null, correspondingAuthor=0, authorType=1, ext={EN=AuthorExt(id=1327017602855055887, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327017598799163887, authorId=1327017602741809675, language=EN, stringName=Weichao Zhang, firstName=Weichao, middleName=null, lastName=Zhang, prefix=null, suffix=null, authorComment=null, nameInitials=null, affiliation=null, department=null, xref=
1, 2, 3, address=
1CAS Key Laboratory of Network Data Science and Technology (Institute of Computing Technology, Chinese Academy of Sciences), Beijing 100190
2University of Chinese Academy of Sciences, Beijing 100049
3Zhongguancun Laboratory, Beijing 100094, bio={"content":"
born in 2000. PhD candidate. His main research interest includes security of large language models
"}, bioImg=null, bioContent=
born in 2000. PhD candidate. His main research interest includes security of large language models
, aboutCorrespAuthor=null), CN=AuthorExt(id=1327017602922164752, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327017598799163887, authorId=1327017602741809675, language=CN, stringName=张伟超, firstName=null, middleName=null, lastName=null, prefix=null, suffix=null, authorComment=null, nameInitials=null, affiliation=null, department=null, xref=
1, 2, 3, address=
1中国科学院网络数据科学与技术重点实验室(中国科学院计算技术研究所) 北京 100190
2中国科学院大学 北京 100049
3中关村实验室 北京 100094, bio={"img":"4kAivzdDzwmNreUKrOEi2Q==","content":"
2000年生. 博士研究生. 主要研究方向为大语言模型安全
"}, bioImg=4kAivzdDzwmNreUKrOEi2Q==, bioContent=
2000年生. 博士研究生. 主要研究方向为大语言模型安全
, aboutCorrespAuthor=null)}, companyList=[AuthorCompany(id=1327017602423042560, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327017598799163887, xref=1, ext=[AuthorCompanyExt(id=1327017602431431169, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327017598799163887, companyId=1327017602423042560, language=EN, country=null, province=null, city=null, postcode=null, companyName=null, departmentName=null, remark=
1CAS Key Laboratory of Network Data Science and Technology (Institute of Computing Technology, Chinese Academy of Sciences), Beijing 100190), AuthorCompanyExt(id=1327017602435625474, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327017598799163887, companyId=1327017602423042560, language=CN, country=null, province=null, city=null, postcode=null, companyName=null, departmentName=null, remark=
1中国科学院网络数据科学与技术重点实验室(中国科学院计算技术研究所) 北京 100190)]), AuthorCompany(id=1327017602540483075, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327017598799163887, xref=2, ext=[AuthorCompanyExt(id=1327017602553065988, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327017598799163887, companyId=1327017602540483075, language=EN, country=null, province=null, city=null, postcode=null, companyName=null, departmentName=null, remark=
2University of Chinese Academy of Sciences, Beijing 100049), AuthorCompanyExt(id=1327017602586620422, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327017598799163887, companyId=1327017602540483075, language=CN, country=null, province=null, city=null, postcode=null, companyName=null, departmentName=null, remark=
2中国科学院大学 北京 100049)]), AuthorCompany(id=1327017602657923591, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327017598799163887, xref=3, ext=[AuthorCompanyExt(id=1327017602666312200, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327017598799163887, companyId=1327017602657923591, language=EN, country=null, province=null, city=null, postcode=null, companyName=null, departmentName=null, remark=
3Zhongguancun Laboratory, Beijing 100094), AuthorCompanyExt(id=1327017602670506505, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327017598799163887, companyId=1327017602657923591, language=CN, country=null, province=null, city=null, postcode=null, companyName=null, departmentName=null, remark=
3中关村实验室 北京 100094)])]), Author(id=1327017603010245138, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327017598799163887, orderNo=1, firstName=null, middleName=null, lastName=null, nameCn=null, orcid=null, stid=null, country=null, authorPic=null, dead=0, email=zhangruqing@ict.ac.cn, emailSecond=null, emailThird=null, correspondingAuthor=1, authorType=1, ext={EN=AuthorExt(id=1327017603094131221, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327017598799163887, authorId=1327017603010245138, language=EN, stringName=Ruqing Zhang, firstName=Ruqing, middleName=null, lastName=Zhang, prefix=null, suffix=null, authorComment=null, nameInitials=null, affiliation=null, department=null, xref=
1, 2, *, address=
1CAS Key Laboratory of Network Data Science and Technology (Institute of Computing Technology, Chinese Academy of Sciences), Beijing 100190
2University of Chinese Academy of Sciences, Beijing 100049, bio={"content":"
born in 1994. PhD, associate professor. Her main research interests include natural language processing and information retrieval
"}, bioImg=null, bioContent=
born in 1994. PhD, associate professor. Her main research interests include natural language processing and information retrieval
, aboutCorrespAuthor=null), CN=AuthorExt(id=1327017603190600214, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327017598799163887, authorId=1327017603010245138, language=CN, stringName=张儒清, firstName=null, middleName=null, lastName=null, prefix=null, suffix=null, authorComment=null, nameInitials=null, affiliation=null, department=null, xref=
1, 2, *, address=
1中国科学院网络数据科学与技术重点实验室(中国科学院计算技术研究所) 北京 100190
2中国科学院大学 北京 100049, bio={"img":"FMjQ6A3P9IqEEzqkVsVP0A==","content":"
1994年生. 博士,副研究员. 主要研究方向为自然语言处理、信息检索
"}, bioImg=FMjQ6A3P9IqEEzqkVsVP0A==, bioContent=
1994年生. 博士,副研究员. 主要研究方向为自然语言处理、信息检索
, aboutCorrespAuthor=null)}, companyList=[AuthorCompany(id=1327017602423042560, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327017598799163887, xref=1, ext=[AuthorCompanyExt(id=1327017602431431169, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327017598799163887, companyId=1327017602423042560, language=EN, country=null, province=null, city=null, postcode=null, companyName=null, departmentName=null, remark=
1CAS Key Laboratory of Network Data Science and Technology (Institute of Computing Technology, Chinese Academy of Sciences), Beijing 100190), AuthorCompanyExt(id=1327017602435625474, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327017598799163887, companyId=1327017602423042560, language=CN, country=null, province=null, city=null, postcode=null, companyName=null, departmentName=null, remark=
1中国科学院网络数据科学与技术重点实验室(中国科学院计算技术研究所) 北京 100190)]), AuthorCompany(id=1327017602540483075, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327017598799163887, xref=2, ext=[AuthorCompanyExt(id=1327017602553065988, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327017598799163887, companyId=1327017602540483075, language=EN, country=null, province=null, city=null, postcode=null, companyName=null, departmentName=null, remark=
2University of Chinese Academy of Sciences, Beijing 100049), AuthorCompanyExt(id=1327017602586620422, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327017598799163887, companyId=1327017602540483075, language=CN, country=null, province=null, city=null, postcode=null, companyName=null, departmentName=null, remark=
2中国科学院大学 北京 100049)])]), Author(id=1327017603261903384, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327017598799163887, orderNo=2, firstName=null, middleName=null, lastName=null, nameCn=null, orcid=null, stid=null, country=null, authorPic=null, dead=0, email=null, emailSecond=null, emailThird=null, correspondingAuthor=0, authorType=1, ext={EN=AuthorExt(id=1327017603358372379, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327017598799163887, authorId=1327017603261903384, language=EN, stringName=Jiafeng Guo, firstName=Jiafeng, middleName=null, lastName=Guo, prefix=null, suffix=null, authorComment=null, nameInitials=null, affiliation=null, department=null, xref=
1, 2, address=
1CAS Key Laboratory of Network Data Science and Technology (Institute of Computing Technology, Chinese Academy of Sciences), Beijing 100190
2University of Chinese Academy of Sciences, Beijing 100049, bio={"content":"
born in 1980. PhD, professor. His main research interests include representation learning, and neural models for information retrieval and filtering
"}, bioImg=null, bioContent=
born in 1980. PhD, professor. His main research interests include representation learning, and neural models for information retrieval and filtering
, aboutCorrespAuthor=null), CN=AuthorExt(id=1327017603463229980, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327017598799163887, authorId=1327017603261903384, language=CN, stringName=郭嘉丰, firstName=null, middleName=null, lastName=null, prefix=null, suffix=null, authorComment=null, nameInitials=null, affiliation=null, department=null, xref=
1, 2, address=
1中国科学院网络数据科学与技术重点实验室(中国科学院计算技术研究所) 北京 100190
2中国科学院大学 北京 100049, bio={"img":"aRsjP7fknI9C0HkijdeiFA==","content":"
1980年生. 博士,研究员. 主要研究方向为表示学习、针对信息检索与过滤的神经模型
"}, bioImg=aRsjP7fknI9C0HkijdeiFA==, bioContent=
1980年生. 博士,研究员. 主要研究方向为表示学习、针对信息检索与过滤的神经模型
, aboutCorrespAuthor=null)}, companyList=[AuthorCompany(id=1327017602423042560, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327017598799163887, xref=1, ext=[AuthorCompanyExt(id=1327017602431431169, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327017598799163887, companyId=1327017602423042560, language=EN, country=null, province=null, city=null, postcode=null, companyName=null, departmentName=null, remark=
1CAS Key Laboratory of Network Data Science and Technology (Institute of Computing Technology, Chinese Academy of Sciences), Beijing 100190), AuthorCompanyExt(id=1327017602435625474, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327017598799163887, companyId=1327017602423042560, language=CN, country=null, province=null, city=null, postcode=null, companyName=null, departmentName=null, remark=
1中国科学院网络数据科学与技术重点实验室(中国科学院计算技术研究所) 北京 100190)]), AuthorCompany(id=1327017602540483075, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327017598799163887, xref=2, ext=[AuthorCompanyExt(id=1327017602553065988, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327017598799163887, companyId=1327017602540483075, language=EN, country=null, province=null, city=null, postcode=null, companyName=null, departmentName=null, remark=
2University of Chinese Academy of Sciences, Beijing 100049), AuthorCompanyExt(id=1327017602586620422, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327017598799163887, companyId=1327017602540483075, language=CN, country=null, province=null, city=null, postcode=null, companyName=null, departmentName=null, remark=
2中国科学院大学 北京 100049)])]), Author(id=1327017603542921758, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327017598799163887, orderNo=3, firstName=null, middleName=null, lastName=null, nameCn=null, orcid=null, stid=null, country=null, authorPic=null, dead=0, email=null, emailSecond=null, emailThird=null, correspondingAuthor=0, authorType=1, ext={EN=AuthorExt(id=1327017603605836321, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327017598799163887, authorId=1327017603542921758, language=EN, stringName=Yixing Fan, firstName=Yixing, middleName=null, lastName=Fan, prefix=null, suffix=null, authorComment=null, nameInitials=null, affiliation=null, department=null, xref=
1, 2, address=
1CAS Key Laboratory of Network Data Science and Technology (Institute of Computing Technology, Chinese Academy of Sciences), Beijing 100190
2University of Chinese Academy of Sciences, Beijing 100049, bio={"content":"
born in 1990. PhD, associate professor. His main research interests include data mining and information retrieval
"}, bioImg=null, bioContent=
born in 1990. PhD, associate professor. His main research interests include data mining and information retrieval
, aboutCorrespAuthor=null), CN=AuthorExt(id=1327017603677139490, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327017598799163887, authorId=1327017603542921758, language=CN, stringName=范意兴, firstName=null, middleName=null, lastName=null, prefix=null, suffix=null, authorComment=null, nameInitials=null, affiliation=null, department=null, xref=
1, 2, address=
1中国科学院网络数据科学与技术重点实验室(中国科学院计算技术研究所) 北京 100190
2中国科学院大学 北京 100049, bio={"img":"nwoUMSd061OVVHyP8RB8Dg==","content":"
1990年生. 博士. 副研究员. 主要研究方向为数据挖掘、信息检索
"}, bioImg=nwoUMSd061OVVHyP8RB8Dg==, bioContent=
1990年生. 博士. 副研究员. 主要研究方向为数据挖掘、信息检索
, aboutCorrespAuthor=null)}, companyList=[AuthorCompany(id=1327017602423042560, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327017598799163887, xref=1, ext=[AuthorCompanyExt(id=1327017602431431169, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327017598799163887, companyId=1327017602423042560, language=EN, country=null, province=null, city=null, postcode=null, companyName=null, departmentName=null, remark=
1CAS Key Laboratory of Network Data Science and Technology (Institute of Computing Technology, Chinese Academy of Sciences), Beijing 100190), AuthorCompanyExt(id=1327017602435625474, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327017598799163887, companyId=1327017602423042560, language=CN, country=null, province=null, city=null, postcode=null, companyName=null, departmentName=null, remark=
1中国科学院网络数据科学与技术重点实验室(中国科学院计算技术研究所) 北京 100190)]), AuthorCompany(id=1327017602540483075, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327017598799163887, xref=2, ext=[AuthorCompanyExt(id=1327017602553065988, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327017598799163887, companyId=1327017602540483075, language=EN, country=null, province=null, city=null, postcode=null, companyName=null, departmentName=null, remark=
2University of Chinese Academy of Sciences, Beijing 100049), AuthorCompanyExt(id=1327017602586620422, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327017598799163887, companyId=1327017602540483075, language=CN, country=null, province=null, city=null, postcode=null, companyName=null, departmentName=null, remark=
2中国科学院大学 北京 100049)])])], keywords=[Keyword(id=1327017603769414179, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327017598799163887, language=EN, orderNo=1, keyword=large language model), Keyword(id=1327017603832328740, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327017598799163887, language=EN, orderNo=2, keyword=model memorization capability), Keyword(id=1327017603899437605, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327017598799163887, language=EN, orderNo=3, keyword=training data copyright), Keyword(id=1327017603945574950, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327017598799163887, language=EN, orderNo=4, keyword=membership inference attack), Keyword(id=1327017604000100903, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327017598799163887, language=EN, orderNo=5, keyword=data contamination), Keyword(id=1327017605715571240, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327017598799163887, language=CN, orderNo=1, keyword=大语言模型), Keyword(id=1327017605786874409, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327017598799163887, language=CN, orderNo=2, keyword=模型记忆能力), Keyword(id=1327017605853983274, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327017598799163887, language=CN, orderNo=3, keyword=训练数据版权), Keyword(id=1327017605912703531, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327017598799163887, language=CN, orderNo=4, keyword=成员推理攻击), Keyword(id=1327017605975618092, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327017598799163887, language=CN, orderNo=5, keyword=数据污染)], refs=[Reference(id=1327017607212937787, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327017598799163887, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=1, rfOrder=0, authorNames=null, journalName=null, refType=null, unstructuredReference=Brown T B, Mann B, Ryder N, et al. Language models are few-shot learners[C]//Proc of the 34th Int Conf on Neural Information Processing Systems. Cambridge, MA: MIT, 2020: 1877−1901, articleTitle=null, refAbstract=null), Reference(id=1327017607288435260, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327017598799163887, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=2, rfOrder=1, authorNames=null, journalName=null, refType=null, unstructuredReference=Achiam J, Adler S, Agarwal S, et al. Gpt-4 technical report[J]. arXiv preprint, arXiv: 2303.08774, 2023, articleTitle=null, refAbstract=null), Reference(id=1327017607359738429, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327017598799163887, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=3, rfOrder=2, authorNames=null, journalName=null, refType=null, unstructuredReference=Touvron H, Martin L, Stone K, et al. Llama 2: Open foundation and fine-tuned chat models[J]. arXiv preprint, arXiv: 2307.09288, 2023, articleTitle=null, refAbstract=null), Reference(id=1327017607435235902, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327017598799163887, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=4, rfOrder=3, authorNames=null, journalName=null, refType=null, unstructuredReference=Bai Jinze, Bai Shuai, Chu Yunfei, et al. Qwen technical report[J]. arXiv preprint, arXiv: 2309.16609, 2023, articleTitle=null, refAbstract=null), Reference(id=1327017607502344767, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327017598799163887, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=5, rfOrder=4, authorNames=null, journalName=null, refType=null, unstructuredReference=Yang Aiyuan, Xiao Bin, Wang Bingning, et al. Baichuan 2: Open large-scale language models[J]. arXiv preprint, arXiv: 2309.10305, 2023, articleTitle=null, refAbstract=null), Reference(id=1327017607565259328, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327017598799163887, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=6, rfOrder=5, authorNames=null, journalName=null, refType=null, unstructuredReference=Mozes M, He Xuanli, Kleinberg B, et al. Use of LLMs for illicit purposes: Threats, prevention measures, and vulnerabilities[J]. arXiv preprint, arXiv: 2308.12833, 2023, articleTitle=null, refAbstract=null), Reference(id=1327017607632368193, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327017598799163887, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=7, rfOrder=6, authorNames=null, journalName=null, refType=null, unstructuredReference=Chang K K, Cramer M, Soni S, et al. Speak, memory: An archaeology of books known to ChatGPT/GPT-4[C]//Proc of the 2023 Conf on Empirical Methods in Natural Language Processing. Stroudsburg, PA: ACL, 2023: 7312−7327, articleTitle=null, refAbstract=null), Reference(id=1327017607720448578, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327017598799163887, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=8, rfOrder=7, authorNames=null, journalName=null, refType=null, unstructuredReference=Dong Yihong, Jiang Xue, Liu Huanyu, et al. Generalization or memorization: Data contamination and trustworthy evaluation for large language models[C]//Proc of the 62nd Annual Meeting of the Association for Computational Linguistics: Findings. Stroudsburg, PA: ACL, 2024: 12039−12050, articleTitle=null, refAbstract=null), Reference(id=1327017607787557443, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327017598799163887, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=9, rfOrder=8, authorNames=null, journalName=null, refType=null, unstructuredReference=Cao Jialun, Zhang Wuqi, Cheung S C. Concerned with data contamination? Assessing countermeasures in code language model[J]. arXiv preprint, arXiv: 2403.16898, 2024, articleTitle=null, refAbstract=null), Reference(id=1327017607951135300, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327017598799163887, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=10, rfOrder=9, authorNames=null, journalName=null, refType=null, unstructuredReference=Shokri R, Stronati M, Song Congzheng, et al. Membership inference attacks against machine learning models[C]//Proc of the 2017 IEEE Symp on Security and Privacy (SP). Piscataway, NJ: IEEE, 2017: 3−18, articleTitle=null, refAbstract=null), Reference(id=1327017608022438469, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327017598799163887, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=11, rfOrder=10, authorNames=null, journalName=null, refType=null, unstructuredReference=Shi Weijia, Ajith A, Xia Mengzhou, et al. Detecting pretraining data from large language models[C/OL]//Proc of the 12th Int Conf on Learning Representations. 2024[2025-03-02]. https://openreview.net/forum?id=PrCUrHJxTC, articleTitle=null, refAbstract=null), Reference(id=1327017608089547334, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327017598799163887, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=12, rfOrder=11, authorNames=null, journalName=null, refType=null, unstructuredReference=Duan M, Suri A, Mireshghallah N, et al. Do membership inference attacks work on large language models?[C/OL]//Proc of the 1st Conf on Language Modeling. 2024[2025-03-02]. https://openreview.net/forum?id=av0D19pSkU, articleTitle=null, refAbstract=null), Reference(id=1327017608160850503, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327017598799163887, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=13, rfOrder=12, authorNames=null, journalName=null, refType=null, unstructuredReference=Carlini N, Tramer F, Wallace E, et al. Extracting training data from large language models[C]//Proc of the 30th USENIX Security Symp. Berkeley, CA: USENIX Association, 2021: 2633−2650, articleTitle=null, refAbstract=null), Reference(id=1327017608219570760, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327017598799163887, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=14, rfOrder=13, authorNames=null, journalName=null, refType=null, unstructuredReference=Carlini N, Chien S, Nasr M, et al. Membership inference attacks from first principles[C]//Proc of the 2022 IEEE Symp on Security and Privacy (SP). Piscataway, NJ: IEEE, 2022: 1897−1914, articleTitle=null, refAbstract=null), Reference(id=1327017608290873929, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327017598799163887, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=15, rfOrder=14, authorNames=null, journalName=null, refType=null, unstructuredReference=Gao L, Biderman S, Black S, et al. The Pile: An 800 GB dataset of diverse text for language modeling[J]. arXiv preprint, arXiv: 2101.00027, 2020, articleTitle=null, refAbstract=null), Reference(id=1327017608353788490, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327017598799163887, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=16, rfOrder=15, authorNames=null, journalName=null, refType=null, unstructuredReference=Zhang Zhexin, Wen Jiaxin, Huang Minlie. Ethicist: Targeted training data extraction through loss smoothed softing and calibrated confidence estimation[C]//Proc of the 61st Annual Meeting of the Association for Computational Linguistics. Stroudsburg, PA: ACL, 2023: 12674−12687, articleTitle=null, refAbstract=null), Reference(id=1327017608483811915, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327017598799163887, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=17, rfOrder=16, authorNames=null, journalName=null, refType=null, unstructuredReference=Yeom S, Giacomelli I, Fredrikson M, et al. Privacy risk in machine learning: Analyzing the connection to overfitting[C]//Proc of the 31st Computer Security Foundations Symp (CSF). Piscataway, NJ: IEEE, 2018: 268−282, articleTitle=null, refAbstract=null), Reference(id=1327017610144756301, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327017598799163887, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=18, rfOrder=17, authorNames=null, journalName=null, refType=null, unstructuredReference=Sablayrolles A, Douze M, Schmid C, et al. White-box vs black-box: Bayes optimal strategies for membership inference[C]//Proc of the 36th Int Conf on Machine Learning. Cambridge, MA: JMLR, 2019: 5558−5567, articleTitle=null, refAbstract=null), Reference(id=1327017610249613902, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327017598799163887, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=19, rfOrder=18, authorNames=null, journalName=null, refType=null, unstructuredReference=Song Congzheng, Shmatikov V. Auditing data provenance in text-generation models[C]//Proc of the 25th ACM SIGKDD Int Conf on Knowledge Discovery & Data Mining. New York: ACM, 2019: 196−206, articleTitle=null, refAbstract=null), Reference(id=1327017610337694287, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327017598799163887, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=20, rfOrder=19, authorNames=null, journalName=null, refType=null, unstructuredReference=Hisamoto S, Post M, Duh K. Membership inference attacks on sequence-to-sequence models: Is my data in your machine translation system[J]. Transactions of the Association for Computational Linguistics, 2020, 8: 49−63, articleTitle=null, refAbstract=null), Reference(id=1327017610480300624, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327017598799163887, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=21, rfOrder=20, authorNames=null, journalName=null, refType=null, unstructuredReference=Jagannatha A, Rawat B P S, Yu Hong. Membership inference attack susceptibility of clinical language models[J]. arXiv preprint, arXiv: 2104.08305, 2021, articleTitle=null, refAbstract=null), Reference(id=1327017610555798097, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327017598799163887, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=22, rfOrder=21, authorNames=null, journalName=null, refType=null, unstructuredReference=Mattern J, Mireshghallah F, Jin Zhijing, et al. Membership inference attacks against language models via neighbourhood comparison[C]//Proc of the 61st Annual Meeting of the Association for Computational Linguistics: Findings. Stroudsburg, PA: ACL, 2023: 11330−11343, articleTitle=null, refAbstract=null), Reference(id=1327017610622906962, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327017598799163887, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=23, rfOrder=22, authorNames=null, journalName=null, refType=null, unstructuredReference=Nasr M, Carlini N, Hayase J, et al. Scalable extraction of training data from (production) language models[J]. arXiv preprint, arXiv: 2311.17035, 2023, articleTitle=null, refAbstract=null), Reference(id=1327017610685821523, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327017598799163887, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=24, rfOrder=23, authorNames=null, journalName=null, refType=null, unstructuredReference=Oren Y, Meister N, Chatterji N, et al. Proving test set contamination in black-box language models[C]//Proc of the 12th Int Conf on Learning Representations. 2024[2025-03-02]. https://openreview.net/forum?id=KS8mIvetg2, articleTitle=null, refAbstract=null), Reference(id=1327017610757124692, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327017598799163887, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=25, rfOrder=24, authorNames=null, journalName=null, refType=null, unstructuredReference=Zhang Weichao, Zhang Ruqing, Guo Jiafeng, et al. Pretraining data detection for large language models: A divergence-based calibration method[C]//Proc of the 2024 Conf on Empirical Methods in Natural Language Processing. Stroudsburg, PA: ACL, 2024: 5263−5274, articleTitle=null, refAbstract=null), Reference(id=1327017610828427861, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327017598799163887, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=26, rfOrder=25, authorNames=null, journalName=null, refType=null, unstructuredReference=Duarte A V, Zhao Xuandong, Oliveira A L, et al. De-cop: Detecting copyrighted content in language models training data[C]//Proc of the 41st Int Conf on Machine Learning. Cambridge, MA: JMLR, 2024: 11940−11956, articleTitle=null, refAbstract=null), Reference(id=1327017610924896854, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327017598799163887, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=27, rfOrder=26, authorNames=null, journalName=null, refType=null, unstructuredReference=Liu Zhenhua, Zhu Tong, Tan Chuanyuan, et al. Probing language models for pre-training data detection[C]//Proc of the 62nd Annual Meeting of the Association for Computational Linguistics. Stroudsburg, PA: ACL, 2024: 1576−1587, articleTitle=null, refAbstract=null), Reference(id=1327017611017171543, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327017598799163887, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=28, rfOrder=27, authorNames=null, journalName=null, refType=null, unstructuredReference=Radford A, Wu J, Child R, et al. Language models are unsupervised multitask learners[J]. OpenAI Blog, 2019, 1(8): 1−24, articleTitle=null, refAbstract=null), Reference(id=1327017611088474712, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327017598799163887, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=29, rfOrder=28, authorNames=null, journalName=null, refType=null, unstructuredReference=Bao Guangsheng, Zhao Yanbin, Teng Zhiyang, et al. Fast-DetectGPT: Efficient zero-shot detection of machine-generated text via conditional probability curvature[C]//Proc of the 12th Int Conf on Learning Representations. 2024[2025-03-02]. https://openreview.net/forum?id=Bpcgcr8E8Z, articleTitle=null, refAbstract=null), Reference(id=1327017611155583577, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327017598799163887, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=30, rfOrder=29, authorNames=null, journalName=null, refType=null, unstructuredReference=Touvron H, Lavril T, Izacard G, et al. Llama: Open and efficient foundation language models[J]. arXiv preprint, arXiv: 2302.13971, 2023, articleTitle=null, refAbstract=null), Reference(id=1327017611235275354, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327017598799163887, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=31, rfOrder=30, authorNames=null, journalName=null, refType=null, unstructuredReference=Black S, Biderman S, Hallahan E, et al. GPT-NeoX−20b: An open-source autoregressive language model[C]//Proc of BigScience Episode. Stroudsburg, PA: ACL, 2022: 95−136, articleTitle=null, refAbstract=null), Reference(id=1327017611327550043, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327017598799163887, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=32, rfOrder=31, authorNames=null, journalName=null, refType=null, unstructuredReference=Zhang Susan, Roller S, Goyal N, et al. OPT: Open pre-trained transformer language models[J]. arXiv preprint, arXiv: 2205.01068, 2022, articleTitle=null, refAbstract=null), Reference(id=1327017611394658908, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327017598799163887, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=33, rfOrder=32, authorNames=null, journalName=null, refType=null, unstructuredReference=Biderman S, Schoelkopf H, Anthony Q G, et al. Pythia: A suite for analyzing large language models across training and scaling[C]//Proc of the 40th Int Conf on Machine Learning. Cambridge, MA: JMLR, 2023: 2397−2430, articleTitle=null, refAbstract=null)], funds=null, companyList=[AuthorCompany(id=1327017602423042560, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327017598799163887, xref=1, ext=[AuthorCompanyExt(id=1327017602431431169, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327017598799163887, companyId=1327017602423042560, language=EN, country=null, province=null, city=null, postcode=null, companyName=null, departmentName=null, remark=
1CAS Key Laboratory of Network Data Science and Technology (Institute of Computing Technology, Chinese Academy of Sciences), Beijing 100190), AuthorCompanyExt(id=1327017602435625474, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327017598799163887, companyId=1327017602423042560, language=CN, country=null, province=null, city=null, postcode=null, companyName=null, departmentName=null, remark=
1中国科学院网络数据科学与技术重点实验室(中国科学院计算技术研究所) 北京 100190)]), AuthorCompany(id=1327017602540483075, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327017598799163887, xref=2, ext=[AuthorCompanyExt(id=1327017602553065988, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327017598799163887, companyId=1327017602540483075, language=EN, country=null, province=null, city=null, postcode=null, companyName=null, departmentName=null, remark=
2University of Chinese Academy of Sciences, Beijing 100049), AuthorCompanyExt(id=1327017602586620422, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327017598799163887, companyId=1327017602540483075, language=CN, country=null, province=null, city=null, postcode=null, companyName=null, departmentName=null, remark=
2中国科学院大学 北京 100049)]), AuthorCompany(id=1327017602657923591, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327017598799163887, xref=3, ext=[AuthorCompanyExt(id=1327017602666312200, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327017598799163887, companyId=1327017602657923591, language=EN, country=null, province=null, city=null, postcode=null, companyName=null, departmentName=null, remark=
3Zhongguancun Laboratory, Beijing 100094), AuthorCompanyExt(id=1327017602670506505, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327017598799163887, companyId=1327017602657923591, language=CN, country=null, province=null, city=null, postcode=null, companyName=null, departmentName=null, remark=
3中关村实验室 北京 100094)])], figs=[ArticleFig(id=1327017606084669997, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327017598799163887, language=EN, label=Figure 1, caption=
An illustrative example to show the contribution of long-range context to token probability increase, figureFileSmall=il1SZhyUBE9fW19xfYh4rQ==, figureFileBig=nWwEVo/aKytnReucNIpXLQ==, tableContent=null), ArticleFig(id=1327017606155973166, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327017598799163887, language=CN, label=图1, caption=
长距离上下文对词元概率提升的贡献度的样例展示, figureFileSmall=il1SZhyUBE9fW19xfYh4rQ==, figureFileBig=nWwEVo/aKytnReucNIpXLQ==, tableContent=null), ArticleFig(id=1327017606315356719, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327017598799163887, language=EN, label=Figure 2, caption=
The impact of model size on detection effect, figureFileSmall=pjZCbpl9bPD7ya4jLblvaA==, figureFileBig=C5q7DRSQyI7kDquBaiNi9g==, tableContent=null), ArticleFig(id=1327017606374076976, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327017598799163887, language=CN, label=图2, caption=
模型大小对检测效果的影响, figureFileSmall=pjZCbpl9bPD7ya4jLblvaA==, figureFileBig=C5q7DRSQyI7kDquBaiNi9g==, tableContent=null), ArticleFig(id=1327017606445380145, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327017598799163887, language=EN, label=Figure 3, caption=
The impact of text length on detection effect, figureFileSmall=uAPbWLHwuNCfoRbKvjW20w==, figureFileBig=6tIRMcrU+WmJ8XlpcNk50g==, tableContent=null), ArticleFig(id=1327017606504100402, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327017598799163887, language=CN, label=图3, caption=
文本长度对检测效果的影响, figureFileSmall=uAPbWLHwuNCfoRbKvjW20w==, figureFileBig=6tIRMcrU+WmJ8XlpcNk50g==, tableContent=null), ArticleFig(id=1327017606583792179, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327017598799163887, language=EN, label=Table 1, caption=
Statistics for WikiMIA Dataset, figureFileSmall=null, figureFileBig=null, tableContent=
| 数据来源 | 文本长度 | 数据集规模 |
|---|
| 注:文本长度是指一段文本的单词数量,数据集规模是指数据的条数. |
| WikiPedia | 32 | 776 |
| 64 | 542 |
| 128 | 250 |
| 256 | 82 |
), ArticleFig(id=1327017606659289652, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327017598799163887, language=CN, label=表1, caption=
WikiMIA数据集的统计信息
, figureFileSmall=null, figureFileBig=null, tableContent=
| 数据来源 | 文本长度 | 数据集规模 |
|---|
| 注:文本长度是指一段文本的单词数量,数据集规模是指数据的条数. |
| WikiPedia | 32 | 776 |
| 64 | 542 |
| 128 | 250 |
| 256 | 82 |
), ArticleFig(id=1327017606722204213, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327017598799163887, language=EN, label=Table 2, caption=
Statistics for MIMIR Dataset, figureFileSmall=null, figureFileBig=null, tableContent=
| 数据来源 | 版本 | 数据集规模 | 训练与非训练文本重叠程度 |
|---|
| 注:数据集规模是指数据的条数. |
Pile- WikiPedia | 7_gram_0.2 | 2 000 | 满足7-gram,重叠率小于等于0.2 |
| 13_gram_0.2 | 2 000 | 满足13-gram,重叠率小于等于0.2 |
| 13_gram_0.8 | 2 000 | 满足13-gram,重叠率小于等于0.8 |
), ArticleFig(id=1327017606797701686, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327017598799163887, language=CN, label=表2, caption=
MIMIR数据集的统计信息
, figureFileSmall=null, figureFileBig=null, tableContent=
| 数据来源 | 版本 | 数据集规模 | 训练与非训练文本重叠程度 |
|---|
| 注:数据集规模是指数据的条数. |
Pile- WikiPedia | 7_gram_0.2 | 2 000 | 满足7-gram,重叠率小于等于0.2 |
| 13_gram_0.2 | 2 000 | 满足13-gram,重叠率小于等于0.2 |
| 13_gram_0.8 | 2 000 | 满足13-gram,重叠率小于等于0.8 |
), ArticleFig(id=1327017606856421943, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327017598799163887, language=EN, label=Table 3, caption=
Experimental Results Comparison on WikiMIA Dataset Detection Under Various Large Language Models, figureFileSmall=null, figureFileBig=null, tableContent=
| 检测方法 | Pythia-2.8B | | OPT-6.7B | | LLaMA-13B | | GPT-NeoX-20B |
|---|
| AUC | TPR@5%FPR | | AUC | TPR@5%FPR | | AUC | TPR@5%FPR | | AUC | TPR@5%FPR |
|---|
| 注:黑体数值表示最优结果,“*”表示本文方法在双尾T测试下相较于现有方法有显著性提升. |
| PPL[13,17] | 0.628 | 0.094 | | 0.625 | 0.130 | | 0.678 | 0.216 | | 0.707 | 0.180 |
| Small Ref[13] | 0.626 | 0.137 | | 0.654 | 0.122 | | 0.658 | 0.151 | | 0.714 | 0.187 |
| Lowercase[13] | 0.571 | 0.072 | | 0.587 | 0.094 | | 0.606 | 0.158 | | 0.680 | 0.130 |
| Zlib[13] | 0.650 | 0.187 | | 0.644 | 0.180 | | 0.697 | 0.187 | | 0.723 | 0.223 |
| Min-K% Prob[11] | 0.660 | 0.137 | | 0.674 | 0.166 | | 0.715 | 0.201 | | 0.756 | 0.216 |
| 本文方法 | 0.683* | 0.201* | | 0.722* | 0.245* | | 0.784* | 0.324* | | 0.758 | 0.223* |
), ArticleFig(id=1327017606923530808, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327017598799163887, language=CN, label=表3, caption=
在多种大语言模型下进行WikiMIA数据集检测的实验结果对比
, figureFileSmall=null, figureFileBig=null, tableContent=
| 检测方法 | Pythia-2.8B | | OPT-6.7B | | LLaMA-13B | | GPT-NeoX-20B |
|---|
| AUC | TPR@5%FPR | | AUC | TPR@5%FPR | | AUC | TPR@5%FPR | | AUC | TPR@5%FPR |
|---|
| 注:黑体数值表示最优结果,“*”表示本文方法在双尾T测试下相较于现有方法有显著性提升. |
| PPL[13,17] | 0.628 | 0.094 | | 0.625 | 0.130 | | 0.678 | 0.216 | | 0.707 | 0.180 |
| Small Ref[13] | 0.626 | 0.137 | | 0.654 | 0.122 | | 0.658 | 0.151 | | 0.714 | 0.187 |
| Lowercase[13] | 0.571 | 0.072 | | 0.587 | 0.094 | | 0.606 | 0.158 | | 0.680 | 0.130 |
| Zlib[13] | 0.650 | 0.187 | | 0.644 | 0.180 | | 0.697 | 0.187 | | 0.723 | 0.223 |
| Min-K% Prob[11] | 0.660 | 0.137 | | 0.674 | 0.166 | | 0.715 | 0.201 | | 0.756 | 0.216 |
| 本文方法 | 0.683* | 0.201* | | 0.722* | 0.245* | | 0.784* | 0.324* | | 0.758 | 0.223* |
), ArticleFig(id=1327017607040971321, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327017598799163887, language=EN, label=Table 4, caption=
Experimental Results Comparison on MIMIR Dataset Detection Under Pythia-2.8B Model, figureFileSmall=null, figureFileBig=null, tableContent=
| 检测方法 | 7_gram_0.2 | | 13_gram_0.2 | | 13_gram_0.8 |
|---|
| AUC | TPR@ 5%FPR | | AUC | TPR@ 5%FPR | | AUC | TPR@ 5%FPR |
|---|
| 注:黑体数值表示最优结果,带下划线的数值是从现有工作中引用. |
| PPL[13,17] | 0.654 | 0.201 | | 0.540 | 0.106 | | 0.521 | 0.042 |
| Small Ref[13] | 0.677 | 0.236 | | | | | 0.530 | 0.060 |
| Zlib[13] | 0.652 | 0.208 | | | | | 0.536 | 0.064 |
| Min-K% Prob[11] | 0.679 | 0.250 | | 0.560 | 0.116 | | 0.526 | 0.044 |
| 本文方法 | 0.665 | 0.227 | | 0.591 | 0.124 | | 0.583 | 0.095 |
), ArticleFig(id=1327017607103885882, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327017598799163887, language=CN, label=表4, caption=
在Pythia-2.8B模型下进行MIMIR数据集检测的实验结果对比
, figureFileSmall=null, figureFileBig=null, tableContent=
| 检测方法 | 7_gram_0.2 | | 13_gram_0.2 | | 13_gram_0.8 |
|---|
| AUC | TPR@ 5%FPR | | AUC | TPR@ 5%FPR | | AUC | TPR@ 5%FPR |
|---|
| 注:黑体数值表示最优结果,带下划线的数值是从现有工作中引用. |
| PPL[13,17] | 0.654 | 0.201 | | 0.540 | 0.106 | | 0.521 | 0.042 |
| Small Ref[13] | 0.677 | 0.236 | | | | | 0.530 | 0.060 |
| Zlib[13] | 0.652 | 0.208 | | | | | 0.536 | 0.064 |
| Min-K% Prob[11] | 0.679 | 0.250 | | 0.560 | 0.116 | | 0.526 | 0.044 |
| 本文方法 | 0.665 | 0.227 | | 0.591 | 0.124 | | 0.583 | 0.095 |
)], attaches=null, journal=Journal(id=1324759606947315908, delFlag=0, nameCn=计算机研究与发展, nameEn=Journal of Computer Research and Development, nameHistory1=null, nameHistory2=null, issn=1000-1239, eissn=null, cn=11-1777/TP, coden=null, periodic=0, language=CN, oaType=null, ccby=null, superviseOffice=null, ownerOffice=null, pubOffice=null, editorOffice=null, officeType=null, aims=null, clcCode=null, officeProv=null, officeCity=null, officeAddr=null, officeZip=null, officeEmail=null, officePhone=null, editDirector=null, officeDirector=null, officeDirectorPhone=null, officeStaffNum=null, officeEmpNum=null, coverPicUrl=k2l8zgqwstuxxevFX2qxVA==, journalPrice=null, startedYear=null, abbrevIsoEn=Journal of Computer Research and Development, journalRemark=null, publicationField=null, createdTime=1793776903141, updatedTime=1793850698612, createdBy=18614031015, updatedBy=13701087609, firstLetterCn=J, firstLetterEn=J, subjectCode=Electronic Information, subjectName=null, subjectCodeEn=Electronic Information, subjectNameEn=null, picCn=k2l8zgqwstuxxevFX2qxVA==, picEn=r6k0Ry+RD+c/VXj5QarmDw==, jcr=null, cjcr=null, exts=[JournalExt(id=1325069127710286688, language=CN, name=计算机研究与发展, nameHistory1=null, nameHistory2=null, managedBy=, sponsoredBy=, publishedBy=, editorOffice=, officeProv=null, officeCity=null, officeAddr=, officeZip=, editDirector=, officeDirector=null, officePhone=null, coverPicUrl=null, journalRemark=, submitArticleUrl=null, websiteUrl=, createdTime=1793850698640, updatedTime=1793850698640, createdBy=13701087609, updatedBy=13701087609, submissionGuidelinesUrl=, submissionAuthorUrl=https://crad.ict.ac.cn/journalx/authorLogOn.action, submissionEditorUrl=https://crad.ict.ac.cn/journalx/editorLogOn.action, submissionReviewUrl=https://crad.ict.ac.cn/journalx/expertLogOn.action, submissionCeEditorUrl=https://crad.ict.ac.cn/journalx/editorInChiefLogOn.action, submissionAeEditorUrl=, option={"copyright":""}), JournalExt(id=1325069127777395553, language=EN, name=Journal of Computer Research and Development, nameHistory1=null, nameHistory2=null, managedBy=, sponsoredBy=, publishedBy=, editorOffice=, officeProv=null, officeCity=null, officeAddr=, officeZip=, editDirector=, officeDirector=null, officePhone=null, coverPicUrl=null, journalRemark=, submitArticleUrl=null, websiteUrl=, createdTime=1793850698656, updatedTime=1793850698656, createdBy=13701087609, updatedBy=13701087609, submissionGuidelinesUrl=, submissionAuthorUrl=https://crad.ict.ac.cn/journalx/authorLogOn.action, submissionEditorUrl=https://crad.ict.ac.cn/journalx/editorLogOn.action, submissionReviewUrl=https://crad.ict.ac.cn/journalx/expertLogOn.action, submissionCeEditorUrl=https://crad.ict.ac.cn/journalx/editorInChiefLogOn.action, submissionAeEditorUrl=, option={"copyright":""})], databaseList=null, tenantJournalId=1324759914146529478, websiteList=[Website(id=1325070300139892771, webName=null, webTitle=null, webDomain=null, webCopyrigh=null, webIpcNo=null, seoTitle=null, seoKeywords=null, seoDescription=null, tenantJournalId=null, journalId=1324759914146529478, journalNameCn=null, journalNameEn=null, grayFlag=null, tenantId=1146029695717560320, platformId=null, journalGroupId=null, journalGroupNameCn=null, journalGroupNameEn=null, type=1, domain=https://castjournals.cast.org.cn/joweb/jsjyjyfz/CN, language=CN, createTime=1793850978169, createBy=18614031015, updateTime=1793851040259, updateBy=18614031015, name=计算机研究与发展-中文, tplId=1146099689490845704, title=计算机研究与发展, delFlag=0, indexPage=/home, props=[WebsiteProps(id=1325070893751349360, tenantId=1146029695717560320, journalId=null, journalGroupId=null, siteId=1325070300139892771, code=articleTextType, value=kx, createTime=1793851119697, updateTime=1793851119697, creator=18614031015, updator=18614031015), WebsiteProps(id=1325070893730377837, tenantId=1146029695717560320, journalId=null, journalGroupId=null, siteId=1325070300139892771, code=banner, value=null, createTime=1793851119692, updateTime=1793851119692, creator=18614031015, updator=18614031015), WebsiteProps(id=1325070893772320883, tenantId=1146029695717560320, journalId=null, journalGroupId=null, siteId=1325070300139892771, code=grayFlag, value=0, createTime=1793851119702, updateTime=1793851119702, creator=18614031015, updator=18614031015), WebsiteProps(id=1325070893721989228, tenantId=1146029695717560320, journalId=null, journalGroupId=null, siteId=1325070300139892771, code=logo, value=https://castjournals.cast.org.cn/joweb/jsjyjyfz/CN/file/pic?fileId=U27tDsy717YF7XSPA+Vwdw==, createTime=1793851119690, updateTime=1793851119690, creator=18614031015, updator=18614031015), WebsiteProps(id=1325070893784903797, tenantId=1146029695717560320, journalId=null, journalGroupId=null, siteId=1325070300139892771, code=minRunFlag, value=0, createTime=1793851119705, updateTime=1793851119705, creator=18614031015, updator=18614031015), WebsiteProps(id=1325070893742960751, tenantId=1146029695717560320, journalId=null, journalGroupId=null, siteId=1325070300139892771, code=picServerUrl, value=https://castjournals.cast.org.cn/joweb/jsjyjyfz/CN/file/pic, createTime=1793851119695, updateTime=1793851119695, creator=18614031015, updator=18614031015), WebsiteProps(id=1325070893780709492, tenantId=1146029695717560320, journalId=null, journalGroupId=null, siteId=1325070300139892771, code=silenceFlag, value=0, createTime=1793851119704, updateTime=1793851119704, creator=18614031015, updator=18614031015), WebsiteProps(id=1325070893738766446, tenantId=1146029695717560320, journalId=null, journalGroupId=null, siteId=1325070300139892771, code=staticResourcePath, value=https://castjournals.cast.org.cn/joweb/cast_kjdb_cn_619/, createTime=1793851119694, updateTime=1793851119694, creator=18614031015, updator=18614031015), WebsiteProps(id=1325070893755543665, tenantId=1146029695717560320, journalId=null, journalGroupId=null, siteId=1325070300139892771, code=themeColor, value=null, createTime=1793851119698, updateTime=1793851119698, creator=18614031015, updator=18614031015), WebsiteProps(id=1325070893763932274, tenantId=1146029695717560320, journalId=null, journalGroupId=null, siteId=1325070300139892771, code=themeStyle, value=null, createTime=1793851119700, updateTime=1793851119700, creator=18614031015, updator=18614031015)]), Website(id=1325070300580294698, webName=null, webTitle=null, webDomain=null, webCopyrigh=null, webIpcNo=null, seoTitle=null, seoKeywords=null, seoDescription=null, tenantJournalId=null, journalId=1324759914146529478, journalNameCn=null, journalNameEn=null, grayFlag=null, tenantId=1146029695717560320, platformId=null, journalGroupId=null, journalGroupNameCn=null, journalGroupNameEn=null, type=1, domain=https://castjournals.cast.org.cn/joweb/jsjyjyfz/EN, language=EN, createTime=1793850978274, createBy=18614031015, updateTime=1793851036638, updateBy=18614031015, name=计算机研究与发展-英文, tplId=1146101810881728533, title=Journal of Computer Research and Development, delFlag=0, indexPage=/home, props=[WebsiteProps(id=1325070925288321148, tenantId=1146029695717560320, journalId=null, journalGroupId=null, siteId=1325070300580294698, code=articleTextType, value=kx, createTime=1793851127216, updateTime=1793851127216, creator=18614031015, updator=18614031015), WebsiteProps(id=1325070925263155321, tenantId=1146029695717560320, journalId=null, journalGroupId=null, siteId=1325070300580294698, code=banner, value=null, createTime=1793851127210, updateTime=1793851127210, creator=18614031015, updator=18614031015), WebsiteProps(id=1325070925309292671, tenantId=1146029695717560320, journalId=null, journalGroupId=null, siteId=1325070300580294698, code=grayFlag, value=0, createTime=1793851127221, updateTime=1793851127221, creator=18614031015, updator=18614031015), WebsiteProps(id=1325070925254766712, tenantId=1146029695717560320, journalId=null, journalGroupId=null, siteId=1325070300580294698, code=logo, value=https://castjournals.cast.org.cn/joweb/jsjyjyfz/EN/file/pic?fileId=U27tDsy717YF7XSPA+Vwdw==, createTime=1793851127208, updateTime=1793851127208, creator=18614031015, updator=18614031015), WebsiteProps(id=1325070925326069889, tenantId=1146029695717560320, journalId=null, journalGroupId=null, siteId=1325070300580294698, code=minRunFlag, value=0, createTime=1793851127225, updateTime=1793851127225, creator=18614031015, updator=18614031015), WebsiteProps(id=1325070925279932539, tenantId=1146029695717560320, journalId=null, journalGroupId=null, siteId=1325070300580294698, code=picServerUrl, value=https://castjournals.cast.org.cn/joweb/jsjyjyfz/EN/file/pic, createTime=1793851127214, updateTime=1793851127214, creator=18614031015, updator=18614031015), WebsiteProps(id=1325070925317681280, tenantId=1146029695717560320, journalId=null, journalGroupId=null, siteId=1325070300580294698, code=silenceFlag, value=0, createTime=1793851127223, updateTime=1793851127223, creator=18614031015, updator=18614031015), WebsiteProps(id=1325070925271543930, tenantId=1146029695717560320, journalId=null, journalGroupId=null, siteId=1325070300580294698, code=staticResourcePath, value=https://castjournals.cast.org.cn/joweb/cast_kjdb_en_623/, createTime=1793851127212, updateTime=1793851127212, creator=18614031015, updator=18614031015), WebsiteProps(id=1325070925292515453, tenantId=1146029695717560320, journalId=null, journalGroupId=null, siteId=1325070300580294698, code=themeColor, value=null, createTime=1793851127217, updateTime=1793851127217, creator=18614031015, updator=18614031015), WebsiteProps(id=1325070925300904062, tenantId=1146029695717560320, journalId=null, journalGroupId=null, siteId=1325070300580294698, code=themeStyle, value=null, createTime=1793851127219, updateTime=1793851127219, creator=18614031015, updator=18614031015)])], journalTitle=计算机研究与发展, weixinUrl=null, journalUrl=https://crad.ict.ac.cn/, iacademicId=null, status=1, seqNo=null, journalTitleEn=Journal of Computer Research and Development, journalPhotoCn=k2l8zgqwstuxxevFX2qxVA==, journalPhotoEn=r6k0Ry+RD+c/VXj5QarmDw==, journalFirstLetter=J, journalRecommend=null, journalNew=null, journalCollection=null, jcrJf=null, cjcrJf=null, jcrJfStr=null, cjcrJfStr=null, submissionFirstDecision=null, sciSubjectClassification=null, casSubjectClassification=null, citeScore=null, totalCitationFrequency=null, icpCode=null, psCode=null, advertisingLicenseCode=null, copyrightInformation=null, country=null, option=, provinceCode=null, provinceName=null, collectFlag=false, interPubPlatform=, interPubPlatformUrl=null), detailUrlCn=https://castjournals.cast.org.cn/joweb/jsjyjyfz/CN/10.7544/issn1000-1239.202440780, detailUrlEn=https://castjournals.cast.org.cn/joweb/jsjyjyfz/EN/10.7544/issn1000-1239.202440780, pdfUrlCn=https://castjournals.cast.org.cn/joweb/jsjyjyfz/CN/PDF/10.7544/issn1000-1239.202440780, pdfUrlEn=https://castjournals.cast.org.cn/joweb/jsjyjyfz/EN/PDF/10.7544/issn1000-1239.202440780, aliStartDate=0, aliEndDate=0, collectionFlag=false, citedCount=null, citedUrl=null, previewStatus=0, delFlag=0, hasFullText=1, orderTime=1764518400000, fullTextJson=null, articleText=null, reference=null)