Article(id=1327017570688938237, tenantId=1146029695717560320, journalId=1324759914146529478, issueId=1327017337032651028, articleNumber=null, orderNo=null, doi=10.7544/issn1000-1239.202440444, pmid=null, cstr=null, oa=null, hot=null, price=null, onlineType=0, articleFormat=0, articleType=null, articleTypeStr=research-article, receivedDate=1717516800000, receivedDateStr=2024-06-05, revisedDate=1744128000000, revisedDateStr=2025-04-09, acceptedDate=null, acceptedDateStr=null, onlineDate=1794315243646, onlineDateStr=2026-11-10, pubDate=1764518400000, pubDateStr=2025-12-01, doiRegisterDate=null, doiRegisterDateStr=null, onlineIssueDate=1794315243646, onlineIssueDateStr=2026-11-10, onlineJustAcceptDate=null, onlineJustAcceptDateStr=null, onlineFirstDate=null, onlineFirstDateStr=null, sourceXml=null, magXml=null, createTime=1794315243646, creator=13701087609, updateTime=1794315243646, updator=13701087609, issue=Issue{id=1327017337032651028, tenantId=1146029695717560320, journalId=1324759914146529478, year='2025', volume='62', issue='12', pageStart='2889', pageEnd='3146', issueExtLink='null', onlineDate='null', pubDate='1764518400000', pubDateStr='2025-12-01', beforeIssueId=null, nextIssueId=null, price=null, status=1, issueComplete=1, articleOrder=1, issueType=-1, specialIssue=null, createTime=1794315187938, creator='13701087609', updateTime=1794377025235, updator='13701087609', preIssue=null, nextIssue=null, articleTotal=null, ext={EN=IssueExt(id=1327276701995872766, tenantId=1146029695717560320, journalId=1324759914146529478, issueId=1327017337032651028, language=EN, specialIssueTitle=, coverIllustrator=null, specialIssueEditor=, specialIssueAbout=), CN=IssueExt(id=1327276701995872767, tenantId=1146029695717560320, journalId=1324759914146529478, issueId=1327017337032651028, language=CN, specialIssueTitle=, coverIllustrator=null, specialIssueEditor=, specialIssueAbout=)}, issueFiles=null, downloadFileDto=null}, startPage=2929, endPage=2950, ext={EN=ArticleExt(id=1327017570877681918, articleId=1327017570688938237, tenantId=1146029695717560320, journalId=1324759914146529478, language=EN, title=Survey of Hallucinations in Large Vision-Language Models: Causes, Evaluations and Mitigations, columnId=1325489891789910943, journalTitle=Journal of Computer Research and Development, columnName=Artificial Intelligence, runingTitle=null, highlight=null, articleAbstract=
LVLMs (large vision-language models) represent a significant advancement in the intersection of natural language processing and computer vision. By integrating pre-trained visual encoders, vision-language adapters, and large language models, LVLMs can understand both visual and textual information, and generate responses in natural language, making them suitable for a range of downstream vision-language tasks such as image captioning and visual question answering. However, these models commonly exhibit hallucinations — generating inaccurate perceptions of image contents. Such hallucinations significantly limit the application of LVLMs in high-stakes domains like medical image diagnosis and autonomous driving. This survey aims to systematically organize and analyze the causes, evaluations, and mitigation strategies of hallucinations to guide research in the field and enhance the safety and reliability of LVLMs in practical applications. It begins with an introduction to the basic concepts of LVLMs and the definition and classification of hallucinations within them. It then explores the causes of hallucinations from four perspectives: training data, training task, visual encoding, and text generation, while also discussing the interactions among these factors. Following this, it discusses mainstream benchmarks for assessing LVLM hallucinations in terms of task setting, data construction, and assessment metrics. Additionally, it examines hallucination mitigating techniques across five aspects: training data, visual perception, training strategy, model inference, and post-hoc corrections. Finally, the review provides directions for future research in the areas of cause analysis, evaluation, and mitigation of hallucinations in LVLMs.
, authors=Xu Li
1, Rui Zhu
1, Xiaolei Chen
1, Jinxuan Wu
2, Yi Zheng
2, Chenghang Lai
1, Yuxuan Liang
2, Bin Li
2, Xiangyang Xue
1, 2, *, authorsList=Xu Li, Rui Zhu, Xiaolei Chen, Jinxuan Wu, Yi Zheng, Chenghang Lai, Yuxuan Liang, Bin Li, Xiangyang Xue, authorCompany=null, correspAuthors=Xiangyang Xue, authorNote=
born in 1998. PhD candidate. His main research interests include multimodal information processing, multimodal large language models, and large vision-language models
born in 2002. PhD candidate. Her main research interests include computer vision and large vision-language models
born in 1999. PhD candidate. His main research interests include computer vision and machine learning
born in 2001. Master candidate. Her main research interests include large multimodal models, and algorithmic fairness and safety
born in 1998. Master candidate. His main research interests include computer vision and large vision language models
born in 1994. PhD candidate. His main research interests include deep learning, machine learning, and multimodal knowledge extraction and reasoning
born in 2001. Master candidate. His main research interests include artificial intelligence and large vision language models
born in 1982. PhD, professor, PhD supervisor. His main research interests include machine learning and vision intelligence
born in 1968. PhD, professor, PhD supervisor. His main research interests include computer vision, multimedia, and machine learning
, correspAuthorsNote=null, copyrightStatement=Copyright © 2025 Journal of Computer Research and Development. All rights reserved., copyrightOwner=null, extLink=null, articleAbsUrl=null, sourceXml=null, magXml=null, pdfUrl=null, pdf=null, pdfFileSize=null, pdfExtLink=null, richHtmlUrl=null, mobilePdfUrl=null, reviewReport=null, pdfFirstPage=null, abstractGraph=null, abstractGraphContent=null, abstractVideo=null, citation=null, cebUrl=null, magXmlContent=null, mapNumber=null, fund=null), CN=ArticleExt(id=1327017574862270746, articleId=1327017570688938237, tenantId=1146029695717560320, journalId=1324759914146529478, language=CN, title=视觉语言大模型的幻觉综述:成因、评估与治理, columnId=1325489892104483745, journalTitle=计算机研究与发展, columnName=人工智能, runingTitle=null, highlight=null, articleAbstract=
视觉语言大模型(large vision-language models,LVLMs)代表了自然语言处理与计算机视觉交叉领域的一项重要进展. 通过结合预训练的视觉编码器、视觉语言适配器和大语言模型,LVLMs能够同时理解图像与文本信息,并通过自然语言进行响应,适用于图像描述、视觉问答等多种视觉语言下游任务. 然而,这类模型普遍存在幻觉现象,即模型对于图像内容进行了错误感知,制约了其在医学图像诊断、自动驾驶等高风险领域的赋能应用. 旨在系统梳理并深入分析幻觉成因、评估方法及治理策略,为LVLMs的可靠性研究提供指导. 首先,介绍LVLMs的基础概念及其幻觉现象的定义与分类;随后,从训练数据、训练任务、视觉编码、文本生成4方面分析LVLMs的幻觉成因,并讨论这些成因间的交互关系;接着,从任务形式、数据构建和评估指标3方面介绍LVLMs的幻觉评估策略;此外,从训练数据、视觉感知、训练策略、模型推理、事后修正5方面讨论LVLMs的幻觉治理技术;最后,为这类幻觉的成因分析、评估和治理3方面提供未来的研究方向.
, authors=李煦
1, 朱睿
1, 陈小磊
1, 伍瑾轩
2, 郑毅
2, 赖承杭
1, 梁宇轩
2, 李斌
2, 薛向阳
1, 2, *, authorsList=李煦, 朱睿, 陈小磊, 伍瑾轩, 郑毅, 赖承杭, 梁宇轩, 李斌, 薛向阳, authorCompany=null, correspAuthors=薛向阳, authorNote=
 |
1998年生. 博士研究生. 主要研究方向为多模态信息处理、多模态大语言模型、视觉语言大模型 |
 |
2002年生. 博士研究生. 主要研究方向为计算机视觉、视觉语言大模型 |
 |
1999年生. 博士研究生. 主要研究方向为计算机视觉、机器学习 |
 |
2001年生. 硕士研究生. 主要研究方向为多模态大模型、算法公平与安全性 |
 |
1998年生. 硕士研究生. 主要研究方向为计算机视觉、视觉语言大模型 |
 |
1994年生. 博士研究生. 主要研究方向为深度学习、机器学习、多模态知识提取与推理 |
 |
2001年生. 硕士研究生. 主要研究方向为人工智能、视觉语言大模型 |
 |
1982年生. 博士,教授,博士生导师. 主要研究方向为机器学习、视觉智能 |
 |
1968年生. 博士,教授,博士生导师. 主要研究方向为计算机视觉、多媒体、机器学习 |
, correspAuthorsNote=
, copyrightStatement=版权所有 © 2025 计算机研究与发展编辑部, copyrightOwner=null, extLink=null, articleAbsUrl=null, sourceXml=YhuTpOktTKFvqQkWROxIVQ==, magXml=V4mkjXF0+1haOvG7XoUktA==, pdfUrl=null, pdf=1l6gkCFPC1+iw7oRPhdDOw==, pdfFileSize=1814249, pdfExtLink=null, richHtmlUrl=null, mobilePdfUrl=null, reviewReport=null, pdfFirstPage=null, abstractGraph=QH2mWQagSxaHYBiUAfi5kg==, abstractGraphContent=null, abstractVideo=null, citation=null, cebUrl=null, magXmlContent=HLkm5W2PZqLciJO1UDBawg==, mapNumber=null, fund=null)}, authors=[Author(id=1327017575264923938, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327017570688938237, orderNo=0, firstName=null, middleName=null, lastName=null, nameCn=null, orcid=null, stid=null, country=null, authorPic=null, dead=0, email=jerryryrylx@gmail.com, emailSecond=null, emailThird=null, correspondingAuthor=0, authorType=1, ext={EN=AuthorExt(id=1327017575348810020, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327017570688938237, authorId=1327017575264923938, language=EN, stringName=Xu Li, firstName=Xu, middleName=null, lastName=Li, prefix=null, suffix=null, authorComment=null, nameInitials=null, affiliation=null, department=null, xref=
1, address=
1Institute of Big Data, Fudan University, Shanghai 200433, bio={"content":"
born in 1998. PhD candidate. His main research interests include multimodal information processing, multimodal large language models, and large vision-language models
"}, bioImg=null, bioContent=
born in 1998. PhD candidate. His main research interests include multimodal information processing, multimodal large language models, and large vision-language models
, aboutCorrespAuthor=null), CN=AuthorExt(id=1327017575420113189, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327017570688938237, authorId=1327017575264923938, language=CN, stringName=李煦, firstName=null, middleName=null, lastName=null, prefix=null, suffix=null, authorComment=null, nameInitials=null, affiliation=null, department=null, xref=
1, address=
1复旦大学大数据研究院 上海 200433, bio={"img":"XSN1PuFGF17JMxJaMx0EtA==","content":"
1998年生. 博士研究生. 主要研究方向为多模态信息处理、多模态大语言模型、视觉语言大模型
"}, bioImg=XSN1PuFGF17JMxJaMx0EtA==, bioContent=
1998年生. 博士研究生. 主要研究方向为多模态信息处理、多模态大语言模型、视觉语言大模型
, aboutCorrespAuthor=null)}, companyList=[AuthorCompany(id=1327017575063597339, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327017570688938237, xref=1, ext=[AuthorCompanyExt(id=1327017575071985948, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327017570688938237, companyId=1327017575063597339, language=EN, country=null, province=null, city=null, postcode=null, companyName=null, departmentName=null, remark=
1Institute of Big Data, Fudan University, Shanghai 200433), AuthorCompanyExt(id=1327017575084568861, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327017570688938237, companyId=1327017575063597339, language=CN, country=null, province=null, city=null, postcode=null, companyName=null, departmentName=null, remark=
1复旦大学大数据研究院 上海 200433)])]), Author(id=1327017575487222055, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327017570688938237, orderNo=1, firstName=null, middleName=null, lastName=null, nameCn=null, orcid=null, stid=null, country=null, authorPic=null, dead=0, email=null, emailSecond=null, emailThird=null, correspondingAuthor=0, authorType=1, ext={EN=AuthorExt(id=1327017575587885353, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327017570688938237, authorId=1327017575487222055, language=EN, stringName=Rui Zhu, firstName=Rui, middleName=null, lastName=Zhu, prefix=null, suffix=null, authorComment=null, nameInitials=null, affiliation=null, department=null, xref=
1, address=
1Institute of Big Data, Fudan University, Shanghai 200433, bio={"content":"
born in 2002. PhD candidate. Her main research interests include computer vision and large vision-language models
"}, bioImg=null, bioContent=
born in 2002. PhD candidate. Her main research interests include computer vision and large vision-language models
, aboutCorrespAuthor=null), CN=AuthorExt(id=1327017575654994218, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327017570688938237, authorId=1327017575487222055, language=CN, stringName=朱睿, firstName=null, middleName=null, lastName=null, prefix=null, suffix=null, authorComment=null, nameInitials=null, affiliation=null, department=null, xref=
1, address=
1复旦大学大数据研究院 上海 200433, bio={"img":"Kyx/e5NBlF1YRl43b8475A==","content":"
2002年生. 博士研究生. 主要研究方向为计算机视觉、视觉语言大模型
"}, bioImg=Kyx/e5NBlF1YRl43b8475A==, bioContent=
2002年生. 博士研究生. 主要研究方向为计算机视觉、视觉语言大模型
, aboutCorrespAuthor=null)}, companyList=[AuthorCompany(id=1327017575063597339, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327017570688938237, xref=1, ext=[AuthorCompanyExt(id=1327017575071985948, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327017570688938237, companyId=1327017575063597339, language=EN, country=null, province=null, city=null, postcode=null, companyName=null, departmentName=null, remark=
1Institute of Big Data, Fudan University, Shanghai 200433), AuthorCompanyExt(id=1327017575084568861, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327017570688938237, companyId=1327017575063597339, language=CN, country=null, province=null, city=null, postcode=null, companyName=null, departmentName=null, remark=
1复旦大学大数据研究院 上海 200433)])]), Author(id=1327017575726297388, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327017570688938237, orderNo=2, firstName=null, middleName=null, lastName=null, nameCn=null, orcid=null, stid=null, country=null, authorPic=null, dead=0, email=null, emailSecond=null, emailThird=null, correspondingAuthor=0, authorType=1, ext={EN=AuthorExt(id=1327017575793406254, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327017570688938237, authorId=1327017575726297388, language=EN, stringName=Xiaolei Chen, firstName=Xiaolei, middleName=null, lastName=Chen, prefix=null, suffix=null, authorComment=null, nameInitials=null, affiliation=null, department=null, xref=
1, address=
1Institute of Big Data, Fudan University, Shanghai 200433, bio={"content":"
born in 1999. PhD candidate. His main research interests include computer vision and machine learning
"}, bioImg=null, bioContent=
born in 1999. PhD candidate. His main research interests include computer vision and machine learning
, aboutCorrespAuthor=null), CN=AuthorExt(id=1327017575860515119, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327017570688938237, authorId=1327017575726297388, language=CN, stringName=陈小磊, firstName=null, middleName=null, lastName=null, prefix=null, suffix=null, authorComment=null, nameInitials=null, affiliation=null, department=null, xref=
1, address=
1复旦大学大数据研究院 上海 200433, bio={"img":"0cq0vbfsUr3aQpsA/Po/lw==","content":"
1999年生. 博士研究生. 主要研究方向为计算机视觉、机器学习
"}, bioImg=0cq0vbfsUr3aQpsA/Po/lw==, bioContent=
1999年生. 博士研究生. 主要研究方向为计算机视觉、机器学习
, aboutCorrespAuthor=null)}, companyList=[AuthorCompany(id=1327017575063597339, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327017570688938237, xref=1, ext=[AuthorCompanyExt(id=1327017575071985948, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327017570688938237, companyId=1327017575063597339, language=EN, country=null, province=null, city=null, postcode=null, companyName=null, departmentName=null, remark=
1Institute of Big Data, Fudan University, Shanghai 200433), AuthorCompanyExt(id=1327017575084568861, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327017570688938237, companyId=1327017575063597339, language=CN, country=null, province=null, city=null, postcode=null, companyName=null, departmentName=null, remark=
1复旦大学大数据研究院 上海 200433)])]), Author(id=1327017575940206897, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327017570688938237, orderNo=3, firstName=null, middleName=null, lastName=null, nameCn=null, orcid=null, stid=null, country=null, authorPic=null, dead=0, email=null, emailSecond=null, emailThird=null, correspondingAuthor=0, authorType=1, ext={EN=AuthorExt(id=1327017576019898675, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327017570688938237, authorId=1327017575940206897, language=EN, stringName=Jinxuan Wu, firstName=Jinxuan, middleName=null, lastName=Wu, prefix=null, suffix=null, authorComment=null, nameInitials=null, affiliation=null, department=null, xref=
2, address=
2School of Computer Science, Fudan University, Shanghai 200438, bio={"content":"
born in 2001. Master candidate. Her main research interests include large multimodal models, and algorithmic fairness and safety
"}, bioImg=null, bioContent=
born in 2001. Master candidate. Her main research interests include large multimodal models, and algorithmic fairness and safety
, aboutCorrespAuthor=null), CN=AuthorExt(id=1327017576082813236, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327017570688938237, authorId=1327017575940206897, language=CN, stringName=伍瑾轩, firstName=null, middleName=null, lastName=null, prefix=null, suffix=null, authorComment=null, nameInitials=null, affiliation=null, department=null, xref=
2, address=
2复旦大学计算机科学技术学院 上海 200438, bio={"img":"UnntlsQF8F00XFw8WkLSMw==","content":"
2001年生. 硕士研究生. 主要研究方向为多模态大模型、算法公平与安全性
"}, bioImg=UnntlsQF8F00XFw8WkLSMw==, bioContent=
2001年生. 硕士研究生. 主要研究方向为多模态大模型、算法公平与安全性
, aboutCorrespAuthor=null)}, companyList=[AuthorCompany(id=1327017575151677726, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327017570688938237, xref=2, ext=[AuthorCompanyExt(id=1327017575160066335, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327017570688938237, companyId=1327017575151677726, language=EN, country=null, province=null, city=null, postcode=null, companyName=null, departmentName=null, remark=
2School of Computer Science, Fudan University, Shanghai 200438), AuthorCompanyExt(id=1327017575168454944, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327017570688938237, companyId=1327017575151677726, language=CN, country=null, province=null, city=null, postcode=null, companyName=null, departmentName=null, remark=
2复旦大学计算机科学技术学院 上海 200438)])]), Author(id=1327017576154116406, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327017570688938237, orderNo=4, firstName=null, middleName=null, lastName=null, nameCn=null, orcid=null, stid=null, country=null, authorPic=null, dead=0, email=null, emailSecond=null, emailThird=null, correspondingAuthor=0, authorType=1, ext={EN=AuthorExt(id=1327017576229613880, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327017570688938237, authorId=1327017576154116406, language=EN, stringName=Yi Zheng, firstName=Yi, middleName=null, lastName=Zheng, prefix=null, suffix=null, authorComment=null, nameInitials=null, affiliation=null, department=null, xref=
2, address=
2School of Computer Science, Fudan University, Shanghai 200438, bio={"content":"
born in 1998. Master candidate. His main research interests include computer vision and large vision language models
"}, bioImg=null, bioContent=
born in 1998. Master candidate. His main research interests include computer vision and large vision language models
, aboutCorrespAuthor=null), CN=AuthorExt(id=1327017576300917049, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327017570688938237, authorId=1327017576154116406, language=CN, stringName=郑毅, firstName=null, middleName=null, lastName=null, prefix=null, suffix=null, authorComment=null, nameInitials=null, affiliation=null, department=null, xref=
2, address=
2复旦大学计算机科学技术学院 上海 200438, bio={"img":"P5d8P9Ll9cocoU1EFILM5Q==","content":"
1998年生. 硕士研究生. 主要研究方向为计算机视觉、视觉语言大模型
"}, bioImg=P5d8P9Ll9cocoU1EFILM5Q==, bioContent=
1998年生. 硕士研究生. 主要研究方向为计算机视觉、视觉语言大模型
, aboutCorrespAuthor=null)}, companyList=[AuthorCompany(id=1327017575151677726, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327017570688938237, xref=2, ext=[AuthorCompanyExt(id=1327017575160066335, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327017570688938237, companyId=1327017575151677726, language=EN, country=null, province=null, city=null, postcode=null, companyName=null, departmentName=null, remark=
2School of Computer Science, Fudan University, Shanghai 200438), AuthorCompanyExt(id=1327017575168454944, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327017570688938237, companyId=1327017575151677726, language=CN, country=null, province=null, city=null, postcode=null, companyName=null, departmentName=null, remark=
2复旦大学计算机科学技术学院 上海 200438)])]), Author(id=1327017576372220219, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327017570688938237, orderNo=5, firstName=null, middleName=null, lastName=null, nameCn=null, orcid=null, stid=null, country=null, authorPic=null, dead=0, email=null, emailSecond=null, emailThird=null, correspondingAuthor=0, authorType=1, ext={EN=AuthorExt(id=1327017576456106301, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327017570688938237, authorId=1327017576372220219, language=EN, stringName=Chenghang Lai, firstName=Chenghang, middleName=null, lastName=Lai, prefix=null, suffix=null, authorComment=null, nameInitials=null, affiliation=null, department=null, xref=
1, address=
1Institute of Big Data, Fudan University, Shanghai 200433, bio={"content":"
born in 1994. PhD candidate. His main research interests include deep learning, machine learning, and multimodal knowledge extraction and reasoning
"}, bioImg=null, bioContent=
born in 1994. PhD candidate. His main research interests include deep learning, machine learning, and multimodal knowledge extraction and reasoning
, aboutCorrespAuthor=null), CN=AuthorExt(id=1327017576523215166, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327017570688938237, authorId=1327017576372220219, language=CN, stringName=赖承杭, firstName=null, middleName=null, lastName=null, prefix=null, suffix=null, authorComment=null, nameInitials=null, affiliation=null, department=null, xref=
1, address=
1复旦大学大数据研究院 上海 200433, bio={"img":"FJWZ9o3i9xQw5WkmrahDNw==","content":"
1994年生. 博士研究生. 主要研究方向为深度学习、机器学习、多模态知识提取与推理
"}, bioImg=FJWZ9o3i9xQw5WkmrahDNw==, bioContent=
1994年生. 博士研究生. 主要研究方向为深度学习、机器学习、多模态知识提取与推理
, aboutCorrespAuthor=null)}, companyList=[AuthorCompany(id=1327017575063597339, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327017570688938237, xref=1, ext=[AuthorCompanyExt(id=1327017575071985948, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327017570688938237, companyId=1327017575063597339, language=EN, country=null, province=null, city=null, postcode=null, companyName=null, departmentName=null, remark=
1Institute of Big Data, Fudan University, Shanghai 200433), AuthorCompanyExt(id=1327017575084568861, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327017570688938237, companyId=1327017575063597339, language=CN, country=null, province=null, city=null, postcode=null, companyName=null, departmentName=null, remark=
1复旦大学大数据研究院 上海 200433)])]), Author(id=1327017576590324032, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327017570688938237, orderNo=6, firstName=null, middleName=null, lastName=null, nameCn=null, orcid=null, stid=null, country=null, authorPic=null, dead=0, email=null, emailSecond=null, emailThird=null, correspondingAuthor=0, authorType=1, ext={EN=AuthorExt(id=1327017576665821506, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327017570688938237, authorId=1327017576590324032, language=EN, stringName=Yuxuan Liang, firstName=Yuxuan, middleName=null, lastName=Liang, prefix=null, suffix=null, authorComment=null, nameInitials=null, affiliation=null, department=null, xref=
2, address=
2School of Computer Science, Fudan University, Shanghai 200438, bio={"content":"
born in 2001. Master candidate. His main research interests include artificial intelligence and large vision language models
"}, bioImg=null, bioContent=
born in 2001. Master candidate. His main research interests include artificial intelligence and large vision language models
, aboutCorrespAuthor=null), CN=AuthorExt(id=1327017576728736067, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327017570688938237, authorId=1327017576590324032, language=CN, stringName=梁宇轩, firstName=null, middleName=null, lastName=null, prefix=null, suffix=null, authorComment=null, nameInitials=null, affiliation=null, department=null, xref=
2, address=
2复旦大学计算机科学技术学院 上海 200438, bio={"img":"C8AmJiSFB39ebHmRbxNU3g==","content":"
2001年生. 硕士研究生. 主要研究方向为人工智能、视觉语言大模型
"}, bioImg=C8AmJiSFB39ebHmRbxNU3g==, bioContent=
2001年生. 硕士研究生. 主要研究方向为人工智能、视觉语言大模型
, aboutCorrespAuthor=null)}, companyList=[AuthorCompany(id=1327017575151677726, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327017570688938237, xref=2, ext=[AuthorCompanyExt(id=1327017575160066335, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327017570688938237, companyId=1327017575151677726, language=EN, country=null, province=null, city=null, postcode=null, companyName=null, departmentName=null, remark=
2School of Computer Science, Fudan University, Shanghai 200438), AuthorCompanyExt(id=1327017575168454944, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327017570688938237, companyId=1327017575151677726, language=CN, country=null, province=null, city=null, postcode=null, companyName=null, departmentName=null, remark=
2复旦大学计算机科学技术学院 上海 200438)])]), Author(id=1327017576795844933, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327017570688938237, orderNo=7, firstName=null, middleName=null, lastName=null, nameCn=null, orcid=null, stid=null, country=null, authorPic=null, dead=0, email=null, emailSecond=null, emailThird=null, correspondingAuthor=0, authorType=1, ext={EN=AuthorExt(id=1327017576875536711, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327017570688938237, authorId=1327017576795844933, language=EN, stringName=Bin Li, firstName=Bin, middleName=null, lastName=Li, prefix=null, suffix=null, authorComment=null, nameInitials=null, affiliation=null, department=null, xref=
2, address=
2School of Computer Science, Fudan University, Shanghai 200438, bio={"content":"
born in 1982. PhD, professor, PhD supervisor. His main research interests include machine learning and vision intelligence
"}, bioImg=null, bioContent=
born in 1982. PhD, professor, PhD supervisor. His main research interests include machine learning and vision intelligence
, aboutCorrespAuthor=null), CN=AuthorExt(id=1327017576959422792, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327017570688938237, authorId=1327017576795844933, language=CN, stringName=李斌, firstName=null, middleName=null, lastName=null, prefix=null, suffix=null, authorComment=null, nameInitials=null, affiliation=null, department=null, xref=
2, address=
2复旦大学计算机科学技术学院 上海 200438, bio={"img":"B2dBohQjxp9UVBq00dXwNw==","content":"
1982年生. 博士,教授,博士生导师. 主要研究方向为机器学习、视觉智能
"}, bioImg=B2dBohQjxp9UVBq00dXwNw==, bioContent=
1982年生. 博士,教授,博士生导师. 主要研究方向为机器学习、视觉智能
, aboutCorrespAuthor=null)}, companyList=[AuthorCompany(id=1327017575151677726, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327017570688938237, xref=2, ext=[AuthorCompanyExt(id=1327017575160066335, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327017570688938237, companyId=1327017575151677726, language=EN, country=null, province=null, city=null, postcode=null, companyName=null, departmentName=null, remark=
2School of Computer Science, Fudan University, Shanghai 200438), AuthorCompanyExt(id=1327017575168454944, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327017570688938237, companyId=1327017575151677726, language=CN, country=null, province=null, city=null, postcode=null, companyName=null, departmentName=null, remark=
2复旦大学计算机科学技术学院 上海 200438)])]), Author(id=1327017577034920266, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327017570688938237, orderNo=8, firstName=null, middleName=null, lastName=null, nameCn=null, orcid=null, stid=null, country=null, authorPic=null, dead=0, email=xyxue@fudan.edu.cn, emailSecond=null, emailThird=null, correspondingAuthor=1, authorType=1, ext={EN=AuthorExt(id=1327017577127194957, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327017570688938237, authorId=1327017577034920266, language=EN, stringName=Xiangyang Xue, firstName=Xiangyang, middleName=null, lastName=Xue, prefix=null, suffix=null, authorComment=null, nameInitials=null, affiliation=null, department=null, xref=
1, 2, *, address=
1Institute of Big Data, Fudan University, Shanghai 200433
2School of Computer Science, Fudan University, Shanghai 200438, bio={"content":"
born in 1968. PhD, professor, PhD supervisor. His main research interests include computer vision, multimedia, and machine learning
"}, bioImg=null, bioContent=
born in 1968. PhD, professor, PhD supervisor. His main research interests include computer vision, multimedia, and machine learning
, aboutCorrespAuthor=null), CN=AuthorExt(id=1327017578771362127, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327017570688938237, authorId=1327017577034920266, language=CN, stringName=薛向阳, firstName=null, middleName=null, lastName=null, prefix=null, suffix=null, authorComment=null, nameInitials=null, affiliation=null, department=null, xref=
1, 2, *, address=
1复旦大学大数据研究院 上海 200433
2复旦大学计算机科学技术学院 上海 200438, bio={"img":"JZR60NO1tJGpX8xWkvfFOg==","content":"
1968年生. 博士,教授,博士生导师. 主要研究方向为计算机视觉、多媒体、机器学习
"}, bioImg=JZR60NO1tJGpX8xWkvfFOg==, bioContent=
1968年生. 博士,教授,博士生导师. 主要研究方向为计算机视觉、多媒体、机器学习
, aboutCorrespAuthor=null)}, companyList=[AuthorCompany(id=1327017575063597339, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327017570688938237, xref=1, ext=[AuthorCompanyExt(id=1327017575071985948, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327017570688938237, companyId=1327017575063597339, language=EN, country=null, province=null, city=null, postcode=null, companyName=null, departmentName=null, remark=
1Institute of Big Data, Fudan University, Shanghai 200433), AuthorCompanyExt(id=1327017575084568861, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327017570688938237, companyId=1327017575063597339, language=CN, country=null, province=null, city=null, postcode=null, companyName=null, departmentName=null, remark=
1复旦大学大数据研究院 上海 200433)]), AuthorCompany(id=1327017575151677726, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327017570688938237, xref=2, ext=[AuthorCompanyExt(id=1327017575160066335, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327017570688938237, companyId=1327017575151677726, language=EN, country=null, province=null, city=null, postcode=null, companyName=null, departmentName=null, remark=
2School of Computer Science, Fudan University, Shanghai 200438), AuthorCompanyExt(id=1327017575168454944, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327017570688938237, companyId=1327017575151677726, language=CN, country=null, province=null, city=null, postcode=null, companyName=null, departmentName=null, remark=
2复旦大学计算机科学技术学院 上海 200438)])])], keywords=[Keyword(id=1327017578876219728, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327017570688938237, language=EN, orderNo=1, keyword=natural language processing), Keyword(id=1327017578943328593, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327017570688938237, language=EN, orderNo=2, keyword=computer vision), Keyword(id=1327017579023020370, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327017570688938237, language=EN, orderNo=3, keyword=large vision-language models), Keyword(id=1327017579111100755, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327017570688938237, language=EN, orderNo=4, keyword=multimodal large language models), Keyword(id=1327017579186598228, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327017570688938237, language=EN, orderNo=5, keyword=hallucinations), Keyword(id=1327017579316621653, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327017570688938237, language=CN, orderNo=1, keyword=自然语言处理), Keyword(id=1327017579404702038, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327017570688938237, language=CN, orderNo=2, keyword=计算机视觉), Keyword(id=1327017579547308375, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327017570688938237, language=CN, orderNo=3, keyword=视觉语言大模型), Keyword(id=1327017579647971672, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327017570688938237, language=CN, orderNo=4, keyword=多模态大语言模型), Keyword(id=1327017579719274841, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327017570688938237, language=CN, orderNo=5, keyword=幻觉)], refs=[Reference(id=1327017581111783782, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327017570688938237, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=1, rfOrder=0, authorNames=null, journalName=null, refType=null, unstructuredReference=Yin Shukang, Fu Chaoyou, Zhao Sirui, et al. A survey on multimodal large language models[J]. arXiv preprint, arXiv: 2306. 13549, 2023, articleTitle=null, refAbstract=null), Reference(id=1327017581183086951, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327017570688938237, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=2, rfOrder=1, authorNames=null, journalName=null, refType=null, unstructuredReference=Touvron H, Lavril T, Lzacard G, et al. LLaMA: Open and efficient foundation language models[J]. arXiv preprint, arXiv: 2302. 13971, 2023, articleTitle=null, refAbstract=null), Reference(id=1327017581262778728, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327017570688938237, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=3, rfOrder=2, authorNames=null, journalName=null, refType=null, unstructuredReference=Touvron H, Martin L, Stone K, et al. LLaMA-2: Open foundation and fine-tuned chat models[J]. arXiv preprint, arXiv: 2307. 09288, 2023, articleTitle=null, refAbstract=null), Reference(id=1327017581329887593, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327017570688938237, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=4, rfOrder=3, authorNames=null, journalName=null, refType=null, unstructuredReference=Radford A, Narasimhan K, Salimans T, et al. Improving language understanding by generative pre-training[EB/OL].[2024-05-16]. https://cdn.openai.com/research-covers/language-unsupervised/language_understanding_paper.pdf, articleTitle=null, refAbstract=null), Reference(id=1327017581405385066, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327017570688938237, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=5, rfOrder=4, authorNames=null, journalName=null, refType=null, unstructuredReference=Radford A, Wu J, Child R, et al. Language models are unsupervised multitask learners[EB/OL].[2024-05-16]. https://cdn.openai.com/better-language-models/language_models_are_unsupervised_multitask_learners.pdf, articleTitle=null, refAbstract=null), Reference(id=1327017581472493931, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327017570688938237, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=6, rfOrder=5, authorNames=null, journalName=null, refType=null, unstructuredReference=Ouyang Long, Wu J, Jiang Xu, et al. Training language models to follow instructions with human feedback[C]//Proc of the 36th Int Conf on Neural Information Processing Systems. New York: Curran Associates, 2022: 27730−27744, articleTitle=null, refAbstract=null), Reference(id=1327017581568962924, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327017570688938237, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=7, rfOrder=6, authorNames=null, journalName=null, refType=null, unstructuredReference=Meta. Introducing Meta LLaMA 3: The most capable openly available LLM to date[EB/OL].[2024-05-16]. https://ai.meta.com/blog/meta-llama-3/, articleTitle=null, refAbstract=null), Reference(id=1327017583246684525, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327017570688938237, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=8, rfOrder=7, authorNames=null, journalName=null, refType=null, unstructuredReference=舒文韬, 李睿潇, 孙天祥, 等. 大型语言模型: 原理、实现与发展[J]. 计算机研究与发展, 2024, 61(2): 351−361, articleTitle=null, refAbstract=null), Reference(id=1327017583338959214, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327017570688938237, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=8, rfOrder=8, authorNames=null, journalName=null, refType=null, unstructuredReference=Shu Wentao, Li Ruixiao, Sun Tianxiang, et al. Large language models: Principles, implementation, and progress[J]. Journal of Computer Research and Development, 2024, 61(2): 351−361 (in Chinese), articleTitle=null, refAbstract=null), Reference(id=1327017583427039599, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327017570688938237, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=9, rfOrder=9, authorNames=null, journalName=null, refType=null, unstructuredReference=Radford A, Kim J W, Hallacy C, et al. Learning transferable visual models from natural language supervision[C]//Proc of the 38th Int Conf on Machine Learning. New York: PMLR, 2021: 8748−8763, articleTitle=null, refAbstract=null), Reference(id=1327017583494148464, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327017570688938237, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=10, rfOrder=10, authorNames=null, journalName=null, refType=null, unstructuredReference=Dosovitskiy A, Beyer L, Kolesnikov A, et al. An image is worth 16×16 words: Transformers for image recognition at scale[C/OL].[2025-04-04]. https://openreview.net/forum?id=YicbFdNTTy, articleTitle=null, refAbstract=null), Reference(id=1327017583619977585, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327017570688938237, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=11, rfOrder=11, authorNames=null, journalName=null, refType=null, unstructuredReference=He Kaiming, Chen Xinlei, Xie Saining, et al. Masked autoencoders are scalable vision learners[C]//Proc of the 34th IEEE/CVF Conf on Computer Vision and Pattern Recognition. Piscataway, NJ: IEEE, 2021: 16000−16009, articleTitle=null, refAbstract=null), Reference(id=1327017583716446578, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327017570688938237, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=12, rfOrder=12, authorNames=null, journalName=null, refType=null, unstructuredReference=Fang Yuxin, Wang Wen, Xie Binhui, et al. Eva: Exploring the limits of masked visual representation learning at scale[C]//Proc of the 36th IEEE/CVF Conf on Computer Vision and Pattern Recognition. Piscataway, NJ: IEEE, 2023: 19358−19369, articleTitle=null, refAbstract=null), Reference(id=1327017583808721267, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327017570688938237, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=13, rfOrder=13, authorNames=null, journalName=null, refType=null, unstructuredReference=Fang Yuxin, Sun Quan, Wang Xinggang, et al. Eva-02: A visual representation for neon genesis[J]. Image Vision Computing, 2024, 149(C): 1−12, articleTitle=null, refAbstract=null), Reference(id=1327017583892607348, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327017570688938237, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=14, rfOrder=14, authorNames=null, journalName=null, refType=null, unstructuredReference=Sun Quan, Fang Yuxin, Wu L, et al. EVA-CLIP: Improved training techniques for clip at scale[J]. arXiv preprint, arXiv: 2303. 15389, 2023, articleTitle=null, refAbstract=null), Reference(id=1327017583955521909, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327017570688938237, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=15, rfOrder=15, authorNames=null, journalName=null, refType=null, unstructuredReference=Wei Haoran, Kong Lingyu, Chen Jinyue, et al. Vary: Scaling up the vision vocabulary for large vision-language models[C]//Proc of the 17th European Conf on Computer Vision. Berlin: Springer, 2024: 1−18, articleTitle=null, refAbstract=null), Reference(id=1327017584026825078, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327017570688938237, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=16, rfOrder=16, authorNames=null, journalName=null, refType=null, unstructuredReference=Tong Shengbang, Liu Zhuang, Zhai Yuexiang, et al. Eyes wide shut? Exploring the visual shortcomings of multimodal LLMs[C]//Proc of the 37th IEEE/CVF Conf on Computer Vision and Pattern Recognition. Piscataway, NJ: IEEE, 2024: 9568−9578, articleTitle=null, refAbstract=null), Reference(id=1327017584135876983, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327017570688938237, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=17, rfOrder=17, authorNames=null, journalName=null, refType=null, unstructuredReference=Liu Haotian, Li Chunyuan, Wu Qingyang, et al. Visual instruction tuning[C]//Proc of the 37th Int Conf on Neural Information Processing Systems. New York: Curran Associates, 2023: 34896−34916, articleTitle=null, refAbstract=null), Reference(id=1327017584223957368, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327017570688938237, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=18, rfOrder=18, authorNames=null, journalName=null, refType=null, unstructuredReference=Chen Gongwei, Shen Leyang, Shao Rui, et al. LION: Empowering multimodal large language model with dual-level visual knowledge[C]//Proc of the 36th IEEE/CVF Conf on Computer Vision and Pattern Recognition. Piscataway, NJ: IEEE, 2023: 26530−26540, articleTitle=null, refAbstract=null), Reference(id=1327017584295260537, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327017570688938237, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=19, rfOrder=19, authorNames=null, journalName=null, refType=null, unstructuredReference=Chen Zhe, Wu Jiannan, Wang Wenhai, et al. InternVL: Scaling up vision foundation models and aligning for generic visual-linguistic tasks[C]//Proc of the 37th IEEE/CVF Conf on Computer Vision and Pattern Recognition. Piscataway, NJ: IEEE, 2024: 24185−24198, articleTitle=null, refAbstract=null), Reference(id=1327017584379146618, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327017570688938237, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=20, rfOrder=20, authorNames=null, journalName=null, refType=null, unstructuredReference=Lu Junyu, Gan Ruyi, Zhang Dixiang, et al. Lyrics: Boosting fine-grained language-vision alignment and comprehension via semantic-aware visual objects[J]. arXiv preprint, arXiv: 2312. 05278, 2023, articleTitle=null, refAbstract=null), Reference(id=1327017584450449787, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327017570688938237, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=21, rfOrder=21, authorNames=null, journalName=null, refType=null, unstructuredReference=Fan Xianran, Ji Tao, Jiang Changhao, et al. MouSi: Poly-visual-expert vision-language models[J]. arXiv preprint, arXiv: 2401. 17221, 2024, articleTitle=null, refAbstract=null), Reference(id=1327017584521752956, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327017570688938237, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=22, rfOrder=22, authorNames=null, journalName=null, refType=null, unstructuredReference=Chen Jun, Zhu Deyao, Shen Xiaoqian, et al. MiniGPT-v2: Large language model as a unified interface for vision-language multi-task learning[J]. arXiv preprint, arXiv: 2310. 09478, 2023, articleTitle=null, refAbstract=null), Reference(id=1327017584597250429, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327017570688938237, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=23, rfOrder=23, authorNames=null, journalName=null, refType=null, unstructuredReference=Zhao Linxi, Deng Yihe, Zhang Weitong, et al. Mitigating object hallucination in large vision-language models via classifier-free guidance[J]. arXiv preprint, arXiv: 2402. 08680, 2024, articleTitle=null, refAbstract=null), Reference(id=1327017584676942206, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327017570688938237, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=24, rfOrder=24, authorNames=null, journalName=null, refType=null, unstructuredReference=Chen Keqin, Zhang Zhao, Zeng Weili, et al. Shikra: Unleashing multimodal LLM’s referential dialogue magic[J]. arXiv preprint, arXiv: 2306. 15195, 2023, articleTitle=null, refAbstract=null), Reference(id=1327017584773411199, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327017570688938237, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=25, rfOrder=25, authorNames=null, journalName=null, refType=null, unstructuredReference=Li Junnan, Li Dongxu, Savarese S, et al. Blip-2: Bootstrapping language-image pre-training with frozen image encoders and large language models[C]//Proc of the 40th Int Conf on Machine Learning. New York: PMLR, 2023: 19730−1974, articleTitle=null, refAbstract=null), Reference(id=1327017584874074496, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327017570688938237, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=26, rfOrder=26, authorNames=null, journalName=null, refType=null, unstructuredReference=Zhang Renrui, Han Jiaming, Liu C, et al. Llama-adapter: Efficient fine-tuning of language models with zero-init attention[J]. arXiv preprint, arXiv: 2303. 16199, 2023, articleTitle=null, refAbstract=null), Reference(id=1327017584962154881, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327017570688938237, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=27, rfOrder=27, authorNames=null, journalName=null, refType=null, unstructuredReference=Zhu Deyao, Chen Jun, Shen Xiaoqian, et al. Minigpt-4: Enhancing vision-language understanding with advanced large language models[J]. arXiv preprint, arXiv: 2304. 10592, 2023, articleTitle=null, refAbstract=null), Reference(id=1327017585029263746, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327017570688938237, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=28, rfOrder=28, authorNames=null, journalName=null, refType=null, unstructuredReference=Gao Peng, Han Jiaming, Zhang Renrui, et al. LLaMA-Adapter v2: Parameter-efficient visual instruction model[J]. arXiv preprint, arXiv: 2304. 15010, 2023, articleTitle=null, refAbstract=null), Reference(id=1327017585100566915, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327017570688938237, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=29, rfOrder=29, authorNames=null, journalName=null, refType=null, unstructuredReference=Dai Wenliang, Li Junnan, Li Dongxu, et al. Instructblip: Towards general-purpose vision-language models with instruction tuning[C]//Proc of the 37th Int Conf on Neural Information Processing Systems. New York: Curran Associates, 2023: 49250−49267, articleTitle=null, refAbstract=null), Reference(id=1327017585176064388, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327017570688938237, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=30, rfOrder=30, authorNames=null, journalName=null, refType=null, unstructuredReference=Luo Gen, Zhou Yiyi, Ren Tianhe, et al. Cheap and quick: Efficient vision-language instruction tuning for large language models[C]//Proc of the 37th Int Conf on Neural Information Processing Systems. New York: Curran Associates, 2023: 29615−29627, articleTitle=null, refAbstract=null), Reference(id=1327017585251561861, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327017570688938237, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=31, rfOrder=31, authorNames=null, journalName=null, refType=null, unstructuredReference=Zeng Yan, Zhang Hanbo, Zheng Jiani, et al. What matters in training a GPT4-style language model with multimodal inputs?[J]. arXiv preprint, arXiv: 2307. 02469, 2023, articleTitle=null, refAbstract=null), Reference(id=1327017585339642246, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327017570688938237, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=32, rfOrder=32, authorNames=null, journalName=null, refType=null, unstructuredReference=Bai Jinze, Bai Shuai, Yang Shusheng, et al. Qwen-VL: A versatile vision-language model for understanding, localization, text reading, and beyond[J]. arXiv preprint, arXiv: 2308. 12966, 2023, articleTitle=null, refAbstract=null), Reference(id=1327017585410945415, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327017570688938237, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=33, rfOrder=33, authorNames=null, journalName=null, refType=null, unstructuredReference=Liu Haotian, Li Chunyuan, Li Yuheng, et al. Improved baselines with visual instruction tuning[C]//Proc of the 36th IEEE/CVF Conf on Computer Vision and Pattern Recognition. Piscataway, NJ: IEEE, 2023: 26286−26296, articleTitle=null, refAbstract=null), Reference(id=1327017585499025800, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327017570688938237, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=34, rfOrder=34, authorNames=null, journalName=null, refType=null, unstructuredReference=Wang Weihan, Lv Qingsong, Yu Wenmeng, et al. CogVLM: Visual expert for pretrained language models[J]. arXiv preprint, arXiv: 2311. 03079, 2023, articleTitle=null, refAbstract=null), Reference(id=1327017585574523273, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327017570688938237, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=35, rfOrder=35, authorNames=null, journalName=null, refType=null, unstructuredReference=Ye Qinghao, Xu Haiyang, Ye Jiabo, et al. mPLUG-Owl2: Revolutionizing multi-modal large language model with modality collaboration[C]//Proc of the 36th IEEE/CVF Conf on Computer Vision and Pattern Recognition. Piscataway, NJ: IEEE, 2023: 13040−13051, articleTitle=null, refAbstract=null), Reference(id=1327017585670992266, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327017570688938237, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=36, rfOrder=36, authorNames=null, journalName=null, refType=null, unstructuredReference=Liu Haotian, Li Chunyuan, Li Yuheng, et al. LLaVA-next: Improved reasoning, OCR, and world knowledge[EB/OL].[2024-05-16]. https://llava-vl.github.io/blog/2024-01-30-llava-next/, articleTitle=null, refAbstract=null), Reference(id=1327017585809404300, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327017570688938237, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=37, rfOrder=37, authorNames=null, journalName=null, refType=null, unstructuredReference=Liu Haotian, Xue Wenyuan, Chen Yifei, et al. A survey on hallucination in large vision-language models[J]. arXiv preprint, arXiv: 2402. 00253, 2024, articleTitle=null, refAbstract=null), Reference(id=1327017585926844813, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327017570688938237, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=38, rfOrder=38, authorNames=null, journalName=null, refType=null, unstructuredReference=Li Chunyuan, Wong C, Zhang Sheng, et al. LLaVA-Med: Training a large language-and-vision assistant for biomedicine in one day[C]//Proc of the 37th Int Conf on Neural Information Processing Systems. New York: Curran Associates, 2023: 28541−28564, articleTitle=null, refAbstract=null), Reference(id=1327017586002342286, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327017570688938237, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=39, rfOrder=39, authorNames=null, journalName=null, refType=null, unstructuredReference=Bai Fan, Du Yuxin, Huang Tiejun, et al. M3D: Advancing 3D medical image analysis with multi-modal large language models[J]. arXiv preprint, arXiv: 2404. 00578, 2024, articleTitle=null, refAbstract=null), Reference(id=1327017586090422671, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327017570688938237, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=40, rfOrder=40, authorNames=null, journalName=null, refType=null, unstructuredReference=Tian Xiaoyu, Gu Junru, Li Bailin, et al. DriveVLM: The convergence of autonomous driving and large vision-language models[J]. arXiv preprint arXiv: 2402. 12289, 2024, articleTitle=null, refAbstract=null), Reference(id=1327017587843641744, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327017570688938237, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=41, rfOrder=41, authorNames=null, journalName=null, refType=null, unstructuredReference=Qian Tianwen, Chen J, Zhuo Linhai, et al. Nuscenes-qa: A multi-modal visual question answering benchmark for autonomous driving scenario[C]//Proc of the 38th AAAI Conf on Artificial Intelligence. Palo Alto, CA: AAAI, 2024: 4542−4550, articleTitle=null, refAbstract=null), Reference(id=1327017587940110737, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327017570688938237, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=42, rfOrder=42, authorNames=null, journalName=null, refType=null, unstructuredReference=Dahl M, Magesh V, Suzgun M, et al. Large legal fictions: Profiling legal hallucinations in large language lodels[J]. Journal of Legal Analysis, 2024, 16(1): 64−93, articleTitle=null, refAbstract=null), Reference(id=1327017588107882898, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327017570688938237, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=43, rfOrder=43, authorNames=null, journalName=null, refType=null, unstructuredReference=Andersland M. Amharic LLaMA and LLaVA: Multimodal LLMs for low resource languages[J]. arXiv preprint, arXiv: 2403.06354, 2024, articleTitle=null, refAbstract=null), Reference(id=1327017588212740499, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327017570688938237, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=44, rfOrder=44, authorNames=null, journalName=null, refType=null, unstructuredReference=Liu Jiazhen, Fu Yuhan, Xie Ruobing, et al. PhD: A prompted visual hallucination evaluation dataset[J]. arXiv preprint, arXiv: 2403. 11116, 2024, articleTitle=null, refAbstract=null), Reference(id=1327017588317598100, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327017570688938237, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=45, rfOrder=45, authorNames=null, journalName=null, refType=null, unstructuredReference=Jiang Chaoya, Ye Wei, Dong Mengfan, et al. Hal-Eval: A universal and fine-grained hallucination evaluation framework for large vision language models[J]. arXiv preprint, arXiv: 2402. 15721, 2024, articleTitle=null, refAbstract=null), Reference(id=1327017588422455701, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327017570688938237, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=46, rfOrder=46, authorNames=null, journalName=null, refType=null, unstructuredReference=Zhang Huixuan, Zhang Junzhe, Wan Xiaojun. Evaluating and mitigating number hallucinations in large vision-language models: A consistency perspective[J]. arXiv preprint, arXiv: 2403. 01373, 2024, articleTitle=null, refAbstract=null), Reference(id=1327017588489564566, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327017570688938237, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=47, rfOrder=47, authorNames=null, journalName=null, refType=null, unstructuredReference=Li Yifan, Du Yifan, Zhou Kun, et al. Evaluating object hallucination in large vision-language models[C]//Proc of the 28th Conf on Empirical Methods in Natural Language Processing. Stroudsburg, PA: ACL, 2023: 292−305, articleTitle=null, refAbstract=null), Reference(id=1327017588586033559, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327017570688938237, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=48, rfOrder=48, authorNames=null, journalName=null, refType=null, unstructuredReference=Liu Fuxiao, Lin K, Li Linjie, et al. Mitigating hallucination in large multi-modal models via robust instruction tuning[C/OL].[2025-04-04]. https://openreview.net/forum?id=J44HfH4JCg, articleTitle=null, refAbstract=null), Reference(id=1327017588665725336, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327017570688938237, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=49, rfOrder=49, authorNames=null, journalName=null, refType=null, unstructuredReference=Lovenia H, Dai Wenliang, Cahyawijaya S, et al. Negative object presence evaluation (NOPE) to measure object hallucination in vision-language models[J]. arXiv preprint, arXiv: 2310. 05338, 2023, articleTitle=null, refAbstract=null), Reference(id=1327017588741222809, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327017570688938237, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=50, rfOrder=50, authorNames=null, journalName=null, refType=null, unstructuredReference=Rohrbach A, Hendricks L A, Burns K, et al. Object hallucination in image captioning[C]//Proc of the 23rd Conf on Empirical Methods in Natural Language Processing. Stroudsburg, PA: ACL, 2018: 4035−4045, articleTitle=null, refAbstract=null), Reference(id=1327017588816720282, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327017570688938237, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=51, rfOrder=51, authorNames=null, journalName=null, refType=null, unstructuredReference=Dai Wenliang, Liu Zihan, Ji Ziwei, et al. Plausible may not be faithful: Probing object hallucination in vision-language pre-training[C]//Proc of the 17th Conf of the European Chapter of the Association for Computational Linguistics. Stroudsburg, PA: ACL, 2023: 2136−2148, articleTitle=null, refAbstract=null), Reference(id=1327017588888023451, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327017570688938237, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=52, rfOrder=52, authorNames=null, journalName=null, refType=null, unstructuredReference=Wang Junyang, Zhou Yiyang, Xu Guohai, et al. Evaluation and analysis of hallucination in large vision-language models[J]. arXiv preprint, arXiv: 2308. 15126, 2023, articleTitle=null, refAbstract=null), Reference(id=1327017588971909532, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327017570688938237, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=53, rfOrder=53, authorNames=null, journalName=null, refType=null, unstructuredReference=Wang Junyang, Wang Yuhang, Xu Guohai, et al. AMBER: An LLM-free multi-dimensional benchmark for MLLMs hallucination evaluation[J]. arXiv preprint, arXiv: 2311. 07397, 2023, articleTitle=null, refAbstract=null), Reference(id=1327017589059989917, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327017570688938237, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=54, rfOrder=54, authorNames=null, journalName=null, refType=null, unstructuredReference=Fu Chaoyou, Chen Peixian, Shen Yunhang, et al. MME: A comprehensive evaluation benchmark for multimodal large language models[J]. arXiv preprint, arXiv: 2306. 13394, 2023, articleTitle=null, refAbstract=null), Reference(id=1327017589118710174, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327017570688938237, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=55, rfOrder=55, authorNames=null, journalName=null, refType=null, unstructuredReference=Hu Hongyu, Zhang Jiyuan, Zhao Minyi, et al. CIEM: Contrastive instruction evaluation method for better instruction tuning[C]//Proc of the 35th Int Conf on Neural Information Processing Systems. New York: Curran Associates, 2023: 1−11, articleTitle=null, refAbstract=null), Reference(id=1327017589202596255, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327017570688938237, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=56, rfOrder=56, authorNames=null, journalName=null, refType=null, unstructuredReference=Zhou Kankan, Lai E, Yeong W B A, et al. ROME: Evaluating pre-trained vision-language models on reasoning beyond visual common sense[C]//Proc of the 28th Findings of the Association for Computational Linguistics: EMNLP 2023. Stroudsburg, PA: ACL, 2023: 10185−10197, articleTitle=null, refAbstract=null), Reference(id=1327017589269705120, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327017570688938237, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=57, rfOrder=57, authorNames=null, journalName=null, refType=null, unstructuredReference=Guan Tianrui, Liu Fuxiao, Wu Xiyang, et al. HallusionBench: An advanced diagnostic suite for entangled language hallucination & visual illusion in large vision-language models[C]//Proc of the 37th IEEE/CVF Conf on Computer Vision and Pattern Recognition. Piscataway, NJ: IEEE, 2024: 14375−14385, articleTitle=null, refAbstract=null), Reference(id=1327017589341008289, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327017570688938237, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=58, rfOrder=58, authorNames=null, journalName=null, refType=null, unstructuredReference=Bitton-Guetta N, Bitton Y, Hessel J, et al. Breaking common sense: WHOOPS! A vision-and-language benchmark of synthetic and compositional images[C]//Proc of the 17th IEEE/CVF Int Conf on Computer Vision. Piscataway, NJ: IEEE, 2023: 2616−2627, articleTitle=null, refAbstract=null), Reference(id=1327017589416505762, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327017570688938237, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=59, rfOrder=59, authorNames=null, journalName=null, refType=null, unstructuredReference=Cui Chenhang, Zhou Yiyang, Yang Xinyu, et al. Holistic analysis of hallucination in GPT−4v (ision): Bias and interference challenges[J]. arXiv preprint, arXiv: 2311. 03287, 2023, articleTitle=null, refAbstract=null), Reference(id=1327017589487808931, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327017570688938237, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=60, rfOrder=60, authorNames=null, journalName=null, refType=null, unstructuredReference=Jing Liqiang, Li Ruosen, Chen Yunmo, et al. FAITHSCORE: Evaluating hallucinations in large vision-language models[J]. arXiv preprint, arXiv: 2311. 01477, 2023, articleTitle=null, refAbstract=null), Reference(id=1327017589550723492, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327017570688938237, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=61, rfOrder=61, authorNames=null, journalName=null, refType=null, unstructuredReference=Gunjal A, Yin Jihan, Bas E. Detecting and preventing hallucinations in large vision language models[C]//Proc of the 38th AAAI Conf on Artificial Intelligence. Palo Alto, CA: AAAI, 2024: 18135−18143, articleTitle=null, refAbstract=null), Reference(id=1327017589626220965, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327017570688938237, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=62, rfOrder=62, authorNames=null, journalName=null, refType=null, unstructuredReference=Wang Bin, Wu Fan, Han Xiao, et al. VIGC: Visual instruction generation and correction[C]//Proc of the 38th AAAI Conf on Artificial Intelligence. Palo Alto, CA: AAAI, 2024: 5309−5317, articleTitle=null, refAbstract=null), Reference(id=1327017589693329830, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327017570688938237, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=63, rfOrder=63, authorNames=null, journalName=null, refType=null, unstructuredReference=Sun Zhiqing, Shen Sheng, Cao Shengcao, et al. Aligning large multimodal models with factually augmented RLHF[C]//Proc of the 18th Findings of the Association for Computational Linguistics: ACL 2024. Stroudsburg, PA: ACL, 2024: 13088−13110, articleTitle=null, refAbstract=null), Reference(id=1327017589764632999, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327017570688938237, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=64, rfOrder=64, authorNames=null, journalName=null, refType=null, unstructuredReference=Yin Shukang, Fu Chaoyou, Zhao Sirui, et al. Woodpecker: Hallucination correction for multimodal large language models[J]. arXiv preprint, arXiv: 2310. 16045, 2023, articleTitle=null, refAbstract=null), Reference(id=1327017589827547560, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327017570688938237, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=65, rfOrder=65, authorNames=null, journalName=null, refType=null, unstructuredReference=Zhou Yiyang, Cui Chenhang, Yoon J, et al. Analyzing and mitigating object hallucination in large vision-language models[J]. arXiv preprint, arXiv: 2310. 00754, 2023, articleTitle=null, refAbstract=null), Reference(id=1327017589898850729, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327017570688938237, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=66, rfOrder=66, authorNames=null, journalName=null, refType=null, unstructuredReference=Leng Sicong, Zhang Hang, Chen Guanzheng, et al. Mitigating object hallucinations in large vision-language models through visual contrastive decoding[C]//Proc of the 37th IEEE/CVF Conf on Computer Vision and Pattern Recognition. Piscataway, NJ: IEEE, 2024: 13872−13882, articleTitle=null, refAbstract=null), Reference(id=1327017589974348202, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327017570688938237, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=67, rfOrder=67, authorNames=null, journalName=null, refType=null, unstructuredReference=Lee S, Park S H, Jo Y, et al. Volcano: Mitigating multimodal hallucination through self-feedback guided revision[C]//Proc of the 18th Conf of the North American Chapter of the Association for Computational Linguistics: Human Language Technologies. Stroudsburg, PA: ACL, 2024: 391−404, articleTitle=null, refAbstract=null), Reference(id=1327017590058234283, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327017570688938237, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=68, rfOrder=68, authorNames=null, journalName=null, refType=null, unstructuredReference=Chen Xi, Djolonga J, Padlewski P, et al. Pali-x: On scaling up a multilingual vision and language model[C]//Proc of the 37th IEEE/CVF Conf on Computer Vision and Pattern Recognition. Piscataway, NJ: IEEE, 2024: 14432−14444, articleTitle=null, refAbstract=null), Reference(id=1327017590121148844, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327017570688938237, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=69, rfOrder=69, authorNames=null, journalName=null, refType=null, unstructuredReference=Alayrac J-B, Donahue J, Luc P, et al. Flamingo: A visual language model for few-shot learning[C]//Proc of the 36th Int Conf on Neural Information Processing Systems. New York: Curran Associates, 2022: 23716−23736, articleTitle=null, refAbstract=null), Reference(id=1327017590213423533, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327017570688938237, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=70, rfOrder=70, authorNames=null, journalName=null, refType=null, unstructuredReference=Chen Xi, Wang Xiao, Changpinyo S, et al. Pali: A jointly-scaled multilingual language-image model[J]. arXiv preprint, arXiv: 2209. 06794, 2022, articleTitle=null, refAbstract=null), Reference(id=1327017590288921006, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327017570688938237, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=71, rfOrder=71, authorNames=null, journalName=null, refType=null, unstructuredReference=Li Bo, Zhang Yuanhan, Chen Liangyu, et al. Mimic-it: Multi-modal in-context instruction tuning[J]. arXiv preprint, arXiv: 2306. 05425, 2023, articleTitle=null, refAbstract=null), Reference(id=1327017590360224175, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327017570688938237, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=72, rfOrder=72, authorNames=null, journalName=null, refType=null, unstructuredReference=Chiang W, Li Zhuohan, Lin Zi, et al. Vicuna: An open-source chatbot impressing GPT-4 with 90%* ChatGPT quality[EB/OL].[2024-05-16]. https://lmsys.org/blog/2023-03-30-vicuna/, articleTitle=null, refAbstract=null), Reference(id=1327017590439915952, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327017570688938237, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=73, rfOrder=73, authorNames=null, journalName=null, refType=null, unstructuredReference=Caffagni D, Cocchi F, Barsellotti L, et al. The revolution of multimodal large language models: A survey[C]//Proc of the 18th Findings of the Association for Computational Linguistics. Stroudsburg, PA: ACL, 2024: 13590−13618, articleTitle=null, refAbstract=null), Reference(id=1327017590532190641, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327017570688938237, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=74, rfOrder=74, authorNames=null, journalName=null, refType=null, unstructuredReference=Zhai Bohan, Yang Shijia, Zhao Xiangchen, et al. HallE-Control: Controlling object hallucination in large multimodal models[J]. arXiv preprint, arXiv: 2310. 01779, 2023, articleTitle=null, refAbstract=null), Reference(id=1327017592193135031, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327017570688938237, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=75, rfOrder=75, authorNames=null, journalName=null, refType=null, unstructuredReference=Zhang Yue, Li Yafu, Cui Leyang, et al. Siren’s song in the AI ocean: A survey on hallucination in large language models[J]. arXiv preprint, arXiv: 2309. 01219, 2023, articleTitle=null, refAbstract=null), Reference(id=1327017592331547064, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327017570688938237, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=76, rfOrder=76, authorNames=null, journalName=null, refType=null, unstructuredReference=Tonmoy S, Zaman S, Jain V, et al. A comprehensive survey of hallucination mitigation techniques in large language models[J]. arXiv preprint, arXiv: 2401. 01313, 2024, articleTitle=null, refAbstract=null), Reference(id=1327017592394461625, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327017570688938237, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=77, rfOrder=77, authorNames=null, journalName=null, refType=null, unstructuredReference=Huang Lei, Yu Weijiang, Ma Weitao, et al. A survey on hallucination in large language models: Principles, taxonomy, challenges, and open questions[J]. arXiv preprint, arXiv: 2311. 05232, 2023, articleTitle=null, refAbstract=null), Reference(id=1327017592474153402, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327017570688938237, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=78, rfOrder=78, authorNames=null, journalName=null, refType=null, unstructuredReference=Chen Lin, Li Jinsong, Dong Xiaoyi, et al. shareGPT4V: Improving large multi-modal models with better captions[C]//Proc of European Conf on Computer Vision. Berlin: Springer, 2024, articleTitle=null, refAbstract=null), Reference(id=1327017592558039483, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327017570688938237, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=79, rfOrder=79, authorNames=null, journalName=null, refType=null, unstructuredReference=OpenAI. GPT-4 is OpenAI’s most advanced system, producing safer and more useful responses[EB/OL].[2024-05-16]. https://openai.com/gpt-4, articleTitle=null, refAbstract=null), Reference(id=1327017592654508476, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327017570688938237, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=80, rfOrder=80, authorNames=null, journalName=null, refType=null, unstructuredReference=Yu Qifan, Li Juncheng, Wei Longhui, et al. HalluciDoctor: Mitigating hallucinatory toxicity in visual instruction data[C]//Proc of the 37th IEEE/CVF Conf on Computer Vision and Pattern Recognition. Piscataway, NJ: IEEE, 2024: 12944−12953, articleTitle=null, refAbstract=null), Reference(id=1327017592721617341, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327017570688938237, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=81, rfOrder=81, authorNames=null, journalName=null, refType=null, unstructuredReference=Chen Zhiyang, Zhu Yousong, Zhan Yufei, et al. Mitigating hallucination in visual language models with visual supervision[J]. arXiv preprint, arXiv: 2311. 16479, 2023, articleTitle=null, refAbstract=null), Reference(id=1327017592805503422, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327017570688938237, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=82, rfOrder=82, authorNames=null, journalName=null, refType=null, unstructuredReference=Chen Liang, Zhao Haozhe, Liu Tianyu, et al. An image is worth 1/2 tokens after layer 2: Plug-and-play inference acceleration for large vision-language models[J].arXiv:2403.06764, articleTitle=null, refAbstract=null), Reference(id=1327017592889389503, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327017570688938237, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=83, rfOrder=83, authorNames=null, journalName=null, refType=null, unstructuredReference=Huang Qidong, Dong Xiaoyi, Zhang Pan, et al. OPERA: Alleviating hallucination in multi-modal large language models via over-trust penalty and retrospection-allocation[C]//Proc of the 37th IEEE/CVF Conf on Computer Vision and Pattern Recognition. Piscataway, NJ: IEEE, 2024: 13418−13427, articleTitle=null, refAbstract=null), Reference(id=1327017592977469888, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327017570688938237, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=84, rfOrder=84, authorNames=null, journalName=null, refType=null, unstructuredReference=Zhang Yifan, Yu Weichen, Wen Qingsong, et al. Debiasing large visual language models[J]. arXiv preprint, arXiv: 2403. 05262, 2024, articleTitle=null, refAbstract=null), Reference(id=1327017593036190145, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327017570688938237, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=85, rfOrder=85, authorNames=null, journalName=null, refType=null, unstructuredReference=Chen Zhaorun, Zhao Zhuokai, Luo Hongyin, et al. HALC: Object hallucination reduction via adaptive focal-contrast decoding[C]//Proc of the 41st Int Conf on Machine Learning. New York: PMLR, 2024: 7824−7846, articleTitle=null, refAbstract=null), Reference(id=1327017593128464834, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327017570688938237, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=86, rfOrder=86, authorNames=null, journalName=null, refType=null, unstructuredReference=Mukherjee A, Chang H. The creative frontier of generative AI: Managing the novelty-usefulness tradeoff[J]. arXiv preprint, arXiv: 2306.03601, 2023, articleTitle=null, refAbstract=null), Reference(id=1327017593224933827, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327017570688938237, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=87, rfOrder=87, authorNames=null, journalName=null, refType=null, unstructuredReference=Wang Xintong, Pan Jingheng, Ding Liang, et al. Mitigating hallucinations in large vision-language models with instruction contrastive decoding[C]//Proc of the 18th Findings of the Association for Computational Linguistics. Stroudsburg, PA: ACL, 2024: 15840−15853, articleTitle=null, refAbstract=null), Reference(id=1327017593300431300, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327017570688938237, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=88, rfOrder=88, authorNames=null, journalName=null, refType=null, unstructuredReference=Krishna R, Zhu Yuke, Groth O, et al. Visual Genome: Connecting language and vision using crowdsourced dense image annotations[J]. International Journal of Computer Vision, 2017, 123(5): 32−73, articleTitle=null, refAbstract=null), Reference(id=1327017593405288901, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327017570688938237, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=89, rfOrder=89, authorNames=null, journalName=null, refType=null, unstructuredReference=Lin T-Y, Maire M, Belongie S, et al. Microsoft COCO: Common objects in context[C]//Proc of the 13th European Conf on Computer Vision. Berlin: Springer, 2014: 740−755, articleTitle=null, refAbstract=null), Reference(id=1327017593510146502, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327017570688938237, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=90, rfOrder=90, authorNames=null, journalName=null, refType=null, unstructuredReference=Chesser L, Hinegardner J, Notes A, et al. The UnSplash dataset[EB/OL].[2024-05-16]. https://github.com/unsplash/datasets, articleTitle=null, refAbstract=null), Reference(id=1327017593707278791, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327017570688938237, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=91, rfOrder=91, authorNames=null, journalName=null, refType=null, unstructuredReference=Deng Jia, Dong Wei, Socher R, et al. ImageNet: A large-scale hierarchical image database[C]//Proc of the 22nd IEEE Conf on Computer Vison and Pattern Recognition. Piscataway, NJ: IEEE, 2009: 248−255, articleTitle=null, refAbstract=null), Reference(id=1327017593791164872, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327017570688938237, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=92, rfOrder=92, authorNames=null, journalName=null, refType=null, unstructuredReference=Schuhmann C, Jitsev J, Vencu R, et al. LAION-Aesthetics[EB/OL].[2024-05-16]. https://laion.ai/blog/laion-asthetics/, articleTitle=null, refAbstract=null), Reference(id=1327017593875050953, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327017570688938237, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=93, rfOrder=93, authorNames=null, journalName=null, refType=null, unstructuredReference=Kuznetsova A, Rom H, Alldrin N, et al. The open images dataset v4: Unified image classification, object detection, and visual relationship detection at scale[J]. International Journal of Computer Vision, 2020, 128(7): 1956−1981, articleTitle=null, refAbstract=null), Reference(id=1327017593963131338, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327017570688938237, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=94, rfOrder=94, authorNames=null, journalName=null, refType=null, unstructuredReference=Arrieta A B, DÍaz-RodrÍguez N, Del-Ser J, et al. Explainable Artificial Intelligence (XAI): Concepts, taxonomies, opportunities and challenges toward responsible AI[J]. Information Fusion, 2020, 58(6): 82−115, articleTitle=null, refAbstract=null), Reference(id=1327017594034434507, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327017570688938237, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=95, rfOrder=95, authorNames=null, journalName=null, refType=null, unstructuredReference=OpenAI. ChatGPT[EB/OL].[2024-05-16]. https: /chat. openai. com/, articleTitle=null, refAbstract=null), Reference(id=1327017594101543372, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327017570688938237, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=96, rfOrder=96, authorNames=null, journalName=null, refType=null, unstructuredReference=Vedantam R, Lawrence-Zitnick C, Parikh D. CIDEr: Consensus-based image description evaluation[C]//Proc of the 28th IEEE/CVF Conf on Computer Vision and Pattern Recognition. Piscataway, NJ: IEEE, 2015: 4566−4575, articleTitle=null, refAbstract=null), Reference(id=1327017594164457933, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327017570688938237, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=97, rfOrder=97, authorNames=null, journalName=null, refType=null, unstructuredReference=Papineni K, Roukos S, Ward T, et al. BLEU: A method for automatic evaluation of machine translation[C]//Proc of the 40th Annual meeting of the Association for Computational Linguistics. Berlin: ACL, 2002: 311−318, articleTitle=null, refAbstract=null), Reference(id=1327017594231566798, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327017570688938237, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=98, rfOrder=98, authorNames=null, journalName=null, refType=null, unstructuredReference=Wang Peng, Yang An, Men Rui, et al. OFA: Unifying architectures, tasks, and modalities through a simple sequence-to-sequence learning framework[C]//Proc of the 39th Int Conf on Machine Learning. New York: PMLR, 2022: 23318−23340, articleTitle=null, refAbstract=null), Reference(id=1327017594290287055, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327017570688938237, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=99, rfOrder=99, authorNames=null, journalName=null, refType=null, unstructuredReference=Wang Wentao, Huang Xuanyao, Roy S K. DeepArt: A benchmark to advance fidelity research in AI-generated content[J]. arXiv preprint, arXiv: 2312. 10407, 2023, articleTitle=null, refAbstract=null), Reference(id=1327017594353201616, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327017570688938237, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=100, rfOrder=100, authorNames=null, journalName=null, refType=null, unstructuredReference=Huang Qingqiu, Xiong Yu, Rao Anyi, et al. MovieNet: A holistic dataset for movie understanding[C]//Proc of the 16th European Conf on Computer Vision. Berlin: Springer, 2020: 709−727, articleTitle=null, refAbstract=null), Reference(id=1327017594420310481, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327017570688938237, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=101, rfOrder=101, authorNames=null, journalName=null, refType=null, unstructuredReference=Zhang Chenyu, Van-Durme B, Li Zhuowan, et al. Visual commonsense in pretrained unimodal and multimodal models[C]//Proc of the 16th Conf of the North American Chapter of the Association for Computational Linguistics: Human Language Technologies. Stroudsburg, PA: ACL, 2022: 5321−5335, articleTitle=null, refAbstract=null), Reference(id=1327017594479030738, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327017570688938237, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=102, rfOrder=102, authorNames=null, journalName=null, refType=null, unstructuredReference=Ramesh A, Dhariwal P, Nichol A, et al. Hierarchical text-conditional image generation with CLIP latents[J]. arXiv preprint, arXiv: 2204. 06125, 2022, articleTitle=null, refAbstract=null), Reference(id=1327017594546139603, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327017570688938237, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=103, rfOrder=103, authorNames=null, journalName=null, refType=null, unstructuredReference=Oppenlaender J. The creativity of text-to-image generation[C]//Proc of the 25th Int Academic Mindtrek Conf. New York: ACM, 2022: 192−202, articleTitle=null, refAbstract=null), Reference(id=1327017594617442772, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327017570688938237, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=104, rfOrder=104, authorNames=null, journalName=null, refType=null, unstructuredReference=Rombach R, Blattmann A, Lorenz D, et al. High-resolution image synthesis with latent diffusion models[C]//Proc of the 35th IEEE/CVF Conf on Computer Vision and Pattern Recognition. Piscataway, NJ: IEEE, 2022: 10684−10695, articleTitle=null, refAbstract=null), Reference(id=1327017594676163029, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327017570688938237, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=105, rfOrder=105, authorNames=null, journalName=null, refType=null, unstructuredReference=Tang B J, Boggust A, Satyanarayan A. Vistext: A benchmark for semantically rich chart captioning[C]//Proc of the 61st Annual Meeting of the Association for Computational Linguistics. Stroudsburg, PA: ACL, 2023: 7268−7298, articleTitle=null, refAbstract=null), Reference(id=1327017594751660502, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327017570688938237, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=106, rfOrder=106, authorNames=null, journalName=null, refType=null, unstructuredReference=Liu Fuxiao, Wang Yinghan, Wang Tianlu, et al. Visual news: Benchmark and challenges in news image captioning[C]//Proc of the 26th Conf on Empirical Methods in Natural Language Processing. Stroudsburg, PA: ACL, 2021: 6761−6771, articleTitle=null, refAbstract=null), Reference(id=1327017594827157975, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327017570688938237, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=107, rfOrder=107, authorNames=null, journalName=null, refType=null, unstructuredReference=Honovich O, Choshen L, Aharoni R, et al. Q2: Evaluating factual consistency in knowledge-grounded dialogues via question generation and question answering[C]//Proc of the 26th Conf on Empirical Methods in Natural Language Processing. Stroudsburg, PA: ACL, 2021: 7856−7870, articleTitle=null, refAbstract=null), Reference(id=1327017594898461144, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327017570688938237, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=108, rfOrder=108, authorNames=null, journalName=null, refType=null, unstructuredReference=Chen Chi, Qin Ruoyu, Luo Fuwen, et al. Position-enhanced visual instruction tuning for multimodal large language models[J]. arXiv preprint, arXiv: 2308. 13437, 2023, articleTitle=null, refAbstract=null), Reference(id=1327017595036873177, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327017570688938237, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=109, rfOrder=109, authorNames=null, journalName=null, refType=null, unstructuredReference=Yang Dingchen, Cao Bowen, Chen Guang, et al. Pensieve: Retrospect-then-compare mitigates visual hallucination[J]. arXiv preprint, arXiv: 2403. 14401, 2024, articleTitle=null, refAbstract=null), Reference(id=1327017596739760602, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327017570688938237, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=110, rfOrder=110, authorNames=null, journalName=null, refType=null, unstructuredReference=Deng Ailin, Chen Zhirui, Hooi B. Seeing is believing: Mitigating hallucination in large vision-language models via CLIP-guided decoding[J]. arXiv preprint, arXiv: 2402. 15300, 2024, articleTitle=null, refAbstract=null), Reference(id=1327017596832035291, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327017570688938237, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=111, rfOrder=111, authorNames=null, journalName=null, refType=null, unstructuredReference=Mitra C, Huang B, Darrell T, et al. Compositional chain-of-thought prompting for large multimodal models[C]//Proc of the 37th IEEE/CVF Conf on Computer Vision and Pattern Recognition Workshops. Piscataway, NJ: IEEE, 2024: 14420−14431, articleTitle=null, refAbstract=null), Reference(id=1327017596936892892, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327017570688938237, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=112, rfOrder=112, authorNames=null, journalName=null, refType=null, unstructuredReference=Wu Junfei, Liu Qiang, Wang Ding, et al. Logical closed loop: Uncovering object hallucinations in large vision-language models[C]//Proc of the 18th Findings of the Association for Computational Linguistics. Stroudsburg, PA: ACL, 2024: 6944−6962, articleTitle=null, refAbstract=null), Reference(id=1327017597008196061, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327017570688938237, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=113, rfOrder=113, authorNames=null, journalName=null, refType=null, unstructuredReference=Goyal Y, Khot T, Summers-Stay D, et al. Making the V in VQA matter[C]//Proc of the 30th IEEE/CVF Conf on Computer Vision and Pattern Recognition. Piscataway, NJ: IEEE, 2017: 6904−6913, articleTitle=null, refAbstract=null), Reference(id=1327017597104665054, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327017570688938237, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=114, rfOrder=114, authorNames=null, journalName=null, refType=null, unstructuredReference=Schwenk D, Khandelwal A, Clark C, et al. A-OKVQA: A benchmark for visual question answering using world knowledge[C]//Proc of the 17th European Conf on Computer Vision. Berilin: Springer, 2022: 146−162, articleTitle=null, refAbstract=null), Reference(id=1327017597171773919, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327017570688938237, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=115, rfOrder=115, authorNames=null, journalName=null, refType=null, unstructuredReference=Young P, Lai A, Hodosh M, et al. From image descriptions to visual denotations: New similarity metrics for semantic inference over event descriptions[J]. Transactions of the Association for Computational Linguistics, 2014, 2: 67−78, articleTitle=null, refAbstract=null), Reference(id=1327017597243077088, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327017570688938237, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=115, rfOrder=116, authorNames=null, journalName=null, refType=null, unstructuredReference=Young P, Lai A, Hodosh M, et al. From image descriptions to visual denotations: New similarity metrics for semantic inference over event descriptions[J]. Transactions of the Association for Computational Linguistics, 2014, 2: 67−78, articleTitle=null, refAbstract=null), Reference(id=1327017597339546081, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327017570688938237, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=116, rfOrder=117, authorNames=null, journalName=null, refType=null, unstructuredReference=Yang Jingkang, Ang Yizhe, Guo Zujin, et al. Panoptic scene graph generation[C]//Proc of the 16th European Conf on Computer Vision. Berilin: Springer, 2022: 178−196, articleTitle=null, refAbstract=null), Reference(id=1327017597402460642, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327017570688938237, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=117, rfOrder=118, authorNames=null, journalName=null, refType=null, unstructuredReference=Zhong Yiwu, Yang Jianwei, Zhang Pengchuan, et al. RegionCLIP: Region-based language-image pretraining[C]//Proc of the 35th IEEE/CVF Conf on Computer Vision and Pattern Recognition. Piscataway, NJ: IEEE, 2022: 16793−16803, articleTitle=null, refAbstract=null), Reference(id=1327017597498929635, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327017570688938237, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=118, rfOrder=119, authorNames=null, journalName=null, refType=null, unstructuredReference=Kirillov A, Mintun E, Ravi N, et al. Segment anything[C]//Proc of the 36th IEEE/CVF Int Conf on Computer Vision. Piscataway, NJ: IEEE, 2023: 4015−4026, articleTitle=null, refAbstract=null), Reference(id=1327017597570232805, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327017570688938237, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=119, rfOrder=120, authorNames=null, journalName=null, refType=null, unstructuredReference=Zhang S, Roller S, Goyal N, et al. OPT: Open pre-trained transformer language models[J]. arXiv preprint, arXiv: 2205. 01068, 2022, articleTitle=null, refAbstract=null), Reference(id=1327017597654118886, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327017570688938237, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=120, rfOrder=121, authorNames=null, journalName=null, refType=null, unstructuredReference=Oquab M, Darcet T, Moutakanni T, et al. DINOv2: Learning robust visual features without supervision[J]. arXiv preprint, arXiv: 2304. 07193, 2023, articleTitle=null, refAbstract=null), Reference(id=1327017597742199271, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327017570688938237, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=121, rfOrder=122, authorNames=null, journalName=null, refType=null, unstructuredReference=Huang Yupan, Lv Tengchao, Cui Lei, et al. LayoutLMv3: Pre-training for document ai with unified text and image masking[C]//Proc of the 30th ACM Int Conf on Multimedia. New York: ACM, 2022: 4083−4091, articleTitle=null, refAbstract=null), Reference(id=1327017597817696744, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327017570688938237, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=122, rfOrder=123, authorNames=null, journalName=null, refType=null, unstructuredReference=Liu Zhuang, Mao Hanzi, Wu Chaoyuan, et al. A convnet for the 2020s[C]//Proc of the 35th IEEE/CVF Conf on Computer Vision and Pattern Recognition. Piscataway, NJ: IEEE, 2022: 11976−11986, articleTitle=null, refAbstract=null), Reference(id=1327017597914165737, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327017570688938237, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=123, rfOrder=124, authorNames=null, journalName=null, refType=null, unstructuredReference=Zhang Youcai, Huang Xinyu, Ma Jinyu, et al. Recognize anything: A strong image tagging model[C]//Proc of the 37th IEEE/CVF Conf on Computer Vision and Pattern Recognition Workshops. Piscataway, NJ: IEEE, 2024: 1724−1732, articleTitle=null, refAbstract=null), Reference(id=1327017597989663210, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327017570688938237, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=124, rfOrder=125, authorNames=null, journalName=null, refType=null, unstructuredReference=Hu E J, Shen Yelong, Wallis P, et al. LoRA: Low-rank adaptation of large language models[C/OL].[2025-04-04]. https://openreview.net/forum?id=nZeVKeeFYf9, articleTitle=null, refAbstract=null), Reference(id=1327017598077743595, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327017570688938237, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=125, rfOrder=126, authorNames=null, journalName=null, refType=null, unstructuredReference=Rafailov R, Sharma A, Mitchell E, et al. Direct preference optimization: Your language model is secretly a reward model[C]//Proc of the 37th Int Conf on Neural Information Processing Systems. New York: Curran Associates, 2023: 53728−53741, articleTitle=null, refAbstract=null), Reference(id=1327017598186795500, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327017570688938237, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=126, rfOrder=127, authorNames=null, journalName=null, refType=null, unstructuredReference=Li X L, Holtzman A, Fried D, et al. Contrastive decoding: Open-ended text generation as optimization[C]//Proc of the 61st Annual Meeting of the Association for Computational Linguistics. Stroudsburg, PA: ACL, 2023: 12286−12312, articleTitle=null, refAbstract=null), Reference(id=1327017598249710061, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327017570688938237, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=127, rfOrder=128, authorNames=null, journalName=null, refType=null, unstructuredReference=Agrawal H, Desai K, Wang Yufei, et al. Nocaps: Novel object captioning at scale[C]//Proc of the 13th IEEE/CVF Int Conf on Computer Vision. Piscataway, NJ: IEEE, 2019: 8948−8957, articleTitle=null, refAbstract=null), Reference(id=1327017598321013230, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327017570688938237, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=128, rfOrder=129, authorNames=null, journalName=null, refType=null, unstructuredReference=Belle V, Papantonis I. Principles and practice of explainable machine learning[J]. Frontiers in Big Data, 2021, 4(7): 1−25, articleTitle=null, refAbstract=null)], funds=null, companyList=[AuthorCompany(id=1327017575063597339, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327017570688938237, xref=1, ext=[AuthorCompanyExt(id=1327017575071985948, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327017570688938237, companyId=1327017575063597339, language=EN, country=null, province=null, city=null, postcode=null, companyName=null, departmentName=null, remark=
1Institute of Big Data, Fudan University, Shanghai 200433), AuthorCompanyExt(id=1327017575084568861, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327017570688938237, companyId=1327017575063597339, language=CN, country=null, province=null, city=null, postcode=null, companyName=null, departmentName=null, remark=
1复旦大学大数据研究院 上海 200433)]), AuthorCompany(id=1327017575151677726, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327017570688938237, xref=2, ext=[AuthorCompanyExt(id=1327017575160066335, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327017570688938237, companyId=1327017575151677726, language=EN, country=null, province=null, city=null, postcode=null, companyName=null, departmentName=null, remark=
2School of Computer Science, Fudan University, Shanghai 200438), AuthorCompanyExt(id=1327017575168454944, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327017570688938237, companyId=1327017575151677726, language=CN, country=null, province=null, city=null, postcode=null, companyName=null, departmentName=null, remark=
2复旦大学计算机科学技术学院 上海 200438)])], figs=[ArticleFig(id=1327017579882852698, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327017570688938237, language=EN, label=Figure 1, caption=
The hallucinatory scenes in LVLMs, figureFileSmall=DsV98KR0ZXdJ5Q32/GVRyg==, figureFileBig=QH2mWQagSxaHYBiUAfi5kg==, tableContent=null), ArticleFig(id=1327017579958350171, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327017570688938237, language=CN, label=图1, caption=
LVLMs中的幻觉场景, figureFileSmall=DsV98KR0ZXdJ5Q32/GVRyg==, figureFileBig=QH2mWQagSxaHYBiUAfi5kg==, tableContent=null), ArticleFig(id=1327017580189036892, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327017570688938237, language=EN, label=Figure 2, caption=
The typical architecture of LVLMs, figureFileSmall=4PEVgGLBsGh7rlI8iKmbMg==, figureFileBig=mubLEFs5NiEbNKRNUo3Kew==, tableContent=null), ArticleFig(id=1327017580260340061, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327017570688938237, language=CN, label=图2, caption=
LVLMs的典型架构, figureFileSmall=4PEVgGLBsGh7rlI8iKmbMg==, figureFileBig=mubLEFs5NiEbNKRNUo3Kew==, tableContent=null), ArticleFig(id=1327017580365197662, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327017570688938237, language=EN, label=Figure 3, caption=
The hallucinations phenomenon in LVLMs, figureFileSmall=uyzR8aeAz6fSE49FnmLf7w==, figureFileBig=iVKPcxbQqpAF6woDqDPFfQ==, tableContent=null), ArticleFig(id=1327017580444889439, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327017570688938237, language=CN, label=图3, caption=
LVLMs中的幻觉现象, figureFileSmall=uyzR8aeAz6fSE49FnmLf7w==, figureFileBig=iVKPcxbQqpAF6woDqDPFfQ==, tableContent=null), ArticleFig(id=1327017580507804000, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327017570688938237, language=EN, label=Figure 4, caption=
Frame work diagram of hallucinations causes in LVLMs, figureFileSmall=Sttxx2QTRDzyglrQ2HxMCg==, figureFileBig=nbUal10pnrdA3uBO2X9Uyg==, tableContent=null), ArticleFig(id=1327017580600078689, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327017570688938237, language=CN, label=图4, caption=
LVLMs幻觉成因框架图, figureFileSmall=Sttxx2QTRDzyglrQ2HxMCg==, figureFileBig=nbUal10pnrdA3uBO2X9Uyg==, tableContent=null), ArticleFig(id=1327017580662993250, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327017570688938237, language=EN, label=Table 1, caption=
Hallucination Evaluation Benchmarks for LVLMs, figureFileSmall=null, figureFileBig=null, tableContent=
| 基准名称 | 任务形式 | 图像来源 | 图像 数量 | 问答对生 成方式 | 问题类型 | 问答对 数量 | 评估幻觉类型 | 评估指标 |
|---|
| CCEval[74] | 描述 | Visual Genome[88] | 1 000 | 无 | 无 | 无 | 存在 | CHAIR、Coverage、平均回答长度、回答中平均物体数量 |
| MME[54] | 问答 | 人工收集 | 1 097 | 人工设计 | 判断 | 2 194 | 存在和属性 | 准确率、Accuracy+ |
| CIEM[55] | 问答 | 任何含图片描 述的数据集 | 由数据集决定 | LLM生成 | 判断 | 由数据集决定 | 存在和属性 | 准确率、精确率、召回率、F1分数、负样本召回率 |
| RAH-Bench[81] | 问答 | MSCOCO[89] | 3 000 | GPT-4生成 | 判断 | 1 500 | 存在、属性和关系 | 精确率、召回率、 F1分数、误报率 |
| HALLUSIONBENCH[57] | 问答 | 人工收集 | 346 | 人工设计 | 判断 | 1 129 | 存在、属性和关系 | 总准确性、图像准确性、 问答对准确性、Yes百分 比差异、误报率 |
| AMBER[53] | 描述和问答 | MSCOCO[89],UnSplash[90] | 1 004 | 人工设计 | 判断 | 15 220 | 存在、属性和关系 | CHAIR、Cover、Hal、 Cog、准确率、精确率、 召回率、F1分数 |
| MMVP[16] | 问答 | ImageNet[91],LAION-Aesthetic[92] | 300 | 人工设计 | 选择 | 300 | 存在、属性和关系 | 准确率 |
| PhD[44] | 问答 | 人工收集 | 7 000 | GPT-3.5 | 判断和问答 | 53 796 | 存在、属性和关系 | 准确率 |
| LLaVA-Bench[17] | 描述和问答 | 人工收集 | 54 | GPT-4 | 问答 | 150 | 存在、属性和关系 | GPT-4打分 |
| WHOOPS[58] | 描述和问答 | 生成图像 | 500 | 人工和自动生成技术 | 选择和问答 | 4 374 | 存在、属性和关系 | CIDEr、BLEU-4、严格 匹配、BERT匹配 |
| LRV-Instruction+GAVIE[48] | 问答 | 人工收集 | 35 000 | GPT-4 | 判断、选择和问答 | 400 000 | 存在、属性和关系 | GPT-4打分 |
| MMHAL-BENCH[63] | 描述和问答 | OpenImages[93] | 96 | 人工设计 | 问答 | 96 | 存在、属性和关系 | GPT-4打分 |
| Bingo[59] | 描述和问答 | 人工收集 | 308 | 人工设计 | 问答 | 370 | 存在、属性和关系 | 准确率 |
| FAITHSCORE-Bench[60] | 描述和问答 | MSCOCO[89] | 2 000 | GPT-4 | 问答 | 3 000 | 存在、属性和关系 | FAITHSCORE |
), ArticleFig(id=1327017580793016675, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327017570688938237, language=CN, label=表1, caption=
LVLMs幻觉评估基准
, figureFileSmall=null, figureFileBig=null, tableContent=
| 基准名称 | 任务形式 | 图像来源 | 图像 数量 | 问答对生 成方式 | 问题类型 | 问答对 数量 | 评估幻觉类型 | 评估指标 |
|---|
| CCEval[74] | 描述 | Visual Genome[88] | 1 000 | 无 | 无 | 无 | 存在 | CHAIR、Coverage、平均回答长度、回答中平均物体数量 |
| MME[54] | 问答 | 人工收集 | 1 097 | 人工设计 | 判断 | 2 194 | 存在和属性 | 准确率、Accuracy+ |
| CIEM[55] | 问答 | 任何含图片描 述的数据集 | 由数据集决定 | LLM生成 | 判断 | 由数据集决定 | 存在和属性 | 准确率、精确率、召回率、F1分数、负样本召回率 |
| RAH-Bench[81] | 问答 | MSCOCO[89] | 3 000 | GPT-4生成 | 判断 | 1 500 | 存在、属性和关系 | 精确率、召回率、 F1分数、误报率 |
| HALLUSIONBENCH[57] | 问答 | 人工收集 | 346 | 人工设计 | 判断 | 1 129 | 存在、属性和关系 | 总准确性、图像准确性、 问答对准确性、Yes百分 比差异、误报率 |
| AMBER[53] | 描述和问答 | MSCOCO[89],UnSplash[90] | 1 004 | 人工设计 | 判断 | 15 220 | 存在、属性和关系 | CHAIR、Cover、Hal、 Cog、准确率、精确率、 召回率、F1分数 |
| MMVP[16] | 问答 | ImageNet[91],LAION-Aesthetic[92] | 300 | 人工设计 | 选择 | 300 | 存在、属性和关系 | 准确率 |
| PhD[44] | 问答 | 人工收集 | 7 000 | GPT-3.5 | 判断和问答 | 53 796 | 存在、属性和关系 | 准确率 |
| LLaVA-Bench[17] | 描述和问答 | 人工收集 | 54 | GPT-4 | 问答 | 150 | 存在、属性和关系 | GPT-4打分 |
| WHOOPS[58] | 描述和问答 | 生成图像 | 500 | 人工和自动生成技术 | 选择和问答 | 4 374 | 存在、属性和关系 | CIDEr、BLEU-4、严格 匹配、BERT匹配 |
| LRV-Instruction+GAVIE[48] | 问答 | 人工收集 | 35 000 | GPT-4 | 判断、选择和问答 | 400 000 | 存在、属性和关系 | GPT-4打分 |
| MMHAL-BENCH[63] | 描述和问答 | OpenImages[93] | 96 | 人工设计 | 问答 | 96 | 存在、属性和关系 | GPT-4打分 |
| Bingo[59] | 描述和问答 | 人工收集 | 308 | 人工设计 | 问答 | 370 | 存在、属性和关系 | 准确率 |
| FAITHSCORE-Bench[60] | 描述和问答 | MSCOCO[89] | 2 000 | GPT-4 | 问答 | 3 000 | 存在、属性和关系 | FAITHSCORE |
), ArticleFig(id=1327017580876902756, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327017570688938237, language=EN, label=Table 2, caption=
Hallucination Management Method for LVLMs, figureFileSmall=null, figureFileBig=null, tableContent=
| 幻觉治理层面 | 幻觉治理思路 | 幻觉治理方法 | 对应的幻觉成因 |
|---|
| 数据侧 | 提升指令数 据的多样性 | LRV-instruction[48] | 训练数据中的统计偏差 |
| CIT[55] | 训练数据中的统计偏差 |
| Consistency Training[46] | 训练数据中的统计偏差 |
增强图像 文本关联度 | VIGC[62] | 训练数据中存在噪声 |
| LLaVA-RLHF[63] | 训练数据中存在噪声 |
| RAI-30K[81] | 训练数据中存在噪声 |
| shareGPT4V[78] | 文本描述不够细致 |
| 改造现有数据集 | HalluciDoctor[80] | 训练数据中存在噪声 |
| 视觉感知侧 | 集成多个视 觉编码器 | PVIT[108] | 细粒度视觉信息感知能力不足 |
| Vary[15] | 细粒度视觉信息感知能力不足 |
| Lyrics[20] | 细粒度视觉信息感知能力不足 |
| MoF[16] | 细粒度视觉信息感知能力不足 |
| MouSi[21] | 细粒度视觉信息感知能力不足 |
利用单个视觉编码器的多层级 视觉特征 | LION[18] | 细粒度视觉信息感知能力不足 |
| 增加视觉编码器的参数规模 | InternVL[19] | 细粒度视觉信息感知能力不足 |
| 训练侧 | 新的监督信号 | ObjMLM[51] | 单一的语言建模训练任务 |
| Mask Prediction[16] | 单一的语言建模训练任务 |
| 强化学习 | LLaVA-RLHF[63] | 单一的语言建模训练任务 |
| FDPO[61] | 单一的语言建模训练任务 |
| 增加任务信息 | MiniGPT-v2[22] | 训练数据中存在统计偏差 |
| 推理侧 | 对比解码 | OPERA[83] | 视觉特征得不到充分关注 |
| VDD[84] | LLM的先验性知识偏差 |
| Pensieve[109] | 训练数据中存在统计偏差 |
| VCD[66] | 自回归解码中的偏差累积 |
| MARINE[23] | 自回归解码中的偏差累积 |
| ICD[87] | 自回归解码中的偏差累积 |
| HALC[85] | 自回归解码中的偏差累积 |
| CGD[110] | 自回归解码中的偏差累积 |
| 思维链 | GroundingCoT[24] | |
| Volcano[67] | |
| CCoT[111] | |
| 事后修正侧 | 基于训练的方法 | LURE[65] | 固有的采样随机性 |
| 无需训练的方法 | Woodpecker[64] | 固有的采样随机性 |
| LogicCheckGPT[112] | 固有的采样随机性 |
), ArticleFig(id=1327017580977566053, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327017570688938237, language=CN, label=表2, caption=
LVLMs幻觉的治理方法
, figureFileSmall=null, figureFileBig=null, tableContent=
| 幻觉治理层面 | 幻觉治理思路 | 幻觉治理方法 | 对应的幻觉成因 |
|---|
| 数据侧 | 提升指令数 据的多样性 | LRV-instruction[48] | 训练数据中的统计偏差 |
| CIT[55] | 训练数据中的统计偏差 |
| Consistency Training[46] | 训练数据中的统计偏差 |
增强图像 文本关联度 | VIGC[62] | 训练数据中存在噪声 |
| LLaVA-RLHF[63] | 训练数据中存在噪声 |
| RAI-30K[81] | 训练数据中存在噪声 |
| shareGPT4V[78] | 文本描述不够细致 |
| 改造现有数据集 | HalluciDoctor[80] | 训练数据中存在噪声 |
| 视觉感知侧 | 集成多个视 觉编码器 | PVIT[108] | 细粒度视觉信息感知能力不足 |
| Vary[15] | 细粒度视觉信息感知能力不足 |
| Lyrics[20] | 细粒度视觉信息感知能力不足 |
| MoF[16] | 细粒度视觉信息感知能力不足 |
| MouSi[21] | 细粒度视觉信息感知能力不足 |
利用单个视觉编码器的多层级 视觉特征 | LION[18] | 细粒度视觉信息感知能力不足 |
| 增加视觉编码器的参数规模 | InternVL[19] | 细粒度视觉信息感知能力不足 |
| 训练侧 | 新的监督信号 | ObjMLM[51] | 单一的语言建模训练任务 |
| Mask Prediction[16] | 单一的语言建模训练任务 |
| 强化学习 | LLaVA-RLHF[63] | 单一的语言建模训练任务 |
| FDPO[61] | 单一的语言建模训练任务 |
| 增加任务信息 | MiniGPT-v2[22] | 训练数据中存在统计偏差 |
| 推理侧 | 对比解码 | OPERA[83] | 视觉特征得不到充分关注 |
| VDD[84] | LLM的先验性知识偏差 |
| Pensieve[109] | 训练数据中存在统计偏差 |
| VCD[66] | 自回归解码中的偏差累积 |
| MARINE[23] | 自回归解码中的偏差累积 |
| ICD[87] | 自回归解码中的偏差累积 |
| HALC[85] | 自回归解码中的偏差累积 |
| CGD[110] | 自回归解码中的偏差累积 |
| 思维链 | GroundingCoT[24] | |
| Volcano[67] | |
| CCoT[111] | |
| 事后修正侧 | 基于训练的方法 | LURE[65] | 固有的采样随机性 |
| 无需训练的方法 | Woodpecker[64] | 固有的采样随机性 |
| LogicCheckGPT[112] | 固有的采样随机性 |
)], attaches=null, journal=Journal(id=1324759606947315908, delFlag=0, nameCn=计算机研究与发展, nameEn=Journal of Computer Research and Development, nameHistory1=null, nameHistory2=null, issn=1000-1239, eissn=null, cn=11-1777/TP, coden=null, periodic=0, language=CN, oaType=null, ccby=null, superviseOffice=null, ownerOffice=null, pubOffice=null, editorOffice=null, officeType=null, aims=null, clcCode=null, officeProv=null, officeCity=null, officeAddr=null, officeZip=null, officeEmail=null, officePhone=null, editDirector=null, officeDirector=null, officeDirectorPhone=null, officeStaffNum=null, officeEmpNum=null, coverPicUrl=k2l8zgqwstuxxevFX2qxVA==, journalPrice=null, startedYear=null, abbrevIsoEn=Journal of Computer Research and Development, journalRemark=null, publicationField=null, createdTime=1793776903141, updatedTime=1793850698612, createdBy=18614031015, updatedBy=13701087609, firstLetterCn=J, firstLetterEn=J, subjectCode=Electronic Information, subjectName=null, subjectCodeEn=Electronic Information, subjectNameEn=null, picCn=k2l8zgqwstuxxevFX2qxVA==, picEn=r6k0Ry+RD+c/VXj5QarmDw==, jcr=null, cjcr=null, exts=[JournalExt(id=1325069127710286688, language=CN, name=计算机研究与发展, nameHistory1=null, nameHistory2=null, managedBy=, sponsoredBy=, publishedBy=, editorOffice=, officeProv=null, officeCity=null, officeAddr=, officeZip=, editDirector=, officeDirector=null, officePhone=null, coverPicUrl=null, journalRemark=, submitArticleUrl=null, websiteUrl=, createdTime=1793850698640, updatedTime=1793850698640, createdBy=13701087609, updatedBy=13701087609, submissionGuidelinesUrl=, submissionAuthorUrl=https://crad.ict.ac.cn/journalx/authorLogOn.action, submissionEditorUrl=https://crad.ict.ac.cn/journalx/editorLogOn.action, submissionReviewUrl=https://crad.ict.ac.cn/journalx/expertLogOn.action, submissionCeEditorUrl=https://crad.ict.ac.cn/journalx/editorInChiefLogOn.action, submissionAeEditorUrl=, option={"copyright":""}), JournalExt(id=1325069127777395553, language=EN, name=Journal of Computer Research and Development, nameHistory1=null, nameHistory2=null, managedBy=, sponsoredBy=, publishedBy=, editorOffice=, officeProv=null, officeCity=null, officeAddr=, officeZip=, editDirector=, officeDirector=null, officePhone=null, coverPicUrl=null, journalRemark=, submitArticleUrl=null, websiteUrl=, createdTime=1793850698656, updatedTime=1793850698656, createdBy=13701087609, updatedBy=13701087609, submissionGuidelinesUrl=, submissionAuthorUrl=https://crad.ict.ac.cn/journalx/authorLogOn.action, submissionEditorUrl=https://crad.ict.ac.cn/journalx/editorLogOn.action, submissionReviewUrl=https://crad.ict.ac.cn/journalx/expertLogOn.action, submissionCeEditorUrl=https://crad.ict.ac.cn/journalx/editorInChiefLogOn.action, submissionAeEditorUrl=, option={"copyright":""})], databaseList=null, tenantJournalId=1324759914146529478, websiteList=[Website(id=1325070300139892771, webName=null, webTitle=null, webDomain=null, webCopyrigh=null, webIpcNo=null, seoTitle=null, seoKeywords=null, seoDescription=null, tenantJournalId=null, journalId=1324759914146529478, journalNameCn=null, journalNameEn=null, grayFlag=null, tenantId=1146029695717560320, platformId=null, journalGroupId=null, journalGroupNameCn=null, journalGroupNameEn=null, type=1, domain=https://castjournals.cast.org.cn/joweb/jsjyjyfz/CN, language=CN, createTime=1793850978169, createBy=18614031015, updateTime=1793851040259, updateBy=18614031015, name=计算机研究与发展-中文, tplId=1146099689490845704, title=计算机研究与发展, delFlag=0, indexPage=/home, props=[WebsiteProps(id=1325070893751349360, tenantId=1146029695717560320, journalId=null, journalGroupId=null, siteId=1325070300139892771, code=articleTextType, value=kx, createTime=1793851119697, updateTime=1793851119697, creator=18614031015, updator=18614031015), WebsiteProps(id=1325070893730377837, tenantId=1146029695717560320, journalId=null, journalGroupId=null, siteId=1325070300139892771, code=banner, value=null, createTime=1793851119692, updateTime=1793851119692, creator=18614031015, updator=18614031015), WebsiteProps(id=1325070893772320883, tenantId=1146029695717560320, journalId=null, journalGroupId=null, siteId=1325070300139892771, code=grayFlag, value=0, createTime=1793851119702, updateTime=1793851119702, creator=18614031015, updator=18614031015), WebsiteProps(id=1325070893721989228, tenantId=1146029695717560320, journalId=null, journalGroupId=null, siteId=1325070300139892771, code=logo, value=https://castjournals.cast.org.cn/joweb/jsjyjyfz/CN/file/pic?fileId=U27tDsy717YF7XSPA+Vwdw==, createTime=1793851119690, updateTime=1793851119690, creator=18614031015, updator=18614031015), WebsiteProps(id=1325070893784903797, tenantId=1146029695717560320, journalId=null, journalGroupId=null, siteId=1325070300139892771, code=minRunFlag, value=0, createTime=1793851119705, updateTime=1793851119705, creator=18614031015, updator=18614031015), WebsiteProps(id=1325070893742960751, tenantId=1146029695717560320, journalId=null, journalGroupId=null, siteId=1325070300139892771, code=picServerUrl, value=https://castjournals.cast.org.cn/joweb/jsjyjyfz/CN/file/pic, createTime=1793851119695, updateTime=1793851119695, creator=18614031015, updator=18614031015), WebsiteProps(id=1325070893780709492, tenantId=1146029695717560320, journalId=null, journalGroupId=null, siteId=1325070300139892771, code=silenceFlag, value=0, createTime=1793851119704, updateTime=1793851119704, creator=18614031015, updator=18614031015), WebsiteProps(id=1325070893738766446, tenantId=1146029695717560320, journalId=null, journalGroupId=null, siteId=1325070300139892771, code=staticResourcePath, value=https://castjournals.cast.org.cn/joweb/cast_kjdb_cn_619/, createTime=1793851119694, updateTime=1793851119694, creator=18614031015, updator=18614031015), WebsiteProps(id=1325070893755543665, tenantId=1146029695717560320, journalId=null, journalGroupId=null, siteId=1325070300139892771, code=themeColor, value=null, createTime=1793851119698, updateTime=1793851119698, creator=18614031015, updator=18614031015), WebsiteProps(id=1325070893763932274, tenantId=1146029695717560320, journalId=null, journalGroupId=null, siteId=1325070300139892771, code=themeStyle, value=null, createTime=1793851119700, updateTime=1793851119700, creator=18614031015, updator=18614031015)]), Website(id=1325070300580294698, webName=null, webTitle=null, webDomain=null, webCopyrigh=null, webIpcNo=null, seoTitle=null, seoKeywords=null, seoDescription=null, tenantJournalId=null, journalId=1324759914146529478, journalNameCn=null, journalNameEn=null, grayFlag=null, tenantId=1146029695717560320, platformId=null, journalGroupId=null, journalGroupNameCn=null, journalGroupNameEn=null, type=1, domain=https://castjournals.cast.org.cn/joweb/jsjyjyfz/EN, language=EN, createTime=1793850978274, createBy=18614031015, updateTime=1793851036638, updateBy=18614031015, name=计算机研究与发展-英文, tplId=1146101810881728533, title=Journal of Computer Research and Development, delFlag=0, indexPage=/home, props=[WebsiteProps(id=1325070925288321148, tenantId=1146029695717560320, journalId=null, journalGroupId=null, siteId=1325070300580294698, code=articleTextType, value=kx, createTime=1793851127216, updateTime=1793851127216, creator=18614031015, updator=18614031015), WebsiteProps(id=1325070925263155321, tenantId=1146029695717560320, journalId=null, journalGroupId=null, siteId=1325070300580294698, code=banner, value=null, createTime=1793851127210, updateTime=1793851127210, creator=18614031015, updator=18614031015), WebsiteProps(id=1325070925309292671, tenantId=1146029695717560320, journalId=null, journalGroupId=null, siteId=1325070300580294698, code=grayFlag, value=0, createTime=1793851127221, updateTime=1793851127221, creator=18614031015, updator=18614031015), WebsiteProps(id=1325070925254766712, tenantId=1146029695717560320, journalId=null, journalGroupId=null, siteId=1325070300580294698, code=logo, value=https://castjournals.cast.org.cn/joweb/jsjyjyfz/EN/file/pic?fileId=U27tDsy717YF7XSPA+Vwdw==, createTime=1793851127208, updateTime=1793851127208, creator=18614031015, updator=18614031015), WebsiteProps(id=1325070925326069889, tenantId=1146029695717560320, journalId=null, journalGroupId=null, siteId=1325070300580294698, code=minRunFlag, value=0, createTime=1793851127225, updateTime=1793851127225, creator=18614031015, updator=18614031015), WebsiteProps(id=1325070925279932539, tenantId=1146029695717560320, journalId=null, journalGroupId=null, siteId=1325070300580294698, code=picServerUrl, value=https://castjournals.cast.org.cn/joweb/jsjyjyfz/EN/file/pic, createTime=1793851127214, updateTime=1793851127214, creator=18614031015, updator=18614031015), WebsiteProps(id=1325070925317681280, tenantId=1146029695717560320, journalId=null, journalGroupId=null, siteId=1325070300580294698, code=silenceFlag, value=0, createTime=1793851127223, updateTime=1793851127223, creator=18614031015, updator=18614031015), WebsiteProps(id=1325070925271543930, tenantId=1146029695717560320, journalId=null, journalGroupId=null, siteId=1325070300580294698, code=staticResourcePath, value=https://castjournals.cast.org.cn/joweb/cast_kjdb_en_623/, createTime=1793851127212, updateTime=1793851127212, creator=18614031015, updator=18614031015), WebsiteProps(id=1325070925292515453, tenantId=1146029695717560320, journalId=null, journalGroupId=null, siteId=1325070300580294698, code=themeColor, value=null, createTime=1793851127217, updateTime=1793851127217, creator=18614031015, updator=18614031015), WebsiteProps(id=1325070925300904062, tenantId=1146029695717560320, journalId=null, journalGroupId=null, siteId=1325070300580294698, code=themeStyle, value=null, createTime=1793851127219, updateTime=1793851127219, creator=18614031015, updator=18614031015)])], journalTitle=计算机研究与发展, weixinUrl=null, journalUrl=https://crad.ict.ac.cn/, iacademicId=null, status=1, seqNo=null, journalTitleEn=Journal of Computer Research and Development, journalPhotoCn=k2l8zgqwstuxxevFX2qxVA==, journalPhotoEn=r6k0Ry+RD+c/VXj5QarmDw==, journalFirstLetter=J, journalRecommend=null, journalNew=null, journalCollection=null, jcrJf=null, cjcrJf=null, jcrJfStr=null, cjcrJfStr=null, submissionFirstDecision=null, sciSubjectClassification=null, casSubjectClassification=null, citeScore=null, totalCitationFrequency=null, icpCode=null, psCode=null, advertisingLicenseCode=null, copyrightInformation=null, country=null, option=, provinceCode=null, provinceName=null, collectFlag=false, interPubPlatform=, interPubPlatformUrl=null), detailUrlCn=https://castjournals.cast.org.cn/joweb/jsjyjyfz/CN/10.7544/issn1000-1239.202440444, detailUrlEn=https://castjournals.cast.org.cn/joweb/jsjyjyfz/EN/10.7544/issn1000-1239.202440444, pdfUrlCn=https://castjournals.cast.org.cn/joweb/jsjyjyfz/CN/PDF/10.7544/issn1000-1239.202440444, pdfUrlEn=https://castjournals.cast.org.cn/joweb/jsjyjyfz/EN/PDF/10.7544/issn1000-1239.202440444, aliStartDate=0, aliEndDate=0, collectionFlag=false, citedCount=null, citedUrl=null, previewStatus=0, delFlag=0, hasFullText=1, orderTime=1764518400000, fullTextJson=null, articleText=null, reference=null)