Article(id=1329456429590753391, tenantId=1146029695717560320, journalId=1324759914146529478, issueId=1329456279791186761, articleNumber=null, orderNo=null, doi=10.7544/issn1000-1239.202221008, pmid=null, cstr=null, oa=null, hot=null, price=null, onlineType=0, articleFormat=0, articleType=null, articleTypeStr=null, receivedDate=1671120000000, receivedDateStr=2022-12-16, revisedDate=1687708800000, revisedDateStr=2023-06-26, acceptedDate=null, acceptedDateStr=null, onlineDate=1794896712920, onlineDateStr=2026-11-17, pubDate=1711900800000, pubDateStr=2024-04-01, doiRegisterDate=null, doiRegisterDateStr=null, onlineIssueDate=1794896712920, onlineIssueDateStr=2026-11-17, onlineJustAcceptDate=null, onlineJustAcceptDateStr=null, onlineFirstDate=null, onlineFirstDateStr=null, sourceXml=null, magXml=null, createTime=1794896712920, creator=13701087609, updateTime=1794896712920, updator=13701087609, issue=Issue{id=1329456279791186761, tenantId=1146029695717560320, journalId=1324759914146529478, year='2024', volume='61', issue='4', pageStart='809', pageEnd='1084', issueExtLink='null', onlineDate='null', pubDate='1711900800000', pubDateStr='2024-04-01', beforeIssueId=null, nextIssueId=null, price=null, status=1, issueComplete=1, articleOrder=1, issueType=-1, specialIssue=null, createTime=1794896677204, creator='13701087609', updateTime=1794899232988, updator='13701087609', preIssue=null, nextIssue=null, articleTotal=null, ext={EN=IssueExt(id=1329466999882756515, tenantId=1146029695717560320, journalId=1324759914146529478, issueId=1329456279791186761, language=EN, specialIssueTitle=, coverIllustrator=null, specialIssueEditor=, specialIssueAbout=), CN=IssueExt(id=1329466999882756516, tenantId=1146029695717560320, journalId=1324759914146529478, issueId=1329456279791186761, language=CN, specialIssueTitle=, coverIllustrator=null, specialIssueEditor=, specialIssueAbout=)}, issueFiles=null, downloadFileDto=null}, startPage=889, endPage=902, ext={EN=ArticleExt(id=1329456429796274288, articleId=1329456429590753391, tenantId=1146029695717560320, journalId=1324759914146529478, language=EN, title=Video Question Answering Scheme Base on Multimodal Knowledge Active Learning, columnId=1325489891789910943, journalTitle=Journal of Computer Research and Development, columnName=Artificial Intelligence, runingTitle=null, highlight=null, articleAbstract=
Video question answering requires models to understand, fuse, and reason about the multimodal data in videos to assist people in quickly retrieving, analyzing, and summarizing complex scenes in videos, becoming a hot research topic in artificial intelligence. However, existing methods lack abilities of obtaining the motion details of visual objects in feature extraction, which may lead to false causality. In addition, in data fusion and reasoning, existing methods lack effective active learning ability, making it difficult to obtain prior knowledge beyond feature extraction, which affects the model’s deep understanding of multimodal content. To address these issues, we propose a multimodal knowledge-based active learning video question answering solution. The solution acquires the semantic correlation of visual targets in image sequences and the dynamic relationship with the surrounding environment to establish the motion trajectory of each visual target. Further, static content is supplemented with dynamic content to provide more accurate video feature expression for data fusion and reasoning. Then, the solution achieves self-improvement and logical thinking focus of multimodal information understanding through knowledge auto-enhancement multimodal data fusion and reasoning model, filling the gap in deep understanding of multimodal content. Experimental results show that the performance of our scheme is better than the most advanced video question answering algorithm, and a large number of ablation and visualization experiments also verify the rationality of this solution.
, authors=Mingyang Liu, Ruomei Wang, Fan Zhou, Ge Lin*, authorsList=Mingyang Liu, Ruomei Wang, Fan Zhou, Ge Lin, authorCompany=null, correspAuthors=Ge Lin, authorNote=
born in 1994. PhD. His main research interests include video comprehension, video question answering, and machine learning
born in 1961. PhD, professor, PhD supervisor. Her main research interests include computer graphics, computer-aided design, and multimedia processing
born in 1976. PhD, professor, PhD supervisor. His main research interests include intelligent home systems, intelligent question answering, and the processing of cross-media data
born in 1980. PhD, associate professor. His main research interests include intelligent home systems and artificial intelligence
, correspAuthorsNote=null, copyrightStatement=Copyright © 2024 Journal of Computer Research and Development. All rights reserved., copyrightOwner=null, extLink=null, articleAbsUrl=null, sourceXml=null, magXml=null, pdfUrl=null, pdf=null, pdfFileSize=null, pdfExtLink=null, richHtmlUrl=null, mobilePdfUrl=null, reviewReport=null, pdfFirstPage=null, abstractGraph=null, abstractGraphContent=null, abstractVideo=null, citation=null, cebUrl=null, magXmlContent=null, mapNumber=null, fund=null), CN=ArticleExt(id=1329456433667616903, articleId=1329456429590753391, tenantId=1146029695717560320, journalId=1324759914146529478, language=CN, title=基于多模态知识主动学习的视频问答方案, columnId=1325489892104483745, journalTitle=计算机研究与发展, columnName=人工智能, runingTitle=null, highlight=null, articleAbstract=
视频问答是人工智能领域的一个热点研究问题. 现有方法在特征提取方面缺乏针对视觉目标运动细节的获取,从而会导致错误因果关系的建立. 此外,在数据融合与推理过程中,现有方法缺乏有效的主动学习能力,难以获取特征提取之外的先验知识,影响了模型对多模态内容的深度理解. 针对这些问题,首先,设计了一种显性多模态特征提取模块,通过获取图像序列中视觉目标的语义关联以及与周围环境的动态关系来建立每个视觉目标的运动轨迹. 进一步通过动态内容对静态内容的补充,为数据融合与推理提供了更加精准的视频特征表达. 其次,提出了知识自增强多模态数据融合与推理模型,实现了多模态信息理解的自我完善和逻辑思维聚焦,增强了对多模态特征的深度理解,减少了对先验知识的依赖. 最后,提出了一种基于多模态知识主动学习的视频问答方案. 实验结果表明,该方案的性能优于现有最先进的视频问答算法,大量的消融和可视化实验也验证了方案的合理性.
, authors=刘明阳, 王若梅, 周凡, 林格*, authorsList=刘明阳, 王若梅, 周凡, 林格, authorCompany=null, correspAuthors=林格, authorNote=
 |
1994年生. 博士. 主要研究方向为视频理解、视频问答、机器学习 |
 |
1961年生. 博士,教授,博士生导师. 主要研究方向为计算机图形学、计算机辅助设计、多媒体处理 |
 |
1976年生. 博士,教授,博士生导师. 主要研究方向为智能家居系统、智能问答、跨媒体数据处理 |
 |
1980年生. 博士,副研究员. 主要研究方向为智慧家庭、人工智能 |
, correspAuthorsNote=
, copyrightStatement=版权所有 © 2024 计算机研究与发展编辑部, copyrightOwner=null, extLink=null, articleAbsUrl=null, sourceXml=3CkTOnvyeToRROQXxC6FJQ==, magXml=t25ZcINnI9Ik590nFfxPnA==, pdfUrl=null, pdf=3BkxO1Xrfefy8EqH9G6EHQ==, pdfFileSize=3263268, pdfExtLink=null, richHtmlUrl=null, mobilePdfUrl=null, reviewReport=null, pdfFirstPage=null, abstractGraph=DKRTqh9fGWV1XJjLhIUFRQ==, abstractGraphContent=null, abstractVideo=null, citation=null, cebUrl=null, magXmlContent=uLXOx7umTrUIpCJkZerI2w==, mapNumber=null, fund=null)}, authors=[Author(id=1329456434036715660, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1329456429590753391, orderNo=0, firstName=null, middleName=null, lastName=null, nameCn=null, orcid=null, stid=null, country=null, authorPic=null, dead=0, email=liumy77@mail2.sysu.edu.cn, emailSecond=null, emailThird=null, correspondingAuthor=0, authorType=1, ext={EN=AuthorExt(id=1329456434145767566, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1329456429590753391, authorId=1329456434036715660, language=EN, stringName=Mingyang Liu, firstName=Mingyang, middleName=null, lastName=Liu, prefix=null, suffix=null, authorComment=null, nameInitials=null, affiliation=null, department=null, xref=null, address=
1National Engineering Research Center of Digital Life, School of Computer Science and Engineering, Sun Yat-sen University, Guangzhou 510006, bio={"content":"
born in 1994. PhD. His main research interests include video comprehension, video question answering, and machine learning
"}, bioImg=null, bioContent=
born in 1994. PhD. His main research interests include video comprehension, video question answering, and machine learning
, aboutCorrespAuthor=null), CN=AuthorExt(id=1329456434212876431, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1329456429590753391, authorId=1329456434036715660, language=CN, stringName=刘明阳, firstName=null, middleName=null, lastName=null, prefix=null, suffix=null, authorComment=null, nameInitials=null, affiliation=null, department=null, xref=null, address=
1中山大学计算机学院国家数字家庭工程技术研究中心 广州 510006, bio={"img":"kJ6K2kuR7zSJqmJZKAAMVA==","content":"
1994年生. 博士. 主要研究方向为视频理解、视频问答、机器学习
"}, bioImg=kJ6K2kuR7zSJqmJZKAAMVA==, bioContent=
1994年生. 博士. 主要研究方向为视频理解、视频问答、机器学习
, aboutCorrespAuthor=null)}, companyList=[AuthorCompany(id=1329456433898303624, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1329456429590753391, xref=1, ext=[AuthorCompanyExt(id=1329456433906692233, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1329456429590753391, companyId=1329456433898303624, language=EN, country=null, province=null, city=null, postcode=null, companyName=null, departmentName=null, remark=
1National Engineering Research Center of Digital Life, School of Computer Science and Engineering, Sun Yat-sen University, Guangzhou 510006), AuthorCompanyExt(id=1329456433915080842, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1329456429590753391, companyId=1329456433898303624, language=CN, country=null, province=null, city=null, postcode=null, companyName=null, departmentName=null, remark=
1中山大学计算机学院国家数字家庭工程技术研究中心 广州 510006)])]), Author(id=1329456434292568209, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1329456429590753391, orderNo=1, firstName=null, middleName=null, lastName=null, nameCn=null, orcid=null, stid=null, country=null, authorPic=null, dead=0, email=null, emailSecond=null, emailThird=null, correspondingAuthor=0, authorType=1, ext={EN=AuthorExt(id=1329456435949318291, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1329456429590753391, authorId=1329456434292568209, language=EN, stringName=Ruomei Wang, firstName=Ruomei, middleName=null, lastName=Wang, prefix=null, suffix=null, authorComment=null, nameInitials=null, affiliation=null, department=null, xref=null, address=
1National Engineering Research Center of Digital Life, School of Computer Science and Engineering, Sun Yat-sen University, Guangzhou 510006, bio={"content":"
born in 1961. PhD, professor, PhD supervisor. Her main research interests include computer graphics, computer-aided design, and multimedia processing
"}, bioImg=null, bioContent=
born in 1961. PhD, professor, PhD supervisor. Her main research interests include computer graphics, computer-aided design, and multimedia processing
, aboutCorrespAuthor=null), CN=AuthorExt(id=1329456436045787284, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1329456429590753391, authorId=1329456434292568209, language=CN, stringName=王若梅, firstName=null, middleName=null, lastName=null, prefix=null, suffix=null, authorComment=null, nameInitials=null, affiliation=null, department=null, xref=null, address=
1中山大学计算机学院国家数字家庭工程技术研究中心 广州 510006, bio={"img":"WZbDXWWVr6ILgG2rROMAmw==","content":"
1961年生. 博士,教授,博士生导师. 主要研究方向为计算机图形学、计算机辅助设计、多媒体处理
"}, bioImg=WZbDXWWVr6ILgG2rROMAmw==, bioContent=
1961年生. 博士,教授,博士生导师. 主要研究方向为计算机图形学、计算机辅助设计、多媒体处理
, aboutCorrespAuthor=null)}, companyList=[AuthorCompany(id=1329456433898303624, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1329456429590753391, xref=1, ext=[AuthorCompanyExt(id=1329456433906692233, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1329456429590753391, companyId=1329456433898303624, language=EN, country=null, province=null, city=null, postcode=null, companyName=null, departmentName=null, remark=
1National Engineering Research Center of Digital Life, School of Computer Science and Engineering, Sun Yat-sen University, Guangzhou 510006), AuthorCompanyExt(id=1329456433915080842, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1329456429590753391, companyId=1329456433898303624, language=CN, country=null, province=null, city=null, postcode=null, companyName=null, departmentName=null, remark=
1中山大学计算机学院国家数字家庭工程技术研究中心 广州 510006)])]), Author(id=1329456436125479062, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1329456429590753391, orderNo=2, firstName=null, middleName=null, lastName=null, nameCn=null, orcid=null, stid=null, country=null, authorPic=null, dead=0, email=null, emailSecond=null, emailThird=null, correspondingAuthor=0, authorType=1, ext={EN=AuthorExt(id=1329456436226142360, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1329456429590753391, authorId=1329456436125479062, language=EN, stringName=Fan Zhou, firstName=Fan, middleName=null, lastName=Zhou, prefix=null, suffix=null, authorComment=null, nameInitials=null, affiliation=null, department=null, xref=null, address=
1National Engineering Research Center of Digital Life, School of Computer Science and Engineering, Sun Yat-sen University, Guangzhou 510006, bio={"content":"
born in 1976. PhD, professor, PhD supervisor. His main research interests include intelligent home systems, intelligent question answering, and the processing of cross-media data
"}, bioImg=null, bioContent=
born in 1976. PhD, professor, PhD supervisor. His main research interests include intelligent home systems, intelligent question answering, and the processing of cross-media data
, aboutCorrespAuthor=null), CN=AuthorExt(id=1329456436297445529, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1329456429590753391, authorId=1329456436125479062, language=CN, stringName=周凡, firstName=null, middleName=null, lastName=null, prefix=null, suffix=null, authorComment=null, nameInitials=null, affiliation=null, department=null, xref=null, address=
1中山大学计算机学院国家数字家庭工程技术研究中心 广州 510006, bio={"img":"wXrGdNqFBQRUXWgDFqRd/w==","content":"
1976年生. 博士,教授,博士生导师. 主要研究方向为智能家居系统、智能问答、跨媒体数据处理
"}, bioImg=wXrGdNqFBQRUXWgDFqRd/w==, bioContent=
1976年生. 博士,教授,博士生导师. 主要研究方向为智能家居系统、智能问答、跨媒体数据处理
, aboutCorrespAuthor=null)}, companyList=[AuthorCompany(id=1329456433898303624, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1329456429590753391, xref=1, ext=[AuthorCompanyExt(id=1329456433906692233, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1329456429590753391, companyId=1329456433898303624, language=EN, country=null, province=null, city=null, postcode=null, companyName=null, departmentName=null, remark=
1National Engineering Research Center of Digital Life, School of Computer Science and Engineering, Sun Yat-sen University, Guangzhou 510006), AuthorCompanyExt(id=1329456433915080842, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1329456429590753391, companyId=1329456433898303624, language=CN, country=null, province=null, city=null, postcode=null, companyName=null, departmentName=null, remark=
1中山大学计算机学院国家数字家庭工程技术研究中心 广州 510006)])]), Author(id=1329456436360360091, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1329456429590753391, orderNo=3, firstName=null, middleName=null, lastName=null, nameCn=null, orcid=null, stid=null, country=null, authorPic=null, dead=0, email=linge3@mail.sysu.edu.cn, emailSecond=null, emailThird=null, correspondingAuthor=1, authorType=1, ext={EN=AuthorExt(id=1329456436431663261, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1329456429590753391, authorId=1329456436360360091, language=EN, stringName=Ge Lin, firstName=Ge, middleName=null, lastName=Lin, prefix=null, suffix=null, authorComment=null, nameInitials=null, affiliation=null, department=null, xref=*, address=
1National Engineering Research Center of Digital Life, School of Computer Science and Engineering, Sun Yat-sen University, Guangzhou 510006, bio={"content":"
born in 1980. PhD, associate professor. His main research interests include intelligent home systems and artificial intelligence
"}, bioImg=null, bioContent=
born in 1980. PhD, associate professor. His main research interests include intelligent home systems and artificial intelligence
, aboutCorrespAuthor=null), CN=AuthorExt(id=1329456436502966430, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1329456429590753391, authorId=1329456436360360091, language=CN, stringName=林格, firstName=null, middleName=null, lastName=null, prefix=null, suffix=null, authorComment=null, nameInitials=null, affiliation=null, department=null, xref=*, address=
1中山大学计算机学院国家数字家庭工程技术研究中心 广州 510006, bio={"img":"bbvU5XyhnezcILa9QQwMfg==","content":"
1980年生. 博士,副研究员. 主要研究方向为智慧家庭、人工智能
"}, bioImg=bbvU5XyhnezcILa9QQwMfg==, bioContent=
1980年生. 博士,副研究员. 主要研究方向为智慧家庭、人工智能
, aboutCorrespAuthor=null)}, companyList=[AuthorCompany(id=1329456433898303624, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1329456429590753391, xref=1, ext=[AuthorCompanyExt(id=1329456433906692233, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1329456429590753391, companyId=1329456433898303624, language=EN, country=null, province=null, city=null, postcode=null, companyName=null, departmentName=null, remark=
1National Engineering Research Center of Digital Life, School of Computer Science and Engineering, Sun Yat-sen University, Guangzhou 510006), AuthorCompanyExt(id=1329456433915080842, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1329456429590753391, companyId=1329456433898303624, language=CN, country=null, province=null, city=null, postcode=null, companyName=null, departmentName=null, remark=
1中山大学计算机学院国家数字家庭工程技术研究中心 广州 510006)])])], keywords=[Keyword(id=1329456436586852511, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1329456429590753391, language=EN, orderNo=1, keyword=video question answering), Keyword(id=1329456436653961376, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1329456429590753391, language=EN, orderNo=2, keyword=data fusion and reasoning), Keyword(id=1329456436733653153, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1329456429590753391, language=EN, orderNo=3, keyword=multimodal active learning), Keyword(id=1329456436800762018, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1329456429590753391, language=EN, orderNo=4, keyword=video details description extraction), Keyword(id=1329456436876259491, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1329456429590753391, language=EN, orderNo=5, keyword=deep learning), Keyword(id=1329456436943368356, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1329456429590753391, language=CN, orderNo=1, keyword=视频问答), Keyword(id=1329456437010477221, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1329456429590753391, language=CN, orderNo=2, keyword=数据融合与推理), Keyword(id=1329456437077586086, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1329456429590753391, language=CN, orderNo=3, keyword=多模态主动学习), Keyword(id=1329456437148889255, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1329456429590753391, language=CN, orderNo=4, keyword=视频细节描述提取), Keyword(id=1329456437215998120, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1329456429590753391, language=CN, orderNo=5, keyword=深度学习)], refs=[Reference(id=1329456442362409152, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1329456429590753391, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=1, rfOrder=0, authorNames=null, journalName=null, refType=null, unstructuredReference=俞俊,汪亮,余宙. 视觉问答技术研究[J]. 计算机研究与发展,2018,55(9):1946−1958, articleTitle=null, refAbstract=null), Reference(id=1329456442458878145, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1329456429590753391, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=1, rfOrder=1, authorNames=null, journalName=null, refType=null, unstructuredReference=Yu Jun, Wang Liang, Yu Zhou. Research on visual question answering techniques[J]. Journal of Computer Research and Development, 2018, 55(9): 1946−1958 (in Chinese), articleTitle=null, refAbstract=null), Reference(id=1329456442567930050, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1329456429590753391, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=2, rfOrder=2, authorNames=null, journalName=null, refType=null, unstructuredReference=张璐,曹峰,梁新彦,等. 基于关联特征传播的跨模态检索[J]. 计算机研究与发展,2022,59(9):1993−2002, articleTitle=null, refAbstract=null), Reference(id=1329456442647621827, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1329456429590753391, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=2, rfOrder=3, authorNames=null, journalName=null, refType=null, unstructuredReference=Zhang Lu, Cao Feng, Liang Xinyan, et al. Cross-modal retrieval with correlation feature propagation[J]. Journal of Computer Research and Development, 2022, 59(9): 1993−2002 (in Chinese), articleTitle=null, refAbstract=null), Reference(id=1329456442706342084, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1329456429590753391, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=3, rfOrder=4, authorNames=null, journalName=null, refType=null, unstructuredReference=李志欣,魏海洋,张灿龙,等. 图像描述生成研究进展[J]. 计算机研究与发展,2021,58(9):1951−1974, articleTitle=null, refAbstract=null), Reference(id=1329456442777645253, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1329456429590753391, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=3, rfOrder=5, authorNames=null, journalName=null, refType=null, unstructuredReference=Li Zhixin, Wei Haiyang, Zhang Canlong, et al. Research progress on image captioning[J]. Journal of Computer Research and Development, 2021, 58(9): 1951−1974 (in Chinese), articleTitle=null, refAbstract=null), Reference(id=1329456442844754118, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1329456429590753391, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=4, rfOrder=6, authorNames=null, journalName=null, refType=null, unstructuredReference=He Kaiming, Zhang Xiangyu, Ren Shaoqing, et al. Deep residual learning for image recognition[C]// Proc of the 34th IEEE Conf on Computer Vision and Pattern Recognition. Piscataway, NJ: IEEE, 2016: 770–778, articleTitle=null, refAbstract=null), Reference(id=1329456442924445895, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1329456429590753391, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=5, rfOrder=7, authorNames=null, journalName=null, refType=null, unstructuredReference=Hara K, Kataoka H, Satoh Y. Can spatiotemporal 3D CNNs retrace the history of 2D CNNs and imageNet[C]// Proc of the 36th IEEE Conf on Computer Vision and Pattern Recognition. Piscataway, NJ: IEEE, 2018: 6546–6555, articleTitle=null, refAbstract=null), Reference(id=1329456442991554760, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1329456429590753391, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=6, rfOrder=8, authorNames=null, journalName=null, refType=null, unstructuredReference=Peter A, He Xiaodong, Buehler C, et al. Bottom-up and top-down attention for image captioning and visual question answering[C]// Proc of the 36th IEEE Conf on Computer Vision and Pattern Recognition. Piscataway, NJ: IEEE, 2018: 6077–6086, articleTitle=null, refAbstract=null), Reference(id=1329456443167715529, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1329456429590753391, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=7, rfOrder=9, authorNames=null, journalName=null, refType=null, unstructuredReference=Lu Jiasen, Yang Jianwei, Batra D, et al. Hierarchical question-image co-attention for visual question answering[C]// Proc of the 30th Int Conf on Neural Information Proc Systems. New York: ACM, 2016: 289–297, articleTitle=null, refAbstract=null), Reference(id=1329456443239018698, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1329456429590753391, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=8, rfOrder=10, authorNames=null, journalName=null, refType=null, unstructuredReference=Gao Jiyang, Ge Runzhou, Chen Kan, et al. Motion appearance co-memory networks for video question answering[C]// Proc of the 36th IEEE Conf on Computer Vision and Pattern Recognition. Piscataway, NJ: IEEE, 2018: 6576–6585, articleTitle=null, refAbstract=null), Reference(id=1329456444887380171, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1329456429590753391, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=9, rfOrder=11, authorNames=null, journalName=null, refType=null, unstructuredReference=Dang L H, Le T, Le V, et al. Hierarchical object-oriented spatio-temporal reasoning for video question answering[C]// Proc of the 30th Int Joint Conf on Artificial Intelligence. San Francisco, CA: Morgan Kaufmann, 2021: 636–642, articleTitle=null, refAbstract=null), Reference(id=1329456445004820684, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1329456429590753391, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=10, rfOrder=12, authorNames=null, journalName=null, refType=null, unstructuredReference=Jiang Jianwen, Chen Ziqiang, Lin Haojie, et al. Divide and conquer: Question-guided spatio-temporal conrmual attention for video question answering[C]// Proc of the 34th AAAI Conf on Artificial Intelligence. Palo Alto, CA: AAAI, 2020: 11101–11108, articleTitle=null, refAbstract=null), Reference(id=1329456445088706765, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1329456429590753391, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=11, rfOrder=13, authorNames=null, journalName=null, refType=null, unstructuredReference=Wang Jianyu, Bao Bingkun, Xu Changsheng. DualVGR: A dual-visual graph reasoning unit for video question answering[J]. IEEE Transactions on Multimedia, 2022, 24: 3369−3380, articleTitle=null, refAbstract=null), Reference(id=1329456445164204238, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1329456429590753391, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=12, rfOrder=14, authorNames=null, journalName=null, refType=null, unstructuredReference=Le T M, Le V, Venkatesh S, et al. Hierarchical conditional relation networks for video question answering[C]// Proc of the 38th IEEE Conf on Computer Vision and Pattern Recognition. Piscataway, NJ: IEEE, 2020: 9969–9978, articleTitle=null, refAbstract=null), Reference(id=1329456445239701711, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1329456429590753391, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=13, rfOrder=15, authorNames=null, journalName=null, refType=null, unstructuredReference=Liu Fei, Liu Jing, Wang Weining, et al. HAIR: Hierarchical visual-semantic relational reasoning for video question answering[C]// Proc of the 18th IEEE/CVF Int Conf on Computer Vision. Piscataway, NJ: IEEE, 2021: 1678–1687, articleTitle=null, refAbstract=null), Reference(id=1329456445311004880, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1329456429590753391, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=14, rfOrder=16, authorNames=null, journalName=null, refType=null, unstructuredReference=Jang Y, Song Y, Yu Y, et al. TGIF-QA: Toward spatiotemporal reasoning in visual question answering[C]// Proc of the 33rd IEEE Conf on Computer Vision and Pattern Recognition. Piscataway, NJ: IEEE, 2017: 1359–1367, articleTitle=null, refAbstract=null), Reference(id=1329456445382308049, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1329456429590753391, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=15, rfOrder=17, authorNames=null, journalName=null, refType=null, unstructuredReference=Xu Dejing, Zhao Zhou, Xiao Jun, et al. Video question answering via gradually refined attention over appearance and motion[C]// Proc of the 25th ACM Int Conf on Multimedia. New York: ACM, 2017: 1645–1653, articleTitle=null, refAbstract=null), Reference(id=1329456445482971346, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1329456429590753391, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=16, rfOrder=18, authorNames=null, journalName=null, refType=null, unstructuredReference=Xu Jun, Mei Tao, Yao Ting, et al. MSRVTT: A large video description dataset for bridging video and language[C]// Proc of the 34th IEEE Conf on Computer Vision and Pattern Recognition. Piscataway, NJ: IEEE, 2016: 5288–5296, articleTitle=null, refAbstract=null), Reference(id=1329456445579440339, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1329456429590753391, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=17, rfOrder=19, authorNames=null, journalName=null, refType=null, unstructuredReference=Simonyan K, Zisserman A. Very deep convolutional networks for large-scale image recognition[J]. arXiv preprint, arXiv: 1409.1556v6, 2015, articleTitle=null, refAbstract=null), Reference(id=1329456445638160596, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1329456429590753391, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=18, rfOrder=20, authorNames=null, journalName=null, refType=null, unstructuredReference=Zhao Zhou, Zhang Zhu, Xiao Shuwen, et al. Open-ended long-form video question answering via adaptive hierarchical reinforced networks[C]// Proc of the 27th Int Joint Conf on Artificial Intelligence. San Francisco, CA: Morgan Kaufmann, 2018: 3683–3689, articleTitle=null, refAbstract=null), Reference(id=1329456445726240981, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1329456429590753391, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=19, rfOrder=21, authorNames=null, journalName=null, refType=null, unstructuredReference=Huang Deng, Chen Peihao, Zeng Runhao, et al. Location-aware graph convolutional networks for video question answering[C]// Proc of the 34th AAAI Conf on Artificial Intelligence. Palo Alto, CA: AAAI, 2020: 11021–11028, articleTitle=null, refAbstract=null), Reference(id=1329456445814321366, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1329456429590753391, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=20, rfOrder=22, authorNames=null, journalName=null, refType=null, unstructuredReference=Seo A, Kang G, Park J, et al. Attend what you need: Motion-appearance synergistic networks for video question answering[C]// Proc of the 59th Annual Meeting of the Association for Computational Linguistics and the 11th Int Joint Conf on Natural Language Processing. Stroudsburg, PA: ACL, 2021: 6167–6177, articleTitle=null, refAbstract=null), Reference(id=1329456445885624535, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1329456429590753391, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=21, rfOrder=23, authorNames=null, journalName=null, refType=null, unstructuredReference=Jiang Pin, Han Yahong. Reasoning with heterogeneous graph alignment for video question answering[C]// Proc of the 34th AAAI Conf on Artificial Intelligence. Palo Alto, CA: AAAI, 2020: 11109–11116, articleTitle=null, refAbstract=null), Reference(id=1329456446061785304, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1329456429590753391, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=22, rfOrder=24, authorNames=null, journalName=null, refType=null, unstructuredReference=Park J, Lee J, Sohn K. Bridge to answer: Structure-aware graph interaction network for video question answering[C]// Proc of the 39th IEEE Conf on Computer Vision and Pattern Recognition. Piscataway, NJ: IEEE, 2021: 15526–15535, articleTitle=null, refAbstract=null), Reference(id=1329456446200197337, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1329456429590753391, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=23, rfOrder=25, authorNames=null, journalName=null, refType=null, unstructuredReference=Russakovsky O, Deng Jia, Su Hao, et al. ImageNet large scale visual recognition challenge[J]. International Journal of Computer Vision, 2015, 115(3): 211−252, articleTitle=null, refAbstract=null), Reference(id=1329456446271500506, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1329456429590753391, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=24, rfOrder=26, authorNames=null, journalName=null, refType=null, unstructuredReference=Will K, Carreira J, Simonyan K, et al. The Kinetics human action video dataset[J]. arXiv preprint, arXiv: 1705.06950, 2017, articleTitle=null, refAbstract=null), Reference(id=1329456446367969499, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1329456429590753391, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=25, rfOrder=27, authorNames=null, journalName=null, refType=null, unstructuredReference=Zeng Pengpeng, Zhang Haonan, Gao Lianli, et al. Video question answering with prior knowledge and object-sensitive learning[J]. IEEE Transactions on Image Processing, 2022, 31: 5936−5948, articleTitle=null, refAbstract=null), Reference(id=1329456446439272668, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1329456429590753391, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=26, rfOrder=28, authorNames=null, journalName=null, refType=null, unstructuredReference=Chen Xinlei, Fang Hao, Lin Tsung-Yi, et al. Microsoft COCO captions: Data collection and evaluation server[J]. arXiv preprint, arXiv: 1504.00325, 2015, articleTitle=null, refAbstract=null), Reference(id=1329456446502187229, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1329456429590753391, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=27, rfOrder=29, authorNames=null, journalName=null, refType=null, unstructuredReference=Lei Jie, Li Linjie, Zhou Luowei, et al. Less is more: ClipBERT for video-and-language learning via sparse sampling[C]// Proc of the 39th IEEE Conf on Computer Vision and Pattern Recognition. Piscataway, NJ: IEEE, 2021: 7331–7341, articleTitle=null, refAbstract=null), Reference(id=1329456446590267614, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1329456429590753391, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=28, rfOrder=30, authorNames=null, journalName=null, refType=null, unstructuredReference=Seo P H, Nagrani A, Schmid C. Look before you speak: Visually conrmualized utterances[C]// Proc of the 39th IEEE Conf on Computer Vision and Pattern Recognition. Piscataway, NJ: IEEE, 2021: 16877–16887, articleTitle=null, refAbstract=null), Reference(id=1329456446653182175, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1329456429590753391, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=29, rfOrder=31, authorNames=null, journalName=null, refType=null, unstructuredReference=Devlin J, Chang Mingwei, Lee K, et al. BERT: Pre-training of deep bidirectional transformers for language understanding[C]// Proc of the 14th Conf of the North American Chapter of the Association for Computational Linguistics. Stroudsburg, PA: ACL, 2019: 4171–4186, articleTitle=null, refAbstract=null), Reference(id=1329456446724485344, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1329456429590753391, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=30, rfOrder=32, authorNames=null, journalName=null, refType=null, unstructuredReference=Vaswani A, Shazeer N, Parmar N, et al. Attention is all you need[C]// Proc of the 31st Int Conf on Neural Information Processing Systems. New York: ACM, 2017: 5998–6008, articleTitle=null, refAbstract=null), Reference(id=1329456446833537249, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1329456429590753391, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=31, rfOrder=33, authorNames=null, journalName=null, refType=null, unstructuredReference=Gao Lianli, Lei Yu, Zeng Pengpeng, et al. Hierarchical representation network with auxiliary tasks for video captioning and video question answering[J]. IEEE Transactions on Image Processing, 2022, 31: 202−215, articleTitle=null, refAbstract=null)], funds=null, companyList=[AuthorCompany(id=1329456433898303624, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1329456429590753391, xref=1, ext=[AuthorCompanyExt(id=1329456433906692233, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1329456429590753391, companyId=1329456433898303624, language=EN, country=null, province=null, city=null, postcode=null, companyName=null, departmentName=null, remark=
1National Engineering Research Center of Digital Life, School of Computer Science and Engineering, Sun Yat-sen University, Guangzhou 510006), AuthorCompanyExt(id=1329456433915080842, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1329456429590753391, companyId=1329456433898303624, language=CN, country=null, province=null, city=null, postcode=null, companyName=null, departmentName=null, remark=
1中山大学计算机学院国家数字家庭工程技术研究中心 广州 510006)])], figs=[ArticleFig(id=1329456437358604457, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1329456429590753391, language=EN, label=Figure 1, caption=
The overview of our proposed video question answering scheme, figureFileSmall=B6At/c+askAF9F7yUmDbmg==, figureFileBig=DKRTqh9fGWV1XJjLhIUFRQ==, tableContent=null), ArticleFig(id=1329456437417324714, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1329456429590753391, language=CN, label=图1, caption=
本文提出的视频问答方案概述, figureFileSmall=B6At/c+askAF9F7yUmDbmg==, figureFileBig=DKRTqh9fGWV1XJjLhIUFRQ==, tableContent=null), ArticleFig(id=1329456437635428523, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1329456429590753391, language=EN, label=Figure 2, caption=
Detail description method of explicit video, figureFileSmall=uLdpTRLhe4usJSzxVv6wPw==, figureFileBig=X1fJhPcQAD+l90oY+9grpQ==, tableContent=null), ArticleFig(id=1329456437702537388, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1329456429590753391, language=CN, label=图2, caption=
显性视频的细节描述方法, figureFileSmall=uLdpTRLhe4usJSzxVv6wPw==, figureFileBig=X1fJhPcQAD+l90oY+9grpQ==, tableContent=null), ArticleFig(id=1329456437782229165, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1329456429590753391, language=EN, label=Figure 3, caption=
Verify the rationality of K value, figureFileSmall=isTEqdEVYWnNarPOzTgupg==, figureFileBig=gxOUiIjwIjee2v9lok8U2g==, tableContent=null), ArticleFig(id=1329456437861920942, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1329456429590753391, language=CN, label=图3, caption=
验证K值的合理性, figureFileSmall=isTEqdEVYWnNarPOzTgupg==, figureFileBig=gxOUiIjwIjee2v9lok8U2g==, tableContent=null), ArticleFig(id=1329456437924835503, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1329456429590753391, language=EN, label=Figure 4, caption=
Verify the rationality of d value, figureFileSmall=zwJYuH/yZXu+H0AnyclKPg==, figureFileBig=MNH+r4HB2hxo02qfuy48ew==, tableContent=null), ArticleFig(id=1329456437987750065, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1329456429590753391, language=CN, label=图4, caption=
验证d值的合理性, figureFileSmall=zwJYuH/yZXu+H0AnyclKPg==, figureFileBig=MNH+r4HB2hxo02qfuy48ew==, tableContent=null), ArticleFig(id=1329456438063247538, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1329456429590753391, language=EN, label=Figure 5, caption=
Visual t-SNE graph for multimodal embedding distribution, figureFileSmall=kUzxMVxqkIiWsDeFnB7WBA==, figureFileBig=gLB4LyFPBFynHHKNolXnDA==, tableContent=null), ArticleFig(id=1329456438121967795, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1329456429590753391, language=CN, label=图5, caption=
用于多模态嵌入分布的可视化t-SNE图, figureFileSmall=kUzxMVxqkIiWsDeFnB7WBA==, figureFileBig=gLB4LyFPBFynHHKNolXnDA==, tableContent=null), ArticleFig(id=1329456438197465268, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1329456429590753391, language=EN, label=Figure 6, caption=
Video question answering result demonstration, figureFileSmall=nWAqRkqJ5V3MM7xBmhRAqA==, figureFileBig=i+zBO5veVoX8I302x5q0Jw==, tableContent=null), ArticleFig(id=1329456438260379829, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1329456429590753391, language=CN, label=图6, caption=
视频问答结果演示, figureFileSmall=nWAqRkqJ5V3MM7xBmhRAqA==, figureFileBig=i+zBO5veVoX8I302x5q0Jw==, tableContent=null), ArticleFig(id=1329456438335877303, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1329456429590753391, language=EN, label=Figure 7, caption=
Visualization of the thinking focus process, figureFileSmall=M2lk7i7asA1kUWmAsHHJOQ==, figureFileBig=8+gZ5TLOEgGy7xcX8FFUng==, tableContent=null), ArticleFig(id=1329456438390403256, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1329456429590753391, language=CN, label=图7, caption=
思维聚焦过程的可视化, figureFileSmall=M2lk7i7asA1kUWmAsHHJOQ==, figureFileBig=8+gZ5TLOEgGy7xcX8FFUng==, tableContent=null), ArticleFig(id=1329456438444929209, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1329456429590753391, language=EN, label=Table 1, caption=
Verify the Rationality of the Network Structures and Their Modules %
, figureFileSmall=null, figureFileBig=null, tableContent=
| 多模态特征提取 | MSVD-QA | MSRVTT-QA |
|---|
| w/o 目标对齐 | 42.2 | 38.1 |
| w/o 静态特征 | 41.2 | 37.6 |
| w/o 动作特征 | 41.5 | 37.8 |
| w/o 视觉目标特征 | 39.8 | 37.3 |
| only 静态特征 | 38.6 | 36.5 |
| only 动作特征 | 36.5 | 35.9 |
| only 视觉目标特征 | 39.5 | 37.2 |
), ArticleFig(id=1329456438503649466, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1329456429590753391, language=CN, label=表1, caption=
验证网络结构及其模块的合理性
, figureFileSmall=null, figureFileBig=null, tableContent=
| 多模态特征提取 | MSVD-QA | MSRVTT-QA |
|---|
| w/o 目标对齐 | 42.2 | 38.1 |
| w/o 静态特征 | 41.2 | 37.6 |
| w/o 动作特征 | 41.5 | 37.8 |
| w/o 视觉目标特征 | 39.8 | 37.3 |
| only 静态特征 | 38.6 | 36.5 |
| only 动作特征 | 36.5 | 35.9 |
| only 视觉目标特征 | 39.5 | 37.2 |
), ArticleFig(id=1329456438566564027, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1329456429590753391, language=EN, label=Table 2, caption=
Verify the Effectiveness of Explicit Detail Feature Extraction %
, figureFileSmall=null, figureFileBig=null, tableContent=
| 网络组成 | MSVD-QA | MSRVTT-QA |
|---|
| 层次结构 | 1层KAFR模块 | 40.8 | 37.7 |
| 2层KAFR模块 | 43.1 | 38.7 |
| 3层KAFR模块 | 42.9 | 38.8 |
| KAFR模块的组成 | 传统数据融合与推理过程 | 39.8 | 36.8 |
| w/o 时序表示学习与推理 | 38.9 | 35.6 |
| w/o 多模态表示再学习 | 40.8 | 37.6 |
| w/o 聚焦表示学习 | 42.4 | 38.1 |
| w/o 汇总表示学习 | 41.6 | 37.9 |
| 聚焦思维→主动学习 | 42.3 | 38.1 |
| 主动学习→聚焦思维 | 43.1 | 38.7 |
), ArticleFig(id=1329456438642061500, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1329456429590753391, language=CN, label=表2, caption=
验证显性细节特征提取的有效性
, figureFileSmall=null, figureFileBig=null, tableContent=
| 网络组成 | MSVD-QA | MSRVTT-QA |
|---|
| 层次结构 | 1层KAFR模块 | 40.8 | 37.7 |
| 2层KAFR模块 | 43.1 | 38.7 |
| 3层KAFR模块 | 42.9 | 38.8 |
| KAFR模块的组成 | 传统数据融合与推理过程 | 39.8 | 36.8 |
| w/o 时序表示学习与推理 | 38.9 | 35.6 |
| w/o 多模态表示再学习 | 40.8 | 37.6 |
| w/o 聚焦表示学习 | 42.4 | 38.1 |
| w/o 汇总表示学习 | 41.6 | 37.9 |
| 聚焦思维→主动学习 | 42.3 | 38.1 |
| 主动学习→聚焦思维 | 43.1 | 38.7 |
), ArticleFig(id=1329456438713364669, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1329456429590753391, language=EN, label=Table 3, caption=
Comparison of Our Method with the Most Advanced Methods on Multiple Video Question Answering Datasets %
, figureFileSmall=null, figureFileBig=null, tableContent=
| 模型 | 预训练 | TGIF-QA | MSVD-QA ↑ | MSRVTT-QA ↑ |
|---|
| Action ↑ | Transition ↑ | FrameQA ↑ | Count ↓ |
|---|
| L-GCN[19] | | 74.3 | 81.1 | 56.3 | 3.95 | 34.3 | |
| HGA[21] | | 75.4 | 81.0 | 55.1 | 4.09 | 34.7 | 35.5 |
| HCRN[12] | | 75.0 | 81.4 | 55.9 | 3.82 | 36.1 | 35.6 |
| HOSTR[9] | | 75.6 | 83.0 | 58.2 | 3.65 | 39.4 | 35.9 |
| MASN[20] | | 84.4 | 87.4 | 59.5 | 3.75 | 38.0 | 35.2 |
| HRNAT[31] | | | | | | 38.2 | 34.9 |
| DualVGR[11] | | | | | | 39.0 | 35.5 |
| PKOL[25] | | 74.6 | 82.3 | 61.8 | 3.67 | 41.1 | 36.9 |
| ClipBERT[27] | √ | 82.8 | 87.8 | 60.3 | | | 37.4 |
| CoMVT[28] | √ | | | | | 42.6 | 39.5 |
| KAFR(本文方法) | | 85.2 | 90.1 | 63.1 | 3.61 | 43.1 | 38.7 |
), ArticleFig(id=1329456440395280575, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1329456429590753391, language=CN, label=表3, caption=
本文方法与多个视频问答数据集上最先进的方法的比较
, figureFileSmall=null, figureFileBig=null, tableContent=
| 模型 | 预训练 | TGIF-QA | MSVD-QA ↑ | MSRVTT-QA ↑ |
|---|
| Action ↑ | Transition ↑ | FrameQA ↑ | Count ↓ |
|---|
| L-GCN[19] | | 74.3 | 81.1 | 56.3 | 3.95 | 34.3 | |
| HGA[21] | | 75.4 | 81.0 | 55.1 | 4.09 | 34.7 | 35.5 |
| HCRN[12] | | 75.0 | 81.4 | 55.9 | 3.82 | 36.1 | 35.6 |
| HOSTR[9] | | 75.6 | 83.0 | 58.2 | 3.65 | 39.4 | 35.9 |
| MASN[20] | | 84.4 | 87.4 | 59.5 | 3.75 | 38.0 | 35.2 |
| HRNAT[31] | | | | | | 38.2 | 34.9 |
| DualVGR[11] | | | | | | 39.0 | 35.5 |
| PKOL[25] | | 74.6 | 82.3 | 61.8 | 3.67 | 41.1 | 36.9 |
| ClipBERT[27] | √ | 82.8 | 87.8 | 60.3 | | | 37.4 |
| CoMVT[28] | √ | | | | | 42.6 | 39.5 |
| KAFR(本文方法) | | 85.2 | 90.1 | 63.1 | 3.61 | 43.1 | 38.7 |
)], attaches=null, journal=Journal(id=1324759606947315908, delFlag=0, nameCn=计算机研究与发展, nameEn=Journal of Computer Research and Development, nameHistory1=null, nameHistory2=null, issn=1000-1239, eissn=null, cn=11-1777/TP, coden=null, periodic=0, language=CN, oaType=null, ccby=null, superviseOffice=null, ownerOffice=null, pubOffice=null, editorOffice=null, officeType=null, aims=null, clcCode=null, officeProv=null, officeCity=null, officeAddr=null, officeZip=null, officeEmail=null, officePhone=null, editDirector=null, officeDirector=null, officeDirectorPhone=null, officeStaffNum=null, officeEmpNum=null, coverPicUrl=k2l8zgqwstuxxevFX2qxVA==, journalPrice=null, startedYear=null, abbrevIsoEn=Journal of Computer Research and Development, journalRemark=null, publicationField=null, createdTime=1793776903141, updatedTime=1793850698612, createdBy=18614031015, updatedBy=13701087609, firstLetterCn=J, firstLetterEn=J, subjectCode=Electronic Information, subjectName=null, subjectCodeEn=Electronic Information, subjectNameEn=null, picCn=k2l8zgqwstuxxevFX2qxVA==, picEn=r6k0Ry+RD+c/VXj5QarmDw==, jcr=null, cjcr=null, exts=[JournalExt(id=1325069127710286688, language=CN, name=计算机研究与发展, nameHistory1=null, nameHistory2=null, managedBy=, sponsoredBy=, publishedBy=, editorOffice=, officeProv=null, officeCity=null, officeAddr=, officeZip=, editDirector=, officeDirector=null, officePhone=null, coverPicUrl=null, journalRemark=, submitArticleUrl=null, websiteUrl=, createdTime=1793850698640, updatedTime=1793850698640, createdBy=13701087609, updatedBy=13701087609, submissionGuidelinesUrl=, submissionAuthorUrl=https://crad.ict.ac.cn/journalx/authorLogOn.action, submissionEditorUrl=https://crad.ict.ac.cn/journalx/editorLogOn.action, submissionReviewUrl=https://crad.ict.ac.cn/journalx/expertLogOn.action, submissionCeEditorUrl=https://crad.ict.ac.cn/journalx/editorInChiefLogOn.action, submissionAeEditorUrl=, option={"copyright":""}), JournalExt(id=1325069127777395553, language=EN, name=Journal of Computer Research and Development, nameHistory1=null, nameHistory2=null, managedBy=, sponsoredBy=, publishedBy=, editorOffice=, officeProv=null, officeCity=null, officeAddr=, officeZip=, editDirector=, officeDirector=null, officePhone=null, coverPicUrl=null, journalRemark=, submitArticleUrl=null, websiteUrl=, createdTime=1793850698656, updatedTime=1793850698656, createdBy=13701087609, updatedBy=13701087609, submissionGuidelinesUrl=, submissionAuthorUrl=https://crad.ict.ac.cn/journalx/authorLogOn.action, submissionEditorUrl=https://crad.ict.ac.cn/journalx/editorLogOn.action, submissionReviewUrl=https://crad.ict.ac.cn/journalx/expertLogOn.action, submissionCeEditorUrl=https://crad.ict.ac.cn/journalx/editorInChiefLogOn.action, submissionAeEditorUrl=, option={"copyright":""})], databaseList=null, tenantJournalId=1324759914146529478, websiteList=[Website(id=1325070300139892771, webName=null, webTitle=null, webDomain=null, webCopyrigh=null, webIpcNo=null, seoTitle=null, seoKeywords=null, seoDescription=null, tenantJournalId=null, journalId=1324759914146529478, journalNameCn=null, journalNameEn=null, grayFlag=null, tenantId=1146029695717560320, platformId=null, journalGroupId=null, journalGroupNameCn=null, journalGroupNameEn=null, type=1, domain=https://castjournals.cast.org.cn/joweb/jsjyjyfz/CN, language=CN, createTime=1793850978169, createBy=18614031015, updateTime=1793851040259, updateBy=18614031015, name=计算机研究与发展-中文, tplId=1146099689490845704, title=计算机研究与发展, delFlag=0, indexPage=/home, props=[WebsiteProps(id=1325070893751349360, tenantId=1146029695717560320, journalId=null, journalGroupId=null, siteId=1325070300139892771, code=articleTextType, value=kx, createTime=1793851119697, updateTime=1793851119697, creator=18614031015, updator=18614031015), WebsiteProps(id=1325070893730377837, tenantId=1146029695717560320, journalId=null, journalGroupId=null, siteId=1325070300139892771, code=banner, value=null, createTime=1793851119692, updateTime=1793851119692, creator=18614031015, updator=18614031015), WebsiteProps(id=1325070893772320883, tenantId=1146029695717560320, journalId=null, journalGroupId=null, siteId=1325070300139892771, code=grayFlag, value=0, createTime=1793851119702, updateTime=1793851119702, creator=18614031015, updator=18614031015), WebsiteProps(id=1325070893721989228, tenantId=1146029695717560320, journalId=null, journalGroupId=null, siteId=1325070300139892771, code=logo, value=https://castjournals.cast.org.cn/joweb/jsjyjyfz/CN/file/pic?fileId=U27tDsy717YF7XSPA+Vwdw==, createTime=1793851119690, updateTime=1793851119690, creator=18614031015, updator=18614031015), WebsiteProps(id=1325070893784903797, tenantId=1146029695717560320, journalId=null, journalGroupId=null, siteId=1325070300139892771, code=minRunFlag, value=0, createTime=1793851119705, updateTime=1793851119705, creator=18614031015, updator=18614031015), WebsiteProps(id=1325070893742960751, tenantId=1146029695717560320, journalId=null, journalGroupId=null, siteId=1325070300139892771, code=picServerUrl, value=https://castjournals.cast.org.cn/joweb/jsjyjyfz/CN/file/pic, createTime=1793851119695, updateTime=1793851119695, creator=18614031015, updator=18614031015), WebsiteProps(id=1325070893780709492, tenantId=1146029695717560320, journalId=null, journalGroupId=null, siteId=1325070300139892771, code=silenceFlag, value=0, createTime=1793851119704, updateTime=1793851119704, creator=18614031015, updator=18614031015), WebsiteProps(id=1325070893738766446, tenantId=1146029695717560320, journalId=null, journalGroupId=null, siteId=1325070300139892771, code=staticResourcePath, value=https://castjournals.cast.org.cn/joweb/cast_kjdb_cn_619/, createTime=1793851119694, updateTime=1793851119694, creator=18614031015, updator=18614031015), WebsiteProps(id=1325070893755543665, tenantId=1146029695717560320, journalId=null, journalGroupId=null, siteId=1325070300139892771, code=themeColor, value=null, createTime=1793851119698, updateTime=1793851119698, creator=18614031015, updator=18614031015), WebsiteProps(id=1325070893763932274, tenantId=1146029695717560320, journalId=null, journalGroupId=null, siteId=1325070300139892771, code=themeStyle, value=null, createTime=1793851119700, updateTime=1793851119700, creator=18614031015, updator=18614031015)]), Website(id=1325070300580294698, webName=null, webTitle=null, webDomain=null, webCopyrigh=null, webIpcNo=null, seoTitle=null, seoKeywords=null, seoDescription=null, tenantJournalId=null, journalId=1324759914146529478, journalNameCn=null, journalNameEn=null, grayFlag=null, tenantId=1146029695717560320, platformId=null, journalGroupId=null, journalGroupNameCn=null, journalGroupNameEn=null, type=1, domain=https://castjournals.cast.org.cn/joweb/jsjyjyfz/EN, language=EN, createTime=1793850978274, createBy=18614031015, updateTime=1793851036638, updateBy=18614031015, name=计算机研究与发展-英文, tplId=1146101810881728533, title=Journal of Computer Research and Development, delFlag=0, indexPage=/home, props=[WebsiteProps(id=1325070925288321148, tenantId=1146029695717560320, journalId=null, journalGroupId=null, siteId=1325070300580294698, code=articleTextType, value=kx, createTime=1793851127216, updateTime=1793851127216, creator=18614031015, updator=18614031015), WebsiteProps(id=1325070925263155321, tenantId=1146029695717560320, journalId=null, journalGroupId=null, siteId=1325070300580294698, code=banner, value=null, createTime=1793851127210, updateTime=1793851127210, creator=18614031015, updator=18614031015), WebsiteProps(id=1325070925309292671, tenantId=1146029695717560320, journalId=null, journalGroupId=null, siteId=1325070300580294698, code=grayFlag, value=0, createTime=1793851127221, updateTime=1793851127221, creator=18614031015, updator=18614031015), WebsiteProps(id=1325070925254766712, tenantId=1146029695717560320, journalId=null, journalGroupId=null, siteId=1325070300580294698, code=logo, value=https://castjournals.cast.org.cn/joweb/jsjyjyfz/EN/file/pic?fileId=U27tDsy717YF7XSPA+Vwdw==, createTime=1793851127208, updateTime=1793851127208, creator=18614031015, updator=18614031015), WebsiteProps(id=1325070925326069889, tenantId=1146029695717560320, journalId=null, journalGroupId=null, siteId=1325070300580294698, code=minRunFlag, value=0, createTime=1793851127225, updateTime=1793851127225, creator=18614031015, updator=18614031015), WebsiteProps(id=1325070925279932539, tenantId=1146029695717560320, journalId=null, journalGroupId=null, siteId=1325070300580294698, code=picServerUrl, value=https://castjournals.cast.org.cn/joweb/jsjyjyfz/EN/file/pic, createTime=1793851127214, updateTime=1793851127214, creator=18614031015, updator=18614031015), WebsiteProps(id=1325070925317681280, tenantId=1146029695717560320, journalId=null, journalGroupId=null, siteId=1325070300580294698, code=silenceFlag, value=0, createTime=1793851127223, updateTime=1793851127223, creator=18614031015, updator=18614031015), WebsiteProps(id=1325070925271543930, tenantId=1146029695717560320, journalId=null, journalGroupId=null, siteId=1325070300580294698, code=staticResourcePath, value=https://castjournals.cast.org.cn/joweb/cast_kjdb_en_623/, createTime=1793851127212, updateTime=1793851127212, creator=18614031015, updator=18614031015), WebsiteProps(id=1325070925292515453, tenantId=1146029695717560320, journalId=null, journalGroupId=null, siteId=1325070300580294698, code=themeColor, value=null, createTime=1793851127217, updateTime=1793851127217, creator=18614031015, updator=18614031015), WebsiteProps(id=1325070925300904062, tenantId=1146029695717560320, journalId=null, journalGroupId=null, siteId=1325070300580294698, code=themeStyle, value=null, createTime=1793851127219, updateTime=1793851127219, creator=18614031015, updator=18614031015)])], journalTitle=计算机研究与发展, weixinUrl=null, journalUrl=https://crad.ict.ac.cn/, iacademicId=null, status=1, seqNo=null, journalTitleEn=Journal of Computer Research and Development, journalPhotoCn=k2l8zgqwstuxxevFX2qxVA==, journalPhotoEn=r6k0Ry+RD+c/VXj5QarmDw==, journalFirstLetter=J, journalRecommend=null, journalNew=null, journalCollection=null, jcrJf=null, cjcrJf=null, jcrJfStr=null, cjcrJfStr=null, submissionFirstDecision=null, sciSubjectClassification=null, casSubjectClassification=null, citeScore=null, totalCitationFrequency=null, icpCode=null, psCode=null, advertisingLicenseCode=null, copyrightInformation=null, country=null, option=, provinceCode=null, provinceName=null, collectFlag=false, interPubPlatform=, interPubPlatformUrl=null), detailUrlCn=https://castjournals.cast.org.cn/joweb/jsjyjyfz/CN/10.7544/issn1000-1239.202221008, detailUrlEn=https://castjournals.cast.org.cn/joweb/jsjyjyfz/EN/10.7544/issn1000-1239.202221008, pdfUrlCn=https://castjournals.cast.org.cn/joweb/jsjyjyfz/CN/PDF/10.7544/issn1000-1239.202221008, pdfUrlEn=https://castjournals.cast.org.cn/joweb/jsjyjyfz/EN/PDF/10.7544/issn1000-1239.202221008, aliStartDate=0, aliEndDate=0, collectionFlag=false, citedCount=null, citedUrl=null, previewStatus=0, delFlag=0, hasFullText=1, orderTime=1711900800000, fullTextJson=null, articleText=null, reference=null)