Article(id=1327012451545100502, tenantId=1146029695717560320, journalId=1324759914146529478, issueId=1327012311115608215, articleNumber=null, orderNo=null, doi=10.7544/issn1000-1239.202330199, pmid=null, cstr=null, oa=null, hot=null, price=null, onlineType=0, articleFormat=0, articleType=null, articleTypeStr=research-article, receivedDate=1679846400000, receivedDateStr=2023-03-27, revisedDate=1705939200000, revisedDateStr=2024-01-23, acceptedDate=null, acceptedDateStr=null, onlineDate=1794314023146, onlineDateStr=2026-11-10, pubDate=1738339200000, pubDateStr=2025-02-01, doiRegisterDate=null, doiRegisterDateStr=null, onlineIssueDate=1794314023146, onlineIssueDateStr=2026-11-10, onlineJustAcceptDate=null, onlineJustAcceptDateStr=null, onlineFirstDate=null, onlineFirstDateStr=null, sourceXml=null, magXml=null, createTime=1794314023146, creator=13701087609, updateTime=1794314023146, updator=13701087609, issue=Issue{id=1327012311115608215, tenantId=1146029695717560320, journalId=1324759914146529478, year='2025', volume='62', issue='2', pageStart='285', pageEnd='541', issueExtLink='null', onlineDate='null', pubDate='1738339200000', pubDateStr='2025-02-01', beforeIssueId=null, nextIssueId=null, price=null, status=1, issueComplete=1, articleOrder=1, issueType=-1, specialIssue=null, createTime=1794313989665, creator='13701087609', updateTime=1794376696965, updator='13701087609', preIssue=null, nextIssue=null, articleTotal=null, ext={EN=IssueExt(id=1327275325060395748, tenantId=1146029695717560320, journalId=1324759914146529478, issueId=1327012311115608215, language=EN, specialIssueTitle=, coverIllustrator=null, specialIssueEditor=, specialIssueAbout=), CN=IssueExt(id=1327275325060395749, tenantId=1146029695717560320, journalId=1324759914146529478, issueId=1327012311115608215, language=CN, specialIssueTitle=, coverIllustrator=null, specialIssueEditor=, specialIssueAbout=)}, issueFiles=null, downloadFileDto=null}, startPage=374, endPage=382, ext={EN=ArticleExt(id=1327012451717066967, articleId=1327012451545100502, tenantId=1146029695717560320, journalId=1324759914146529478, language=EN, title=Multimodal Sentiment Analysis Method for Sentimental Semantic Inconsistency, columnId=1325489891789910943, journalTitle=Journal of Computer Research and Development, columnName=Artificial Intelligence, runingTitle=null, highlight=null, articleAbstract=
Multimodal sentiment analysis is a multimodal task that uses multiple modalities of subjective information to analyze sentiment. In some scenarios, the sentimental expression in different modalities is inconsistent, even contradictory, which will weaken the effect of multimodal collaborative decision-making. In this paper, a multimodal learning method is proposed to learn the modal feature representations with consistent sentimental semantics. In order to improve the common feature representation of different modalities and learn the dynamic interaction between modalities without affecting the original information, we first learn the common feature representation of each modality, and then use cross attention to enable one modality to effectively obtain auxiliary information from the common feature representations of other modalities. In multimodal fusion, we propose a multimodal attention, which is used to weighted concatenate modal feature representations, in order to increase the expression of contributed modalities and suppress the influence of weak modalities. The experimental results of the proposed method on the sentiment analysis datasets MOSI, MOSEI, CH-SIMS are better than those of the compared models, indicating the necessity and rationality of considering the problem of sentimental semantic inconsistency in multimodal sentiment analysis.
, authors=Yuanyi Luo, Rui Wu*, Jiafeng Liu, Xianglong Tang, authorsList=Yuanyi Luo, Rui Wu, Jiafeng Liu, Xianglong Tang, authorCompany=null, correspAuthors=Rui Wu, authorNote=
born in 1996. PhD candidate. His main research interest includes multimodal learning
born in 1976. PhD, associate professor, PhD supervisor. His main research interests include pattern recognition and multimodal learning
born in 1968. PhD, associate professor. His main research interests include pattern recognition and machine learning
born in 1960. PhD, professor, PhD supervisor. His main research interests include pattern recognition and computer vision
, correspAuthorsNote=null, copyrightStatement=Copyright © 2025 Journal of Computer Research and Development. All rights reserved., copyrightOwner=null, extLink=null, articleAbsUrl=null, sourceXml=null, magXml=null, pdfUrl=null, pdf=null, pdfFileSize=null, pdfExtLink=null, richHtmlUrl=null, mobilePdfUrl=null, reviewReport=null, pdfFirstPage=null, abstractGraph=null, abstractGraphContent=null, abstractVideo=null, citation=null, cebUrl=null, magXmlContent=null, mapNumber=null, fund=null), CN=ArticleExt(id=1327012453193461990, articleId=1327012451545100502, tenantId=1146029695717560320, journalId=1324759914146529478, language=CN, title=面向情感语义不一致的多模态情感分析方法, columnId=1325489892104483745, journalTitle=计算机研究与发展, columnName=人工智能, runingTitle=null, highlight=null, articleAbstract=
多模态情感分析是利用多种模式的主观信息对情感进行分析判断的一种多模态任务. 情感表达具有主观性,在某些场景下不同模态的情感表达不一致,甚至存在相悖的情况,这会削弱多模态协同决策的效果. 针对不同模态间情感语义不一致的问题,提出一种多模态学习方法,学习情感语义表达一致的模态特征表示. 为了在不影响模态原始信息的同时,提高各模态的共性特征表达并增加模态间的动态交互,首先学习每个模态的共性特征表示,然后利用交叉注意力使单个模态能有效从其余模态的共性特征表示中获取辅助信息. 在模态融合模块,以软注意力机制为基础提出模态注意力,对情感语义表达一致的各模态特征表示进行加权连接,以增大强模态的表达,抑制弱模态对任务的影响. 提出的模型在情感分析数据集MOSI,MOSEI,CH-SIMS上的实验结果均优于对比模型,表明在多模态情感分析任务中考虑情感语义不一致问题的必要性与合理性.
, authors=罗渊贻, 吴锐*, 刘家锋, 唐降龙, authorsList=罗渊贻, 吴锐, 刘家锋, 唐降龙, authorCompany=null, correspAuthors=吴锐, authorNote=
 |
1996年生. 博士研究生. 主要研究方向为多模态学习 |
 |
1976年生. 博士,副教授,博士生导师. 主要研究方向为模式识别、多模态学习 |
 |
1968年生. 博士,副教授. 主要研究方向为模式识别、机器学习 |
 |
1960年生. 博士,教授,博士生导师. 主要研究方向为模式识别、计算机视觉 |
, correspAuthorsNote=
, copyrightStatement=版权所有 © 2025 计算机研究与发展编辑部, copyrightOwner=null, extLink=null, articleAbsUrl=null, sourceXml=QvIkqOUcnfQPWBKBhilytg==, magXml=Y4jq2abGr4UL12fgzjFOTQ==, pdfUrl=null, pdf=W3WCSnWHy2Im3Js0Sv6i1w==, pdfFileSize=1064152, pdfExtLink=null, richHtmlUrl=null, mobilePdfUrl=null, reviewReport=null, pdfFirstPage=null, abstractGraph=rgkmpsFP7L/QCcYZ270oNg==, abstractGraphContent=null, abstractVideo=null, citation=null, cebUrl=null, magXmlContent=tXuaI4VQCvTAbMRYVXr33w==, mapNumber=null, fund=null)}, authors=[Author(id=1327012453558366443, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012451545100502, orderNo=0, firstName=null, middleName=null, lastName=null, nameCn=null, orcid=null, stid=null, country=null, authorPic=null, dead=0, email=luoyy@stu.hit.edu.cn, emailSecond=null, emailThird=null, correspondingAuthor=0, authorType=1, ext={EN=AuthorExt(id=1327012453667418349, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012451545100502, authorId=1327012453558366443, language=EN, stringName=Yuanyi Luo, firstName=Yuanyi, middleName=null, lastName=Luo, prefix=null, suffix=null, authorComment=null, nameInitials=null, affiliation=null, department=null, xref=null, address=College of Computer Science and Technology, Harbin Institute of Technology, Harbin 150006, bio={"content":"
born in 1996. PhD candidate. His main research interest includes multimodal learning
"}, bioImg=null, bioContent=
born in 1996. PhD candidate. His main research interest includes multimodal learning
, aboutCorrespAuthor=null), CN=AuthorExt(id=1327012453726138606, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012451545100502, authorId=1327012453558366443, language=CN, stringName=罗渊贻, firstName=null, middleName=null, lastName=null, prefix=null, suffix=null, authorComment=null, nameInitials=null, affiliation=null, department=null, xref=null, address=哈尔滨工业大学计算机科学与技术学院 哈尔滨 150006, bio={"img":"m0zPg7xcidyAY+C3o1tbnQ==","content":"
1996年生. 博士研究生. 主要研究方向为多模态学习
"}, bioImg=m0zPg7xcidyAY+C3o1tbnQ==, bioContent=
1996年生. 博士研究生. 主要研究方向为多模态学习
, aboutCorrespAuthor=null)}, companyList=[AuthorCompany(id=1327012453457703143, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012451545100502, xref=null, ext=[AuthorCompanyExt(id=1327012453474480360, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012451545100502, companyId=1327012453457703143, language=EN, country=null, province=null, city=null, postcode=null, companyName=null, departmentName=null, remark=College of Computer Science and Technology, Harbin Institute of Technology, Harbin 150006), AuthorCompanyExt(id=1327012453487063273, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012451545100502, companyId=1327012453457703143, language=CN, country=null, province=null, city=null, postcode=null, companyName=null, departmentName=null, remark=哈尔滨工业大学计算机科学与技术学院 哈尔滨 150006)])]), Author(id=1327012453814218992, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012451545100502, orderNo=1, firstName=null, middleName=null, lastName=null, nameCn=null, orcid=null, stid=null, country=null, authorPic=null, dead=0, email=simple@hit.edu.cn, emailSecond=null, emailThird=null, correspondingAuthor=1, authorType=1, ext={EN=AuthorExt(id=1327012453998768370, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012451545100502, authorId=1327012453814218992, language=EN, stringName=Rui Wu, firstName=Rui, middleName=null, lastName=Wu, prefix=null, suffix=null, authorComment=null, nameInitials=null, affiliation=null, department=null, xref=*, address=College of Computer Science and Technology, Harbin Institute of Technology, Harbin 150006, bio={"content":"
born in 1976. PhD, associate professor, PhD supervisor. His main research interests include pattern recognition and multimodal learning
"}, bioImg=null, bioContent=
born in 1976. PhD, associate professor, PhD supervisor. His main research interests include pattern recognition and multimodal learning
, aboutCorrespAuthor=null), CN=AuthorExt(id=1327012454091043059, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012451545100502, authorId=1327012453814218992, language=CN, stringName=吴锐, firstName=null, middleName=null, lastName=null, prefix=null, suffix=null, authorComment=null, nameInitials=null, affiliation=null, department=null, xref=*, address=哈尔滨工业大学计算机科学与技术学院 哈尔滨 150006, bio={"img":"PH5J3LEicyxrWYmDOhRR3A==","content":"
1976年生. 博士,副教授,博士生导师. 主要研究方向为模式识别、多模态学习
"}, bioImg=PH5J3LEicyxrWYmDOhRR3A==, bioContent=
1976年生. 博士,副教授,博士生导师. 主要研究方向为模式识别、多模态学习
, aboutCorrespAuthor=null)}, companyList=[AuthorCompany(id=1327012453457703143, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012451545100502, xref=null, ext=[AuthorCompanyExt(id=1327012453474480360, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012451545100502, companyId=1327012453457703143, language=EN, country=null, province=null, city=null, postcode=null, companyName=null, departmentName=null, remark=College of Computer Science and Technology, Harbin Institute of Technology, Harbin 150006), AuthorCompanyExt(id=1327012453487063273, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012451545100502, companyId=1327012453457703143, language=CN, country=null, province=null, city=null, postcode=null, companyName=null, departmentName=null, remark=哈尔滨工业大学计算机科学与技术学院 哈尔滨 150006)])]), Author(id=1327012454174929141, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012451545100502, orderNo=2, firstName=null, middleName=null, lastName=null, nameCn=null, orcid=null, stid=null, country=null, authorPic=null, dead=0, email=null, emailSecond=null, emailThird=null, correspondingAuthor=0, authorType=1, ext={EN=AuthorExt(id=1327012454267203831, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012451545100502, authorId=1327012454174929141, language=EN, stringName=Jiafeng Liu, firstName=Jiafeng, middleName=null, lastName=Liu, prefix=null, suffix=null, authorComment=null, nameInitials=null, affiliation=null, department=null, xref=null, address=College of Computer Science and Technology, Harbin Institute of Technology, Harbin 150006, bio={"content":"
born in 1968. PhD, associate professor. His main research interests include pattern recognition and machine learning
"}, bioImg=null, bioContent=
born in 1968. PhD, associate professor. His main research interests include pattern recognition and machine learning
, aboutCorrespAuthor=null), CN=AuthorExt(id=1327012454334312696, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012451545100502, authorId=1327012454174929141, language=CN, stringName=刘家锋, firstName=null, middleName=null, lastName=null, prefix=null, suffix=null, authorComment=null, nameInitials=null, affiliation=null, department=null, xref=null, address=哈尔滨工业大学计算机科学与技术学院 哈尔滨 150006, bio={"img":"YK2/i++s2qGvYmOApDBAyw==","content":"
1968年生. 博士,副教授. 主要研究方向为模式识别、机器学习
"}, bioImg=YK2/i++s2qGvYmOApDBAyw==, bioContent=
1968年生. 博士,副教授. 主要研究方向为模式识别、机器学习
, aboutCorrespAuthor=null)}, companyList=[AuthorCompany(id=1327012453457703143, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012451545100502, xref=null, ext=[AuthorCompanyExt(id=1327012453474480360, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012451545100502, companyId=1327012453457703143, language=EN, country=null, province=null, city=null, postcode=null, companyName=null, departmentName=null, remark=College of Computer Science and Technology, Harbin Institute of Technology, Harbin 150006), AuthorCompanyExt(id=1327012453487063273, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012451545100502, companyId=1327012453457703143, language=CN, country=null, province=null, city=null, postcode=null, companyName=null, departmentName=null, remark=哈尔滨工业大学计算机科学与技术学院 哈尔滨 150006)])]), Author(id=1327012454460141818, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012451545100502, orderNo=3, firstName=null, middleName=null, lastName=null, nameCn=null, orcid=null, stid=null, country=null, authorPic=null, dead=0, email=null, emailSecond=null, emailThird=null, correspondingAuthor=0, authorType=1, ext={EN=AuthorExt(id=1327012454539833596, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012451545100502, authorId=1327012454460141818, language=EN, stringName=Xianglong Tang, firstName=Xianglong, middleName=null, lastName=Tang, prefix=null, suffix=null, authorComment=null, nameInitials=null, affiliation=null, department=null, xref=null, address=College of Computer Science and Technology, Harbin Institute of Technology, Harbin 150006, bio={"content":"
born in 1960. PhD, professor, PhD supervisor. His main research interests include pattern recognition and computer vision
"}, bioImg=null, bioContent=
born in 1960. PhD, professor, PhD supervisor. His main research interests include pattern recognition and computer vision
, aboutCorrespAuthor=null), CN=AuthorExt(id=1327012454606942461, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012451545100502, authorId=1327012454460141818, language=CN, stringName=唐降龙, firstName=null, middleName=null, lastName=null, prefix=null, suffix=null, authorComment=null, nameInitials=null, affiliation=null, department=null, xref=null, address=哈尔滨工业大学计算机科学与技术学院 哈尔滨 150006, bio={"img":"uAy4ykO7JsICd1XwAzDbmA==","content":"
1960年生. 博士,教授,博士生导师. 主要研究方向为模式识别、计算机视觉
"}, bioImg=uAy4ykO7JsICd1XwAzDbmA==, bioContent=
1960年生. 博士,教授,博士生导师. 主要研究方向为模式识别、计算机视觉
, aboutCorrespAuthor=null)}, companyList=[AuthorCompany(id=1327012453457703143, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012451545100502, xref=null, ext=[AuthorCompanyExt(id=1327012453474480360, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012451545100502, companyId=1327012453457703143, language=EN, country=null, province=null, city=null, postcode=null, companyName=null, departmentName=null, remark=College of Computer Science and Technology, Harbin Institute of Technology, Harbin 150006), AuthorCompanyExt(id=1327012453487063273, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012451545100502, companyId=1327012453457703143, language=CN, country=null, province=null, city=null, postcode=null, companyName=null, departmentName=null, remark=哈尔滨工业大学计算机科学与技术学院 哈尔滨 150006)])])], keywords=[Keyword(id=1327012454695022846, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012451545100502, language=EN, orderNo=1, keyword=multimodal sentiment analysis), Keyword(id=1327012454757937407, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012451545100502, language=EN, orderNo=2, keyword=common features), Keyword(id=1327012454829240576, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012451545100502, language=EN, orderNo=3, keyword=sentimental semantic inconsistency), Keyword(id=1327012454896349441, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012451545100502, language=EN, orderNo=4, keyword=attention mechanism), Keyword(id=1327012454971846914, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012451545100502, language=CN, orderNo=1, keyword=多模态情感分析), Keyword(id=1327012455038955779, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012451545100502, language=CN, orderNo=2, keyword=共性特征), Keyword(id=1327012455106064644, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012451545100502, language=CN, orderNo=3, keyword=情感语义不一致), Keyword(id=1327012455185756421, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012451545100502, language=CN, orderNo=4, keyword=注意力机制)], refs=[Reference(id=1327012457429709085, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012451545100502, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=1, rfOrder=0, authorNames=null, journalName=null, refType=null, unstructuredReference=陈龙,管子玉,何金红,等. 情感分类研究进展[J]. 计算机研究与发展,2017,54(6):1150−1170, articleTitle=null, refAbstract=null), Reference(id=1327012457505206558, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012451545100502, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=1, rfOrder=1, authorNames=null, journalName=null, refType=null, unstructuredReference=Chen Long, Guan Ziyu, He Jinhong, et al. A survey on sentiment classification[J]. Journal of Computer Research and Development, 2017, 54(6): 1150−1170 (in Chinese), articleTitle=null, refAbstract=null), Reference(id=1327012457597481247, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012451545100502, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=2, rfOrder=2, authorNames=null, journalName=null, refType=null, unstructuredReference=Yu Wenmeng, Xu Hua, Meng Fanyang, et al. CH-SIMS: A Chinese multimodal sentiment analysis dataset with fine-grained annotation of modality[C]//Proc of the 58th Annual Meeting of the Association for Computational Linguistics. Stroudsburg, PA: ACL, 2020: 3718−3727, articleTitle=null, refAbstract=null), Reference(id=1327012457672978720, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012451545100502, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=3, rfOrder=3, authorNames=null, journalName=null, refType=null, unstructuredReference=Tsai Y, Bai S, Liang P P, et al. Multimodal transformer for unaligned multimodal language sequences[C]//Proc of the 57th Int Conf on Association for Computational Linguistics. Stroudsburg, PA: ACL, 2019: 6558−6569, articleTitle=null, refAbstract=null), Reference(id=1327012457731698977, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012451545100502, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=4, rfOrder=4, authorNames=null, journalName=null, refType=null, unstructuredReference=Bertasius G, Wang Heng, Torresani L. Is space-time attention all you need for video understanding[J]. arXiv preprint, arXiv: 2102.05095, 2021, articleTitle=null, refAbstract=null), Reference(id=1327012457790419234, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012451545100502, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=5, rfOrder=5, authorNames=null, journalName=null, refType=null, unstructuredReference=Zellinger W, Grubinger T, Lughofer E, et al. Central moment discrepancy (CMD) for domain-invariant representation learning[J]. arXiv preprint, arXiv: 1702.08811, 2017, articleTitle=null, refAbstract=null), Reference(id=1327012457853333795, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012451545100502, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=6, rfOrder=6, authorNames=null, journalName=null, refType=null, unstructuredReference=Sahay S, Kumar S H, Xia Rui, et al. Multimodal relational tensor network for sentiment and emotion classification[C]//Proc of the 1st Int Conf on Association for Computational Linguistics. Stroudsburg, PA: ACL, 2018: 20−27, articleTitle=null, refAbstract=null), Reference(id=1327012457916248356, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012451545100502, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=7, rfOrder=7, authorNames=null, journalName=null, refType=null, unstructuredReference=Zadeh A, Chen Minghai, Poria S, et al. Tensor fusion network for multimodal sentiment analysis[C]//Proc of the Int Conf on Empirical Methods in Natural Language Processing. Stroudsburg, PA: ACL, 2017: 1103−1114, articleTitle=null, refAbstract=null), Reference(id=1327012457995940133, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012451545100502, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=8, rfOrder=8, authorNames=null, journalName=null, refType=null, unstructuredReference=Liu Zhun, Shen Ying, Lakshminarasimhan V B, et al. Efficient low-rank multimodal fusion with modality-specific factors[C]//Proc of the Int Conf on Association for Computational Linguistics. Stroudsburg, PA: ACL, 2018: 2247−2256, articleTitle=null, refAbstract=null), Reference(id=1327012458071437606, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012451545100502, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=9, rfOrder=9, authorNames=null, journalName=null, refType=null, unstructuredReference=Hazarika D, Zimmermann R, Poria S. MISA: Modality-invariant and -specific representations for multimodal sentiment analysis[J]. arXiv preprint, arXiv: 2005.03545, 2020, articleTitle=null, refAbstract=null), Reference(id=1327012458146935079, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012451545100502, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=10, rfOrder=10, authorNames=null, journalName=null, refType=null, unstructuredReference=Rahman W, Hasan M K, Lee S, et al. Integrating multimodal information in large pretrained transformers[C]//Proc of the 58th Conf on Association for Computational Linguistics. Stroudsburg, PA: ACL, 2020: 2359−2369, articleTitle=null, refAbstract=null), Reference(id=1327012459820462376, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012451545100502, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=11, rfOrder=11, authorNames=null, journalName=null, refType=null, unstructuredReference=Yu Wenmeng, Xu Hua, Yuan Ziqi, et al. Learning modality-specific representations with self-supervised multi-task learning for multimodal sentiment analysis[C]//Proc of the Conf on Artificial Intelligence. Palo Alto, CA: AAAI, 2021: 10790−10797, articleTitle=null, refAbstract=null), Reference(id=1327012460005011753, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012451545100502, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=12, rfOrder=12, authorNames=null, journalName=null, refType=null, unstructuredReference=程艳,尧磊波,张光河,等. 基于注意力机制的多通道CNN和BiGRU的文本情感倾向性分析[J]. 计算机研究与发展,2020,57(12):2583−2595, articleTitle=null, refAbstract=null), Reference(id=1327012460197949738, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012451545100502, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=12, rfOrder=13, authorNames=null, journalName=null, refType=null, unstructuredReference=Cheng Yan, Yao Leibo, Zhang Guanghe, et al. Text sentiment orientation analysis of multi-channels CNN and BiGRU based on attention mechanism[J]. Journal of Computer Research and Development, 2020, 57(12): 2583−2595 (in Chinese), articleTitle=null, refAbstract=null), Reference(id=1327012460319584555, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012451545100502, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=13, rfOrder=14, authorNames=null, journalName=null, refType=null, unstructuredReference=Bahdanau D, Cho K, Bengio Y. Neural machine translation by jointly learning to align and translate[J]. arXiv preprint, arXiv: 1409.0473, 2014, articleTitle=null, refAbstract=null), Reference(id=1327012460432830764, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012451545100502, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=14, rfOrder=15, authorNames=null, journalName=null, refType=null, unstructuredReference=Mnih V, Heess N, Graves A, et al. Recurrent models of visual attention[C]//Proc of the 27th Int Conf on Neural Information Processing Systems. Cambridge, MA: NeurIPS, 2014: 2204–2212, articleTitle=null, refAbstract=null), Reference(id=1327012460520911149, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012451545100502, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=15, rfOrder=16, authorNames=null, journalName=null, refType=null, unstructuredReference=Zhang Zhizheng, Lan Cuiling, Zeng Wenjun, et al. Relation-aware global attention for person re-identification[C]//Proc of the IEEE Conf on Computer Vision and Pattern Recognition. Piscataway, NJ: IEEE, 2020: 3183–3192, articleTitle=null, refAbstract=null), Reference(id=1327012460600602926, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012451545100502, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=16, rfOrder=17, authorNames=null, journalName=null, refType=null, unstructuredReference=Xu K, Ba J L, Kiros R, et al. Show, attend and tell: Neural image caption generation with visual attention[C]//Proc of the Int Conf on Machine Learning. New York: ACM, 2015: 2048−2057, articleTitle=null, refAbstract=null), Reference(id=1327012460667711791, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012451545100502, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=17, rfOrder=18, authorNames=null, journalName=null, refType=null, unstructuredReference=Guo Menghao, Xu Tianxing, Liu Jiangjiang, et al. Attention mechanisms in computer vision: A survey[J]. Computational Visual Media, 2022, 8(3): 331−368, articleTitle=null, refAbstract=null), Reference(id=1327012460734820657, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012451545100502, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=18, rfOrder=19, authorNames=null, journalName=null, refType=null, unstructuredReference=Li Jianing, Wang Jingdong, Gao Wen, et al. Global–local temporal representations for video person re-identification[C]//Proc of the IEEE Int Conf on Computer Vision. Piscataway, NJ: IEEE, 2019: 3957–3966, articleTitle=null, refAbstract=null), Reference(id=1327012460801929522, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012451545100502, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=19, rfOrder=20, authorNames=null, journalName=null, refType=null, unstructuredReference=Xiang Long, Gan Chuang, Melo G, et al. Multimodal keyless attention fusion for video classification[C]//Proc of the 32nd Conf on Artificial Intelligence. Palo Alto, CA: AAAI, 2018: 7202−7209, articleTitle=null, refAbstract=null), Reference(id=1327012460864844083, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012451545100502, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=20, rfOrder=21, authorNames=null, journalName=null, refType=null, unstructuredReference=Ghosal D, Akhtar M S, Chauhan D, et al. Contextual inter-modal attention for multi-modal sentiment analysis[C]//Proc of the 2018 Conf on Empirical Methods in Natural Language Processing. Stroudsburg, PA: ACL, 2018: 3454−3466, articleTitle=null, refAbstract=null), Reference(id=1327012460927758644, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012451545100502, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=21, rfOrder=22, authorNames=null, journalName=null, refType=null, unstructuredReference=Ye Junjie, Zhou Jie, Tian Junfeng, et al. Sentiment-aware multimodal pre-training for multimodal sentiment analysis[J]. Knowledge-Based Systems, 2022, 258: 110021, articleTitle=null, refAbstract=null), Reference(id=1327012461011644725, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012451545100502, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=22, rfOrder=23, authorNames=null, journalName=null, refType=null, unstructuredReference=Sun Zhongkai, Sarma P, Sethares W, et al. Learning relationships between text, audio, and video via deep canonical correlation for multimodal language analysis[C]//Proc of the Conf on Artificial Intelligence. Palo Alto, CA: AAAI, 2020: 8992−8999, articleTitle=null, refAbstract=null), Reference(id=1327012461108113718, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012451545100502, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=23, rfOrder=24, authorNames=null, journalName=null, refType=null, unstructuredReference=Devlin J, Chang Mingwei, Lee K, et al. BERT: Pre-training of deep bidirectional transformers for language understanding[C]//Proc of the Conf on Association for Computational Linguistics. Stroudsburg, PA: ACL, 2019: 4171–4186, articleTitle=null, refAbstract=null), Reference(id=1327012461208777015, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012451545100502, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=24, rfOrder=25, authorNames=null, journalName=null, refType=null, unstructuredReference=Zadeh A, Zellers R, Pincus E, et al. MOSI: Multimodal corpus of sentiment intensity and subjectivity analysis in online opinion videos[J]. arXiv preprint, arXiv: 1606.06259, 2016, articleTitle=null, refAbstract=null), Reference(id=1327012461280080184, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012451545100502, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=25, rfOrder=26, authorNames=null, journalName=null, refType=null, unstructuredReference=Zadeh A, Liang P P, Vanbriesen J, et al. Multimodal language analysis in the wild: CMU-MOSEI dataset and interpretable dynamic fusion graph[C]//Proc of the 56th Conf on Association for Computational Linguistics. Stroudsburg, PA: ACL, 2018: 2236−2246, articleTitle=null, refAbstract=null), Reference(id=1327012461347189049, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012451545100502, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=26, rfOrder=27, authorNames=null, journalName=null, refType=null, unstructuredReference=Sun Hao, Wang Hongyi, Liu Jiaqing, et al. CubeMLP: An MLP-based model for multimodal sentiment analysis and depression estimation[C]//Proc of the 30th ACM Int Conf on Multimedia. New York: ACM, 2022: 3722−3729, articleTitle=null, refAbstract=null)], funds=null, companyList=[AuthorCompany(id=1327012453457703143, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012451545100502, xref=null, ext=[AuthorCompanyExt(id=1327012453474480360, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012451545100502, companyId=1327012453457703143, language=EN, country=null, province=null, city=null, postcode=null, companyName=null, departmentName=null, remark=College of Computer Science and Technology, Harbin Institute of Technology, Harbin 150006), AuthorCompanyExt(id=1327012453487063273, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012451545100502, companyId=1327012453457703143, language=CN, country=null, province=null, city=null, postcode=null, companyName=null, departmentName=null, remark=哈尔滨工业大学计算机科学与技术学院 哈尔滨 150006)])], figs=[ArticleFig(id=1327012455303196934, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012451545100502, language=EN, label=Figure 1, caption=
Multimodal sentiment analysis model, figureFileSmall=QljLCmHYEadBM1k+rmkUlg==, figureFileBig=rgkmpsFP7L/QCcYZ270oNg==, tableContent=null), ArticleFig(id=1327012455382888711, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012451545100502, language=CN, label=图1, caption=
多模态情感分析模型, figureFileSmall=QljLCmHYEadBM1k+rmkUlg==, figureFileBig=rgkmpsFP7L/QCcYZ270oNg==, tableContent=null), ArticleFig(id=1327012455580021000, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012451545100502, language=EN, label=Figure 2, caption=
Structure diagram of modal attention, figureFileSmall=kRN/DlXRTHMyxHOb++/kDA==, figureFileBig=ARR8l5S1r/BKSvvI/UFXnw==, tableContent=null), ArticleFig(id=1327012455718433033, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012451545100502, language=CN, label=图2, caption=
模态注意力结构图, figureFileSmall=kRN/DlXRTHMyxHOb++/kDA==, figureFileBig=ARR8l5S1r/BKSvvI/UFXnw==, tableContent=null), ArticleFig(id=1327012455798124810, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012451545100502, language=EN, label=Table 1, caption=
Influence of Sample Data with Sentimental Semantic Inconsistency Between Modalities on Multimodal Sentiment Analysis Tasks, figureFileSmall=null, figureFileBig=null, tableContent=
| 模型 | CH-SIMS数据集 |
|---|
| 数据样本 | 2分类 | 3分类 | F1 | MAE | Corr |
|---|
| LF-DNN | org | 0.792 | 0.644 | 0.802 | 0.458 | 0.558 |
| (–)d | 0.830 | 0.669 | 0.831 | 0.408 | 0.659 |
| (re)$ {\mathrm{d}}_{170} $ | 0.804 | 0.668 | 0.811 | 0.418 | 0.639 |
| (re)$ {\mathrm{d}}_{228} $ | 0.793 | 0.667 | 0.802 | 0.432 | 0.619 |
| (re)$ {\mathrm{d}}_{\mathrm{t}\mathrm{e}\mathrm{s}\mathrm{t}} $ | 0.699 | 0.588 | 0.712 | 0.530 | 0.403 |
| MulT | org | 0.784 | 0.647 | 0.799 | 0.492 | 0.555 |
| (–)d | 0.790 | 0.634 | 0.807 | 0.492 | 0.589 |
| (re)$ {\mathrm{d}}_{228} $ | 0.770 | 0.628 | 0.790 | 0.515 | 0.544 |
), ArticleFig(id=1327012455861039371, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012451545100502, language=CN, label=表1, caption=
模态间情感语义不一致的样本数据对多模态情感分析任务的影响
, figureFileSmall=null, figureFileBig=null, tableContent=
| 模型 | CH-SIMS数据集 |
|---|
| 数据样本 | 2分类 | 3分类 | F1 | MAE | Corr |
|---|
| LF-DNN | org | 0.792 | 0.644 | 0.802 | 0.458 | 0.558 |
| (–)d | 0.830 | 0.669 | 0.831 | 0.408 | 0.659 |
| (re)$ {\mathrm{d}}_{170} $ | 0.804 | 0.668 | 0.811 | 0.418 | 0.639 |
| (re)$ {\mathrm{d}}_{228} $ | 0.793 | 0.667 | 0.802 | 0.432 | 0.619 |
| (re)$ {\mathrm{d}}_{\mathrm{t}\mathrm{e}\mathrm{s}\mathrm{t}} $ | 0.699 | 0.588 | 0.712 | 0.530 | 0.403 |
| MulT | org | 0.784 | 0.647 | 0.799 | 0.492 | 0.555 |
| (–)d | 0.790 | 0.634 | 0.807 | 0.492 | 0.589 |
| (re)$ {\mathrm{d}}_{228} $ | 0.770 | 0.628 | 0.790 | 0.515 | 0.544 |
), ArticleFig(id=1327012455936536844, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012451545100502, language=EN, label=Table 2, caption=
Hyper-Parameter Settings in Dataset, figureFileSmall=null, figureFileBig=null, tableContent=
| 超参数 | MOSI | MOSEI | SIMS |
|---|
| 注:“batch size”是单次训练的样本个数;“learning rate”是神经网络的学习率;“hidden size”是隐藏层的维度大小;“depth”是时空注意力和时间注意力中的学习层数;“epoch”是训练迭代次数;“weight”是共性特征表示学习的损失函数系数;“early stop”是为防止过拟合而设置的提前结束循环条件. |
| batch size | 32 | 64 | 64 |
| learning rate | 0.0001 | 0.0001 | 0.0001 |
| hidden size | 256 | 256 | 256 |
| depth | 12 | 12 | 18 |
| epoch | 60 | 100 | 100 |
| weight | 0.5 | 0.5 | 0.8 |
| early stop | 20 | 20 | 20 |
), ArticleFig(id=1327012456024617229, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012451545100502, language=CN, label=表2, caption=
数据集的超参数设置
, figureFileSmall=null, figureFileBig=null, tableContent=
| 超参数 | MOSI | MOSEI | SIMS |
|---|
| 注:“batch size”是单次训练的样本个数;“learning rate”是神经网络的学习率;“hidden size”是隐藏层的维度大小;“depth”是时空注意力和时间注意力中的学习层数;“epoch”是训练迭代次数;“weight”是共性特征表示学习的损失函数系数;“early stop”是为防止过拟合而设置的提前结束循环条件. |
| batch size | 32 | 64 | 64 |
| learning rate | 0.0001 | 0.0001 | 0.0001 |
| hidden size | 256 | 256 | 256 |
| depth | 12 | 12 | 18 |
| epoch | 60 | 100 | 100 |
| weight | 0.5 | 0.5 | 0.8 |
| early stop | 20 | 20 | 20 |
), ArticleFig(id=1327012456100114703, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012451545100502, language=EN, label=Table 3, caption=
Comparison Results of Methods Based on CMU-MOSI Dataset, figureFileSmall=null, figureFileBig=null, tableContent=
| 模型 | 2分类 | 5分类 | 7分类 | F1 | MAE | Corr |
|---|
| 注:上标为“1”的数据来源于文献[9],上标为“2”的数据来源于文献[11],上标为“3”的数据来源于文献[26],上标为“*”的数据结果是根据可获取的开源代码进行复现得出. |
| TFN1 | 0.808 | | | 0.807 | 0. 901 | 0.698 |
| LMF1 | 0.825 | | | 0.824 | 0. 917 | 0.695 |
| MulT1 | 0.834 | | | 0.828 | 0.871 | 0.698 |
| MISA1 | 0.834 | | | 0.836 | 0.783 | 0.761 |
| MAG-BERT2 | 0.843 | | | 0.843 | 0.731 | 0.789 |
| Self-MM2 | 0.860 | | | 0.860 | 0.713 | 0.798 |
| CubeMLP3 | 0.856 | | 0.455 | 0.855 | 0.770 | 0.767 |
| MAG-BERT* | 0.835 | 0.501 | 0.438 | 0.836 | 0.749 | 0.783 |
| Self-MM* | 0.840 | 0.526 | 0.458 | 0.841 | 0.724 | 0.791 |
| 本文模型 | 0.866 | 0.532 | 0.474 | 0.867 | 0.709 | 0.796 |
), ArticleFig(id=1327012456242721040, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012451545100502, language=CN, label=表3, caption=
基于CMU-MOSI数据集的方法对比结果
, figureFileSmall=null, figureFileBig=null, tableContent=
| 模型 | 2分类 | 5分类 | 7分类 | F1 | MAE | Corr |
|---|
| 注:上标为“1”的数据来源于文献[9],上标为“2”的数据来源于文献[11],上标为“3”的数据来源于文献[26],上标为“*”的数据结果是根据可获取的开源代码进行复现得出. |
| TFN1 | 0.808 | | | 0.807 | 0. 901 | 0.698 |
| LMF1 | 0.825 | | | 0.824 | 0. 917 | 0.695 |
| MulT1 | 0.834 | | | 0.828 | 0.871 | 0.698 |
| MISA1 | 0.834 | | | 0.836 | 0.783 | 0.761 |
| MAG-BERT2 | 0.843 | | | 0.843 | 0.731 | 0.789 |
| Self-MM2 | 0.860 | | | 0.860 | 0.713 | 0.798 |
| CubeMLP3 | 0.856 | | 0.455 | 0.855 | 0.770 | 0.767 |
| MAG-BERT* | 0.835 | 0.501 | 0.438 | 0.836 | 0.749 | 0.783 |
| Self-MM* | 0.840 | 0.526 | 0.458 | 0.841 | 0.724 | 0.791 |
| 本文模型 | 0.866 | 0.532 | 0.474 | 0.867 | 0.709 | 0.796 |
), ArticleFig(id=1327012456318218513, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012451545100502, language=EN, label=Table 4, caption=
Comparison Results of Models Based on CMU-MOSEI Dataset, figureFileSmall=null, figureFileBig=null, tableContent=
| 模型 | 2分类 | 5分类 | 7分类 | F1 | MAE | Corr |
|---|
| 注:黑体数字表示最优结果. 上标为“1”的数据来源于文献[9],上标为“2”的数据来源于文献[11],上标为“3”的数据来源于文献[26],上标为“*”的数据结果是根据可获取的开源代码进行复现得出. |
| TFN1 | 0.825 | | | 0.821 | 0.593 | 0.700 |
| LMF1 | 0.820 | | | 0.821 | 0.623 | 0.677 |
| MulT1 | 0.825 | | | 0.823 | 0.580 | 0.703 |
| MISA2 | 0.842 | | | 0.840 | 0.568 | 0.724 |
| MAG-BERT2 | 0.852 | | | 0.851 | 0.539 | 0.753 |
| Self-MM2 | 0.852 | | | 0.853 | 0.530 | 0.765 |
| CubeMLP3 | 0.851 | | 0.549 | 0.845 | 0.529 | 0.760 |
| MAG-BERT* | 0.851 | 0.545 | 0.527 | 0.850 | 0.543 | 0.760 |
| Self-MM* | 0.852 | 0.545 | 0.529 | 0.851 | 0.539 | 0.766 |
| 本文模型 | 0.854 | 0.547 | 0.531 | 0.854 | 0.537 | 0.767 |
), ArticleFig(id=1327012456456630546, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012451545100502, language=CN, label=表4, caption=
基于CMU-MOSEI数据集的模型对比结果
, figureFileSmall=null, figureFileBig=null, tableContent=
| 模型 | 2分类 | 5分类 | 7分类 | F1 | MAE | Corr |
|---|
| 注:黑体数字表示最优结果. 上标为“1”的数据来源于文献[9],上标为“2”的数据来源于文献[11],上标为“3”的数据来源于文献[26],上标为“*”的数据结果是根据可获取的开源代码进行复现得出. |
| TFN1 | 0.825 | | | 0.821 | 0.593 | 0.700 |
| LMF1 | 0.820 | | | 0.821 | 0.623 | 0.677 |
| MulT1 | 0.825 | | | 0.823 | 0.580 | 0.703 |
| MISA2 | 0.842 | | | 0.840 | 0.568 | 0.724 |
| MAG-BERT2 | 0.852 | | | 0.851 | 0.539 | 0.753 |
| Self-MM2 | 0.852 | | | 0.853 | 0.530 | 0.765 |
| CubeMLP3 | 0.851 | | 0.549 | 0.845 | 0.529 | 0.760 |
| MAG-BERT* | 0.851 | 0.545 | 0.527 | 0.850 | 0.543 | 0.760 |
| Self-MM* | 0.852 | 0.545 | 0.529 | 0.851 | 0.539 | 0.766 |
| 本文模型 | 0.854 | 0.547 | 0.531 | 0.854 | 0.537 | 0.767 |
), ArticleFig(id=1327012456582459667, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012451545100502, language=EN, label=Table 5, caption=
Comparison Results of Models Based on CH-SIMS Dataset, figureFileSmall=null, figureFileBig=null, tableContent=
| 模型 | 多任务 | 2分类 | F1 |
|---|
| 注:上标为“2”的数据来源于文献[11],上标为“*”的数据结果是根据可获取的开源代码进行复现得出. |
| TFN2 | × | 0.798 | 0.802 |
| LMF2 | × | 0.794 | 0.787 |
| Self-MM2 | √ | 0.807 | 0.808 |
| Self-MM* | √ | 0.792 | 0.789 |
| 本文模型 | √ | 0.812 | 0.825 |
), ArticleFig(id=1327012456725066004, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012451545100502, language=CN, label=表5, caption=
基于CH-SIMS数据集的模型对比结果
, figureFileSmall=null, figureFileBig=null, tableContent=
| 模型 | 多任务 | 2分类 | F1 |
|---|
| 注:上标为“2”的数据来源于文献[11],上标为“*”的数据结果是根据可获取的开源代码进行复现得出. |
| TFN2 | × | 0.798 | 0.802 |
| LMF2 | × | 0.794 | 0.787 |
| Self-MM2 | √ | 0.807 | 0.808 |
| Self-MM* | √ | 0.792 | 0.789 |
| 本文模型 | √ | 0.812 | 0.825 |
), ArticleFig(id=1327012456804757781, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012451545100502, language=EN, label=Table 6, caption=
Analysis Results of Models Based on CH-SIMS Dataset, figureFileSmall=null, figureFileBig=null, tableContent=
| 样本 | 实际值 | | 预测值 |
|---|
| T | A | V | M | | T | A | V | M |
|---|
| 注:“T”表示文本模态,“A”表示音频模态,“V”表示视频模态,“M”表示多模型. |
| N38 | –0.2 | –0.6 | 0.4 | –1.0 | | –0.4 | –0.2 | –0.2 | –0.6 |
| N63 | –0.6 | –0.2 | 0.4 | –0.6 | | –0.3 | –0.2 | –0.2 | –0.5 |
| N187 | –0.6 | 0.2 | –1.0 | –0.8 | | –0.5 | –0.1 | –0.3 | –0.7 |
| N732 | –0.8 | –0.6 | 0.6 | –0.6 | | –0.4 | –0.2 | –0.2 | –0.6 |
| N2081 | –0.2 | 0.6 | –0.4 | –1.0 | | –0.1 | –0.2 | –0.2 | –0.2 |
), ArticleFig(id=1327012456876060950, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012451545100502, language=CN, label=表6, caption=
基于CH-SIMS数据集的模型分析结果
, figureFileSmall=null, figureFileBig=null, tableContent=
| 样本 | 实际值 | | 预测值 |
|---|
| T | A | V | M | | T | A | V | M |
|---|
| 注:“T”表示文本模态,“A”表示音频模态,“V”表示视频模态,“M”表示多模型. |
| N38 | –0.2 | –0.6 | 0.4 | –1.0 | | –0.4 | –0.2 | –0.2 | –0.6 |
| N63 | –0.6 | –0.2 | 0.4 | –0.6 | | –0.3 | –0.2 | –0.2 | –0.5 |
| N187 | –0.6 | 0.2 | –1.0 | –0.8 | | –0.5 | –0.1 | –0.3 | –0.7 |
| N732 | –0.8 | –0.6 | 0.6 | –0.6 | | –0.4 | –0.2 | –0.2 | –0.6 |
| N2081 | –0.2 | 0.6 | –0.4 | –1.0 | | –0.1 | –0.2 | –0.2 | –0.2 |
), ArticleFig(id=1327012456947364119, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012451545100502, language=EN, label=Table 7, caption=
Cosine Similarity Comparison of Modality Features Before and After Training, figureFileSmall=null, figureFileBig=null, tableContent=
| 数据集 | 训练前 | 训练后 |
|---|
| MOSI | –0.0118 | 0.0314 |
| MOSEI | –0.0316 | 0.0176 |
| CH-SIMS | 0.0499 | 0.0835 |
), ArticleFig(id=1327012457010278680, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012451545100502, language=CN, label=表7, caption=
模态特征在训练前后的余弦相似度对比
, figureFileSmall=null, figureFileBig=null, tableContent=
| 数据集 | 训练前 | 训练后 |
|---|
| MOSI | –0.0118 | 0.0314 |
| MOSEI | –0.0316 | 0.0176 |
| CH-SIMS | 0.0499 | 0.0835 |
), ArticleFig(id=1327012457094164761, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012451545100502, language=EN, label=Table 8, caption=
Comparison Results of Methods, figureFileSmall=null, figureFileBig=null, tableContent=
| 计算方法 | 2分类 | F1 | MAE | Corr | 训练时间/s |
|---|
| CMD | 0.866 | 0.867 | 0.709 | 0.796 | 11 |
| MMD | 0.854 | 0.854 | 0.765 | 0.787 | 15 |
| KL | 0.846 | 0.847 | 0.770 | 0.764 | 9 |
), ArticleFig(id=1327012457178050842, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012451545100502, language=CN, label=表8, caption=
方法对比结果
, figureFileSmall=null, figureFileBig=null, tableContent=
| 计算方法 | 2分类 | F1 | MAE | Corr | 训练时间/s |
|---|
| CMD | 0.866 | 0.867 | 0.709 | 0.796 | 11 |
| MMD | 0.854 | 0.854 | 0.765 | 0.787 | 15 |
| KL | 0.846 | 0.847 | 0.770 | 0.764 | 9 |
), ArticleFig(id=1327012457261936923, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012451545100502, language=EN, label=Table 9, caption=
Results of Ablation Study, figureFileSmall=null, figureFileBig=null, tableContent=
| 模型 | 2分类 | F1 | MAE | Corr |
|---|
| 本文模型 | 0.867 | 0.867 | 0.709 | 0.796 |
| (–)时空注意力 | 0.836 | 0.835 | 0.763 | 0.786 |
| (–)交叉注意力 | 0.839 | 0.839 | 0.762 | 0.769 |
| (–)模态注意力 | 0.845 | 0.844 | 0.747 | 0.788 |
), ArticleFig(id=1327012457333240092, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1327012451545100502, language=CN, label=表9, caption=
消融实验结果
, figureFileSmall=null, figureFileBig=null, tableContent=
| 模型 | 2分类 | F1 | MAE | Corr |
|---|
| 本文模型 | 0.867 | 0.867 | 0.709 | 0.796 |
| (–)时空注意力 | 0.836 | 0.835 | 0.763 | 0.786 |
| (–)交叉注意力 | 0.839 | 0.839 | 0.762 | 0.769 |
| (–)模态注意力 | 0.845 | 0.844 | 0.747 | 0.788 |
)], attaches=null, journal=Journal(id=1324759606947315908, delFlag=0, nameCn=计算机研究与发展, nameEn=Journal of Computer Research and Development, nameHistory1=null, nameHistory2=null, issn=1000-1239, eissn=null, cn=11-1777/TP, coden=null, periodic=0, language=CN, oaType=null, ccby=null, superviseOffice=null, ownerOffice=null, pubOffice=null, editorOffice=null, officeType=null, aims=null, clcCode=null, officeProv=null, officeCity=null, officeAddr=null, officeZip=null, officeEmail=null, officePhone=null, editDirector=null, officeDirector=null, officeDirectorPhone=null, officeStaffNum=null, officeEmpNum=null, coverPicUrl=k2l8zgqwstuxxevFX2qxVA==, journalPrice=null, startedYear=null, abbrevIsoEn=Journal of Computer Research and Development, journalRemark=null, publicationField=null, createdTime=1793776903141, updatedTime=1793850698612, createdBy=18614031015, updatedBy=13701087609, firstLetterCn=J, firstLetterEn=J, subjectCode=Electronic Information, subjectName=null, subjectCodeEn=Electronic Information, subjectNameEn=null, picCn=k2l8zgqwstuxxevFX2qxVA==, picEn=r6k0Ry+RD+c/VXj5QarmDw==, jcr=null, cjcr=null, exts=[JournalExt(id=1325069127710286688, language=CN, name=计算机研究与发展, nameHistory1=null, nameHistory2=null, managedBy=, sponsoredBy=, publishedBy=, editorOffice=, officeProv=null, officeCity=null, officeAddr=, officeZip=, editDirector=, officeDirector=null, officePhone=null, coverPicUrl=null, journalRemark=, submitArticleUrl=null, websiteUrl=, createdTime=1793850698640, updatedTime=1793850698640, createdBy=13701087609, updatedBy=13701087609, submissionGuidelinesUrl=, submissionAuthorUrl=https://crad.ict.ac.cn/journalx/authorLogOn.action, submissionEditorUrl=https://crad.ict.ac.cn/journalx/editorLogOn.action, submissionReviewUrl=https://crad.ict.ac.cn/journalx/expertLogOn.action, submissionCeEditorUrl=https://crad.ict.ac.cn/journalx/editorInChiefLogOn.action, submissionAeEditorUrl=, option={"copyright":""}), JournalExt(id=1325069127777395553, language=EN, name=Journal of Computer Research and Development, nameHistory1=null, nameHistory2=null, managedBy=, sponsoredBy=, publishedBy=, editorOffice=, officeProv=null, officeCity=null, officeAddr=, officeZip=, editDirector=, officeDirector=null, officePhone=null, coverPicUrl=null, journalRemark=, submitArticleUrl=null, websiteUrl=, createdTime=1793850698656, updatedTime=1793850698656, createdBy=13701087609, updatedBy=13701087609, submissionGuidelinesUrl=, submissionAuthorUrl=https://crad.ict.ac.cn/journalx/authorLogOn.action, submissionEditorUrl=https://crad.ict.ac.cn/journalx/editorLogOn.action, submissionReviewUrl=https://crad.ict.ac.cn/journalx/expertLogOn.action, submissionCeEditorUrl=https://crad.ict.ac.cn/journalx/editorInChiefLogOn.action, submissionAeEditorUrl=, option={"copyright":""})], databaseList=null, tenantJournalId=1324759914146529478, websiteList=[Website(id=1325070300139892771, webName=null, webTitle=null, webDomain=null, webCopyrigh=null, webIpcNo=null, seoTitle=null, seoKeywords=null, seoDescription=null, tenantJournalId=null, journalId=1324759914146529478, journalNameCn=null, journalNameEn=null, grayFlag=null, tenantId=1146029695717560320, platformId=null, journalGroupId=null, journalGroupNameCn=null, journalGroupNameEn=null, type=1, domain=https://castjournals.cast.org.cn/joweb/jsjyjyfz/CN, language=CN, createTime=1793850978169, createBy=18614031015, updateTime=1793851040259, updateBy=18614031015, name=计算机研究与发展-中文, tplId=1146099689490845704, title=计算机研究与发展, delFlag=0, indexPage=/home, props=[WebsiteProps(id=1325070893751349360, tenantId=1146029695717560320, journalId=null, journalGroupId=null, siteId=1325070300139892771, code=articleTextType, value=kx, createTime=1793851119697, updateTime=1793851119697, creator=18614031015, updator=18614031015), WebsiteProps(id=1325070893730377837, tenantId=1146029695717560320, journalId=null, journalGroupId=null, siteId=1325070300139892771, code=banner, value=null, createTime=1793851119692, updateTime=1793851119692, creator=18614031015, updator=18614031015), WebsiteProps(id=1325070893772320883, tenantId=1146029695717560320, journalId=null, journalGroupId=null, siteId=1325070300139892771, code=grayFlag, value=0, createTime=1793851119702, updateTime=1793851119702, creator=18614031015, updator=18614031015), WebsiteProps(id=1325070893721989228, tenantId=1146029695717560320, journalId=null, journalGroupId=null, siteId=1325070300139892771, code=logo, value=https://castjournals.cast.org.cn/joweb/jsjyjyfz/CN/file/pic?fileId=U27tDsy717YF7XSPA+Vwdw==, createTime=1793851119690, updateTime=1793851119690, creator=18614031015, updator=18614031015), WebsiteProps(id=1325070893784903797, tenantId=1146029695717560320, journalId=null, journalGroupId=null, siteId=1325070300139892771, code=minRunFlag, value=0, createTime=1793851119705, updateTime=1793851119705, creator=18614031015, updator=18614031015), WebsiteProps(id=1325070893742960751, tenantId=1146029695717560320, journalId=null, journalGroupId=null, siteId=1325070300139892771, code=picServerUrl, value=https://castjournals.cast.org.cn/joweb/jsjyjyfz/CN/file/pic, createTime=1793851119695, updateTime=1793851119695, creator=18614031015, updator=18614031015), WebsiteProps(id=1325070893780709492, tenantId=1146029695717560320, journalId=null, journalGroupId=null, siteId=1325070300139892771, code=silenceFlag, value=0, createTime=1793851119704, updateTime=1793851119704, creator=18614031015, updator=18614031015), WebsiteProps(id=1325070893738766446, tenantId=1146029695717560320, journalId=null, journalGroupId=null, siteId=1325070300139892771, code=staticResourcePath, value=https://castjournals.cast.org.cn/joweb/cast_kjdb_cn_619/, createTime=1793851119694, updateTime=1793851119694, creator=18614031015, updator=18614031015), WebsiteProps(id=1325070893755543665, tenantId=1146029695717560320, journalId=null, journalGroupId=null, siteId=1325070300139892771, code=themeColor, value=null, createTime=1793851119698, updateTime=1793851119698, creator=18614031015, updator=18614031015), WebsiteProps(id=1325070893763932274, tenantId=1146029695717560320, journalId=null, journalGroupId=null, siteId=1325070300139892771, code=themeStyle, value=null, createTime=1793851119700, updateTime=1793851119700, creator=18614031015, updator=18614031015)]), Website(id=1325070300580294698, webName=null, webTitle=null, webDomain=null, webCopyrigh=null, webIpcNo=null, seoTitle=null, seoKeywords=null, seoDescription=null, tenantJournalId=null, journalId=1324759914146529478, journalNameCn=null, journalNameEn=null, grayFlag=null, tenantId=1146029695717560320, platformId=null, journalGroupId=null, journalGroupNameCn=null, journalGroupNameEn=null, type=1, domain=https://castjournals.cast.org.cn/joweb/jsjyjyfz/EN, language=EN, createTime=1793850978274, createBy=18614031015, updateTime=1793851036638, updateBy=18614031015, name=计算机研究与发展-英文, tplId=1146101810881728533, title=Journal of Computer Research and Development, delFlag=0, indexPage=/home, props=[WebsiteProps(id=1325070925288321148, tenantId=1146029695717560320, journalId=null, journalGroupId=null, siteId=1325070300580294698, code=articleTextType, value=kx, createTime=1793851127216, updateTime=1793851127216, creator=18614031015, updator=18614031015), WebsiteProps(id=1325070925263155321, tenantId=1146029695717560320, journalId=null, journalGroupId=null, siteId=1325070300580294698, code=banner, value=null, createTime=1793851127210, updateTime=1793851127210, creator=18614031015, updator=18614031015), WebsiteProps(id=1325070925309292671, tenantId=1146029695717560320, journalId=null, journalGroupId=null, siteId=1325070300580294698, code=grayFlag, value=0, createTime=1793851127221, updateTime=1793851127221, creator=18614031015, updator=18614031015), WebsiteProps(id=1325070925254766712, tenantId=1146029695717560320, journalId=null, journalGroupId=null, siteId=1325070300580294698, code=logo, value=https://castjournals.cast.org.cn/joweb/jsjyjyfz/EN/file/pic?fileId=U27tDsy717YF7XSPA+Vwdw==, createTime=1793851127208, updateTime=1793851127208, creator=18614031015, updator=18614031015), WebsiteProps(id=1325070925326069889, tenantId=1146029695717560320, journalId=null, journalGroupId=null, siteId=1325070300580294698, code=minRunFlag, value=0, createTime=1793851127225, updateTime=1793851127225, creator=18614031015, updator=18614031015), WebsiteProps(id=1325070925279932539, tenantId=1146029695717560320, journalId=null, journalGroupId=null, siteId=1325070300580294698, code=picServerUrl, value=https://castjournals.cast.org.cn/joweb/jsjyjyfz/EN/file/pic, createTime=1793851127214, updateTime=1793851127214, creator=18614031015, updator=18614031015), WebsiteProps(id=1325070925317681280, tenantId=1146029695717560320, journalId=null, journalGroupId=null, siteId=1325070300580294698, code=silenceFlag, value=0, createTime=1793851127223, updateTime=1793851127223, creator=18614031015, updator=18614031015), WebsiteProps(id=1325070925271543930, tenantId=1146029695717560320, journalId=null, journalGroupId=null, siteId=1325070300580294698, code=staticResourcePath, value=https://castjournals.cast.org.cn/joweb/cast_kjdb_en_623/, createTime=1793851127212, updateTime=1793851127212, creator=18614031015, updator=18614031015), WebsiteProps(id=1325070925292515453, tenantId=1146029695717560320, journalId=null, journalGroupId=null, siteId=1325070300580294698, code=themeColor, value=null, createTime=1793851127217, updateTime=1793851127217, creator=18614031015, updator=18614031015), WebsiteProps(id=1325070925300904062, tenantId=1146029695717560320, journalId=null, journalGroupId=null, siteId=1325070300580294698, code=themeStyle, value=null, createTime=1793851127219, updateTime=1793851127219, creator=18614031015, updator=18614031015)])], journalTitle=计算机研究与发展, weixinUrl=null, journalUrl=https://crad.ict.ac.cn/, iacademicId=null, status=1, seqNo=null, journalTitleEn=Journal of Computer Research and Development, journalPhotoCn=k2l8zgqwstuxxevFX2qxVA==, journalPhotoEn=r6k0Ry+RD+c/VXj5QarmDw==, journalFirstLetter=J, journalRecommend=null, journalNew=null, journalCollection=null, jcrJf=null, cjcrJf=null, jcrJfStr=null, cjcrJfStr=null, submissionFirstDecision=null, sciSubjectClassification=null, casSubjectClassification=null, citeScore=null, totalCitationFrequency=null, icpCode=null, psCode=null, advertisingLicenseCode=null, copyrightInformation=null, country=null, option=, provinceCode=null, provinceName=null, collectFlag=false, interPubPlatform=, interPubPlatformUrl=null), detailUrlCn=https://castjournals.cast.org.cn/joweb/jsjyjyfz/CN/10.7544/issn1000-1239.202330199, detailUrlEn=https://castjournals.cast.org.cn/joweb/jsjyjyfz/EN/10.7544/issn1000-1239.202330199, pdfUrlCn=https://castjournals.cast.org.cn/joweb/jsjyjyfz/CN/PDF/10.7544/issn1000-1239.202330199, pdfUrlEn=https://castjournals.cast.org.cn/joweb/jsjyjyfz/EN/PDF/10.7544/issn1000-1239.202330199, aliStartDate=0, aliEndDate=0, collectionFlag=false, citedCount=null, citedUrl=null, previewStatus=0, delFlag=0, hasFullText=1, orderTime=1738339200000, fullTextJson=null, articleText=null, reference=null)