Article(id=1329419929369030861, tenantId=1146029695717560320, journalId=1324759914146529478, issueId=1329419793389695520, articleNumber=null, orderNo=null, doi=10.7544/issn1000-1239.202550544, pmid=null, cstr=null, oa=null, hot=null, price=null, onlineType=0, articleFormat=0, articleType=null, articleTypeStr=null, receivedDate=1752940800000, receivedDateStr=2025-07-20, revisedDate=1765728000000, revisedDateStr=2025-12-15, acceptedDate=1767715200000, acceptedDateStr=2026-01-07, onlineDate=1794888010589, onlineDateStr=2026-11-17, pubDate=1780243200000, pubDateStr=2026-06-01, doiRegisterDate=null, doiRegisterDateStr=null, onlineIssueDate=1794888010589, onlineIssueDateStr=2026-11-17, onlineJustAcceptDate=null, onlineJustAcceptDateStr=null, onlineFirstDate=null, onlineFirstDateStr=null, sourceXml=null, magXml=null, createTime=1794888010589, creator=13701087609, updateTime=1794888010589, updator=13701087609, issue=Issue{id=1329419793389695520, tenantId=1146029695717560320, journalId=1324759914146529478, year='2026', volume='63', issue='6', pageStart='1377', pageEnd='1638', issueExtLink='null', onlineDate='null', pubDate='1780243200000', pubDateStr='2026-06-01', beforeIssueId=null, nextIssueId=null, price=null, status=1, issueComplete=1, articleOrder=1, issueType=-1, specialIssue=null, createTime=1794887978168, creator='13701087609', updateTime=1794892139295, updator='13701087609', preIssue=null, nextIssue=null, articleTotal=null, ext={EN=IssueExt(id=1329437246815580885, tenantId=1146029695717560320, journalId=1324759914146529478, issueId=1329419793389695520, language=EN, specialIssueTitle=, coverIllustrator=null, specialIssueEditor=, specialIssueAbout=), CN=IssueExt(id=1329437246815580886, tenantId=1146029695717560320, journalId=1324759914146529478, issueId=1329419793389695520, language=CN, specialIssueTitle=, coverIllustrator=null, specialIssueEditor=, specialIssueAbout=)}, issueFiles=null, downloadFileDto=null}, startPage=1478, endPage=1491, ext={EN=ArticleExt(id=1329419929566163151, articleId=1329419929369030861, tenantId=1146029695717560320, journalId=1324759914146529478, language=EN, title=Multimodal Emotion Recognition Method Based on Mixture of Fusion Experts, columnId=1325489891789910943, journalTitle=Journal of Computer Research and Development, columnName=Artificial Intelligence, runingTitle=null, highlight=null, articleAbstract=
Multimodal emotion recognition aims to integrate data from multiple modalities for accurate inference of emotional states. Existing research either treats all modalities equally or adopts fixed fusion strategies based on a single modality, failing to adequately address the imbalance of modality contributions. To tackle this, we propose a dynamic multimodal fusion method MoMFE based on the mixture of experts (MoE) framework, incorporating an adaptive router module and a multimodal fusion expert module. This method dynamically evaluates the contribution of each modality and selects appropriate fusion strategies accordingly. The adaptive router mechanism dynamically assesses modality contributions through inter-modal correlation analysis and dynamic weighting to guide the selection of fusion experts, while an expert-guided loss function is integrated to further optimize the expert selection process. The multimodal fusion experts perform complementary fusion for different modality contribution combinations and introduce a shared expert to mitigate the loss of global information and reduce parameter redundancy. Comparative experiments on three benchmark datasets (MER2024, CMU-MOSEI, and CH-SIMS) for multimodal emotion recognition demonstrate that MoMFE method outperforms state-of-the-art (SOTA) multimodal fusion methods in core metrics, including binary emotion recognition accuracy (Acc-2) and F1 score. Notably, the method achieves an average improvement of approximately 2% on CH-SIMS dataset.
, authors=Haifeng Chen
1, Xi Li
1, Yan Li
2, Jian Li
3, *, Lang He
4, Dongmei Jiang
5, authorsList=Haifeng Chen, Xi Li, Yan Li, Jian Li, Lang He, Dongmei Jiang, authorCompany=null, correspAuthors=Jian Li, authorNote=
born in 1988. PhD, lecturer. Member of CCF. His main research interests include affective computing, multimodal emotion recognition, emotion recognition in conversations, multimodal depression analysis, and facial action unit recognition
born in 2001. Master candidate. Student member of CCF. Her main research interest includes multimodal emotion recognition
born in 1998. Master candidate. His main research interests include machine learning and multimodal understanding
born in 1975. PhD, professor. His main research interests include computer vision and digital twin
born in 1983. PhD, associate professor. His main research interests include multimodal depression recognition and analysis, discrete emotion analysis, dimensional emotion recognition, and image processing
born in 1973. PhD, professor. Her main research interests include audio-visual fusion based speech processing, audio-visual emotion analysis and recognition, and facial animation synthesis of speakers with expressions
, correspAuthorsNote=null, copyrightStatement=Copyright © 2026 Journal of Computer Research and Development. All rights reserved., copyrightOwner=null, extLink=null, articleAbsUrl=null, sourceXml=null, magXml=null, pdfUrl=null, pdf=null, pdfFileSize=null, pdfExtLink=null, richHtmlUrl=null, mobilePdfUrl=null, reviewReport=null, pdfFirstPage=null, abstractGraph=null, abstractGraphContent=null, abstractVideo=null, citation=null, cebUrl=null, magXmlContent=null, mapNumber=null, fund=null), CN=ArticleExt(id=1329419931608789222, articleId=1329419929369030861, tenantId=1146029695717560320, journalId=1324759914146529478, language=CN, title=基于混合融合专家的多模态情感识别方法, columnId=1325489892104483745, journalTitle=计算机研究与发展, columnName=人工智能, runingTitle=null, highlight=null, articleAbstract=
多模态情感识别旨在整合不同模态数据以推断情感状态。现有研究通常平等对待各模态或依赖单一模态设计固定策略的融合方法,难以应对模态数据贡献度不均衡的问题。为此,提出一种动态的多模态混合融合方法(MoMFE),并基于混合专家模型框架设计了自适应路由模块和多模态融合专家模块,动态评估各模态的贡献度并选择相应的融合策略。其中,自适应路由机制通过模态间相关性分析动态评估模态贡献度,并为融合专家的选择提供指导,同时引入专家引导损失,进一步优化融合专家选择;多模态融合专家则针对不同模态贡献度组合进行互补融合,并通过引入共享专家以避免全局信息丢失和参数冗余。基于MER2024,CMU-MOSEI,CH-SIMS数据集上的多模态情感识别任务的对比实验表明,相比其他先进的多模态融合方法,MoMFE方法在情感识别准确率(Acc-2)、F1分数等核心评估指标上均取得优异的性能。
, authors=陈海丰
1, 李溪
1, 李岩
2, 李健
3, *, 何浪
4, 蒋冬梅
5, authorsList=陈海丰, 李溪, 李岩, 李健, 何浪, 蒋冬梅, authorCompany=null, correspAuthors=李健, authorNote=
作者贡献声明
陈海丰提出方法思路以及修改意见;李溪实现方法设计并撰写论文;李岩负责部分实验以及论文修改;李健负责实验监督以及论文修改;何浪负责项目管理以及论文修改;蒋冬梅负责项目监督以及论文修改。
 |
1988年生。博士,讲师。CCF会员。主要研究方向为情感计算、多模态情感识别、对话中的情感识别、多模态抑郁症分析、面部动作单元识别 |
 |
2001年生。硕士研究生。CCF学生会员。主要研究方向为多模态情感识别 |
 |
1998年生。硕士研究生。主要研究方向为机器学习、多模态理解 |
 |
1975年生。博士,教授。主要研究方向为计算机视觉、数字孪生 |
 |
1983年生。博士,副教授。主要研究方向为多模态抑郁症识别和分析、离散情感分析、维度情感识别、图像处理 |
 |
1973年生。博士,教授。主要研究方向为听视觉融合的语音处理、听视觉情感分析与识别、带表情的说话人面部动画合成 |
, correspAuthorsNote=
, copyrightStatement=版权所有 © 2026 计算机研究与发展编辑部, copyrightOwner=null, extLink=null, articleAbsUrl=null, sourceXml=4J0wv8sD9ne0T9Y0hbQDPg==, magXml=vlu7352g9oll082ZhvDdiQ==, pdfUrl=null, pdf=30ZAYQc1pf3aOAl5e9Luyw==, pdfFileSize=1319565, pdfExtLink=null, richHtmlUrl=null, mobilePdfUrl=null, reviewReport=null, pdfFirstPage=null, abstractGraph=rbp7quKlsEnVafS+IPjGFw==, abstractGraphContent=null, abstractVideo=null, citation=null, cebUrl=null, magXmlContent=J3ZmzYeQiovZm72t8BUCZw==, mapNumber=null, fund=null)}, authors=[Author(id=1329419932334403832, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1329419929369030861, orderNo=0, firstName=null, middleName=null, lastName=null, nameCn=null, orcid=null, stid=null, country=null, authorPic=null, dead=0, email=chenhaifeng@sust.edu.cn, emailSecond=null, emailThird=null, correspondingAuthor=0, authorType=1, ext={EN=AuthorExt(id=1329419932418289914, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1329419929369030861, authorId=1329419932334403832, language=EN, stringName=Haifeng Chen, firstName=Haifeng, middleName=null, lastName=Chen, prefix=null, suffix=null, authorComment=null, nameInitials=null, affiliation=null, department=null, xref=
1, address=
1School of Electronic Information and Artificial Intelligence, Shaanxi University of Science & Technology, Xi’an 710021, bio={"content":"
born in 1988. PhD, lecturer. Member of CCF. His main research interests include affective computing, multimodal emotion recognition, emotion recognition in conversations, multimodal depression analysis, and facial action unit recognition
"}, bioImg=null, bioContent=
born in 1988. PhD, lecturer. Member of CCF. His main research interests include affective computing, multimodal emotion recognition, emotion recognition in conversations, multimodal depression analysis, and facial action unit recognition
, aboutCorrespAuthor=null), CN=AuthorExt(id=1329419932489593083, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1329419929369030861, authorId=1329419932334403832, language=CN, stringName=陈海丰, firstName=null, middleName=null, lastName=null, prefix=null, suffix=null, authorComment=null, nameInitials=null, affiliation=null, department=null, xref=
1, address=
1陕西科技大学电子信息与人工智能学院 西安 710021, bio={"img":"O6v6TIhveb2oZj/xzAMCyA==","content":"
1988年生。博士,讲师。CCF会员。主要研究方向为情感计算、多模态情感识别、对话中的情感识别、多模态抑郁症分析、面部动作单元识别
"}, bioImg=O6v6TIhveb2oZj/xzAMCyA==, bioContent=
1988年生。博士,讲师。CCF会员。主要研究方向为情感计算、多模态情感识别、对话中的情感识别、多模态抑郁症分析、面部动作单元识别
, aboutCorrespAuthor=null)}, companyList=[AuthorCompany(id=1329419931822698727, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1329419929369030861, xref=1, ext=[AuthorCompanyExt(id=1329419931831087336, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1329419929369030861, companyId=1329419931822698727, language=EN, country=null, province=null, city=null, postcode=null, companyName=null, departmentName=null, remark=
1School of Electronic Information and Artificial Intelligence, Shaanxi University of Science & Technology, Xi’an 710021), AuthorCompanyExt(id=1329419931839475945, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1329419929369030861, companyId=1329419931822698727, language=CN, country=null, province=null, city=null, postcode=null, companyName=null, departmentName=null, remark=
1陕西科技大学电子信息与人工智能学院 西安 710021)])]), Author(id=1329419932548313341, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1329419929369030861, orderNo=1, firstName=null, middleName=null, lastName=null, nameCn=null, orcid=null, stid=null, country=null, authorPic=null, dead=0, email=null, emailSecond=null, emailThird=null, correspondingAuthor=0, authorType=1, ext={EN=AuthorExt(id=1329419932619616511, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1329419929369030861, authorId=1329419932548313341, language=EN, stringName=Xi Li, firstName=Xi, middleName=null, lastName=Li, prefix=null, suffix=null, authorComment=null, nameInitials=null, affiliation=null, department=null, xref=
1, address=
1School of Electronic Information and Artificial Intelligence, Shaanxi University of Science & Technology, Xi’an 710021, bio={"content":"
born in 2001. Master candidate. Student member of CCF. Her main research interest includes multimodal emotion recognition
"}, bioImg=null, bioContent=
born in 2001. Master candidate. Student member of CCF. Her main research interest includes multimodal emotion recognition
, aboutCorrespAuthor=null), CN=AuthorExt(id=1329419932686725376, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1329419929369030861, authorId=1329419932548313341, language=CN, stringName=李溪, firstName=null, middleName=null, lastName=null, prefix=null, suffix=null, authorComment=null, nameInitials=null, affiliation=null, department=null, xref=
1, address=
1陕西科技大学电子信息与人工智能学院 西安 710021, bio={"img":"m4VIMPi/oNkdn9KNShSmzA==","content":"
2001年生。硕士研究生。CCF学生会员。主要研究方向为多模态情感识别
"}, bioImg=m4VIMPi/oNkdn9KNShSmzA==, bioContent=
2001年生。硕士研究生。CCF学生会员。主要研究方向为多模态情感识别
, aboutCorrespAuthor=null)}, companyList=[AuthorCompany(id=1329419931822698727, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1329419929369030861, xref=1, ext=[AuthorCompanyExt(id=1329419931831087336, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1329419929369030861, companyId=1329419931822698727, language=EN, country=null, province=null, city=null, postcode=null, companyName=null, departmentName=null, remark=
1School of Electronic Information and Artificial Intelligence, Shaanxi University of Science & Technology, Xi’an 710021), AuthorCompanyExt(id=1329419931839475945, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1329419929369030861, companyId=1329419931822698727, language=CN, country=null, province=null, city=null, postcode=null, companyName=null, departmentName=null, remark=
1陕西科技大学电子信息与人工智能学院 西安 710021)])]), Author(id=1329419932774805762, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1329419929369030861, orderNo=2, firstName=null, middleName=null, lastName=null, nameCn=null, orcid=null, stid=null, country=null, authorPic=null, dead=0, email=null, emailSecond=null, emailThird=null, correspondingAuthor=0, authorType=1, ext={EN=AuthorExt(id=1329419932854497540, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1329419929369030861, authorId=1329419932774805762, language=EN, stringName=Yan Li, firstName=Yan, middleName=null, lastName=Li, prefix=null, suffix=null, authorComment=null, nameInitials=null, affiliation=null, department=null, xref=
2, address=
2School of Computer Science, Northwestern Polytechnical University, Xi’an 710072, bio={"content":"
born in 1998. Master candidate. His main research interests include machine learning and multimodal understanding
"}, bioImg=null, bioContent=
born in 1998. Master candidate. His main research interests include machine learning and multimodal understanding
, aboutCorrespAuthor=null), CN=AuthorExt(id=1329419932925800709, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1329419929369030861, authorId=1329419932774805762, language=CN, stringName=李岩, firstName=null, middleName=null, lastName=null, prefix=null, suffix=null, authorComment=null, nameInitials=null, affiliation=null, department=null, xref=
2, address=
2西北工业大学计算机学院 西安 710072, bio={"img":"Bwmsy3UZLO9e+qARuRfiwQ==","content":"
1998年生。硕士研究生。主要研究方向为机器学习、多模态理解
"}, bioImg=Bwmsy3UZLO9e+qARuRfiwQ==, bioContent=
1998年生。硕士研究生。主要研究方向为机器学习、多模态理解
, aboutCorrespAuthor=null)}, companyList=[AuthorCompany(id=1329419931923362026, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1329419929369030861, xref=2, ext=[AuthorCompanyExt(id=1329419931935944939, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1329419929369030861, companyId=1329419931923362026, language=EN, country=null, province=null, city=null, postcode=null, companyName=null, departmentName=null, remark=
2School of Computer Science, Northwestern Polytechnical University, Xi’an 710072), AuthorCompanyExt(id=1329419931940139244, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1329419929369030861, companyId=1329419931923362026, language=CN, country=null, province=null, city=null, postcode=null, companyName=null, departmentName=null, remark=
2西北工业大学计算机学院 西安 710072)])]), Author(id=1329419932997103879, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1329419929369030861, orderNo=3, firstName=null, middleName=null, lastName=null, nameCn=null, orcid=null, stid=null, country=null, authorPic=null, dead=0, email=lijianjsj@sust.edu.cn, emailSecond=null, emailThird=null, correspondingAuthor=1, authorType=1, ext={EN=AuthorExt(id=1329419933089378569, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1329419929369030861, authorId=1329419932997103879, language=EN, stringName=Jian Li, firstName=Jian, middleName=null, lastName=Li, prefix=null, suffix=null, authorComment=null, nameInitials=null, affiliation=null, department=null, xref=
3, *, address=
3School of Electrical and Control Engineering, Shaanxi University of Science & Technology, Xi’an 710021, bio={"content":"
born in 1975. PhD, professor. His main research interests include computer vision and digital twin
"}, bioImg=null, bioContent=
born in 1975. PhD, professor. His main research interests include computer vision and digital twin
, aboutCorrespAuthor=null), CN=AuthorExt(id=1329419933152293130, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1329419929369030861, authorId=1329419932997103879, language=CN, stringName=李健, firstName=null, middleName=null, lastName=null, prefix=null, suffix=null, authorComment=null, nameInitials=null, affiliation=null, department=null, xref=
3, *, address=
3陕西科技大学电气与控制工程学院 西安 710021, bio={"img":"oxZUsu9gMuKzRosUQspqeg==","content":"
1975年生。博士,教授。主要研究方向为计算机视觉、数字孪生
"}, bioImg=oxZUsu9gMuKzRosUQspqeg==, bioContent=
1975年生。博士,教授。主要研究方向为计算机视觉、数字孪生
, aboutCorrespAuthor=null)}, companyList=[AuthorCompany(id=1329419932091134190, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1329419929369030861, xref=3, ext=[AuthorCompanyExt(id=1329419932099522799, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1329419929369030861, companyId=1329419932091134190, language=EN, country=null, province=null, city=null, postcode=null, companyName=null, departmentName=null, remark=
3School of Electrical and Control Engineering, Shaanxi University of Science & Technology, Xi’an 710021), AuthorCompanyExt(id=1329419932112105712, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1329419929369030861, companyId=1329419932091134190, language=CN, country=null, province=null, city=null, postcode=null, companyName=null, departmentName=null, remark=
3陕西科技大学电气与控制工程学院 西安 710021)])]), Author(id=1329419933227790604, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1329419929369030861, orderNo=4, firstName=null, middleName=null, lastName=null, nameCn=null, orcid=null, stid=null, country=null, authorPic=null, dead=0, email=null, emailSecond=null, emailThird=null, correspondingAuthor=0, authorType=1, ext={EN=AuthorExt(id=1329419933328453902, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1329419929369030861, authorId=1329419933227790604, language=EN, stringName=Lang He, firstName=Lang, middleName=null, lastName=He, prefix=null, suffix=null, authorComment=null, nameInitials=null, affiliation=null, department=null, xref=
4, address=
4School of Computer Science & Technology, Xi’an University of Posts and Telecommunications, Xi’an 710061, bio={"content":"
born in 1983. PhD, associate professor. His main research interests include multimodal depression recognition and analysis, discrete emotion analysis, dimensional emotion recognition, and image processing
"}, bioImg=null, bioContent=
born in 1983. PhD, associate professor. His main research interests include multimodal depression recognition and analysis, discrete emotion analysis, dimensional emotion recognition, and image processing
, aboutCorrespAuthor=null), CN=AuthorExt(id=1329419933399757071, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1329419929369030861, authorId=1329419933227790604, language=CN, stringName=何浪, firstName=null, middleName=null, lastName=null, prefix=null, suffix=null, authorComment=null, nameInitials=null, affiliation=null, department=null, xref=
4, address=
4西安邮电大学计算机学院 西安 710061, bio={"img":"tYH1eHu03Lbb9ZCtJxiZCw==","content":"
1983年生。博士,副教授。主要研究方向为多模态抑郁症识别和分析、离散情感分析、维度情感识别、图像处理
"}, bioImg=tYH1eHu03Lbb9ZCtJxiZCw==, bioContent=
1983年生。博士,副教授。主要研究方向为多模态抑郁症识别和分析、离散情感分析、维度情感识别、图像处理
, aboutCorrespAuthor=null)}, companyList=[AuthorCompany(id=1329419932175020273, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1329419929369030861, xref=4, ext=[AuthorCompanyExt(id=1329419932191797490, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1329419929369030861, companyId=1329419932175020273, language=EN, country=null, province=null, city=null, postcode=null, companyName=null, departmentName=null, remark=
4School of Computer Science & Technology, Xi’an University of Posts and Telecommunications, Xi’an 710061), AuthorCompanyExt(id=1329419932200186099, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1329419929369030861, companyId=1329419932175020273, language=CN, country=null, province=null, city=null, postcode=null, companyName=null, departmentName=null, remark=
4西安邮电大学计算机学院 西安 710061)])]), Author(id=1329419933471060241, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1329419929369030861, orderNo=5, firstName=null, middleName=null, lastName=null, nameCn=null, orcid=null, stid=null, country=null, authorPic=null, dead=0, email=null, emailSecond=null, emailThird=null, correspondingAuthor=0, authorType=1, ext={EN=AuthorExt(id=1329419933596889363, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1329419929369030861, authorId=1329419933471060241, language=EN, stringName=Dongmei Jiang, firstName=Dongmei, middleName=null, lastName=Jiang, prefix=null, suffix=null, authorComment=null, nameInitials=null, affiliation=null, department=null, xref=
5, address=
5Pengcheng Laboratory, Shenzhen, Guangdong 518055, bio={"content":"
born in 1973. PhD, professor. Her main research interests include audio-visual fusion based speech processing, audio-visual emotion analysis and recognition, and facial animation synthesis of speakers with expressions
"}, bioImg=null, bioContent=
born in 1973. PhD, professor. Her main research interests include audio-visual fusion based speech processing, audio-visual emotion analysis and recognition, and facial animation synthesis of speakers with expressions
, aboutCorrespAuthor=null), CN=AuthorExt(id=1329419933668192532, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1329419929369030861, authorId=1329419933471060241, language=CN, stringName=蒋冬梅, firstName=null, middleName=null, lastName=null, prefix=null, suffix=null, authorComment=null, nameInitials=null, affiliation=null, department=null, xref=
5, address=
5鹏城实验室 广东深圳 518055, bio={"img":"EREeNgT5sQem+X5oaK3f2A==","content":"
1973年生。博士,教授。主要研究方向为听视觉融合的语音处理、听视觉情感分析与识别、带表情的说话人面部动画合成
"}, bioImg=EREeNgT5sQem+X5oaK3f2A==, bioContent=
1973年生。博士,教授。主要研究方向为听视觉融合的语音处理、听视觉情感分析与识别、带表情的说话人面部动画合成
, aboutCorrespAuthor=null)}, companyList=[AuthorCompany(id=1329419932263100660, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1329419929369030861, xref=5, ext=[AuthorCompanyExt(id=1329419932271489269, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1329419929369030861, companyId=1329419932263100660, language=EN, country=null, province=null, city=null, postcode=null, companyName=null, departmentName=null, remark=
5Pengcheng Laboratory, Shenzhen, Guangdong 518055), AuthorCompanyExt(id=1329419932279877878, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1329419929369030861, companyId=1329419932263100660, language=CN, country=null, province=null, city=null, postcode=null, companyName=null, departmentName=null, remark=
5鹏城实验室 广东深圳 518055)])])], keywords=[Keyword(id=1329419933764661525, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1329419929369030861, language=EN, orderNo=1, keyword=emotion recognition), Keyword(id=1329419933840158998, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1329419929369030861, language=EN, orderNo=2, keyword=multimodal fusion), Keyword(id=1329419933903073559, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1329419929369030861, language=EN, orderNo=3, keyword=mixture of experts), Keyword(id=1329419933970182424, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1329419929369030861, language=EN, orderNo=4, keyword=adaptive router), Keyword(id=1329419934058262809, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1329419929369030861, language=EN, orderNo=5, keyword=cross-modal attention), Keyword(id=1329419934146343194, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1329419929369030861, language=CN, orderNo=1, keyword=情感识别), Keyword(id=1329419934272172315, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1329419929369030861, language=CN, orderNo=2, keyword=多模态融合), Keyword(id=1329419934360252700, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1329419929369030861, language=CN, orderNo=3, keyword=混合专家), Keyword(id=1329419934423167261, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1329419929369030861, language=CN, orderNo=4, keyword=自适应路由), Keyword(id=1329419934536413470, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1329419929369030861, language=CN, orderNo=5, keyword=跨模态注意力)], refs=[Reference(id=1329419936662925623, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1329419929369030861, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=1, rfOrder=0, authorNames=null, journalName=null, refType=null, unstructuredReference=Das R, Singh T D. Multimodal sentiment analysis: A survey of methods, trends, and challenges[J]. ACM Computing Surveys, 2023, 55(13): 1−38, articleTitle=null, refAbstract=null), Reference(id=1329419936734228792, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1329419929369030861, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=2, rfOrder=1, authorNames=null, journalName=null, refType=null, unstructuredReference=Poria S, Hazarika D, Majumder N, et al. Beneath the tip of the iceberg: Current challenges and new directions in sentiment analysis research[J]. IEEE Transactions on Affective Computing, 2020, 14(1): 108−132, articleTitle=null, refAbstract=null), Reference(id=1329419936792949049, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1329419929369030861, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=3, rfOrder=2, authorNames=null, journalName=null, refType=null, unstructuredReference=Islam M S, Kabir M N, Ghani N A, et al. Challenges and future in deep learning for sentiment analysis: A comprehensive review and a proposed novel hybrid approach[J]. Artificial Intelligence Review, 2024, 57(62): 1−79, articleTitle=null, refAbstract=null), Reference(id=1329419936864252218, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1329419929369030861, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=4, rfOrder=3, authorNames=null, journalName=null, refType=null, unstructuredReference=Huang Jian, Ji Yanli, Qin Zhen, et al. Dominant single-modal supplementary fusion (SIMSUF) for multimodal sentiment analysis[J]. IEEE Transactions on Multimedia, 2023, 26: 8383−8394, articleTitle=null, refAbstract=null), Reference(id=1329419936927166779, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1329419929369030861, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=5, rfOrder=4, authorNames=null, journalName=null, refType=null, unstructuredReference=Lian Zhen, Sun Haiyang, Sun Licai, et al. MER2024: Semi-supervised learning, noise robustness, and open-vocabulary multimodal emotion recognition[C]//Proc of the 2nd Int Workshop on Multimodal and Responsible Affective Computing. New York: ACM, 2024: 41−48, articleTitle=null, refAbstract=null), Reference(id=1329419936985887036, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1329419929369030861, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=6, rfOrder=5, authorNames=null, journalName=null, refType=null, unstructuredReference=罗渊贻, 吴锐, 刘家锋, 等. 面向情感语义不一致的多模态情感分析方法[J]. 计算机研究与发展, 2025, 62(2): 374−382, articleTitle=null, refAbstract=null), Reference(id=1329419937052995901, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1329419929369030861, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=6, rfOrder=6, authorNames=null, journalName=null, refType=null, unstructuredReference=Luo Yuanyi, Wu Rui, Liu Jiafeng, et al. Multimodal sentiment analysis method for sentimental semantic inconsistency[J]. Journal of Computer Research and Development, 2025, 62(2): 374−382 (in Chinese), articleTitle=null, refAbstract=null), Reference(id=1329419937128493374, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1329419929369030861, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=7, rfOrder=7, authorNames=null, journalName=null, refType=null, unstructuredReference=Liang Yang. A dynamic weighted fusion model for multimodal sentiment analysis[J]. Signal, Image and Video Processing, 2025, 19(8): 609−623, articleTitle=null, refAbstract=null), Reference(id=1329419937191407935, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1329419929369030861, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=8, rfOrder=8, authorNames=null, journalName=null, refType=null, unstructuredReference=Feng Xinyu, Lin Yuming, He Lihua, et al. Knowledge-Guided dynamic modality attention fusion framework for multimodal sentiment analysis[J]. arXiv preprint, arXiv: 2410.04491, 2024, articleTitle=null, refAbstract=null), Reference(id=1329419937275294016, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1329419929369030861, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=9, rfOrder=9, authorNames=null, journalName=null, refType=null, unstructuredReference=申锦涛. 基于模态交互和动态融合的多模态情感识别研究[D]. 沈阳: 辽宁大学, 2024, articleTitle=null, refAbstract=null), Reference(id=1329419937354985793, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1329419929369030861, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=9, rfOrder=10, authorNames=null, journalName=null, refType=null, unstructuredReference=Shen Jintao. Research on multimodal emotion recognition based on modal interaction and dynamic fusion[D]. Shenyang: Liaoning University, 2024 (in Chinese), articleTitle=null, refAbstract=null), Reference(id=1329419937434677570, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1329419929369030861, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=10, rfOrder=11, authorNames=null, journalName=null, refType=null, unstructuredReference=Zhuang Yan, Zhang Yanru, Hu Zheng, et al. GLoMo: Global-local modal fusion for multimodal sentiment analysis[C]//Proc of the 32nd ACM Int Conf on Multimedia. New York: ACM, 2024: 1800−1809, articleTitle=null, refAbstract=null), Reference(id=1329419937501786435, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1329419929369030861, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=11, rfOrder=12, authorNames=null, journalName=null, refType=null, unstructuredReference=Zhao Xueliang, Wang Mingyang, Tan Yingchun, et al. TGMoE: A text guided mixture-of-experts model for multimodal sentiment analysis[J]. International Journal of Advanced Computer Science & Applications, 2024, 15(8): 1227−1234, articleTitle=null, refAbstract=null), Reference(id=1329419937585672516, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1329419929369030861, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=12, rfOrder=13, authorNames=null, journalName=null, refType=null, unstructuredReference=贾熹滨, 曾檬, 米庆, 等. 领域对齐对抗的无监督跨领域文本情感分析算法[J]. 计算机研究与发展, 2022, 59(6): 1225−1270, articleTitle=null, refAbstract=null), Reference(id=1329419937652781381, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1329419929369030861, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=12, rfOrder=14, authorNames=null, journalName=null, refType=null, unstructuredReference=Jia Xibin, Zeng Meng, Mi Qing, et al. Domain alignment adversarial unsupervised cross-domain text sentiment analysis algorithm[J]. Journal of Computer Research and Development, 2022, 59(6): 1225−1270 (in Chinese), articleTitle=null, refAbstract=null), Reference(id=1329419937753444678, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1329419929369030861, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=13, rfOrder=15, authorNames=null, journalName=null, refType=null, unstructuredReference=Chen Haifeng, Zhang Peng, Guo Chujia, et al. Facial action unit representation based on self-supervised learning with ensembled priori constraints[J]. IEEE Transactions on Image Processing, 2024, 33: 5045−5069, articleTitle=null, refAbstract=null), Reference(id=1329419937828942151, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1329419929369030861, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=14, rfOrder=16, authorNames=null, journalName=null, refType=null, unstructuredReference=Wang Yuhua, Shen Guang, Xu Yuezhu, et al. Learning mutual correlation in multimodal transformer for speech emotion recognition[C/OL]//Proc of the 2021 Interspeech. 2021: 4518−4522.[2025-04-16]. https://www.isca-archive.org/interspeech_2021/wang21ga_interspeech.html, articleTitle=null, refAbstract=null), Reference(id=1329419937891856712, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1329419929369030861, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=15, rfOrder=17, authorNames=null, journalName=null, refType=null, unstructuredReference=Pennington J, Socher R, Manning C, et al. GloVe: Global vectors for word representation[C]//Proc of the 2014 Conf on Empirical Methods in Natural Language Processing. Stroudsburg, PA: ACL, 2014: 1532−1543, articleTitle=null, refAbstract=null), Reference(id=1329419938055434569, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1329419929369030861, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=16, rfOrder=18, authorNames=null, journalName=null, refType=null, unstructuredReference=Devlin J, Chang Mingwei, Lee K, et al. BERT: Pre-training of deep bidirectional transformers for language understanding[C]//Proc of the 2019 Conf of the North American Chapter of the Association for Computational Linguistics: Human Language Technologies. Stroudsburg, PA: ACL, 2019: 4171−4186, articleTitle=null, refAbstract=null), Reference(id=1329419938130932042, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1329419929369030861, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=17, rfOrder=19, authorNames=null, journalName=null, refType=null, unstructuredReference=Dosovitskiy A, Beyer L, Kolesnikov A, et al. An image is worth 16x16 words: Transformers for image recognition at scale[J]. arXiv preprint, arXiv: 2010.11929, 2020, articleTitle=null, refAbstract=null), Reference(id=1329419938210623819, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1329419929369030861, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=18, rfOrder=20, authorNames=null, journalName=null, refType=null, unstructuredReference=Hsu W N, Bolte B, Tsai Y H, et al. Hubert: Self-supervised speech representation learning by masked prediction of hidden units[J]. IEEE/ACM Transactions on Audio, Speech, and Language Processing, 2021, 29: 3451−3460, articleTitle=null, refAbstract=null), Reference(id=1329419938286121292, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1329419929369030861, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=19, rfOrder=21, authorNames=null, journalName=null, refType=null, unstructuredReference=Radford A, Kim J W, Xu Tao, et al. Robust speech recognition via large-scale weak supervision[C]//Proc of the 38th Int Conf on Machine Learning. New York: ACM, 2023: 28492−28518, articleTitle=null, refAbstract=null), Reference(id=1329419938340647245, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1329419929369030861, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=20, rfOrder=22, authorNames=null, journalName=null, refType=null, unstructuredReference=Wei Han, Chen Hui, Poria S. Improving multimodal fusion with hierarchical mutual information maximization for multimodal sentiment analysis[C]//Proc of the 2021 Conf on Empirical Methods in Natural Language Processing. Stroudsburg, PA: ACL, 2021: 9180–9192, articleTitle=null, refAbstract=null), Reference(id=1329419938403561806, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1329419929369030861, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=21, rfOrder=23, authorNames=null, journalName=null, refType=null, unstructuredReference=Cai Yunrui, Wu Zhiyong, Jia Jia, et al. Lora-Mer: Low-rank adaptation of pre-trained speech models for multimodal emotion recognition using mutual information[C/OL]//Proc of the 2024 Interspeech. 2024: 4658−4662.[2025-04-15]. https://www.isca-archive.org/interspeech_2024/cai24b_interspeech.html, articleTitle=null, refAbstract=null), Reference(id=1329419938462282063, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1329419929369030861, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=22, rfOrder=24, authorNames=null, journalName=null, refType=null, unstructuredReference=Kumar A, Vepa J. Gated mechanism for attention based multi modal sentiment analysis[C]//Proc of the 2020 IEEE Int Conf on Acoustics, Speech and Signal Processing. Piscataway, NJ: IEEE, 2020: 4477−4481, articleTitle=null, refAbstract=null), Reference(id=1329419938546168144, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1329419929369030861, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=23, rfOrder=25, authorNames=null, journalName=null, refType=null, unstructuredReference=Majumder N, Hazarika D, Gelbukh A, et al. Multimodal sentiment analysis using hierarchical fusion with context modeling[J]. Knowledge-Based Systems, 2018, 161: 124−133, articleTitle=null, refAbstract=null), Reference(id=1329419938613277009, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1329419929369030861, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=24, rfOrder=26, authorNames=null, journalName=null, refType=null, unstructuredReference=Jacobs R A, Jordan M I, Nowlan S J, et al. Adaptive mixtures of local experts[J]. Neural Computation, 1991, 3(1): 79−87, articleTitle=null, refAbstract=null), Reference(id=1329419938734911826, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1329419929369030861, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=25, rfOrder=27, authorNames=null, journalName=null, refType=null, unstructuredReference=Fedus W, Zoph B, Shazeer N. Switch Transformers: Scaling to trillion parameter models with simple and efficient sparsity[J]. Journal of Machine Learning Research, 2022, 23(120): 1−39, articleTitle=null, refAbstract=null), Reference(id=1329419938810409299, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1329419929369030861, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=26, rfOrder=28, authorNames=null, journalName=null, refType=null, unstructuredReference=Vaswani A, Shazeer N, Parmar N, et al. Attention is all you need[C]//Proc of the 31st Int Conf on Neural Information Processing Systems. New York: ACM, 2017: 6000−6010, articleTitle=null, refAbstract=null), Reference(id=1329419938911072596, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1329419929369030861, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=27, rfOrder=29, authorNames=null, journalName=null, refType=null, unstructuredReference=Lepikhin D, Lee H J, Xu Yuanzhong, et al. Gshard: Scaling giant models with conditional computation and automatic sharding[C/OL]//Proc of the 9th Int Conf on Learning Representations. 2021: 1−23.[2025-04-23]. https://openreview.net/forum?id=qrwe7XHTmYb, articleTitle=null, refAbstract=null), Reference(id=1329419938982375765, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1329419929369030861, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=28, rfOrder=30, authorNames=null, journalName=null, refType=null, unstructuredReference=Du Nan, Huang Yanping, Dai A M, et al. GlaM: Efficient scaling of language models with mixture-of-experts[C]//Proc of the 39th Int Conf on Machine Learning. New York: ACM, 2022: 5547−5569, articleTitle=null, refAbstract=null), Reference(id=1329419939045290326, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1329419929369030861, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=29, rfOrder=31, authorNames=null, journalName=null, refType=null, unstructuredReference=Cai Weilin, Jiang Juyong, Wang Fan, et al. A survey on mixture of experts in large language models[J]. IEEE Transactions on Knowledge and Data Engineering, 2025, 37(7): 3896−3915, articleTitle=null, refAbstract=null), Reference(id=1329419939095621975, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1329419929369030861, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=30, rfOrder=32, authorNames=null, journalName=null, refType=null, unstructuredReference=Sarkar R, Liang Hanxue, Fan Zhiwen, et al. Edge-MoE: Memory-efficient multi-task vision transformer architecture with task-level sparsity via mixture-of-experts[C]//Proc of the 2023 IEEE/ACM Int Conf on Computer Aided Design. Piscataway, NJ: IEEE, 2023: 1−9, articleTitle=null, refAbstract=null), Reference(id=1329419939166925144, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1329419929369030861, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=31, rfOrder=33, authorNames=null, journalName=null, refType=null, unstructuredReference=Lin X V, Shrivastava A, Liang Luo, et al. Moma: Efficient early-fusion pre-training with mixture of modality-aware experts[J]. arXiv preprint, arXiv: 2407.21770, 2024, articleTitle=null, refAbstract=null), Reference(id=1329419939234034009, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1329419929369030861, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=32, rfOrder=34, authorNames=null, journalName=null, refType=null, unstructuredReference=Han Xing, Nguyen H, Harris C, et al. FuseMoE: Mixture-of-experts transformers for fleximodal fusion[C]//Proc of the 38th Int Conf on Neural Information Processing Systems, New York: ACM, 2025: 67850–67900, articleTitle=null, refAbstract=null), Reference(id=1329419939301142874, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1329419929369030861, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=33, rfOrder=35, authorNames=null, journalName=null, refType=null, unstructuredReference=Xiong Congxiong, Chen Hao, Zheng Hao, et al. Mome: Mixture of multimodal experts for cancer survival prediction[C]//Proc of the 27th Int Conf on Medical Image Computing and Computer-Assisted Intervention. Springer: Berlin, 2024: 318−328, articleTitle=null, refAbstract=null), Reference(id=1329419939364057435, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1329419929369030861, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=34, rfOrder=36, authorNames=null, journalName=null, refType=null, unstructuredReference=Shazeer N, Mirhoseini A, Maziarz K, et al. Outrageously large neural networks: The sparsely-gated mixture-of-experts layer[J]. arXiv preprint, arXiv: 1701.06538, 2017, articleTitle=null, refAbstract=null), Reference(id=1329419939422777692, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1329419929369030861, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=35, rfOrder=37, authorNames=null, journalName=null, refType=null, unstructuredReference=Dai Damai, Deng Chengqi, Zhao Chenggang, et al. Deepseek-Moe: Towards ultimate expert specialization in mixture-of-experts language models[C]//Proc of the 62nd Annual Meeting of the Association for Computational Linguistics. Stroudsburg, PA: ACL, 2024: 1290−1297, articleTitle=null, refAbstract=null), Reference(id=1329419939481497949, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1329419929369030861, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=36, rfOrder=38, authorNames=null, journalName=null, refType=null, unstructuredReference=Zadeh A B, Liang P P, Poria S, et al. Multimodal language analysis in the wild: CMU-MOSEI dataset and interpretable dynamic fusion graph[C]//Proc of the 56th Annual Meeting of the Association for Computational Linguistics. Stroudsburg, PA: ACL, 2018: 2236−2246, articleTitle=null, refAbstract=null), Reference(id=1329419939544412510, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1329419929369030861, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=37, rfOrder=39, authorNames=null, journalName=null, refType=null, unstructuredReference=Yu Wenmeng, Xu Hua, Meng Fanyang, et al. CH-SIMS: A Chinese multimodal sentiment analysis dataset with fine-grained annotation of modality[C]//Proc of the 58th Annual Meeting of the Association for Computational Linguistics. Stroudsburg, PA: ACL, 2020: 3718−3727, articleTitle=null, refAbstract=null), Reference(id=1329419939607327071, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1329419929369030861, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=38, rfOrder=40, authorNames=null, journalName=null, refType=null, unstructuredReference=Baltrusaitis T, Zadeh A, Lim Y C, et al. OpenFace 2.0: Facial behavior analysis toolkit[C]//Proc of the 13th IEEE Int Conf on Automatic Face & Gesture Recognition. Piscataway, NJ: IEEE, 2018: 59−66, articleTitle=null, refAbstract=null), Reference(id=1329419939674435936, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1329419929369030861, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=39, rfOrder=41, authorNames=null, journalName=null, refType=null, unstructuredReference=Radford A, Kim J W, Hallacy C, et al. Learning transferable visual models from natural language supervision[C]//Proc of the 38th Int Conf on Machine Learning. New York: ACM, 2021: 8748−8763, articleTitle=null, refAbstract=null), Reference(id=1329419939728961889, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1329419929369030861, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=40, rfOrder=42, authorNames=null, journalName=null, refType=null, unstructuredReference=Zhao Zhixian, Chen Haifeng, Li Xi, et al. Improving multimodal emotion recognition by leveraging acoustic adaptation and visual alignment[C]//Proc of the 2nd Int Workshop on Multimodal and Responsible Affective Computing. New York: ACM, 2024: 67−71, articleTitle=null, refAbstract=null), Reference(id=1329419939787682146, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1329419929369030861, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=41, rfOrder=43, authorNames=null, journalName=null, refType=null, unstructuredReference=Yang Aiyuan, Xiao Bin, Wang Bingning, et al. Baichuan 2: Open large-scale language models[J]. arXiv preprint, arXiv: 2309.10305, 2023, articleTitle=null, refAbstract=null), Reference(id=1329419939867373923, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1329419929369030861, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=42, rfOrder=44, authorNames=null, journalName=null, refType=null, unstructuredReference=Li Tianyi, Liu Daming. MPID: A modality-preserving and interaction-driven fusion network for multimodal sentiment analysis[C]//Proc of the 31st Int Conf on Computational Linguistics. Stroudsburg, PA: ACL, 2025: 4313−4322, articleTitle=null, refAbstract=null), Reference(id=1329419939926094180, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1329419929369030861, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=43, rfOrder=45, authorNames=null, journalName=null, refType=null, unstructuredReference=Xin Feng, Gong Wenjuan, Shi Tingbo, et al. Gated fusion network with progressive modality enhancement for sentiment analysis[J]. IEEE Access, 2024, 12: 165810−165821, articleTitle=null, refAbstract=null), Reference(id=1329419939984814437, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1329419929369030861, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=44, rfOrder=46, authorNames=null, journalName=null, refType=null, unstructuredReference=Fan Qi, Li Yong, Xin Yi, et al. Leveraging contrastive learning and self-training for multimodal emotion recognition with limited labeled samples[C]//Proc of the 2nd Int Workshop on Multimodal and Responsible Affective Computing. New York: ACM, 2024: 72−77, articleTitle=null, refAbstract=null), Reference(id=1329419940110643558, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1329419929369030861, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=45, rfOrder=47, authorNames=null, journalName=null, refType=null, unstructuredReference=Gao Fei, Shi Pujin. Audio-Guided fusion techniques for multimodal emotion analysis[C]//Proc of the 2nd Int Workshop on Multimodal and Responsible Affective Computing. New York: ACM, 2024: 62−66, articleTitle=null, refAbstract=null), Reference(id=1329419940194529639, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1329419929369030861, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=46, rfOrder=48, authorNames=null, journalName=null, refType=null, unstructuredReference=Lei Ling, He Wangjun, Zheng Qiuyan, et al. Multimodal sentiment analysis based on expert mixing of subtask representations[J]. IEEE Access, 2025, 13: 44278−44287, articleTitle=null, refAbstract=null), Reference(id=1329419940295192936, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1329419929369030861, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=47, rfOrder=49, authorNames=null, journalName=null, refType=null, unstructuredReference=Liu Yihe, Yuan Ziqi, Mao Huisheng, et al. Make acoustic and visual cues matter: CH-SIMS v2. 0 dataset and av-mixup consistent module[C]//Proc of the 24th Int Conf on Multimodal Interaction. New York: ACM, 2022: 247−258, articleTitle=null, refAbstract=null)], funds=null, companyList=[AuthorCompany(id=1329419931822698727, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1329419929369030861, xref=1, ext=[AuthorCompanyExt(id=1329419931831087336, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1329419929369030861, companyId=1329419931822698727, language=EN, country=null, province=null, city=null, postcode=null, companyName=null, departmentName=null, remark=
1School of Electronic Information and Artificial Intelligence, Shaanxi University of Science & Technology, Xi’an 710021), AuthorCompanyExt(id=1329419931839475945, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1329419929369030861, companyId=1329419931822698727, language=CN, country=null, province=null, city=null, postcode=null, companyName=null, departmentName=null, remark=
1陕西科技大学电子信息与人工智能学院 西安 710021)]), AuthorCompany(id=1329419931923362026, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1329419929369030861, xref=2, ext=[AuthorCompanyExt(id=1329419931935944939, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1329419929369030861, companyId=1329419931923362026, language=EN, country=null, province=null, city=null, postcode=null, companyName=null, departmentName=null, remark=
2School of Computer Science, Northwestern Polytechnical University, Xi’an 710072), AuthorCompanyExt(id=1329419931940139244, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1329419929369030861, companyId=1329419931923362026, language=CN, country=null, province=null, city=null, postcode=null, companyName=null, departmentName=null, remark=
2西北工业大学计算机学院 西安 710072)]), AuthorCompany(id=1329419932091134190, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1329419929369030861, xref=3, ext=[AuthorCompanyExt(id=1329419932099522799, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1329419929369030861, companyId=1329419932091134190, language=EN, country=null, province=null, city=null, postcode=null, companyName=null, departmentName=null, remark=
3School of Electrical and Control Engineering, Shaanxi University of Science & Technology, Xi’an 710021), AuthorCompanyExt(id=1329419932112105712, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1329419929369030861, companyId=1329419932091134190, language=CN, country=null, province=null, city=null, postcode=null, companyName=null, departmentName=null, remark=
3陕西科技大学电气与控制工程学院 西安 710021)]), AuthorCompany(id=1329419932175020273, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1329419929369030861, xref=4, ext=[AuthorCompanyExt(id=1329419932191797490, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1329419929369030861, companyId=1329419932175020273, language=EN, country=null, province=null, city=null, postcode=null, companyName=null, departmentName=null, remark=
4School of Computer Science & Technology, Xi’an University of Posts and Telecommunications, Xi’an 710061), AuthorCompanyExt(id=1329419932200186099, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1329419929369030861, companyId=1329419932175020273, language=CN, country=null, province=null, city=null, postcode=null, companyName=null, departmentName=null, remark=
4西安邮电大学计算机学院 西安 710061)]), AuthorCompany(id=1329419932263100660, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1329419929369030861, xref=5, ext=[AuthorCompanyExt(id=1329419932271489269, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1329419929369030861, companyId=1329419932263100660, language=EN, country=null, province=null, city=null, postcode=null, companyName=null, departmentName=null, remark=
5Pengcheng Laboratory, Shenzhen, Guangdong 518055), AuthorCompanyExt(id=1329419932279877878, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1329419929369030861, companyId=1329419932263100660, language=CN, country=null, province=null, city=null, postcode=null, companyName=null, departmentName=null, remark=
5鹏城实验室 广东深圳 518055)])], figs=[ArticleFig(id=1329419934632882463, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1329419929369030861, language=EN, label=Figure 1, caption=
Classification statistics of CH-SIMS dataset samples and classification examples, figureFileSmall=CiT4ecHDdGb36rKzQJ1sPg==, figureFileBig=rbp7quKlsEnVafS+IPjGFw==, tableContent=null), ArticleFig(id=1329419934720962848, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1329419929369030861, language=CN, label=图1, caption=
CH-SIMS数据集样本分类统计以及分类示例, figureFileSmall=CiT4ecHDdGb36rKzQJ1sPg==, figureFileBig=rbp7quKlsEnVafS+IPjGFw==, tableContent=null), ArticleFig(id=1329419934934872353, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1329419929369030861, language=EN, label=Figure 2, caption=
The method model diagram, figureFileSmall=svCd1Is6a88YAAzR9NdfWw==, figureFileBig=EfgMy270oRC1s008SZintQ==, tableContent=null), ArticleFig(id=1329419935010369826, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1329419929369030861, language=CN, label=图2, caption=
方法模型图, figureFileSmall=svCd1Is6a88YAAzR9NdfWw==, figureFileBig=EfgMy270oRC1s008SZintQ==, tableContent=null), ArticleFig(id=1329419935090061603, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1329419929369030861, language=EN, label=Figure 3, caption=
Expert activation rates under different router settings on CH-SIMS dataset, figureFileSmall=KID1z6hqes97q531fTGWrw==, figureFileBig=pzB8RagRTl/bLQbBj/V3Mw==, tableContent=null), ArticleFig(id=1329419935178141988, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1329419929369030861, language=CN, label=图3, caption=
CH-SIMS数据集上不同路由设置下的专家激活率, figureFileSmall=KID1z6hqes97q531fTGWrw==, figureFileBig=pzB8RagRTl/bLQbBj/V3Mw==, tableContent=null), ArticleFig(id=1329419935262028069, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1329419929369030861, language=EN, label=Figure 4, caption=
Performance of each fusion expert on CH-SIMS classification dataset, figureFileSmall=wrd8ElTWRVhOoWOFGI3EFQ==, figureFileBig=YZzyaFDkZhOTA/47rwzC/A==, tableContent=null), ArticleFig(id=1329419935350108454, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1329419929369030861, language=CN, label=图4, caption=
CH-SIMS分类数据集上各融合专家的表现, figureFileSmall=wrd8ElTWRVhOoWOFGI3EFQ==, figureFileBig=YZzyaFDkZhOTA/47rwzC/A==, tableContent=null), ArticleFig(id=1329419935429800231, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1329419929369030861, language=EN, label=Figure 5, caption=
Assessment results of d and α for MoMFE on CH-SIMS dataset, figureFileSmall=tDq6GT2UvFPuK6LltEif4g==, figureFileBig=gPY1b1P/+QZCICHVAiMWTA==, tableContent=null), ArticleFig(id=1329419935505297704, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1329419929369030861, language=CN, label=图5, caption=
CH-SIMS数据集上d与α对MoMFE的评估结果, figureFileSmall=tDq6GT2UvFPuK6LltEif4g==, figureFileBig=gPY1b1P/+QZCICHVAiMWTA==, tableContent=null), ArticleFig(id=1329419935580795177, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1329419929369030861, language=EN, label=Table 1, caption=
Introduction of Datasets
, figureFileSmall=null, figureFileBig=null, tableContent=
| 数据集 | 训练集 | 验证集 | 测试集 | 语言 |
|---|
| MER2024 | 5030 | | 1169 | 中文 |
| CMU-MOSEI | 16326 | 1871 | 4659 | 英文 |
| CH-SIMS | 1368 | 456 | 457 | 中文 |
), ArticleFig(id=1329419935635321130, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1329419929369030861, language=CN, label=表1, caption=
数据集介绍
, figureFileSmall=null, figureFileBig=null, tableContent=
| 数据集 | 训练集 | 验证集 | 测试集 | 语言 |
|---|
| MER2024 | 5030 | | 1169 | 中文 |
| CMU-MOSEI | 16326 | 1871 | 4659 | 英文 |
| CH-SIMS | 1368 | 456 | 457 | 中文 |
), ArticleFig(id=1329419935702429995, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1329419929369030861, language=EN, label=Table 2, caption=
Results of Different Methods on MER2024 Dataset %
, figureFileSmall=null, figureFileBig=null, tableContent=
| 方法 | 训练及验证 F1 | 测试 F1 | Acc-6 |
|---|
| MR-CCL | 86.53 | 88.25 | |
| LAA | 83.85 | 88.90 | |
| AGT | 83.22 | 89.83 | |
| MoMFE(本文) | 83.28 | 89.88 | 90.01 |
), ArticleFig(id=1329419935790510380, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1329419929369030861, language=CN, label=表2, caption=
不同方法在MER2024数据集上的结果
, figureFileSmall=null, figureFileBig=null, tableContent=
| 方法 | 训练及验证 F1 | 测试 F1 | Acc-6 |
|---|
| MR-CCL | 86.53 | 88.25 | |
| LAA | 83.85 | 88.90 | |
| AGT | 83.22 | 89.83 | |
| MoMFE(本文) | 83.28 | 89.88 | 90.01 |
), ArticleFig(id=1329419935882785069, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1329419929369030861, language=EN, label=Table 3, caption=
Results of Different Methods on CMU-MOSEI and CH-SIMS Datasets %
, figureFileSmall=null, figureFileBig=null, tableContent=
| 方法 | CMU-MOSEI数据集 | | CH-SIMS数据集 |
|---|
| Mae | Corr | Acc-7 | Acc-2 | F1 | | Mae | Corr | Acc-5 | Acc-3 | Acc-2 | F1 |
|---|
| MPID | 53.3 | 77.6 | 53.64 | 85.10 | 85.30 | | 42.1 | 61.8 | 44.21 | | 81.56 | 81.91 |
| GFNPME | 52.8 | 77.1 | 53.90 | 86.52 | 86.26 | | 40.2 | 61.6 | | | 83.15 | 83.10 |
| SIMSUF | 52.9 | 77.2 | 53.68 | 86.23 | 86.12 | | | | | | | |
| KuDA | 52.9 | 77.6 | 52.89 | 86.46 | 86.59 | | 40.8 | 61.3 | 43.54 | 66.52 | 80.74 | 80.71 |
| DWE | 52.6 | 76.8 | 54.80 | 85.97 | 85.98 | | 41.6 | 59.3 | 42.90 | | 80.74 | 80.54 |
| AC-MoE | | | | | | | 31.5 | 73.2 | | | 82.05 | 82.04 |
| TGMoE | 53.5 | 75.7 | 53.70 | 85.51 | 85.86 | | | | | | | |
| FuseMoE | 54.0 | 81.0 | | 86.41 | 85.43 | | | | | | | |
| GLoMo | 53.9 | 77.1 | 55.00 | 86.50 | 86.40 | | | | | | | |
| MoMFE(本文) | 53.4 | 78.8 | 54.72 | 86.83 | 86.69 | | 37.7 | 75.1 | 41.38 | 71.99 | 84.33 | 84.51 |
), ArticleFig(id=1329419935945699630, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1329419929369030861, language=CN, label=表3, caption=
不同方法在CMU-MOSEI和CH-SIMS数据集上的结果
, figureFileSmall=null, figureFileBig=null, tableContent=
| 方法 | CMU-MOSEI数据集 | | CH-SIMS数据集 |
|---|
| Mae | Corr | Acc-7 | Acc-2 | F1 | | Mae | Corr | Acc-5 | Acc-3 | Acc-2 | F1 |
|---|
| MPID | 53.3 | 77.6 | 53.64 | 85.10 | 85.30 | | 42.1 | 61.8 | 44.21 | | 81.56 | 81.91 |
| GFNPME | 52.8 | 77.1 | 53.90 | 86.52 | 86.26 | | 40.2 | 61.6 | | | 83.15 | 83.10 |
| SIMSUF | 52.9 | 77.2 | 53.68 | 86.23 | 86.12 | | | | | | | |
| KuDA | 52.9 | 77.6 | 52.89 | 86.46 | 86.59 | | 40.8 | 61.3 | 43.54 | 66.52 | 80.74 | 80.71 |
| DWE | 52.6 | 76.8 | 54.80 | 85.97 | 85.98 | | 41.6 | 59.3 | 42.90 | | 80.74 | 80.54 |
| AC-MoE | | | | | | | 31.5 | 73.2 | | | 82.05 | 82.04 |
| TGMoE | 53.5 | 75.7 | 53.70 | 85.51 | 85.86 | | | | | | | |
| FuseMoE | 54.0 | 81.0 | | 86.41 | 85.43 | | | | | | | |
| GLoMo | 53.9 | 77.1 | 55.00 | 86.50 | 86.40 | | | | | | | |
| MoMFE(本文) | 53.4 | 78.8 | 54.72 | 86.83 | 86.69 | | 37.7 | 75.1 | 41.38 | 71.99 | 84.33 | 84.51 |
), ArticleFig(id=1329419936037974319, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1329419929369030861, language=EN, label=Table 4, caption=
Experimental Results Under Different Router Settings %
, figureFileSmall=null, figureFileBig=null, tableContent=
| 路由机制 | MER2024数据集 | | CH-SIMS数据集 |
|---|
| Acc-6 | F1 | | Acc-2 | F1 |
|---|
| Softmax路由 | 87.68 | 87.45 | | 81.14 | 81.19 |
| ARM路由 | 89.39 | 88.81 | | 83.36 | 83.11 |
| ARM路由+Lna | 90.01 | 89.88 | | 84.33 | 84.51 |
), ArticleFig(id=1329419936096694576, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1329419929369030861, language=CN, label=表4, caption=
不同路由设置下的实验结果
, figureFileSmall=null, figureFileBig=null, tableContent=
| 路由机制 | MER2024数据集 | | CH-SIMS数据集 |
|---|
| Acc-6 | F1 | | Acc-2 | F1 |
|---|
| Softmax路由 | 87.68 | 87.45 | | 81.14 | 81.19 |
| ARM路由 | 89.39 | 88.81 | | 83.36 | 83.11 |
| ARM路由+Lna | 90.01 | 89.88 | | 84.33 | 84.51 |
), ArticleFig(id=1329419936151220529, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1329419929369030861, language=EN, label=Table 5, caption=
Experimental Results of Different Expert Combinations Setting %
, figureFileSmall=null, figureFileBig=null, tableContent=
| 多模态融合专家 | | CH-SIMS数据集 |
|---|
| DF | LF | EF | SF | | Acc-3 | Acc-2 | F1 |
|---|
| √ | √ | | | | 68.19 | 80.39 | 80.37 |
| √ | √ | | √ | | 69.37 | 81.33 | 81.53 |
| √ | | √ | | | 69.10 | 82.35 | 82.39 |
| √ | | √ | √ | | 70.48 | 82.39 | 82.77 |
| √ | √ | | | 70.07 | 82.13 | 82.55 |
| √ | √ | √ | | 70.39 | 82.53 | 82.68 |
| √ | √ | √ | | | 71.25 | 83.42 | 83.71 |
| √ | √ | √ | √ | | 71.99 | 84.33 | 84.51 |
), ArticleFig(id=1329419936209940786, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1329419929369030861, language=CN, label=表5, caption=
不同专家组合设置的实验结果
, figureFileSmall=null, figureFileBig=null, tableContent=
| 多模态融合专家 | | CH-SIMS数据集 |
|---|
| DF | LF | EF | SF | | Acc-3 | Acc-2 | F1 |
|---|
| √ | √ | | | | 68.19 | 80.39 | 80.37 |
| √ | √ | | √ | | 69.37 | 81.33 | 81.53 |
| √ | | √ | | | 69.10 | 82.35 | 82.39 |
| √ | | √ | √ | | 70.48 | 82.39 | 82.77 |
| √ | √ | | | 70.07 | 82.13 | 82.55 |
| √ | √ | √ | | 70.39 | 82.53 | 82.68 |
| √ | √ | √ | | | 71.25 | 83.42 | 83.71 |
| √ | √ | √ | √ | | 71.99 | 84.33 | 84.51 |
), ArticleFig(id=1329419936285438259, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1329419929369030861, language=EN, label=Table 6, caption=
Experimental Results of Model Performance with Missing Modalities %
, figureFileSmall=null, figureFileBig=null, tableContent=
模态缺失 类型 | 缺失样本 比例 | MER2024数据集 | | CMU-MOSEI数据集 | | CH-SIMS数据集 |
|---|
| Acc | F1 | | Acc-2 | F1 | | Acc-2 | F1 |
|---|
单模态 缺失 | 10 | 89.27±0.07 | 88.83±0.12 | | 86.12±0.12 | 86.25±0.08 | | 83.81±0.11 | 84.03±0.14 |
| 30 | 88.01±0.12 | 87.55±0.17 | | 84.89±0.24 | 85.02±0.27 | | 83.11±0.17 | 83.37±0.21 |
| 50 | 86.43±0.10 | 86.13±0.05 | | 84.14±0.20 | 84.37±0.16 | | 82.69±0.26 | 82.84±0.19 |
双模态 缺失 | 10 | 87.32±0.03 | 87.10±0.10 | | 84.26±0.13 | 84.53±0.12 | | 83.28±0.12 | 83.81±0.15 |
| 30 | 86.52±0.11 | 86.31±0.18 | | 83.52±0.24 | 83.88±0.17 | | 82.03±0.06 | 82.44±0.09 |
| 50 | 84.35±0.09 | 83.87±0.23 | | 83.04±0.15 | 83.29±0.18 | | 81.42±0.25 | 81.75±0.14 |
), ArticleFig(id=1329419936373518644, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1329419929369030861, language=CN, label=表6, caption=
缺失模态下模型性能的实验结果
, figureFileSmall=null, figureFileBig=null, tableContent=
模态缺失 类型 | 缺失样本 比例 | MER2024数据集 | | CMU-MOSEI数据集 | | CH-SIMS数据集 |
|---|
| Acc | F1 | | Acc-2 | F1 | | Acc-2 | F1 |
|---|
单模态 缺失 | 10 | 89.27±0.07 | 88.83±0.12 | | 86.12±0.12 | 86.25±0.08 | | 83.81±0.11 | 84.03±0.14 |
| 30 | 88.01±0.12 | 87.55±0.17 | | 84.89±0.24 | 85.02±0.27 | | 83.11±0.17 | 83.37±0.21 |
| 50 | 86.43±0.10 | 86.13±0.05 | | 84.14±0.20 | 84.37±0.16 | | 82.69±0.26 | 82.84±0.19 |
双模态 缺失 | 10 | 87.32±0.03 | 87.10±0.10 | | 84.26±0.13 | 84.53±0.12 | | 83.28±0.12 | 83.81±0.15 |
| 30 | 86.52±0.11 | 86.31±0.18 | | 83.52±0.24 | 83.88±0.17 | | 82.03±0.06 | 82.44±0.09 |
| 50 | 84.35±0.09 | 83.87±0.23 | | 83.04±0.15 | 83.29±0.18 | | 81.42±0.25 | 81.75±0.14 |
), ArticleFig(id=1329419936453210421, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1329419929369030861, language=EN, label=Table 7, caption=
Experimental Results of Noise Robustness %
, figureFileSmall=null, figureFileBig=null, tableContent=
模态加噪 类型 | 加噪样本 比例 | MER2024数据集 |
|---|
| Acc | F1 |
|---|
语音模态 加噪 | 10 | 88.57±0.15 | 88.21±0.13 |
| 30 | 87.44±0.05 | 87.10±0.08 |
| 50 | 86.23±0.14 | 85.95±0.15 |
视觉模态 加噪 | 10 | 89.11±0.08 | 88.85±0.10 |
| 30 | 88.64±0.06 | 88.26±0.05 |
| 50 | 87.34±0.11 | 87.05±0.13 |
语音+视觉 加噪 | 10 | 88.23±0.07 | 87.94±0.09 |
| 30 | 87.29±0.15 | 87.12±0.11 |
| 50 | 85.86±0.08 | 85.62±0.05 |
), ArticleFig(id=1329419936541290806, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1329419929369030861, language=CN, label=表7, caption=
噪声鲁棒性实验结果
, figureFileSmall=null, figureFileBig=null, tableContent=
模态加噪 类型 | 加噪样本 比例 | MER2024数据集 |
|---|
| Acc | F1 |
|---|
语音模态 加噪 | 10 | 88.57±0.15 | 88.21±0.13 |
| 30 | 87.44±0.05 | 87.10±0.08 |
| 50 | 86.23±0.14 | 85.95±0.15 |
视觉模态 加噪 | 10 | 89.11±0.08 | 88.85±0.10 |
| 30 | 88.64±0.06 | 88.26±0.05 |
| 50 | 87.34±0.11 | 87.05±0.13 |
语音+视觉 加噪 | 10 | 88.23±0.07 | 87.94±0.09 |
| 30 | 87.29±0.15 | 87.12±0.11 |
| 50 | 85.86±0.08 | 85.62±0.05 |
)], attaches=null, journal=Journal(id=1324759606947315908, delFlag=0, nameCn=计算机研究与发展, nameEn=Journal of Computer Research and Development, nameHistory1=null, nameHistory2=null, issn=1000-1239, eissn=null, cn=11-1777/TP, coden=null, periodic=0, language=CN, oaType=null, ccby=null, superviseOffice=null, ownerOffice=null, pubOffice=null, editorOffice=null, officeType=null, aims=null, clcCode=null, officeProv=null, officeCity=null, officeAddr=null, officeZip=null, officeEmail=null, officePhone=null, editDirector=null, officeDirector=null, officeDirectorPhone=null, officeStaffNum=null, officeEmpNum=null, coverPicUrl=k2l8zgqwstuxxevFX2qxVA==, journalPrice=null, startedYear=null, abbrevIsoEn=Journal of Computer Research and Development, journalRemark=null, publicationField=null, createdTime=1793776903141, updatedTime=1793850698612, createdBy=18614031015, updatedBy=13701087609, firstLetterCn=J, firstLetterEn=J, subjectCode=Electronic Information, subjectName=null, subjectCodeEn=Electronic Information, subjectNameEn=null, picCn=k2l8zgqwstuxxevFX2qxVA==, picEn=r6k0Ry+RD+c/VXj5QarmDw==, jcr=null, cjcr=null, exts=[JournalExt(id=1325069127710286688, language=CN, name=计算机研究与发展, nameHistory1=null, nameHistory2=null, managedBy=, sponsoredBy=, publishedBy=, editorOffice=, officeProv=null, officeCity=null, officeAddr=, officeZip=, editDirector=, officeDirector=null, officePhone=null, coverPicUrl=null, journalRemark=, submitArticleUrl=null, websiteUrl=, createdTime=1793850698640, updatedTime=1793850698640, createdBy=13701087609, updatedBy=13701087609, submissionGuidelinesUrl=, submissionAuthorUrl=https://crad.ict.ac.cn/journalx/authorLogOn.action, submissionEditorUrl=https://crad.ict.ac.cn/journalx/editorLogOn.action, submissionReviewUrl=https://crad.ict.ac.cn/journalx/expertLogOn.action, submissionCeEditorUrl=https://crad.ict.ac.cn/journalx/editorInChiefLogOn.action, submissionAeEditorUrl=, option={"copyright":""}), JournalExt(id=1325069127777395553, language=EN, name=Journal of Computer Research and Development, nameHistory1=null, nameHistory2=null, managedBy=, sponsoredBy=, publishedBy=, editorOffice=, officeProv=null, officeCity=null, officeAddr=, officeZip=, editDirector=, officeDirector=null, officePhone=null, coverPicUrl=null, journalRemark=, submitArticleUrl=null, websiteUrl=, createdTime=1793850698656, updatedTime=1793850698656, createdBy=13701087609, updatedBy=13701087609, submissionGuidelinesUrl=, submissionAuthorUrl=https://crad.ict.ac.cn/journalx/authorLogOn.action, submissionEditorUrl=https://crad.ict.ac.cn/journalx/editorLogOn.action, submissionReviewUrl=https://crad.ict.ac.cn/journalx/expertLogOn.action, submissionCeEditorUrl=https://crad.ict.ac.cn/journalx/editorInChiefLogOn.action, submissionAeEditorUrl=, option={"copyright":""})], databaseList=null, tenantJournalId=1324759914146529478, websiteList=[Website(id=1325070300139892771, webName=null, webTitle=null, webDomain=null, webCopyrigh=null, webIpcNo=null, seoTitle=null, seoKeywords=null, seoDescription=null, tenantJournalId=null, journalId=1324759914146529478, journalNameCn=null, journalNameEn=null, grayFlag=null, tenantId=1146029695717560320, platformId=null, journalGroupId=null, journalGroupNameCn=null, journalGroupNameEn=null, type=1, domain=https://castjournals.cast.org.cn/joweb/jsjyjyfz/CN, language=CN, createTime=1793850978169, createBy=18614031015, updateTime=1793851040259, updateBy=18614031015, name=计算机研究与发展-中文, tplId=1146099689490845704, title=计算机研究与发展, delFlag=0, indexPage=/home, props=[WebsiteProps(id=1325070893751349360, tenantId=1146029695717560320, journalId=null, journalGroupId=null, siteId=1325070300139892771, code=articleTextType, value=kx, createTime=1793851119697, updateTime=1793851119697, creator=18614031015, updator=18614031015), WebsiteProps(id=1325070893730377837, tenantId=1146029695717560320, journalId=null, journalGroupId=null, siteId=1325070300139892771, code=banner, value=null, createTime=1793851119692, updateTime=1793851119692, creator=18614031015, updator=18614031015), WebsiteProps(id=1325070893772320883, tenantId=1146029695717560320, journalId=null, journalGroupId=null, siteId=1325070300139892771, code=grayFlag, value=0, createTime=1793851119702, updateTime=1793851119702, creator=18614031015, updator=18614031015), WebsiteProps(id=1325070893721989228, tenantId=1146029695717560320, journalId=null, journalGroupId=null, siteId=1325070300139892771, code=logo, value=https://castjournals.cast.org.cn/joweb/jsjyjyfz/CN/file/pic?fileId=U27tDsy717YF7XSPA+Vwdw==, createTime=1793851119690, updateTime=1793851119690, creator=18614031015, updator=18614031015), WebsiteProps(id=1325070893784903797, tenantId=1146029695717560320, journalId=null, journalGroupId=null, siteId=1325070300139892771, code=minRunFlag, value=0, createTime=1793851119705, updateTime=1793851119705, creator=18614031015, updator=18614031015), WebsiteProps(id=1325070893742960751, tenantId=1146029695717560320, journalId=null, journalGroupId=null, siteId=1325070300139892771, code=picServerUrl, value=https://castjournals.cast.org.cn/joweb/jsjyjyfz/CN/file/pic, createTime=1793851119695, updateTime=1793851119695, creator=18614031015, updator=18614031015), WebsiteProps(id=1325070893780709492, tenantId=1146029695717560320, journalId=null, journalGroupId=null, siteId=1325070300139892771, code=silenceFlag, value=0, createTime=1793851119704, updateTime=1793851119704, creator=18614031015, updator=18614031015), WebsiteProps(id=1325070893738766446, tenantId=1146029695717560320, journalId=null, journalGroupId=null, siteId=1325070300139892771, code=staticResourcePath, value=https://castjournals.cast.org.cn/joweb/cast_kjdb_cn_619/, createTime=1793851119694, updateTime=1793851119694, creator=18614031015, updator=18614031015), WebsiteProps(id=1325070893755543665, tenantId=1146029695717560320, journalId=null, journalGroupId=null, siteId=1325070300139892771, code=themeColor, value=null, createTime=1793851119698, updateTime=1793851119698, creator=18614031015, updator=18614031015), WebsiteProps(id=1325070893763932274, tenantId=1146029695717560320, journalId=null, journalGroupId=null, siteId=1325070300139892771, code=themeStyle, value=null, createTime=1793851119700, updateTime=1793851119700, creator=18614031015, updator=18614031015)]), Website(id=1325070300580294698, webName=null, webTitle=null, webDomain=null, webCopyrigh=null, webIpcNo=null, seoTitle=null, seoKeywords=null, seoDescription=null, tenantJournalId=null, journalId=1324759914146529478, journalNameCn=null, journalNameEn=null, grayFlag=null, tenantId=1146029695717560320, platformId=null, journalGroupId=null, journalGroupNameCn=null, journalGroupNameEn=null, type=1, domain=https://castjournals.cast.org.cn/joweb/jsjyjyfz/EN, language=EN, createTime=1793850978274, createBy=18614031015, updateTime=1793851036638, updateBy=18614031015, name=计算机研究与发展-英文, tplId=1146101810881728533, title=Journal of Computer Research and Development, delFlag=0, indexPage=/home, props=[WebsiteProps(id=1325070925288321148, tenantId=1146029695717560320, journalId=null, journalGroupId=null, siteId=1325070300580294698, code=articleTextType, value=kx, createTime=1793851127216, updateTime=1793851127216, creator=18614031015, updator=18614031015), WebsiteProps(id=1325070925263155321, tenantId=1146029695717560320, journalId=null, journalGroupId=null, siteId=1325070300580294698, code=banner, value=null, createTime=1793851127210, updateTime=1793851127210, creator=18614031015, updator=18614031015), WebsiteProps(id=1325070925309292671, tenantId=1146029695717560320, journalId=null, journalGroupId=null, siteId=1325070300580294698, code=grayFlag, value=0, createTime=1793851127221, updateTime=1793851127221, creator=18614031015, updator=18614031015), WebsiteProps(id=1325070925254766712, tenantId=1146029695717560320, journalId=null, journalGroupId=null, siteId=1325070300580294698, code=logo, value=https://castjournals.cast.org.cn/joweb/jsjyjyfz/EN/file/pic?fileId=U27tDsy717YF7XSPA+Vwdw==, createTime=1793851127208, updateTime=1793851127208, creator=18614031015, updator=18614031015), WebsiteProps(id=1325070925326069889, tenantId=1146029695717560320, journalId=null, journalGroupId=null, siteId=1325070300580294698, code=minRunFlag, value=0, createTime=1793851127225, updateTime=1793851127225, creator=18614031015, updator=18614031015), WebsiteProps(id=1325070925279932539, tenantId=1146029695717560320, journalId=null, journalGroupId=null, siteId=1325070300580294698, code=picServerUrl, value=https://castjournals.cast.org.cn/joweb/jsjyjyfz/EN/file/pic, createTime=1793851127214, updateTime=1793851127214, creator=18614031015, updator=18614031015), WebsiteProps(id=1325070925317681280, tenantId=1146029695717560320, journalId=null, journalGroupId=null, siteId=1325070300580294698, code=silenceFlag, value=0, createTime=1793851127223, updateTime=1793851127223, creator=18614031015, updator=18614031015), WebsiteProps(id=1325070925271543930, tenantId=1146029695717560320, journalId=null, journalGroupId=null, siteId=1325070300580294698, code=staticResourcePath, value=https://castjournals.cast.org.cn/joweb/cast_kjdb_en_623/, createTime=1793851127212, updateTime=1793851127212, creator=18614031015, updator=18614031015), WebsiteProps(id=1325070925292515453, tenantId=1146029695717560320, journalId=null, journalGroupId=null, siteId=1325070300580294698, code=themeColor, value=null, createTime=1793851127217, updateTime=1793851127217, creator=18614031015, updator=18614031015), WebsiteProps(id=1325070925300904062, tenantId=1146029695717560320, journalId=null, journalGroupId=null, siteId=1325070300580294698, code=themeStyle, value=null, createTime=1793851127219, updateTime=1793851127219, creator=18614031015, updator=18614031015)])], journalTitle=计算机研究与发展, weixinUrl=null, journalUrl=https://crad.ict.ac.cn/, iacademicId=null, status=1, seqNo=null, journalTitleEn=Journal of Computer Research and Development, journalPhotoCn=k2l8zgqwstuxxevFX2qxVA==, journalPhotoEn=r6k0Ry+RD+c/VXj5QarmDw==, journalFirstLetter=J, journalRecommend=null, journalNew=null, journalCollection=null, jcrJf=null, cjcrJf=null, jcrJfStr=null, cjcrJfStr=null, submissionFirstDecision=null, sciSubjectClassification=null, casSubjectClassification=null, citeScore=null, totalCitationFrequency=null, icpCode=null, psCode=null, advertisingLicenseCode=null, copyrightInformation=null, country=null, option=, provinceCode=null, provinceName=null, collectFlag=false, interPubPlatform=, interPubPlatformUrl=null), detailUrlCn=https://castjournals.cast.org.cn/joweb/jsjyjyfz/CN/10.7544/issn1000-1239.202550544, detailUrlEn=https://castjournals.cast.org.cn/joweb/jsjyjyfz/EN/10.7544/issn1000-1239.202550544, pdfUrlCn=https://castjournals.cast.org.cn/joweb/jsjyjyfz/CN/PDF/10.7544/issn1000-1239.202550544, pdfUrlEn=https://castjournals.cast.org.cn/joweb/jsjyjyfz/EN/PDF/10.7544/issn1000-1239.202550544, aliStartDate=0, aliEndDate=0, collectionFlag=false, citedCount=null, citedUrl=null, previewStatus=0, delFlag=0, hasFullText=1, orderTime=1780243200000, fullTextJson=null, articleText=null, reference=null)