Article(id=1325490414429545387, tenantId=1146029695717560320, journalId=1324759914146529478, issueId=1325489890678420381, articleNumber=null, orderNo=null, doi=10.7544/issn1000-1239.202440694, pmid=null, cstr=null, oa=null, hot=null, price=null, onlineType=0, articleFormat=0, articleType=null, articleTypeStr=research-article, receivedDate=1724860800000, receivedDateStr=2024-08-29, revisedDate=1745856000000, revisedDateStr=2025-04-29, acceptedDate=1749571200000, acceptedDateStr=2025-06-11, onlineDate=1793951141220, onlineDateStr=2026-11-06, pubDate=1767196800000, pubDateStr=2026-01-01, doiRegisterDate=null, doiRegisterDateStr=null, onlineIssueDate=1793951141220, onlineIssueDateStr=2026-11-06, onlineJustAcceptDate=null, onlineJustAcceptDateStr=null, onlineFirstDate=null, onlineFirstDateStr=null, sourceXml=null, magXml=null, createTime=1793951141220, creator=13701087609, updateTime=1793951141220, updator=13701087609, issue=Issue{id=1325489890678420381, tenantId=1146029695717560320, journalId=1324759914146529478, year='2026', volume='63', issue='1', pageStart='1', pageEnd='272', issueExtLink='null', onlineDate='null', pubDate='1767196800000', pubDateStr='2026-01-01', beforeIssueId=null, nextIssueId=null, price=null, status=1, issueComplete=1, articleOrder=1, issueType=-1, specialIssue=null, createTime=1793951016348, creator='13701087609', updateTime=1793952522017, updator='13701087609', preIssue=null, nextIssue=null, articleTotal=null, ext={EN=IssueExt(id=1325496205974839692, tenantId=1146029695717560320, journalId=1324759914146529478, issueId=1325489890678420381, language=EN, specialIssueTitle=, coverIllustrator=null, specialIssueEditor=, specialIssueAbout=), CN=IssueExt(id=1325496205974839693, tenantId=1146029695717560320, journalId=1324759914146529478, issueId=1325489890678420381, language=CN, specialIssueTitle=, coverIllustrator=null, specialIssueEditor=, specialIssueAbout=)}, issueFiles=null, downloadFileDto=null}, startPage=90, endPage=122, ext={EN=ArticleExt(id=1325490425359901614, articleId=1325490414429545387, tenantId=1146029695717560320, journalId=1324759914146529478, language=EN, title=Knowledge Distillation in Visual Algorithms: A Survey, columnId=1325489891789910943, journalTitle=Journal of Computer Research and Development, columnName=Artificial Intelligence, runingTitle=null, highlight=null, articleAbstract=
Knowledge distillation, as a key technique in deep learning, achieves model compression and acceleration by transferring knowledge from a large teacher model to a smaller student model. Under the premise of maintaining performance, this technology significantly reduces the requirements of computational resources and storage, and facilitates the deployment of high-performance models on resource-constrained edge devices. Firstly, we provide a systematic review of the recent research in knowledge distillation and categorize it from two perspectives: the type of knowledge and teacher-student model architectures. We comprehensively summarize the distillation methods based on three typical types of knowledge: output feature knowledge, intermediate feature knowledge, and relational feature knowledge, as well as distillation methods based on CNN to CNN architecture, CNN to ViT (vision Transformer) architecture, ViT to CNN architecture, and ViT to ViT architecture. Next, we explore various learning paradigms such as offline distillation, online distillation, self-distillation, data-free distillation, multi-teacher distillation, and assistant distillation. Then, we summarize distillation optimization methods based on the distillation process, knowledge structure, temperature coefficient, and loss functions and analyze improvements in distillation brought by adversarial techniques, automated machine learning, reinforcement learning, and diffusion models, and conclude with the implementation of distillation technology in common applications. Despite significant advancements in knowledge distillation, numerous challenges remain in both practical applications and theoretical research. Finally, we provide an in-depth analysis of these issues and offer insights into future development directions.
, authors=Haiwei Pan, Fengming Yu, Kejia Zhang*, Haiyan Lan, Qingyu Meng, Zhe Li, authorsList=Haiwei Pan, Fengming Yu, Kejia Zhang, Haiyan Lan, Qingyu Meng, Zhe Li, authorCompany=null, correspAuthors=Kejia Zhang, authorNote=
born in 1975. PhD, professor, PhD supervisor. Senior member of CCF. His main research interests include remote sensing image processing, medical image processing, data mining, machine learning, and deep learning
born in 1998. PhD candidate. Student member of CCF. His main research interests include knowledge distillation, model compression, and machine learning
born in 1981. PhD, associate professor, PhD supervisor. Member of CCF. His main research interests include machine learning, smart healthcare, and big data
born in 1983. PhD, lecturer, master supervisor. Senior member of CCF. Her main research interests include signal processing and information security
born in 1998. Master. His main research interests include model compression, natural language processing, and machine learning
born in 1999. PhD candidate. Student member of CCF. His main research interests include computer vision, image generation, and multimodal image processing
, correspAuthorsNote=null, copyrightStatement=Copyright © 2026 Journal of Computer Research and Development. All rights reserved., copyrightOwner=null, extLink=null, articleAbsUrl=null, sourceXml=null, magXml=null, pdfUrl=null, pdf=null, pdfFileSize=null, pdfExtLink=null, richHtmlUrl=null, mobilePdfUrl=null, reviewReport=null, pdfFirstPage=null, abstractGraph=null, abstractGraphContent=null, abstractVideo=null, citation=null, cebUrl=null, magXmlContent=null, mapNumber=null, fund=null), CN=ArticleExt(id=1325490573792124945, articleId=1325490414429545387, tenantId=1146029695717560320, journalId=1324759914146529478, language=CN, title=面向视觉算法的知识蒸馏研究综述, columnId=1325489892104483745, journalTitle=计算机研究与发展, columnName=人工智能, runingTitle=null, highlight=null, articleAbstract=
知识蒸馏作为深度学习中的关键技术,通过将大型教师模型的知识传递给较小的学生模型,实现了模型的压缩与加速。在保证性能的前提下,显著减少了计算资源和存储需求,促进了高性能模型在资源受限的边缘设备上的部署。围绕知识蒸馏的最新研究进展进行了系统性的综述,从知识类型和师生模型架构2个角度对知识蒸馏进行分类,详细汇总了输出特征知识、中间特征知识、关系特征知识3种典型知识类型的蒸馏方法,以及卷积架构到卷积架构、卷积架构到ViT(vision Transformer)架构、ViT架构到卷积架构和ViT架构到ViT架构的蒸馏方法;探讨了离线蒸馏、在线蒸馏、自蒸馏、无数据蒸馏、多教师蒸馏和助理蒸馏的学习方式;归纳了基于蒸馏过程、知识结构、温度系数及损失函数的蒸馏优化方法,分析了对抗性技术、自动机器学习、强化学习和扩散模型对蒸馏的改进,并总结了蒸馏技术在常见应用中的实现。尽管知识蒸馏取得了显著进展,但在实际应用和理论研究中仍面临诸多挑战。最后,对这些问题进行了深入分析,并对未来发展方向提出了见解。
, authors=潘海为, 于丰铭, 张可佳*, 兰海燕, 孟庆宇, 李哲, authorsList=潘海为, 于丰铭, 张可佳, 兰海燕, 孟庆宇, 李哲, authorCompany=null, correspAuthors=张可佳, authorNote=
 |
1975年生。博士,教授,博士生导师。CCF高级会员。主要研究方向为遥感图像处理、医学图像处理、数据挖掘、机器学习、深度学习 |
 |
1998年生。博士研究生。CCF学生会员。主要研究方向为知识蒸馏、模型压缩、机器学习 |
 |
1981年生。博士,副教授,博士生导师。CCF会员。主要研究方向为机器学习、智慧医疗、大数据 |
 |
1983年生。博士,讲师,硕士生导师。CCF高级会员。主要研究方向为信号处理、信息安全 |
 |
1998年生。硕士。主要研究方向为模型压缩、自然语言处理、机器学习 |
 |
1999年生。博士研究生。CCF学生会员。主要研究方向为计算机视觉、图像生成、多模态图像处理 |
, correspAuthorsNote=
, copyrightStatement=版权所有 © 2026 计算机研究与发展编辑部, copyrightOwner=null, extLink=null, articleAbsUrl=null, sourceXml=Cg+Yj0fWJF4mVglgl1ZhNw==, magXml=+dbfugX4DU/JND2BgNBVhw==, pdfUrl=null, pdf=XYICTV+GezntksZjoVsUNw==, pdfFileSize=2175625, pdfExtLink=null, richHtmlUrl=null, mobilePdfUrl=null, reviewReport=null, pdfFirstPage=null, abstractGraph=c5TffK5Y2OVaiAIy+407Gg==, abstractGraphContent=null, abstractVideo=null, citation=null, cebUrl=null, magXmlContent=ik4womjjWLX17AEJT9QBPg==, mapNumber=null, fund=null)}, authors=[Author(id=1325490584328216600, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1325490414429545387, orderNo=0, firstName=null, middleName=null, lastName=null, nameCn=null, orcid=null, stid=null, country=null, authorPic=null, dead=0, email=panhaiwei@hrbeu.edu.cn, emailSecond=null, emailThird=null, correspondingAuthor=0, authorType=1, ext={EN=AuthorExt(id=1325490586010132506, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1325490414429545387, authorId=1325490584328216600, language=EN, stringName=Haiwei Pan, firstName=Haiwei, middleName=null, lastName=Pan, prefix=null, suffix=null, authorComment=null, nameInitials=null, affiliation=null, department=null, xref=null, address=College of Computer Science and Technology, Harbin Engineering University, Harbin 150001, bio={"content":"
born in 1975. PhD, professor, PhD supervisor. Senior member of CCF. His main research interests include remote sensing image processing, medical image processing, data mining, machine learning, and deep learning
"}, bioImg=null, bioContent=
born in 1975. PhD, professor, PhD supervisor. Senior member of CCF. His main research interests include remote sensing image processing, medical image processing, data mining, machine learning, and deep learning
, aboutCorrespAuthor=null), CN=AuthorExt(id=1325490588136644635, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1325490414429545387, authorId=1325490584328216600, language=CN, stringName=潘海为, firstName=null, middleName=null, lastName=null, prefix=null, suffix=null, authorComment=null, nameInitials=null, affiliation=null, department=null, xref=null, address=哈尔滨工程大学计算机科学与技术学院 哈尔滨 150001, bio={"img":"RvLRdxKhIw4RYAitV5qsfw==","content":"
1975年生。博士,教授,博士生导师。CCF高级会员。主要研究方向为遥感图像处理、医学图像处理、数据挖掘、机器学习、深度学习
"}, bioImg=RvLRdxKhIw4RYAitV5qsfw==, bioContent=
1975年生。博士,教授,博士生导师。CCF高级会员。主要研究方向为遥感图像处理、医学图像处理、数据挖掘、机器学习、深度学习
, aboutCorrespAuthor=null)}, companyList=[AuthorCompany(id=1325490582310756372, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1325490414429545387, xref=null, ext=[AuthorCompanyExt(id=1325490582361088021, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1325490414429545387, companyId=1325490582310756372, language=EN, country=null, province=null, city=null, postcode=null, companyName=null, departmentName=null, remark=College of Computer Science and Technology, Harbin Engineering University, Harbin 150001), AuthorCompanyExt(id=1325490582587580438, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1325490414429545387, companyId=1325490582310756372, language=CN, country=null, province=null, city=null, postcode=null, companyName=null, departmentName=null, remark=哈尔滨工程大学计算机科学与技术学院 哈尔滨 150001)])]), Author(id=1325490589852114973, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1325490414429545387, orderNo=1, firstName=null, middleName=null, lastName=null, nameCn=null, orcid=null, stid=null, country=null, authorPic=null, dead=0, email=null, emailSecond=null, emailThird=null, correspondingAuthor=0, authorType=1, ext={EN=AuthorExt(id=1325490593136255008, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1325490414429545387, authorId=1325490589852114973, language=EN, stringName=Fengming Yu, firstName=Fengming, middleName=null, lastName=Yu, prefix=null, suffix=null, authorComment=null, nameInitials=null, affiliation=null, department=null, xref=null, address=College of Computer Science and Technology, Harbin Engineering University, Harbin 150001, bio={"content":"
born in 1998. PhD candidate. Student member of CCF. His main research interests include knowledge distillation, model compression, and machine learning
"}, bioImg=null, bioContent=
born in 1998. PhD candidate. Student member of CCF. His main research interests include knowledge distillation, model compression, and machine learning
, aboutCorrespAuthor=null), CN=AuthorExt(id=1325490594843336737, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1325490414429545387, authorId=1325490589852114973, language=CN, stringName=于丰铭, firstName=null, middleName=null, lastName=null, prefix=null, suffix=null, authorComment=null, nameInitials=null, affiliation=null, department=null, xref=null, address=哈尔滨工程大学计算机科学与技术学院 哈尔滨 150001, bio={"img":"4cLSSbHqrsnl91CyY3GZyA==","content":"
1998年生。博士研究生。CCF学生会员。主要研究方向为知识蒸馏、模型压缩、机器学习
"}, bioImg=4cLSSbHqrsnl91CyY3GZyA==, bioContent=
1998年生。博士研究生。CCF学生会员。主要研究方向为知识蒸馏、模型压缩、机器学习
, aboutCorrespAuthor=null)}, companyList=[AuthorCompany(id=1325490582310756372, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1325490414429545387, xref=null, ext=[AuthorCompanyExt(id=1325490582361088021, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1325490414429545387, companyId=1325490582310756372, language=EN, country=null, province=null, city=null, postcode=null, companyName=null, departmentName=null, remark=College of Computer Science and Technology, Harbin Engineering University, Harbin 150001), AuthorCompanyExt(id=1325490582587580438, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1325490414429545387, companyId=1325490582310756372, language=CN, country=null, province=null, city=null, postcode=null, companyName=null, departmentName=null, remark=哈尔滨工程大学计算机科学与技术学院 哈尔滨 150001)])]), Author(id=1325490597276033062, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1325490414429545387, orderNo=2, firstName=null, middleName=null, lastName=null, nameCn=null, orcid=null, stid=null, country=null, authorPic=null, dead=0, email=kejiazhang@hrbeu.edu.cn, emailSecond=null, emailThird=null, correspondingAuthor=1, authorType=1, ext={EN=AuthorExt(id=1325490598555295784, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1325490414429545387, authorId=1325490597276033062, language=EN, stringName=Kejia Zhang, firstName=Kejia, middleName=null, lastName=Zhang, prefix=null, suffix=null, authorComment=null, nameInitials=null, affiliation=null, department=null, xref=*, address=College of Computer Science and Technology, Harbin Engineering University, Harbin 150001, bio={"content":"
born in 1981. PhD, associate professor, PhD supervisor. Member of CCF. His main research interests include machine learning, smart healthcare, and big data
"}, bioImg=null, bioContent=
born in 1981. PhD, associate professor, PhD supervisor. Member of CCF. His main research interests include machine learning, smart healthcare, and big data
, aboutCorrespAuthor=null), CN=AuthorExt(id=1325490599914250281, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1325490414429545387, authorId=1325490597276033062, language=CN, stringName=张可佳, firstName=null, middleName=null, lastName=null, prefix=null, suffix=null, authorComment=null, nameInitials=null, affiliation=null, department=null, xref=*, address=哈尔滨工程大学计算机科学与技术学院 哈尔滨 150001, bio={"img":"2DN+pW3Sz4H4IJYowRSC0A==","content":"
1981年生。博士,副教授,博士生导师。CCF会员。主要研究方向为机器学习、智慧医疗、大数据
"}, bioImg=2DN+pW3Sz4H4IJYowRSC0A==, bioContent=
1981年生。博士,副教授,博士生导师。CCF会员。主要研究方向为机器学习、智慧医疗、大数据
, aboutCorrespAuthor=null)}, companyList=[AuthorCompany(id=1325490582310756372, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1325490414429545387, xref=null, ext=[AuthorCompanyExt(id=1325490582361088021, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1325490414429545387, companyId=1325490582310756372, language=EN, country=null, province=null, city=null, postcode=null, companyName=null, departmentName=null, remark=College of Computer Science and Technology, Harbin Engineering University, Harbin 150001), AuthorCompanyExt(id=1325490582587580438, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1325490414429545387, companyId=1325490582310756372, language=CN, country=null, province=null, city=null, postcode=null, companyName=null, departmentName=null, remark=哈尔滨工程大学计算机科学与技术学院 哈尔滨 150001)])]), Author(id=1325490602325975083, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1325490414429545387, orderNo=3, firstName=null, middleName=null, lastName=null, nameCn=null, orcid=null, stid=null, country=null, authorPic=null, dead=0, email=null, emailSecond=null, emailThird=null, correspondingAuthor=0, authorType=1, ext={EN=AuthorExt(id=1325490605404594222, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1325490414429545387, authorId=1325490602325975083, language=EN, stringName=Haiyan Lan, firstName=Haiyan, middleName=null, lastName=Lan, prefix=null, suffix=null, authorComment=null, nameInitials=null, affiliation=null, department=null, xref=null, address=College of Computer Science and Technology, Harbin Engineering University, Harbin 150001, bio={"content":"
born in 1983. PhD, lecturer, master supervisor. Senior member of CCF. Her main research interests include signal processing and information security
"}, bioImg=null, bioContent=
born in 1983. PhD, lecturer, master supervisor. Senior member of CCF. Her main research interests include signal processing and information security
, aboutCorrespAuthor=null), CN=AuthorExt(id=1325490607765987377, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1325490414429545387, authorId=1325490602325975083, language=CN, stringName=兰海燕, firstName=null, middleName=null, lastName=null, prefix=null, suffix=null, authorComment=null, nameInitials=null, affiliation=null, department=null, xref=null, address=哈尔滨工程大学计算机科学与技术学院 哈尔滨 150001, bio={"img":"LcbrCCD2Mr8BKhKqmKb0kw==","content":"
1983年生。博士,讲师,硕士生导师。CCF高级会员。主要研究方向为信号处理、信息安全
"}, bioImg=LcbrCCD2Mr8BKhKqmKb0kw==, bioContent=
1983年生。博士,讲师,硕士生导师。CCF高级会员。主要研究方向为信号处理、信息安全
, aboutCorrespAuthor=null)}, companyList=[AuthorCompany(id=1325490582310756372, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1325490414429545387, xref=null, ext=[AuthorCompanyExt(id=1325490582361088021, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1325490414429545387, companyId=1325490582310756372, language=EN, country=null, province=null, city=null, postcode=null, companyName=null, departmentName=null, remark=College of Computer Science and Technology, Harbin Engineering University, Harbin 150001), AuthorCompanyExt(id=1325490582587580438, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1325490414429545387, companyId=1325490582310756372, language=CN, country=null, province=null, city=null, postcode=null, companyName=null, departmentName=null, remark=哈尔滨工程大学计算机科学与技术学院 哈尔滨 150001)])]), Author(id=1325490611121430579, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1325490414429545387, orderNo=4, firstName=null, middleName=null, lastName=null, nameCn=null, orcid=null, stid=null, country=null, authorPic=null, dead=0, email=null, emailSecond=null, emailThird=null, correspondingAuthor=0, authorType=1, ext={EN=AuthorExt(id=1325490612409081910, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1325490414429545387, authorId=1325490611121430579, language=EN, stringName=Qingyu Meng, firstName=Qingyu, middleName=null, lastName=Meng, prefix=null, suffix=null, authorComment=null, nameInitials=null, affiliation=null, department=null, xref=null, address=College of Computer Science and Technology, Harbin Engineering University, Harbin 150001, bio={"content":"
born in 1998. Master. His main research interests include model compression, natural language processing, and machine learning
"}, bioImg=null, bioContent=
born in 1998. Master. His main research interests include model compression, natural language processing, and machine learning
, aboutCorrespAuthor=null), CN=AuthorExt(id=1325490614904692792, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1325490414429545387, authorId=1325490611121430579, language=CN, stringName=孟庆宇, firstName=null, middleName=null, lastName=null, prefix=null, suffix=null, authorComment=null, nameInitials=null, affiliation=null, department=null, xref=null, address=哈尔滨工程大学计算机科学与技术学院 哈尔滨 150001, bio={"img":"lbXEooh6sVk23lCVpi7YtQ==","content":"
1998年生。硕士。主要研究方向为模型压缩、自然语言处理、机器学习
"}, bioImg=lbXEooh6sVk23lCVpi7YtQ==, bioContent=
1998年生。硕士。主要研究方向为模型压缩、自然语言处理、机器学习
, aboutCorrespAuthor=null)}, companyList=[AuthorCompany(id=1325490582310756372, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1325490414429545387, xref=null, ext=[AuthorCompanyExt(id=1325490582361088021, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1325490414429545387, companyId=1325490582310756372, language=EN, country=null, province=null, city=null, postcode=null, companyName=null, departmentName=null, remark=College of Computer Science and Technology, Harbin Engineering University, Harbin 150001), AuthorCompanyExt(id=1325490582587580438, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1325490414429545387, companyId=1325490582310756372, language=CN, country=null, province=null, city=null, postcode=null, companyName=null, departmentName=null, remark=哈尔滨工程大学计算机科学与技术学院 哈尔滨 150001)])]), Author(id=1325490616150401082, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1325490414429545387, orderNo=5, firstName=null, middleName=null, lastName=null, nameCn=null, orcid=null, stid=null, country=null, authorPic=null, dead=0, email=null, emailSecond=null, emailThird=null, correspondingAuthor=0, authorType=1, ext={EN=AuthorExt(id=1325490617878454333, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1325490414429545387, authorId=1325490616150401082, language=EN, stringName=Zhe Li, firstName=Zhe, middleName=null, lastName=Li, prefix=null, suffix=null, authorComment=null, nameInitials=null, affiliation=null, department=null, xref=null, address=College of Computer Science and Technology, Harbin Engineering University, Harbin 150001, bio={"content":"
born in 1999. PhD candidate. Student member of CCF. His main research interests include computer vision, image generation, and multimodal image processing
"}, bioImg=null, bioContent=
born in 1999. PhD candidate. Student member of CCF. His main research interests include computer vision, image generation, and multimodal image processing
, aboutCorrespAuthor=null), CN=AuthorExt(id=1325490619988189246, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1325490414429545387, authorId=1325490616150401082, language=CN, stringName=李哲, firstName=null, middleName=null, lastName=null, prefix=null, suffix=null, authorComment=null, nameInitials=null, affiliation=null, department=null, xref=null, address=哈尔滨工程大学计算机科学与技术学院 哈尔滨 150001, bio={"img":"2EItHUXjrCrylIjyNkGmfQ==","content":"
1999年生。博士研究生。CCF学生会员。主要研究方向为计算机视觉、图像生成、多模态图像处理
"}, bioImg=2EItHUXjrCrylIjyNkGmfQ==, bioContent=
1999年生。博士研究生。CCF学生会员。主要研究方向为计算机视觉、图像生成、多模态图像处理
, aboutCorrespAuthor=null)}, companyList=[AuthorCompany(id=1325490582310756372, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1325490414429545387, xref=null, ext=[AuthorCompanyExt(id=1325490582361088021, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1325490414429545387, companyId=1325490582310756372, language=EN, country=null, province=null, city=null, postcode=null, companyName=null, departmentName=null, remark=College of Computer Science and Technology, Harbin Engineering University, Harbin 150001), AuthorCompanyExt(id=1325490582587580438, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1325490414429545387, companyId=1325490582310756372, language=CN, country=null, province=null, city=null, postcode=null, companyName=null, departmentName=null, remark=哈尔滨工程大学计算机科学与技术学院 哈尔滨 150001)])])], keywords=[Keyword(id=1325490624123772991, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1325490414429545387, language=EN, orderNo=1, keyword=knowledge distillation), Keyword(id=1325490625482727488, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1325490414429545387, language=EN, orderNo=2, keyword=model compression), Keyword(id=1325490628737507393, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1325490414429545387, language=EN, orderNo=3, keyword=deep learning), Keyword(id=1325490629685420098, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1325490414429545387, language=EN, orderNo=4, keyword=convolutional neural network), Keyword(id=1325490633766477891, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1325490414429545387, language=EN, orderNo=5, keyword=vision Transformer), Keyword(id=1325490635532279877, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1325490414429545387, language=CN, orderNo=1, keyword=知识蒸馏), Keyword(id=1325490637251944518, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1325490414429545387, language=CN, orderNo=2, keyword=模型压缩), Keyword(id=1325490639244238919, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1325490414429545387, language=CN, orderNo=3, keyword=深度学习), Keyword(id=1325490642570321994, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1325490414429545387, language=CN, orderNo=4, keyword=卷积神经网络), Keyword(id=1325490643845390412, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1325490414429545387, language=CN, orderNo=5, keyword=视觉Transformer)], refs=[Reference(id=1325490767606718615, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1325490414429545387, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=1, rfOrder=0, authorNames=null, journalName=null, refType=null, unstructuredReference=Lecun Y, Bottou L, Bengio Y, et al. Gradient-based learning applied to document recognition[J]. Proceedings of the IEEE, 1998, 86(11): 2278−2324, articleTitle=null, refAbstract=null), Reference(id=1325490769661927575, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1325490414429545387, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=2, rfOrder=1, authorNames=null, journalName=null, refType=null, unstructuredReference=Krizhevsky A, Sutskever I, Hinton G E. ImageNet classification with deep convolutional neural networks[C]//Proc of the 26th Annual Conference on Neural Information Processing Systems. Cambridge, MA: MIT, 2012: 1106−1114, articleTitle=null, refAbstract=null), Reference(id=1325490772237230233, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1325490414429545387, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=3, rfOrder=2, authorNames=null, journalName=null, refType=null, unstructuredReference=Simonyan K, Zisserman A. Very deep convolutional networks for large-scale image recognition[J]. arXiv preprint, arXiv: 1409.1556, 2014, articleTitle=null, refAbstract=null), Reference(id=1325490774401491098, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1325490414429545387, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=4, rfOrder=3, authorNames=null, journalName=null, refType=null, unstructuredReference=Szegedy C, Liu Wei, Jia Yangqing, et al. Going deeper with convolutions[C]//Proc of 2015 IEEE/CVF Conf on Computer Vision and Pattern Recognition. Piscataway, NJ: IEEE, 2015: 1−9, articleTitle=null, refAbstract=null), Reference(id=1325490778558046363, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1325490414429545387, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=5, rfOrder=4, authorNames=null, journalName=null, refType=null, unstructuredReference=Szegedy C, Vanhoucke V, Ioffe S, et al. Rethinking the inception architecture for computer vision[C]//Proc of 2016 IEEE/CVF Conf on Computer Vision and Pattern Recognition. Piscataway, NJ: IEEE, 2016: 2818−2826, articleTitle=null, refAbstract=null), Reference(id=1325490780659392669, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1325490414429545387, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=6, rfOrder=5, authorNames=null, journalName=null, refType=null, unstructuredReference=Szegedy C, Ioffe S, Vanhoucke V, et al. Inception-v4, inception-ResNet and the impact of residual connections on learning[C]//Proc of the 31st AAAI Conf on Artificial Intelligence. Palo Alto, CA: AAAI, 2017: 4278−4284, articleTitle=null, refAbstract=null), Reference(id=1325490783310192798, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1325490414429545387, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=7, rfOrder=6, authorNames=null, journalName=null, refType=null, unstructuredReference=He Kaiming, Zhang Xiangyu, Ren Shaoqing, et al. Deep residual learning for image recognition[C]//Proc of 2016 IEEE/CVF Conf on Computer Vision and Pattern Recognition. Piscataway, NJ: IEEE, 2016: 770−778, articleTitle=null, refAbstract=null), Reference(id=1325490785193435297, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1325490414429545387, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=8, rfOrder=7, authorNames=null, journalName=null, refType=null, unstructuredReference=Huang Gao, Liu Zhuang, van der M L, et al. Densely connected convolutional networks[C]//Proc of 2017 IEEE/CVF Conf on Computer Vision and Pattern Recognition. Piscataway, NJ: IEEE, 2017: 4700−4708, articleTitle=null, refAbstract=null), Reference(id=1325490787286392994, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1325490414429545387, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=9, rfOrder=8, authorNames=null, journalName=null, refType=null, unstructuredReference=Howard A G, Zhu Menglong, Chen Bo, et al. MobileNets: Efficient convolutional neural networks for mobile vision applications[J]. arXiv preprint, arXiv: 1704.04861, 2017, articleTitle=null, refAbstract=null), Reference(id=1325490789912027301, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1325490414429545387, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=10, rfOrder=9, authorNames=null, journalName=null, refType=null, unstructuredReference=Sandler M, Howard A, Zhu Menglong, et al. MobileNetV2: Inverted residuals and linear bottlenecks[C]//Proc of 2018 IEEE/CVF Conf on Computer Vision and Pattern Recognition. Piscataway, NJ: IEEE, 2018: 4510−4520, articleTitle=null, refAbstract=null), Reference(id=1325490792768348326, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1325490414429545387, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=11, rfOrder=10, authorNames=null, journalName=null, refType=null, unstructuredReference=Howard A, Sandler M, Chu G, et al. Searching for MobileNetV3[C]//Proc of 2019 IEEE/CVF Int Conf on Computer Vision. Piscataway, NJ: IEEE, 2019: 1314−1324, articleTitle=null, refAbstract=null), Reference(id=1325490795268153512, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1325490414429545387, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=12, rfOrder=11, authorNames=null, journalName=null, refType=null, unstructuredReference=Tan Mingxing, Le Q. EfficientNet: Rethinking model scaling for convolutional neural networks[C]//Proc of the 36th Int Conf on Machine Learning. New York: ACM, 2019: 6105−6114, articleTitle=null, refAbstract=null), Reference(id=1325490797344333993, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1325490414429545387, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=13, rfOrder=12, authorNames=null, journalName=null, refType=null, unstructuredReference=Tan Mingxing, Le Q. EfficientNetV2: Smaller models and faster training[C]//Proc of the 36th Int Conf on Machine Learning. New York: ACM, 2021: 10096−10106, articleTitle=null, refAbstract=null), Reference(id=1325490799449874604, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1325490414429545387, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=14, rfOrder=13, authorNames=null, journalName=null, refType=null, unstructuredReference=Vaswani A, Shazeer N, Parmar N, et al. Attention is all you need[C]//Proc of the 31st Annual Conf on Neural Information Processing Systems. Cambridge, MA: MIT, 2017: 5998−6008, articleTitle=null, refAbstract=null), Reference(id=1325490801983234222, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1325490414429545387, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=15, rfOrder=14, authorNames=null, journalName=null, refType=null, unstructuredReference=Dosovitskiy A, Beyer L, Kolesnikov A, et al. An image is worth 16×16 words: Transformers for image recognition at scale[J]. arXiv preprint, arXiv: 2010.11929, 2021, articleTitle=null, refAbstract=null), Reference(id=1325490804571119791, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1325490414429545387, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=16, rfOrder=15, authorNames=null, journalName=null, refType=null, unstructuredReference=Liu Ze, Lin Yutong, Cao Yue, et al. Swin Transformer: Hierarchical vision Transformer using shifted windows[C]//Proc of 2021 IEEE/CVF Int Conf on Computer Vision. Piscataway, NJ: IEEE, 2021: 10012−10022, articleTitle=null, refAbstract=null), Reference(id=1325490808719286451, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1325490414429545387, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=17, rfOrder=16, authorNames=null, journalName=null, refType=null, unstructuredReference=Touvron H, Cord M, Douze M, et al. Training data-efficient image transformers & distillation through attention[C]//Proc of the 38th Int Conf on Machine Learning. New York: ACM, 2021: 10347−10357, articleTitle=null, refAbstract=null), Reference(id=1325490811672076468, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1325490414429545387, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=18, rfOrder=17, authorNames=null, journalName=null, refType=null, unstructuredReference=Wang Wenhai, Xie Enze, Li Xiang, et al. Pyramid vision transformer: A versatile backbone for dense prediction without convolutions[C]//Proc of 2021 IEEE/CVF Int Conf on Computer Vision. Piscataway, NJ: IEEE, 2021: 568−578, articleTitle=null, refAbstract=null), Reference(id=1325490814201241782, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1325490414429545387, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=19, rfOrder=18, authorNames=null, journalName=null, refType=null, unstructuredReference=Radford A, Kim J W, Hallacy C, et al. Learning transferable visual models from natural language supervision[C]//Proc of the 38th Int Conf on Machine Learning. New York: ACM, 2021: 8748−8763, articleTitle=null, refAbstract=null), Reference(id=1325490817103700151, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1325490414429545387, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=20, rfOrder=19, authorNames=null, journalName=null, refType=null, unstructuredReference=Ramesh A, Pavlov M, Goh G, et al. Zero-shot text-to-image generation[C]//Proc of the 38th Int Conf on Machine Learning. New York: ACM, 2021: 8821−8831, articleTitle=null, refAbstract=null), Reference(id=1325490819246989498, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1325490414429545387, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=21, rfOrder=20, authorNames=null, journalName=null, refType=null, unstructuredReference=Hinton G, Vinyals O, Dean J. Distilling the knowledge in a neural network[J]. arXiv preprint, arXiv: 1503.02531, 2015, articleTitle=null, refAbstract=null), Reference(id=1325490822183002299, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1325490414429545387, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=22, rfOrder=21, authorNames=null, journalName=null, refType=null, unstructuredReference=Gou Jianping, Yu Baosheng, Maybank S J, et al. Knowledge distillation: A survey[J]. International Journal of Computer Vision, 2021, 129(6): 1789−1819, articleTitle=null, refAbstract=null), Reference(id=1325490824712167612, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1325490414429545387, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=23, rfOrder=22, authorNames=null, journalName=null, refType=null, unstructuredReference=Wang Lin, Yoon K J. Knowledge distillation and student-teacher learning for visual intelligence: A review and new outlooks[J]. IEEE Transactions on Pattern Analysis and Machine Intelligence, 2022, 44(6): 3048−3068, articleTitle=null, refAbstract=null), Reference(id=1325490827589460158, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1325490414429545387, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=24, rfOrder=23, authorNames=null, journalName=null, refType=null, unstructuredReference=黄震华, 杨顺志, 林威, 等. 知识蒸馏研究综述[J]. 计算机学报, 2022, 45(3): 624−653, articleTitle=null, refAbstract=null), Reference(id=1325490831314002111, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1325490414429545387, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=24, rfOrder=24, authorNames=null, journalName=null, refType=null, unstructuredReference=Huang Zhenhua, Yang Shunzhi, Lin Wei, et al. Knowledge distillation: A survey[J]. Chinese Journal of Computers, 2022, 45(3): 624−653 (in Chinese), articleTitle=null, refAbstract=null), Reference(id=1325490833503428801, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1325490414429545387, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=25, rfOrder=25, authorNames=null, journalName=null, refType=null, unstructuredReference=邵仁荣, 刘宇昂, 张伟, 等. 深度学习中知识蒸馏研究综述[J]. 计算机学报, 2022, 45(8): 1638−1673, articleTitle=null, refAbstract=null), Reference(id=1325490835130818755, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1325490414429545387, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=25, rfOrder=26, authorNames=null, journalName=null, refType=null, unstructuredReference=Shao Renrong, Liu Yuang, Zhang Wei, et al. A survey of knowledge distillation in deep learning[J]. Chinese Journal of Computers, 2022, 45(8): 1638−1673 (in Chinese), articleTitle=null, refAbstract=null), Reference(id=1325490836863066310, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1325490414429545387, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=26, rfOrder=27, authorNames=null, journalName=null, refType=null, unstructuredReference=司兆峰, 齐洪钢. 知识蒸馏方法研究与应用综述[J]. 中国图象图形学报, 2023, 28(9): 2817−2832, articleTitle=null, refAbstract=null), Reference(id=1325490838914080968, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1325490414429545387, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=26, rfOrder=28, authorNames=null, journalName=null, refType=null, unstructuredReference=Si Zhaofeng, Qi Honggang. Survey on knowledge distillation and its application[J]. Journal of Image and Graphics, 2023, 28(9): 2817−2832 (in Chinese), articleTitle=null, refAbstract=null), Reference(id=1325490840575025353, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1325490414429545387, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=27, rfOrder=29, authorNames=null, journalName=null, refType=null, unstructuredReference=Oord A, Li Yazhe, Vinyals O. Representation learning with contrastive predictive coding[J]. arXiv preprint, arXiv: 1807.03748, 2019, articleTitle=null, refAbstract=null), Reference(id=1325490843989188811, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1325490414429545387, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=28, rfOrder=30, authorNames=null, journalName=null, refType=null, unstructuredReference=Bucila C, Caruana R, Niculescu-Mizil A. Model compression[C]//Proc of the 12th ACM SIGKDD Int Conf on Knowledge Discovery and Data Mining. New York: ACM, 2006: 535−541, articleTitle=null, refAbstract=null), Reference(id=1325490846447050956, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1325490414429545387, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=29, rfOrder=31, authorNames=null, journalName=null, refType=null, unstructuredReference=Sau B B, Balasubramanian V N. Deep model compression: Distilling knowledge from noisy teachers[J]. arXiv preprint, arXiv: 1610.09650, 2016, articleTitle=null, refAbstract=null), Reference(id=1325490848598728910, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1325490414429545387, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=30, rfOrder=32, authorNames=null, journalName=null, refType=null, unstructuredReference=Zhang Ying, Xiang Tao, Hospedales T M, et al. Deep mutual learning[C]//Proc of 2018 IEEE/CVF Conf on Computer Vision and Pattern Recognition. Piscataway, NJ: IEEE, 2018: 4320−4328, articleTitle=null, refAbstract=null), Reference(id=1325490851027230927, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1325490414429545387, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=31, rfOrder=33, authorNames=null, journalName=null, refType=null, unstructuredReference=Furlanello T, Lipton Z, Tschannen M, et al. Born again neural networks[C]//Proc of the 35th Int Conf on Machine Learning. New York: ACM, 2018: 1607−1616, articleTitle=null, refAbstract=null), Reference(id=1325490852361019601, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1325490414429545387, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=32, rfOrder=34, authorNames=null, journalName=null, refType=null, unstructuredReference=Yang Chenglin, Xie Lingxi, Qiao Siyuan, et al. Training deep neural networks in generations: A more tolerant teacher educates better students[C]//Proc of the 33rd AAAI Conf on Artificial Intelligence. Palo Alto, CA: AAAI, 2019: 5628−5635, articleTitle=null, refAbstract=null), Reference(id=1325490853963243730, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1325490414429545387, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=33, rfOrder=35, authorNames=null, journalName=null, refType=null, unstructuredReference=Lee H, Hwang S J, Shin J. Self-supervised label augmentation via input transformations[C]//Proc of the 37th Int Conf on Machine Learning. New York: ACM, 2020: 5714−5724, articleTitle=null, refAbstract=null), Reference(id=1325490856945393876, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1325490414429545387, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=34, rfOrder=36, authorNames=null, journalName=null, refType=null, unstructuredReference=Yang Jing, Martinez B, Bulat A, et al. Knowledge distillation via softmax regression representation learning[C/OL]//Proc of 9th Int Conf on Learning Representations, New York: Ithaca, 2020[2024-08-20]. https://openreview.net/forum?id=ZzwDy_wiWv, articleTitle=null, refAbstract=null), Reference(id=1325490859482947797, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1325490414429545387, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=35, rfOrder=37, authorNames=null, journalName=null, refType=null, unstructuredReference=Ba J, Caruana R. Do deep nets really need to be deep?[C]//Proc of the 28th Annual Conf on Neural Information Processing Systems. Cambridge, MA: MIT, 2014: 2654−2662, articleTitle=null, refAbstract=null), Reference(id=1325490861131309272, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1325490414429545387, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=36, rfOrder=38, authorNames=null, journalName=null, refType=null, unstructuredReference=Zhou Guorui, Fan Ying, Cui Runpeng, et al. Rocket launching: A universal and efficient framework for training well-performing light net[C]//Proc of the 32nd AAAI Conf on Artificial Intelligence. Palo Alto, CA: AAAI, 2018: 4580−4587, articleTitle=null, refAbstract=null), Reference(id=1325490864449003737, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1325490414429545387, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=37, rfOrder=39, authorNames=null, journalName=null, refType=null, unstructuredReference=Son W, Na J, Choi J, et al. Densely guided knowledge distillation using multiple teacher assistants[C]//Proc of 2021 IEEE/CVF Int Conf on Computer Vision. Piscataway, NJ: IEEE, 2021: 9375−9384, articleTitle=null, refAbstract=null), Reference(id=1325490866198028508, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1325490414429545387, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=38, rfOrder=40, authorNames=null, journalName=null, refType=null, unstructuredReference=Mirzadeh S I, Farajtabar M, Li Ang, et al. Improved knowledge distillation via teacher assistant[C]//Proc of the 34th AAAI Conf on Artificial Intelligence. Palo Alto, CA: AAAI, 2020: 5191−5198, articleTitle=null, refAbstract=null), Reference(id=1325490868228071645, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1325490414429545387, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=39, rfOrder=41, authorNames=null, journalName=null, refType=null, unstructuredReference=Li Xuewei, Li Songyuan, Omar B, et al. ResKD: Residual-guided knowledge distillation[J]. IEEE Transactions on Image Processing, 2021, 30: 4735−4746, articleTitle=null, refAbstract=null), Reference(id=1325490870413304031, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1325490414429545387, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=40, rfOrder=42, authorNames=null, journalName=null, refType=null, unstructuredReference=Romero A, Ballas N, Kahou S E, et al. FitNets: Hints for thin deep nets[J]. arXiv preprint, arXiv: 1412.6550, 2015, articleTitle=null, refAbstract=null), Reference(id=1325490871705149664, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1325490414429545387, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=41, rfOrder=43, authorNames=null, journalName=null, refType=null, unstructuredReference=Zagoruyko S, Komodakis N. Paying more attention to attention: Improving the performance of convolutional neural networks via attention transfer[J]. arXiv preprint, arXiv: 1612.03928, 2017, articleTitle=null, refAbstract=null), Reference(id=1325490873340928227, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1325490414429545387, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=42, rfOrder=44, authorNames=null, journalName=null, refType=null, unstructuredReference=Lin Sihao, Xie Hongwei, Wang Bing, et al. Knowledge distillation via the target-aware transformer[C]//Proc of 2022 IEEE/CVF Conf on Computer Vision and Pattern Recognition. Piscataway, NJ: IEEE, 2022: 10915−10924, articleTitle=null, refAbstract=null), Reference(id=1325490875412914404, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1325490414429545387, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=43, rfOrder=45, authorNames=null, journalName=null, refType=null, unstructuredReference=Lao Shanshan, Song Guanglu, Liu Boxiao, et al. Masked autoencoders are stronger knowledge distillers[C]//Proc of 2023 IEEE/CVF Int Conf on Computer Vision. Piscataway, NJ: IEEE, 2023: 6361−6370, articleTitle=null, refAbstract=null), Reference(id=1325490877505872104, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1325490414429545387, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=44, rfOrder=46, authorNames=null, journalName=null, refType=null, unstructuredReference=Guo Ziyao, Yan Haonan, Li Hui, et al. Class attention transfer based knowledge distillation[C]//Proc of 2023 IEEE/CVF Conf on Computer Vision and Pattern Recognition. Piscataway, NJ: IEEE, 2023: 11868−11877, articleTitle=null, refAbstract=null), Reference(id=1325490879150039273, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1325490414429545387, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=45, rfOrder=47, authorNames=null, journalName=null, refType=null, unstructuredReference=Zhang Yuan, Huang Tao, Liu Jiaming, et al. FreeKD: Knowledge distillation via semantic frequency prompt[C]//Proc of 2024 IEEE/CVF Conf on Computer Vision and Pattern Recognition. Piscataway, NJ: IEEE, 2024: 15931−15940, articleTitle=null, refAbstract=null), Reference(id=1325490881675010284, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1325490414429545387, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=46, rfOrder=48, authorNames=null, journalName=null, refType=null, unstructuredReference=Jia Menglin, Tang Luming, Chen B, et al. Visual prompt tuning[C]//Proc of the 17th European Conf on Computer Vision. Berlin: Springer, 2022: 709−727, articleTitle=null, refAbstract=null), Reference(id=1325490884233535725, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1325490414429545387, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=47, rfOrder=49, authorNames=null, journalName=null, refType=null, unstructuredReference=Huang Zehao, Wang Naiyan. Like what you like: Knowledge distill via neuron selectivity transfer[J]. arXiv preprint, arXiv: 1707.01219, 2017, articleTitle=null, refAbstract=null), Reference(id=1325490886724952303, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1325490414429545387, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=48, rfOrder=50, authorNames=null, journalName=null, refType=null, unstructuredReference=Heo B, Lee M, Yun S, et al. Knowledge transfer via distillation of activation boundaries formed by hidden neurons[C]//Proc of the 33rd AAAI Conf on Artificial Intelligence. Palo Alto, CA: AAAI, 2019: 3779−3787, articleTitle=null, refAbstract=null), Reference(id=1325490888364925171, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1325490414429545387, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=49, rfOrder=51, authorNames=null, journalName=null, refType=null, unstructuredReference=Shen Chengchao, Wang Xinchao, Song Jie, et al. Amalgamating knowledge towards comprehensive classification[C]//Proc of the 33rd AAAI Conf on Artificial Intelligence. Palo Alto, CA: AAAI, 2019: 3068−3075, articleTitle=null, refAbstract=null), Reference(id=1325490890126532852, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1325490414429545387, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=50, rfOrder=52, authorNames=null, journalName=null, refType=null, unstructuredReference=Wei Yi, Pan Xinyu, Qin Hongwei, et al. Quantization mimic: Towards very tiny CNN for object detection[C]//Proc of the 15th European Conf on Computer Vision. Berlin: Springer, 2018: 274−290, articleTitle=null, refAbstract=null), Reference(id=1325490892626338037, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1325490414429545387, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=51, rfOrder=53, authorNames=null, journalName=null, refType=null, unstructuredReference=Ahn S, Hu S X, Damianou A, et al. Variational information distillation for knowledge transfer[C]//Proc of 2019 IEEE/CVF Conf on Computer Vision and Pattern Recognition. Piscataway, NJ: IEEE, 2019: 9155−9163, articleTitle=null, refAbstract=null), Reference(id=1325490894727684342, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1325490414429545387, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=52, rfOrder=54, authorNames=null, journalName=null, refType=null, unstructuredReference=Heo B, Kim J, Yun S, et al. A comprehensive overhaul of feature distillation[C]//Proc of 2019 IEEE/CVF Int Conf on Computer Vision. Piscataway, NJ: IEEE, 2019: 1921−1930, articleTitle=null, refAbstract=null), Reference(id=1325490898427060472, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1325490414429545387, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=53, rfOrder=55, authorNames=null, journalName=null, refType=null, unstructuredReference=Yim J, Joo D, Bae J, et al. A gift from knowledge distillation: Fast optimization, network minimization and transfer learning[C]//Proc of 2017 IEEE/CVF Conf on Computer Vision and Pattern Recognition. Piscataway, NJ: IEEE, 2017: 7130−7138, articleTitle=null, refAbstract=null), Reference(id=1325490902696861945, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1325490414429545387, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=54, rfOrder=56, authorNames=null, journalName=null, refType=null, unstructuredReference=Srinivas S, Fleuret F. Knowledge transfer with Jacobian matching[C]//Proc of the 35th Int Conf on Machine Learning. New York: ACM, 2018: 4730−4738, articleTitle=null, refAbstract=null), Reference(id=1325490905993584890, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1325490414429545387, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=55, rfOrder=57, authorNames=null, journalName=null, refType=null, unstructuredReference=Kim J, Park S, Kwak N. Paraphrasing complex network: Network compression via factor transfer[C]//Proc of the 32nd Annual Conf on Neural Information Processing Systems. Cambridge, MA: MIT, 2018: 2765−2774, articleTitle=null, refAbstract=null), Reference(id=1325490907352539391, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1325490414429545387, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=56, rfOrder=58, authorNames=null, journalName=null, refType=null, unstructuredReference=Yang Chuanguang, An Zhulin, Cai Linhang, et al. Hierarchical self-supervised augmented knowledge distillation[C]//Proc of the 30th Int Joint Conf on Artificial Intelligence. San Francisco, CA: Morgan Kaufmann, 2021: 1217−1223, articleTitle=null, refAbstract=null), Reference(id=1325490910590542081, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1325490414429545387, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=57, rfOrder=59, authorNames=null, journalName=null, refType=null, unstructuredReference=Tian Yonglong, Krishnan D, Isola P. Contrastive representation distillation[J]. arXiv preprint, arXiv: 1910.10699, 2022, articleTitle=null, refAbstract=null), Reference(id=1325490913199399171, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1325490414429545387, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=58, rfOrder=60, authorNames=null, journalName=null, refType=null, unstructuredReference=Chen Defang, Mei Jian Ping, Zhang Hailin, et al. Knowledge distillation with the reused teacher classifier[C]//Proc of 2022 IEEE/CVF Conf on Computer Vision and Pattern Recognition. Piscataway, NJ: IEEE, 2022: 11923−11932, articleTitle=null, refAbstract=null), Reference(id=1325490915720175878, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1325490414429545387, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=59, rfOrder=61, authorNames=null, journalName=null, refType=null, unstructuredReference=Wang Xiaobo, Fu Tianyu, Liao Shengcai, et al. Exclusivity-consistency regularized knowledge distillation for face recognition[C]//Proc of the 16th European Conf on Computer Vision. Berlin: Springer, 2020: 325−342, articleTitle=null, refAbstract=null), Reference(id=1325490918555525384, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1325490414429545387, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=60, rfOrder=62, authorNames=null, journalName=null, refType=null, unstructuredReference=Chen Pengguang, Liu Shu, Zhao Hengshuang, et al. Distilling knowledge via knowledge review[C]//Proc of 2021 IEEE/CVF Conf on Computer Vision and Pattern Recognition. Piscataway, NJ: IEEE, 2021: 5006−5015, articleTitle=null, refAbstract=null), Reference(id=1325490920753340682, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1325490414429545387, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=61, rfOrder=63, authorNames=null, journalName=null, refType=null, unstructuredReference=You Shan, Xu Chang, Xu Chao, et al. Learning from multiple teacher networks[C]//Proc of the 23rd ACM SIGKDD Int Conf on Knowledge Discovery and Data Mining. New York: ACM, 2017: 1285−1294, articleTitle=null, refAbstract=null), Reference(id=1325490924519825676, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1325490414429545387, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=62, rfOrder=64, authorNames=null, journalName=null, refType=null, unstructuredReference=Zhang Chenrui, Peng Yuxin. Better and faster: Knowledge transfer from multiple self-supervised learning tasks via graph distillation for video classification[C]//Proc of the 27th Int Joint Conf on Artificial Intelligence. San Francisco, CA: Morgan Kaufmann, 2018: 1135−1141, articleTitle=null, refAbstract=null), Reference(id=1325490926235296014, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1325490414429545387, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=63, rfOrder=65, authorNames=null, journalName=null, refType=null, unstructuredReference=Chen Defang, Mei Jianping, Zhang Yuan, et al. Cross-layer distillation with semantic calibration[C]//Proc of the 35th AAAI Conf on Artificial Intelligence. Palo Alto, CA: AAAI, 2021: 7028−7036, articleTitle=null, refAbstract=null), Reference(id=1325490928366002449, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1325490414429545387, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=64, rfOrder=66, authorNames=null, journalName=null, refType=null, unstructuredReference=Liu Li, Huang Qingle, Lin Sihao, et al. Exploring inter-channel correlation for diversity-preserved knowledge distillation[C]//Proc of 2021 IEEE/CVF Int Conf on Computer Vision. Piscataway, NJ: IEEE, 2021: 8251−8260, articleTitle=null, refAbstract=null), Reference(id=1325490929989198098, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1325490414429545387, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=65, rfOrder=67, authorNames=null, journalName=null, refType=null, unstructuredReference=Jin Ying, Wang Jiaqi, Lin Dahua. Multi-level logit distillation[C]//Proc of 2023 IEEE/CVF Conf on Computer Vision and Pattern Recognition. Piscataway, NJ: IEEE, 2023: 24276−24285, articleTitle=null, refAbstract=null), Reference(id=1325490933235589396, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1325490414429545387, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=66, rfOrder=68, authorNames=null, journalName=null, refType=null, unstructuredReference=Chen Yuntao, Wang Naiyan, Zhang Zhaoxiang. DarkRank: Accelerating deep metric learning via cross sample similarities transfer[C]//Proc of the 32nd AAAI Conf on Artificial Intelligence. Palo Alto, CA: AAAI, 2018: 2852−2859, articleTitle=null, refAbstract=null), Reference(id=1325490935383073046, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1325490414429545387, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=67, rfOrder=69, authorNames=null, journalName=null, refType=null, unstructuredReference=Liu Yufan, Cao Jiajiong, Li Bing, et al. Knowledge distillation via instance relationship graph[C]//Proc of 2019 IEEE/CVF Conf on Computer Vision and Pattern Recognition. Piscataway, NJ: IEEE, 2019: 7089−7097, articleTitle=null, refAbstract=null), Reference(id=1325490938277142808, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1325490414429545387, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=68, rfOrder=70, authorNames=null, journalName=null, refType=null, unstructuredReference=Peng Baoyun, Jin Xiao, Li Dongsheng, et al. Correlation congruence for knowledge distillation[C]//Proc of 2019 IEEE/CVF Int Conf on Computer Vision. Piscataway, NJ: IEEE, 2019: 5006−5015, articleTitle=null, refAbstract=null), Reference(id=1325490940869222681, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1325490414429545387, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=69, rfOrder=71, authorNames=null, journalName=null, refType=null, unstructuredReference=Tung F, Mori G. Similarity-preserving knowledge distillation[C]//Proc of 2019 IEEE/CVF Int Conf on Computer Vision. Piscataway, NJ: IEEE, 2019: 1365−1374, articleTitle=null, refAbstract=null), Reference(id=1325490944228860187, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1325490414429545387, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=70, rfOrder=72, authorNames=null, journalName=null, refType=null, unstructuredReference=Xu Haohang, Fang Jiemin, Zhang Xiaopeng, et al. Bag of instances aggregation boosts self-supervised distillation[J]. arXiv preprint, arXiv: 2107.01691, 2022, articleTitle=null, refAbstract=null), Reference(id=1325490947076792604, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1325490414429545387, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=71, rfOrder=73, authorNames=null, journalName=null, refType=null, unstructuredReference=刘静, 郑铜亚, 郝沁汾. 图知识蒸馏综述: 算法分类与应用分析[J]. 软件学报, 2023, 35(2): 675−710, articleTitle=null, refAbstract=null), Reference(id=1325490949652095262, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1325490414429545387, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=71, rfOrder=74, authorNames=null, journalName=null, refType=null, unstructuredReference=Liu Jing, Zheng Tongya, Hao Qinfen. Survey on knowledge distillation with graph: Algorithms classification and application analysis[J]. Journal of Software, 2023, 35(2): 675−710 (in Chinese), articleTitle=null, refAbstract=null), Reference(id=1325490951724081439, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1325490414429545387, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=72, rfOrder=75, authorNames=null, journalName=null, refType=null, unstructuredReference=Park W, Kim D, Lu Yan, et al. Relational knowledge distillation[C]//Proc of 2019 IEEE/CVF Conf on Computer Vision and Pattern Recognition. Piscataway, NJ: IEEE, 2019: 3967−3976, articleTitle=null, refAbstract=null), Reference(id=1325490953020121376, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1325490414429545387, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=73, rfOrder=76, authorNames=null, journalName=null, refType=null, unstructuredReference=Ren Sucheng, Gao Zhengqi, Hua Tianyu, et al. Co-advise: Cross inductive bias distillation[C]//Proc of 2022 IEEE/CVF Conf on Computer Vision and Pattern Recognition. Piscataway, NJ: IEEE, 2022: 16752−16761, articleTitle=null, refAbstract=null), Reference(id=1325490955939356962, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1325490414429545387, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=74, rfOrder=77, authorNames=null, journalName=null, refType=null, unstructuredReference=Li Duo, Hu Jie, Wang Changhu, et al. Involution: Inverting the inherence of convolution for visual recognition[C]//Proc of 2021 IEEE/CVF Conf on Computer Vision and Pattern Recognition. Piscataway, NJ: IEEE, 2021: 12316−12325, articleTitle=null, refAbstract=null), Reference(id=1325490957642244388, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1325490414429545387, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=75, rfOrder=78, authorNames=null, journalName=null, refType=null, unstructuredReference=Zhao Borui, Song Renjie, Liang Jiajun. Cumulative spatial knowledge distillation for vision transformers[C]//Proc of 2023 IEEE/CVF Int Conf on Computer Vision. Piscataway, NJ: IEEE, 2023: 6123−6132, articleTitle=null, refAbstract=null), Reference(id=1325490959668093222, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1325490414429545387, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=76, rfOrder=79, authorNames=null, journalName=null, refType=null, unstructuredReference=Liu Yufan, Cao Jiajiong, Li Bing, et al. Cross-architecture knowledge distillation[C]//Proc of the 16th Asian Conf on Computer Vision. Berlin: Springer, 2022: 179−195, articleTitle=null, refAbstract=null), Reference(id=1325490961375174951, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1325490414429545387, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=77, rfOrder=80, authorNames=null, journalName=null, refType=null, unstructuredReference=Zhao Weisong, Zhu Xiangyu, He Zhixiang, et al. Cross-architecture distillation for face recognition[C]//Proc of the 31st ACM Int Conf on Multimedia. New York: ACM, 2023: 8076−8085, articleTitle=null, refAbstract=null), Reference(id=1325490965133271337, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1325490414429545387, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=78, rfOrder=81, authorNames=null, journalName=null, refType=null, unstructuredReference=Hao Zhiwei, Guo Jianyuan, Han Kai, et al. One-for-all: Bridge the gap between heterogeneous architectures in knowledge distillation[C]//Proc of the 37th Annual Conf on Neural Information Processing Systems. Cambridge, MA: MIT, 2023: 79570−79582, articleTitle=null, refAbstract=null), Reference(id=1325490967729545515, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1325490414429545387, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=79, rfOrder=82, authorNames=null, journalName=null, refType=null, unstructuredReference=Li Guopeng, Wang Qiang, Yan Ke, et al. Fuse before transfer: Knowledge fusion for heterogeneous distillation[C]// Proc of 2025 IEEE/CVF Int Conf on Computer Vision. Piscataway, NJ: IEEE, 2025: 3445-3454, articleTitle=null, refAbstract=null), Reference(id=1325490970262905132, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1325490414429545387, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=80, rfOrder=83, authorNames=null, journalName=null, refType=null, unstructuredReference=Ni Jianyuan, Tang Hao, Shang Yuzhang, et al. Adaptive cross-architecture mutual knowledge distillation[C]//Proc of the 18th Int Conf on Automatic Face and Gesture Recognition. Piscataway, NJ: IEEE, 2024: 1−5, articleTitle=null, refAbstract=null), Reference(id=1325490974851473710, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1325490414429545387, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=81, rfOrder=84, authorNames=null, journalName=null, refType=null, unstructuredReference=Zheng Zhimeng, Huang Tao, Li Gongsheng, et al. Promoting CNNs with cross-architecture knowledge distillation for efficient monocular depth estimation[J]. arXiv preprint, arXiv: 2404.16386, 2024, articleTitle=null, refAbstract=null), Reference(id=1325490978974474543, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1325490414429545387, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=82, rfOrder=85, authorNames=null, journalName=null, refType=null, unstructuredReference=Wu Kan, Zhang Jinnian, Peng Houwen, et al. TinyViT: Fast pretraining distillation for small vision transformers[C]//Proc of the 17th European Conf on Computer Vision. Berlin: Springer, 2022: 68−85, articleTitle=null, refAbstract=null), Reference(id=1325490981117763889, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1325490414429545387, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=83, rfOrder=86, authorNames=null, journalName=null, refType=null, unstructuredReference=Zhang Jinnian, Peng Houwen, Wu Kan, et al. MiniViT: Compressing vision transformers with weight multiplexing[C]//Proc of 2022 IEEE/CVF Conf on Computer Vision and Pattern Recognition. Piscataway, NJ: IEEE, 2022: 12135−12144, articleTitle=null, refAbstract=null), Reference(id=1325490983223304498, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1325490414429545387, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=84, rfOrder=87, authorNames=null, journalName=null, refType=null, unstructuredReference=Yang Zhendong, Li Zhe, Zeng Ailing, et al. ViTKD: Practical guidelines for ViT feature knowledge distillation[J]. arXiv preprint, arXiv: 2209.02432, 2022, articleTitle=null, refAbstract=null), Reference(id=1325490987023343926, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1325490414429545387, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=85, rfOrder=88, authorNames=null, journalName=null, refType=null, unstructuredReference=Zhao Tianyang, Singh K Y, Appalaraju S, et al. No head left behind–multi-head alignment distillation for transformers[C]//Proc of the 38th AAAI Conf on Artificial Intelligence. Palo Alto, CA: AAAI, 2024: 7514−7524, articleTitle=null, refAbstract=null), Reference(id=1325490989896442167, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1325490414429545387, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=86, rfOrder=89, authorNames=null, journalName=null, refType=null, unstructuredReference=Ren Sucheng, Wei Fangyun, Zhang Zheng, et al. TinyMIM: An empirical study of distilling MIM pre-trained models[C]//Proc of 2023 IEEE/CVF Conf on Computer Vision and Pattern Recognition. Piscataway, NJ: IEEE, 2023: 3687−3697, articleTitle=null, refAbstract=null), Reference(id=1325490992811483449, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1325490414429545387, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=87, rfOrder=90, authorNames=null, journalName=null, refType=null, unstructuredReference=Lin Han, Han Guangxing, Ma Jiawei, et al. Supervised masked knowledge distillation for few-shot transformers[C]//Proc of 2023 IEEE/CVF Conf on Computer Vision and Pattern Recognition. Piscataway, NJ: IEEE, 2023: 19649−19659, articleTitle=null, refAbstract=null), Reference(id=1325490997060313402, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1325490414429545387, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=88, rfOrder=91, authorNames=null, journalName=null, refType=null, unstructuredReference=Bai Yutong, Wang Zeyu, Xiao Junfei, et al. Masked autoencoders enable efficient knowledge distillers[C]//Proc of 2023 IEEE/CVF Conf on Computer Vision and Pattern Recognition. Piscataway, NJ: IEEE, 2023: 24256−24265, articleTitle=null, refAbstract=null), Reference(id=1325491001632104765, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1325490414429545387, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=89, rfOrder=92, authorNames=null, journalName=null, refType=null, unstructuredReference=He Kaiming, Chen Xinlei, Xie Saining, et al. Masked autoencoders are scalable vision learners[C]//Proc of 2022 IEEE/CVF Conf on Computer Vision and Pattern Recognition. Piscataway, NJ: IEEE, 2022: 16000−16009, articleTitle=null, refAbstract=null), Reference(id=1325491004538757441, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1325490414429545387, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=90, rfOrder=93, authorNames=null, journalName=null, refType=null, unstructuredReference=Zhao Zhiyu, Huang Bingkun, Xing Sen, et al. Asymmetric masked distillation for pre-training small foundation models[C]//Proc of 2024 IEEE/CVF Conf on Computer Vision and Pattern Recognition. Piscataway, NJ: IEEE, 2024: 18516−18526, articleTitle=null, refAbstract=null), Reference(id=1325491005499253059, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1325490414429545387, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=91, rfOrder=94, authorNames=null, journalName=null, refType=null, unstructuredReference=Ye Hanjia, Lu Su, Zhan Dechuan. Distilling cross-task knowledge via relationship matching[C]//Proc of 2020 IEEE/CVF Conf on Computer Vision and Pattern Recognition. Piscataway, NJ: IEEE, 2020: 12396−12405, articleTitle=null, refAbstract=null), Reference(id=1325491006677852484, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1325490414429545387, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=92, rfOrder=95, authorNames=null, journalName=null, refType=null, unstructuredReference=Lê H Â, Pham M T. Self-training and multi-task learning for limited data: Evaluation study on object detection[C]//Proc of 2019 IEEE/CVF Int Conf on Computer Vision. Piscataway, NJ: IEEE, 2023: 1003−1009, articleTitle=null, refAbstract=null), Reference(id=1325491009219600709, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1325490414429545387, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=93, rfOrder=96, authorNames=null, journalName=null, refType=null, unstructuredReference=Li Deng, Wu Aming, Han Yahong, et al. Prototype-guided cross-task knowledge distillation for large-scale models[J]. arXiv preprint, arXiv: 2212.13180, 2022, articleTitle=null, refAbstract=null), Reference(id=1325491011283198279, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1325490414429545387, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=94, rfOrder=97, authorNames=null, journalName=null, refType=null, unstructuredReference=Auty D, Miles R, Kolbeinsson B, et al. Learning to project for cross-task knowledge distillation[J]. arXiv preprint, arXiv: 2403.14494, 2024, articleTitle=null, refAbstract=null), Reference(id=1325491014282125640, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1325490414429545387, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=95, rfOrder=98, authorNames=null, journalName=null, refType=null, unstructuredReference=Yuan Mingkuan, Peng Yuxin. CKD: Cross-task knowledge distillation for text-to-image synthesis[J]. IEEE Transactions on Multimedia, 2020, 22(8): 1955−1968, articleTitle=null, refAbstract=null), Reference(id=1325491016714821962, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1325490414429545387, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=96, rfOrder=99, authorNames=null, journalName=null, refType=null, unstructuredReference=Bayramli B, Ding Yue, Lu Hongtao. Integrating semantic segmentation model for self-supervised scene flow estimation via cross task distillation[C]//Proc of 2024 Int Joint Conf on Neural Networks. Piscataway, NJ: IEEE, 2024: 1−8, articleTitle=null, refAbstract=null), Reference(id=1325491019432730955, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1325490414429545387, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=97, rfOrder=100, authorNames=null, journalName=null, refType=null, unstructuredReference=Wang Lin, Chae Y, Yoon S H, et al. EvDistill: Asynchronous events to end-task learning via bidirectional reconstruction-guided cross-modal knowledge distillation[C]//Proc of 2021 IEEE/CVF Conf on Computer Vision and Pattern Recognition. Piscataway, NJ: IEEE, 2021: 608−619, articleTitle=null, refAbstract=null), Reference(id=1325491020586164557, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1325490414429545387, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=98, rfOrder=101, authorNames=null, journalName=null, refType=null, unstructuredReference=Qian Biao, Wang Yang, Yin Hongzhi, et al. Switchable online knowledge distillation[C]//Proc of the 17th European Conf on Computer Vision. Berlin: Springer, 2022: 449−466, articleTitle=null, refAbstract=null), Reference(id=1325491023023055183, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1325490414429545387, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=99, rfOrder=102, authorNames=null, journalName=null, refType=null, unstructuredReference=Yang Chuanguang, An Zhulin, Zhou Helong, et al. Online knowledge distillation via mutual contrastive learning for visual recognition[J]. IEEE Transactions on Pattern Analysis and Machine Intelligence, 2023, 45(8): 10212−10227, articleTitle=null, refAbstract=null), Reference(id=1325491025510277456, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1325490414429545387, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=100, rfOrder=103, authorNames=null, journalName=null, refType=null, unstructuredReference=Jacob G M, Agarwal V, Stenger B. Online knowledge distillation for multi-task learning[C]//Proc of 2023 IEEE/CVF Winter Conf on Applications of Computer Vision. Piscataway, NJ: IEEE, 2023: 2358−2367, articleTitle=null, refAbstract=null), Reference(id=1325491028119134546, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1325490414429545387, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=101, rfOrder=104, authorNames=null, journalName=null, refType=null, unstructuredReference=Rao Jun, Meng Xv, Ding Liang, et al. Parameter-efficient and student-friendly knowledge distillation[J]. IEEE Transactions on Multimedia, 2024, 26: 4230−4241, articleTitle=null, refAbstract=null), Reference(id=1325491030144983379, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1325490414429545387, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=102, rfOrder=105, authorNames=null, journalName=null, refType=null, unstructuredReference=Anil R, Pereyra G, Passos A, et al. Large scale distributed neural network training through online distillation[J]. arXiv preprint, arXiv: 1804.03235, 2020, articleTitle=null, refAbstract=null), Reference(id=1325491031910785364, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1325490414429545387, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=103, rfOrder=106, authorNames=null, journalName=null, refType=null, unstructuredReference=Song Guocong, Chai Wei. Collaborative learning for deep neural networks[C]//Proc of the 32nd Annual Conf on Neural Information Processing Systems. Cambridge, MA: MIT, 2018: 1837−1846, articleTitle=null, refAbstract=null), Reference(id=1325491033500426582, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1325490414429545387, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=104, rfOrder=107, authorNames=null, journalName=null, refType=null, unstructuredReference=Chen Defang, Mei Jianping, Wang Can, et al. Online knowledge distillation with diverse peers[C]//Proc of the 34th AAAI Conf on Artificial Intelligence. Palo Alto, CA: AAAI, 2020: 3430−3437, articleTitle=null, refAbstract=null), Reference(id=1325491036184781143, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1325490414429545387, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=105, rfOrder=108, authorNames=null, journalName=null, refType=null, unstructuredReference=Gong Linrui, Lin Shaohui, Zhang Baochang, et al. Adaptive hierarchy-branch fusion for online knowledge distillation[C]//Proc of the 37th AAAI Conf on Artificial Intelligence. Palo Alto, CA: AAAI, 2023: 7731−7739, articleTitle=null, refAbstract=null), Reference(id=1325491038550368602, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1325490414429545387, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=106, rfOrder=109, authorNames=null, journalName=null, refType=null, unstructuredReference=Wu Guile, Gong Shaogang. Peer collaborative learning for online knowledge distillation[C]//Proc of the 35th AAAI Conf on Artificial Intelligence. Palo Alto, CA: AAAI, 2021: 10302−10310, articleTitle=null, refAbstract=null), Reference(id=1325491040706240860, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1325490414429545387, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=107, rfOrder=110, authorNames=null, journalName=null, refType=null, unstructuredReference=Li Shaojie, Lin Mingbao, Wang Yan, et al. Distilling a powerful student model via online knowledge distillation[J]. IEEE Transactions on Neural Networks and Learning Systems, 2023, 34(11): 8743−8752, articleTitle=null, refAbstract=null), Reference(id=1325491043549978973, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1325490414429545387, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=108, rfOrder=111, authorNames=null, journalName=null, refType=null, unstructuredReference=Zhang Linfeng, Song Jiebo, Gao Anni, et al. Be your own teacher: Improve the performance of convolutional neural networks via self distillation[C]//Proc of 2019 IEEE/CVF Int Conf on Computer Vision. Piscataway, NJ: IEEE, 2019: 3712−3721, articleTitle=null, refAbstract=null), Reference(id=1325491048167907679, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1325490414429545387, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=109, rfOrder=112, authorNames=null, journalName=null, refType=null, unstructuredReference=Lan Xu, Zhu Xiatian, Gong Shaogang. Self-referenced deep learning[C]//Proc of the 14th Asian Conf on Computer Vision. Berlin: Springer, 2018: 284−300, articleTitle=null, refAbstract=null), Reference(id=1325491050676101474, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1325490414429545387, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=110, rfOrder=113, authorNames=null, journalName=null, refType=null, unstructuredReference=Yang Chenglin, Xie Lingxi, Su Chi, et al. Snapshot distillation: Teacher-student optimization in one generation[C]//Proc of 2019 IEEE/CVF Conf on Computer Vision and Pattern Recognition. Piscataway, NJ: IEEE, 2019: 2854−2863, articleTitle=null, refAbstract=null), Reference(id=1325491054530666853, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1325490414429545387, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=111, rfOrder=114, authorNames=null, journalName=null, refType=null, unstructuredReference=Mobahi H, Farajtabar M, Bartlett P L. Self-distillation amplifies regularization in Hilbert space[C]//Proc of the 34th Annual Conf on Neural Information Processing Systems. Cambridge, MA: MIT, 2020: 3351−3361, articleTitle=null, refAbstract=null), Reference(id=1325491056652984678, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1325490414429545387, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=112, rfOrder=115, authorNames=null, journalName=null, refType=null, unstructuredReference=Shen Yiqing, Xu Liwu, Yang Yuzhe, et al. Self-distillation from the last mini-batch for consistency regularization[C]//Proc of 2022 IEEE/CVF Conf on Computer Vision and Pattern Recognition. Piscataway, NJ: IEEE, 2022: 11933−11942, articleTitle=null, refAbstract=null), Reference(id=1325491060771791211, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1325490414429545387, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=113, rfOrder=116, authorNames=null, journalName=null, refType=null, unstructuredReference=Gou Jianping, Xiong Xiangshuo, Yu Baosheng, et al. Multi-target knowledge distillation via student self-reflection[J]. International Journal of Computer Vision, 2023, 131(7): 1857−1874, articleTitle=null, refAbstract=null), Reference(id=1325491063695221100, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1325490414429545387, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=114, rfOrder=117, authorNames=null, journalName=null, refType=null, unstructuredReference=余鹰, 危伟, 汤洪, 等. 多层次知识自蒸馏联合多步骤训练的细粒度图像识别[J]. 计算机研究与发展, 2023, 60(8): 1834−1845, articleTitle=null, refAbstract=null), Reference(id=1325491068774523250, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1325490414429545387, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=114, rfOrder=118, authorNames=null, journalName=null, refType=null, unstructuredReference=Yu Ying, Wei Wei, Tang Hong, et al. Multi-stage training with multi-level knowledge self-distillation for fine-grained image recognition[J]. Journal of Computer Research and Development, 2023, 60(8): 1834−1845 (in Chinese), articleTitle=null, refAbstract=null), Reference(id=1325491072935272819, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1325490414429545387, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=115, rfOrder=119, authorNames=null, journalName=null, refType=null, unstructuredReference=Lopes R G, Fenu S, Starner T. Data-free knowledge distillation for deep neural networks[J]. arXiv preprint, arXiv: 1710.07535, 2017, articleTitle=null, refAbstract=null), Reference(id=1325491076697563510, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1325490414429545387, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=116, rfOrder=120, authorNames=null, journalName=null, refType=null, unstructuredReference=Nayak G K, Mopuri K R, Shaj V, et al. Zero-shot knowledge distillation in deep networks[C]//Proc of the 36th Int Conf on Machine Learning. New York: ACM, 2019: 4743−4751, articleTitle=null, refAbstract=null), Reference(id=1325491078874407287, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1325490414429545387, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=117, rfOrder=121, authorNames=null, journalName=null, refType=null, unstructuredReference=Fang Gongfan, Mo Kanya, Wang Xinchao, et al. Up to 100× faster data-free knowledge distillation[C]//Proc of the 36th AAAI Conf on Artificial Intelligence. Palo Alto, CA: AAAI, 2022: 6597−6604, articleTitle=null, refAbstract=null), Reference(id=1325491082166935928, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1325490414429545387, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=118, rfOrder=122, authorNames=null, journalName=null, refType=null, unstructuredReference=Yu Shikang, Chen Jiachen, Han Hu, et al. Data-free knowledge distillation via feature exchange and activation region constraint[C]//Proc of 2023 IEEE/CVF Conf on Computer Vision and Pattern Recognition. Piscataway, NJ: IEEE, 2023: 24266−24275, articleTitle=null, refAbstract=null), Reference(id=1325491085505601916, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1325490414429545387, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=119, rfOrder=123, authorNames=null, journalName=null, refType=null, unstructuredReference=Li Jingru, Zhou Sheng, Li Liangcheng, et al. Dynamic data-free knowledge distillation by easy-to-hard learning strategy[J]. Information Sciences, 2023, 642: 119202, articleTitle=null, refAbstract=null), Reference(id=1325491087187517822, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1325490414429545387, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=120, rfOrder=124, authorNames=null, journalName=null, refType=null, unstructuredReference=Wang Yang, Qian Biao, Liu Haipeng, et al. Unpacking the gap box against data-free knowledge distillation[J]. IEEE Transactions on Pattern Analysis and Machine Intelligence, 2024, 46(9): 6280−6291, articleTitle=null, refAbstract=null), Reference(id=1325491089817346432, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1325490414429545387, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=121, rfOrder=125, authorNames=null, journalName=null, refType=null, unstructuredReference=张晶, 鞠佳良, 任永功. 基于双生成器网络的Data-Free知识蒸馏[J]. 计算机研究与发展, 2023, 60(7): 1615−1627, articleTitle=null, refAbstract=null), Reference(id=1325491091469902210, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1325490414429545387, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=121, rfOrder=126, authorNames=null, journalName=null, refType=null, unstructuredReference=Zhang Jing, Ju Jialiang, Ren Yonggong. Double-generators network for data-free knowledge distillation[J]. Journal of Computer Research and Development, 2023, 60(7): 1615−1627 (in Chinese), articleTitle=null, refAbstract=null), Reference(id=1325491094829539716, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1325490414429545387, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=122, rfOrder=127, authorNames=null, journalName=null, refType=null, unstructuredReference=Chen Hanting, Wang Yunhe, Xu Chang, et al. Data-free learning of student networks[C]//Proc of 2019 IEEE/CVF Int Conf on Computer Vision. Piscataway, NJ: IEEE, 2019: 3513−3521, articleTitle=null, refAbstract=null), Reference(id=1325491096842805637, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1325490414429545387, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=123, rfOrder=128, authorNames=null, journalName=null, refType=null, unstructuredReference=Goodfellow I, Pouget-Abadie J, Mirza M, et al. Generative adversarial nets[C]//Proc of the 28th Annual Conf on Neural Information Processing Systems. Cambridge, MA: MIT, 2014: 2672−2680, articleTitle=null, refAbstract=null), Reference(id=1325491099749458311, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1325490414429545387, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=124, rfOrder=129, authorNames=null, journalName=null, refType=null, unstructuredReference=Fang Gongfan, Song Jie, Shen Chengchao, et al. Data-free adversarial distillation[J]. arXiv preprint, arXiv: 1912.11006, 2020, articleTitle=null, refAbstract=null), Reference(id=1325491102303789449, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1325490414429545387, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=125, rfOrder=130, authorNames=null, journalName=null, refType=null, unstructuredReference=Luo Liangchen, Sandler M, Lin Zi, et al. Large-scale generative data-free distillation[J]. arXiv preprint, arXiv: 2012.05578, 2020, articleTitle=null, refAbstract=null), Reference(id=1325491104455467404, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1325490414429545387, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=126, rfOrder=131, authorNames=null, journalName=null, refType=null, unstructuredReference=Qu Xiaoyang, Wang Jianzong, Xiao Jing. Enhancing data-free adversarial distillation with activation regularization and virtual interpolation[C]//Proc of 2021 IEEE Int Conf on Acoustics, Speech and Signal Processing. Piscataway, NJ: IEEE, 2021: 3340−3344, articleTitle=null, refAbstract=null), Reference(id=1325491108230341005, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1325490414429545387, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=127, rfOrder=132, authorNames=null, journalName=null, refType=null, unstructuredReference=Micaelli P, Storkey A J. Zero-shot knowledge transfer via adversarial belief matching[C]//Proc of the 33rd Annual Conf on Neural Information Processing Systems. Cambridge, MA: MIT, 2019: 9547−9557, articleTitle=null, refAbstract=null), Reference(id=1325491109492826512, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1325490414429545387, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=128, rfOrder=133, authorNames=null, journalName=null, refType=null, unstructuredReference=Do K, Le T H, Nguyen D, et al. Momentum adversarial distillation: Handling large distribution shifts in data-free knowledge distillation[C]//Proc of the 36th Annual Conf on Neural Information Processing Systems. Cambridge, MA: MIT, 2022: 10055−10067, articleTitle=null, refAbstract=null), Reference(id=1325491111996826001, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1325490414429545387, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=129, rfOrder=134, authorNames=null, journalName=null, refType=null, unstructuredReference=Zhao Haoran, Sun Xin, Dong Junyu, et al. Dual discriminator adversarial distillation for data-free model compression[J]. International Journal of Machine Learning and Cybernetics, 2021, 13(5): 1213−1230, articleTitle=null, refAbstract=null), Reference(id=1325491113657770386, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1325490414429545387, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=130, rfOrder=135, authorNames=null, journalName=null, refType=null, unstructuredReference=Shao Renrong, Zhang Wei, Wang Jun. Conditional pseudo-supervised contrast for data-free knowledge distillation[J]. Pattern Recognition, 2023, 143: 109781, articleTitle=null, refAbstract=null), Reference(id=1325491117365535123, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1325490414429545387, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=131, rfOrder=136, authorNames=null, journalName=null, refType=null, unstructuredReference=Patel G, Mopuri K R, Qiu Qiang. Learning to retain while acquiring: Combating distribution-shift in adversarial data-free knowledge distillation[C]//Proc of 2023 IEEE/CVF Conf on Computer Vision and Pattern Recognition. Piscataway, NJ: IEEE, 2023: 7786−7794, articleTitle=null, refAbstract=null), Reference(id=1325491120393822612, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1325490414429545387, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=132, rfOrder=137, authorNames=null, journalName=null, refType=null, unstructuredReference=Fukuda T, Suzuki M, Kurata G, et al. Efficient knowledge distillation from an ensemble of teachers[C]//Proc of the 18th Annual Conf of the Int Speech Communication Association. Grenoble, FR: ISCA, 2017: 3697−3701, articleTitle=null, refAbstract=null), Reference(id=1325491121673085333, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1325490414429545387, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=133, rfOrder=138, authorNames=null, journalName=null, refType=null, unstructuredReference=Tarvainen A, Valpola H. Mean teachers are better role models: Weight-averaged consistency targets improve semi-supervised deep learning results[C]//Proc of the 31st Annual Conf on Neural Information Processing Systems. Cambridge, MA: MIT, 2017: 1195−1204, articleTitle=null, refAbstract=null), Reference(id=1325491122931376534, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1325490414429545387, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=134, rfOrder=139, authorNames=null, journalName=null, refType=null, unstructuredReference=Xiang Liuyu, Ding Guiguang, Han Jungong. Learning from multiple experts: Self-paced knowledge distillation for long-tailed classification[C]//Proc of the 16th European Conf on Computer Vision. Berlin: Springer, 2020: 247−263, articleTitle=null, refAbstract=null), Reference(id=1325491125414404503, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1325490414429545387, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=135, rfOrder=140, authorNames=null, journalName=null, refType=null, unstructuredReference=Nguyen L T, Lee K, Shim B. Stochasticity and skip connection improve knowledge transfer[C]//Proc of the 28th European Signal Processing Conf. Piscataway, NJ: IEEE, 2021: 1537−1541, articleTitle=null, refAbstract=null), Reference(id=1325491127859683736, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1325490414429545387, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=136, rfOrder=141, authorNames=null, journalName=null, refType=null, unstructuredReference=Shi Luyao, Jiang Ning, Tang Jialiang, et al. Correlation guided multi-teacher knowledge distillation[C]//Proc of the 30th Int Conf on Neural Information Processing. Berlin: Springer, 2023: 562−574, articleTitle=null, refAbstract=null), Reference(id=1325491131236098457, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1325490414429545387, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=137, rfOrder=142, authorNames=null, journalName=null, refType=null, unstructuredReference=Pham C, Hoang T, Do T T. Collaborative multi-teacher knowledge distillation for learning low bit-width deep neural networks[C]//Proc of 2023 IEEE/CVF Winter Conf on Applications of Computer Vision. Piscataway, NJ: IEEE, 2023: 6424−6432, articleTitle=null, refAbstract=null), Reference(id=1325491133396165019, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1325490414429545387, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=138, rfOrder=143, authorNames=null, journalName=null, refType=null, unstructuredReference=Zhang Hailin, Chen Defang, Wang Can. Confidence-aware multi-teacher knowledge distillation[C]//Proc of 2022 IEEE Int Conf on Acoustics, Speech and Signal Processing. Piscataway, NJ: IEEE, 2022: 4498−4502, articleTitle=null, refAbstract=null), Reference(id=1325491135426208157, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1325490414429545387, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=139, rfOrder=144, authorNames=null, journalName=null, refType=null, unstructuredReference=Cheng Xin, Zhang Zhiqiang, Weng Wei, et al. DE-MKD: Decoupled multi-teacher knowledge distillation based on entropy[J]. Mathematics, 2024, 12(11): 1672, articleTitle=null, refAbstract=null), Reference(id=1325491137175232927, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1325490414429545387, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=140, rfOrder=145, authorNames=null, journalName=null, refType=null, unstructuredReference=Klingner M, Borse S, Kumar V R, et al. X3KD: Knowledge distillation across modalities, tasks and stages for multi-camera 3D object detection[C]//Proc of 2023 IEEE/CVF Conf on Computer Vision and Pattern Recognition. Piscataway, NJ: IEEE, 2023: 13343−13353, articleTitle=null, refAbstract=null), Reference(id=1325491138416746913, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1325490414429545387, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=141, rfOrder=146, authorNames=null, journalName=null, refType=null, unstructuredReference=Zhao Lingjun, Song Jingyu, Skinner Katherine A. CRKD: Enhanced camera-radar object detection with cross-modality knowledge distillation[C]//Proc of 2024 IEEE/CVF Conf on Computer Vision and Pattern Recognition. Piscataway, NJ: IEEE, 2024: 15470−15480, articleTitle=null, refAbstract=null), Reference(id=1325491139607929250, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1325490414429545387, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=142, rfOrder=147, authorNames=null, journalName=null, refType=null, unstructuredReference=Wang Yunhe, Xu Chang, Xu Chao, et al. Adversarial learning of portable student networks[C]//Proc of the 32nd AAAI Conf on Artificial Intelligence. Palo Alto, CA: AAAI, 2018: 4260−4267, articleTitle=null, refAbstract=null), Reference(id=1325491140509704611, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1325490414429545387, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=143, rfOrder=148, authorNames=null, journalName=null, refType=null, unstructuredReference=Passalis N, Tzelepi M, Tefas A. Heterogeneous knowledge distillation using information flow modeling[C]//Proc of 2020 IEEE/CVF Conf on Computer Vision and Pattern Recognition. Piscataway, NJ: IEEE, 2020: 2339−2348, articleTitle=null, refAbstract=null), Reference(id=1325491143076618662, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1325490414429545387, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=144, rfOrder=149, authorNames=null, journalName=null, refType=null, unstructuredReference=Ganta D P, Gupta H D, Sheng V S. Knowledge distillation via weighted ensemble of teaching assistants[C]//Proc of 2021 IEEE Int Conf on Big Knowledge. Piscataway, NJ: IEEE, 2021: 30−37, articleTitle=null, refAbstract=null), Reference(id=1325491144754340263, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1325490414429545387, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=145, rfOrder=150, authorNames=null, journalName=null, refType=null, unstructuredReference=Gao Mengya, Wang Yujun, Wan Liang. Residual error based knowledge distillation[J]. Neurocomputing, 2021, 433: 154−161, articleTitle=null, refAbstract=null), Reference(id=1325491148013314473, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1325490414429545387, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=146, rfOrder=151, authorNames=null, journalName=null, refType=null, unstructuredReference=Morikawa T, Kameyama K. Multi-stage model compression using teacher assistant and distillation with hint-based training[C]//Proc of 2022 IEEE Int Conf on Pervasive Computing and Communications Workshops and other Affiliated Events. Piscataway, NJ: IEEE, 2022: 484−490, articleTitle=null, refAbstract=null), Reference(id=1325491149749756331, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1325490414429545387, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=147, rfOrder=152, authorNames=null, journalName=null, refType=null, unstructuredReference=He Limuxuan, Hua Bei. Semi-supervised knowledge distillation via teaching assistant[J]. Highlights in Science, Engineering and Technology, 2023, 72: 429−436, articleTitle=null, refAbstract=null), Reference(id=1325491152186646956, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1325490414429545387, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=148, rfOrder=153, authorNames=null, journalName=null, refType=null, unstructuredReference=Zhao Lei, Ng W W Y, Zhang Jianjun. Distillation of ensemble adaptive teacher assistant knowledge[J]. International Journal of Machine Learning and Cybernetics, 2025, 16(10): 8063-8078, articleTitle=null, refAbstract=null), Reference(id=1325491154283798957, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1325490414429545387, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=149, rfOrder=154, authorNames=null, journalName=null, refType=null, unstructuredReference=Huo Fushuo, Xu Wenchao, Guo Jingcai, et al. C2KD: Bridging the modality gap for cross-modal knowledge distillation[C]//Proc of 2024 IEEE/CVF Conf on Computer Vision and Pattern Recognition. Piscataway, NJ: IEEE, 2024: 16006−16015, articleTitle=null, refAbstract=null), Reference(id=1325491156074766766, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1325490414429545387, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=150, rfOrder=155, authorNames=null, journalName=null, refType=null, unstructuredReference=Cho J H, Hariharan B. On the efficacy of knowledge distillation[C]//Proc of 2019 IEEE/CVF Int Conf on Computer Vision. Piscataway, NJ: IEEE, 2019: 4793−4801, articleTitle=null, refAbstract=null), Reference(id=1325491156901044655, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1325490414429545387, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=151, rfOrder=156, authorNames=null, journalName=null, refType=null, unstructuredReference=Yuan Li, Tay F E, Li Guilin, et al. Revisiting knowledge distillation via label smoothing regularization[C]//Proc of 2020 IEEE/CVF Conf on Computer Vision and Pattern Recognition. Piscataway, NJ: IEEE, 2020: 3902−3910, articleTitle=null, refAbstract=null), Reference(id=1325491158062866864, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1325490414429545387, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=152, rfOrder=157, authorNames=null, journalName=null, refType=null, unstructuredReference=Stanton S, Izmailov P, Kirichenko P, et al. Does knowledge distillation really work?[C]//Proc of the 35th Annual Conf on Neural Information Processing Systems. Cambridge, MA: MIT, 2021: 6906−6919, articleTitle=null, refAbstract=null), Reference(id=1325491160671723954, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1325490414429545387, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=153, rfOrder=158, authorNames=null, journalName=null, refType=null, unstructuredReference=He Ruifei, Sun Shuyang, Yang Jihan, et al. Knowledge distillation as efficient pre-training: Faster convergence, higher data-efficiency, and better transferability[C]//Proc of 2022 IEEE/CVF Conf on Computer Vision and Pattern Recognition. Piscataway, NJ: IEEE, 2022: 9151−9161, articleTitle=null, refAbstract=null), Reference(id=1325491161967763892, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1325490414429545387, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=154, rfOrder=159, authorNames=null, journalName=null, refType=null, unstructuredReference=Shen Zhiqiang, Xing E. A fast knowledge distillation framework for visual recognition[C]//Proc of the 17th European Conf on Computer Vision. Berlin: Springer, 2022: 673−690, articleTitle=null, refAbstract=null), Reference(id=1325491163989418421, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1325490414429545387, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=155, rfOrder=160, authorNames=null, journalName=null, refType=null, unstructuredReference=Song Jie, Chen Ying, Ye Jingwen, et al. Spot-adaptive knowledge distillation[J]. IEEE Transactions on Image Processing, 2022, 31: 3359−3370, articleTitle=null, refAbstract=null), Reference(id=1325491165688111542, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1325490414429545387, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=156, rfOrder=161, authorNames=null, journalName=null, refType=null, unstructuredReference=Zhang Tianli, Xue Mengqi, Zhang Jiangtao, et al. Generalization matters: Loss minima flattening via parameter hybridization for efficient online knowledge distillation[C]//Proc of 2023 IEEE/CVF Conf on Computer Vision and Pattern Recognition. Piscataway, NJ: IEEE, 2023: 20176−20185, articleTitle=null, refAbstract=null), Reference(id=1325491166967374264, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1325490414429545387, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=157, rfOrder=162, authorNames=null, journalName=null, refType=null, unstructuredReference=Wang Jiabao, Chen Yuming, Zheng Zhaohui, et al. CrossKD: Cross-head knowledge distillation for object detection[C]//Proc of 2024 IEEE/CVF Conf on Computer Vision and Pattern Recognition. Piscataway, NJ: IEEE, 2024: 16520−16530, articleTitle=null, refAbstract=null), Reference(id=1325491168137585081, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1325490414429545387, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=158, rfOrder=163, authorNames=null, journalName=null, refType=null, unstructuredReference=Xu Guodong, Liu Ziwei, Li Xiaoxiao, et al. Knowledge distillation meets self-supervision[C]//Proc of the 16th European Conf on Computer Vision. Berlin: Springer, 2020: 588−604, articleTitle=null, refAbstract=null), Reference(id=1325491169437819322, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1325490414429545387, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=159, rfOrder=164, authorNames=null, journalName=null, refType=null, unstructuredReference=Li Chenxin, Lin Mingbao, Ding Zhiyuan, et al. Knowledge condensation distillation[C]//Proc of the 17th European Conf on Computer Vision. Berlin: Springer, 2022: 19−35, articleTitle=null, refAbstract=null), Reference(id=1325491171077792187, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1325490414429545387, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=160, rfOrder=165, authorNames=null, journalName=null, refType=null, unstructuredReference=Huang Tao, You Shan, Wang Fei, et al. Knowledge distillation from a stronger teacher[C]//Proc of the 36th Annual Conference on Neural Information Processing Systems. Cambridge, MA: MIT, 2022: 33716−33727, articleTitle=null, refAbstract=null), Reference(id=1325491172453523900, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1325490414429545387, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=161, rfOrder=166, authorNames=null, journalName=null, refType=null, unstructuredReference=Choi J, Cho H, Cheung S, et al. ORC: Network group-based knowledge distillation using online role change[C]//Proc of 2023 IEEE/CVF Int Conf on Computer Vision. Piscataway, NJ: IEEE, 2023: 17335−17344, articleTitle=null, refAbstract=null), Reference(id=1325491174009610686, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1325490414429545387, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=162, rfOrder=167, authorNames=null, journalName=null, refType=null, unstructuredReference=Shen Zhiqiang. FerKD: Surgical label adaptation for efficient distillation[C]//Proc of 2023 IEEE/CVF Int Conf on Computer Vision. Piscataway, NJ: IEEE, 2023: 1666−1675, articleTitle=null, refAbstract=null), Reference(id=1325491175683137983, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1325490414429545387, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=163, rfOrder=168, authorNames=null, journalName=null, refType=null, unstructuredReference=Miles R, Elezi I, Deng Jiankang. VkD: Improving knowledge distillation using orthogonal projections[C]//Proc of 2024 IEEE/CVF Conf on Computer Vision and Pattern Recognition. Piscataway, NJ: IEEE, 2024: 15720−15730, articleTitle=null, refAbstract=null), Reference(id=1325491178669482432, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1325490414429545387, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=164, rfOrder=169, authorNames=null, journalName=null, refType=null, unstructuredReference=Miles R, Mikolajczyk K. Understanding the role of the projector in knowledge distillation[C]//Proc of the 38th AAAI Conf on Artificial Intelligence. Palo Alto, CA: AAAI, 2024: 4233−4241, articleTitle=null, refAbstract=null), Reference(id=1325491180758245826, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1325490414429545387, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=165, rfOrder=170, authorNames=null, journalName=null, refType=null, unstructuredReference=Sun Wujie, Chen Defang, Lyu Siwei, et al. Knowledge distillation with refined logits[J]. arXiv preprint, arXiv: 2408.07703, 2024, articleTitle=null, refAbstract=null), Reference(id=1325491183316771267, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1325490414429545387, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=166, rfOrder=171, authorNames=null, journalName=null, refType=null, unstructuredReference=Xu Kunran, Rui Lai, Li Yishi, et al. Feature normalized knowledge distillation for image classification[C]//Proc of the 16th European Conf on Computer Vision. Berlin: Springer, 2020: 664−680, articleTitle=null, refAbstract=null), Reference(id=1325491184898023877, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1325490414429545387, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=167, rfOrder=172, authorNames=null, journalName=null, refType=null, unstructuredReference=Jafari A, Rezagholizadeh M, Sharma P, et al. Annealing knowledge distillation[C]//Proc of the 16th Conf of the European Chapter of the Association for Computational Linguistics. Stroudsburg, PA: ACL, 2021: 2493−2504, articleTitle=null, refAbstract=null), Reference(id=1325491187838230982, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1325490414429545387, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=168, rfOrder=173, authorNames=null, journalName=null, refType=null, unstructuredReference=Li Zheng, Li Xiang, Yang Lingfeng, et al. Curriculum temperature for knowledge distillation[C]//Proc of the 37th AAAI Conf on Artificial Intelligence. Palo Alto, CA: AAAI, 2023: 1504−1512, articleTitle=null, refAbstract=null), Reference(id=1325491189142659528, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1325490414429545387, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=169, rfOrder=174, authorNames=null, journalName=null, refType=null, unstructuredReference=Chi Zhihao, Zheng Tu, Li Hengjia, et al. NormKD: Normalized logits for knowledge distillation[J]. arXiv preprint, arXiv: 2308.00520, 2023, articleTitle=null, refAbstract=null), Reference(id=1325491190396756425, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1325490414429545387, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=170, rfOrder=175, authorNames=null, journalName=null, refType=null, unstructuredReference=Wei Yukang, Bai Yu. Dynamic temperature knowledge distillation[J]. arXiv preprint, arXiv: 2404.12711, 2024, articleTitle=null, refAbstract=null), Reference(id=1325491192477131211, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1325490414429545387, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=171, rfOrder=176, authorNames=null, journalName=null, refType=null, unstructuredReference=Sun Shangquan, Ren Wenqi, Li Jingzhi, et al. Logit standardization in knowledge distillation[C]//Proc of 2024 IEEE/CVF Conf on Computer Vision and Pattern Recognition. Piscataway, NJ: IEEE, 2024: 15731−15740, articleTitle=null, refAbstract=null), Reference(id=1325491194163241420, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1325490414429545387, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=172, rfOrder=177, authorNames=null, journalName=null, refType=null, unstructuredReference=Zhao Borui, Cui Quan, Song Renjie, et al. Decoupled knowledge distillation[C]//Proc of 2022 IEEE/CVF Conf on Computer Vision and Pattern Recognition. Piscataway, NJ: IEEE, 2022: 11943−11952, articleTitle=null, refAbstract=null), Reference(id=1325491196738544078, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1325490414429545387, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=173, rfOrder=178, authorNames=null, journalName=null, refType=null, unstructuredReference=Yang Zhendong, Zeng Ailing, Li Zhe, et al. From knowledge distillation to self-knowledge distillation: A unified approach with normalized loss and customized soft labels[C]//Proc of 2023 IEEE/CVF Int Conf on Computer Vision. Piscataway, NJ: IEEE, 2023: 17139−17148, articleTitle=null, refAbstract=null), Reference(id=1325491197971669455, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1325490414429545387, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=174, rfOrder=179, authorNames=null, journalName=null, refType=null, unstructuredReference=Park H, Lee J S. Adaptive explicit knowledge transfer for knowledge distillation[J]. arXiv preprint, arXiv: 2409.01679, 2024, articleTitle=null, refAbstract=null), Reference(id=1325491199221572048, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1325490414429545387, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=175, rfOrder=180, authorNames=null, journalName=null, refType=null, unstructuredReference=Zhao Borui, Cui Quan, Song Renjie, et al. DOT: A distillation-oriented trainer[C]//Proc of 2023 IEEE/CVF Int Conf on Computer Vision. Piscataway, NJ: IEEE, 2023: 6166−6175, articleTitle=null, refAbstract=null), Reference(id=1325491201255809490, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1325490414429545387, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=176, rfOrder=181, authorNames=null, journalName=null, refType=null, unstructuredReference=Shen Zhiqiang, He Zhankui, Xue Xiangyang. MEAL: Multi-model ensemble via adversarial learning[C]//Proc of the 33rd AAAI Conf on Artificial Intelligence. Palo Alto, CA: AAAI, 2019: 4886−4893, articleTitle=null, refAbstract=null), Reference(id=1325491202992251347, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1325490414429545387, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=177, rfOrder=182, authorNames=null, journalName=null, refType=null, unstructuredReference=Li Peng, Shu Chang, Xie Yuan, et al. Hierarchical knowledge squeezed adversarial network compression[C]//Proc of the 34th AAAI Conf on Artificial Intelligence. Palo Alto, CA: AAAI, 2020: 11370−11377, articleTitle=null, refAbstract=null), Reference(id=1325491206284779989, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1325490414429545387, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=178, rfOrder=183, authorNames=null, journalName=null, refType=null, unstructuredReference=Ma Haoyu, Chen Tianlong, Hu T K, et al. Undistillable: Making a nasty teacher that cannot teach students[J]. arXiv preprint, arXiv: 2105.07381, 2021, articleTitle=null, refAbstract=null), Reference(id=1325491208407097816, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1325490414429545387, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=179, rfOrder=184, authorNames=null, journalName=null, refType=null, unstructuredReference=Heo B, Lee M, Yun S, et al. Knowledge distillation with adversarial samples supporting decision boundary[C]//Proc of the 33rd AAAI Conf on Artificial Intelligence. Palo Alto, CA: AAAI, 2019: 3771−3778, articleTitle=null, refAbstract=null), Reference(id=1325491210575552985, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1325490414429545387, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=180, rfOrder=185, authorNames=null, journalName=null, refType=null, unstructuredReference=Nguyen-Duc T, Le T, Zhao He, et al. Adversarial local distribution regularization for knowledge distillation[C]//Proc of 2023 IEEE/CVF Winter Conf on Applications of Computer Vision. Piscataway, NJ: IEEE, 2023: 4670−4679, articleTitle=null, refAbstract=null), Reference(id=1325491213071163867, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1325490414429545387, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=181, rfOrder=186, authorNames=null, journalName=null, refType=null, unstructuredReference=Bai Tao, Zhao Jun, Wen Bihan. Guided adversarial contrastive distillation for robust students[J]. IEEE Transactions on Information Forensics and Security, 2024, 19: 9643−9655, articleTitle=null, refAbstract=null), Reference(id=1325491215982010845, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1325490414429545387, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=182, rfOrder=187, authorNames=null, journalName=null, refType=null, unstructuredReference=Huang Bo, Chen Mingyang, Wang Yi, et al. Boosting accuracy and robustness of student models via adaptive adversarial distillation[C]//Proc of 2023 IEEE/CVF Conf on Computer Vision and Pattern Recognition. Piscataway, NJ: IEEE, 2023: 24668−24677, articleTitle=null, refAbstract=null), Reference(id=1325491218053997022, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1325490414429545387, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=183, rfOrder=188, authorNames=null, journalName=null, refType=null, unstructuredReference=Yue Xinli, Mou Ningping, Wang Qian, et al. Revisiting adversarial robustness distillation from the perspective of robust fairness[C]//Proc of the 37th Annual Conf on Neural Information Processing Systems. Cambridge, MA: MIT, 2023: 30390−30401, articleTitle=null, refAbstract=null), Reference(id=1325491220981621216, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1325490414429545387, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=184, rfOrder=189, authorNames=null, journalName=null, refType=null, unstructuredReference=Wang Yuzheng, Chen Zhaoyu, Yang Dingkang, et al. Adversarial contrastive distillation with adaptive denoising[C]//Proc of 2023 IEEE Int Conf on Acoustics, Speech and Signal Processing, Piscataway, NJ: IEEE, 2023: 1−5, articleTitle=null, refAbstract=null), Reference(id=1325491223590478306, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1325490414429545387, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=185, rfOrder=190, authorNames=null, journalName=null, refType=null, unstructuredReference=Li Changlin, Peng Jiefeng, Yuan Liuchun, et al. Block-wisely supervised neural architecture search with knowledge distillation[C]//Proc of 2020 IEEE/CVF Conf on Computer Vision and Pattern Recognition. Piscataway, NJ: IEEE, 2020: 1986−1995, articleTitle=null, refAbstract=null), Reference(id=1325491225322725859, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1325490414429545387, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=186, rfOrder=191, authorNames=null, journalName=null, refType=null, unstructuredReference=Liu Yu, Jia Xuhui, Tan Mingxing, et al. Search to distill: Pearls are everywhere but not the eyes[C]//Proc of 2020 IEEE/CVF Conf on Computer Vision and Pattern Recognition. Piscataway, NJ: IEEE, 2020: 7536−7545, articleTitle=null, refAbstract=null), Reference(id=1325491227717673446, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1325490414429545387, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=187, rfOrder=192, authorNames=null, journalName=null, refType=null, unstructuredReference=Kang M, Mun J, Han B. Towards oracle knowledge distillation with neural architecture search[C]//Proc of the 34th AAAI Conf on Artificial Intelligence. Palo Alto, CA: AAAI, 2020: 4404−4411, articleTitle=null, refAbstract=null), Reference(id=1325491229407977959, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1325490414429545387, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=188, rfOrder=193, authorNames=null, journalName=null, refType=null, unstructuredReference=Wei Zimian, Pan Hengyue, Li Lujun, et al. TVT: Training-free vision transformer search on tiny datasets[J]. arXiv preprint, arXiv: 2311.14337, 2023, articleTitle=null, refAbstract=null), Reference(id=1325491232343990761, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1325490414429545387, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=189, rfOrder=194, authorNames=null, journalName=null, refType=null, unstructuredReference=Dong Peijie, Li Lujun, Wei Zimian. DisWOT: Student architecture search for distillation without training[C]//Proc of 2023 IEEE/CVF Conf on Computer Vision and Pattern Recognition. Piscataway, NJ: IEEE, 2023: 11898−11908, articleTitle=null, refAbstract=null), Reference(id=1325491234038489578, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1325490414429545387, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=190, rfOrder=195, authorNames=null, journalName=null, refType=null, unstructuredReference=Lu Liming, Chen Zhenghan, Lu Xiaoyu, et al. UniADS: Universal architecture-distiller search for distillation gap[C]//Proc of the 38th AAAI Conf on Artificial Intelligence. Palo Alto, CA: AAAI, 2024: 14167−14174, articleTitle=null, refAbstract=null), Reference(id=1325491236597015021, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1325490414429545387, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=191, rfOrder=196, authorNames=null, journalName=null, refType=null, unstructuredReference=Li Lujun, Dong Peijie, Wei Zimian, et al. Automated knowledge distillation via monte carlo tree search[C]//Proc of 2023 IEEE/CVF Int Conf on Computer Vision. Piscataway, NJ: IEEE, 2023: 17367−17378, articleTitle=null, refAbstract=null), Reference(id=1325491237867889135, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1325490414429545387, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=192, rfOrder=197, authorNames=null, journalName=null, refType=null, unstructuredReference=Sohl-Dickstein J, Weiss E A, Maheswaranathan N, et al. Deep unsupervised learning using nonequilibrium thermodynamics[C]//Proc of the 32nd Int Conf on Machine Learning. New York: ACM, 2015: 2256−2265, articleTitle=null, refAbstract=null), Reference(id=1325491239046488560, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1325490414429545387, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=193, rfOrder=198, authorNames=null, journalName=null, refType=null, unstructuredReference=Ho J, Jain A, Abbeel P. Denoising diffusion probabilistic models[C]//Proc of the 34th Annual Conf on Neural Information Processing Systems. Cambridge, MA: MIT, 2020: 6840−6851, articleTitle=null, refAbstract=null), Reference(id=1325491241533710835, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1325490414429545387, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=194, rfOrder=199, authorNames=null, journalName=null, refType=null, unstructuredReference=Huang Tao, Zhang Yuan, Zheng Mingkai, et al. Knowledge diffusion for distillation[C]//Proc of the 37th Annual Conf on Neural Information Processing Systems. Cambridge, MA: MIT, 2023, 36: 65299−65316, articleTitle=null, refAbstract=null), Reference(id=1325491242473234933, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1325490414429545387, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=195, rfOrder=200, authorNames=null, journalName=null, refType=null, unstructuredReference=Guo Qiushan, Mu Yao, Chen Jianyu, et al. Scale-equivalent distillation for semi-supervised object detection[C]//Proc of 2022 IEEE/CVF Conf on Computer Vision and Pattern Recognition. Piscataway, NJ: IEEE, 2022: 14502−14511, articleTitle=null, refAbstract=null), Reference(id=1325491244465529334, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1325490414429545387, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=196, rfOrder=201, authorNames=null, journalName=null, refType=null, unstructuredReference=Zhou Huajun, Qiao Bo, Yang Lingxiao, et al. Texture-guided saliency distilling for unsupervised salient object detection[C]//Proc of 2023 IEEE/CVF Conf on Computer Vision and Pattern Recognition. Piscataway, NJ: IEEE, 2023: 7257−7267, articleTitle=null, refAbstract=null), Reference(id=1325491246629790200, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1325490414429545387, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=197, rfOrder=202, authorNames=null, journalName=null, refType=null, unstructuredReference=Wang Luting, Liu Yi, Du Penghui, et al. Object-aware distillation pyramid for open-vocabulary object detection[C]//Proc of 2023 IEEE/CVF Conf on Computer Vision and Pattern Recognition. Piscataway, NJ: IEEE, 2023: 11186−11196, articleTitle=null, refAbstract=null), Reference(id=1325491247900664313, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1325490414429545387, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=198, rfOrder=203, authorNames=null, journalName=null, refType=null, unstructuredReference=Li Liangqi, Miao Jiaxu, Shi Dahu, et al. Distilling DETR with visual-linguistic knowledge for open-vocabulary object detection[C]//Proc of 2023 IEEE/CVF Int Conf on Computer Vision. Piscataway, NJ: IEEE, 2023: 6478−6487, articleTitle=null, refAbstract=null), Reference(id=1325491250748596731, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1325490414429545387, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=199, rfOrder=204, authorNames=null, journalName=null, refType=null, unstructuredReference=Zhou Shengchao, Liu Weizhou, Hu Chen, et al. UniDistill: A universal cross-modality knowledge distillation framework for 3D object detection in bird’s-eye view[C]//Proc of 2023 IEEE/CVF Conf on Computer Vision and Pattern Recognition. Piscataway, NJ: IEEE, 2023: 5116−5125, articleTitle=null, refAbstract=null), Reference(id=1325491252870914556, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1325490414429545387, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=200, rfOrder=205, authorNames=null, journalName=null, refType=null, unstructuredReference=Wu Aming, Deng Cheng. Single-domain generalized object detection in urban scene via cyclic-disentangled self-distillation[C]//Proc of 2022 IEEE/CVF Conf on Computer Vision and Pattern Recognition. Piscataway, NJ: IEEE, 2022: 837−846, articleTitle=null, refAbstract=null), Reference(id=1325491255907590654, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1325490414429545387, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=201, rfOrder=206, authorNames=null, journalName=null, refType=null, unstructuredReference=Lao Shanshan, Song Guanglu, Liu Boxiao, et al. UniKD: Universal knowledge distillation for mimicking homogeneous or heterogeneous object detectors[C]//Proc of 2023 IEEE/CVF Int Conf on Computer Vision. Piscataway, NJ: IEEE, 2023: 6362−6372, articleTitle=null, refAbstract=null), Reference(id=1325491258298343935, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1325490414429545387, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=202, rfOrder=207, authorNames=null, journalName=null, refType=null, unstructuredReference=Zeng Jia, Chen Li, Deng Hanming, et al. Distilling focal knowledge from imperfect expert for 3D object detection[C]//Proc of 2023 IEEE/CVF Conf on Computer Vision and Pattern Recognition. Piscataway, NJ: IEEE, 2023: 992−1001, articleTitle=null, refAbstract=null), Reference(id=1325491260529713664, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1325490414429545387, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=203, rfOrder=208, authorNames=null, journalName=null, refType=null, unstructuredReference=Liu Zhe, Ye Xiaoqing, Tan Xiao, et al. StereoDistill: Pick the cream from LiDAR for distilling stereo-based 3D object detection[C]//Proc of the 37th AAAI Conf on Artificial Intelligence. Palo Alto, CA: AAAI, 2023: 1790−1798, articleTitle=null, refAbstract=null), Reference(id=1325491264237478401, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1325490414429545387, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=204, rfOrder=209, authorNames=null, journalName=null, refType=null, unstructuredReference=Wu Zizhang, Wu Yunzhe, Pu Jian, et al. Attention-based depth distillation with 3D-aware positional encoding for monocular 3D object detection[C]//Proc of the 37th AAAI Conf on Artificial Intelligence. Palo Alto, CA: AAAI, 2023: 2892−2900, articleTitle=null, refAbstract=null), Reference(id=1325491265957143042, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1325490414429545387, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=205, rfOrder=210, authorNames=null, journalName=null, refType=null, unstructuredReference=Wang Zeyu, Li Dingwen, Luo Chenxu, et al. DistillBEV: Boosting multi-camera 3D object detection with cross-modal knowledge distillation[C]//Proc of 2023 IEEE/CVF Int Conf on Computer Vision. Piscataway, NJ: IEEE, 2023: 8603−8612, articleTitle=null, refAbstract=null), Reference(id=1325491268805075459, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1325490414429545387, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=206, rfOrder=211, authorNames=null, journalName=null, refType=null, unstructuredReference=Zhang Yifan, Dong Zhen, Yang Huanrui, et al. QD-BEV: Quantization-aware view-guided distillation for multi-view 3D object detection[C]//Proc of 2023 IEEE/CVF Int Conf on Computer Vision. Piscataway, NJ: IEEE, 2023: 3802−3812, articleTitle=null, refAbstract=null), Reference(id=1325491270562488836, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1325490414429545387, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=207, rfOrder=212, authorNames=null, journalName=null, refType=null, unstructuredReference=Zhao Haimei, Zhang Qiming, Zhao Shanshan, et al. SimDistill: Simulated multi-modal distillation for BEV 3D object detection[C]//Proc of the 38th AAAI Conf on Artificial Intelligence. Palo Alto, CA: AAAI, 2024: 7460−7468, articleTitle=null, refAbstract=null), Reference(id=1325491273829851653, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1325490414429545387, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=208, rfOrder=213, authorNames=null, journalName=null, refType=null, unstructuredReference=Li Yanjing, Xu Sheng, Lin Mingbao, et al. Representation disparity-aware distillation for 3D object detection[C]//Proc of 2023 IEEE/CVF Int Conf on Computer Vision. Piscataway, NJ: IEEE, 2023: 6692−6701, articleTitle=null, refAbstract=null), Reference(id=1325491276421931526, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1325490414429545387, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=209, rfOrder=214, authorNames=null, journalName=null, refType=null, unstructuredReference=Gambashidze A, Dadukin A, Golyadkin M, et al. Weak-to-strong 3D object detection with X-ray distillation[C]//Proc of 2024 IEEE/CVF Conf on Computer Vision and Pattern Recognition. Piscataway, NJ: IEEE, 2024: 15055−15064, articleTitle=null, refAbstract=null), Reference(id=1325491278103847432, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1325490414429545387, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=210, rfOrder=215, authorNames=null, journalName=null, refType=null, unstructuredReference=Huang Xun, Wu Hai, Li Xin, et al. Sunshine to rainstorm: Cross-weather knowledge distillation for robust 3D object detection[C]//Proc of the 38th AAAI Conf on Artificial Intelligence. Palo Alto, CA: AAAI, 2024: 2409−2416, articleTitle=null, refAbstract=null), Reference(id=1325491279710265865, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1325490414429545387, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=211, rfOrder=216, authorNames=null, journalName=null, refType=null, unstructuredReference=Ji Deyi, Wang Haoran, Tao Mingyuan, et al. Structural and statistical texture knowledge distillation for semantic segmentation[C]//Proc of 2022 IEEE/CVF Conf on Computer Vision and Pattern Recognition. Piscataway, NJ: IEEE, 2022: 16855−16864, articleTitle=null, refAbstract=null), Reference(id=1325491281471873546, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1325490414429545387, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=212, rfOrder=217, authorNames=null, journalName=null, refType=null, unstructuredReference=Gao Huan, Guo Jichang, Wang Guoli, et al. Cross-domain correlation distillation for unsupervised domain adaptation in nighttime semantic segmentation[C]//Proc of 2022 IEEE/CVF Conf on Computer Vision and Pattern Recognition. Piscataway, NJ: IEEE, 2022: 9903−9913, articleTitle=null, refAbstract=null), Reference(id=1325491283040543243, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1325490414429545387, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=213, rfOrder=218, authorNames=null, journalName=null, refType=null, unstructuredReference=Yang Chuanguang, Zhou Helong, An Zhulin, et al. Cross-image relational knowledge distillation for semantic segmentation[C]//Proc of 2022 IEEE/CVF Conf on Computer Vision and Pattern Recognition. Piscataway, NJ: IEEE, 2022: 12309−12318, articleTitle=null, refAbstract=null), Reference(id=1325491285330633228, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1325490414429545387, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=214, rfOrder=219, authorNames=null, journalName=null, refType=null, unstructuredReference=Peng Bohao, Tian Zhuotao, Wu Xiaoyang, et al. Hierarchical dense correlation distillation for few-shot segmentation[C]//Proc of 2023 IEEE/CVF Conf on Computer Vision and Pattern Recognition. Piscataway, NJ: IEEE, 2023: 23641−23651, articleTitle=null, refAbstract=null), Reference(id=1325491286840582670, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1325490414429545387, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=215, rfOrder=220, authorNames=null, journalName=null, refType=null, unstructuredReference=Liu Yuang, Zhang Wei, Wang Jun. Self-decoupling and ensemble distillation for efficient segmentation[C]//Proc of the 37th AAAI Conf on Artificial Intelligence. Palo Alto, CA: AAAI, 2023: 1772−1780, articleTitle=null, refAbstract=null), Reference(id=1325491288195342863, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1325490414429545387, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=216, rfOrder=221, authorNames=null, journalName=null, refType=null, unstructuredReference=Hou Yuenan, Zhu Xinge, Ma Yuexin, et al. Point-to-voxel knowledge distillation for LiDAR semantic segmentation[C]//Proc of 2022 IEEE/CVF Conf on Computer Vision and Pattern Recognition. Piscataway, NJ: IEEE, 2022: 8469−8478, articleTitle=null, refAbstract=null), Reference(id=1325491290233774608, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1325490414429545387, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=217, rfOrder=222, authorNames=null, journalName=null, refType=null, unstructuredReference=Chen Zisheng, Xu Hongbin, Chen Weitao, et al. PointDC: Unsupervised semantic segmentation of 3D point clouds via cross-modal distillation and super-voxel clustering[C]//Proc of 2023 IEEE/CVF Int Conf on Computer Vision. Piscataway, NJ: IEEE, 2023: 14244−14253, articleTitle=null, refAbstract=null), Reference(id=1325491291966022161, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1325490414429545387, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=218, rfOrder=223, authorNames=null, journalName=null, refType=null, unstructuredReference=Xu Jianqing, Li Shen, Deng Ailin, et al. Probabilistic knowledge distillation of face ensembles[C]//Proc of 2023 IEEE/CVF Conf on Computer Vision and Pattern Recognition. Piscataway, NJ: IEEE, 2023: 3489−3498, articleTitle=null, refAbstract=null), Reference(id=1325491293593412114, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1325490414429545387, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=219, rfOrder=224, authorNames=null, journalName=null, refType=null, unstructuredReference=Li Jingzhi, Guo Zidong, Li Hui, et al. Rethinking feature-based knowledge distillation for face recognition[C]//Proc of 2023 IEEE/CVF Conf on Computer Vision and Pattern Recognition. Piscataway, NJ: IEEE, 2023: 20156−20165, articleTitle=null, refAbstract=null), Reference(id=1325491295644426771, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1325490414429545387, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=220, rfOrder=225, authorNames=null, journalName=null, refType=null, unstructuredReference=Yu Zhipeng, Liu Jiaheng, Qin Haoyu, et al. ICD-face: Intra-class compactness distillation for face recognition[C]//Proc of 2023 IEEE/CVF Int Conf on Computer Vision. Piscataway, NJ: IEEE, 2023: 20985−20995, articleTitle=null, refAbstract=null), Reference(id=1325491296994992660, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1325490414429545387, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=221, rfOrder=226, authorNames=null, journalName=null, refType=null, unstructuredReference=Zhao Weisong, Zhu Xiangyu, Guo Kaiwen, et al. Grouped knowledge distillation for deep face recognition[C]//Proc of the 37th AAAI Conf on Artificial Intelligence. Palo Alto, CA: AAAI, 2023: 3615−3623, articleTitle=null, refAbstract=null), Reference(id=1325491299515769365, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1325490414429545387, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=222, rfOrder=227, authorNames=null, journalName=null, refType=null, unstructuredReference=Zhang Zhenduo, Chen Yongru, Yang Wenming, et al. Pose-invariant face recognition via adaptive angular distillation[C]//Proc of the 36th AAAI Conf on Artificial Intelligence. Palo Alto, CA: AAAI, 2022: 3390−3398, articleTitle=null, refAbstract=null), Reference(id=1325491301583561238, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1325490414429545387, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=223, rfOrder=228, authorNames=null, journalName=null, refType=null, unstructuredReference=Hu Siyuan, Wang Zheng, Hu Peng, et al. PrefAce: Face-centric pretraining with self-structure aware distillation[C]//Proc of the 38th AAAI Conf on Artificial Intelligence. Palo Alto, CA: AAAI, 2024: 12538−12546, articleTitle=null, refAbstract=null), Reference(id=1325491304439882263, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1325490414429545387, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=224, rfOrder=229, authorNames=null, journalName=null, refType=null, unstructuredReference=Deng Hanqiu, Li Xingyu. Anomaly detection via reverse distillation from one-class embedding[C]//Proc of 2022 IEEE/CVF Conf on Computer Vision and Pattern Recognition. Piscataway, NJ: IEEE, 2022: 9727−9736, articleTitle=null, refAbstract=null), Reference(id=1325491306553811480, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1325490414429545387, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=225, rfOrder=230, authorNames=null, journalName=null, refType=null, unstructuredReference=Tien T D, Nguyen A T, Tran N H, et al. Revisiting reverse distillation for anomaly detection[C]//Proc of 2023 IEEE/CVF Conf on Computer Vision and Pattern Recognition. Piscataway, NJ: IEEE, 2023: 24511−24520, articleTitle=null, refAbstract=null), Reference(id=1325491308265087513, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1325490414429545387, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=226, rfOrder=231, authorNames=null, journalName=null, refType=null, unstructuredReference=Gu Zhihao, Liu Liang, Chen Xu, et al. Remembering normality: Memory-guided knowledge distillation for unsupervised anomaly detection[C]//Proc of 2023 IEEE/CVF Int Conf on Computer Vision. Piscataway, NJ: IEEE, 2023: 16355−16363, articleTitle=null, refAbstract=null), Reference(id=1325491309217194523, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1325490414429545387, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=227, rfOrder=232, authorNames=null, journalName=null, refType=null, unstructuredReference=Binh L M, Woo S. ADD: Frequency attention and multi-view based knowledge distillation to detect low-quality compressed deepfake images[C]//Proc of the 36th AAAI Conf on Artificial Intelligence. Palo Alto, CA: AAAI, 2022: 122−130, articleTitle=null, refAbstract=null), Reference(id=1325491309963780636, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1325490414429545387, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=228, rfOrder=233, authorNames=null, journalName=null, refType=null, unstructuredReference=Ma Ye, Jiang Xu, Guan Nan, et al. Anomaly detection based on multi-teacher knowledge distillation[J]. Journal of Systems Architecture, 2023, 138: 102861, articleTitle=null, refAbstract=null), Reference(id=1325491312035766814, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1325490414429545387, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=229, rfOrder=234, authorNames=null, journalName=null, refType=null, unstructuredReference=Ristea N C, Croitoru F A, Ionescu R T, et al. Self-distilled masked auto-encoders are efficient video anomaly detectors[C]//Proc of 2024 IEEE/CVF Conf on Computer Vision and Pattern Recognition. Piscataway, NJ: IEEE, 2024: 15984−15995, articleTitle=null, refAbstract=null), Reference(id=1325491313386332703, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1325490414429545387, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=230, rfOrder=235, authorNames=null, journalName=null, refType=null, unstructuredReference=邢鹏, 蒋鑫, 潘永华, 等. 基于特征约束蒸馏学习的视觉异常检测[J]. 软件学报, 2022, 34(9): 4378−4391, articleTitle=null, refAbstract=null), Reference(id=1325491315370238496, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1325490414429545387, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=230, rfOrder=236, authorNames=null, journalName=null, refType=null, unstructuredReference=Xing Peng, Jiang Xin, Pan Yonghua, et al. Feature constrained restricted distillation learning for visual anomaly detection[J]. Journal of Software, 2022, 34(9): 4378−4391 (in Chinese), articleTitle=null, refAbstract=null), Reference(id=1325491317857460769, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1325490414429545387, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=231, rfOrder=237, authorNames=null, journalName=null, refType=null, unstructuredReference=Ju Chen, Zheng Kunhao, Liu Jinxiang, et al. Distilling vision-language pre-training to collaborate with weakly-supervised temporal action localization[C]//Proc of 2023 IEEE/CVF Conf on Computer Vision and Pattern Recognition. Piscataway, NJ: IEEE, 2023: 14751−14762, articleTitle=null, refAbstract=null), Reference(id=1325491318755041826, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1325490414429545387, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=232, rfOrder=238, authorNames=null, journalName=null, refType=null, unstructuredReference=Wu Mingrui, Gu Jiaxin, Shen Yunhang, et al. End-to-end zero-shot HOI detection via vision and language knowledge distillation[C]//Proc of the 37th AAAI Conf on Artificial Intelligence. Palo Alto, CA: AAAI, 2023: 2839−2846, articleTitle=null, refAbstract=null), Reference(id=1325491320009138723, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1325490414429545387, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=233, rfOrder=239, authorNames=null, journalName=null, refType=null, unstructuredReference=Monti A, Porrello A, Calderara S, et al. How many observations are enough? Knowledge distillation for trajectory forecasting[C]//Proc of 2018 IEEE/CVF Conf on Computer Vision and Pattern Recognition. Piscataway, NJ: IEEE, 2022: 6543−6552, articleTitle=null, refAbstract=null), Reference(id=1325491321720414756, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1325490414429545387, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=234, rfOrder=240, authorNames=null, journalName=null, refType=null, unstructuredReference=Ma Hengbo, Li Jiachen, Hosseini R, et al. Multi-objective diverse human motion prediction with knowledge distillation[C]//Proc of 2022 IEEE/CVF Conf on Computer Vision and Pattern Recognition. Piscataway, NJ: IEEE, 2022: 8151−8161, articleTitle=null, refAbstract=null), Reference(id=1325491323804983845, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1325490414429545387, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=235, rfOrder=241, authorNames=null, journalName=null, refType=null, unstructuredReference=Yang C, Kong K, Min S, et al. SEFD: Learning to distill complex pose and occlusion[C]//Proc of 2023 IEEE/CVF Int Conf on Computer Vision. Piscataway, NJ: IEEE, 2023: 14895−14906, articleTitle=null, refAbstract=null), Reference(id=1325491326736802342, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1325490414429545387, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=236, rfOrder=242, authorNames=null, journalName=null, refType=null, unstructuredReference=Ye Suhang, Zhang Yingyi, Hu Jie, et al. DistilPose: Tokenized pose regression with heatmap distillation[C]//Proc of 2023 IEEE/CVF Conf on Computer Vision and Pattern Recognition. Piscataway, NJ: IEEE, 2023: 2163−2172, articleTitle=null, refAbstract=null), Reference(id=1325491328817177128, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1325490414429545387, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=237, rfOrder=243, authorNames=null, journalName=null, refType=null, unstructuredReference=Chen Sichen, Zhang Yingyi, Huang Siming, et al. SDPose: Tokenized pose estimation via circulation-guide self-distillation[C]//Proc of 2024 IEEE/CVF Conf on Computer Vision and Pattern Recognition. Piscataway, NJ: IEEE, 2024: 1082−1090, articleTitle=null, refAbstract=null), Reference(id=1325491331795132970, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1325490414429545387, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=238, rfOrder=244, authorNames=null, journalName=null, refType=null, unstructuredReference=Lu Yichen, Wang Mei, Deng Weihong. Augmented geometric distillation for data-free incremental person ReID[C]//Proc of 2022 IEEE/CVF Conf on Computer Vision and Pattern Recognition. Piscataway, NJ: IEEE, 2022: 7319−7328, articleTitle=null, refAbstract=null), Reference(id=1325491332977926701, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1325490414429545387, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=239, rfOrder=245, authorNames=null, journalName=null, refType=null, unstructuredReference=Shang Chao, Li Hongliang, Meng Fanman, et al. Incrementer: Transformer for class-incremental semantic segmentation with knowledge distillation focusing on old class[C]//Proc of 2023 IEEE/CVF Conf on Computer Vision and Pattern Recognition. Piscataway, NJ: IEEE, 2023: 7214−7224, articleTitle=null, refAbstract=null), Reference(id=1325491335075078703, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1325490414429545387, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=240, rfOrder=246, authorNames=null, journalName=null, refType=null, unstructuredReference=Phan M H, Ta T A, Phung S L, et al. Class similarity weighted knowledge distillation for continual semantic segmentation[C]//Proc of 2022 IEEE/CVF Conf on Computer Vision and Pattern Recognition. Piscataway, NJ: IEEE, 2022: 16845−16854, articleTitle=null, refAbstract=null), Reference(id=1325491335985242672, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1325490414429545387, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=241, rfOrder=247, authorNames=null, journalName=null, refType=null, unstructuredReference=Gu Peiyan, Zhang Chuyu, Xu Ruijie, et al. Class-relation knowledge distillation for novel class discovery[C]//Proc of 2023 IEEE/CVF Int Conf on Computer Vision. Piscataway, NJ: IEEE, 2023: 16428−16437, articleTitle=null, refAbstract=null), Reference(id=1325491337616826929, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1325490414429545387, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=242, rfOrder=248, authorNames=null, journalName=null, refType=null, unstructuredReference=Li Jiyong, Azizov D, Li Yang, et al. Contrastive continual learning with importance sampling and prototype-instance relation distillation[C]//Proc of the 38th AAAI Conf on Artificial Intelligence. Palo Alto, CA: AAAI, 2024: 13554−13562, articleTitle=null, refAbstract=null), Reference(id=1325491340246655538, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1325490414429545387, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=243, rfOrder=249, authorNames=null, journalName=null, refType=null, unstructuredReference=Zhao Linglan, Lu Jing, Xu Yunlu, et al. Few-shot class-incremental learning via class-aware bilateral distillation[C]//Proc of 2023 IEEE/CVF Conf on Computer Vision and Pattern Recognition. Piscataway, NJ: IEEE, 2023: 11838−11847, articleTitle=null, refAbstract=null), Reference(id=1325491341844685364, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1325490414429545387, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=244, rfOrder=250, authorNames=null, journalName=null, refType=null, unstructuredReference=Yan Hongwei, Wang Liyuan, Ma Kaisheng, et al. Orchestrate latent expertise: Advancing online continual learning with multi-level supervision and reverse self-distillation[C]//Proc of 2024 IEEE/CVF Conf on Computer Vision and Pattern Recognition. Piscataway, NJ: IEEE, 2024: 23670−23680, articleTitle=null, refAbstract=null), Reference(id=1325491343971197493, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1325490414429545387, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=245, rfOrder=251, authorNames=null, journalName=null, refType=null, unstructuredReference=Ye Fei, Bors A G. Continual variational autoencoder via continual generative knowledge distillation[C]//Proc of the 37th AAAI Conf on Artificial Intelligence. Palo Alto, CA: AAAI, 2023: 10918−10926, articleTitle=null, refAbstract=null), Reference(id=1325491346374533687, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1325490414429545387, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=246, rfOrder=252, authorNames=null, journalName=null, refType=null, unstructuredReference=Wen Haitao, Pan Lili, Dai Yu, et al. Class incremental learning with multi-teacher distillation[C]//Proc of 2024 IEEE/CVF Conf on Computer Vision and Pattern Recognition. Piscataway, NJ: IEEE, 2024: 28443−28452, articleTitle=null, refAbstract=null), Reference(id=1325491348505240120, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1325490414429545387, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=247, rfOrder=253, authorNames=null, journalName=null, refType=null, unstructuredReference=Yang Ze, Li Ruibo, Ling Evan, et al. Label-guided knowledge distillation for continual semantic segmentation on 2D images and 3D point clouds[C]//Proc of 2023 IEEE/CVF Int Conf on Computer Vision. Piscataway, NJ: IEEE, 2023: 18555−18566, articleTitle=null, refAbstract=null), Reference(id=1325491349805474361, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1325490414429545387, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=248, rfOrder=254, authorNames=null, journalName=null, refType=null, unstructuredReference=Zhang Lin, Shen Li, Ding Liang, et al. Fine-tuning global model via data-free knowledge distillation for non-IID federated learning[C]//Proc of 2022 IEEE/CVF Conf on Computer Vision and Pattern Recognition. Piscataway, NJ: IEEE, 2022: 10164−10173, articleTitle=null, refAbstract=null), Reference(id=1325491354033332796, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1325490414429545387, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=249, rfOrder=255, authorNames=null, journalName=null, refType=null, unstructuredReference=Shen Yiqing, Zhou Yuyin, Yu Lequan. CD2-pFed: Cyclic distillation-guided channel decoupling for model personalization in federated learning[C]//Proc of 2022 IEEE/CVF Conf on Computer Vision and Pattern Recognition. Piscataway, NJ: IEEE, 2022: 10031−10040, articleTitle=null, refAbstract=null), Reference(id=1325491356034015805, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1325490414429545387, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=250, rfOrder=256, authorNames=null, journalName=null, refType=null, unstructuredReference=Kim H, Kwak Y, Jung M, et al. ProtoFL: Unsupervised federated learning via prototypical distillation[C]//Proc of 2023 IEEE/CVF Int Conf on Computer Vision. Piscataway, NJ: IEEE, 2023: 6447−6456, articleTitle=null, refAbstract=null), Reference(id=1325491357791429182, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1325490414429545387, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=251, rfOrder=257, authorNames=null, journalName=null, refType=null, unstructuredReference=Lu Yang, Chen Lin, Zhang Yonggang, et al. Federated learning with extremely noisy clients via negative distillation[C]//Proc of the 38th AAAI Conf on Artificial Intelligence. Palo Alto, CA: AAAI, 2024: 14184−14192, articleTitle=null, refAbstract=null), Reference(id=1325491358970028607, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1325490414429545387, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=252, rfOrder=258, authorNames=null, journalName=null, refType=null, unstructuredReference=Wang Haozhao, Li Yichen, Xu Wenchao, et al. DaFKD: Domain-aware federated knowledge distillation[C]//Proc of 2023 IEEE/CVF Conf on Computer Vision and Pattern Recognition. Piscataway, NJ: IEEE, 2023: 20412−20421, articleTitle=null, refAbstract=null), Reference(id=1325491360249291329, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1325490414429545387, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=253, rfOrder=259, authorNames=null, journalName=null, refType=null, unstructuredReference=乐俊青, 谭州勇, 张迪, 等. 面向车联网数据持续共享的安全高效联邦学习[J]. 计算机研究与发展, 2024, 61(9): 2199−2212, articleTitle=null, refAbstract=null), Reference(id=1325491362371609154, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1325490414429545387, doi=null, pmid=null, pmcid=null, year=null, volume=null, issue=null, pageStart=null, pageEnd=null, url=null, language=null, rfNumber=253, rfOrder=260, authorNames=null, journalName=null, refType=null, unstructuredReference=Le Junqing, Tan Zhouyong, Zhang Di, et al. Secure and efficient federated learning for continuous IoV data sharing[J]. Journal of Computer Research and Development, 2024, 61(9): 2199−2212 (in Chinese), articleTitle=null, refAbstract=null)], funds=null, companyList=[AuthorCompany(id=1325490582310756372, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1325490414429545387, xref=null, ext=[AuthorCompanyExt(id=1325490582361088021, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1325490414429545387, companyId=1325490582310756372, language=EN, country=null, province=null, city=null, postcode=null, companyName=null, departmentName=null, remark=College of Computer Science and Technology, Harbin Engineering University, Harbin 150001), AuthorCompanyExt(id=1325490582587580438, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1325490414429545387, companyId=1325490582310756372, language=CN, country=null, province=null, city=null, postcode=null, companyName=null, departmentName=null, remark=哈尔滨工程大学计算机科学与技术学院 哈尔滨 150001)])], figs=[ArticleFig(id=1325490651911037005, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1325490414429545387, language=EN, label=Figure 1, caption=
Overview of the general structure in our paper, figureFileSmall=vKzwRXKs5ApH7FR+XmapLg==, figureFileBig=c5TffK5Y2OVaiAIy+407Gg==, tableContent=null), ArticleFig(id=1325490653899137102, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1325490414429545387, language=CN, label=图1, caption=
本文总体结构概览, figureFileSmall=vKzwRXKs5ApH7FR+XmapLg==, figureFileBig=c5TffK5Y2OVaiAIy+407Gg==, tableContent=null), ArticleFig(id=1325490662329688145, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1325490414429545387, language=EN, label=Figure 2, caption=
Convolutional network model architecture, figureFileSmall=Yk9wvvvidmUB3FfHgjYh8Q==, figureFileBig=8O/FUP1mV1W3XRNNqaGzyg==, tableContent=null), ArticleFig(id=1325490665261506643, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1325490414429545387, language=CN, label=图2, caption=
卷积网络模型架构, figureFileSmall=Yk9wvvvidmUB3FfHgjYh8Q==, figureFileBig=8O/FUP1mV1W3XRNNqaGzyg==, tableContent=null), ArticleFig(id=1325490667761311828, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1325490414429545387, language=EN, label=Figure 3, caption=
ViT model architecture, figureFileSmall=Ganfc+xyx+7S3OBDTdQnqw==, figureFileBig=F3qCwuMK9AeFotTO/nRZ7A==, tableContent=null), ArticleFig(id=1325490670747656277, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1325490414429545387, language=CN, label=图3, caption=
ViT 模型架构, figureFileSmall=Ganfc+xyx+7S3OBDTdQnqw==, figureFileBig=F3qCwuMK9AeFotTO/nRZ7A==, tableContent=null), ArticleFig(id=1325490675306864729, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1325490414429545387, language=EN, label=Figure 4, caption=
Basic architecture of knowledge distillation, figureFileSmall=CgGn37Ckj6BoVj2Ca+CyVQ==, figureFileBig=hUfpVIuRW0+hVzcM99njXw==, tableContent=null), ArticleFig(id=1325490678230294619, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1325490414429545387, language=CN, label=图4, caption=
知识蒸馏的基本架构, figureFileSmall=CgGn37Ckj6BoVj2Ca+CyVQ==, figureFileBig=hUfpVIuRW0+hVzcM99njXw==, tableContent=null), ArticleFig(id=1325490680742682716, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1325490414429545387, language=EN, label=Figure 5, caption=
Illustration of distillation method based on probability distribution output, figureFileSmall=4l6kdGw2qC6m2+a1/xtslw==, figureFileBig=vzDbLtXU4bFAnh0lzo1WNw==, tableContent=null), ArticleFig(id=1325490682911137886, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1325490414429545387, language=CN, label=图5, caption=
基于概率分布输出的蒸馏方法示意图, figureFileSmall=4l6kdGw2qC6m2+a1/xtslw==, figureFileBig=vzDbLtXU4bFAnh0lzo1WNw==, tableContent=null), ArticleFig(id=1325490685423525983, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1325490414429545387, language=EN, label=Figure 6, caption=
Illustration of distillation method based on Logits output, figureFileSmall=+aiJvHYdwQ0u4xLxD8kYZg==, figureFileBig=2FVY/JHgbd+8NYY8/jAdpg==, tableContent=null), ArticleFig(id=1325490688774774880, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1325490414429545387, language=CN, label=图6, caption=
基于Logits 输出的蒸馏方法示意图, figureFileSmall=+aiJvHYdwQ0u4xLxD8kYZg==, figureFileBig=2FVY/JHgbd+8NYY8/jAdpg==, tableContent=null), ArticleFig(id=1325490692176355427, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1325490414429545387, language=EN, label=Figure 7, caption=
Illustration of distillation method based on intermediate features, figureFileSmall=jdXbTvM+r75PoaGPZP8Eng==, figureFileBig=NfsXNY/f8RVwEYyS1RnSdA==, tableContent=null), ArticleFig(id=1325490694193815652, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1325490414429545387, language=CN, label=图7, caption=
基于中间特征的蒸馏方法示意图, figureFileSmall=jdXbTvM+r75PoaGPZP8Eng==, figureFileBig=NfsXNY/f8RVwEYyS1RnSdA==, tableContent=null), ArticleFig(id=1325490696265801829, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1325490414429545387, language=EN, label=Figure 8, caption=
Illustration of distillation method based on relational features, figureFileSmall=GhWAzPMHYuEvjmrePAaYOw==, figureFileBig=gv+za33hpcbv0k+VQgDEAQ==, tableContent=null), ArticleFig(id=1325490698434256999, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1325490414429545387, language=CN, label=图8, caption=
基于关系特征的蒸馏方法示意图, figureFileSmall=GhWAzPMHYuEvjmrePAaYOw==, figureFileBig=gv+za33hpcbv0k+VQgDEAQ==, tableContent=null), ArticleFig(id=1325490701760340072, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1325490414429545387, language=EN, label=Figure 9, caption=
Illustration of CNN2ViT method, figureFileSmall=D+ExZABaGV0jMdjIWM/QMQ==, figureFileBig=+auuYPcBGDx3G4CkKwZloA==, tableContent=null), ArticleFig(id=1325490705585545321, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1325490414429545387, language=CN, label=图9, caption=
CNN2ViT 方法示意图, figureFileSmall=D+ExZABaGV0jMdjIWM/QMQ==, figureFileBig=+auuYPcBGDx3G4CkKwZloA==, tableContent=null), ArticleFig(id=1325490708508975210, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1325490414429545387, language=EN, label=Figure 10, caption=
Illustration of ViT2CNN method, figureFileSmall=P5ibvUjHhLhNTU1G6eFrGA==, figureFileBig=8jhK7oZ3s3xTbTalk1xl8A==, tableContent=null), ArticleFig(id=1325490712212545643, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1325490414429545387, language=CN, label=图10, caption=
ViT2CNN 方法示意图, figureFileSmall=P5ibvUjHhLhNTU1G6eFrGA==, figureFileBig=8jhK7oZ3s3xTbTalk1xl8A==, tableContent=null), ArticleFig(id=1325490714007707758, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1325490414429545387, language=EN, label=Figure 11, caption=
Illustration of ViT2ViT method, figureFileSmall=othK7q2yk35X2lXiBoddew==, figureFileBig=uEQVNRlCo2te9zFzISKcyA==, tableContent=null), ArticleFig(id=1325490716012585072, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1325490414429545387, language=CN, label=图11, caption=
ViT2ViT 方法示意图, figureFileSmall=othK7q2yk35X2lXiBoddew==, figureFileBig=uEQVNRlCo2te9zFzISKcyA==, tableContent=null), ArticleFig(id=1325490718550138994, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1325490414429545387, language=EN, label=Figure 12, caption=
Style of learning of knowledge distillation, figureFileSmall=MrjZRXQBim+O6xuxePHusw==, figureFileBig=SG9UoU5unQoBPtPd6lkXmA==, tableContent=null), ArticleFig(id=1325490720659873907, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1325490414429545387, language=CN, label=图12, caption=
知识蒸馏的学习方式, figureFileSmall=MrjZRXQBim+O6xuxePHusw==, figureFileBig=SG9UoU5unQoBPtPd6lkXmA==, tableContent=null), ArticleFig(id=1325490723579109493, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1325490414429545387, language=EN, label=Figure 13, caption=
Illustration of optimized location of knowledge distillation, figureFileSmall=ChTSfxEWdBwJILhAFGLbBQ==, figureFileBig=Xsj0Af1j6zjs/gh5QH4J+A==, tableContent=null), ArticleFig(id=1325490725630124151, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1325490414429545387, language=CN, label=图13, caption=
知识蒸馏的优化位置示意图, figureFileSmall=ChTSfxEWdBwJILhAFGLbBQ==, figureFileBig=Xsj0Af1j6zjs/gh5QH4J+A==, tableContent=null), ArticleFig(id=1325490727853105273, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1325490414429545387, language=EN, label=Table 1, caption=
Reference Table of Distance Symbols, figureFileSmall=null, figureFileBig=null, tableContent=
| 符号缩写 | 英文全称 | 中文名称 |
|---|
| ${{L}_1}$ | ${{L}_{1}}{ \text{-Norm}}$ | 曼哈顿距离 |
| ${{L}_2}$ | $ {{L}_{2}}{ \text{-Norm}} $ | 欧氏距离 |
| ${{L}_\infty }$ | $ {{L}_\infty }{ \text{-Norm}} $ | 切比雪夫距离 |
| CE | cross entropy | 交叉熵 |
| KL | Kullback-Leibler divergence | KL散度 |
| MSE | mean-square error | 均方误差 |
| MMD | maximum mean discrepancy | 最大均值差异 |
| EMD | earth mover’s distance | 推土机距离 |
| InfoNCE[27] | information noise-contrastive estimation | |
), ArticleFig(id=1325490729874759803, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1325490414429545387, language=CN, label=表1, caption=
距离符号对照表
, figureFileSmall=null, figureFileBig=null, tableContent=
| 符号缩写 | 英文全称 | 中文名称 |
|---|
| ${{L}_1}$ | ${{L}_{1}}{ \text{-Norm}}$ | 曼哈顿距离 |
| ${{L}_2}$ | $ {{L}_{2}}{ \text{-Norm}} $ | 欧氏距离 |
| ${{L}_\infty }$ | $ {{L}_\infty }{ \text{-Norm}} $ | 切比雪夫距离 |
| CE | cross entropy | 交叉熵 |
| KL | Kullback-Leibler divergence | KL散度 |
| MSE | mean-square error | 均方误差 |
| MMD | maximum mean discrepancy | 最大均值差异 |
| EMD | earth mover’s distance | 推土机距离 |
| InfoNCE[27] | information noise-contrastive estimation | |
), ArticleFig(id=1325490732852715644, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1325490414429545387, language=EN, label=Table 2, caption=
Summary of Knowledge Distillation Methods Based on Output Features, figureFileSmall=null, figureFileBig=null, tableContent=
| 方法来源 | 知识的来源 | 距离度量 | 关键技术 |
|---|
| 注:L2,KL,CE分别表示欧氏距离、KL散度和交叉熵。 |
| 文献[35] | Logits输出 | ${{L}_2}$ | 提出模仿复杂模型的Logits输出,获得与复杂模型精度相当的轻量模型 |
| 文献[21] | 概率分布输出 | KL | 通过温度系数软化标签强调暗知识的作用 |
| 文献[29] | 概率分布输出 | ${{L}_2}$ | 提出对教师模型概率分布输出添加噪声,模拟同专业的不同教师模型,缓解知识多样性问题 |
| 文献[30] | 概率分布输出 | KL | 提出让多个模型互相学习来获得更有效更紧凑的模型 |
| 文献[36] | Logits输出 | ${{L}_2}$ | 通过共享师生模型架构的低级别参数来降低推理所需要的时间 |
| 文献[31] | 概率分布输出 | CE | 提出了一种新的学习范式,通过序列化建模获得高性能的学生模型 |
| 文献[32] | 概率分布输出 | 最高分差 | 考虑了次要类别概率分布输出的知识,缓解严格教师模型带来的性能损失 |
| 文献[33] | 概率分布输出 | CE | 提出标签与增强标签联合优化,处理样本经过数据增强后不一定线性可分的问题 |
| 文献[34] | 概率分布输出 倒数第2层输出 | CE L2 | 提出将表示学习和分类解耦,利用教师分类器辅助学生模型训练,改善学生模型的特征表示 |
| 文献[37] | Logits输出概率分布输出 | CE KL | 始终将上级教师模型的知识作为多级下级学生模型的指导信息,缓解教师模型错误累计问题 |
| 文献[39] | 师生Logits输出差 | KL | 利用师生模型Logits输出的差异作为知识来缩减师生模型之间的能力差距 |
), ArticleFig(id=1325490735323160703, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1325490414429545387, language=CN, label=表2, caption=
基于输出特征的知识蒸馏方法汇总
, figureFileSmall=null, figureFileBig=null, tableContent=
| 方法来源 | 知识的来源 | 距离度量 | 关键技术 |
|---|
| 注:L2,KL,CE分别表示欧氏距离、KL散度和交叉熵。 |
| 文献[35] | Logits输出 | ${{L}_2}$ | 提出模仿复杂模型的Logits输出,获得与复杂模型精度相当的轻量模型 |
| 文献[21] | 概率分布输出 | KL | 通过温度系数软化标签强调暗知识的作用 |
| 文献[29] | 概率分布输出 | ${{L}_2}$ | 提出对教师模型概率分布输出添加噪声,模拟同专业的不同教师模型,缓解知识多样性问题 |
| 文献[30] | 概率分布输出 | KL | 提出让多个模型互相学习来获得更有效更紧凑的模型 |
| 文献[36] | Logits输出 | ${{L}_2}$ | 通过共享师生模型架构的低级别参数来降低推理所需要的时间 |
| 文献[31] | 概率分布输出 | CE | 提出了一种新的学习范式,通过序列化建模获得高性能的学生模型 |
| 文献[32] | 概率分布输出 | 最高分差 | 考虑了次要类别概率分布输出的知识,缓解严格教师模型带来的性能损失 |
| 文献[33] | 概率分布输出 | CE | 提出标签与增强标签联合优化,处理样本经过数据增强后不一定线性可分的问题 |
| 文献[34] | 概率分布输出 倒数第2层输出 | CE L2 | 提出将表示学习和分类解耦,利用教师分类器辅助学生模型训练,改善学生模型的特征表示 |
| 文献[37] | Logits输出概率分布输出 | CE KL | 始终将上级教师模型的知识作为多级下级学生模型的指导信息,缓解教师模型错误累计问题 |
| 文献[39] | 师生Logits输出差 | KL | 利用师生模型Logits输出的差异作为知识来缩减师生模型之间的能力差距 |
), ArticleFig(id=1325490738250784898, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1325490414429545387, language=EN, label=Table 3, caption=
Summary of Distillation Methods Based on Intermediate Features, figureFileSmall=null, figureFileBig=null, tableContent=
| 方法来源 | 知识的来源 | 知识的转换 | 距离度量 | 关键技术 |
|---|
| 注:${{L}_1}$和${{L}_2}$表示曼哈顿距离和欧氏距离,KL和CE表示KL散度和交叉熵,MMD表示最大均值差异,dp[52]表示修正的欧氏距离。 |
| 文献[40] | 中间层特征图 | 线性层 | ${{L}_2}$ | 提出从教师模型中间层抽取知识来改善训练过程和学生模型性能 |
| 文献[41] | 注意力图 | 归一化层 | ${{L}_2}$ | 利用网络层块注意力图转移特征知识模仿人类视觉系统来提高网络性能 |
| 文献[47] | 神经元激活模式 | 核方法 | MMD | 提出转移教师模型神经元激活模式来传递知识以提高蒸馏知识的有效性 |
| 文献[50] | 中间层特征图 | 线性层 | ${{L}_2}$ | 通过结合量化和蒸馏2种方法来专注于非常微小网络的训练 |
| 文献[51] | 中间层特征图 | | 互信息 | 提出知识转移信息论框架,将知识转移表述为师生模型的互信息最大化 |
| 文献[52] | 中间层特征图 | margin-ReLU | dp | 提出新的激活函数来保存教师模型信息,引入新距离度量用于特征蒸馏 |
| 文献[59] | 概率分布输出中间层特征图 | 线性层 | CE,${{L}_2}$ | 鼓励同一卷积层的不同卷积核生成多样性特征图, 缓解学生模型性能低下的问题 |
| 文献[56] | 联合分布特征中间层特征图概率分布输出 | 线性层 | CE,KL | 将联合标签增强自蒸馏方法迁移到传统蒸馏,缓解自监督对比 学习对原始类表征的损害 |
| 文献[34] | 倒数第2层特征 概率分布输出 | 线性层 | ${{L}_2}$,CE | 提出将表示学习和分类解耦,利用教师分类器辅助学生模型训练, 改善学生模型的特征表示 |
| 文献[60] | 金字塔网络特征图 | 注意力聚合 | ${{L}_2}$ | 提出跨阶段连接路径,学生模型高阶层学习教师模型低层 特征以提高模型的性能表现 |
| 文献[58] | 倒数第2层特征教师模型分类器 | 线性层 | ${{L}_2}$ | 提出对齐师生模型倒数第2层特征并直接复用教师模型分类器 来降低蒸馏算法的复杂性 |
| 文献[42] | 中间层特征图 | 相似加权线性层 | ${{L}_2}$ | 利用教师模型对学生模型特征重建,解决模型架构差异带来的语义偏差 |
| 文献[43] | 金字塔网络特征图 | 随机掩码、线性层 | ${{L}_2}$ | 利用掩码图像机制进行知识传递,提高教师模型知识的完整性和有效性 |
| 文献[44] | 类别注意力图 | 归一化层 | ${{L}_2}$ | 提出类别注意力图来进行知识的迁移,缓解蒸馏可解释性差的问题 |
| 文献[45] | 中间层频率特征 | 线性层 | ${{L}_1}$ | 通过离散小波变换获得频率域特征,缓解空间域连续下采样的信息丢失 |
), ArticleFig(id=1325490742264733828, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1325490414429545387, language=CN, label=表3, caption=
基于中间特征的蒸馏方法汇总
, figureFileSmall=null, figureFileBig=null, tableContent=
| 方法来源 | 知识的来源 | 知识的转换 | 距离度量 | 关键技术 |
|---|
| 注:${{L}_1}$和${{L}_2}$表示曼哈顿距离和欧氏距离,KL和CE表示KL散度和交叉熵,MMD表示最大均值差异,dp[52]表示修正的欧氏距离。 |
| 文献[40] | 中间层特征图 | 线性层 | ${{L}_2}$ | 提出从教师模型中间层抽取知识来改善训练过程和学生模型性能 |
| 文献[41] | 注意力图 | 归一化层 | ${{L}_2}$ | 利用网络层块注意力图转移特征知识模仿人类视觉系统来提高网络性能 |
| 文献[47] | 神经元激活模式 | 核方法 | MMD | 提出转移教师模型神经元激活模式来传递知识以提高蒸馏知识的有效性 |
| 文献[50] | 中间层特征图 | 线性层 | ${{L}_2}$ | 通过结合量化和蒸馏2种方法来专注于非常微小网络的训练 |
| 文献[51] | 中间层特征图 | | 互信息 | 提出知识转移信息论框架,将知识转移表述为师生模型的互信息最大化 |
| 文献[52] | 中间层特征图 | margin-ReLU | dp | 提出新的激活函数来保存教师模型信息,引入新距离度量用于特征蒸馏 |
| 文献[59] | 概率分布输出中间层特征图 | 线性层 | CE,${{L}_2}$ | 鼓励同一卷积层的不同卷积核生成多样性特征图, 缓解学生模型性能低下的问题 |
| 文献[56] | 联合分布特征中间层特征图概率分布输出 | 线性层 | CE,KL | 将联合标签增强自蒸馏方法迁移到传统蒸馏,缓解自监督对比 学习对原始类表征的损害 |
| 文献[34] | 倒数第2层特征 概率分布输出 | 线性层 | ${{L}_2}$,CE | 提出将表示学习和分类解耦,利用教师分类器辅助学生模型训练, 改善学生模型的特征表示 |
| 文献[60] | 金字塔网络特征图 | 注意力聚合 | ${{L}_2}$ | 提出跨阶段连接路径,学生模型高阶层学习教师模型低层 特征以提高模型的性能表现 |
| 文献[58] | 倒数第2层特征教师模型分类器 | 线性层 | ${{L}_2}$ | 提出对齐师生模型倒数第2层特征并直接复用教师模型分类器 来降低蒸馏算法的复杂性 |
| 文献[42] | 中间层特征图 | 相似加权线性层 | ${{L}_2}$ | 利用教师模型对学生模型特征重建,解决模型架构差异带来的语义偏差 |
| 文献[43] | 金字塔网络特征图 | 随机掩码、线性层 | ${{L}_2}$ | 利用掩码图像机制进行知识传递,提高教师模型知识的完整性和有效性 |
| 文献[44] | 类别注意力图 | 归一化层 | ${{L}_2}$ | 提出类别注意力图来进行知识的迁移,缓解蒸馏可解释性差的问题 |
| 文献[45] | 中间层频率特征 | 线性层 | ${{L}_1}$ | 通过离散小波变换获得频率域特征,缓解空间域连续下采样的信息丢失 |
), ArticleFig(id=1325490745783754885, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1325490414429545387, language=EN, label=Table 4, caption=
Summary of Distillation Methods Based on Relational Features, figureFileSmall=null, figureFileBig=null, tableContent=
| 方法来源 | 知识的来源 | 相似性知识 | 距离度量 | 关键技术 |
|---|
| 注:${{L}_2}$,${{L}_\infty }$分别表示欧氏距离和切比雪夫距离,EMD,MMD,MSE,KL分别表示推土机距离、最大均值差异、均方误差、KL散度, InfoNCE[27]表示一项对比损失。 |
| 文献[61] | 中间层特征图 | ${{L}_2}$ | ${{L}_\infty }$ | 利用不同样本的中间层特征之间的相对差异作为知识,提高知识的多样性 |
| 文献[53] | 中间层特征图 | 哈达玛乘积 | ${{L}_2}$ | 提出将教师模型求解问题流程抽象为关系矩阵来作为知识提高学生模型的性能 |
| 文献[62] | Logits输出 中间层特征图 | Logits图表示图 | EMD MMD | 利用教师模型Logits输出特征和中间特征构造图结构,解决特征之间互补性的缺失问题 |
| 文献[66] | 样本特征向量 | ${{L}_2}$ | KL | 提出一种新的知识类型,给出样本相似性打分函数并根据相似性来进行蒸馏 |
| 文献[67] | 不同层样本 特征向量 | 实例关系图 | ${{L}_2}$ | 利用实例特征构造图关系知识,解决由于师生模型架构不同 导致的性能下降和类间差异性低的问题 |
| 文献[68] | 样本特征向量 | 核方法 | ${{L}_2}$ | 考虑样本内容一致性和样本关系一致性,解除师生模型的实例强约束限制 |
| 文献[69] | 中间层特征图 | 转置相乘 | ${{L}_2}$ | 利用样本相似性向学生模型传递教师模型的激活模式,推动知识获取的发展 |
| 文献[57] | 样本特征向量 | 互信息 | 最大互信息 | 引入对比损失表达样本关系,处理先前方法忽略教师模型结构信息的问题 |
| 文献[63] | 中间层特征图 | 转置相乘 | MSE | 利用相似度矩阵和注意力机制捕获中间层语义关联,解决手动关联不匹配问题 |
| 文献[64] | 中间层特征图 | 转置相乘 | ${{L}_2}$ | 提出通道间相似性知识蒸馏,解决教师模型特征多样性不足的问题 |
| 文献[70] | 样本特征向量 | 余弦相似度 | InfoNCE | 提出词袋聚合蒸馏传递关系知识,解决基于对比学习方法优化困难的问题 |
| 文献[65] | 概率分布输出 中间层特征图 | 转置相乘 | KL L2 | 从样本实例、批次和类别角度构造关系知识,解决基于概率 分布输出蒸馏方法性能欠佳的问题 |
), ArticleFig(id=1325490747897684103, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1325490414429545387, language=CN, label=表4, caption=
基于关系特征的蒸馏方法汇总
, figureFileSmall=null, figureFileBig=null, tableContent=
| 方法来源 | 知识的来源 | 相似性知识 | 距离度量 | 关键技术 |
|---|
| 注:${{L}_2}$,${{L}_\infty }$分别表示欧氏距离和切比雪夫距离,EMD,MMD,MSE,KL分别表示推土机距离、最大均值差异、均方误差、KL散度, InfoNCE[27]表示一项对比损失。 |
| 文献[61] | 中间层特征图 | ${{L}_2}$ | ${{L}_\infty }$ | 利用不同样本的中间层特征之间的相对差异作为知识,提高知识的多样性 |
| 文献[53] | 中间层特征图 | 哈达玛乘积 | ${{L}_2}$ | 提出将教师模型求解问题流程抽象为关系矩阵来作为知识提高学生模型的性能 |
| 文献[62] | Logits输出 中间层特征图 | Logits图表示图 | EMD MMD | 利用教师模型Logits输出特征和中间特征构造图结构,解决特征之间互补性的缺失问题 |
| 文献[66] | 样本特征向量 | ${{L}_2}$ | KL | 提出一种新的知识类型,给出样本相似性打分函数并根据相似性来进行蒸馏 |
| 文献[67] | 不同层样本 特征向量 | 实例关系图 | ${{L}_2}$ | 利用实例特征构造图关系知识,解决由于师生模型架构不同 导致的性能下降和类间差异性低的问题 |
| 文献[68] | 样本特征向量 | 核方法 | ${{L}_2}$ | 考虑样本内容一致性和样本关系一致性,解除师生模型的实例强约束限制 |
| 文献[69] | 中间层特征图 | 转置相乘 | ${{L}_2}$ | 利用样本相似性向学生模型传递教师模型的激活模式,推动知识获取的发展 |
| 文献[57] | 样本特征向量 | 互信息 | 最大互信息 | 引入对比损失表达样本关系,处理先前方法忽略教师模型结构信息的问题 |
| 文献[63] | 中间层特征图 | 转置相乘 | MSE | 利用相似度矩阵和注意力机制捕获中间层语义关联,解决手动关联不匹配问题 |
| 文献[64] | 中间层特征图 | 转置相乘 | ${{L}_2}$ | 提出通道间相似性知识蒸馏,解决教师模型特征多样性不足的问题 |
| 文献[70] | 样本特征向量 | 余弦相似度 | InfoNCE | 提出词袋聚合蒸馏传递关系知识,解决基于对比学习方法优化困难的问题 |
| 文献[65] | 概率分布输出 中间层特征图 | 转置相乘 | KL L2 | 从样本实例、批次和类别角度构造关系知识,解决基于概率 分布输出蒸馏方法性能欠佳的问题 |
), ArticleFig(id=1325490749529268361, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1325490414429545387, language=EN, label=Table 5, caption=
Summary of ViT Architecture Relative Distillation Methods, figureFileSmall=null, figureFileBig=null, tableContent=
| 方法来源 | 知识的来源 | 知识的转换 | 架构 | 距离度量 | 关键技术 |
|---|
| 注:KL,CE,MSE,${{L}_{1}}$,${{L}_{2}}$分别表示KL散度、交叉熵、均方误差、曼哈顿距离和欧氏距离。 |
| 文献[17] | CNN概率分布输出 | 蒸馏token | CNN→ViT | KL | 引入一个蒸馏token接收卷积教师模型的知识, 解决ViT在数据不足时的泛化性问题 |
| 文献[76] | QKV矩阵 | 线性层 | ViT→CNN | ${{L}_2}$ | 提出交叉注意力分组线性映射对齐师生模型特征, 缓解跨架构蒸馏的语义对齐问题 |
| 文献[73] | CNN概率分布输出、 INN概率分布输出 | 蒸馏token | CNN→ViT | KL | 引入多个蒸馏token接收卷积网络和内卷网络的知识, 探究了归纳偏差的作用 |
| 文献[82] | 分类token | 稀疏编码 | ViT→ViT | CE | 将教师模型推理与蒸馏分离,缓解资源有限设备上的局限性 |
| 文献[83] | 分类token、QKV 矩阵、MLP输出 | 自注意力矩阵 | ViT→ViT | CE | 提出权重复用和权重蒸馏,缓解在资源有限设备上的适用性 |
| 文献[84] | MLP输出 | 线性层、掩码 | ViT→ViT | ${{L}_2}$ | 提出了一种基于特征的ViT蒸馏方法,缓解CNN架构的 蒸馏方法无法应用到ViT架构上的问题 |
| 文献[75] | CNN Logits输出 | 全局池化层 | CNN→ViT | KL | 提出累计空间知识蒸馏,缓解跨架构语义对齐和CNN 知识抑制模型后期训练的问题 |
| 文献[86] | QKV矩阵 | 自注意力矩阵 | ViT→ViT | KL | 将MIM的知识转移到较小的模型中,解决小模型 只能从预训练方法中获益的问题 |
| 文献[87] | 分类token、 图像掩码块 | 无 | ViT→ViT | CE | 通过分类token和MIM损失进行自监督训练, 缓解小样本下ViT的性能损失问题 |
| 文献[88] | MLP输出 | 线性层 | ViT→ViT | ${{L}_1}$ | 利用MAE预训练模型提取知识,缓解蒸馏计算效率的问题 |
| 文献[90] | MLP输出 | 线性层 MHA | ViT→ViT | ${{L}_1}$ | 提出非对称掩码蒸馏,缓解教师模型高掩码率信息丢失问题 |
| 文献[77] | MHA特征图 | 自注意力矩阵 | ViT→CNN | MSE | 将像素级特征映射到具有统一感受野的局部特征中, 解决师生模型特征的对齐问题 |
| 文献[85] | MHA特征图 | 余弦相似度 | ViT→ViT | ${{L}_1}$ | 提出多头注意力对齐策略,解决师生模型输出头对齐问题 |
| 文献[78] | MLP输出 中间层特征图 | 特征投影层 | ViT→CNN CNN→ViT | KL | 提出通过将特征映射到Logits空间,解决异构模型的特征对齐问题 |
| 文献[79] | MLP输出中间 层特征图 | 助理模型 | ViT→CNN CNN→ViT | InfoNCE KL | 引入一个辅助模型来促进异构教师模型和学生模型之间的特征传递 |
| 文献[80] | MHA特征、 分类token | 线性层 | ViT→CNN | KL CE | 通过多个不同归纳偏差的学生模型相互学习,来缓解跨架构蒸馏的性能问题 |
| 文献[81] | MLP输出 | 特征适应模块 | ViT→CNN | ${{L}_2}$ | 提出特征适应块处理教师模型特征,缓解异构特征的不匹配 |
), ArticleFig(id=1325490751211184267, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1325490414429545387, language=CN, label=表5, caption=
ViT 架构相关蒸馏方法汇总
, figureFileSmall=null, figureFileBig=null, tableContent=
| 方法来源 | 知识的来源 | 知识的转换 | 架构 | 距离度量 | 关键技术 |
|---|
| 注:KL,CE,MSE,${{L}_{1}}$,${{L}_{2}}$分别表示KL散度、交叉熵、均方误差、曼哈顿距离和欧氏距离。 |
| 文献[17] | CNN概率分布输出 | 蒸馏token | CNN→ViT | KL | 引入一个蒸馏token接收卷积教师模型的知识, 解决ViT在数据不足时的泛化性问题 |
| 文献[76] | QKV矩阵 | 线性层 | ViT→CNN | ${{L}_2}$ | 提出交叉注意力分组线性映射对齐师生模型特征, 缓解跨架构蒸馏的语义对齐问题 |
| 文献[73] | CNN概率分布输出、 INN概率分布输出 | 蒸馏token | CNN→ViT | KL | 引入多个蒸馏token接收卷积网络和内卷网络的知识, 探究了归纳偏差的作用 |
| 文献[82] | 分类token | 稀疏编码 | ViT→ViT | CE | 将教师模型推理与蒸馏分离,缓解资源有限设备上的局限性 |
| 文献[83] | 分类token、QKV 矩阵、MLP输出 | 自注意力矩阵 | ViT→ViT | CE | 提出权重复用和权重蒸馏,缓解在资源有限设备上的适用性 |
| 文献[84] | MLP输出 | 线性层、掩码 | ViT→ViT | ${{L}_2}$ | 提出了一种基于特征的ViT蒸馏方法,缓解CNN架构的 蒸馏方法无法应用到ViT架构上的问题 |
| 文献[75] | CNN Logits输出 | 全局池化层 | CNN→ViT | KL | 提出累计空间知识蒸馏,缓解跨架构语义对齐和CNN 知识抑制模型后期训练的问题 |
| 文献[86] | QKV矩阵 | 自注意力矩阵 | ViT→ViT | KL | 将MIM的知识转移到较小的模型中,解决小模型 只能从预训练方法中获益的问题 |
| 文献[87] | 分类token、 图像掩码块 | 无 | ViT→ViT | CE | 通过分类token和MIM损失进行自监督训练, 缓解小样本下ViT的性能损失问题 |
| 文献[88] | MLP输出 | 线性层 | ViT→ViT | ${{L}_1}$ | 利用MAE预训练模型提取知识,缓解蒸馏计算效率的问题 |
| 文献[90] | MLP输出 | 线性层 MHA | ViT→ViT | ${{L}_1}$ | 提出非对称掩码蒸馏,缓解教师模型高掩码率信息丢失问题 |
| 文献[77] | MHA特征图 | 自注意力矩阵 | ViT→CNN | MSE | 将像素级特征映射到具有统一感受野的局部特征中, 解决师生模型特征的对齐问题 |
| 文献[85] | MHA特征图 | 余弦相似度 | ViT→ViT | ${{L}_1}$ | 提出多头注意力对齐策略,解决师生模型输出头对齐问题 |
| 文献[78] | MLP输出 中间层特征图 | 特征投影层 | ViT→CNN CNN→ViT | KL | 提出通过将特征映射到Logits空间,解决异构模型的特征对齐问题 |
| 文献[79] | MLP输出中间 层特征图 | 助理模型 | ViT→CNN CNN→ViT | InfoNCE KL | 引入一个辅助模型来促进异构教师模型和学生模型之间的特征传递 |
| 文献[80] | MHA特征、 分类token | 线性层 | ViT→CNN | KL CE | 通过多个不同归纳偏差的学生模型相互学习,来缓解跨架构蒸馏的性能问题 |
| 文献[81] | MLP输出 | 特征适应模块 | ViT→CNN | ${{L}_2}$ | 提出特征适应块处理教师模型特征,缓解异构特征的不匹配 |
), ArticleFig(id=1325490755422265484, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1325490414429545387, language=EN, label=Table 6, caption=
Summary of Learning Methods of Knowledge Distillation, figureFileSmall=null, figureFileBig=null, tableContent=
| 方法来源 | 学习方法 | 关键技术 |
|---|
| 文献[115] | 无数据学习 | 提出利用网络的元数据蒸馏,无需访问原始数据,解决数据隐私问题 |
| 文献[30] | 互学习 | 提出让多个模型互相学习其输出,在没有强大教师模型也能提高模型性能 |
| 文献[102] | 共享学习 | 使每个模型在不相交的数据子集上训练并迁移其他模型知识来降低复杂性 |
| 文献[103] | 协同学习 | 提出协同学习,在相同数据上训练多个分类器,提高模型的泛化能力 |
| 文献[36] | 共享学习 | 通过共享师生模型架构的低级别参数来降低推理所需要的时间 |
| 文献[122] | 无数据学习 | 提出一种利用GAN合成数据的方法,缓解蒸馏中的隐私问题 |
| 文献[109] | 自学习 | 提出将模型中发现的知识提炼回自身来避免训练计算开销大的教师模型 |
| 文献[108] | 自学习 | 提出自蒸馏的概念来缓解网络规模增加带来的计算存储成本 |
| 文献[116] | 无数据学习 | 将softmax函数建模为狄利克雷分布抽取教师模型数据分布,缓解数据隐私问题 |
| 文献[110] | 自学习 | 将模型训练早期的知识转移到模型训练后期来降低训练的计算资源消耗 |
| 文献[104] | 协同学习 | 提出多个辅助网络与主网络进行2级蒸馏,提高多样性来缓解学生模型同质化问题 |
| 文献[111] | 自学习 | 对自蒸馏进行理论分析,自蒸馏迭代逐渐限制表示解决方案的基函数数量 |
| 文献[38] | 助理学习 | 提出通过助理模型缓解师生模型的能力差距问题 |
| 文献[143] | 助理学习 | 通过助理模型建立师生模型层之间1对1的匹配关系,缓解师生模型的能力差距问题 |
| 文献[106] | 协作学习 | 构建分支网络,分支增强后与另一分支分类器集成后互相学习来构建高水平教师模型 |
| 文献[144] | 助理学习 | 集成了多个助理模型来缩减师生模型性能差异,实现了更好的蒸馏性能 |
| 文献[145] | 助理学习 | 提出通过助理模型学习师生模型之间的残差来弥合两者学习能力的差异 |
| 文献[37] | 助理学习 | 利用高层助理模型和教师模型指导低层助理模型和学生模型缓解错误累计问题 |
| 文献[128] | 无数据学习 | 提出动量对抗蒸馏来缓解合成数据的分布偏移问题 |
| 文献[98] | 互学习 | 通过自适应切换阈值互换师生模型角色,缓解师生模型能力差距带来的性能影响 |
| 文献[129] | 无数据学习 | 提出通过双判别器对抗蒸馏来解决数据不可用的问题 |
| 文献[112] | 自学习 | 通过利用上一次迭代结果更新自蒸馏模型来保持模型的一致性 |
| 文献[130] | 无数据学习 | 利用条件生成对抗网络改进合成样本的歧义性和多样性不足的问题 |
| 文献[117] | 无数据学习 | 提出利用重用共享的共同特征来解决数据合成效率低下的问题 |
| 文献[146] | 助理学习 | 同时考虑了输出和中间特征来训练助理模型,更好地缓解了师生模型之间的性能差异 |
| 文献[101] | 互学习 | 在教师模型中添加可学习适配器模块来缓解师生模型预测分布尖锐度差异 |
| 文献[99] | 互学习 | 提出对比互学习框架来从其他网络学习对比知识以增强特征表示能力 |
| 文献[131] | 无数据学习 | 提出一个元学习启发框架来解决生成的伪样本分布不稳定的问题 |
| 文献[100] | 互学习 | 提出自适应特征蒸馏和在线任务加权来缓解多任务学习中准确率受损问题 |
| 文献[105] | 协作学习 | 提出自适应分支融合框架提高模型多样性来缓解学生模型集成的同质化 |
| 文献[113] | 自学习 | 将模型前几层知识蒸馏到后几层,通过自我反思进行更有效的学习 |
| 文献[118] | 无数据学习 | 提出信道特征交换和多尺度区域一致性,处理数据合成在多样性和效率方面的局限性 |
| 文献[119] | 无数据学习 | 通过逐渐生成由易到难的伪样本,模拟人类学习方式解决学习不充分不稳定的问题 |
| 文献[147] | 助理学习 | 令助理模型和学生模型相互学习,通过离线在线联合蒸馏的方法缓解师生模型差异 |
| 文献[148] | 助理学习 | 构造自适应助理模型,提供多样化知识并缩小师生模型之间的能力差距 |
| 文献[120] | 无数据学习 | 利用师生模型的差异生成样本来解决无数据蒸馏中样本质量不理想的问题 |
| 文献[79] | 助理学习 | 通过助理模型来进行跨架构的蒸馏,弥合异构模型之间的巨大特征差异 |
), ArticleFig(id=1325490758492496016, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1325490414429545387, language=CN, label=表6, caption=
知识蒸馏的学习方式汇总
, figureFileSmall=null, figureFileBig=null, tableContent=
| 方法来源 | 学习方法 | 关键技术 |
|---|
| 文献[115] | 无数据学习 | 提出利用网络的元数据蒸馏,无需访问原始数据,解决数据隐私问题 |
| 文献[30] | 互学习 | 提出让多个模型互相学习其输出,在没有强大教师模型也能提高模型性能 |
| 文献[102] | 共享学习 | 使每个模型在不相交的数据子集上训练并迁移其他模型知识来降低复杂性 |
| 文献[103] | 协同学习 | 提出协同学习,在相同数据上训练多个分类器,提高模型的泛化能力 |
| 文献[36] | 共享学习 | 通过共享师生模型架构的低级别参数来降低推理所需要的时间 |
| 文献[122] | 无数据学习 | 提出一种利用GAN合成数据的方法,缓解蒸馏中的隐私问题 |
| 文献[109] | 自学习 | 提出将模型中发现的知识提炼回自身来避免训练计算开销大的教师模型 |
| 文献[108] | 自学习 | 提出自蒸馏的概念来缓解网络规模增加带来的计算存储成本 |
| 文献[116] | 无数据学习 | 将softmax函数建模为狄利克雷分布抽取教师模型数据分布,缓解数据隐私问题 |
| 文献[110] | 自学习 | 将模型训练早期的知识转移到模型训练后期来降低训练的计算资源消耗 |
| 文献[104] | 协同学习 | 提出多个辅助网络与主网络进行2级蒸馏,提高多样性来缓解学生模型同质化问题 |
| 文献[111] | 自学习 | 对自蒸馏进行理论分析,自蒸馏迭代逐渐限制表示解决方案的基函数数量 |
| 文献[38] | 助理学习 | 提出通过助理模型缓解师生模型的能力差距问题 |
| 文献[143] | 助理学习 | 通过助理模型建立师生模型层之间1对1的匹配关系,缓解师生模型的能力差距问题 |
| 文献[106] | 协作学习 | 构建分支网络,分支增强后与另一分支分类器集成后互相学习来构建高水平教师模型 |
| 文献[144] | 助理学习 | 集成了多个助理模型来缩减师生模型性能差异,实现了更好的蒸馏性能 |
| 文献[145] | 助理学习 | 提出通过助理模型学习师生模型之间的残差来弥合两者学习能力的差异 |
| 文献[37] | 助理学习 | 利用高层助理模型和教师模型指导低层助理模型和学生模型缓解错误累计问题 |
| 文献[128] | 无数据学习 | 提出动量对抗蒸馏来缓解合成数据的分布偏移问题 |
| 文献[98] | 互学习 | 通过自适应切换阈值互换师生模型角色,缓解师生模型能力差距带来的性能影响 |
| 文献[129] | 无数据学习 | 提出通过双判别器对抗蒸馏来解决数据不可用的问题 |
| 文献[112] | 自学习 | 通过利用上一次迭代结果更新自蒸馏模型来保持模型的一致性 |
| 文献[130] | 无数据学习 | 利用条件生成对抗网络改进合成样本的歧义性和多样性不足的问题 |
| 文献[117] | 无数据学习 | 提出利用重用共享的共同特征来解决数据合成效率低下的问题 |
| 文献[146] | 助理学习 | 同时考虑了输出和中间特征来训练助理模型,更好地缓解了师生模型之间的性能差异 |
| 文献[101] | 互学习 | 在教师模型中添加可学习适配器模块来缓解师生模型预测分布尖锐度差异 |
| 文献[99] | 互学习 | 提出对比互学习框架来从其他网络学习对比知识以增强特征表示能力 |
| 文献[131] | 无数据学习 | 提出一个元学习启发框架来解决生成的伪样本分布不稳定的问题 |
| 文献[100] | 互学习 | 提出自适应特征蒸馏和在线任务加权来缓解多任务学习中准确率受损问题 |
| 文献[105] | 协作学习 | 提出自适应分支融合框架提高模型多样性来缓解学生模型集成的同质化 |
| 文献[113] | 自学习 | 将模型前几层知识蒸馏到后几层,通过自我反思进行更有效的学习 |
| 文献[118] | 无数据学习 | 提出信道特征交换和多尺度区域一致性,处理数据合成在多样性和效率方面的局限性 |
| 文献[119] | 无数据学习 | 通过逐渐生成由易到难的伪样本,模拟人类学习方式解决学习不充分不稳定的问题 |
| 文献[147] | 助理学习 | 令助理模型和学生模型相互学习,通过离线在线联合蒸馏的方法缓解师生模型差异 |
| 文献[148] | 助理学习 | 构造自适应助理模型,提供多样化知识并缩小师生模型之间的能力差距 |
| 文献[120] | 无数据学习 | 利用师生模型的差异生成样本来解决无数据蒸馏中样本质量不理想的问题 |
| 文献[79] | 助理学习 | 通过助理模型来进行跨架构的蒸馏,弥合异构模型之间的巨大特征差异 |
), ArticleFig(id=1325490760480596113, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1325490414429545387, language=EN, label=Table 7, caption=
Summary of Optimization Methods for Knowledge Distillation, figureFileSmall=null, figureFileBig=null, tableContent=
| 方法来源 | 优化位置 | 优化方法 | 关键技术 |
|---|
| 文献[150] | 训练过程 | 提前停止 | 提出提前停止优化方法,缓解蒸馏损失后期对模型的损害 |
| 文献[176] | 损失函数 | 对抗性技术 | 提出利用分块训练损失来从教师模型中提取多样化知识,缓解知识单一的问题 |
| 文献[179] | 训练过程 | 对抗性技术 | 利用对抗攻击发现决策边界样本,传递准确的边界信息来提升性能 |
| 文献[151] | 训练过程 | 正则化约束 | 性能较弱的模型能够提供正则化约束给性能较强的模型来提升性能 |
| 文献[38] | 训练过程 | 辅助模型 | 引入一个助理模型来弥合师生模型的性能差距,缓解师生模型能力差距带来的性能下降 |
| 文献[158] | 知识结构 | 特征筛选 | 提出选择性转移策略,利用样本变换带来的暗知识以提高模型性能 |
| 文献[166] | 温度系数 | 动态温度 | 利用倒数第2层特征的${{L}_2}$范数作为温度系数来缓解标签噪声的影响 |
| 文献[177] | 训练过程 | 对抗性技术 | 设计任务驱动的注意力机制,构造紧凑的知识结构以减少信息丢失 |
| 文献[186] | 模型架构 | 强化学习 | 设计奖励函数寻找教师模型的最佳学生模型,缓解师生模型能力差距带来的性能问题 |
| 文献[187] | 模型架构 | 自动机器学习 | 提出一种基于预言机的搜索算法来缓解师生模型能力差距带来的性能问题 |
| 文献[185] | 模型架构 | 自动机器学习 | 提出一种分块的搜索评估方案以保证候选框架得到公平的训练 |
| 文献[167] | 温度系数 | 动态温度 | 通过退火函数生成动态温度来缓解师生模型能力差距过大带来的性能下降 |
| 文献[152] | 训练过程 | | 综合探究影响蒸馏性能的原因,优化不佳是主要原因 |
| 文献[178] | 损失函数 | 对抗性技术 | 提出自我破坏知识蒸馏来缓解蒸馏知识产权暴露的风险 |
| 文献[172] | 损失函数 | 解耦 | 提出将损失函数解耦来缓解经典蒸馏损失本身内部的冲突博弈 |
| 文献[153] | 训练过程 | 非参数化对齐 | 提出通过奇异值分解对齐师生模型来缓解投射层性能效率不佳的问题 |
| 文献[154] | 训练过程 | 推理蒸馏分离 | 通过保存教师模型推理的数据增强及输出来缓解蒸馏过程教师模型推理的开销 |
| 文献[159] | 知识结构 | 特征筛选 | 提出知识凝聚蒸馏,对知识进行分级来处理蒸馏知识冗余的问题 |
| 文献[160] | 知识结构 | 皮尔森系数 | 利用师生模型预测输出类别的相对关系来处理师生模型预测的差异问题 |
| 文献[112] | 知识结构 | 批次结果重用 | 提出小批量自蒸馏算法,利用上一批次的信息缓解难以并行化的问题 |
| 文献[155] | 训练过程 | 特征路由 | 提出点自适应位置蒸馏来处理师生模型蒸馏对齐位置不确定的问题 |
| 文献[188] | 模型架构 | 自动机器学习 | 提出小规模数据集ViT搜索方案来缓解师生模型能力差距带来的性能问题 |
| 文献[191] | 训练设置 | 自动机器学习 | 构造蒙特卡洛搜索空间探寻最佳设置来缓解手工设计蒸馏方案的压力 |
| 文献[189] | 模型架构 | 自动机器学习 | 通过衡量样本间相似性寻找最佳师生模型架构来解决师生模型能力差距带来的性能下降 |
| 文献[180] | 训练过程 | 对抗性技术 | 通过构造给定输入球约束内对抗样本集合挖掘决策边界来提升模型性能 |
| 文献[181] | 训练过程 | 对抗性技术 | 提出一种重加权策略的引导对抗对比蒸馏来帮助学生模型继承对抗鲁棒性 |
| 文献[182] | 训练过程 | 对抗性技术 | 设计了自适应对抗性蒸馏方案来缓解先前模型过度修正的问题 |
| 文献[184] | 训练过程 | 对抗性技术 | 利用多个实例之间的结构化关系来缓解教师模型不稳定预测的影响 |
| 文献[173] | 损失函数 | 解耦 | 通过对解耦后的输出归一化来处理师生模型输出不一致的问题 |
| 文献[175] | 损失函数 | 解耦 | 调整优化器动量值让蒸馏损失占据主导,打破不同损失的冲突博弈 |
| 文献[162] | 知识结构 | 数据筛选 | 通过困难样本挖掘和校准来缓解教师模型软标签置信度可靠性问题 |
| 文献[156] | 训练过程 | 正则化约束 | 提出混合权重和间歇融合寻找更平坦的最优解来提高模型泛化性 |
| 文献[161] | 知识结构 | 困难样本挖掘 | 通过筛选易错样本加强学习来缓解教师模型传播错误知识的风险 |
| 文献[168] | 温度系数 | 对抗温度 | 利用温度系数调整任务难度来更合理地为学生模型设置学习进程 |
| 文献[169] | 温度系数 | 标准化Logits温度 | 将Logits输出标准差作为温度系数来缓解样本知识传递不充分的问题 |
| 文献[163] | 知识结构 | 正交投影层 | 提出在Stiefel流形上构建正交投影层来提高知识传递效率 |
| 文献[170] | 温度系数 | 师生模型差异温度 | 提出通过平衡输出特征的平滑度来解决不同难度样本的知识转移问题 |
| 文献[171] | 温度系数 | 标准化Logits温度 | 提出对Logits输出进行标准化来解除师生模型输出在范围和方差上的耦合 |
| 文献[183] | 训练过程 | 对抗性技术 | 提出公平的鲁棒性蒸馏框架来解决鲁棒性的类别差异性 |
| 文献[194] | 训练过程 | 扩散模型 | 提出扩散模型去噪模块来解决蒸馏特征中的噪声问题 |
| 文献[190] | 模型架构 | 自动机器学习 | 通过遗传算法寻找最佳师生模型架构来缓解师生模型能力差距带来的性能问题 |
| 文献[174] | 损失函数 | 解耦、序列化优化 | 将真实标签和教师模型标签的优化任务序列化进行,缓解不同分布优化任务的冲突 |
| 文献[157] | 训练过程 | 优化路径调整 | 通过将学生模型与教师模型交叉蒸馏来缓解真实标签与教师模型错误分类之间的冲突问题 |
| 文献[165] | 知识结构 | 错误标签掩码 | 利用掩码屏蔽教师模型的错误分类来减少教师模型的错误输出对学生模型的负面影响 |
), ArticleFig(id=1325490763500494996, tenantId=1146029695717560320, journalId=1324759914146529478, articleId=1325490414429545387, language=CN, label=表7, caption=
知识蒸馏的优化方法汇总
, figureFileSmall=null, figureFileBig=null, tableContent=
| 方法来源 | 优化位置 | 优化方法 | 关键技术 |
|---|
| 文献[150] | 训练过程 | 提前停止 | 提出提前停止优化方法,缓解蒸馏损失后期对模型的损害 |
| 文献[176] | 损失函数 | 对抗性技术 | 提出利用分块训练损失来从教师模型中提取多样化知识,缓解知识单一的问题 |
| 文献[179] | 训练过程 | 对抗性技术 | 利用对抗攻击发现决策边界样本,传递准确的边界信息来提升性能 |
| 文献[151] | 训练过程 | 正则化约束 | 性能较弱的模型能够提供正则化约束给性能较强的模型来提升性能 |
| 文献[38] | 训练过程 | 辅助模型 | 引入一个助理模型来弥合师生模型的性能差距,缓解师生模型能力差距带来的性能下降 |
| 文献[158] | 知识结构 | 特征筛选 | 提出选择性转移策略,利用样本变换带来的暗知识以提高模型性能 |
| 文献[166] | 温度系数 | 动态温度 | 利用倒数第2层特征的${{L}_2}$范数作为温度系数来缓解标签噪声的影响 |
| 文献[177] | 训练过程 | 对抗性技术 | 设计任务驱动的注意力机制,构造紧凑的知识结构以减少信息丢失 |
| 文献[186] | 模型架构 | 强化学习 | 设计奖励函数寻找教师模型的最佳学生模型,缓解师生模型能力差距带来的性能问题 |
| 文献[187] | 模型架构 | 自动机器学习 | 提出一种基于预言机的搜索算法来缓解师生模型能力差距带来的性能问题 |
| 文献[185] | 模型架构 | 自动机器学习 | 提出一种分块的搜索评估方案以保证候选框架得到公平的训练 |
| 文献[167] | 温度系数 | 动态温度 | 通过退火函数生成动态温度来缓解师生模型能力差距过大带来的性能下降 |
| 文献[152] | 训练过程 | | 综合探究影响蒸馏性能的原因,优化不佳是主要原因 |
| 文献[178] | 损失函数 | 对抗性技术 | 提出自我破坏知识蒸馏来缓解蒸馏知识产权暴露的风险 |
| 文献[172] | 损失函数 | 解耦 | 提出将损失函数解耦来缓解经典蒸馏损失本身内部的冲突博弈 |
| 文献[153] | 训练过程 | 非参数化对齐 | 提出通过奇异值分解对齐师生模型来缓解投射层性能效率不佳的问题 |
| 文献[154] | 训练过程 | 推理蒸馏分离 | 通过保存教师模型推理的数据增强及输出来缓解蒸馏过程教师模型推理的开销 |
| 文献[159] | 知识结构 | 特征筛选 | 提出知识凝聚蒸馏,对知识进行分级来处理蒸馏知识冗余的问题 |
| 文献[160] | 知识结构 | 皮尔森系数 | 利用师生模型预测输出类别的相对关系来处理师生模型预测的差异问题 |
| 文献[112] | 知识结构 | 批次结果重用 | 提出小批量自蒸馏算法,利用上一批次的信息缓解难以并行化的问题 |
| 文献[155] | 训练过程 | 特征路由 | 提出点自适应位置蒸馏来处理师生模型蒸馏对齐位置不确定的问题 |
| 文献[188] | 模型架构 | 自动机器学习 | 提出小规模数据集ViT搜索方案来缓解师生模型能力差距带来的性能问题 |
| 文献[191] | 训练设置 | 自动机器学习 | 构造蒙特卡洛搜索空间探寻最佳设置来缓解手工设计蒸馏方案的压力 |
| 文献[189] | 模型架构 | 自动机器学习 | 通过衡量样本间相似性寻找最佳师生模型架构来解决师生模型能力差距带来的性能下降 |
| 文献[180] | 训练过程 | 对抗性技术 | 通过构造给定输入球约束内对抗样本集合挖掘决策边界来提升模型性能 |
| 文献[181] | 训练过程 | 对抗性技术 | 提出一种重加权策略的引导对抗对比蒸馏来帮助学生模型继承对抗鲁棒性 |
| 文献[182] | 训练过程 | 对抗性技术 | 设计了自适应对抗性蒸馏方案来缓解先前模型过度修正的问题 |
| 文献[184] | 训练过程 | 对抗性技术 | 利用多个实例之间的结构化关系来缓解教师模型不稳定预测的影响 |
| 文献[173] | 损失函数 | 解耦 | 通过对解耦后的输出归一化来处理师生模型输出不一致的问题 |
| 文献[175] | 损失函数 | 解耦 | 调整优化器动量值让蒸馏损失占据主导,打破不同损失的冲突博弈 |
| 文献[162] | 知识结构 | 数据筛选 | 通过困难样本挖掘和校准来缓解教师模型软标签置信度可靠性问题 |
| 文献[156] | 训练过程 | 正则化约束 | 提出混合权重和间歇融合寻找更平坦的最优解来提高模型泛化性 |
| 文献[161] | 知识结构 | 困难样本挖掘 | 通过筛选易错样本加强学习来缓解教师模型传播错误知识的风险 |
| 文献[168] | 温度系数 | 对抗温度 | 利用温度系数调整任务难度来更合理地为学生模型设置学习进程 |
| 文献[169] | 温度系数 | 标准化Logits温度 | 将Logits输出标准差作为温度系数来缓解样本知识传递不充分的问题 |
| 文献[163] | 知识结构 | 正交投影层 | 提出在Stiefel流形上构建正交投影层来提高知识传递效率 |
| 文献[170] | 温度系数 | 师生模型差异温度 | 提出通过平衡输出特征的平滑度来解决不同难度样本的知识转移问题 |
| 文献[171] | 温度系数 | 标准化Logits温度 | 提出对Logits输出进行标准化来解除师生模型输出在范围和方差上的耦合 |
| 文献[183] | 训练过程 | 对抗性技术 | 提出公平的鲁棒性蒸馏框架来解决鲁棒性的类别差异性 |
| 文献[194] | 训练过程 | 扩散模型 | 提出扩散模型去噪模块来解决蒸馏特征中的噪声问题 |
| 文献[190] | 模型架构 | 自动机器学习 | 通过遗传算法寻找最佳师生模型架构来缓解师生模型能力差距带来的性能问题 |
| 文献[174] | 损失函数 | 解耦、序列化优化 | 将真实标签和教师模型标签的优化任务序列化进行,缓解不同分布优化任务的冲突 |
| 文献[157] | 训练过程 | 优化路径调整 | 通过将学生模型与教师模型交叉蒸馏来缓解真实标签与教师模型错误分类之间的冲突问题 |
| 文献[165] | 知识结构 | 错误标签掩码 | 利用掩码屏蔽教师模型的错误分类来减少教师模型的错误输出对学生模型的负面影响 |
)], attaches=null, journal=Journal(id=1324759606947315908, delFlag=0, nameCn=计算机研究与发展, nameEn=Journal of Computer Research and Development, nameHistory1=null, nameHistory2=null, issn=1000-1239, eissn=null, cn=11-1777/TP, coden=null, periodic=0, language=CN, oaType=null, ccby=null, superviseOffice=null, ownerOffice=null, pubOffice=null, editorOffice=null, officeType=null, aims=null, clcCode=null, officeProv=null, officeCity=null, officeAddr=null, officeZip=null, officeEmail=null, officePhone=null, editDirector=null, officeDirector=null, officeDirectorPhone=null, officeStaffNum=null, officeEmpNum=null, coverPicUrl=k2l8zgqwstuxxevFX2qxVA==, journalPrice=null, startedYear=null, abbrevIsoEn=Journal of Computer Research and Development, journalRemark=null, publicationField=null, createdTime=1793776903141, updatedTime=1793850698612, createdBy=18614031015, updatedBy=13701087609, firstLetterCn=J, firstLetterEn=J, subjectCode=Electronic Information, subjectName=null, subjectCodeEn=Electronic Information, subjectNameEn=null, picCn=k2l8zgqwstuxxevFX2qxVA==, picEn=r6k0Ry+RD+c/VXj5QarmDw==, jcr=null, cjcr=null, exts=[JournalExt(id=1325069127710286688, language=CN, name=计算机研究与发展, nameHistory1=null, nameHistory2=null, managedBy=, sponsoredBy=, publishedBy=, editorOffice=, officeProv=null, officeCity=null, officeAddr=, officeZip=, editDirector=, officeDirector=null, officePhone=null, coverPicUrl=null, journalRemark=, submitArticleUrl=null, websiteUrl=, createdTime=1793850698640, updatedTime=1793850698640, createdBy=13701087609, updatedBy=13701087609, submissionGuidelinesUrl=, submissionAuthorUrl=https://crad.ict.ac.cn/journalx/authorLogOn.action, submissionEditorUrl=https://crad.ict.ac.cn/journalx/editorLogOn.action, submissionReviewUrl=https://crad.ict.ac.cn/journalx/expertLogOn.action, submissionCeEditorUrl=https://crad.ict.ac.cn/journalx/editorInChiefLogOn.action, submissionAeEditorUrl=, option={"copyright":""}), JournalExt(id=1325069127777395553, language=EN, name=Journal of Computer Research and Development, nameHistory1=null, nameHistory2=null, managedBy=, sponsoredBy=, publishedBy=, editorOffice=, officeProv=null, officeCity=null, officeAddr=, officeZip=, editDirector=, officeDirector=null, officePhone=null, coverPicUrl=null, journalRemark=, submitArticleUrl=null, websiteUrl=, createdTime=1793850698656, updatedTime=1793850698656, createdBy=13701087609, updatedBy=13701087609, submissionGuidelinesUrl=, submissionAuthorUrl=https://crad.ict.ac.cn/journalx/authorLogOn.action, submissionEditorUrl=https://crad.ict.ac.cn/journalx/editorLogOn.action, submissionReviewUrl=https://crad.ict.ac.cn/journalx/expertLogOn.action, submissionCeEditorUrl=https://crad.ict.ac.cn/journalx/editorInChiefLogOn.action, submissionAeEditorUrl=, option={"copyright":""})], databaseList=null, tenantJournalId=1324759914146529478, websiteList=[Website(id=1325070300139892771, webName=null, webTitle=null, webDomain=null, webCopyrigh=null, webIpcNo=null, seoTitle=null, seoKeywords=null, seoDescription=null, tenantJournalId=null, journalId=1324759914146529478, journalNameCn=null, journalNameEn=null, grayFlag=null, tenantId=1146029695717560320, platformId=null, journalGroupId=null, journalGroupNameCn=null, journalGroupNameEn=null, type=1, domain=https://castjournals.cast.org.cn/joweb/jsjyjyfz/CN, language=CN, createTime=1793850978169, createBy=18614031015, updateTime=1793851040259, updateBy=18614031015, name=计算机研究与发展-中文, tplId=1146099689490845704, title=计算机研究与发展, delFlag=0, indexPage=/home, props=[WebsiteProps(id=1325070893751349360, tenantId=1146029695717560320, journalId=null, journalGroupId=null, siteId=1325070300139892771, code=articleTextType, value=kx, createTime=1793851119697, updateTime=1793851119697, creator=18614031015, updator=18614031015), WebsiteProps(id=1325070893730377837, tenantId=1146029695717560320, journalId=null, journalGroupId=null, siteId=1325070300139892771, code=banner, value=null, createTime=1793851119692, updateTime=1793851119692, creator=18614031015, updator=18614031015), WebsiteProps(id=1325070893772320883, tenantId=1146029695717560320, journalId=null, journalGroupId=null, siteId=1325070300139892771, code=grayFlag, value=0, createTime=1793851119702, updateTime=1793851119702, creator=18614031015, updator=18614031015), WebsiteProps(id=1325070893721989228, tenantId=1146029695717560320, journalId=null, journalGroupId=null, siteId=1325070300139892771, code=logo, value=https://castjournals.cast.org.cn/joweb/jsjyjyfz/CN/file/pic?fileId=U27tDsy717YF7XSPA+Vwdw==, createTime=1793851119690, updateTime=1793851119690, creator=18614031015, updator=18614031015), WebsiteProps(id=1325070893784903797, tenantId=1146029695717560320, journalId=null, journalGroupId=null, siteId=1325070300139892771, code=minRunFlag, value=0, createTime=1793851119705, updateTime=1793851119705, creator=18614031015, updator=18614031015), WebsiteProps(id=1325070893742960751, tenantId=1146029695717560320, journalId=null, journalGroupId=null, siteId=1325070300139892771, code=picServerUrl, value=https://castjournals.cast.org.cn/joweb/jsjyjyfz/CN/file/pic, createTime=1793851119695, updateTime=1793851119695, creator=18614031015, updator=18614031015), WebsiteProps(id=1325070893780709492, tenantId=1146029695717560320, journalId=null, journalGroupId=null, siteId=1325070300139892771, code=silenceFlag, value=0, createTime=1793851119704, updateTime=1793851119704, creator=18614031015, updator=18614031015), WebsiteProps(id=1325070893738766446, tenantId=1146029695717560320, journalId=null, journalGroupId=null, siteId=1325070300139892771, code=staticResourcePath, value=https://castjournals.cast.org.cn/joweb/cast_kjdb_cn_619/, createTime=1793851119694, updateTime=1793851119694, creator=18614031015, updator=18614031015), WebsiteProps(id=1325070893755543665, tenantId=1146029695717560320, journalId=null, journalGroupId=null, siteId=1325070300139892771, code=themeColor, value=null, createTime=1793851119698, updateTime=1793851119698, creator=18614031015, updator=18614031015), WebsiteProps(id=1325070893763932274, tenantId=1146029695717560320, journalId=null, journalGroupId=null, siteId=1325070300139892771, code=themeStyle, value=null, createTime=1793851119700, updateTime=1793851119700, creator=18614031015, updator=18614031015)]), Website(id=1325070300580294698, webName=null, webTitle=null, webDomain=null, webCopyrigh=null, webIpcNo=null, seoTitle=null, seoKeywords=null, seoDescription=null, tenantJournalId=null, journalId=1324759914146529478, journalNameCn=null, journalNameEn=null, grayFlag=null, tenantId=1146029695717560320, platformId=null, journalGroupId=null, journalGroupNameCn=null, journalGroupNameEn=null, type=1, domain=https://castjournals.cast.org.cn/joweb/jsjyjyfz/EN, language=EN, createTime=1793850978274, createBy=18614031015, updateTime=1793851036638, updateBy=18614031015, name=计算机研究与发展-英文, tplId=1146101810881728533, title=Journal of Computer Research and Development, delFlag=0, indexPage=/home, props=[WebsiteProps(id=1325070925288321148, tenantId=1146029695717560320, journalId=null, journalGroupId=null, siteId=1325070300580294698, code=articleTextType, value=kx, createTime=1793851127216, updateTime=1793851127216, creator=18614031015, updator=18614031015), WebsiteProps(id=1325070925263155321, tenantId=1146029695717560320, journalId=null, journalGroupId=null, siteId=1325070300580294698, code=banner, value=null, createTime=1793851127210, updateTime=1793851127210, creator=18614031015, updator=18614031015), WebsiteProps(id=1325070925309292671, tenantId=1146029695717560320, journalId=null, journalGroupId=null, siteId=1325070300580294698, code=grayFlag, value=0, createTime=1793851127221, updateTime=1793851127221, creator=18614031015, updator=18614031015), WebsiteProps(id=1325070925254766712, tenantId=1146029695717560320, journalId=null, journalGroupId=null, siteId=1325070300580294698, code=logo, value=https://castjournals.cast.org.cn/joweb/jsjyjyfz/EN/file/pic?fileId=U27tDsy717YF7XSPA+Vwdw==, createTime=1793851127208, updateTime=1793851127208, creator=18614031015, updator=18614031015), WebsiteProps(id=1325070925326069889, tenantId=1146029695717560320, journalId=null, journalGroupId=null, siteId=1325070300580294698, code=minRunFlag, value=0, createTime=1793851127225, updateTime=1793851127225, creator=18614031015, updator=18614031015), WebsiteProps(id=1325070925279932539, tenantId=1146029695717560320, journalId=null, journalGroupId=null, siteId=1325070300580294698, code=picServerUrl, value=https://castjournals.cast.org.cn/joweb/jsjyjyfz/EN/file/pic, createTime=1793851127214, updateTime=1793851127214, creator=18614031015, updator=18614031015), WebsiteProps(id=1325070925317681280, tenantId=1146029695717560320, journalId=null, journalGroupId=null, siteId=1325070300580294698, code=silenceFlag, value=0, createTime=1793851127223, updateTime=1793851127223, creator=18614031015, updator=18614031015), WebsiteProps(id=1325070925271543930, tenantId=1146029695717560320, journalId=null, journalGroupId=null, siteId=1325070300580294698, code=staticResourcePath, value=https://castjournals.cast.org.cn/joweb/cast_kjdb_en_623/, createTime=1793851127212, updateTime=1793851127212, creator=18614031015, updator=18614031015), WebsiteProps(id=1325070925292515453, tenantId=1146029695717560320, journalId=null, journalGroupId=null, siteId=1325070300580294698, code=themeColor, value=null, createTime=1793851127217, updateTime=1793851127217, creator=18614031015, updator=18614031015), WebsiteProps(id=1325070925300904062, tenantId=1146029695717560320, journalId=null, journalGroupId=null, siteId=1325070300580294698, code=themeStyle, value=null, createTime=1793851127219, updateTime=1793851127219, creator=18614031015, updator=18614031015)])], journalTitle=计算机研究与发展, weixinUrl=null, journalUrl=https://crad.ict.ac.cn/, iacademicId=null, status=1, seqNo=null, journalTitleEn=Journal of Computer Research and Development, journalPhotoCn=k2l8zgqwstuxxevFX2qxVA==, journalPhotoEn=r6k0Ry+RD+c/VXj5QarmDw==, journalFirstLetter=J, journalRecommend=null, journalNew=null, journalCollection=null, jcrJf=null, cjcrJf=null, jcrJfStr=null, cjcrJfStr=null, submissionFirstDecision=null, sciSubjectClassification=null, casSubjectClassification=null, citeScore=null, totalCitationFrequency=null, icpCode=null, psCode=null, advertisingLicenseCode=null, copyrightInformation=null, country=null, option=, provinceCode=null, provinceName=null, collectFlag=false, interPubPlatform=, interPubPlatformUrl=null), detailUrlCn=https://castjournals.cast.org.cn/joweb/jsjyjyfz/CN/10.7544/issn1000-1239.202440694, detailUrlEn=https://castjournals.cast.org.cn/joweb/jsjyjyfz/EN/10.7544/issn1000-1239.202440694, pdfUrlCn=https://castjournals.cast.org.cn/joweb/jsjyjyfz/CN/PDF/10.7544/issn1000-1239.202440694, pdfUrlEn=https://castjournals.cast.org.cn/joweb/jsjyjyfz/EN/PDF/10.7544/issn1000-1239.202440694, aliStartDate=0, aliEndDate=0, collectionFlag=false, citedCount=null, citedUrl=null, previewStatus=0, delFlag=0, hasFullText=1, orderTime=1767196800000, fullTextJson=null, articleText=null, reference=null)