-
[pdf]
[bibtex]@InProceedings{Zhang_2025_WACV, author = {Zhang, Xinxi and Wen, Song and Han, Ligong and Juefei-Xu, Felix and Srivastava, Akash and Huang, Junzhou and Pavlovic, Vladimir and Wang, Hao and Tao, Molei and Metaxas, Dimitris}, title = {SODA: Spectral Orthogonal Decomposition Adaptation for Diffusion Models}, booktitle = {Proceedings of the Winter Conference on Applications of Computer Vision (WACV)}, month = {February}, year = {2025}, pages = {4665-4682} }
SODA: Spectral Orthogonal Decomposition Adaptation for Diffusion Models
Abstract
Adapting large-scale pre-trained generative models in a parameter-efficient manner is gaining traction. Traditional methods like low rank adaptation achieve parameter efficiency by imposing constraints but may not be optimal for tasks requiring high representation capacity. We propose a novel spectrum-aware adaptation framework for generative models. Our method adjusts both singular values and their basis vectors of pretrained weights. Using the Kronecker product and efficient Stiefel optimizers we achieve parameter-efficient adaptation of orthogonal matrices. Specifically we introduce Spectral Orthogonal Decomposition Adaptation (SODA) which balances computational efficiency and representation capacity. Extensive evaluations on text-to-image diffusion models demonstrate SODA's effectiveness offering a spectrum-aware alternative to existing fine-tuning methods.
Related Material