@InProceedings{maskey-EtAl:2022:SIGUL,
  author    = {Maskey, Utsav  and  Bhatta, Manish  and  Bhatt, Shiva  and  Dhungel, Sanket  and  Bal, Bal Krishna},
  title     = {Nepali Encoder Transformers: An Analysis of Auto Encoding Transformer Language Models for Nepali Text Classification},
  booktitle      = {Proceedings of the the 1st Annual Meeting of the ELRA/ISCA Special Interest Group on Under-Resourced Languages},
  month          = {June},
  year           = {2022},
  address        = {Marseille, France},
  publisher      = {European Language Resources Association},
  pages     = {106--111},
  abstract  = {Language model pre-training has significantly impacted NLP and resulted in performance gains on many NLP-related tasks, but comparative study of different approaches on many low-resource languages seems to be missing. This paper attempts to investigate appropriate methods for pretraining a Transformer-based model for the Nepali language. We focus on the language-specific aspects that need to be considered for modeling. Although some language models have been trained for Nepali, the study is far from sufficient. We train three distinct Transformer-based masked language models for Nepali text sequences: distilbert-base (Sanh et al., 2019) for its efficiency and minuteness, deberta-base (P. He et al., 2020) for its capability of modeling the dependency of nearby token pairs and XLM-ROBERTa (Conneau et al., 2020) for its capabilities to handle multilingual downstream tasks. We evaluate and compare these models with other Transformer-based models on a downstream classification task with an aim to suggest an effective strategy for training low-resource language models and their fine-tuning.},
  url       = {https://aclanthology.org/2022.sigul-1.14}
}