Abstract

High-quality multilingual training data is essential for effectively pretraining large language models (LLMs). Yet, the availability of suitable open-source multilingual datasets remains limited. Existing state-of-the-art datasets mostly rely on heuristic filtering methods, restricting both their cross-lingual transferability and scalability. Here, we introduce JQL, a systematic approach that efficiently curates diverse and high-quality multilingual data at scale while significantly reducing computational demands. JQL distills LLMs’ annotation capabilities into lightweight annotators based on pretrained multilingual embeddings. These models exhibit robust multilingual and cross-lingual performance, even for languages and scripts unseen during training. Evaluated empirically across 35 languages, the resulting annotation pipeline substantially outperforms current heuristic filtering methods like Fineweb2. JQL notably enhances downstream model training quality and increases data retention rates. Our research provides practical insights and valuable resources for multilingual data curation, raising the standards of multilingual dataset development.

BibTeX

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14

@inproceedings{aliJudgingQualityLanguages2025b,
  title = {Judging {{Quality Across Languages}}: {{A Multilingual Approach}} to {{Pretraining Data Filtering}} with {{Language Models}}},
  shorttitle = {Judging {{Quality Across Languages}}},
  booktitle = {Proceedings of the 2025 {{Conference}} on {{Empirical Methods}} in {{Natural Language Processing}}},
  author = {Ali, Mehdi and Brack, Manuel and L{\"u}bbering, Max and Wendt, Elias and Khan, Abbas Goher and Rutmann, Richard and Jude, Alex and Kraus, Maurice and Weber, Alexander Arno and Stollenwerk, Felix and Kacz{\'e}r, David and Mai, Florian and Flek, Lucie and Sifa, Rafet and {Flores-Herr}, Nicolas and Koehler, Joachim and Schramowski, Patrick and Fromm, Michael and Kersting, Kristian},
  year = 2025,
  pages = {8870--8909},
  publisher = {Association for Computational Linguistics},
  address = {Suzhou, China},
  doi = {10.18653/v1/2025.emnlp-main.449},
  urldate = {2026-06-27},
  langid = {english},
  file = {/Users/me/Zotero/storage/5IE6855P/Ali et al. - 2025 - Judging Quality Across Languages A Multilingual Approach to Pretraining Data Filtering with Languag.pdf}
}