publications

2026

1. Arxiv

   IndicIFEval: A Benchmark for Verifiable Instruction-Following Evaluation in 14 Indic Languages

   Thanmay Jayakumar, Mohammed Safi Ur Rahman Khan, Raj Dabre, Ratish Puduppully, and Anoop Kunchukuttan

   Under Review, 2026

   Bib PDF Data

   @article{jayakumar2026indicifeval,
     title = {IndicIFEval: A Benchmark for Verifiable Instruction-Following Evaluation in 14 Indic Languages},
     author = {Jayakumar, Thanmay and Khan, Mohammed Safi Ur Rahman and Dabre, Raj and Puduppully, Ratish and Kunchukuttan, Anoop},
     year = {2026},
     journal = {Under Review},
     data = {http://github.com/ai4bharat/IndicIFEval},
   }

2. COLM 2026

   Seeing Isn’t Believing: Uncovering Blind Spots in Evaluator Vision-Language Models

   Mohammed Safi Ur Rahman Khan, Sanjay Suryanarayanan, Tushar Anand, and Mitesh M. Khapra

   COLM 2026, 2026

   Bib PDF Code Data

   @article{khan2026seeing,
     title = {Seeing Isn't Believing: Uncovering Blind Spots in Evaluator Vision-Language Models},
     author = {Khan, Mohammed Safi Ur Rahman and Suryanarayanan, Sanjay and Anand, Tushar and Khapra, Mitesh M.},
     year = {2026},
     journal = {COLM 2026},
     data = {https://huggingface.co/datasets/ai4bharat/Focus},
   }

3. Interspeech 2026

   Preferences of a Voice-First Nation: Large-Scale Pairwise Evaluation and Preference Analysis for TTS in Indian Languages

   Srija Anand, Ashwin Sankar, Ishvinder Sethi, Aaditya Pareek, Kartik Rajput, Gaurav Yadav, Nikhil Narasimhan, Adish Pandya, Deepon Halder, Mohammed Safi Ur Rahman Khan, Praveen S, Shobhit Banga, and Mitesh M Khapra

   Interspeech 2026, 2026

   Bib PDF Data

   @article{anand2026preferences,
     title = {Preferences of a Voice-First Nation: Large-Scale Pairwise Evaluation and Preference Analysis for TTS in Indian Languages},
     author = {Anand, Srija and Sankar, Ashwin and Sethi, Ishvinder and Pareek, Aaditya and Rajput, Kartik and Yadav, Gaurav and Narasimhan, Nikhil and Pandya, Adish and Halder, Deepon and Khan, Mohammed Safi Ur Rahman and S, Praveen and Banga, Shobhit and Khapra, Mitesh M},
     year = {2026},
     journal = {Interspeech 2026},
     data = {https://huggingface.co/datasets/ai4bharat/SpeechArenaBench},
   }

2025

1. ACL 2025

   FairI Tales: Evaluation of Fairness in Indian Contexts with a Focus on Bias and Stereotypes

   Janki Atul Nawale^*, Mohammed Safi Ur Rahman Khan^*, Janani D, Mansi Gupta, Danish Pruthi, and Mitesh M. Khapra

   Proceedings of the 63rd Annual Meeting of the Association for Computational Linguistics (Volume 1: Long Papers), 2025

   Bib PDF Code Data

   @article{nawale2025fairi,
     title = {FairI Tales: Evaluation of Fairness in Indian Contexts with a Focus on Bias and Stereotypes},
     author = {Nawale, Janki Atul and Khan, Mohammed Safi Ur Rahman and D, Janani and Gupta, Mansi and Pruthi, Danish and Khapra, Mitesh M.},
     year = {2025},
     journal = {Proceedings of the 63rd Annual Meeting of the Association for Computational Linguistics (Volume 1: Long Papers)},
     data = {https://huggingface.co/datasets/ai4bharat/Indic-Bias},
   }

2. ACL 2025

   Can Vision-Language Models Evaluate Handwritten Math?

   Oikantik Nath, Hanani Bathina, Mohammed Safi Ur Rahman Khan, and Mitesh M. Khapra

   Proceedings of the 63rd Annual Meeting of the Association for Computational Linguistics (Volume 1: Long Papers), 2025

   Bib PDF Code Data

   @article{nath2025visionlanguage,
     title = {Can Vision-Language Models Evaluate Handwritten Math?},
     author = {Nath, Oikantik and Bathina, Hanani and Khan, Mohammed Safi Ur Rahman and Khapra, Mitesh M.},
     year = {2025},
     journal = {Proceedings of the 63rd Annual Meeting of the Association for Computational Linguistics (Volume 1: Long Papers)},
     data = {https://huggingface.co/datasets/ai4bharat/FERMAT},
   }

3. ACL 2025

   Cross-Lingual Auto Evaluation for Assessing Multilingual LLMs

   Sumanth Doddapaneni^*, Mohammed Safi Ur Rahman Khan^*, Dilip Venkatesh, Raj Dabre, Anoop Kunchukuttan, and Mitesh M. Khapra

   Proceedings of the 63rd Annual Meeting of the Association for Computational Linguistics (Volume 1: Long Papers), 2025

   Bib PDF Code Data

   @article{doddapaneni2024crosslingual,
     title = {Cross-Lingual Auto Evaluation for Assessing Multilingual LLMs},
     author = {Doddapaneni, Sumanth and Khan, Mohammed Safi Ur Rahman and Venkatesh, Dilip and Dabre, Raj and Kunchukuttan, Anoop and Khapra, Mitesh M.},
     year = {2025},
     journal = {Proceedings of the 63rd Annual Meeting of the Association for Computational Linguistics (Volume 1: Long Papers)},
     data = {https://huggingface.co/collections/ai4bharat/cia-suite-66ea9a7e18a6c70bd8de27a1},
   }

4. IJCNLP-AACL 2025

   Pralekha: An Indic Document Alignment Evaluation Benchmark

   Sanjay Suryanarayanan, Haiyue Song, Mohammed Safi Ur Rahman Khan, Anoop Kunchukuttan, and Raj Dabre

   Proceedings of the 13th International Joint Conference on Natural Language Processing and the 3rd Conference of the Asia-Pacific Chapter of the Association for Computational Linguistics, 2025

   Bib PDF Code Data

   @article{suryanarayanan2024pralekha,
     title = {Pralekha: An Indic Document Alignment Evaluation Benchmark},
     author = {Suryanarayanan, Sanjay and Song, Haiyue and Khan, Mohammed Safi Ur Rahman and Kunchukuttan, Anoop and Dabre, Raj},
     year = {2025},
     journal = {Proceedings of the 13th International Joint Conference on Natural Language Processing and the 3rd Conference of the Asia-Pacific Chapter of the Association for Computational Linguistics},
     data = {https://huggingface.co/datasets/ai4bharat/Pralekha},
   }

5. ACL 2025

   Towards Building Large Scale Datasets and State-of-the-Art Automatic Speech Translation Systems for 14 Indian Languages

   Sparsh Jain, Ashwin Sankar, Devilal Choudhary, Dhairya Suman, Nikhil Narasimhan, Mohammed Safi Ur Rahman Khan, Anoop Kunchukuttan, Mitesh M Khapra, and Raj Dabre

   Proceedings of the 63rd Annual Meeting of the Association for Computational Linguistics (Volume 1: Long Papers), 2025

   Bib PDF Code Data

   @article{jain2024bhasaanuvaad,
     title = {Towards Building Large Scale Datasets and State-of-the-Art Automatic Speech Translation Systems for 14 Indian Languages},
     author = {Jain, Sparsh and Sankar, Ashwin and Choudhary, Devilal and Suman, Dhairya and Narasimhan, Nikhil and Khan, Mohammed Safi Ur Rahman and Kunchukuttan, Anoop and Khapra, Mitesh M and Dabre, Raj},
     year = {2025},
     journal = {Proceedings of the 63rd Annual Meeting of the Association for Computational Linguistics (Volume 1: Long Papers)},
     data = {https://huggingface.co/collections/ai4bharat/bhasaanuvaad-672b3790b6470eab68b1cb87},
   }

6. NAACL 2025

   MILU: A Multi-task Indic Language Understanding Benchmark

   Sshubam Verma, Mohammed Safi Ur Rahman Khan, Vishwajeet Kumar, Rudra Murthy, and Jaydeep Sen

   Proceedings of the 2025 Conference of the Nations of the Americas Chapter of the Association for Computational Linguistics, 2025

   Bib PDF Code Data

   @article{verma2024milu,
     title = {MILU: A Multi-task Indic Language Understanding Benchmark},
     author = {Verma, Sshubam and Khan, Mohammed Safi Ur Rahman and Kumar, Vishwajeet and Murthy, Rudra and Sen, Jaydeep},
     year = {2025},
     journal = {Proceedings of the 2025 Conference of the Nations of the Americas Chapter of the Association for Computational Linguistics},
     data = {https://huggingface.co/datasets/ai4bharat/MILU},
   }

2024

1. EMNLP 2024

   

   Finding Blind Spots in Evaluator LLMs with Interpretable Checklists

   Sumanth Doddapaneni^*, Mohammed Safi Ur Rahman Khan^*, Sshubam Verma, and Mitesh M. Khapra

   Proceedings of the 2024 Conference on Empirical Methods in Natural Language Processing, Nov 2024

   Outstanding Paper Bib PDF Code Data

   EMNLP-2024 Outstanding Paper Award

   @article{doddapaneni2024finding,
     title = {Finding Blind Spots in Evaluator LLMs with Interpretable Checklists},
     author = {Doddapaneni, Sumanth and Khan, Mohammed Safi Ur Rahman and Verma, Sshubam and Khapra, Mitesh M.},
     year = {2024},
     journal = {Proceedings of the 2024 Conference on Empirical Methods in Natural Language Processing},
     month = nov,
     address = {Miami, Florida, USA},
     publisher = {Association for Computational Linguistics},
     pages = {16279--16309},
     data = {https://huggingface.co/datasets/ai4bharat/FBI},
   }

2. ACL 2024

   

   IndicLLMSuite: A Blueprint for Creating Pre-training and Fine-Tuning Datasets for Indian Languages

   Mohammed Safi Ur Rahman Khan^*, Priyam Mehta^*, Ananth Sankar, Umashankar Kumaravelan, Sumanth Doddapaneni, Suriyaprasaad G, Varun Balan G, Sparsh Jain, Anoop Kunchukuttan, Pratyush Kumar, Raj Dabre, and Mitesh M. Khapra

   Proceedings of the 62nd Annual Meeting of the Association for Computational Linguistics (Volume 1: Long Papers), Aug 2024

   Outstanding Paper Bib PDF Code Data

   ACL-2024 Outstanding Paper Award

   @article{khan2024indicllmsuite,
     title = {IndicLLMSuite: A Blueprint for Creating Pre-training and Fine-Tuning Datasets for Indian Languages},
     author = {Khan, Mohammed Safi Ur Rahman and Mehta, Priyam and Sankar, Ananth and Kumaravelan, Umashankar and Doddapaneni, Sumanth and G, Suriyaprasaad and G, Varun Balan and Jain, Sparsh and Kunchukuttan, Anoop and Kumar, Pratyush and Dabre, Raj and Khapra, Mitesh M.},
     year = {2024},
     journal = {Proceedings of the 62nd Annual Meeting of the Association for Computational Linguistics (Volume 1: Long Papers)},
     month = aug,
     address = {Bangkok, Thailand},
     publisher = {Association for Computational Linguistics},
     pages = {15831--15879},
     data = {https://huggingface.co/collections/ai4bharat/indicllmsuite-65ee7d225c337fcfa0991707},
   }

3. arXiv

   Airavata: Introducing Hindi Instruction-tuned LLM

   Jay Gala, Thanmay Jayakumar, Jaavid Aktar Husain, Aswanth Kumar M, Mohammed Safi Ur Rahman Khan, Diptesh Kanojia, Ratish Puduppully, Mitesh M. Khapra, Raj Dabre, Rudra Murthy, and Anoop Kunchukuttan

   arXiv preprint arXiv: 2401.15006, Aug 2024

   Bib PDF Code Data

   @article{gala2024airavata,
     title = {Airavata: Introducing Hindi Instruction-tuned LLM},
     author = {Gala, Jay and Jayakumar, Thanmay and Husain, Jaavid Aktar and M, Aswanth Kumar and Khan, Mohammed Safi Ur Rahman and Kanojia, Diptesh and Puduppully, Ratish and Khapra, Mitesh M. and Dabre, Raj and Murthy, Rudra and Kunchukuttan, Anoop},
     year = {2024},
     journal = {arXiv preprint arXiv: 2401.15006},
     data = {https://huggingface.co/datasets/ai4bharat/indic-instruct-data-v0.1},
   }