writing

Google Scholar

ACL Anthology

SemanticScholar

OAR@UoM (open access repo)

Academia.edu

ORCiD

DBLP

publications

Publications with the keyword: evaluation

View all publications

2020
[29]
Gradations of error severity in automatic image description (; ; ; ; ; and ), In Proceedings of the 13th International Conference on Natural Language Genration (INLG'20), Association for Computational Linguistics, . 
@inproceedings{inlg2020errorseverity,
	Title = {Gradations of error severity in automatic image description},
	Author = {van Miltenburg, E and Lu, W-T and Krahmer, E and Gatt, A and Chen, G and Li, L and van Deemter, K},
	Booktitle = {Proceedings of the 13th International Conference on Natural Language Genration (INLG'20)},
	Year = {2020},
	Address = {Dublin, Ireland},
	Pages = {398-411},
	Publisher = {Association for Computational Linguistics},	
	Url = {https://www.aclweb.org/anthology/2020.inlg-1.45.pdf},
	Keywords = {natural language generation, vision and language, evaluation}		
}
[28]
Unmasking Contextual Stereotypes: Measuring and Mitigating BERT's Gender Bias (; and ), In Proceedings of the 2nd Workshop on Gender Bias in Natural Language Processing (GeBNLP 2020), Association for Computational Linguistics, . 
@inproceedings{gbnlp2020gender,
	Title = {Unmasking Contextual Stereotypes: Measuring and Mitigating {BERT}'s Gender Bias},
	Author = {Bartl, M and Nissim, M and Gatt, A},
	Booktitle = {Proceedings of the 2nd Workshop on Gender Bias in Natural Language Processing (GeBNLP 2020)},
	Year = {2020},
	Url = {https://www.aclweb.org/anthology/2020.gebnlp-1.1.pdf},
	Address = {Barcelona, Spain},
	Publisher = {Association for Computational Linguistics},	
	eprinttype = {arxiv},
	eprint={https://arxiv.org/abs/2010.14534},
	Keywords = {bias, evaluation}		
}
[arxiv]
[27]
On the interaction of automatic evaluation and task framing in headline style transfer (; ; ; ; and ), In Proceedings of the 1st Workshop on Evaluating NLG Evaluation (EvalNLGEval'20), Association for Computational Linguistics, . 
@inproceedings{evalnlgeval2020style,
	Title = {On the interaction of automatic evaluation and task framing in headline style transfer},
	Author = {De Mattei, L and Cafagna, M and Lai, H and Nissim, M and Dell'Orletta, F and Gatt, A},
	Booktitle = {Proceedings of the 1st Workshop on Evaluating NLG Evaluation (EvalNLGEval'20)},
	Year = {2020},
	Url = {https://evalnlg-workshop.github.io/papers/EvalNLGEval_2020_paper_8.pdf},
	Address = {Dublin, Ireland},
	Publisher = {Association for Computational Linguistics},
	eprinttype = {arxiv},
	eprint = {https://arxiv.org/abs/2101.01634},
	Keywords = {natural language generation, evaluation, style}		
}
[arxiv]
[26]
Human evaluation of automatically generated text: Current trends and best practice guidelines (; ; and ), Computer Speech and Language, volume 67, . 
@article{csleval2020,
	Title = {Human evaluation of automatically generated text: Current trends and best practice guidelines},
	Author = {van der Lee, C and Gatt, A and van Miltenburg, E and Krahmer, E},
	Year = {2020},
	Journal = {Computer Speech and Language},
	Volume = {67},
	Issue = {May, 2021},
	Url = {https://www.sciencedirect.com/science/article/abs/pii/S088523082030084X},
	Year = {2020},
	Doi = {https://doi.org/10.1016/j.csl.2020.101151},
	Keywords = {natural language generation, evaluation}
}
[doi]
[25]
CHANGE-IT: Change headlines, adapt news, generate (; ; ; and ), In Proceedings of the 7th Evaluation Campaign of Natural Language Processing and Speech Tools for Italian (EVALITA'20), . 
@inproceedings{changie-evalita2020,
	Title = {CHANGE-IT: Change headlines, adapt news, generate},
	Author = {De Mattei, L and Cafagna, M and Dell'Orletta, F and Nissim, M and Gatt, A},
	Booktitle = {Proceedings of the 7th Evaluation Campaign of Natural Language Processing and Speech Tools for Italian (EVALITA'20)},
	Year = {2020},
	Url = {http://ceur-ws.org/Vol-2765/paper169.pdf},
	Keywords = {natural language generation, evaluation}		
}
2019
[24]
Best Practices for the Human Evaluation of Automatically Generated Text (; ; ; and ), In Proceedings of the 12th International Conference on Natural Language Generation (INLG'19), Association for Computational Linguistics, . 
@inproceedings{inlg2019evaluation,
	Author = {van der Lee, C and Gatt, A and van Miltenburg, E and Wubben, S and Krahmer, E},
	Title = {Best Practices for the Human Evaluation of Automatically Generated Text},
	Booktitle = {Proceedings of the 12th International Conference on Natural Language Generation (INLG'19)},
	Year = {2019},
	Url = {https://www.aclweb.org/anthology/W19-8643/},
	Address = {Tokyo, Japan},
	supptype = {Supplementary data (XLSX format)},
	supplink = {https://www.aclweb.org/anthology/attachments/W19-8643.Supplementary_Attachment.xlsx},
	Publisher = {Association for Computational Linguistics},	
	Keywords = {natural language generation, evaluation}			
}
[Supplementary data (XLSX format)]
2012
[23]
Generation of referring expressions: Assessing the incremental algorithm (; ; and ), Cognitive science, volume 36, . 
@article{deemter2012generation,
	Author = {van Deemter, K and Gatt, A and van der Sluis, I and Power, R},
	Journal = {Cognitive science},
	Number = {5},
	Pages = {799--836},
	Title = {Generation of referring expressions: Assessing the incremental algorithm},
	Url = {https://staff.um.edu.mt/albert.gatt/pubs/incrementality_final-19-04-2011.pdf},
	Volume = {36},
	Year = {2012},
	Keywords = {natural language generation, referring expressions, evaluation}}
[22]
Assessing the Incremental Algorithm: a Response to Krahmer et al. (; ; and ), Cognitive science, volume 36, . 
@article{deemter2012assessing,
	Author = {van Deemter, K and Gatt, A and van der Sluis, I and Power, R},
	Journal = {Cognitive science},
	Number = {5},
	Pages = {842--845},
	Title = {Assessing the Incremental Algorithm: a Response to Krahmer et al.},
	Url = {https://staff.um.edu.mt/albert.gatt/pubs/incrementality_krahmer-response.pdf},
	Volume = {36},
	Year = {2012},
	Keywords = {referring expressions, natural language generation,evaluation}}
[21]
A repository of data and evaluation resources for natural language generation ( and ), In Proceedings of the 8th Language Resources and Evaluation Conference (LREC'12), ELRA, . 
@inproceedings{belz2012repository,
	Address = {Istanbul, Turkey},
	Author = {Belz, Anja and Gatt, Albert},
	Booktitle = {Proceedings of the 8th Language Resources and Evaluation Conference (LREC'12)},
	Publisher = {ELRA},
	Title = {A repository of data and evaluation resources for natural language generation},
	Url = {https://staff.um.edu.mt/albert.gatt/pubs/lrec2012_genchal.pdf},
	Year = {2012},
	Keywords = {language resources, evaluation, natural language generation}}
2011
[20]
What is in a text and what does it do: Qualitative Evaluations of an NLG system–the BT-Nurse–using content analysis and discourse analysis. (; ; ; ; ; and ), In Proceedings of the 13th European Workshop on Natural Language Generation (ENLG'11), Association for Computational Linguistics, . 
@inproceedings{sambaraju2011text,
	Address = {Nancy, France},
	Author = {Sambaraju, R and Reiter, E and Logie, R and McKinlay, A and McVittie, C and Gatt, A and Sykes, C},
	Booktitle = {Proceedings of the 13th European Workshop on Natural Language Generation (ENLG'11)},
	Pages = {22--31},
	Publisher = {Association for Computational Linguistics},
	Title = {What is in a text and what does it do: Qualitative Evaluations of an NLG system--the BT-Nurse--using content analysis and discourse analysis.},
	Url = {http://aclweb.org/anthology/W/W11/W11-2804.pdf},
	Year = {2011},
	Keywords = {natural language generation, data-to-text, evaluation}}
2010
[19]
Textual properties and task based evaluation: investigating the role of surface properties, structure and content ( and ), In Proceedings of the 6th International Natural Language Generation Conference (INLG'10), Association for Computational Linguistics, . 
@inproceedings{gatt2010textual,
	Address = {Dublin, Ireland},
	Author = {Gatt, A and Portet, F},
	Booktitle = {Proceedings of the 6th International Natural Language Generation Conference (INLG'10)},
	Pages = {57--65},
	Publisher = {Association for Computational Linguistics},
	Title = {Textual properties and task based evaluation: investigating the role of surface properties, structure and content},
	Url = {http://staff.um.edu.mt/albert.gatt/pubs/inlg2010-bt45-eval.pdf},
	Year = {2010},
	Keywords = {natural language generation, data-to-text, evaluation}}
[18]
Introducing shared tasks to NLG: the TUNA shared task evaluation challenges ( and ), Chapter in Empirical methods in natural language generation: Data-oriented methods and empirical evaluation (Krahmer, E.; Theune, M., eds.), Springer-Verlag, . 
@incollection{gatt2010introducing,
	Address = {Berlin and Heidelberg},
	Author = {Gatt, Albert and Belz, Anja},
	Booktitle = {Empirical methods in natural language generation: Data-oriented methods and empirical evaluation},
	Editor = {E. Krahmer and M. Theune},
	Pages = {264--293},
	Publisher = {Springer-Verlag},
	Title = {Introducing shared tasks to NLG: the TUNA shared task evaluation challenges},
	Url = {http://staff.um.edu.mt/albert.gatt/pubs/tunareg-bkchapter.pdf},
	Year = {2010},
	Keywords = {evaluation, natural language generation, referring expressions}}
[17]
Generating referring expressions in context: The GREC task evaluation challenges (; ; and ), Chapter in Empirical methods in natural language generation (Krahmer, E.; Theune, M., eds.), Springer, . 
@incollection{belz2010generating,
	Address = {Berlin and Heidelberg},
	Author = {Belz, A and Kow, E and Viethen, J and Gatt, A},
	Booktitle = {Empirical methods in natural language generation},
	Editor = {E. Krahmer and M. Theune},
	Pages = {294--327},
	Publisher = {Springer},
	Title = {Generating referring expressions in context: The GREC task evaluation challenges},
	Url = {http://staff.um.edu.mt/albert.gatt/pubs/grec-bkchapter.pdf},
	Year = {2010},
	Keywords = {natural language generation, referring expressions, evaluation, anaphora}}
2009
[16]
Beyond DICE: Measuring the quality of a referring expression ( and ), In Proceedings of the Workshop on Production of Referring Expressions: Bridging Computational and Psycholinguistic Approaches (PRE-CogSci'09), . 
@inproceedings{van2009beyond,
	Address = {Amsterdam, The Netherlands},
	Author = {van Deemter, K and Gatt, A},
	Booktitle = {Proceedings of the Workshop on Production of Referring Expressions: Bridging Computational and Psycholinguistic Approaches (PRE-CogSci'09)},
	Title = {Beyond DICE: Measuring the quality of a referring expression},
	Url = {http://staff.um.edu.mt/albert.gatt/pubs/pre-Dice.pdf},
	Year = {2009},
	Keywords = {natural language generation, referring expressions, evaluation}}
[15]
A hearer-oriented evaluation of referring expression generation (; ; ; and ), In Proceedings of the 12th European Workshop on Natural Language Generation, Association for Computational Linguistics, . 
@inproceedings{khan2009hearer,
	Address = {Athens. Greece},
	Author = {Khan, IH and van Deemter, K and Ritchie, G and Gatt, A and Cleland, AA},
	Booktitle = {Proceedings of the 12th European Workshop on Natural Language Generation},
	Pages = {98--101},
	Publisher = {Association for Computational Linguistics},
	Title = {A hearer-oriented evaluation of referring expression generation},
	Url = {http://staff.um.edu.mt/albert.gatt/pubs/enlg2009-hearer-eval.pdf},
	Year = {2009},
	Keywords = {natural language generation, referring expressions, evaluation, psycholinguistics}}
[14]
The TUNA-REG Challenge 2009: Overview and evaluation results (; and ), In Proceedings of the 12th European Workshop on Natural Language Generation (ENLG'09), Association for Computational Linguistics, . 
@inproceedings{gatt2009tuna,
	Address = {Athens, Greece},
	Author = {Gatt, Albert and Belz, Anja and Kow, Eric},
	Booktitle = {Proceedings of the 12th European Workshop on Natural Language Generation (ENLG'09)},
	Pages = {174--182},
	Publisher = {Association for Computational Linguistics},
	Title = {The TUNA-REG Challenge 2009: Overview and evaluation results},
	Url = {http://staff.um.edu.mt/albert.gatt/pubs/enlg2009-tunareg.pdf},
	Year = {2009},
	Keywords = {evaluation, referring expressions, natural language generation}}
[13]
Text Content and Task Performance in the Evaluation of a Natural Language Generation System. ( and ), In Proceedings of the Conference on Recent Advances in Natural Language Processing (RANLP'09), . 
@inproceedings{gatt2009text,
	Address = {Borovets, Bulgaria},
	Author = {Gatt, A and Portet, F},
	Booktitle = {Proceedings of the Conference on Recent Advances in Natural Language Processing (RANLP'09)},
	Pages = {107--112},
	Title = {Text Content and Task Performance in the Evaluation of a Natural Language Generation System.},
	Url = {http://staff.um.edu.mt/albert.gatt/pubs/ranlp2009.pdf},
	Year = {2009},
	Keywords = {natural language generation, data-to-text, evaluation}}
2008
[12]
Towards a balanced corpus of multimodal referring expressions in dialogue (; ; and ), In Proceedings of the AISB 2008 Convention Communication, Interaction and Social Intelligence, AISB, . 
@inproceedings{van2008towards,
	Address = {Aberdeen, UK},
	Author = {van der Sluis, I and Piwek, P and Gatt, A and Bangerter, A},
	Booktitle = {Proceedings of the AISB 2008 Convention Communication, Interaction and Social Intelligence},
	Publisher = {AISB},
	Title = {Towards a balanced corpus of multimodal referring expressions in dialogue},
	Url = {http://staff.um.edu.mt/albert.gatt/pubs/mog2008-pointing.pdf},
	Year = {2008},
	Keywords = {natural language generation, referring expressions, dialogue, evaluation, language resources}}
[11]
The importance of narrative and other lessons from an evaluation of an NLG system that summarises clinical data (; ; and ), In Proceedings of the 5th International Natural Language Generation Conference (INLG'08), Association for Computational Linguistics, . 
@inproceedings{reiter2008importance,
	Address = {Salt Fork, OH},
	Author = {Reiter, Ehud and Gatt, Albert and Portet, Fran{\c{c}}ois and Van Der Meulen, Marian},
	Booktitle = {Proceedings of the 5th International Natural Language Generation Conference (INLG'08)},
	Pages = {147--156},
	Publisher = {Association for Computational Linguistics},
	Title = {The importance of narrative and other lessons from an evaluation of an NLG system that summarises clinical data},
	Url = {http://staff.um.edu.mt/albert.gatt/pubs/inlg08-bt45.pdf},
	Year = {2008},
	Keywords = {natural language generation, data-to-text, evaluation}}
[10]
Xml format guidelines for the tuna corpus (; and ), Technical report, Technical report, Dept of Computing Science, University of Aberdeen, . 
@techreport{gatt2008xml,
	Author = {Gatt, A and van der Sluis, I and van Deemter, K},
	Institution = {Technical report, Dept of Computing Science, University of Aberdeen},
	Title = {Xml format guidelines for the tuna corpus},
	Url = {http://www.csd.abdn.ac.uk/~ agatt/home/pubs/tunaFormat.pdf},
	Year = {2008},
	Keywords = {language resources, evaluation, referring expressions, natural language generation}}
[9]
The TUNA Challenge 2008: Overview and evaluation results (; and ), In Proceedings of the 5th International Natural Language Generation Conference (INLG'08), Association for Computational Linguistics, . 
@inproceedings{gatt2008tuna,
	Address = {Columbus, OH},
	Author = {Gatt, Albert and Belz, Anja and Kow, Eric},
	Booktitle = {Proceedings of the 5th International Natural Language Generation Conference (INLG'08)},
	Pages = {198--206},
	Publisher = {Association for Computational Linguistics},
	Title = {The TUNA Challenge 2008: Overview and evaluation results},
	Url = {http://staff.um.edu.mt/albert.gatt/pubs/reg-tuna-report.pdf},
	Year = {2008},
	Keywords = {natural language generation, referring expressions, evaluation}}
[8]
Attribute selection for referring expression generation: New algorithms and evaluation methods ( and ), In Proceedings of the 5th International Conference on Natural Language Generation (INLG'08), Association for Computational Linguistics, . 
@inproceedings{gatt2008attribute,
	Address = {Columbus, OH},
	Author = {Gatt, Albert and Belz, Anja},
	Booktitle = {Proceedings of the 5th International Conference on Natural Language Generation (INLG'08)},
	Pages = {50--58},
	Publisher = {Association for Computational Linguistics},
	Title = {Attribute selection for referring expression generation: New algorithms and evaluation methods},
	Url = {http://staff.um.edu.mt/albert.gatt/pubs/inlg08-asgre.pdf},
	Year = {2008},
	Keywords = {natural language generation, referring expressions, evaluation}}
[7]
Intrinsic vs. extrinsic evaluation measures for referring expression generation ( and ), In Proceedings of the 46th Annual Meeting of the Association for Computational Linguistics (ACL'08), Association for Computational Linguistics, . 
@inproceedings{belz2008intrinsic,
	Address = {Columbus, OH},
	Author = {Belz, Anja and Gatt, Albert},
	Booktitle = {Proceedings of the 46th Annual Meeting of the Association for Computational Linguistics (ACL'08)},
	Pages = {197--200},
	Publisher = {Association for Computational Linguistics},
	Title = {Intrinsic vs. extrinsic evaluation measures for referring expression generation},
	Url = {http://staff.um.edu.mt/albert.gatt/pubs/acl08-eval.pdf},
	Year = {2008},
	Keywords = {natural language generation, referring expressions, evaluation}}
2007
[6]
Evaluating algorithms for the generation of referring expressions: Going beyond toy domains (; and ), In Proceedings of the International Conference on Recent Advances in Natural Language Processing (RANLP'07), RANLP, . 
@inproceedings{van2007evaluating,
	Address = {Borovets, Bulgaria},
	Author = {Van der Sluis, Ielka and Gatt, Albert and Van Deemter, Kees},
	Booktitle = {Proceedings of the International Conference on Recent Advances in Natural Language Processing (RANLP'07)},
	Publisher = {RANLP},
	Title = {Evaluating algorithms for the generation of referring expressions: Going beyond toy domains},
	Url = {http://staff.um.edu.mt/albert.gatt/pubs/ranlp2007.pdf},
	Year = {2007},
	Keywords = {natural language generation, referring expressions, evaluation}}
[5]
Content determination in GRE: Evaluating the evaluator ( and ), In Proceedings of the 2nd UCNLG Workshop: Language Generation and Machine Translation, Association for Computational Linguistics, . 
@inproceedings{van2007content,
	Address = {Copenhagen, Denmark},
	Author = {van Deemter, Kees and Gatt, Albert},
	Booktitle = {Proceedings of the 2nd UCNLG Workshop: Language Generation and Machine Translation},
	Publisher = {Association for Computational Linguistics},
	Title = {Content determination in GRE: Evaluating the evaluator},
	Url = {http://staff.um.edu.mt/albert.gatt/pubs/diceEvaluation_final.pdf},
	Year = {2007},
	Keywords = {natural language generation, referring expressions, evaluation}}
[4]
Evaluating algorithms for the generation of referring expressions using a balanced corpus (; and ), In Proceedings of the 11th European Workshop on Natural Language Generation (ENLG'07), Association for Computational Linguistics, . 
@inproceedings{gatt2007evaluating,
	Address = {Schloss Dagstuhl, Germany},
	Author = {Gatt, A and van der Sluis, I and vann Deemter, K},
	Booktitle = {Proceedings of the 11th European Workshop on Natural Language Generation (ENLG'07)},
	Pages = {49--56},
	Publisher = {Association for Computational Linguistics},
	Title = {Evaluating algorithms for the generation of referring expressions using a balanced corpus},
	Url = {http://staff.um.edu.mt/albert.gatt/pubs/enlg2007.pdf},
	Year = {2007},
	Keywords = {natural language generation, referring expressions, evaluation}}
[3]
Corpus-based evaluation of Referring Expressions Generation (; and ), In Proceedings of the Workshop on Shared Tasks and Comparative Evaluation in NLG, . 
@inproceedings{gatt2007corpus,
	Address = {Arlington, VA},
	Author = {Gatt, Albert and Van Der Sluis, Ielka and Van Deemter, Kees},
	Booktitle = {Proceedings of the Workshop on Shared Tasks and Comparative Evaluation in NLG},
	Title = {Corpus-based evaluation of Referring Expressions Generation},
	Url = {http://www.ling.ohio-state.edu/nlgeval07/papers/Gatt-vanderSluis-vanDeemter.pdf},
	Year = {2007},
	Keywords = {natural language generation, referring expressions, evaluation}}
[2]
The attribute selection for GRE challenge: Overview and evaluation results ( and ), In Proceedings of UCNLG+MT: Language Generation and Machine Translation, Association for Computational Linguistics, . 
@inproceedings{belz2007attribute,
	Address = {Copenhagen, Denmark},
	Author = {Belz, Anja and Gatt, Albert},
	Booktitle = {Proceedings of UCNLG+MT: Language Generation and Machine Translation},
	Pages = {75--83},
	Publisher = {Association for Computational Linguistics},
	Title = {The attribute selection for GRE challenge: Overview and evaluation results},
	Url = {http://staff.um.edu.mt/albert.gatt/pubs/asgre2007.pdf},
	Year = {2007},
	Keywords = {natural language generation, referring expressions, evaluation}}
2006
[1]
Building a semantically transparent corpus for the generation of referring expressions (; and ), In Proceedings of the 4th International Natural Language Generation Conference (INLG'06), Association for Computational Linguistics, . 
@inproceedings{van2006building,
	Address = {Sydney, Australia},
	Author = {van Deemter, K and van der Sluis, I and Gatt, A},
	Booktitle = {Proceedings of the 4th International Natural Language Generation Conference (INLG'06)},
	Pages = {130--132},
	Publisher = {Association for Computational Linguistics},
	Title = {Building a semantically transparent corpus for the generation of referring expressions},
	Url = {http://staff.um.edu.mt/albert.gatt/pubs/semTransCorpus-INLG06.pdf},
	Year = {2006},
	Keywords = {language resources, natural language generation, referring expressions, evaluation}}