Abstract We present a framework for machine translation evaluation using neural networks in a pairwise setting, where the goal is to select the better translation from a pair of hypotheses, given the reference translation. In this framework, lexical, syntactic and semantic information from the reference and the two hypotheses is embedded into compact distributed vector representations, and fed into a multi-layer neural network that models nonlinear interactions between each of the hypotheses and the reference, as well as between the two hypotheses. We experiment with the benchmark datasets from the \WMT\ Metrics shared task, on which we obtain the best results published so far, with the basic network configuration. We also perform a series of experiments to analyze and understand the contribution of the different components of the network. We evaluate variants and extensions, including fine-tuning of the semantic embeddings, and sentence-based representations modeled with convolutional and recurrent neural networks. In summary, the proposed framework is flexible and generalizable, allows for efficient learning and scoring, and provides an \MT\ evaluation metric that correlates with human judgments, and is on par with the state of the art.

@article{GuzmanCSL2016, abstract = {Abstract We present a framework for machine translation evaluation using neural networks in a pairwise setting, where the goal is to select the better translation from a pair of hypotheses, given the reference translation. In this framework, lexical, syntactic and semantic information from the reference and the two hypotheses is embedded into compact distributed vector representations, and fed into a multi-layer neural network that models nonlinear interactions between each of the hypotheses and the reference, as well as between the two hypotheses. We experiment with the benchmark datasets from the \{WMT\} Metrics shared task, on which we obtain the best results published so far, with the basic network configuration. We also perform a series of experiments to analyze and understand the contribution of the different components of the network. We evaluate variants and extensions, including fine-tuning of the semantic embeddings, and sentence-based representations modeled with convolutional and recurrent neural networks. In summary, the proposed framework is flexible and generalizable, allows for efficient learning and scoring, and provides an \{MT\} evaluation metric that correlates with human judgments, and is on par with the state of the art.}, author = {Guzm\'{a}n,Francisco and Joty, Shafiq and Màrquez,Lluís and Nakov, Preslav}, doi = {http://dx.doi.org/10.1016/j.csl.2016.12.005}, issn = {0885-2308}, journal = {Computer Speech & Language}, link = {http://www.sciencedirect.com/science/article/pii/S0885230816301693}, title = {Machine translation evaluation with neural networks}, year = {2016} }

@inproceedings{nakov-marquez-guzman:2016:EMNLP2016, address = {Austin, Texas}, author = {Nakov, Preslav and M\`{a}rquez, Llu\'{i}s and Guzm\'{a}n, Francisco}, booktitle = {Proceedings of the 2016 Conference on Empirical Methods in Natural Language Processing}, link = {https://aclweb.org/anthology/D16-1165}, month = {November}, pages = {1586--1597}, publisher = {Association for Computational Linguistics}, title = {It Takes Three to Tango: Triangulation Approach to Answer Ranking in Community Question Answering}, year = {2016} }

@inproceedings{guzman-marquez-nakov:2016:P16-2, address = {Berlin, Germany}, author = {Guzm\'{a}n, Francisco and M\`{a}rquez, Llu\'{i}s and Nakov, Preslav}, booktitle = {Proceedings of the 54th Annual Meeting of the Association for Computational Linguistics}, link = {http://anthology.aclweb.org/P16-2075}, month = {August}, pages = {460--466}, publisher = {Association for Computational Linguistics}, title = {Machine Translation Evaluation Meets Community Question Answering}, year = {2016} }

In this paper, we take a closer look at the MT evaluation process from a glass-box perspective using eye-tracking. We analyze two aspects of the evaluation task: the background of evaluators (monolingual or bilingual) and the sources of information available, and we evaluate them using time and consistency as criteria. Our findings show that monolinguals are slower but more consistent than bilinguals, especially when only target language information is available. When exposed to various sources of information, evaluators in general take more time and in the case of monolinguals, there is a drop in consistency. Our findings suggest that to have consistent and cost effective MT evaluations, it is better to use monolinguals with only target language information.

@InProceedings{sajjad-EtAl:2016:N16-1, author = {Sajjad, Hassan and Guzm\'{a}n, Francisco and Durrani, Nadir and Abdelali, Ahmed and Bouamor, Houda and Temnikova, Irina and Vogel, Stephan}, title = {Eyes Don't Lie: Predicting Machine Translation Quality Using Eye Movement}, booktitle = {Proceedings of the 2016 Conference of the North American Chapter of the Association for Computational Linguistics: Human Language Technologies}, month = {June}, year = {2016}, address = {San Diego, California}, publisher = {Association for Computational Linguistics}, pages = {1082--1088}, url = {http://www.aclweb.org/anthology/N16-1125} }

In this paper, we take a closer look at the MT evaluation process from a glass-box perspective using eye-tracking. We analyze two aspects of the evaluation task: the background of evaluators (monolingual or bilingual) and the sources of information available, and we evaluate them using time and consistency as criteria. Our findings show that monolinguals are slower but more consistent than bilinguals, especially when only target language information is available. When exposed to various sources of information, evaluators in general take more time and in the case of monolinguals, there is a drop in consistency. Our findings suggest that to have consistent and cost effective MT evaluations, it is better to use monolinguals with only target language information.

@inproceedings{guzman-EtAl:2015:WMT, abstract = {In this paper, we take a closer look at the MT evaluation process from a glass-box perspective using eye-tracking. We analyze two aspects of the evaluation task: the background of evaluators (monolingual or bilingual) and the sources of information available, and we evaluate them using time and consistency as criteria. Our findings show that monolinguals are slower but more consistent than bilinguals, especially when only target language information is available. When exposed to various sources of information, evaluators in general take more time and in the case of monolinguals, there is a drop in consistency. Our findings suggest that to have consistent and cost effective MT evaluations, it is better to use monolinguals with only target language information.}, address = {Lisbon, Portugal}, author = {Guzm\'{a}n, Francisco and Abdelali, Ahmed and Temnikova, Irina and Sajjad, Hassan and Vogel, Stephan}, booktitle = {Proceedings of the Tenth Workshop on Statistical Machine Translation}, link = {http://aclweb.org/anthology/W15-3059}, month = {September}, pages = {457--466}, publisher = {Association for Computational Linguistics}, title = {How do Humans Evaluate Machine Translation}, year = {2015} }

few typos were corrected in this version

We present a novel framework for machine translation evaluation using neural networks in a pairwise setting, where the goal is to select the better translation from a pair of hypotheses, given the reference translation. In this framework, lexical, syntactic and semantic information from the reference and the two hypotheses is compacted into relatively small distributed vector representations, and fed into a multi-layer neural network that models the interaction between each of the hypotheses and the reference, as well as between the two hypotheses. These compact representations are in turn based on word and sentence embeddings, which are learned using neural networks. The framework is flexible, allows for efficient learning and classification, and yields correlation with humans that rivals the state of the art.

@InProceedings{guzman2015-ACL, author = {Guzm\'{a}n, Francisco and Joty, Shafiq and M\`{a}rquez, Llu\'{i}s and Nakov, Preslav }, title = {Pairwise Neural Machine Translation Evaluation}, booktitle = {Proceedings of the 53rd Annual Meeting of the Association for Computational Linguistics and The 7th International Joint Conference of the Asian Federation of Natural Language Processing ({ACL}'15)}, month = {July}, year = {2015}, address = {Beijing, China}, publisher = {Association for Computational Linguistics}, pages = {805--814}, url = {http://www.aclweb.org/anthology/P15-1078}, Abstract = {We present a novel framework for machine translation evaluation using neural networks in a pairwise setting, where the goal is to select the better translation from a pair of hypotheses, given the reference translation. In this framework, lexical, syntactic and semantic information from the reference and the two hypotheses is compacted into relatively small distributed vector representations, and fed into a multi-layer neural network that models the interaction between each of the hypotheses and the reference, as well as between the two hypotheses. These compact representations are in turn based on word and sentence embeddings, which are learned using neural networks. The framework is flexible, allows for efficient learning and classification, and yields correlation with humans that rivals the state of the art.} }

We present a pairwise learning-to-rank approach to machine translation evaluation that learns to differentiate better from worse translations in the context of a given reference. We integrate several layers of linguistic information encapsulated in tree-based structures, making use of both the reference and the system output simultaneously, thus bringing our ranking closer to how humans evaluate translations. Most importantly, instead of deciding upfront which types of features are important, we use the learning framework of preference re-ranking kernels to learn the features automatically. The evaluation results show that learning in the proposed framework yields better correlation with humans than computing the direct similarity over the same type of structures. Also, we show our structural kernel learning (SKL) can be a general framework for MT evaluation, in which syntactic and semantic information can be naturally incorporated.

@inproceedings{guzman-EtAl:2014:EMNLP2014, abstract = {We present a pairwise learning-to-rank approach to machine translation evaluation that learns to differentiate better from worse translations in the context of a given reference. We integrate several layers of linguistic information encapsulated in tree-based structures, making use of both the reference and the system output simultaneously, thus bringing our ranking closer to how humans evaluate translations. Most importantly, instead of deciding upfront which types of features are important, we use the learning framework of preference re-ranking kernels to learn the features automatically. The evaluation results show that learning in the proposed framework yields better correlation with humans than computing the direct similarity over the same type of structures. Also, we show our structural kernel learning (SKL) can be a general framework for MT evaluation, in which syntactic and semantic information can be naturally incorporated.}, address = {Doha, Qatar}, author = {Guzm\'{a}n, Francisco and Joty, Shafiq and M\`{a}rquez, Llu\'{i}s and Moschitti, Alessandro and Nakov, Preslav and Nicosia, Massimo}, booktitle = {Proceedings of the 2014 Conference on Empirical Methods in Natural Language Processing (EMNLP'14)}, link = {http://www.aclweb.org/anthology/D14-1027}, month = {October}, pages = {214--220}, publisher = {Association for Computational Linguistics}, title = {Learning to Differentiate Better from Worse Translations}, year = {2014} }

We present experiments in using discourse structure for improving machine translation evaluation. We first design two discourse-aware similarity measures, which use all-subtree kernels to compare discourse parse trees in accordance with the Rhetorical Structure Theory. Then, we show that these measures can help improve a number of existing machine translation evaluation metrics both at the segmentand at the system-level. Rather than proposing a single new metric, we show that discourse information is complementary to the state-of-the-art evaluation metrics, and thus should be taken into account in the development of future richer evaluation metrics.

@inproceedings{guzman-EtAl:2014:P14-1, abstract = {We present experiments in using discourse structure for improving machine translation evaluation. We first design two discourse-aware similarity measures, which use all-subtree kernels to compare discourse parse trees in accordance with the Rhetorical Structure Theory. Then, we show that these measures can help improve a number of existing machine translation evaluation metrics both at the segmentand at the system-level. Rather than proposing a single new metric, we show that discourse information is complementary to the state-of-the-art evaluation metrics, and thus should be taken into account in the development of future richer evaluation metrics.}, address = {Baltimore, Maryland, USA}, author = {Guzm\'{a}n, Francisco and Joty, Shafiq and M\`{a}rquez, Llu\'{i}s and Nakov, Preslav}, booktitle = {Proceedings of the 52nd Annual Meeting of the Association for Computational Linguistics ({ACL}'14)}, link = {http://www.aclweb.org/anthology/P/P14/P14-1065}, month = {June}, pages = {687--698}, publisher = {Association for Computational Linguistics}, title = {Using Discourse Structure Improves Machine Translation Evaluation}, year = {2014} }

In this paper, we introduce a new parallel corpus of subtitles of educational videos: the AMARA corpus for online educational content. We crawl a multilingual collection community generated subtitles, and present the results of processing the Arabic–English portion of the data, which yields a parallel corpus of about 2.6M Arabic and 3.9M English words. We explore different approaches to align the segments, and extrinsically evaluate the resulting parallel corpus on the standard TED-talks tst-2010. We observe that the data can be successfully used for this task, and also observe an absolute improvement of 1.6 BLEU when it is used in combination with TED data. Finally, we analyze some of the specific challenges when translating the educational content.

@inproceedings{guzman:2013:amara, abstract = {In this paper, we introduce a new parallel corpus of subtitles of educational videos: the AMARA corpus for online educational content. We crawl a multilingual collection community generated subtitles, and present the results of processing the Arabic–English portion of the data, which yields a parallel corpus of about 2.6M Arabic and 3.9M English words. We explore different approaches to align the segments, and extrinsically evaluate the resulting parallel corpus on the standard TED-talks tst-2010. We observe that the data can be successfully used for this task, and also observe an absolute improvement of 1.6 BLEU when it is used in combination with TED data. Finally, we analyze some of the specific challenges when translating the educational content.}, address = {Heidelberg, Germany}, author = {Guzm{\'a}n, Francisco and Sajjad, Hassan and Abdelali, Ahmed and Vogel, Stephan}, booktitle = {Proceedings of the 10th International Workshop on Spoken Language Translation {(IWSLT'13})}, month = {December}, title = {The {AMARA} Corpus: Building Resources for Translating the Web's Educational Content}, volume = {13}, year = {2013} }

While experimenting with tuning on long sentences, we made an unexpected discovery: that PRO falls victim to monsters -overly long negative examples with very low BLEU+1 scores, which are unsuitable for learning and can cause testing BLEU to drop by several points absolute. We propose several effective ways to address the problem, using length- and BLEU+1- based cut-offs, outlier filters, stochastic sampling, and random acceptance. The best of these fixes not only slay and protect against monsters, but also yield higher stability for PRO as well as improved test-time BLEU scores. Thus, we recommend them to anybody using PRO, monster-believer or not.

@inproceedings{nakov-guzman-vogel:2013:Short, abstract = {While experimenting with tuning on long sentences, we made an unexpected discovery: that PRO falls victim to monsters -overly long negative examples with very low BLEU+1 scores, which are unsuitable for learning and can cause testing BLEU to drop by several points absolute. We propose several effective ways to address the problem, using length- and BLEU+1- based cut-offs, outlier filters, stochastic sampling, and random acceptance. The best of these fixes not only slay and protect against monsters, but also yield higher stability for PRO as well as improved test-time BLEU scores. Thus, we recommend them to anybody using PRO, monster-believer or not.}, address = {Sofia, Bulgaria}, author = {Nakov, Preslav and Guzm{\'a}n, Francisco and Vogel, Stephan}, booktitle = {Proceedings of the 51st Annual Meeting of the Association for Computational Linguistics ({ACL'13})}, link = {http://www.aclweb.org/anthology/P13-2003}, month = {August}, pages = {12--17}, title = {A Tale about {PRO} and Monsters}, year = {2013} }

We study a problem with pairwise ranking optimization (PRO): that it tends to yield too short translations. We find that this is partially due to the inadequate smoothing in PRO’s BLEU+1, which boosts the precision component of BLEU but leaves the brevity penalty unchanged, thus destroying the balance between the two, compared to BLEU. It is also partially due to PRO optimizing for a sentence-level score without a global view on the overall length, which introducing a bias towards short translations; we show that letting PRO optimize a corpus-level BLEU yields a perfect length. Finally, we find some residual bias due to the interaction of PRO with BLEU+1: such a bias does not exist for a version of MIRA with sentence-level BLEU+1. We propose several ways to fix the length problem of PRO, including smoothing the brevity penalty, scaling the effective reference length, grounding the precision component, and unclipping the brevity penalty, which yield sizable improvements in test BLEU on two Arabic-English datasets: IWSLT (+0.65) and NIST (+0.37).

@inproceedings{nakov-guzman-vogel:2012:PAPERS, author = {Nakov, Preslav and Guzm{\'a}n, Francisco and Vogel, Stephan}, title = {Optimizing for Sentence-Level {BLEU}+1 Yields Short Translations}, booktitle = {Proceedings of the 24rd International Conference on Computational Linguistics (COLING) 2012 }, month = {December}, year = {2012}, address = {Mumbai, India}, publisher = {The {COLING} 2012 Organizing Committee}, pages = {1979--1994}, url = {http://www.aclweb.org/anthology/C12-1121}, abstract = {We study a problem with pairwise ranking optimization (PRO): that it tends to yield too short translations. We find that this is partially due to the inadequate smoothing in PRO’s BLEU+1, which boosts the precision component of BLEU but leaves the brevity penalty unchanged, thus destroying the balance between the two, compared to BLEU. It is also partially due to PRO optimizing for a sentence-level score without a global view on the overall length, which introducing a bias towards short translations; we show that letting PRO optimize a corpus-level BLEU yields a perfect length. Finally, we find some residual bias due to the interaction of PRO with BLEU+1: such a bias does not exist for a version of MIRA with sentence-level BLEU+1. We propose several ways to fix the length problem of PRO, including smoothing the brevity penalty, scaling the effective reference length, grounding the precision component, and unclipping the brevity penalty, which yield sizable improvements in test BLEU on two Arabic-English datasets: IWSLT (+0.65) and NIST (+0.37).} }

@incollection{francisco:2011:word, Author = {Francisco Guzm{\'a}n, Qin Gao, Jan Niehues and Stephan Vogel}, Booktitle = {Handbook of Natural Language Processing and Machine Translation: DARPA global autonomous language exploitation.}, Pages = {164--175}, editor = {Olive, Joseph and Christianson, Caitlin and McCary, John}, publisher={Springer Science \& Business Media} Title = {Word Alignment Revisited}, Year = {2011}}

In this paper we study in detail the relation between word alignment and phrase extraction. First, we analyze different word alignments according to several characteristics and compare them to hand-aligned data. Secondly, we analyzed the phrase-pairs generated by these alignments. We observed that the number of unaligned words has a large impact on the characteristics of the phrase table. A manual evaluation of phrase pair quality showed that the increase in the number of unaligned words results in a lower quality. Finally, we present translation results from using the number of unaligned words as features from which we obtain up to 2BP of improvement.

@inproceedings{guzman:2009:reassessment, Author = {Guzm{\'a}n, Francisco and Gao, Qin and Vogel, Stephan}, Booktitle = {The Twelfth Machine Translation Summit ({MTSummit XII})}, Organization = {International Association for Machine Translation}, Title = {Reassessment of the Role of Phrase Extraction in {PBSMT} }, address = {Ottawa, Canada}, Month = {August}, Year = {2009}, Abstract ={In this paper we study in detail the relation between word alignment and phrase extraction. First, we analyze different word alignments according to several characteristics and compare them to hand-aligned data. Secondly, we analyzed the phrase-pairs generated by these alignments. We observed that the number of unaligned words has a large impact on the characteristics of the phrase table. A manual evaluation of phrase pair quality showed that the increase in the number of unaligned words results in a lower quality. Finally, we present translation results from using the number of unaligned words as features from which we obtain up to 2BP of improvement.}}