@article {7482, title = {Assessing the Capacity of Transformer to Abstract Syntactic Representations: A Contrastive Analysis Based on Long-distance Agreement}, journal = {Transactions of the Association for Computational Linguistics}, volume = {11}, year = {2023}, pages = {18-33}, doi = {10.1162/tacl_a_00531}, url = {https://hal.science/hal-03957265}, author = {Li, Bingzhi and Wisniewski, Guillaume and Beno{\^\i}t Crabb{\'e}} } @article {liAssessingCapacityTransformer2023, title = {Assessing the Capacity of Transformer to Abstract Syntactic Representations: A Contrastive Analysis Based on Long-distance Agreement}, journal = {Transactions of the Association for Computational Linguistics}, volume = {10}, year = {2023}, author = {Li, Bingzhi and Wisniewski, Guillaume and Beno{\^\i}t Crabb{\'e}} } @inproceedings {liHowDistributedAre2022a, title = {How Distributed Are Distributed Representations? An Observation on the Locality of Syntactic Information in Verb Agreement Tasks}, year = {2022}, month = {may}, pages = {501{\textendash}507}, publisher = {Association for Computational Linguistics}, address = {Dublin, Ireland}, abstract = {

This work addresses the question of the localization of syntactic information encoded in the transformers representations. We tackle this question from two perspectives, considering the object-past participle agreement in French, by identifying, first, in which part of the sentence and, second, in which part of the representation the syntactic information is encoded. The results of our experiments, using probing, causal analysis and feature selection method, show that syntactic information is encoded locally in a way consistent with the French grammar.

}, author = {Li, Bingzhi and Wisniewski, Guillaume and Beno{\^\i}t Crabb{\'e}} } @inproceedings {li:halshs-03755082, title = {How Distributed are Distributed Representations? An Observation on the Locality of Syntactic Information in Verb Agreement Tasks}, year = {2022}, month = {May}, pages = {501-507}, publisher = {Association for Computational Linguistics}, address = {Dublin, France}, doi = {10.18653/v1/2022.acl-short.54}, url = {https://shs.hal.science/halshs-03755082}, author = {Li, Bingzhi and Wisniewski, Guillaume and Beno{\^\i}t Crabb{\'e}} } @inproceedings {7233, title = {How Distributed are Distributed Representations? An Observation on the Locality of Syntactic Information in Verb Agreement Tasks}, year = {2022}, pages = {501{\textendash}507}, publisher = {Association for Computational Linguistics}, address = {Dublin, Ireland}, abstract = {

This work addresses the question of the localization of syntactic information encoded in the transformers representations. We tackle this question from two perspectives, considering the object-past participle agreement in French, by identifying, first, in which part of the sentence and, second, in which part of the representation the syntactic information is encoded. The results of our experiments, using probing, causal analysis and feature selection method, show that syntactic information is encoded locally in a way consistent with the French grammar.

}, doi = {10.18653/v1/2022.acl-short.54}, url = {https://aclanthology.org/2022.acl-short.54}, author = {Li, Bingzhi and Wisniewski, Guillaume and Beno{\^\i}t Crabb{\'e}} } @inproceedings {li:halshs-03755082, title = {How Distributed are Distributed Representations? An Observation on the Locality of Syntactic Information in Verb Agreement Tasks}, year = {2022}, month = {May}, pages = {501-507}, publisher = {Association for Computational Linguistics}, address = {Dublin, France}, doi = {10.18653/v1/2022.acl-short.54}, url = {https://halshs.archives-ouvertes.fr/halshs-03755082}, author = {Li, Bingzhi and Wisniewski, Guillaume and Beno{\^\i}t Crabb{\'e}} } @inproceedings {li:hal-03701480, title = {Les repr{\'e}sentations distribu{\'e}es sont-elles vraiment distribu{\'e}es ? Observations sur la localisation de l{\textquoteright}information syntaxique dans les t{\^a}ches d{\textquoteright}accord du verbe en fran{\c c}ais}, year = {2022}, pages = {384-391}, publisher = {ATALA}, address = {Avignon, France}, abstract = {

Ce travail aborde la question de la localisation de l\&$\#$39;information syntaxique qui est encod\ée dans les repr\ésentations de transformers. En consid\érant la t\âche d\&$\#$39;accord objet-participe pass\é en fran\çais, les r\ésultats de nos sondes linguistiques montrent que les informations n\écessaires pour accomplir la t\âche sont encod\ées d\&$\#$39;une mani\ère locale dans les repr\ésentations de mots entre l\&$\#$39;ant\éc\édent du pronom relatif objet et le participe pass\é cible. En plus, notre analyse causale montre que le mod\èle s\&$\#$39;appuie essentiellement sur les \él\éments linguistiquement motiv\és (i.e. ant\éc\édent et pronom relatif) pour pr\édire le nombre du participe pass\é.

}, keywords = {analyse causale., information syntaxique, repr{\'e}sentation distribu{\'e}e, transformers}, url = {https://hal.archives-ouvertes.fr/hal-03701480}, author = {Li, Bingzhi and Wisniewski, Guillaume and Beno{\^\i}t Crabb{\'e}}, editor = {Est{\`e}ve, Yannick and Jim{\'e}nez, Tania and Parcollet, Titouan and Zanon Boito, Marcely} } @inproceedings {li:halshs-03197072, title = {Are Neural Networks Extracting Linguistic Properties or Memorizing Training Data? An Observation with a Multilingual Probe for Predicting Tense}, year = {2021}, address = {Kiev (on line), Ukraine}, url = {https://shs.hal.science/halshs-03197072}, author = {Li, Bingzhi and Wisniewski, Guillaume} } @inproceedings {7232, title = {Are Neural Networks Extracting Linguistic Properties or Memorizing Training Data? An Observation with a Multilingual Probe for Predicting Tense}, year = {2021}, pages = {3080{\textendash}3089}, publisher = {Association for Computational Linguistics}, address = {Online}, abstract = {

We evaluate the ability of Bert embeddings to represent tense information, taking French and Chinese as a case study. In French, the tense information is expressed by verb morphology and can be captured by simple surface information. On the contrary, tense interpretation in Chinese is driven by abstract, lexical, syntactic and even pragmatic information. We show that while French tenses can easily be predicted from sentence representations, results drop sharply for Chinese, which suggests that Bert is more likely to memorize shallow patterns from the training data rather than uncover abstract properties.

}, doi = {10.18653/v1/2021.eacl-main.269}, url = {https://aclanthology.org/2021.eacl-main.269}, author = {Li, Bingzhi and Wisniewski, Guillaume} } @inproceedings {li:halshs-03197072, title = {Are Neural Networks Extracting Linguistic Properties or Memorizing Training Data? An Observation with a Multilingual Probe for Predicting Tense}, year = {2021}, address = {Kiev (on line), Ukraine}, url = {https://halshs.archives-ouvertes.fr/halshs-03197072}, author = {Li, Bingzhi and Wisniewski, Guillaume} } @inproceedings {liAreTransformersModern2021, title = {Are Transformers a Modern Version of ELIZA? Observations on French Object Verb Agreement}, year = {2021}, month = {nov}, pages = {4599{\textendash}4610}, publisher = {Association for Computational Linguistics}, address = {Online and Punta Cana, Dominican Republic}, abstract = {

Many recent works have demonstrated that unsupervised sentence representations of neural networks encode syntactic information by observing that neural language models are able to predict the agreement between a verb and its subject. We take a critical look at this line of research by showing that it is possible to achieve high accuracy on this agreement task with simple surface heuristics, indicating a possible flaw in our assessment of neural networks\&$\#$39; syntactic ability. Our fine-grained analyses of results on the long-range French object-verb agreement show that contrary to LSTMs, Transformers are able to capture a non-trivial amount of grammatical structure.

}, author = {Li, Bingzhi and Wisniewski, Guillaume and Beno{\^\i}t Crabb{\'e}} } @inproceedings {li:halshs-03755089, title = {Are Transformers a Modern Version of ELIZA? Observations on French Object Verb Agreement}, year = {2021}, month = {Nov}, address = {Punta Cana, Dominican Republic}, doi = {10.18653/v1/2021.emnlp-main.377}, url = {https://shs.hal.science/halshs-03755089}, author = {Li, Bingzhi and Wisniewski, Guillaume and Beno{\^\i}t Crabb{\'e}} } @inproceedings {7231, title = {Are Transformers a Modern Version of ELIZA? Observations on French Object Verb Agreement}, year = {2021}, pages = {4599{\textendash}4610}, publisher = {Association for Computational Linguistics}, address = {Online and Punta Cana, Dominican Republic}, abstract = {

Many recent works have demonstrated that unsupervised sentence representations of neural networks encode syntactic information by observing that neural language models are able to predict the agreement between a verb and its subject. We take a critical look at this line of research by showing that it is possible to achieve high accuracy on this agreement task with simple surface heuristics, indicating a possible flaw in our assessment of neural networks\&$\#$39; syntactic ability. Our fine-grained analyses of results on the long-range French object-verb agreement show that contrary to LSTMs, Transformers are able to capture a non-trivial amount of grammatical structure.

}, doi = {10.18653/v1/2021.emnlp-main.377}, url = {https://aclanthology.org/2021.emnlp-main.377}, author = {Li, Bingzhi and Wisniewski, Guillaume and Beno{\^\i}t Crabb{\'e}} } @inproceedings {li:halshs-03755089, title = {Are Transformers a Modern Version of ELIZA? Observations on French Object Verb Agreement}, year = {2021}, address = {Punta Cana, Dominican Republic}, doi = {10.18653/v1/2021.emnlp-main.377}, url = {https://halshs.archives-ouvertes.fr/halshs-03755089}, author = {Li, Bingzhi and Wisniewski, Guillaume and Beno{\^\i}t Crabb{\'e}} }