@inproceedings {7233, title = {How Distributed are Distributed Representations? An Observation on the Locality of Syntactic Information in Verb Agreement Tasks}, year = {2022}, pages = {501{\textendash}507}, publisher = {Association for Computational Linguistics}, address = {Dublin, Ireland}, abstract = {

This work addresses the question of the localization of syntactic information encoded in the transformers representations. We tackle this question from two perspectives, considering the object-past participle agreement in French, by identifying, first, in which part of the sentence and, second, in which part of the representation the syntactic information is encoded. The results of our experiments, using probing, causal analysis and feature selection method, show that syntactic information is encoded locally in a way consistent with the French grammar.

}, doi = {10.18653/v1/2022.acl-short.54}, url = {https://aclanthology.org/2022.acl-short.54}, author = {Li, Bingzhi and Wisniewski, Guillaume and Beno{\^\i}t Crabb{\'e}} } @inproceedings {li:halshs-03755082, title = {How Distributed are Distributed Representations? An Observation on the Locality of Syntactic Information in Verb Agreement Tasks}, year = {2022}, month = {May}, pages = {501-507}, publisher = {Association for Computational Linguistics}, address = {Dublin, France}, doi = {10.18653/v1/2022.acl-short.54}, url = {https://shs.hal.science/halshs-03755082}, author = {Li, Bingzhi and Wisniewski, Guillaume and Beno{\^\i}t Crabb{\'e}} } @inproceedings {li:halshs-03755082, title = {How Distributed are Distributed Representations? An Observation on the Locality of Syntactic Information in Verb Agreement Tasks}, year = {2022}, month = {May}, pages = {501-507}, publisher = {Association for Computational Linguistics}, address = {Dublin, France}, doi = {10.18653/v1/2022.acl-short.54}, url = {https://halshs.archives-ouvertes.fr/halshs-03755082}, author = {Li, Bingzhi and Wisniewski, Guillaume and Beno{\^\i}t Crabb{\'e}} } @inproceedings {liHowDistributedAre2022a, title = {How Distributed Are Distributed Representations? An Observation on the Locality of Syntactic Information in Verb Agreement Tasks}, year = {2022}, month = {may}, pages = {501{\textendash}507}, publisher = {Association for Computational Linguistics}, address = {Dublin, Ireland}, abstract = {

This work addresses the question of the localization of syntactic information encoded in the transformers representations. We tackle this question from two perspectives, considering the object-past participle agreement in French, by identifying, first, in which part of the sentence and, second, in which part of the representation the syntactic information is encoded. The results of our experiments, using probing, causal analysis and feature selection method, show that syntactic information is encoded locally in a way consistent with the French grammar.

}, author = {Li, Bingzhi and Wisniewski, Guillaume and Beno{\^\i}t Crabb{\'e}} } @inproceedings {simoulin:hal-03601412, title = {How Many Layers and Why? An Analysis of the Model Depth in Transformers}, year = {2021}, address = {Bangkok, Thailand}, url = {https://hal.archives-ouvertes.fr/hal-03601412}, author = {Simoulin, Antoine and Beno{\^\i}t Crabb{\'e}} } @inproceedings {simoulinHowManyLayers2021, title = {How Many Layers and Why? An Analysis of the Model Depth in Transformers}, year = {2021}, pages = {221{\textendash}228}, publisher = {Association for Computational Linguistics}, address = {Online}, abstract = {

In this study, we investigate the role of the multiple layers in deep transformer models. We design a variant of Albert that dynamically adapts the number of layers for each token of the input. The key specificity of Albert is that weights are tied across layers. Therefore, the stack of encoder layers iteratively repeats the application of the same transformation function on the input. We interpret the repetition of this application as an iterative process where the token contextualized representations are progressively refined. We analyze this process at the token level during pre-training, fine-tuning, and inference. We show that tokens do not require the same amount of iterations and that difficult or crucial tokens for the task are subject to more iterations.

}, author = {Simoulin, Antoine and Beno{\^\i}t Crabb{\'e}} }