@mastersthesis {simoulin:tel-03791935, title = {Sentence embeddings and their relation with sentence structures}, year = {2022}, school = {Universit{\'e} Paris Cit{\'e}}, type = {phdTheses}, keywords = {Apprentissage profond, Deep learning, Natural Language Processing, Plongements de phrases, R{\'e}seaux de neurones structur{\'e}s, Sentence embeddings, Structured neural networks, Traitement automatique des langues naturelles}, url = {https://hal.archives-ouvertes.fr/tel-03791935}, author = {Simoulin, Antoine} } @inproceedings {simoulinUnifyingParsingTreeStructured2022, title = {Unifying Parsing and Tree-Structured Models for Generating Sentence Semantic Representations}, year = {2022}, pages = {267{\textendash}276}, publisher = {Association for Computational Linguistics}, address = {Hybrid: Seattle, Washington + Online}, abstract = {

We introduce a novel tree-based model that learns its composition function together with its structure. The architecture produces sentence embeddings by composing words according to an induced syntactic tree. The parsing and the composition functions are explicitly connected and, therefore, learned jointly. As a result, the sentence embedding is computed according to an interpretable linguistic pattern and may be used on any downstream task. We evaluate our encoder on downstream tasks, and we observe that it outperforms tree-based models relying on external parsers. In some configurations, it is even competitive with Bert base model. Our model is capable of supporting multiple parser architectures. We exploit this property to conduct an ablation study by comparing different parser initializations. We explore to which extent the trees produced by our model compare with linguistic structures and how this initialization impacts downstream performances. We empirically observe that downstream supervision troubles producing stable parses and preserving linguistically relevant structures.

}, author = {Simoulin, Antoine and Beno{\^\i}t Crabb{\'e}} } @inproceedings {simoulinContrastingDistinctStructured2021, title = {Contrasting Distinct Structured Views to Learn Sentence Embeddings}, year = {2021}, pages = {71{\textendash}79}, publisher = {Association for Computational Linguistics}, address = {Online}, abstract = {We propose a self-supervised method that builds sentence embeddings from the combination of diverse explicit syntactic structures of a sentence. We assume structure is crucial to building consistent representations as we expect sentence meaning to be a function of both syntax and semantic aspects. In this perspective, we hypothesize that some linguistic representations might be better adapted given the considered task or sentence. We, therefore, propose to learn individual representation functions for different syntactic frameworks jointly. Again, by hypothesis, all such functions should encode similar semantic information differently and consequently, be complementary for building better sentential semantic embeddings. To assess such hypothesis, we propose an original contrastive multi-view framework that induces an explicit interaction between models during the training phase. We make experiments combining various structures such as dependency, constituency, or sequential schemes. Our results outperform comparable methods on several tasks from standard sentence embedding benchmarks.}, author = {Simoulin, Antoine and Beno{\^\i}t Crabb{\'e}} } @inproceedings {simoulin:hal-03601428, title = {Contrasting distinct structured views to learn sentence embeddings}, year = {2021}, address = {Kyiv, Ukraine}, url = {https://hal.archives-ouvertes.fr/hal-03601428}, author = {Simoulin, Antoine and Beno{\^\i}t Crabb{\'e}} } @inproceedings {simoulinHowManyLayers2021, title = {How Many Layers and Why? An Analysis of the Model Depth in Transformers}, year = {2021}, pages = {221{\textendash}228}, publisher = {Association for Computational Linguistics}, address = {Online}, abstract = {

In this study, we investigate the role of the multiple layers in deep transformer models. We design a variant of Albert that dynamically adapts the number of layers for each token of the input. The key specificity of Albert is that weights are tied across layers. Therefore, the stack of encoder layers iteratively repeats the application of the same transformation function on the input. We interpret the repetition of this application as an iterative process where the token contextualized representations are progressively refined. We analyze this process at the token level during pre-training, fine-tuning, and inference. We show that tokens do not require the same amount of iterations and that difficult or crucial tokens for the task are subject to more iterations.

}, author = {Simoulin, Antoine and Beno{\^\i}t Crabb{\'e}} } @inproceedings {simoulin:hal-03601412, title = {How Many Layers and Why? An Analysis of the Model Depth in Transformers}, year = {2021}, address = {Bangkok, Thailand}, url = {https://hal.archives-ouvertes.fr/hal-03601412}, author = {Simoulin, Antoine and Beno{\^\i}t Crabb{\'e}} } @inproceedings {simoulin:hal-03265900, title = {Un mod{\`e}le Transformer G{\'e}n{\'e}ratif Pr{\'e}-entrain{\'e} pour le ______ fran{\c c}ais}, year = {2021}, pages = {246-255}, publisher = {ATALA}, address = {Lille, France}, keywords = {fran{\c c}ais., G{\'e}n{\'e}ratif, GPT, Pr{\'e}-entra{\^\i}n{\'e}, Transformer}, url = {https://hal.archives-ouvertes.fr/hal-03265900}, author = {Simoulin, Antoine and Beno{\^\i}t Crabb{\'e}}, editor = {Denis, Pascal and Grabar, Natalia and Fraisse, Amel and Cardon, R{\'e}mi and Jacquemin, Bernard and Kergosien, Eric and Balvet, Antonio} } @inproceedings {simoulinModeleTransformerGeneratif2021, title = {Un mod{\`e}le Transformer G{\'e}n{\'e}ratif Pr{\'e}-entrain{\'e} pour le fran{\c c}ais Generative Pre-trained Transformer in French (We introduce a French adaptation from the well-known GPT model)}, year = {2021}, month = {jun}, pages = {246{\textendash}255}, publisher = {ATALA}, address = {Lille, France}, abstract = {

Nous proposons une adaptation en fran\çais du fameux mod\èle Generative Pre-trained Transformer (GPT). Ce dernier appartient \à la cat\égorie des architectures transformers qui ont significativement transform\é les m\éthodes de traitement automatique du langage. Ces architectures sont en particulier pr\é-entra\în\ées sur des t\âches auto-supervis\ées et sont ainsi sp\écifiques pour une langue donn\ée. Si certaines sont disponibles en fran\çais, la plupart se d\éclinent avant tout en anglais. GPT est particuli\èrement efficace pour les t\âches de g\én\ération de texte. Par ailleurs, il est possible de l\&$\#$39;appliquer \à de nombreux cas d\&$\#$39;usages. Ses propri\ét\és g\én\ératives singuli\ères permettent de l\&$\#$39;utiliser dans des conditions originales comme l\&$\#$39;apprentissage sans exemple qui ne suppose aucune mise \à jour des poids du mod\èle, ou modification de l\&$\#$39;architecture.

}, author = {Simoulin, Antoine and Beno{\^\i}t Crabb{\'e}} }