@article {7536, title = {Leading voices: Dialogue semantics, cognitive science, and the polyphonic structure of multimodal interaction}, journal = {Language and Cognition}, volume = {15}, year = {2023}, pages = {148{\textendash}172}, abstract = {

The neuro-cognition of multimodal interaction---the embedded, embodied, predictive processing of vocal and non-vocal communicative behaviour---has developed into an important sub-field of cognitive science.
\%
It leaves a glaring lacuna, however, namely the dearth of a precise investigation of the meanings of the verbal and non-verbal communication signals that constitute multimodal interaction.
\%
Cognitively construable dialogue semantics provides a detailed and context-aware notion of meaning, and thereby contributes content-based identity conditions needed for distinguishing syntactically or form-based defined multimodal constituents.
\%
We exemplify this by means of two novel empirical examples: \emph{dissociated} uses of negative polarity utterances and head shaking, and attentional clarification requests addressing speaker/hearer roles.
\%
On this view, interlocutors are described as co-active agents, thereby motivating a replacement of \emph{sequential turn organisation} as a basic organizing principle with notions of \emph{leading} and \emph{accompanying voices}.
\%
The \emph{Multimodal Serialization Hypothesis} is formulated: Multimodal natural language processing is driven in part by a notion of \emph{vertical relevance}---relevance of utterances occurring simultaneously---which we suggest supervenes on sequential ({\textquoteleft}horizontal\&$\#$39;) relevance---relevance of utterances succeeding each other temporally.

}, doi = {10.1017/langcog.2022.30}, author = {L{\"u}cking, Andy and Ginzburg, Jonathan} }