@ARTICLE{BONFIGLI_2024_ARTICLE_BBMD_518430, 
 AUTHOR = {Bonfigli, A. and Bacco, L. and Merone, M. and Dell'Orletta, F.}, 
 TITLE = {From pre-training to fine-tuning: An in-depth analysis of Large Language Models in the biomedical domain}, 
 YEAR = {2024}, 
 ABSTRACT = {In this study, we delve into the adaptation and effectiveness of Transformer-based, pre-trained Large Language Models (LLMs) within the biomedical domain, a field that poses unique challenges due to its complexity and the specialized nature of its data. Building on the foundation laid by the transformative architecture of Transformers, we investigate the nuanced dynamics of LLMs through a multifaceted lens, focusing on two domain-specific tasks, i. e., Natural Language Inference (NLI) and Named Entity Recognition (NER). Our objective is to bridge the knowledge gap regarding how these models’ downstream performances correlate with their capacity to encapsulate task-relevant information. To achieve this goal, we probed and analyzed the inner encoding and attention mechanisms in LLMs, both encoder-and decoder-based, tailored for either general or biomedical-specific applications. This examination occurs before and after the models are fine-tuned across various data volumes. Our findings reveal that the models’ downstream effectiveness is intricately linked to specific patterns within their internal mechanisms, shedding light on the nuanced ways in which LLMs process and apply knowledge in the biomedical context. The source code for this paper is available at https: //github. com/agnesebonfigli99/LLMs-in-the-Biomedical-Domain}, 
 KEYWORDS = {Biomedical domain, Domain adaptation, Large Language Models}, 
 URL = {https://iris.cnr.it/handle/20.500.14243/518430}, 
 VOLUME = {157}, 
 DOI = {10.1016/j.artmed.2024.103003}, 
 ISSN = {0933-3657}, 
 JOURNAL = {ARTIFICIAL INTELLIGENCE IN MEDICINE}, 
}

@ARTICLE{BACCO_2023_ARTICLE_BDLMN_439016, 
 AUTHOR = {Bacco, L. and Dell'Orletta, F. and Lai, H. and Merone, M. and Nissim, M.}, 
 TITLE = {A text style transfer system for reducing the physician-patient expertise gap: An analysis with automatic and human evaluations}, 
 YEAR = {2023}, 
 ABSTRACT = {Physicians and patients often come from different backgrounds and have varying levels of education, which can result in communication difficulties in the healthcare process. To address this expertise gap, we present a "Text Style Transfer" system. Our system uses Semantic Textual Similarity techniques based on Sentence Transformers models to create pseudo-parallel datasets from a large, non-parallel corpus of lay and expert texts. This approach allowed us to train a denoising autoencoder model (BART), overcoming the limitations of previous systems. Our extensive analysis, which includes both automatic metrics and human evaluations from both lay (patients) and expert (physicians) individuals, shows that our system outperforms state-of-the-art models and is comparable to human-provided gold references in some cases}, 
 KEYWORDS = {Natural language processing, Text style transfer, Text simplification}, 
 PAGES = {1-18}, 
 URL = {https://www.sciencedirect.com/science/article/pii/S0957417423013763}, 
 VOLUME = {233}, 
 DOI = {10.1016/j.eswa.2023.120874}, 
 ISSN = {0957-4174}, 
 JOURNAL = {EXPERT SYSTEMS WITH APPLICATIONS}, 
}

@ARTICLE{BACCO_2022_ARTICLE_BRADVVDMPD_446362, 
 AUTHOR = {Bacco, L. and Russo, F. and Ambrosio, L. and D'Antoni, F. and Vollero, L. and Vadala, G. and Dell'Orletta, F. and Merone, M. and Papalia, R. and Denaro, V.}, 
 TITLE = {Natural language processing in low back pain and spine diseases: A systematic review}, 
 YEAR = {2022}, 
 ABSTRACT = {Natural Language Processing (NLP) is a discipline at the intersection between Computer Science (CS), Artificial Intelligence (AI), and Linguistics that leverages unstructured human-interpretable (natural) language text. In recent years, it gained momentum also in health-related applications and research. Although preliminary, studies concerning Low Back Pain (LBP) and other related spine disorders with relevant applications of NLP methodologies have been reported in the literature over the last few years. It motivated us to systematically review the literature comprised of two major public databases, PubMed and Scopus. To do so, we first formulated our research question following the PICO guidelines. Then, we followed a PRISMA-like protocol by performing a search query including terminologies of both technical (e. g., natural language and computational linguistics) and clinical (e. g., lumbar and spine surgery) domains. We collected 221 non-duplicated studies, 16 of which were eligible for our analysis. In this work, we present these studies divided into sub-categories, from both tasks and exploited models' points of view. Furthermore, we report a detailed description of techniques used to extract and process textual features and the several evaluation metrics used to assess the performance of the NLP models. However, what is clear from our analysis is that additional studies on larger datasets are needed to better define the role of NLP in the care of patients with spinal disorders}, 
 KEYWORDS = {natural language processing, Low Back Pain, Survey}, 
 URL = {http://www.scopus.com/record/display.url?eid=2-s2.0-85135163810\&origin=inward}, 
 VOLUME = {9}, 
 DOI = {10.3389/fsurg.2022.957085}, 
 ISSN = {2296-875X}, 
 JOURNAL = {FRONTIERS IN SURGERY}, 
}

@ARTICLE{BACCO_2021_ARTICLE_BCDM_444101, 
 AUTHOR = {Bacco, L. and Cimino, A. and Dell'Orletta, F. and Merone, M.}, 
 TITLE = {Explainable sentiment analysis: A hierarchical transformer-based extractive summarization approach}, 
 YEAR = {2021}, 
 ABSTRACT = {In recent years, the explainable artificial intelligence (XAI) paradigm is gaining wide research interest. The natural language processing (NLP) community is also approaching the shift of paradigm: building a suite of models that provide an explanation of the decision on some main task, without affecting the performances. It is not an easy job for sure, especially when very poorly interpretable models are involved, like the almost ubiquitous (at least in the NLP literature of the last years) transformers. Here, we propose two different transformer-based methodologies exploiting the inner hierarchy of the documents to perform a sentiment analysis task while extracting the most important (with regards to the model decision) sentences to build a summary as the explanation of the output. For the first architecture, we placed two transformers in cascade and leveraged the attention weights of the second one to build the summary. For the other architecture, we employed a single transformer to classify the single sentences in the document and then combine the probability scores of each to perform the classification and then build the summary. We compared the two methodologies by using the IMDB dataset, both in terms of classification and explainability performances. To assess the explainability part, we propose two kinds of metrics, based on benchmarking the models' summaries with human annotations. We recruited four independent operators to annotate few documents retrieved from the original dataset. Furthermore, we conducted an ablation study to highlight how implementing some strategies leads to important improvements on the explainability performance of the cascade transformers model}, 
 KEYWORDS = {Natural Language Processing, Sentiment Analysis, Explainable IA}, 
 URL = {http://www.scopus.com/record/display.url?eid=2-s2.0-85114289346\&origin=inward}, 
 VOLUME = {10}, 
 DOI = {10.3390/electronics10182195}, 
 ISSN = {2079-9292}, 
 JOURNAL = {ELECTRONICS}, 
}

@INPROCEEDINGS{BACCO_2020_INPROCEEDINGS_BCPMD_401373, 
 AUTHOR = {Bacco, L. and Cimino, A. and Paulon, L. and Merone, M. and Dell'Orletta, F.}, 
 TITLE = {A Machine Learning approach for Sentiment Analysis for Italian Reviews in Healthcare}, 
 YEAR = {2020}, 
 ABSTRACT = {In this paper, we present our approach to the task of binary sentiment classification for Italian reviews in healthcare domain. We first collected a new dataset for such domain. Then, we compared the results obtained by two different systems, one including a Support Vector Machine and one with BERT. For the first one, we linguistic pre-processed the dataset to extract hand-crafted features exploited by the classifier. For the second one, we oversampled the dataset to achieve better results. Our results show that the SVM-based system, without the worry of having to oversample, has better performance than the BERT-based one, achieving anF1-score of 91. 21%}, 
 KEYWORDS = {natural language processing, sentiment analisys}, 
 URL = {https://iris.cnr.it/handle/20.500.14243/401373}, 
 CONFERENCE_NAME = {Seventh Italian Conference on Computational Linguistics (CLiC-it 2020)}, 
}