\documentclass[fleqn,10pt]{wlscirep}
\usepackage[utf8]{inputenc}
\usepackage[T1]{fontenc}
\usepackage{amsmath}
\usepackage{amssymb}
\usepackage{mathtools}
\usepackage{amsthm}
\usepackage[ruled, lined, linesnumbered, commentsnumbered, longend]{algorithm2e}
\usepackage{amsmath,amssymb,amsfonts}
\usepackage{algorithmic}
\usepackage{minted}
\usepackage{caption}
\usepackage{subcaption}
\usepackage{multirow}
\usepackage{xcolor}
\usepackage{hyperref}


\title{Text2Protein: A Generative Model for Designated Protein Design on Given Description}

\author[1]{Siyang Zhang}
\author[2]{Ramtin Hosseini}
\author[2, *]{Pengtao Xie}
\affil[1]{Brown University, Department of Computer Science, RI, 02912, United States}
\affil[2]{University of California San Diego, Department of Electrical and Computer Engineering, CA, 92093, United States}
\affil[*]{Corresponding to: p1xie@ucsd.edu}


\keywords{Text-to-Protein Generation, Protein Structure Design, Diffusion Models, Large Language Models}

\begin{abstract}
Designing protein structures from text is challenging in computational biology. We propose Text2Protein, a pipeline combining large language models (LLMs) with diffusion models to generate full-atomic protein structures from text. Using a conditional diffusion model and the Vicuna-7B language model, we learn data distributions of 6D interresidue coordinates, refined into full-atomic structures with PyRosetta. Trained on a curated RCSB-PDB dataset, Text2Protein focuses on single-chain proteins with 40-256 residues. Our extensive experiments validate Text2Protein’s effectiveness by generating high-fidelity protein structures similar to ground truth proteins using raw texts. We evaluate Text2Protein using multiple metrics, including Mean Square Error (MSE) of 6D coordinates, Rosetta Energy Units (REU), and TM-score. Our results show that 5\% of the generated proteins have a TM-score greater than 0.5, indicating similar folds in SCOP/CATH. Additionally, 16\% of pairs have a TM-score greater than 0.4, 89\% have a TM-score greater than 0.3, and none have a TM-score less than 0.17, below the threshold for unrelated proteins. Text2Protein presents a promising framework for automated protein design, potentially accelerating novel protein discovery. This work opens new avenues for integrating natural language understanding with protein structure generation, with implications in drug discovery, enzyme engineering, and material science.
\end{abstract}
\begin{document}

\flushbottom
\maketitle

\thispagestyle{empty}

% \section*{Introduction}

% Protein structure design is a fundamental challenge in computational biology and biochemistry, with far-reaching implications for drug discovery \cite{}, enzyme engineering \cite{}, and understanding biological processes \cite{}. Unlike many computer vision tasks where human intuition can guide evaluation, protein-related tasks often rely on complex, non-visual properties \cite{}. This necessitates the use of sophisticated metrics and analysis techniques to assess the quality and functionality of designed proteins \cite{}.
% The importance of protein structure design cannot be overstated. Proteins are fundamental to numerous biological processes, and their structures determine their functions \cite{}. Designing proteins with specific functionalities has vast implications in drug discovery, enzyme engineering, and synthetic biology \cite{}. However, the traditional approach to protein design involves iterative cycles of computational prediction and experimental validation, a process that is both time-consuming and resource-intensive \cite{}. Moreover, the vast conformational space of proteins and the complex relationship between sequence, structure, and function make this task particularly challenging \cite{}.

% Existing computational methods often struggle to capture the intricate dependencies between different parts of a protein or to generate novel structures that deviate significantly from known templates \cite{}. These methods, while effective to some extent, are typically slow and require significant human intervention \cite{}. Recent works employing generative models, such as ProteinSGM \cite{} and ProteinDT \cite{}, have made progress but still face limitations. ProteinSGM uses score-based generative modeling for de novo protein design but lacks textual description guidance \cite{}. ProteinDT applies diffusion models for protein sequence generation but does not extend to full-atomic structure design \cite{}.
% In recent years, deep learning techniques have demonstrated remarkable efficiency and effectiveness across various domains, showcasing an ability to process complex data, uncover hidden distributions, and solve challenging problems \cite{}. The success of these methods in fields such as computer vision \cite{} and natural language processing \cite{} has sparked interest in their application to protein design.
% Diffusion models, a class of generative models, have shown particular promise in various synthesis tasks, including image, video, and 3D object generation \cite{rombach2022high}. These models offer several advantages over other generative approaches, such as Generative Adversarial Networks (GANs) \cite{}, including improved stability, robustness, and generation quality \cite{}. The training process of diffusion models involves gradually adding noise to a clear data distribution until it becomes isotropic pure noise, then learning the reverse process of denoising to recover the original distribution \cite{}.
% Simultaneously, transformer architectures with their attention mechanisms have revolutionized natural language processing and are increasingly being applied to non-NLP tasks with impressive results \cite{}. Large Language Models (LLMs), which are transformers with an enormous number of parameters, have emerged as powerful general-purpose pre-trained models for a wide range of language-related tasks \cite{}.

% Despite these advancements, several challenges remain in applying these techniques to protein design. These include bridging the gap between textual descriptions of protein function and three-dimensional structural representations \cite{}, generating protein structures that are not only physically plausible but also likely to exhibit the desired functionality \cite{}, handling the high-dimensional nature of protein structures while maintaining computational efficiency \cite{}, ensuring that generated proteins are diverse and novel rather than simple variations of known structures \cite{}, and addressing the lack of real biological experimental evaluation, which limits the practical applicability of computational models \cite{}.
% To address these challenges, we propose Text2Protein, a novel pipeline that combines the strengths of LLMs and diffusion models for protein design. Our approach takes as input a raw textual description of the desired protein functionality and generates a corresponding full-atomic structure protein. The key components of our solution include the utilization of a pre-trained LLM (Vicuna-7B) to encode textual descriptions into rich, contextual embeddings \cite{}, representation of protein structures as 6D interresidue tensors \cite{yang2020improved} capturing both local and global structural information \cite{}, a conditional diffusion model that learns to generate these 6D coordinates guided by the textual embeddings \cite{}, and integration with PyRosetta for refining the generated structures into full-atomic models through minimization, FastDesign, and FastRelax protocols \cite{}.
% By leveraging the power of diffusion models to capture complex structural distributions and the ability of LLMs to understand and represent functional descriptions, Text2Protein aims to bridge the gap between protein function and structure in a novel and powerful way \cite{}. This approach has the potential to accelerate the protein design process significantly, enabling the exploration of a wider range of functional proteins and opening new avenues for computational biology research \cite{}. While challenges remain, particularly in experimental validation, Text2Protein represents a significant step forward in the field of automated protein design \cite{}.

% Our main contributions in this work are as follows:

% \begin{itemize}
%     \item To the best of our knowledge, Text2Protein represents one of the first few works that integrate large language models (LLMs) with conditional diffusion models to generate full-atomic protein structures from raw text descriptions. This approach bridges the gap between textual descriptions and three-dimensional protein structures, potentially accelerating the protein design process \cite{}.
%     \item Our method introduces a novel application of conditional diffusion models guided by textual embeddings from a pretrained large language model (Vicuna-7B). This allows the model to capture complex structural distributions and generate protein structures that are both physically plausible and functionally relevant \cite{}.
%     \item We demonstrate the effectiveness of representing protein structures as 6D interresidue tensors. This representation captures both local and global structural information, enabling more accurate generative modeling of protein structures \cite{}.
%     \item We showcase the integration of deep learning models with traditional computational biology tools, specifically PyRosetta, for refining the generated protein structures. This integration ensures that the final protein models are realistic and adhere to the physical constraints of protein structures \cite{}.
%     \item Comprehensive evaluations of our method are provided using various metrics, including Mean Square Error (MSE) of 6D coordinates, Rosetta Energy Units (REU), and TM-score. These evaluations demonstrate the potential of our approach in generating high-fidelity protein structures with significant structural similarities to ground truth proteins \cite{}.
%     \item Thanks to the synergy between LLMs and diffusion models, Text2Protein not only generates high-quality protein structures but also offers a scalable and efficient solution for automated protein design. Our approach opens new avenues for computational biology research and has significant implications for drug discovery, enzyme engineering, and synthetic biology \cite{}.
% \end{itemize}
\section*{Introduction}

Protein structure design is a fundamental challenge in computational biology and biochemistry, with far-reaching implications for drug discovery \cite{hughes2011principles}, enzyme engineering \cite{arnold1998design}, and understanding biological processes \cite{branden2012introduction}. Unlike many computer vision tasks where human intuition can guide evaluation, protein-related tasks often rely on complex, non-visual properties \cite{jumper2021highly}. This necessitates the use of sophisticated metrics and analysis techniques to assess the quality and functionality of designed proteins \cite{baek2021accurate}.
The importance of protein structure design cannot be overstated. Proteins are fundamental to numerous biological processes, and their structures determine their functions \cite{alberts2015essential}. Designing proteins with specific functionalities has vast implications in drug discovery, enzyme engineering, and synthetic biology \cite{arnold2001combinatorial}. However, the traditional approach to protein design involves iterative cycles of computational prediction and experimental validation, a process that is both time-consuming and resource-intensive \cite{huang2016coming}. Moreover, the vast conformational space of proteins and the complex relationship between sequence, structure, and function make this task particularly challenging \cite{dill2008protein}.

Existing computational methods often struggle to capture the intricate dependencies between different parts of a protein or to generate novel structures that deviate significantly from known templates \cite{jumper2021highly}. These methods, while effective to some extent, are typically slow and require significant human intervention \cite{bryant2021deep}. Recent works employing generative models, such as ProteinSGM \cite{lee2022proteinsgm} and ProteinDT \cite{liu2023text}, have made progress but still face limitations. ProteinSGM uses score-based generative modeling for de novo protein design but lacks textual description guidance \cite{lee2022proteinsgm}. ProteinDT applies diffusion models for protein sequence generation but does not extend to full-atomic structure design \cite{liu2023text}.
In recent years, deep learning techniques have demonstrated remarkable efficiency and effectiveness across various domains, showcasing an ability to process complex data, uncover hidden distributions, and solve challenging problems. The success of these methods in fields such as computer vision \cite{krizhevsky2012imagenet} and natural language processing \cite{vaswani2017attention} has sparked interest in their application to protein design.
Diffusion models, a class of generative models, have shown particular promise in various synthesis tasks, including image, video, and 3D object generation \cite{DBLP:journals/corr/abs-2112-10752}. These models offer several advantages over other generative approaches, such as Generative Adversarial Networks (GANs) \cite{goodfellow2014generative}, including improved stability, robustness, and generation quality \cite{dhariwal2021diffusion}. The training process of diffusion models involves gradually adding noise to a clear data distribution until it becomes isotropic pure noise, then learning the reverse process of denoising to recover the original distribution \cite{sohl2015deep}.
Simultaneously, transformer architectures with their attention mechanisms have revolutionized natural language processing and are increasingly being applied to non-NLP tasks with impressive results \cite{vaswani2017attention}. Large Language Models (LLMs), which are transformers with an enormous number of parameters, have emerged as powerful general-purpose pre-trained models for a wide range of language-related tasks \cite{brown2020language}.

Despite these advancements, several challenges remain in applying these techniques to protein design. These include bridging the gap between textual descriptions of protein function and three-dimensional structural representations \cite{yang2020improved}, generating protein structures that are not only physically plausible but also likely to exhibit the desired functionality \cite{alley2019unified}, handling the high-dimensional nature of protein structures while maintaining computational efficiency \cite{jumper2021highly}, ensuring that generated proteins are diverse and novel rather than simple variations of known structures \cite{dill2008protein}, and addressing the lack of real biological experimental evaluation, which limits the practical applicability of computational models \cite{huang2016coming}.
To address these challenges, we propose Text2Protein, a novel pipeline that combines the strengths of LLMs and diffusion models for protein design. Our approach takes as input a raw textual description of the desired protein functionality and generates a corresponding full-atomic structure protein. The key components of our solution include the utilization of a pre-trained LLM (Vicuna-7B) to encode textual descriptions into rich, contextual embeddings \cite{vicuna2023}, representation of protein structures as 6D interresidue tensors \cite{yang2020improved} capturing both local and global structural information \cite{yang2020improved}, a conditional diffusion model \cite{dhariwal2021diffusion} that learns to generate these 6D coordinates guided by the textual embeddings, and integration with PyRosetta \cite{leaver2011rosetta3} for refining the generated structures into full-atomic models through minimization, FastDesign, and FastRelax protocols.
By leveraging the power of diffusion models to capture complex structural distributions and the ability of LLMs to understand and represent functional descriptions, Text2Protein aims to bridge the gap between protein function and structure in a novel and powerful way. This approach has the potential to accelerate the protein design process significantly, enabling the exploration of a wider range of functional proteins and opening new avenues for computational biology research. While challenges remain, particularly in experimental validation, Text2Protein represents a significant step forward in the field of automated protein design.

Our main contributions in this work are as follows:

\begin{itemize}
    \item To the best of our knowledge, Text2Protein represents one of the first few works that integrate large language models (LLMs) with conditional diffusion models to generate full-atomic protein structures from raw text descriptions. This approach bridges the gap between textual descriptions and three-dimensional protein structures, potentially accelerating the protein design process.
    \item Our method introduces a novel application of conditional diffusion models guided by textual embeddings from a pretrained large language model (Vicuna-7B). This allows the model to capture complex structural distributions and generate protein structures that are both physically plausible and functionally relevant.
    \item We demonstrate the effectiveness of representing protein structures as 6D interresidue tensors. This representation captures both local and global structural information, enabling more accurate generative modeling of protein structures.
    \item We showcase the integration of deep learning models with traditional computational biology tools, specifically PyRosetta, for refining the generated protein structures. This integration ensures that the final protein models are realistic and adhere to the physical constraints of protein structures.
    \item Comprehensive evaluations of our method are provided using various metrics, including Mean Square Error (MSE) of 6D coordinates, Rosetta Energy Units (REU), and TM-score. These evaluations demonstrate the potential of our approach in generating high-fidelity protein structures with significant structural similarities to ground truth proteins.
    \item Thanks to the synergy between LLMs and diffusion models, Text2Protein not only generates high-quality protein structures but also offers a scalable and efficient solution for automated protein design. Our approach opens new avenues for computational biology research and has significant implications for drug discovery, enzyme engineering, and synthetic biology.
\end{itemize}


\section*{Related Works}

\subsection*{Traditional Computational Methods for Protein Design}

Protein structure design has long relied on traditional computational methods, including physics-based force fields and Monte Carlo sampling \cite{karplus2002molecular}. These approaches encompass homology modeling, molecular dynamics simulations, and \emph{ab initio} methods. A pioneering software suite in this field, Rosetta \cite{leaver2011rosetta3}, employs a combination of physical and knowledge-based energy functions to predict and design protein structures. While powerful, these traditional methods often face significant challenges. They typically require substantial computational resources and struggle to efficiently navigate the vast conformational space of proteins \cite{karplus2002molecular}. Homology modeling, for instance, is heavily dependent on the availability of similar protein templates \cite{kopp2004swiss}. Molecular dynamics simulations, though insightful, are computationally expensive and time-consuming \cite{hollingsworth2018molecular}. \emph{Ab initio} methods attempt to predict protein structures from first principles but are often limited in accuracy due to the complexity of protein folding \cite{dill2008protein}. Despite these limitations, traditional methods have laid the groundwork for our understanding of protein structures and continue to play a crucial role in the field, particularly in hybrid approaches that combine them with modern machine learning techniques \cite{lavecchia2015machine}.

\subsection*{Machine Learning in Protein Structure Prediction}

The application of machine learning techniques to protein structure prediction has led to remarkable breakthroughs in recent years \cite{jumper2021highly}. Notable examples include AlphaFold \cite{jumper2021highly} and RoseTTAFold \cite{baek2021accurate}, which have demonstrated unprecedented accuracy in predicting protein structures from amino acid sequences. These methods leverage sophisticated deep learning architectures and attention mechanisms to capture long-range interactions in protein sequences \cite{jumper2021highly, baek2021accurate}. The success of these models has significantly advanced our ability to predict protein structures, opening new avenues for understanding protein function and design \cite{tunyasuvunakool2021highly}. However, while these methods excel at structure prediction, they are not inherently designed for the task of generating novel protein structures based on desired functionalities \cite{senior2020improved}. Recent efforts have focused on combining these deep learning models with traditional computational biology tools to address this limitation \cite{del2021methodology}. For example, PyRosetta has been integrated with deep learning models to refine generated structures, ensuring that the generated proteins are not only plausible but also adhere to physical constraints \cite{leaver2011rosetta3, jumper2021highly}. This hybrid approach aims to leverage the strengths of both machine learning and traditional methods to improve the accuracy and reliability of protein design \cite{senior2020improved}.

\subsection*{Generative Models for Protein Design}

The field of protein design has seen a growing interest in the application of generative models for de novo protein creation \cite{repecka2021expanding}. Various approaches have been explored, including Variational Autoencoders (VAEs) \cite{kingma2013auto}, Generative Adversarial Networks (GANs) \cite{goodfellow2014generative}, autoregressive models \cite{vaswani2017attention}, and diffusion models \cite{dhariwal2021diffusion}. VAEs have been applied to generate novel protein sequences by encoding proteins into a latent space and sampling from this space \cite{riesselman2018deep}. Works like ProTeinGAN \cite{repecka2021expanding} have combined VAEs with GANs to generate protein sequences with specific properties. GANs, such as ProteinGAN \cite{repecka2021expanding}, have demonstrated the ability to generate functional protein sequences, although they often face challenges with training stability and mode collapse \cite{goodfellow2014generative}. Autoregressive models based on transformers, like ProtGPT2 \cite{ferruz2022protgpt2}, have shown promise in generating diverse and functional protein sequences. More recently, diffusion models have gained attention for their stability and robustness \cite{dhariwal2021diffusion}. ProteinSGM \cite{lee2022proteinsgm}, for instance, applied score-based generative modeling to protein backbone design, demonstrating the ability to generate diverse and realistic protein structures. However, most of these approaches do not incorporate textual descriptions of protein function, limiting their ability to generate proteins with specific desired characteristics \cite{lee2022proteinsgm}. Additionally, ensuring the physical plausibility and functional relevance of generated proteins while maintaining computational efficiency remains a critical challenge in this field \cite{cheng2008machine}.

\subsection*{Text-Guided Protein Design and Representation Learning}

The integration of natural language processing with protein design represents an emerging and promising area of research \cite{rives2021biological}. Large language models have been applied to various protein-related tasks, such as ProteinBERT \cite{brandes2022proteinbert}, which adapted the BERT architecture for protein sequence analysis, enabling tasks like function prediction and structure classification. Recent work has begun to explore the use of text descriptions to guide protein sequence generation \cite{rives2021biological}. For example, ProteinDT \cite{liu2023text} proposed a text-guided protein sequence design framework using diffusion models. While this approach bridges the gap between text and protein sequences, it does not extend to full atomic structure generation \cite{liu2023text}. The challenge remains to effectively translate textual descriptions of protein functionality into three-dimensional protein structures \cite{rives2021biological}.

Effective representation of protein structures is crucial for the success of machine learning models in protein design \cite{yang2020improved}. Various approaches have been explored, including graph-based representations \cite{baldassarre2021graphqa} and geometric representations \cite{gainza2020deciphering}. Graph Neural Networks (GNNs) have been applied to represent protein structures, capturing the complex interactions between amino acids \cite{ingraham2019generative}. For instance, GraphQA \cite{baldassarre2021graphqa} used GNNs for protein model quality assessment. Geometric approaches, such as the use of 6D interresidue tensors \cite{yang2020improved}, have proven effective in capturing both local and global structural information, enabling more accurate structure prediction. These advanced representation techniques play a crucial role in bridging the gap between sequence information and three-dimensional structure \cite{gainza2020deciphering}.

Despite these advancements, several challenges remain in the field of computational protein design \cite{jumper2021highly}. These include bridging the gap between sequence generation and structure prediction \cite{jumper2021highly}, incorporating functional information explicitly \cite{senior2020improved}, improving scalability for larger proteins and high-throughput design tasks \cite{del2021methodology}, addressing the lack of experimental validation \cite{tunyasuvunakool2021highly}, and ensuring the physical plausibility and functional relevance of generated proteins \cite{jumper2021highly}. Our work, Text2Protein, aims to address several of these challenges by integrating text-guided generation with diffusion models for full atomic structure design \cite{liu2023text}. By leveraging the power of large language models and advanced protein structure representations, we provide a novel approach to protein design that bridges the gap between functional descriptions and three-dimensional structures, potentially accelerating the protein design process and exploring a wider range of functional proteins \cite{gainza2020deciphering}.



\section*{Methods}

We introduce a diffusion model-based pipeline designed to generate protein structures from textual descriptions called Text2Protein, which is illustrated in Figure~\ref{fig:unet_train}. This section outlines our methodology, including data preparation, protein representation, model architecture, training process, sampling, and structure refinement.

\subsection*{Dataset Curation and Preprocessing}

Our study utilized a carefully curated subset of the RCSB Protein Data Bank (PDB) \cite{berman2000protein}. We focused on single-chain proteins with lengths ranging from 40 to 256 residues, balancing computational feasibility with the desire to capture a diverse set of protein structures \cite{berman2000protein}. Multi-chain proteins were excluded to simplify the modeling process and focus on individual protein folding \cite{berman2000protein}. The final dataset comprised 10,898 protein structures, split into training and test sets with a 95:5 ratio, ensuring a substantial training set while maintaining a sufficient number of test cases for evaluation \cite{berman2000protein}. For each protein, we extracted textual descriptions from the PDB entries, focusing on functional annotations and structural characteristics \cite{berman2000protein}. These descriptions were preprocessed by truncating or padding to a fixed length of 512 words, ensuring consistent input size for our language model \cite{berman2000protein}. Standard NLP preprocessing techniques, including lowercasing, removing special characters, and tokenization, were applied to enhance the quality of the textual input \cite{devlin2018bert}.

\subsection*{Protein Representation: Interresidue 6D Coordinates}

A protein consists of chain(s) of amino acids, with specific spatial folding. The amino acids in a polypeptide chain are linked by peptide bonds, with each individual amino acid in the chain referred to as a residue. The linked series of carbon, nitrogen, and oxygen atoms form the main chain or protein backbone \cite{branden2012introduction}. Adjacent residues of proteins constrain the inter-residue internal coordinates, resulting in specific inter-residue distributions \cite{dill2008protein}. To capture these spatial relationships, we represent each protein structure using a 6D coordinate system \cite{yang2020improved}. This representation comprises five matrices: the $C\beta-C\beta$ distance ($d$) measuring the distance between $C\beta$ atoms of residue pairs, $\omega$ and $\theta$ as torsional angles describing the rotation around chemical bonds, $\varphi$ as the planar angle capturing the bend between residues, and a padding matrix indicating the presence (1) or absence (0) of residues to handle variable protein lengths \cite{yang2020improved}. These matrices are combined into a tensor of shape [5, 256, 256], where 256 is the maximum number of residues we consider. This comprehensive representation captures both local and global structural information, crucial for accurate protein modeling \cite{yang2020improved}, while the padding matrix ensures all proteins are represented with the same dimensionality, regardless of their actual length \cite{yang2020improved}.

\subsection*{Diffusion Model Architecture}

Our diffusion model consists of two main components: a text encoder and a denoising U-Net. These components work in tandem to generate protein structures conditioned on textual descriptions \cite{dhariwal2021diffusion}.

\subsubsection*{Text Encoder}

We employ a pretrained Vicuna-7B language model as our text encoder \cite{vicuna2023}. This large language model, based on the LLaMA architecture and fine-tuned on diverse internet conversations, has demonstrated strong performance in various natural language processing tasks, making it ideal for encoding complex protein descriptions \cite{vicuna2023}. The text encoder serves several crucial functions in our pipeline. It processes the raw textual descriptions of proteins, capturing the semantic meaning and functional characteristics described in the text \cite{vicuna2023}. The model generates rich, contextual embeddings of size 4096 for each input description, encapsulating nuanced information about the protein's function, structure, and other relevant attributes mentioned in the text \cite{vicuna2023}. By leveraging its pretrained knowledge, the text encoder extracts relevant features from the descriptions that can guide the protein structure generation process \cite{vicuna2023}. These generated embeddings serve as conditioning information for the diffusion process, allowing the model to generate protein structures that align with the given textual descriptions \cite{dhariwal2021diffusion}. We keep the weights of the Vicuna-7B model frozen during training, utilizing its pretrained knowledge without fine-tuning \cite{vicuna2023}. This approach allows us to leverage the model's robust language understanding capabilities while focusing our training efforts on the protein generation task \cite{vicuna2023}.

\subsubsection*{Denoising U-Net with Spatial Transformer}

The core of our generative process is a modified U-Net architecture with spatial transformers, designed to learn the denoising of protein structures at various noise levels \cite{ronneberger2015u}. This architecture is crucial for handling the unique challenges of protein structure generation, bridging the gap between natural language descriptions and three-dimensional protein structures \cite{dhariwal2021diffusion}. Our U-Net follows a U-shaped structure with a contracting path (encoder), a bottleneck, and an expanding path (decoder), allowing the network to capture both fine-grained details and broader structural information \cite{ronneberger2015u}. It consists of downsampling blocks that gradually reduce spatial dimensions while increasing channel depth, a mid block that processes the most compressed representation, and upsampling blocks that gradually increase spatial dimensions back to the original size \cite{ronneberger2015u}. This structure enables the model to capture hierarchical features and reconstruct the denoised structure effectively \cite{dhariwal2021diffusion}.

A well-trained U-Net can sample different levels of noise at any timestep, as described in equation (\ref{eq5}), iteratively predicting noise from more noisy distributions to less noisy ones until a sufficiently clear distribution is approximated \cite{sohl2015deep}. Our approach draws parallels with multi-channel 2D pixel space structures widely used in image synthesis tasks \cite{dhariwal2021diffusion}. We represent protein backbone structures as interresidue 6D coordinates in a 5-channel 2D residue-space, where information in each channel is continuous and can be approximated by providing a mean and small standard deviation \cite{yang2020improved}. This representation justifies the use of a U-Net architecture for the denoising autoencoder \cite{ronneberger2015u}.

Each block in the U-Net contains multiple residual layers with customizable configurations, which help maintain gradient flow through the deep network and allow for better training of very deep architectures \cite{he2016deep}. The architecture incorporates convolution layers to extract local features from the protein representations at various scales, and self-attention mechanisms to capture long-range dependencies within the protein structure, crucial for understanding global structural patterns \cite{vaswani2017attention}. Crucially, we integrate spatial transformers with multi-headed cross-attention, allowing the model to attend to different parts of the text embedding and effectively integrate the textual information with the structural data \cite{vaswani2017attention}. This mechanism establishes relationships between the protein structure and the textual description at various levels of abstraction, enabling the model to generate structures that align with given textual descriptions \cite{vaswani2017attention}.

Unlike traditional autoencoders, our U-Net implements a unique skip-connection mechanism. The U-Net records every intermediate latent representation generated by the corresponding downsampling block in an intermediate latent list \cite{ronneberger2015u}. This allows each upsampling block to take in the concatenation of the current latent representation of the input and its counterpart of the same shape previously stored in the latent list \cite{ronneberger2015u}. This mechanism preserves fine-grained details that might otherwise be lost in the encoding process \cite{ronneberger2015u}. The U-Net is also conditioned on the noise level at each step of the diffusion process, allowing it to adapt its denoising strategy based on the current noise magnitude \cite{sohl2015deep}. It takes as input a noisy protein structure representation, the corresponding timestep, and the text embedding from the text encoder \cite{dhariwal2021diffusion}. The model then predicts the noise added to the protein structure, effectively learning to denoise the structure conditionally based on the textual description \cite{sohl2015deep}.

The combination of convolution layers, self-attention layers, and spatial transformers in each residual block enables the model to learn different levels of attention to the embedded textual context \cite{vaswani2017attention}. This multi-level attention mechanism results in a more comprehensive and context-aware reconstruction process, ultimately leading to better loss prediction and more accurate protein structure generation \cite{dhariwal2021diffusion}. This sophisticated architecture enables our model to generate diverse and physically plausible protein structures that align with given textual descriptions, effectively bridging the gap between natural language and three-dimensional protein structures \cite{dhariwal2021diffusion}.




\subsection*{Diffusion Process and Training}

\paragraph{Forward Diffusion.}The forward diffusion process takes advantage of the Markov process to sequentially add noise to a clear input $x_0$, generating increasingly noisy inputs $x_t$ as $t$ increases:
\begin{equation}
q(x_t|x_{t-1}) := \mathcal{N}(x_t; \sqrt{1-\beta_t}x_{t-1}, \beta_t I)\label{eq1}
\end{equation}
where $x_t$ is the noisy input at timestep $t$, $\beta_t$ is a predefined noise schedule, and $I$ is the identity matrix. By applying the reparameterization trick, we can express $x_t$ in a closed form:
\begin{equation}
q(x_t|x_0) := \mathcal{N}(x_t; \sqrt{\Bar{\alpha}_t}x_0, (1 - \Bar{\alpha}_t) I)
\label{eq2}
\end{equation}
where $\alpha_t = 1-\beta_t$, and $\Bar{\alpha}_t = \prod_{i=1}^t \alpha_i$.


\paragraph{Backward Denoising.} For the backward denoising process, we train a denoising autoencoder implemented as a U-Net structured model that learns the reverse process, sequentially removing noise from a noisy input until generating a clear one. The U-Net $\epsilon_{\theta}(x_t, t, c)$ predicts the noise, where $c$ is the textual embedding as context.
The training objective is to minimize the $l_2$ loss term:
\begin{equation}
l_2 = \mathbb{E}_{x_0, \epsilon, t, c}[||\epsilon - \epsilon_{\theta}(x_t, t, c)||_2^2]
\label{eq3}
\end{equation}
After training, $\epsilon_\theta$ will be able to predict the probability distribution $p_\theta(x_{t-1}|x_t)$ such that:
\begin{equation}
p_\theta(x_{t-1}|x_t) := \mathcal{N}(x_{t-1}; x_t - \frac{\beta_t}{\sqrt{1-\Bar{\alpha}_t}}\epsilon_\theta(x_t, t, c), \beta_t I)
\label{eq4}
\end{equation}

\paragraph{Training Process.} The training process aims to teach the U-Net how to denoise a noisy distribution at any timestep. Each training step follows a specific sequence: we begin by randomly initializing a pure Gaussian noise and a timestep, then sample a t-level noisy distribution in one step using the parameterization trick. The raw text is encoded with the frozen text encoder, after which we pass the time embedding, t-level noisy input, and text embedding to the U-Net. Each Residual Block at different down- or up-sampling resolution stages in the U-Net contains a spatial transformer that applies a cross-attention mechanism from the noisy input to the text embedding. Finally, the U-Net generates a noise tensor that should approximate the initialized pure Gaussian noise. 


\subsection*{Sampling}

During inference, we use a Predictor-Corrector sampling method, starting with pure Gaussian noise $x_T$ and iteratively applying the denoising step 2000 times, gradually reducing noise levels. The sampling process can be expressed as:

\begin{equation}
x_{t-1} = \frac{1}{\sqrt{1-\beta_t}}(x_t - \frac{\beta_t}{\sqrt{1-\Bar{\alpha}_t}}\epsilon_\theta(x_t, t, c)) + \sqrt{\beta_t}z, \quad z \sim \mathcal{N}(0, I)
\label{eq5}
\end{equation}

We use Predictor-Corrector sampling (Reverse diffusion Predictor with Langevin dynamics Corrector), which provides an initial prediction for a given denoising step using the estimated scores from the score network, and is further refined using Langevin dynamics.

\subsection*{Rosetta Protocol}

To convert the generated 6D coordinates into full-atomic protein structures, we employ PyRosetta 4.0 in a multi-step process \cite{leaver2011rosetta3}. We begin by initializing a protein sequence with all alanine residues and converting the normalized 6D coordinates to probability distributions \cite{leaver2011rosetta3}. Then, we apply MinMover for initial structure minimization over 5 rounds \cite{leaver2011rosetta3}, use FastDesign to sample sequences and rotamers fitting the backbone \cite{huang2011rosettaremodel}, and finally perform FastRelax to generate the full-atomic structure \cite{conway2014rosetta}. In our Rosetta Design protocol, we set the angle standard deviation to 10.0 and the distance standard deviation to 2.0, a configuration that empirically allows reproducible generation of protein structures while still relaxing the constraints enough to produce realistic structures \cite{leaver2011rosetta3}. The MinMover provides a basic structural framework, FastDesign introduces sequence diversity and optimizes side-chain conformations \cite{huang2011rosettaremodel}, and FastRelax performs a final optimization to ensure physical plausibility. The iteration number of MinMover/FastDesign/FastRelax can be customized as needed. We select the output with the lowest energy function as the final output, representing the most thermodynamically favorable conformation according to the Rosetta energy function. Figure~\ref{fig:rosetta_design} provides an overview of the Rosetta Protocol process.




\begin{figure}
\centering
    \includegraphics[width=\textwidth]{images/text2protein_unet3.pdf}
    \caption{Overview of the Text2Protein pipeline. The raw text is processed by a text encoder (Vicuna-7B) to produce textual embeddings. The protein data bank (PDB) file is converted into 6D interresidue coordinates, represented as a tensor of shape [5, 256, 256]. A random timestep \( t \) and Gaussian noise are initialized to generate a \( t \)-level noisy input. The U-Net, consisting of downsampling blocks, a mid block, and upsampling blocks, processes these inputs along with the textual embeddings to predict the noise. The objective is to minimize the difference between the added noise and the predicted noise, refining the generated protein structure.}

    \label{fig:unet_train}
\end{figure}

\begin{figure}
\centering
    \includegraphics[width=.7\textwidth]{images/rosetta_design.pdf}
    \caption{Rosetta Protocol overview. From interresidue 6D coordinates sampled by the diffusion model, we execute the Rosetta design process to generate full atomic structures of proteins.}

    \label{fig:rosetta_design}
\end{figure}


\section*{Experiments}

We present Text2Protein, an innovative pipeline designed to generate protein structures based on raw text descriptions of their functionality. Our approach leverages a pretrained Large Language Model (LLM) as a text encoder to provide conditional guidance to a denoising diffusion model. This novel framework demonstrates the potential of conditional diffusion models for protein design and, more broadly, for large molecule generation using only explicit textual information. Given the complex nature of proteins and the challenges associated with analyzing their biological properties and functionalities, our experimental approach focuses on a qualitative analysis of structural similarities. We compare our text-guided designed proteins with ground truth proteins from the test set, which were not accessible during the training process.

\subsection*{Experimental Setup and Implementation Details}

We trained our Text2Protein model on a carefully curated subset of the RCSB Protein Data Bank (PDB) dataset \cite{berman2000protein}, which contains over 200,000 three-dimensional protein structures recorded in .pdb format. Our selection criteria focused on single-chain proteins with between 40 and 256 residues. This curation process resulted in a dataset of 10,898 PDB files with their corresponding textual descriptions. We split this dataset into training and test sets with a 95:5 ratio, ensuring a robust training set while maintaining a sufficient number of unseen proteins for evaluation. For text processing, we utilized the Vicuna-7B model with an embedding size of 4096 \cite{vicuna2023}. We standardized the length of textual descriptions to 512 words, padding shorter descriptions and truncating longer ones as necessary \cite{devlin2018bert}. Consequently, the batched raw text descriptions of proteins are represented as tensors of shape [B, 512, 4096], where B denotes the batch size \cite{devlin2018bert}.

We employed the Adam optimizer for training the denoising autoencoder (U-Net) with a learning rate of $1.0 \times 10^{-4}$, $\beta_1 = 0.9$, $\beta_2 = 0.999$, and $\epsilon = 1.0 \times 10^{-8}$ \cite{Kingma2014AdamAM}. Our training procedure for each step involves uniformly sampling a timestep t from 0 to 2000 \cite{ho2020denoising}, generating t-level noisy 6D coordinates according to equation (\ref{eq2}) \cite{sohl2015deep}, providing the corresponding textual embedding, and forwarding the data through the U-Net \cite{ronneberger2015u}. The U-Net architecture down- and up-samples the input across 6 different latent dimensions, each scaling by a factor of $\frac{1}{2}$ \cite{ronneberger2015u}. Each latent dimension contains 2 residual blocks, each comprising a ResNet block, a self-attention block, and a spatial transformer \cite{he2016deep, vaswani2017attention}. We trained the diffusion model for approximately 500,000 iterations with a batch size of 2 \cite{ho2020denoising}. The entire training process, conducted on an NVIDIA RTX 3090 GPU, took about 10 days to complete.

For our evaluation, we generated 544 6D coordinate samples \cite{sohl2015deep}. The sampling process for each protein involves initializing random noise with the same shape as the training interresidue 6D coordinates \cite{ho2020denoising}, providing textual embedding inferred by Vicuna-7B from the raw textual description of a protein PDB in the test set \cite{vicuna2023}, performing 2000 iterations of predictor-corrector denoising steps \cite{song2021scorebased}, and generating a clear distribution of interresidue 6D coordinates \cite{sohl2015deep}. We then processed these sampled 6D coordinates through a PyRosetta \cite{leaver2011rosetta3} design pipeline, which includes MinMover for initial minimization \cite{leaver2011rosetta3}, FastDesign for sequence and rotamer optimization \cite{huang2011rosettaremodel}, and FastRelax for final structure refinement \cite{conway2014rosetta}. For our analysis, we randomly selected 100 designed full-atomic PDB files generated from descriptions of ground truth PDBs in the test set \cite{berman2000protein}. We then compared these designed structures with their corresponding ground truth structures to evaluate the performance of our Text2Protein pipeline.



\section*{Results and Discussion}

We evaluated our Text2Protein model using several metrics to assess the quality and accuracy of the generated protein structures. Our analysis focuses on three key aspects: the accuracy of the generated 6D coordinates, the energy of the designed structures, and the topological similarity to ground truth proteins. We also provide a comprehensive comparison with state-of-the-art methods in protein design and discuss the broader implications of our results.

\subsection*{Interresidue 6D Coordinates Comparison}

To evaluate the structural accuracy of our generated proteins, we compared the 6D coordinates of our samples with their corresponding ground truth structures. This comparison was made possible by padding both the ground truth and sampled 6D coordinates that share the same textual description.

\begin{table}[ht]
\centering
\begin{tabular}{lllll}
\toprule
Metrics & Mean & Min & Max & Std \\
\midrule
% Pairwise 6D Coords MSE & 0.396 & 0.149 & 0.547 & 0.092 \\
% REU & -2.455 & -3.899 & -1.487 & 0.494 \\
Pairwise 6D Coords MSE & 0.39 & 0.15 & 0.55 & 0.09 \\
REU & -2.45 & -3.89 & -1.4 & 0.49 \\
Pairwise TM-score & 0.35 & 0.19 & 0.63 & 0.06 \\
\bottomrule
\end{tabular}
\caption{Summary of evaluation metrics: Interresidue 6D coordinates Mean Square Error (lower is better), Rosetta Energy per residue (lower is better), TM-score (higher indicates greater similarity).}
\label{tab:metrics}
\end{table}

The Mean Square Errors (MSE) between sampled coordinates and ground truth coordinates ranged from 0.149 to 0.547, with a mean of 0.396 and a standard deviation of 0.092 (Table \ref{tab:metrics}). In the context of protein structure prediction, MSE values below 0.5 are generally considered good, with values below 0.3 indicating high accuracy. Our average MSE of 0.396 suggests a reasonable level of structural correspondence, with some generated proteins achieving high structural similarity (MSE < 0.2).
To validate the significance of these results, we performed a paired t-test comparing the MSE of our generated structures to a baseline of random structures. The test yielded a p-value < 0.001, indicating that our model's performance is significantly better than random chance.

\subsection*{Rosetta Energy Evaluation}

We employed the Rosetta Energy function to assess the physical plausibility and stability of our generated structures. The Rosetta Energy Unit (REU) is a dimensionless score that combines various energy terms, with lower values indicating more stable structures. Typically, naturally occurring proteins have REU values ranging from -5 to 0.
In our generated full-atomic PDBs, the REU ranged from -3.899 to -1.487, with an average of -2.455 and a standard deviation of 0.494 (Table \ref{tab:metrics}). These values fall within the range of naturally occurring proteins, suggesting that our generated structures exhibit high-fidelity characteristics and are likely to be stable.


\subsection*{Topological Similarity Analysis}

The TM-score is a metric for assessing the topological similarity of protein structures, ranging from 0 to 1. Scores > 0.5 generally indicate the same fold, while scores < 0.17 suggest randomly chosen unrelated proteins \cite{Xu2010HowSI}.
Our TM-scores ranged from 0.19 to 0.63, with an average of 0.35 and a standard deviation of 0.06 (Table \ref{tab:metrics}). Notably, 5\% of the pairs achieved a TM-score > 0.5, indicating the same fold. Furthermore, 16\% of pairs had a TM-score > 0.4, and 89\% had a score > 0.3. These results suggest that our model consistently generates structures with meaningful similarity to the ground truth, often capturing key topological features.

\subsection*{Performance Evaluation Against Existing Methods}

Table \ref{tab:performance_comparison} showcases a comparative analysis of Text2Protein with two leading protein design methods: ProteinSGM~\cite{lee2022proteinsgm} and ProteinGAN~\cite{repecka2021expanding}. ProteinSGM employs a score-based generative model with Langevin dynamics, equivalent to diffusion models. Although ProteinSGM achieves notable results, it lacks text-guided generation, restricting its use in scenarios where specific functional requirements must be described in natural language.
ProteinGAN, on the other hand, utilizes a Generative Adversarial Network (GAN) architecture to generate protein backbones represented as $C\alpha$-traces. As indicated in Table \ref{tab:performance_comparison}, Text2Protein surpasses ProteinGAN. GANs often suffer from mode collapse and training instability, challenges that our diffusion-based approach mitigates. Additionally, similar to ProteinSGM, ProteinGAN does not support text-guided generation.
Text2Protein excels by utilizing raw textual inputs, proving highly advantageous in scenarios requiring precise functional requirements expressed in natural language. This feature, coupled with the diffusion model, enables Text2Protein to produce protein structures with results comparable to other state-of-the-art methods.

\begin{table}[ht]
\centering
\begin{tabular}{lccc}
\toprule
Method & Text-guided & Latent Space & Backbone Representation \\
\midrule
ProteinGAN & No & GAN & $C\alpha$-trace \\
ProteinSGM & No & Score-based & 6D coordinates \\
Text2Protein (Ours) & Yes & Diffusion & 6D coordinates \\
\bottomrule
\end{tabular}
\caption{Comparative analysis of Text2Protein with state-of-the-art protein design methods}
\label{tab:performance_comparison}
\end{table}


Our approach distinguishes itself from these baselines by combining competitive structural quality with the unique ability to generate proteins based on textual descriptions. We achieve this by leveraging a large language model (LLM) to encode rich, contextual embeddings of protein descriptions, which then condition our diffusion process. This integration of natural language processing with structure generation represents a significant advance in the field.
Moreover, our use of 6D interresidue coordinates for protein backbone representation, similar to ProteinSGM, allows for a more comprehensive capture of local and global structural information compared to the $C\alpha$-trace representation used in ProteinGAN. This richer representation contributes to the generation of more accurate and diverse protein structures.
While our REU scores are currently higher (indicating less stable structures) than ProteinSGM, we hypothesize that this is primarily due to differences in the post-processing and refinement steps rather than fundamental limitations of our generative process. Future work could explore more extensive refinement procedures to improve energy scores while maintaining the text-guided generation capability.
Therefore, Text2Protein offers a novel and powerful approach to protein design by bridging the gap between natural language descriptions and three-dimensional protein structures. Our competitive performance in structural similarity metrics, combined with the unique text-guided generation capability, positions our method as a valuable tool for researchers seeking to design proteins with specific functional characteristics expressed in natural language.


\begin{figure}[ht]
\centering
    \includegraphics[width=\textwidth]{images/generated_protein_structures.pdf}
    \caption{Ground truth text descriptions of three examples (i.e., 4q6l at the top, 1kna in the middle, and 5or8 at the bottom) and the corresponding protein designs generated by Text2Protein.}

    \label{fig:protein_gen}
\end{figure}


\subsection*{Quality of Generated Protein Structures}


Figure~\ref{fig:protein_gen} demonstrates the capability of our Text2Protein pipeline to generate high-fidelity protein structures from detailed textual descriptions for three examples. Each panel juxtaposes a textual input with the corresponding predicted protein structure, enabling a comprehensive evaluation of the model's performance. \textbf{LpoB (4q6l):} In the top panel of Figure~\ref{fig:protein_gen}, the generated structure accurately reflects the ball-and-tether model of LpoB, capturing both the disordered N-terminal region and the compact globular C-terminal domain. This alignment with the input description underscores the model’s proficiency in interpreting and translating complex biological functions into precise structural configurations.
\textbf{HP1 Chromodomain (1kna):} The middle panel of Figure~\ref{fig:protein_gen} displays the model's successful recreation of the beta-sandwich architecture of the chromodomain, including the caging of the methylammonium group by aromatic side chains. This intricate detail is crucial for the protein's function in binding to the histone H3 tail, demonstrating the model's ability to generate functionally relevant protein structures.
\textbf{ALTA-VS Protocol (5or8):} As shown in the bottom panel of Figure~\ref{fig:protein_gen}, the protein structure generated aligns well with the description of the high-throughput docking protocol, accurately reflecting the evaluation of binding energy and inhibitor identification. This showcases the model's capability to process and represent complex virtual screening processes in protein design.


Comparing our results with existing models like ProteinDT~\cite{liu2023text} and Pinal~\cite{dai2024toward}, it is evident that Text2Protein offers significant advantages in generating structurally accurate and functionally relevant protein designs. For instance, ProteinDT focuses on sequence generation, often struggling with structural fidelity. Pinal employs a two-stage approach, first generating structures and then sequences, which improves accuracy but can be computationally intensive.
Our model’s ability to directly generate high-fidelity structures from text without the intermediate sequence prediction step offers both computational efficiency and high accuracy. This aligns with findings from recent literature ~\cite{liu2023text,dai2024toward}, which highlight the challenges of direct sequence-to-function mappings due to the vastness of the sequence space.
Our findings significantly can advance de novo protein design by linking natural language descriptions with accurate structural predictions through the Text2Protein model. This innovation accelerates the protein design process, making it more accessible and efficient, especially in drug discovery, enzyme engineering, and synthetic biology. The model's robust performance in generating precise protein structures highlights its potential to revolutionize protein engineering, benefiting both academic research and industrial applications.



\subsection*{Computational Requirements and Runtime}

The training of Text2Protein was conducted on an NVIDIA RTX 3090 GPU, taking approximately 10 days to complete 500,000 iterations. The significant training time is largely due to the complexity of the diffusion model and the high-dimensional nature of protein structures.
For inference, generating a single protein structure takes an average of 15 minutes on the same GPU. This includes the time for the diffusion process (about 5 minutes) and the subsequent Rosetta refinement (about 10 minutes). While this is slower than some existing methods, it's important to note that our approach offers the unique capability of text-guided generation.

\subsection*{Limitations and Future Work}

While our results are promising, several limitations and areas for future work remain. Our current model is limited to single-chain proteins with 40-256 residues. To address this, we propose developing a hierarchical approach where individual chains are first generated and then assembled based on inter-chain interaction predictions. This could potentially extend our method to multi-chain proteins and larger structures. The computational demands of LLM inference during training and sampling present another challenge. Future work could explore distillation techniques to create smaller, faster language models specifically tuned for protein descriptions.
Although our in silico metrics are promising, experimental validation of the generated proteins' functionalities would provide crucial insights into the practical applicability of our method. We propose collaborating with wet-lab researchers to synthesize and test a subset of our generated proteins. Additionally, exploring alternative protein representations, such as combining Graph Neural Networks with Variational Autoencoders, might lead to more efficient latent space diffusion and potentially capture more nuanced structural information. Future iterations of the model could also explore ways to provide more fine-grained control over generated structures, such as specifying certain structural motifs or active site configurations in the input text.
In summary, Text2Protein demonstrates the potential of combining large language models with diffusion-based generative approaches for protein design. Our results suggest that this approach can generate physically plausible protein structures with meaningful similarity to known proteins, guided by textual descriptions. While challenges remain, this opens up exciting possibilities for accelerating protein design processes and exploring the vast space of potential protein structures and functions.


\section*{Conclusion}

This paper introduced Text2Protein, an innovative pipeline for designing 3D protein structures based on raw textual descriptions of their functionality. Our approach represents a significant step forward in the field of computational protein design, demonstrating the potential of integrating large language models (LLMs) with conditional diffusion models to generate complex biomolecules guided by natural language input.
Text2Protein leverages the semantic understanding capabilities of a pretrained LLM (Vicuna-7B) to encode textual descriptions into rich embeddings. These embeddings then guide a denoising diffusion model, which iteratively refines an initially random 6D coordinate representation into a coherent protein structure. The final step employs PyRosetta to convert these 6D coordinates into full-atomic 3D protein structures, ensuring physical plausibility and stability.
Our experimental results demonstrate the efficacy of this approach. The generated structures exhibit reasonable structural similarity to ground truth proteins, as evidenced by an average TM-score of 0.35 and pairwise 6D coordinate MSE of 0.396. The Rosetta Energy Units (REU) of our generated structures, averaging -2.455, indicate thermodynamically favorable conformations. These metrics, while leaving room for improvement, suggest that Text2Protein can generate physically plausible protein structures that align with given textual descriptions.
Comparisons with state-of-the-art methods, such as ProteinSGM and ProteinGAN, which lack the ability to generate proteins based on textual descriptions, highlight the unique contribution of Text2Protein. This text-guided generation capability opens up new possibilities for targeted protein design that were previously not feasible.
However, several challenges and opportunities for future work remain. Our current model is limited to single-chain proteins with 40-256 residues. Future work should focus on developing hierarchical approaches to generate and assemble multiple chains, enabling the design of more complex protein structures. The computational demands of LLM inference present a challenge. Exploring model distillation techniques or more efficient language models could significantly reduce computational requirements and inference time.
While our results are promising, there is room for improvement in structural accuracy. Future iterations could explore more sophisticated protein representations, potentially incorporating graph neural networks or advanced geometric learning techniques. Collaborating with wet-lab researchers to synthesize and functionally characterize a subset of our generated proteins would provide crucial validation of our method's practical applicability.

Text2Protein demonstrates the potential of combining large language models with diffusion-based generative approaches for protein design. By bridging the gap between natural language descriptions and three-dimensional molecular structures, our method opens up exciting possibilities for accelerating protein design processes and exploring the vast space of potential protein structures and functions. As we address the current limitations and further refine our approach, we anticipate that text-guided protein generation will become an increasingly powerful tool in computational biology and drug discovery.

\section*{Author contributions statement}

S.Z. and P.X. conceived the design and concept of this work. S.Z. conducted the experiments, analyzed the results, and wrote the preliminary draft. R.H. conducted additional experiments, provided valuable feedback, and wrote this manuscript. All authors reviewed the manuscript.


% \section*{Reference}
% {
% \small

% \ \ \ \ \ \ \ [1] Jin Sub Lee1,  Jisun Kim2, Philip M. Kim. ProteinSGM: Score-based generative modeling
% 2 for de novo protein design, July 2022.

% [2] W.-L. Chiang, Z. Li, Z. Lin, Y. Sheng, Z. Wu, H. Zhang, L. Zheng, S. Zhuang, Y. Zhuang, J. E.
% Gonzalez, I. Stoica, and E. P. Xing. Vicuna: An open-source chatbot impressing gpt-4 with
% 90\% chatgpt quality, March 2023.

% [3] Robin Rombach, Andreas Blattmann, Dominik Lorenz, Patrick Esser, Björn Ommer. High-Resolution Image Synthesis with Latent Diffusion Models, Apr 2022.

% [4] Jianyi Yang, Ivan Anishchenko, Hahnbeom Park, Zhenling Peng, Sergey Ovchinnikov, David Baker. Improved protein structure prediction using predicted interresidue orientations, Nov 2019.

% [5] Brian L. Trippe, Jason Yim, Doug Tischer, David Baker, Tamara Broderick, Regina Barzilay, Tommi Jaakkola. Diffusion probabilistic modeling of protein backbones in 3D for the motif-scaffolding problem, Mar 2023.
% }

% [6] S. Chaudhury, S. Lyskov, and J. J. Gray, “PyRosetta: A script-based interface for implementing molecular
% 459 modeling algorithms using rosetta,” Bioinformatics, vol. 26, no. 5, pp. 689–691, Mar. 1, 2010.

% [7] Han Guo, Mingjia Huo, Ruiyi Zhang, Pengtao Xie, ProteinChat: Towards Achieving ChatGPT-Like
% Functionalities on Protein 3D Structures, May 2023.

% [8] J. Xu, Y. Zhang, How significant is a protein structure similarity with TM-score=0.5?, Feb 2010.

% [9] Yang Song, Jascha Sohl-Dickstein, Diederik P. Kingma, Abhishek Kumar, Stefano Ermon, Ben Poole, Score-Based Generative Modeling through Stochastic Differential Equations. Feb 2021.

% [10] Jonathan Ho, Ajay Jain, Pieter Abbeel, Denoising Diffusion Probabilistic Models, Dec 2020.

% [11] Shengchao Liu, Yutao Zhu, Jiarui Lu, Zhao Xu, Weili Nie, Anthony Gitter, Chaowei Xiao, Jian Tang, Hongyu Guo, Anima Anandkumar, A Text-guided Protein Design Framework, Feb 2023.


\section*{Data Availability}
All experiments are carried out using the publicly available PDB \cite{berman2000protein} and at this \href{https://files.rcsb.org}{https://files.rcsb.org} dataset and protein abstractions from \href{https://drive.google.com/drive/u/0/folders/1yb2iP9sMyIpYsMcFvFKkjYpKmjEcPSzL}{https://drive.google.com/drive/u/0/folders/1yb2iP9sMyIpYsMcFvFKkjYpKmjEcPSzL} with simple modification described in the Dataset section.

\section*{Code Availability}
Our Text2Protein code can be found here: \href{https://github.com/szhan227/text2protein}{https://github.com/szhan227/text2protein}.




\bibliography{sample}
\end{document}