%Version 3 October 2023
% See section 11 of the User Manual for version history
%
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%                                                                 %%
%% Please do not use \input{...} to include other tex files.       %%
%% Submit your LaTeX manuscript as one .tex document.              %%
%%                                                                 %%
%% All additional figures and files should be attached             %%
%% separately and not embedded in the \TeX\ document itself.       %%
%%                                                                 %%
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

%%\documentclass[referee,sn-basic]{sn-jnl}% referee option is meant for double line spacing

%%=======================================================%%
%% to print line numbers in the margin use lineno option %%
%%=======================================================%%

%%\documentclass[lineno,sn-basic]{sn-jnl}% Basic Springer Nature Reference Style/Chemistry Reference Style

%%======================================================%%
%% to compile with pdflatex/xelatex use pdflatex option %%
%%======================================================%%

%%\documentclass[pdflatex,sn-basic]{sn-jnl}% Basic Springer Nature Reference Style/Chemistry Reference Style


%%Note: the following reference styles support Namedate and Numbered referencing. By default the style follows the most common style. To switch between the options you can add or remove Numbered in the optional parenthesis. 
%%The option is available for: sn-basic.bst, sn-vancouver.bst, sn-chicago.bst%  
 
%%\documentclass[sn-nature]{sn-jnl}% Style for submissions to Nature Portfolio journals
%%\documentclass[sn-basic]{sn-jnl}% Basic Springer Nature Reference Style/Chemistry Reference Style
\documentclass[sn-mathphys-num]{sn-jnl}% Math and Physical Sciences Numbered Reference Style 
%%\documentclass[sn-mathphys-ay]{sn-jnl}% Math and Physical Sciences Author Year Reference Style
%%\documentclass[sn-aps]{sn-jnl}% American Physical Society (APS) Reference Style
%%\documentclass[sn-vancouver,Numbered]{sn-jnl}% Vancouver Reference Style
%%\documentclass[sn-apa]{sn-jnl}% APA Reference Style 
%%\documentclass[sn-chicago]{sn-jnl}% Chicago-based Humanities Reference Style

%%%% Standard Packages
%%<additional latex packages if required can be included here>
\usepackage{longtable}%
\usepackage{graphicx}%
\usepackage{multirow}%
\usepackage{amsmath,amssymb,amsfonts}%
\usepackage{amsthm}%
\usepackage{mathrsfs}%
\usepackage[title]{appendix}%
\usepackage{xcolor}%
\usepackage{textcomp}%
\usepackage{manyfoot}%
\usepackage{booktabs}%
\usepackage{algorithm}%
\usepackage{algorithmicx}%
\usepackage{algpseudocode}%
\usepackage{listings}%
\usepackage{array}%
\usepackage{hhline}%
\usepackage{amsmath}  % For math environments
\usepackage{breqn}    % For the dmath environment
\usepackage{graphicx}


%%%%

%%%%%=============================================================================%%%%
%%%%  Remarks: This template is provided to aid authors with the preparation
%%%%  of original research articles intended for submission to journals published 
%%%%  by Springer Nature. The guidance has been prepared in partnership with 
%%%%  production teams to conform to Springer Nature technical requirements. 
%%%%  Editorial and presentation requirements differ among journal portfolios and 
%%%%  research disciplines. You may find sections in this template are irrelevant 
%%%%  to your work and are empowered to omit any such section if allowed by the 
%%%%  journal you intend to submit to. The submission guidelines and policies 
%%%%  of the journal take precedence. A detailed User Manual is available in the 
%%%%  template package for technical guidance.
%%%%%=============================================================================%%%%

%% as per the requirement new theorem styles can be included as shown below
\theoremstyle{thmstyleone}%
\newtheorem{theorem}{Theorem}%  meant for continuous numbers
%%\newtheorem{theorem}{Theorem}[section]% meant for sectionwise numbers
%% optional argument [theorem] produces theorem numbering sequence instead of independent numbers for Proposition
\newtheorem{proposition}[theorem]{Proposition}% 
%%\newtheorem{proposition}{Proposition}% to get separate numbers for theorem and proposition etc.

\theoremstyle{thmstyletwo}%
\newtheorem{example}{Example}%
\newtheorem{remark}{Remark}%
\theoremstyle{thmstylethree}%
\newtheorem{definition}{Definition}%

\raggedbottom
%%\unnumbered% uncomment this for unnumbered level heads

\begin{document}

\title[Article Title]{Optimizing Software Crowdsourcing Requirements Design through Machine Learning}

%%=============================================================%%
%% GivenName	-> \fnm{Joergen W.}
%% Particle	-> \spfx{van der} -> surname prefix
%% FamilyName	-> \sur{Ploeg}
%% Suffix	-> \sfx{IV}
%% \author*[1,2]{\fnm{Joergen W.} \spfx{van der} \sur{Ploeg} 
%%  \sfx{IV}}\email{iauthor@gmail.com}
%%=============================================================%%

\author*[1]{\fnm{Turki} \sur{Alelyani}}\email{tnalelyani@nu.edu.sa}

% \author[2,3]{\fnm{Second} \sur{Author}}\email{iiauthor@gmail.com}
% \equalcont{These authors contributed equally to this work.}

% \author[1,2]{\fnm{Third} \sur{Author}}\email{iiiauthor@gmail.com}
% \equalcont{These authors contributed equally to this work.}

\affil*[1]{\orgdiv{Department of Information Systems, College of Computer Science and Information Systems}, \orgname{Najran University}, \orgaddress{\city{Najran}, \postcode{1988}, \country{Saudi Arabia}}}

 
% \affil[2]{\orgdiv{Department}, \orgname{Organization}, \orgaddress{\street{Street}, \city{City}, \postcode{10587}, \state{State}, \country{Country}}}

% \affil[3]{\orgdiv{Department}, \orgname{Organization}, \orgaddress{\street{Street}, \city{City}, \postcode{610101}, \state{State}, \country{Country}}}

%%==================================%%
%% Sample for unstructured abstract %%
%%==================================%%

\abstract{Crowdsourcing, a distributed computing model on the internet, has opened new avenues for software development. Traditional in-house software development has transformed into more decentralized approaches, where software tasks are distributed through open calls on the internet. Crowdsourcing leverages human intelligence, augmented with machine learning, to produce software solutions. However, several challenges need to be tackled. For instance, designing software requirements is a major challenge during the development process. This research investigates the design aspects of the notable crowdsourcing platform TopCoder and applies machine learning approaches to identify design factors that affect developers’ decision-making during the selection process. My approach shows that design factors can be extracted to estimate software task prices and provide actionable insights into the factors influencing task success. The results will guide future research and crowdsourcing managers in applying these approaches to extract design factors and understand their impact on the development process.}

%%================================%%
%% Sample for structured abstract %%
%%================================%%

% \abstract{\textbf{Purpose:} The abstract serves both as a general introduction to the topic and as a brief, non-technical summary of the main results and their implications. The abstract must not include subheadings (unless expressly permitted in the journal's Instructions to Authors), equations or citations. As a guide the abstract should not exceed 200 words. Most journals do not set a hard limit however authors are advised to check the author instructions for the journal they are submitting to.
% 
% \textbf{Methods:} The abstract serves both as a general introduction to the topic and as a brief, non-technical summary of the main results and their implications. The abstract must not include subheadings (unless expressly permitted in the journal's Instructions to Authors), equations or citations. As a guide the abstract should not exceed 200 words. Most journals do not set a hard limit however authors are advised to check the author instructions for the journal they are submitting to.
% 
% \textbf{Results:} The abstract serves both as a general introduction to the topic and as a brief, non-technical summary of the main results and their implications. The abstract must not include subheadings (unless expressly permitted in the journal's Instructions to Authors), equations or citations. As a guide the abstract should not exceed 200 words. Most journals do not set a hard limit however authors are advised to check the author instructions for the journal they are submitting to.
% 
% \textbf{Conclusion:} The abstract serves both as a general introduction to the topic and as a brief, non-technical summary of the main results and their implications. The abstract must not include subheadings (unless expressly permitted in the journal's Instructions to Authors), equations or citations. As a guide the abstract should not exceed 200 words. Most journals do not set a hard limit however authors are advised to check the author instructions for the journal they are submitting to.}

\keywords{Software, Crowdsourcing, Task, Design, Requirements, Machine Learning} 

%%\pacs[JEL Classification]{D8, H51}

%%\pacs[MSC Classification]{35A01, 65L10, 65L12, 65L20, 65L70}

\maketitle

\section{Introduction}\label{sec1}

Human intelligence has been leveraged by prominent crowdsourcing platforms like TopCoder, Upwork, Fiverr, and Amazon Mechanical Turk amongst others in the quest to provide solutions affecting the world today. Crowdsourcing has effectively; albeit with challenges, managed to tap talents from the crowd thereby leading to rapid expansion in software design and development. These crowdsourcing platforms have managed to connect skilled individuals to prospective employers virtually across the world [1]. Talented individuals from different parts of the world are pulled together and allowed to showcase their expertise in providing solutions to complex problems. Software development leads as the main focus for such platforms. By eliminating primary costs like commuter allowance and space, the platforms are able to deliver high-quality deliverables with reduced costs. Employees are only allowed to work at their convenience time which leads to high productivity. The major contributors to the overall functional efficacy of the platforms include flexibility, specialty-oriented tasks, and the availability of surplus skilled labor [31]. However, the operational model of the platforms is not without challenges and errors. Quality control can be compromised by powerful contributors like limited context and understanding of objective goals by the crowd.

Ensuring the production of high-quality deliverables consistently is the major challenge affecting crowdsourcing platforms [2]. Managing a pool of experts towards one goal turns out difficult owing to the way these platforms work [3]. Designing engaging specifications or ‘requirements’ for specific objectives through selected individuals becomes challenging. Designing such elements cannot be achieved without the threat of onerous overhead. The design nature of one crowdsourcing platform differs from another, engaging within the sites is also limited in several aspects. The restrictive form of design adopted by these platforms limits interactions and also makes some tasks impossible to do. For example, some experts might find it difficult to fully exhaust their skills and therefore abandon the system or offer limited services thereby compromising performance [3]. 

Quality assurance control happens to be the most important aspect of software development [4]. The ultimate goal of software design and development is efficiency and its ability to serve the intended purpose to perfection. The outcome of a software development process is based on the amount of work put in design and not specifically on what contributors give out [5]. Software design, development, and implementation is a process that involves incentives, description layout, complexity, workflow, task clarity among others. These process descriptions can be handled singly or approached in unison. Task description or instructions can be challenging for the worker to understand thereby leading to poor concept acquisition [6]. If the instructions were not well decoded, then the overall workability of the software to be designed is jeopardized. When the platform interface is poorly designed, requesters are likely to face difficulties getting the necessary support from either the owner of the project or the lead consultant of the project. 

The impact different software design approaches have on the quality of software development deliverables is not specified. Despite research by leading scholars, the relationship between participant behavior and crowdsourcing platform efficiency remains inconclusive. Dubey et al. [7] suggest that platform design impacts software performance; clear task descriptions, effectively presented by employers and understood by workers, are crucial for performance and quality assurance. While initial software design can involve decomposing task descriptions, outlining workflow, and defining rewards, Dubey et al. [7] call for further research to determine the precise influence of task design on performance and quality.

Crowdsourcing software is a complex system with economic, social, and technical components. Research on how diverse design techniques to influence crowd performance is necessary to improve its design, enabling the development of sophisticated software, and limit potential risk [8]. Crowdsourcing is described as a sociotechnical system defined by a set of interrelated elements in this study, which builds on our earlier work.

The proposed study builds upon my previous work [9] to examine how various software crowdsourcing architecture characteristics influence crowdsourcing efficiency. This work creates a framework of characteristics that impact the success of software crowdsourcing tasks. The study employs machine learning methods to analyze 4,859 software tasks and derives design parameters that software crowdsourcing managers can utilize to create software projects.

To achieve the study's objectives, the Context-Centric Price (CCP) technique was proposed. CCP is a pricing method based on clear mathematical and linguistic aspects of software crowdsourcing tasks. Next, the Socio-Technical Design (STD) framework was introduced, comprising a comprehensive collection of features to investigate their effects on software crowdsourcing performance. Finally, a large-scale, real-world software crowdsourcing dataset was used to validate the two proposed models.

Relevant studies have primarily focused on two aspects of technological systems: requirements and complexity, while overlooking the rich features in job requirements. Using machine-learning approaches to investigate social and technological circumstances in a real crowdsourcing scenario with a large dataset will help us gain a better understanding of how different design variables affect crowdsourcing platforms. This understanding can assist crowdsourcing managers in making informed design choices. Additionally, software crowdsourcing platforms must detect design aspects and mitigate the risks these features pose.


% The Introduction section, of referenced text \cite{bib1} expands on the background of the work (some overlap with the Abstract is acceptable). The introduction should not include subheadings.

% Springer Nature does not impose a strict layout as standard however authors are advised to check the individual requirements for the journal they are planning to submit to as there may be journal-level preferences. When preparing your text please also be aware that some stylistic choices are not supported in full text XML (publication version), including coloured font. These will not be replicated in the typeset article if it is accepted. 
\section{Background}\label{sec2}
\subsection{Crowdsourcing Incentive Design}\label{sec2}

Crowdsourcing is rapidly gaining ground in recent years with advances being made in technology. The modern world is aggressively embracing artificial intelligence with machine learning as the engine behind AI. Research has been conducted from different perspectives with regard to crowdsourcing and trying to unravel the inner working mechanisms. Scholars are eager to answer the question as to how people can effectively collaborate and work virtually. Worker motivation has been under study for a significant period for it has been found to have a higher stake in employee performance [10]. Worker motivation can be studied from different perspectives. Motivation can be classified as either extrinsic motivation or intrinsic. Intrinsic motivation occurs when a pool of experts is driven by curiosity to learn and know different aspects of things. In this study, participants were not motivated by intrinsic incentive when no extrinsic motivation was provided (no payment). Individuals are organically motivated to contribute and make some works in a crowdsourcing situation. Another sort of intrinsic drive is achievement motivation, which arises when participation in activity leads to a sense of accomplishment. According to Stol et al [11], individuals who have a sense of personal gratification when obtaining certain coding skills, for example, are intrinsically motivated and will have a pleasurable experience.

On the other hand, intrinsic motivation to experience stimulation arises when a person participates in the activity because it is exciting or enjoyable. Workers are paid for accomplishing needed activities in paid crowd work. TopCoder (the largest software crowdsourcing platform), Amazon Mechanical Turk, Upwork, Fiverr, Utest, and other crowdsourcing platforms provide workers with monetary incentives in exchange for the projects they handle. 

According to Finnerty et al, [12], there are different incentives provided with the aim of improving the quality of software development outcomes. For example, workers can be given a chance to evaluate their peers’ work, this makes people the workers to feel part of the system. Alelyani et al, [6], on the other hand, argue that workers value quality evaluation feedback and that it can help them achieve better achievements. Despite their success in attracting talented individuals to take part and produce various products, studies show that crowdsourcing platforms face challenges in designing effective incentive systems for their various tasks. These challenges can be due to requesters' constraints and the online market setting of a particular project. As a result, further research is needed to better understand crowd behavior and build appropriate mechanisms.

\subsection{Crowdsourcing Participants’ Behavior}

Participant behavior has gained traction among researchers in recent years. The scholars aim to find out the relationship between participant’s behavior and the quality of work. Prior work studied the dependability of crowd workers in selecting and handling various projects in the crowdsourcing platforms [13]. Workers with high-reliability ratings are more precise in the types of rewards they choose, whereas workers with low-reliability ratings are more general. According to the findings, the most dependable workers in the high-reliability group are more driven to swiftly register for a task and complete it in the shortest possible period.

The contest outcome on crowdsourcing was previously determined by a game theory model. The game theory model majorly focused on the outcome of a single contest by determining the reward value [12]. In another related scholarly work [14], conducted research aimed at determining the effects of the reputation system being applied by TopCoder. The TopCoder reputation system touched on the behavior of the participants. Individual characteristics, as well as project payment and the number of project requirements, were found to be significant predictors of project quality in the study. In addition, the researchers discovered evidence of contestants' strategic behavior. According to Moussawi and Koufaris [15], the competitive round of the contest puts high-rated participants up against the tougher competition. To make things easier for themselves, they go first in the contest's registration phase, registering for specific projects ahead of time.
     
{According to the findings, TopCoder's reputation and cheap talk methods improve the allocative efficiency of concurrent all-pay contests and should be evaluated for implementation in other crowdsourcing platforms. A recent study looked at the impact of developer behavior and community structure on software crowdsourcing methods using social network analysis. The study discovered that TopCoder users engage in temporal burst patterns of online interaction, which leads to a resemblance in the type and level of participation at TopCoder [13].}

\subsection{Crowdsourcing Task Design}
According to Goodman and Paolacci, [16], recent studies have conclusively agreed that the data collection interface design directly relates to the quality of the work to be delivered by the worker. The graphical user interfaces of any crowdsourcing platform are an important factor when it comes to quality assurance. Prior research used the task choice decision (TCD) model and found that there is a close correlation between task design and user productivity [17]. As one part of the study focused on determining the relationship between work deliverables and task design, the other focused was studying the effects such designs have on quality. The study concluded that there is a need to apply different designs specific to different tasks. However, the revelations from the above studies do not close the possibility of more studies being conducted on the stated topic. 
A study on crowdsourcing relatedness and growth took a different approach to problem design. Moussawi and Koufaris [15], investigated how the kind and scale of the tasks influence the level of creativity, complexity, and skill necessary to perform them. A pay vs. time trade-off was determined to be the key determinant for selecting tasks to do in this study. More research on the types of incentive mechanisms that should be used when developing activities is needed, according to the findings. Another study looked at how task design, such as instructions and user interface, can influence workers' perceptions of the job [15].
The difficulties of cognitive load theory and its importance for building interface systems were discussed by Vaz, Steinmacher, and Marczak [18]. When users interact with the interface, one part of cognitive stress, according to the study, can address the concept of memory burden. Considering users' knowledge and minimizing redundant content is one strategy to reduce cognitive load. Sarma et al [19] defined task quality in crowdsourcing as a set of components that can be used to assess the quality of an assignment. Time spent on a task, job length, task resources, and price options is some of these elements. The findings of applying a regression model to the evaluated tasks reveal that the previously described criteria play a role in determining task quality.

\subsection{Decision Making and Task Uncertainty}
Software crowdsourcing tasks are being misinterpreted by workers due to the emerging uncertainties caused during posting on the platforms. Crowdsourcing participants have exhibited risk behavior through unintentional deviations from quality assurance. According to Vaz, Steinmacher, and Marczak [18], researchers are concerned about the participants’ risk behaviors emanating from poorly designed ‘requirements’ posting interfaces. The completion of the software crowdsourcing tasks is deemed uncertain if the task description is misinterpreted. The quality of the deliverables is significantly threatened by the participants’ risks. Crowdsourcing systems need workers to make hazardous judgments by selecting from a range of options, each of which has a chance of yielding related payoffs. The TopCoder model will be used to better describe the software crowdsourcing decision-making process. 

In general, a developer can express interest in any task by signing up for a challenge, which indicates that the developer is competing with others on that assignment. Performance and skill ratings for the developer are also displayed. Multiple micro activities are typically chained together into workflows for quality assurance and to complete difficult jobs. In the "Task Preparation" phase, such workflows may divide bigger jobs into smaller subtasks, and then recompose subtask solutions into an overall work product. To avoid any ambiguity or confusion in the decomposed task, this part of the workflow should follow a methodical procedure. Failure to do so would jeopardize the development process, thus impacting the crowdsourcing experience.
Existing platforms do not always follow a regular procedure for breaking down huge projects into smaller tasks, setting up requirements, pricing each task, or managing inter-dependencies [20]. Workers may have ambiguity because of these challenges when attempting to register for a task. Following the preparatory phase, tasks are made public and open to the public for registration and contribution. The perception of task design by crowd participants is a difficult subject to answer because there are always some differences in task design and preparation.
Participants have a set amount of time to complete and submit projects after registering for them. A task is considered unsuccessful if it receives no submissions. The term "task hunger" refers to the lack of submissions for a task [20]. 

Following submission, the peer-review process begins. All contributions to a challenge are vetted by review boards, which are made up of experienced. Individuals picked by the site. If the task's score after the review phase was greater than {75\%,} it was deemed successful; otherwise, it was canceled. Developers are notified confidentially of their ratings and have the option to appeal if they are unhappy with their present results. It is important to note that this "appeal phase" can only be used once.  Finally, all of the developers' scores are made public, and the reviewers provide ratings.
The above workflow demonstrates how crowdsourcing technologies allowed task requesters and workers to transcend geographical barriers, reducing expert shortages and scalability. Crowdsourcing can help to assign the appropriate assignment to the right people at the right price by utilizing the internet. Crowdsourcing platforms, on the other hand, are still grappling with how to better build a system that can reduce task ambiguity, which could compromise the quality of their decision-making [4]. 
Previous research looked at how crowdsourcing platforms might improve their present methods by collecting all components of crowdsourcing market decision-making. This entails developing several models that synthesize all aspects that can be combined to improve decision-making. With the development of various software crowdsourcing platforms, decision making has become more popular. Its use has spread across a variety of fields, including operations research, economics, and computer science.
Each platform's management can define the interaction model. Requestors can freely interact with available workers on some platforms. Other platforms, on the other hand, prohibit this type of connection between workers and requestors, making the platform the only route via which both sides must communicate to complete the task. As a result, various modeling challenges should be considered in this category. For example, the process of assigning workers to jobs based on factors such as worker reliability, feedback, experience, and talents [13].
One of the key roadblocks to software crowdsourcing platforms is the modeling of crowdsourcing incentives [21]. The task's success may be jeopardized if the task's price is overestimated or underestimated. As a result, various studies have attempted to solve this issue in the context of software crowdsourcing. Task requestors and workers are also concerned about performance optimization. Given the current market rivalry, platforms must maximize the utility of requestors to ensure their long-term viability. Workers' management, on the other hand, is a difficult task because there are constantly issues about their dependability, motivation, and involvement.

\subsection{Socio Technical Systems}
According to the Socio-Technical System (STS) design model, the technical and social aspects of software design are key factors influencing positive productivity [3]. The interdependencies between the various components of a socio-technical system are primarily structured. The qualities and interactions of these two components should be considered in any effective system. As described by Sherief et al. [22], any management information system design should be framed within the STS design methodology. The STS method applies to both current and new work systems. In a crowdsourcing context, a platform comprises two interconnected systems: social and technological.

Tasks, technology, and design are all part of the technical system, which transforms input into output. The social system comprises people, skills, beliefs, connections, and reward systems. Therefore, any system design or redesign must consider both systems holistically. According to Sherief et al. [19], optimal fit is achieved through a design process aimed at the joint optimization of these subsystems, which is the cornerstone of the socio-technical system approach. In other words, an organizational system can only maximize performance if the interdependence of its subsystems is openly recognized.
Prior research identifies software crowdsourcing as a socio-technical system, where individuals, technologies, and labor activities are interconnected within the platform [6]. The interaction among requesters, developers, and platforms is essential for system success, increasing the interdependence between system design and user engagement.

Prior research [23] has studied the impact of five risk characteristics on crowdsourcing performance through self-reporting. Workers, requestors, relationships, requirements, and task complexity were all identified as potential sources of risk. Research investigating the impact of these components on the process is crucial yet surprisingly understudied, as system design might influence participant behavior. Existing studies indicate that social subsystem risks reduce crowdsourcing performance, suggesting that various forms of risk collectively affect performance. Many studies have discovered a link between crowdsourcing performance and factors such as task design, price, worker experience, skills, motivation, and other aspects of the crowdsourcing system. Crowdsourcing performance is measured by the degree of effective work completion [13]. Given the substantial technological and social uncertainties in crowdsourcing, this paper hypothesizes that crowdsourcing risk relates to several design features in task description, incentive, communication, and engagement tactics, based on past research.
Prior research studied the impact of five risk characteristics on crowdsourcing performance in a self-reporting study. Workers, requestors, relationships, requirements, and task complexity were all mentioned as possible sources of danger. Research that could investigate the impact of these components on the process must be investigated, a topic that is surprisingly understudied, because system design might influence participant behavior. Existing research indicated that social subsystem risk reduces crowdsourcing performance, indicating that several forms of risks combine to effect performance.

Numerous studies have identified correlations between crowdsourcing performance and several elements of the crowdsourcing system, such as task design, price, worker experience, skills, and motivation [18]. Crowdsourcing performance is defined as the degree of effective work completion. Based on prior research and acknowledging the significant technological and social uncertainties inherent in crowdsourcing, this paper hypothesizes a relationship between crowdsourcing risk and design features encompassing task description, incentives, communication strategies, and engagement tactics.

Table 1 compares prior literature with the outcomes of the current study, highlighting attributes influencing task success in software crowdsourcing platforms. The study confirms findings on task presentation quality, clarity, readability, and incentive design, while extending insights with machine learning models like Price2. Key factors such as external resources, technology relevance, and reputation systems align with prior research, validating their importance. The results provide actionable recommendations to enhance task design and platform performance.



\subsection{Literature-Derived Factors and Their Empirical Validation}

To ground the development of the proposed task design framework, I reviewed key studies in software crowdsourcing. Several recurring factors emerged, including task clarity, incentive structures, cognitive load, and social dynamics. These were mapped to broader design categories presented in this study. Furthermore, the empirical findings validated the practical relevance of these factors, as most showed strong associations with task success in the machine learning models.

Table 1 summarizes each literature-derived factor, its source, the key insight from prior research, how it aligns with this study’s design categories, and whether it was empirically supported by the results.




% \subsection{Summary of Literature-Derived Factors}

% To ground the development of the proposed task design framework, I reviewed key studies in software crowdsourcing. Several recurring factors emerged across the literature, such as task clarity, incentive structures, social elements, and cognitive load. These factors were synthesized and organized into broader design categories that inform the framework presented in Table~\ref{tab:table3}. 

% Table~\ref{tab:lit_factors} summarizes these literature-derived factors alongside their original sources, a brief description, and how they map to the design categories proposed in this study.

% \begin{table}[ht]
% \centering
% \caption{Literature-Derived Factors and Their Mapping to Task Design Categories}
% \begin{tabular}{|p{3.8cm}|p{4.2cm}|p{5.2cm}|p{4cm}|}
% \hline
% \textbf{Factor from Literature} & \textbf{Source(s)} & \textbf{Summary} & \textbf{Mapped to Design Category} \\
% \hline
% Task Presentation Quality & Goodman \& Paolacci (2014) & Clear, readable instructions boost engagement and task success & Textual, Readability \\
% \hline
% Task Clarity & Vaz et al.\ (2019); Sarma et al.\ (2016) & Unambiguous tasks improve quality & Textual, Readability, Software Engineering \\
% \hline
% Incentive Design & Finnerty et al.\ (2013); Alelyani et al.\ (2020) & Monetary rewards and peer feedback motivate participation & Social (Reward) \\
% \hline
% Reputation Systems & Archak (2010); Moussawi \& Koufaris (2015) & Trusted requestors attract more developers & Organization \\
% \hline
% Task Complexity & Moussawi \& Koufaris (2015); Dubey et al.\ (2016) & Complex tasks should be decomposed & Software Engineering, Design \\
% \hline
% Worker Motivation & Rogstadius et al.\ (2011); Stol et al.\ (2014) & Both intrinsic and extrinsic motivators improve performance & Social \\
% \hline
% Technology Relevance & Alelyani et al.\ (2016); Mao et al.\ (2015) & Tasks using popular techs get higher engagement & Application \\
% \hline
% External Resources & Goodman \& Paolacci (2014); Vaz et al.\ (2019) & GitHub, forums, and links clarify task requirements & External \\
% \hline
% Cognitive Load (Readability) & Vaz et al.\ (2019); Sarma et al.\ (2016) & Simplified language reduces cognitive burden & Readability \\
% \hline
% \end{tabular}
% \label{tab:lit_factors}
% \end{table}



% \usepackage{array, longtable, booktabs, xcolor} % Add this to your preamble if not already included

\begin{longtable}{@{}p{0.2\linewidth}p{0.2\linewidth}p{0.3\linewidth}p{0.3\linewidth}@{}}
\caption{Attributes, prior literature Findings, and current Study outcomes} \\
\toprule
\textbf{Attribute} & \textbf{Source} & \textbf{Prior Findings} & \textbf{Current Outcomes} \\ 
\midrule
\endfirsthead
\caption*{(Continued)} \\
\toprule
\textbf{Attribute} & \textbf{Source} & \textbf{Prior Findings} & \textbf{Current Outcomes} \\ 
\midrule
\endhead
\textbf{Task Presentation Quality} & Goodman \& Paolacci (2014) & Readable and structured tasks improve engagement and success. & \textbf{Flesch Reading Ease} and \textbf{SMOG Index} predict success. \\
\hline
\textbf{Task Clarity} & Vaz et al. (2019); Sarma et al. (2016) & Clear instructions reduce ambiguity and improve quality. & Clarity and low ambiguity increased success rates. \\
\hline
\textbf{Incentive Design} & Finnerty et al. (2013); Alelyani et al. (2020) & Monetary rewards and peer feedback enhance motivation. & Competitive pricing (\textbf{Price2}) influenced success. \\
\hline
\textbf{Readability Metrics} & Kincaid et al. (1975); McLaughlin (1969) & Readability indices like ARI predict success. & \textbf{Fog Index} and \textbf{Flesch Score} strongly correlated with success. \\
\hline
\textbf{Social Engagement} & Stol \& Fitzgerald (2014); Archak (2010) & Peer interactions and reputation systems boost motivation. & Social features like \textbf{engagement} improved outcomes. \\
\hline
\textbf{Task Complexity} & Moussawi \& Koufaris (2015); Dubey et al. (2016) & Complex tasks need decomposition for better results. & Clear sub-tasks increased completion rates. \\
\hline
\textbf{Worker Motivation} & Rogstadius et al. (2011); Stol et al. (2014) & Intrinsic and extrinsic motivators improve performance. & Pricing and clarity boosted participation. \\
\hline
\textbf{Technology Relevance} & Alelyani et al. (2016); Mao et al. (2015) & Popular technologies attract higher engagement. & \textbf{Web} and \textbf{Mobile} tasks were more successful. \\
\hline
\textbf{External Resources} & Goodman \& Paolacci (2014); Vaz et al. (2019) & Links like GitHub improve clarity and success. & \textbf{GitHub} and forums improved outcomes. \\
\hline
\textbf{Reputation Systems} & Archak (2010); Moussawi \& Koufaris (2015) & Requester reputation impacts task success. & Reputed requesters (\textbf{NASA}, \textbf{IBM}) saw higher participation. \\
\hline
\textbf{Cognitive Load} & Vaz et al. (2019); Sarma et al. (2016) & Simplified instructions reduce cognitive load. & Lower load, measured by readability and clarity, improved success. \\
\hline
\bottomrule
\end{longtable}

% \subsection{Summary of Literature-Derived Factors}

% To establish a foundation for the design factors proposed in this study, I conducted a comprehensive review of the literature on software crowdsourcing. Several studies have examined the elements that influence task success, developer engagement, and outcome quality in crowdsourced environments. By synthesizing these works, I identified recurring factors that span across themes such as task clarity, incentive design, reputation, technical relevance, and readability.

% Table~\ref{tab:lit_factors} presents a summary of the most prominent factors identified in prior research. Each factor is listed alongside its key source(s), a brief summary of its role, and the corresponding category it maps to in the framework I propose in Table 1. This comparative view provides a bridge between established knowledge and the set of design variables derived in this study through empirical analysis of TopCoder tasks.

% \subsection{Summary of Literature-Derived Factors}

% To establish a foundation for the design factors proposed in this study, I conducted a comprehensive review of the literature on software crowdsourcing. Several studies have examined the elements that influence task success, developer engagement, and outcome quality in crowdsourced environments. By synthesizing these works, I identified recurring factors that span across themes such as task clarity, incentive design, reputation, technical relevance, and readability.

% Table~\ref{tab:lit_factors} presents a summary of the most prominent factors identified in prior research. Each factor is listed alongside its key source(s), a brief summary of its role, and the corresponding category it maps to in the framework I propose in Table~\ref{tab:table3}. This comparative view provides a bridge between established knowledge and the set of design variables derived in this study through empirical analysis of TopCoder tasks.

% \begin{table}[ht]
% \centering
% \caption{Factors Derived from Literature and Their Mapping to Design Factors in Table 3}
% \begin{tabular}{|p{3.8cm}|p{4.2cm}|p{5.2cm}|p{4cm}|}
% \hline
% \textbf{Factor from Literature} & \textbf{Source(s)} & \textbf{Summary} & \textbf{Mapped to Table 3 Category} \\
% \hline
% Task Presentation Quality & Goodman \& Paolacci (2014) & Clear, readable instructions boost engagement and task success & Textual, Readability \\
% \hline
% Task Clarity & Vaz et al.\ (2019); Sarma et al.\ (2016) & Unambiguous tasks improve quality & Textual, Readability, Software Engineering \\
% \hline
% Incentive Design & Finnerty et al.\ (2013); Alelyani et al.\ (2020) & Monetary rewards and peer feedback motivate participation & Social (Reward) \\
% \hline
% Reputation Systems & Archak (2010); Moussawi \& Koufaris (2015) & Trusted requestors attract more developers & Organization \\
% \hline
% Task Complexity & Moussawi \& Koufaris (2015); Dubey et al.\ (2016) & Complex tasks should be decomposed & Software Engineering, Design \\
% \hline
% Worker Motivation & Rogstadius et al.\ (2011); Stol et al.\ (2014) & Both intrinsic and extrinsic motivators improve performance & Social \\
% \hline
% Technology Relevance & Alelyani et al.\ (2016); Mao et al.\ (2015) & Tasks using popular techs get higher engagement & Application \\
% \hline
% External Resources & Goodman \& Paolacci (2014); Vaz et al.\ (2019) & GitHub, forums, and links clarify task requirements & External \\
% \hline
% Cognitive Load (Readability) & Vaz et al.\ (2019); Sarma et al.\ (2016) & Simplified language reduces cognitive burden & Readability \\
% \hline
% \end{tabular}
% \label{tab:lit_factors}
% \end{table}



\section{Study Objective}
Designing a software crowdsourcing strategy involves selecting design elements that meet the goals of both work requesters and developers. As crowdsourcing platforms continue to grow at scale, techniques to automate and streamline the process to achieve the desired outcomes must be developed. Prior research often overlooks crucial elements in software crowdsourcing, particularly task requirements design, due to the lack of a systematic mechanism to measure and enhance design quality early on. Consequently, it is essential to provide appropriate tasks to assist developers in making task selection decisions.
\\
This study builds on my prior work, Context-Centric Pricing, and Socio-Technical Design techniques [9], to propose design variables that facilitate software crowdsourcing task design, aiming to achieve the following objectives:
\\

\textbf{1. Assess Task Prices:} The study offers a crowdsourcing approach to evaluate task prices and improve upon the previous work, demonstrating that the proposed approach has a higher level of validation and generalizability.

\textbf{2. Assess Performance:} Social and technical aspects can be used to assess the performance of software crowdsourcing task design, allowing for a more methodical approach to crowdsourcing platforms.




In this study, I address the following research questions:


\textbf{1. RQ1:} How can we systematically extract features of software crowdsourcing tasks to estimate their price?

\textbf{3. RQ2:} Which task features influence task success on software crowdsourcing platforms?

\section{Overview of the Proposed Approach}

Task requirements are the primary sources of information used by crowdsourcing platforms to design prospective tasks in the early stages of the software crowdsourcing process. This study develops an approach utilizing Natural Language Processing (NLP) methods to leverage the available data, including both explicit and hidden properties. Previous research typically focuses on textual aspects or explicit categories to understand software crowdsourcing design. In contrast, the proposed models demonstrate how to create a crowdsourcing task that is both effective and efficient.

\section{Study Design}

A holistic framework is offered that incorporates several types of data: textual, numerical, and categorical, to investigate the impacts of software crowdsourcing design aspects on crowd performance and pricing estimates. To extract model features, NLP approaches and machine learning algorithms are employed. Figure 1 illustrates the proposed approach, which comprises three main parts:

1. Data Preprocessing

2. Natural Language Processing (NLP)

3. Model Results

\begin{enumerate}

\item Data Preprocessing: Stop-words and missing metadata are removed from each document using NLTK, a platform for constructing Python programs that work with natural language. Stop-words that are unrelated to our context, such as prepositions, conjunctions, and articles, are frequently present in documents with textual descriptions.

\item Natural Language Processing (NLP): NLP techniques are employed to extract the target characteristics. One-Hot Encoding is used to convert categorical input into numerical features.

\item Model Results: After successfully preprocessing the data and extracting the target characteristics, the model is developed as shown in Figure 1.

\end{enumerate}


\begin{figure}[h]
\centering
\includegraphics[width=0.9\textwidth]{fig28}
\caption{Study proposed approach.}\label{fig1}
\end{figure}




\subsection{Dataset}


Data was collected from TopCoder, a prominent software crowdsourcing platform, between January 2014 and January 2015. The sample consisted of nearly 7,800 developers and 4,859 software tasks. Each task included various metadata, such as descriptions, prizes, start and end dates, technologies used, and details about the developers who enrolled and completed the tasks. The outcome of each task was classified based on whether it was completed after being attempted. Table 2 presents the descriptive statistics of the collected data.
To address the imbalance issue and increase the representation of the minority class, the Synthetic Minority Over-Sampling Technique (SMOTE) was applied. Approximately 80\% of the data consisted of "finished" tasks, which could reduce the effectiveness of training models. As a result, the data was balanced between the two classes: finished and canceled tasks.


% \documentclass{article}
% \usepackage{array} % If not already included, for p{} column type

% \begin{document}

\begin{table}
\centering
\caption{Descriptive statistics of the collected data}

\begin{tabular}{p{2.0cm} p{2.0cm} p{2.0cm} p{2.0cm} p{2.0cm}}
\hline
~          & Max    & Mean & STD & Min  \\ 
\hline
Tasks      & 4859   & -    & -        & -         \\ 
\hline
Reward     & \$6700 & 865  & 894      & 10        \\ 
\hline
Days       & 71     & 15   & 12       & 1         \\ 
\hline
Words      & 2748   & 361  & 285      & 10        \\ 
\hline
Links      & 58     & 2.32 & 3.50     & 0         \\ 
\hline
Technology & 12     & 3    & 1.77     & 1         \\
\hline
\end{tabular}
\end{table}

% \end{document}




%  \begin{table}
% \centering
% \caption{Descriptive statistics of the collected data}

% \begin{tabular}{p{2.0cm} p{2.0cm} p{2.0cm} p{2.0cm} p{2.0cm}}
% % Your table content here
% \hline
% ~          & Max    & Mean & ~~~~ STD & ~~~~ Min  \\ 
% \hline
% Tasks      & 4859   & -    & -        & -         \\ 
% \hline
% Reward     & \$6700 & ~865 & 894      & ~10       \\ 
% \hline
% Days       & 71     & 15   & 12       & 1         \\ 
% \hline
% Words      & ~2748  & ~361 & 285      & 10        \\ 
% \hline
% Links      & 58     & 2.32 & 3.50     & 0         \\ 
% \hline
% Technology & 12     & 3    & 1.77     & 1         \\
% \hline
% \end{table}
% \end{tabular}

\subsection{Identifying task design factors} 

This research focuses on identifying and implementing requirements-based aspects in task design that can enhance the performance of software crowdsourcing platforms. The proposed criteria in the framework are intended to assist specifically in task pricing, task type, determining its location in the software development life cycle, and identifying design factors that may contribute to task success.

The proposed model (Figure 1) can manage the increasing amount and complexity of crowdsourcing activities. Textual features, such as the number of sentences, nouns, connections, readability, and understandability ratings, were extracted using NLP techniques. The Linguistic Inquiry and Word Count (LIWC) tool was also used to retrieve user stories (by counting the number of functions each task includes) and to assess the social feature rate, as well as to determine the level of certainty for each task through semantic analysis.

A user story is a casual natural language description of a software feature [20]. Finally, task domain aspects were retrieved and classified into the following categories: application, design, software engineering, organization, external resources, textual, and readability. The criteria were tagged and segmented using the Social Named Entity Recognition (NER) algorithm to reveal valuable insights. The collected design factors will be explained in the following section.
 


% \begin{figure}[h]
% \centering
% \includegraphics[width=1.2\textwidth]{fig29.png}
% \caption{Extracted design factors and correlation among predictors factors.}\label{fig2}
% \end{figure}


1. Task domain: This diagram depicts the various traits associated with each task, as well as the unique features observed in software engineering. Different components of text, such as syntax and semantics, can be translated and interpreted with NLP models. NER was used in the task domain to crowdsource software engineering knowledge, develop word- and phrase-level models, and categorize sections of raw text into relevant features. The uniformity of software engineering information was ensured through an iterative process.

The main procedures for extracting Task Domain categories included:
(a) NER revealed different segmentations tagged based on their context (e.g., people, place, object).
(b) A cleaning process removed stop-words and keywords that did not add value to the analysis, such as "guideline submission" and "project goal."
(c) NER was reapplied to the cleaned data to produce a new version that better described the software context.
(d) A manual process corrected any tagging inconsistent with the software engineering context.
(e) The entire analyzed text was ordered based on their occurrence. For example, "Network" appeared more frequently than "update." Keywords with fewer occurrences were excluded from the analysis.
(f) A rule-based approach was used to map attributes or keywords to each task.
(g) Finally, a list of attributes for five extracted categories was revealed. Table 3 illustrates eight categories with their attributes.

Five categories were defined for the task domain: (1) application, (2) design, (3) software engineering, (4) organization, and (5) external. The Application category covers different types of applications, such as App, Network, Software, and Web. Design includes Architecture, Dashboard, Module, Prototype, Screen, UI, Wireframe, and Workflow. Software Engineering refers to domain-specific contexts, including user stories, Tech, Bug Fix, Design, Develop, Documentation, Requirements, and Update. The Organization category represents the top task requestors who can impact crowd participation and performance. The External category encompasses a wide range of resources incorporated in the task to provide additional information, serving as a reliable source for a complete understanding of task requirements.

2. Textual: To further assess the type of presentation or task design, three metrics were used to measure the type of task presentation: Sentences, Headers, and Nouns.

3. Readability: Five readability metrics established in previous studies were used, presuming that complex or low-readability tasks were poorly designed, causing users to lose interest. To evaluate each task, the following metrics were used:


(a) Automated Reading Index: Proposed by [24] to measure the understandability of a given text, it is calculated using the average number of characters per word and words per sentence.



% 1.	Task Domain: This diagram depicts the various traits associated with each work, as well as the special features seen in software engineering. Different components of text, such as syntax and semantics, can be translated and interpreted with NLP models. We used NER in the Task Domain to crowdsource software engineering knowledge, develop word-level and phrase-level models, and categorize the sections of raw text into relevant features. The uniformity of software engineering information was ensured through an iterative process.

% Our main procedures for extracting Task Domain categories were: 
% (a)	NER revealed different segmentations tagged based on their context. (i.e., people, place, object).
% (b)	NER revealed different segmentations tagged based on their context. (i.e., people, place, object).
% (c)	A cleaning process removed stop-words and some keywords that did not add values to the analysis, e.g., “guidelines submission”, “project goal.” 
% (d)	NER was reapplied on the cleaned data to reveal a new version that better described the software context. 
% (e)	A manual process corrected any tagging that was inconsistent with the software engineering context.
% (f)	The entire analyzed text was ordered based on their occurrence. For example, “Network” appears more than “update”. Keywords with fewer occurrences were excluded from the analysis.
% (g)	We employed a rule-based approach to help map the attributes or keywords of with each task.  
% (h)	Finally, a list of attributes for five extracted categories was revealed. (Table 2 illustrates 8 categories with their attributes.)

% We defined five categories for Task Domain: (1) Application, (2) Design, (3) Software Engineering, (4) Organization, and (5) External. Application covers different types of applications such as App, Network, Software, and Web. Design covers Architecture, Dashboard, Module, Prototype, Screen, UI, Wireframe, and Workflow. Software Engineering refers to a domain-specific context, including user stories, Tech, Bug Fix, Design, Develop, Documentation, Requirements, and Update. The top task requestors who can impact crowd involvement and performance are represented by the organization. External refers to a wide range of resources that are incorporated in the task to provide extra information, and so can be a source of reliable information for a complete understanding of Task requirements.

% 2.	Textual: To give further assessments of the type of presentation or task design, we used three metrics to measure the task presentation type - Sentences, Headers, and Nouns.
% 3.	Readability: We used five readability metrics established in prior studies, presuming that complex or low-readability tasks were designed poorly, causing users to become disengaged from doing them. To evaluate each task, the following metrics were used:

% \item (a)	Automated Reading Index: Proposed by [114] to measure the understandability of a given text, it is calculated by using the average numbers of characters per word and words per sentence.  

\begin{equation}
% \begin{split}
    A R I=4.71\left(\frac{\text { characters }}{\text {words}}\right)+0.5+\left(\frac{\text { words }}{\text { sentences }}\right)-21.43
% \end{split}
\end{equation}



\begin{itemize}
    \item (b) Flesch Reading Ease Score: Proposed by [25], it ranges between 0 to 100 
    (a higher score means the text is easier to comprehend) and is one of the most widely recognized 
    readability scoring metrics.
\end{itemize}


% \item (b)	Flesch Reading Ease Score: Proposed by [25], it ranges between 0 to 100 (a higher score means the text is easier to comprehend) and is one of the most widely recognized readability scoring metrics.    

% use dmath or \small font to adjust in one line
\begin{equation}
      F R E S=206.835-1.015\left(\frac{\text{ total words }}{\text { total sentences}}\right)-84.6+\left(\frac{\text{ total sylables}}{\text { total words}}\right)
\end{equation}

\begin{itemize}

\item (c)	Flesch Kincaid Grade Level: This is similar to the Flesch Reading Ease Score except that it uses the syllables in words to calculate the readability score. 
\end{itemize}

\begin{equation}
    F K G L=0.39 .\left(\frac{\text { total words }}{\text {total sentences}}\right)+11.8 .\left(\frac{\text {total sylables}}{\text {total words}}\right)-15.9
\end{equation}


\begin{itemize}
\item (d)	Coleman Liau Index: This was proposed by Coleman et al. [26] to calculate the readability score from characters, words, and sentences. L is the average number of characters per 200 words, and S is the average number of sentences per 100 words. 
\end{itemize}


\begin{equation}
    C L I=0.0588 * L-0.296 * S-15.8
\end{equation}

\begin{itemize}

\item (e)	SMOG (Simple Measure of Gobbledygook) Index: Proposed by [27], this readability test uses words with high number if syllables to determine readability. 
\end{itemize}


\begin{equation}
    S M O G=1.0430 * \sqrt{\text { polysylables * } \frac{30}{\text { sentences }}+3.1291}
\end{equation}

\begin{itemize}
\item (f)	Gunning Fog Index: This estimates the years of education necessary to comprehend the text. 
\end{itemize}


\begin{equation}
    \text {Gunning Fog Index}=0.4\left[\frac{\text { words }}{\text { sentences }}+100 * \frac{\text { complex words }}{\text {words}}\right]
\end{equation}

\begin{itemize}
\item (g)	Certain: Sentiment analysis by Linguistic Inquiry and Word Count Tool to determine the level of certainty per each task [28]. 
\end{itemize}




\begin{itemize}
    \item \textbf{\textit{4. Social:}} Two features are employed in the social category: (1) Engagement and (2) Reward. Sentiment orientation of a task is relied upon, with the Linguistic Inquiry and Word Count (LIWC) tool calculating the score based on the emotional context within a task. The reward feature represents the price associated with each task.
\end{itemize}


% \begin{equation}
% \begin{split}
%     A R I=4.71\left(\frac{\text { characters }}{\text {words}}\right)+0.5+\left(\frac{\text { words }}{\text { sentences }}\right)-21.43
% \end{split}
% \end{equation}

% \begin{itemize}
% \item (b)	Flesch Reading Ease Score: Proposed by [116], it ranges between 0 to 100 (a higher score means the text is easier to comprehend) and is one of the most widely recognized readability scoring metrics.    
% \end{itemize}

% % use dmath or \small font to adjust in one line
% \begin{dmath}
%       F R E S=206.835-1.015\left(\frac{\text{ total words }}{\text { total sentences}}\right)-84.6+\left(\frac{\text{ total sylables}}{\text { total words}}\right)
% \end{dmath}
  


% \normalsize

% \begin{itemize}
% \item (c)	Flesch Kincaid Grade Level: This is similar to the Flesch Reading Ease Score except that it uses the syllables in words to calculate the readability score. 
% \end{itemize}

% \begin{equation}
%     F K G L=0.39 .\left(\frac{\text { total words }}{\text {total sentences}}\right)+11.8 .\left(\frac{\text {total sylables}}{\text {total words}}\right)-15.9
% \end{equation}

% \begin{itemize}
% \item (d)	Coleman Liau Index: This was proposed by Coleman et al. [115] to calculate the readability score from characters, words, and sentences. L is the average number of characters per 200 words, and S is the average number of sentences per 100 words. 
% \end{itemize}

% \begin{equation}
%     C L I=0.0588 * L-0.296 * S-15.8
% \end{equation}


% \begin{itemize}
% \item (e)	SMOG (Simple Measure of Gobbledygook) Index: Proposed by [117], this readability test uses words with high number if syllables to determine readability. 
% \end{itemize}

% \begin{equation}
%     S M O G=1.0430 * \sqrt{\text { polysylables * } \frac{30}{\text { sentences }}+3.1291}
% \end{equation}

% \begin{itemize}
% \item (f)	Gunning Fog Index: This estimates the years of education necessary to comprehend the text. 
% \end{itemize}

% \begin{equation}
%     \text {Gunning Fog Index}=0.4\left[\frac{\text { words }}{\text { sentences }}+100 * \frac{\text { complex words }}{\text {words}}\right]
% \end{equation}


% \begin{itemize}
% \item (g)	Certain: Sentiment analysis by Linguistic Inquiry and Word Count Tool to determine the level of certainty per each task [18]. 
% \end{itemize}

% \begin{itemize}
% \item \textbf{\textit{4. Social:}} We employ two features in the social category: (1) Engagement and (2) Reward. We rely on sentiment orientation of a task. Specifically, Linguistic Inquiry and Word Count Tool calculates the score based on emotion context within a task, while reward represents the price that is associated with each task.   
% \end{itemize}



\subsection{Machine Learning Classifiers and Performance/Measure Evaluation}


To answer RQ1, a multiple linear regression model was built to obtain the best-fit equation for software task pricing, labeled as Price1 and Price2. A correlation analysis was performed among the predictors to test for collinearity (Figure 2). The values were not close to 1, confirming that collinearity does not affect the model predictors. To compare the new approach in this study with the previous pricing approach in [20], Price1 included the factors used in the previous regression model tested in [20]. The regression model (Price2) quantifies the factors that contribute to crowdsourcing task-pricing estimation, including the task domain attributes. The models are enumerated as the following equations:


\begin{multline}
    \text { Price } 1=\beta_{0}+\beta_{1}(\text {User Story })+\beta_{2}(\text { Algorithm })+\beta_{3}(\text { Tech })\\+\beta_{4}(\text { Nouns })+\beta_{5}(\text{Links})+ \underline{\varepsilon}
\end{multline}


\begin{dmath}
     \text { Price } 2=\beta_{0}+\beta_{1}(\text {User Story })+\beta_{2}(\text { Algorithm })+\beta_{3}(\text { Tech })+\beta_{4}(\text { Nouns })\\+\beta_{5}(\text{Links})+ \beta_{6}(\text { Days })+ \beta_{7}(\text { Architecture })+ \beta_{8}(\text { Dashboard })+\beta_{9}(\text { Forums })\\+\beta_{10}(\text { GitHub })+\beta_{11}(\text { GitLab })+\beta_{12}(\text { Google })+\beta_{13}(\text { HTTP })+\beta_{14}(\text { IBM })\\+\beta_{15}(\text { Module })+ \beta_{16}(\text { NASA })+\beta_{17}(\text { Network })+\beta_{18}(\text { Prototype })+\beta_{19}(\text { Screen })\\+\beta_{20}(\text { Software }) + \beta_{21}(\text { UI })+\beta_{22}(\text { Web })+\beta_{23}(\text { Wireframe })+\beta_{24}(\text { wflow })\\+ \beta_{25}(\text { bfix })+\beta_{26}(\text { Design })+\beta_{27}(\text { Develop })+\beta_{28}(\text { Documentation })\\+\beta_{29}(\text { Requirements })+\beta_{30}(\text { update }) +\underline{\varepsilon}
\end{dmath}
   

Equation 8 addresses the prediction of task price in software crowdsourcing platforms regarding 32 attributes, Table 3. 

To address RQ2, the study ran three different classifiers on the dataset to determine which design factors worked best to classify successful and cancelled tasks. Logistic Regression (LR), Random Forest (RF), and Neural Network (NN) predicted whether a task was completed or cancelled. 

\begin{figure}[h]
\centering
\includegraphics[width=\textwidth]{fig29.png}
\caption{Extracted design factors and correlation among predictors factors.}\label{fig2}
\end{figure}



% \begin{table}
% \centering
% \caption{Design Factors}
% \begin{tabular}{|p{3cm}|p{8cm}|}
% \hline
% Category & Keywords \\ 
% \hline
% \multirow{4}{*}{Application} 
%     & App \\ 
%     & Network \\ 
%     & Web \\ 
%     & Mobile \\ \hline
% \multirow{8}{*}{Design} 
%     & Architecture \\
%     & Dashboard \\ 
%     & Module \\ 
%     & Prototype \\ 
%     & Screen \\ 
%     & UI \\ 
%     & Wireframe \\ 
%     & Workflow \\ \hline
% \multirow{9}{*}{Software Engineering} 
%     & User Stories \\ 
%     & Tech \\ 
%     & Bug Fix \\ 
%     & Design \\ 
%     & Develop \\ 
%     & Documentation \\ 
%     & Requirements \\ 
%     & Update \\ 
%     & Days \\ \hline
% \multirow{3}{*}{Organization} 
%     & Google \\ 
%     & IBM \\ 
%     & NASA \\ \hline
% \multirow{5}{*}{External} 
%     & Forums \\ 
%     & Git \\ 
%     & GitHub \\ 
%     & GitLab \\ 
%     & HTTP \\ \hline
% \multirow{3}{*}{Textual} 
%     & Sentences \\ 
%     & Headers \\ 
%     & Nouns \\ \hline
% \multirow{7}{*}{Readability} 
%     & Automated Reading Index \\ 
%     & Flesch Reading Ease Score \\ 
%     & Flesch Kincaid Grade Level \\ 
%     & Gunning Fog Index \\ 
%     & SMOG Index \\ 
%     & Coleman Liau Index \\ 
%     & Certain \\ \hline
% \multirow{2}{*}{Social} 
%     & Social Engagement \\ 
%     & Reward \\ \hline
% \end{tabular}
% \end{table}


\begin{table}[ht]
\centering
\caption{Design Categories and Associated Keywords Extracted from Task Descriptions}
\begin{tabular}{|p{4cm}|p{9.5cm}|}
\hline
\textbf{Design Category} & \textbf{Representative Keywords} \\
\hline
Application & App, Network, Web, Mobile \\
\hline
Design & Architecture, Dashboard, Module, Prototype, Screen, UI, Wireframe, Workflow \\
\hline
Software Engineering & User Stories, Tech, Bug Fix, Design, Develop, Documentation, Requirements, Update, Days \\
\hline
Organization & Google, IBM, NASA \\
\hline
External & Forums, Git, GitHub, GitLab, HTTP \\
\hline
Textual & Sentences, Headers, Nouns \\
\hline
Readability & Automated Reading Index, Flesch Reading Ease Score, Flesch-Kincaid Grade Level, \newline Gunning Fog Index, SMOG Index, Coleman-Liau Index, Certain \\
\hline
Social & Social Engagement, Reward \\
\hline
\end{tabular}
\label{tab:design_factors}
\end{table}


\subsection{Machine Learning Classifiers}

To address the objectives of this study, three classifiers were implemented to analyze and compare the effectiveness of the predictive models in classifying outcomes: Logistic Regression (LR), Random Forest (RF), and Neural Network (NN). These classifiers were chosen based on their distinct strengths and their suitability for the dataset at hand.

1. Logistic Regression (LR)
Logistic Regression is a linear model commonly used for binary classification tasks. In this study, LR was employed as a baseline model due to its simplicity and interpretability. The model applied L2 regularization to prevent overfitting and was optimized using the Newton-Conjugate Gradient solver. Logistic Regression processed the features extracted during preprocessing, providing insight into linear relationships between the predictors and the outcome. While the model achieved reasonable accuracy, its performance was limited in capturing non-linear patterns in the data, resulting in lower precision and recall compared to the other classifiers.

2. Random Forest (RF)
Random Forest, an ensemble learning method, was utilized for its robustness and ability to handle non-linear relationships. The model was composed of 100 decision trees, each trained on a bootstrapped subset of the training data, with random feature selection at each split. This approach enhanced generalization by reducing overfitting. Hyperparameters such as the maximum depth of trees and the minimum number of samples per split were optimized using grid search. RF demonstrated improved performance over LR, particularly in recall and robustness to noisy data, although it fell short of the performance achieved by the Neural Network.

3. Neural Network (NN)
The Neural Network model was the most complex and demonstrated the best overall performance. It consisted of three fully connected hidden layers with ReLU activation functions, enabling the model to capture intricate non-linear patterns in the data. The output layer employed a softmax activation function for multi-class classification. The model was trained using the Adam optimizer with a learning rate of 0.01, a batch size of 32, and for 32 epochs, although training details such as the number of epochs for other models are not provided. To address overfitting, dropout regularization was applied with a rate of 0.2. This configuration allowed the NN to achieve the highest accuracy, precision, and recall among the three classifiers, highlighting its suitability for the complexity of the dataset.

\subsection{Models Performance and Measure Evaluation}

To quantify and assess the quality of the applied machine learning algorithms, several model evaluation measures were used. Cross-validation, commonly applied to assess the accuracy of proposed models, was employed. This method is often used in supervised machine learning algorithms to hold out part of the available data. In this study, the data was divided into 10 parts, with each part used as a test set while the remaining parts were used to train the model. This process was repeated for all combinations of train-test splits. This type of assessment has been used in previous studies where the goal was to predict future events [29].

To evaluate model performance, an accuracy classification score was used (Equation 9). This score is the ratio of the number of correctly predicted instances to the total number of instances in the dataset, multiplied by 100 to render the ratio as a percentage. Additionally, two measures, recall and precision, were applied. Precision (Equation 10) is the number of true positives divided by the total number of elements labeled as belonging to the class. Recall (Equation 11) is the number of true positives divided by the total number of elements that actually belong to the class.

\begin{equation}
    \text { Accuracy }=\frac{t p+t n}{t p+t n+f p+f n}
\end{equation}

\begin{equation}
  \text { Precision }=\frac{t p}{(t p+f p)} 
\end{equation}

\begin{equation}
    \text {Recall}=\frac{t p}{(t p+f n)}
\end{equation}
    
Precision and recall are combined into a single measure called F-Measure—the weighted average of the precision and recall. It reaches its best at 1 and its worst at 0. Equation 6 shows how F-Measure can be computed. 

\begin{equation}
\text{F-measure} = \frac{2 \cdot \text{Precision} \cdot \text{Recall}}{\text{Precision} + \text{Recall}}
\end{equation}


\section{Results}

Multiple linear regression analysis was used to develop two models for predicting software crowdsourcing task prices, providing insights into the underlying features that estimate task price. Price1 involves only seven predictors, mostly text descriptor features, as shown in Table 4. Price2 demonstrates improved results by including task domain-specific features, as shown in Table 5.

This section presents the results of the regression models and classification models used to analyze software crowdsourcing task designs. The evaluation metrics for regression models include the coefficient of determination (R²), Akaike Information Criterion (AIC), and Mean Squared Error (MSE), while classification models are assessed based on accuracy, precision, recall, and F1-score. 

Table 4 shows a stepwise regression procedure on TopCoder data, which eliminates non-significant factors. In Equation 8, where price (P) was predicted by the numbers of user stories, technologies, platforms, sentences, headers, and nouns, results showed that the effects of the number of user stories (US) (t-statistics = 28.75, p-values = 0.00), technologies (L) (t-statistics = -5.48, p-values = 0.00), platforms (t-statistics = 8.51, p-values = 0.00), days (t-statistics = -43.92, p-values = 0.00), sentences (t-statistics = -2.80, p-values = 0.00), headers (t-statistics = 4.41, p-values = 0.00), and nouns (t-statistics = 8.11, p-values = 0.00) are statistically significant (\(R^2 = 48\%\)).

The regression results in Equation 7 reveal that the coefficient $\beta_{1}$ =22.72, which signifies whether the number of user stories is increased where the intercept is $\beta_{0}$ =1238.35 (pricing). This result indicates that user stories factor contributes to increased pricing. This also aligns with previous findings on TopCoder data that was collected in 2003. The coefficient $\beta_{2}$ =95.96, which represents the number of technologies, is increased where the intercept is $\beta_{0}$ =1238.35(pricing). Also, when the number of platforms increased $\beta_{3}$ =178.89, the price also did.

 $\beta_{4}$ =-35.32, which represents the number of days for the task to be completed, is decreased where the intercept is $\beta_{0}$ =1238.35  (pricing). This may indicate that more days reflect lower cost. However, this finding may reveal that crowdsourcing requestors are willing to pay more for a fast result. The coefficient $\beta_{5}$=-2.593 shows that tasks with fewer numbers of sentences contribute to increasing task pricing. Finally, the coefficients $\beta_{6}$=6.701 and $\beta_{7}$ =1.738, which represent numbers of headers and sentences respectively, are increased when the intercept is $\beta_{0}$ =1238.35  (pricing).    
 
\begin{table}
\centering
\caption{Multiple linear regression (Price 1)}
\begin{tabular}{lllll} 
% \cline{2-5}
\multirow{3}{*}{} & \multicolumn{4}{l}{Regression Coefficients}    \\ 
\hline
% \cline{2-5}
  \multirow{3}{*}{Variable}                   &     \multicolumn{4}{l}{R2 = \%48~ AIC = 67043.11}  
                      \\ 
\hline
% \cline{2-5}
                          & B         & STD     & t value & P              \\ 
\hline
(Intercept)               & 1238.3595 & 69.7994 & 17.742  & 0.00           \\ 
\hline
User Stories              & 22.7250   & 0.790   & 28.753  & 0.000          \\ 
\hline
\# Tech                   & 95.9684   & 9.813   & -5.483  & 0.000          \\ 
\hline
\#Plat                    & 178.8948  & 21.009  & 8.515   & 0.000          \\ 
\hline
days                      & -35.3231  & 0.804   & -43.920 & 0.000          \\ 
\hline
sentences                 & -2.5939   & 0.924   & -2.806  & 0.005          \\ 
\hline
headers                   & 6.7013    & 1.518   & 4.416   & 0.000          \\ 
\hline
nouns                     & 1.7389    & 0.214   & 8.112   & 0.000          \\
\hline
\end{tabular}
\end{table}

In Equation 8, where price (P) was predicted by 32 predictors in seven categories: Application (4 features), Design (8 features), Software Engineering (9 features), Organization (3 features), External (5 features), Textual (3 features). Results show that the effects of task domain features significantly improve task price estimation, \( R^2 = 77\% \).

Table 5 shows that price (P) can be predicated by (1) the Application type: App (t-statistics = 6.379, p-values = 0.00), Network (t-statistics = 3.528, p-values = 0.00), Software (t-statistics = 3.204, p-values = 0.00), Web (t-statistics = -4.134, p-values = 0.00). 

(2) Design features: Architecture (t-statistics = 12.109, p-values = 0.00), Dashboard (t-statistics = 4.918, p-values = 0.00), Module (t-statistics = 3.968, p-values = 0.00), Prototype (t-statistics = 5.701, p-values = 0.00), Screen (t-statistics = -6.624, p-values = 0.00), UI (t-statistics = -5.826, p-values = 0.00), and Wireframe (t-statistics = 3.451, p-values = 0.00). 

(3) Software Engineering features: User Stories (t-statistics = 20.568, p-values = 0.00), Technology (t-statistics = 5.042, p-values = 0.00), Bug Fix (t-statistics = -7.275, p-values = 0.00), Design (t-statistics = 5.215, p-values = 0.00), Develop (t-statistics = 3.172, p-values = 0.002), Documentation (t-statistics = 3.702, p-values = 0.00), Requirements (t-statistics = 6.379, p-values = 0.00), and Days (t-statistics = 3.464, p-values = 0.00). 

(4) Organization: NASA (t-statistics = 3.518, p-values = 0.00). 

(5) External: Links (t-statistics = 4.296, p-values = 0.00), GitHub (t-statistics = -2.758, p-values = 0.006), GitLab (t-statistics = -3.951, p-values = 0.00), and HTTP (t-statistics = 6.220, p-values = 0.00). 

The regression results in Equation 8 indicate Application features App ($\beta_{7}$ = 22.72), Network ($\beta_{8}$ = 22.72), and Software ($\beta_{9}$ = 22.72) contribute to increased pricing ($\beta_{0}$ = 1139.27),where Web ($\beta_{10}$ = 22.72) on the other hand decreased where the pricing intercept is ($\beta_{0}$ = 1139.27). Design features Architecture ($\beta_{11}$ = 466.9354), Dashboard ($\beta_{12}$ = 212.8277), Module ($\beta_{13}$ = 131.0977), and Prototype ($\beta_{14}$ = 79.6454) increase where the pricing intercept is $\beta_{0}$ = 1139.27. Screen ($\beta_{15}$ = -163.3953) and UI ($\beta_{16}$ = -341.1581) decrease where Wireframe ($\beta_{17}$ = 199.6591) increase when the pricing $\beta_{0}$ = 1139.2730 increases. 

Software Engineering features: User Stories ($\beta_{18}$ = 199.6591), Technology ($\beta_{19}$ = 199.6591), Bug Fix ($\beta_{20}$ = 199.6591), Design ($\beta_{21}$ = 199.6591), Develop $\beta_{22}$ = 199.6591), Documentation ($\beta_{23}$ = 199.6591), Requirements ($\beta_{24}$ = 199.6591), and Days ($\beta_{25}$ = 199.6591) increase when price increases ($\beta_{0}$ = 1139.27). Similarly,  Organization: NASA ($\beta_{26}$ = 199.6591), and External: Links ($\beta_{27}$ = 199.6591), GitHub ($\beta_{28}$ = 199.6591), GitLab ($\beta_{29}$ = 199.6591), and HTTP ($\beta_{26}$ = 199.6591) increase when the intercept $\beta_{0}$ = 1139.2730 increases. 

Table~\ref{tab:regression_metrics} summarizes the performance of the two regression models, Price1 and Price2, using R² and Mean Squared Error (MSE) as evaluation metrics. The results highlight significant differences between the two models, reflecting the impact of incorporating domain-specific features.

Price1, which relies on text descriptor features, achieved an R² of 0.48, indicating that 48\% of the variance in task pricing is explained by the model. However, its Mean Squared Error (MSE) of 25 suggests a moderate level of prediction error, emphasizing the limitations of using only generic task features for price estimation.

In contrast, Price2 leverages domain-specific features, resulting in a substantially higher R² of 0.77. This indicates that 77\% of the variance in task pricing is explained by the model, reflecting its ability to capture more complex pricing patterns. Additionally, the MSE for Price2 is significantly lower at 12, demonstrating improved prediction accuracy and a reduced margin of error compared to Price1.

These findings emphasize the importance of domain-specific features in improving the predictive capability of regression models for software crowdsourcing task pricing. By incorporating these features, Price2 effectively captures nuanced relationships that are not evident in generic descriptors, making it a superior model for this context.


\begin{table}
\centering
\caption{Multiple linear regression (Price 2)}
\begin{tabular}{lllll} 
% \cline{2-5}
\multirow{3}{*}{} & \multicolumn{4}{l}{Regression Coefficients}    \\ 
\hline
% \cline{2-5}
\multirow{3}{*}{Variable}     &     \multicolumn{4}{l}{R2 = \%77~ AIC = 612131.42}  \\ \hline
% \cline{2-5}
                          & B         & STD     & t value & P                                                                    \\ 
\hline
(Intercept)               & 1139.2730 & 35.0759 & 29.954  & 0.00                                                                 \\ 
\hline
sentences                 & -2.4226   & 1.497   & -2.690  & 0.007                                                                \\ 
\hline
headers                   & 6.7050    & 0.213   & 4.479   & 0.000                                                                \\ 
\hline
nouns                     & 1.3387    & 9.282   & 6.281   & 0.000                                                                \\ 
\hline
links                     & 39.8744   & 0.885   & 4.296   & 0.000                                                                \\ 
\hline
User Stories              & 18.2030   & 19.571  & 20.568  & 0.000                                                                \\ 
\hline
Platform                  & 134.4831  & 6.067   & 6.872   & 0.000                                                                \\ 
\hline
Technology                & 30.5884   & 0.829   & 5.042   & 0.000                                                                \\ 
\hline
days                      & -30.6089  & 9.447   & -36.915 & 0.000                                                                \\ 
\hline
APP                       & 140.2262  & 38.560  & 6.379   & 0.000                                                                \\ 
\hline
Architecture              & 466.9354  & 43.273  & 12.109  & 0.000                                                                \\ 
\hline
Dashboard                 & 212.8277  & 13.250  & 4.918   & 0.000                                                                \\ 
\hline
Design                    & 69.0933   & 30.784  & 5.215   & 0.000                                                                \\ 
\hline
GitHub                    & -84.8905  & 35.917  & -2.758  & 0.006                                                                \\ 
\hline
GitLab                    & -141.9097 & 22.501  & -3.951  & 0.000                                                                \\ 
\hline
HTTP                      & 139.9543  & 80.064  & 6.220   & 0.000                                                                \\ 
\hline
IBM                       & 505.9828  & 33.037  & 6.320   & 0.000                                                                \\ 
\hline
Module                    & 131.0977  & 76.862  & 3.968   & 0.000                                                                \\ 
\hline
NASA                      & 270.3672  & 53.171  & 3.518   & 0.000                                                                \\ 
\hline
Network                   & 187.5927  & 13.971  & 3.528   & 0.000                                                                \\ 
\hline
Prototype                 & 79.6454   & 24.668  & 5.701   & 0.000                                                                \\ 
\hline
Screen                    & -163.3953 & 35.706  & -6.624  & 0.000                                                                \\ 
\hline
Software                  & 114.3915  & 58.556  & 3.204   & 0.001                                                                \\ 
\hline
UI                        & -341.1581 & 12.495  & -5.826  & 0.000                                                                \\ 
\hline
Web                       & -51.6525  & 57.863  & -4.134  & 0.000                                                                \\ 
\hline
Wireframe                 & 199.6591  & 10.027  & 3.451   & 0.001                                                                \\ 
\hline
bug\_fix                  & -72.9505  & 13.250  & -7.275  & 0.000                                                                \\ 
\hline
design                    & 69.0933   & 10.724  & 5.215   & 0.000                                                                \\ 
\hline
develop                   & 34.0180   & 10.738  & 3.172   & 0.002                                                                \\ 
\hline
documentation             & 39.7463   & 13.971  & 3.702   & 0.000                                                                \\ 
\hline
requirement               & 74.8951   & 32.370  & 3.464   & 0.001                                                                \\
\hline
\end{tabular}
\end{table}

\begin{table}
\centering
\caption{Performance Metrics of Regression Models}
\label{tab:regression_metrics}
\begin{tabular}{|l|c|c|} 
\hline
Model & R² & Mean Squared Error (MSE) \\ 
\hline
Price1         & 0.48        & 25                               \\ 
Price2         & 0.77        & 12                               \\ 
\hline
\end{tabular}
\end{table}



\begin{itemize}

\item RQ2: Which actionable factors in software crowdsourcing task design affect its success? 

\end{itemize}

 

To answer this research question, the study ran Logistic Regression (LR), Random Forest (RF), and Neural Network (NN) on eight task-design categories (Table 7): Application (4 features), Design (7 features), Software Engineering (8 features), Organization (3 features), External (5 features), Textual (3 features), Readability (7), and Social (2) to determine which features worked best to classify completed and cancelled tasks. 

Table 7 summarizes the findings of the empirical study on TopCoder data. For this analysis we used 80\% of the dataset for training and 20\% for testing, and we reported the results on 10-fold cross validation. We analyzed the results according to accuracy measure, precision, recall, and F-Measure. NN scored 81\%, the highest mean accuracy, or ratio of the number of correctly predicted instances divided by the total number of instances. NN achieves the recall of 79\% and precision of 80\%, showing that NN substantially outperformed the other models and the extracted task design features help identify software crowdsourcing task success. This finding confirmed previous studies’ conclusions that NN is an outstanding text classifier [85]. RF and LR scored 80\% and 76\%, respectively, achieving the recall of 79\% and precision of 79\% and 76\%.   These results confirm that the proposed design factors significantly predict the success of software crowdsourcing task success. 

To further investigate the performance of LR, RF and NN and the factors that contributed to the prediction success, we examined the feature importance, which measures how each feature is independent from or dependent on its class. Our findings show that the top 20 features in the classification results are: Days, GitLab, GitHub, Software, Headers, Web, SMOG Index, Git, Mobile, APP, Repository, Forums, Gunning Fog Index, Social, Screen, Certain, Wordcount, Flesch Reading Ease Score, Email, and NASA, Table 8.




% \usepackage{colortbl}


\begin{table}
\centering
\caption{Performance Accuracy of Task Design Models}
\begin{tabular}{lllll} 
\hline\
Algorithm           & Accuracy & Precision & Recall & F1 – Score  \\ 
\hline
Logistic Regression & 76\%     & 76\%      & 76\%   & 75\%        \\ 
\hline
Random Forest       & 80\%     & 79\%      & 79\%   & 79\%        \\ 
\hline
Neural Network      & 81\%     & 80\%      & 79\%   & 80\%        \\
\hline
\end{tabular}
\end{table}





% \usepackage{multirow}
% \usepackage{colortbl}


\begin{table}
\centering
\caption{Task Design Factors Ranking}
\begin{tabular}{!{\color{black}\vrule}l!{\color{black}\vrule}l!{\color{black}\vrule}l!{\color{black}\vrule}l!{\color{black}\vrule}} 
\hline
Feature                                                                & \begin{tabular}[c]{@{}l@{}} Rank \\(ANOVA) \end{tabular} & Feature                                                             & \begin{tabular}[c]{@{}l@{}} Rank \\(ANOVA) \end{tabular}  \\ 
\hline
Days                                                                   & 82.72                                                    & testing                                                             & 1.58                                                      \\ 
\hline
GitLab                                                                 & 52.98                                                    & sentences                                                           & 1.48                                                      \\ 
\hline
GitHub                                                                 & 31.57                                                    & Network                                                             & 1.44                                                      \\ 
\hline
Software                                                               & 31.12                                                    & verification                                                        & 1.37                                                      \\ 
\hline
Headers                                                                & 30.24                                                    & prototype                                                           & 1.11                                                      \\ 
\hline
Web                                                                    & 25.17                                                    & Prototype                                                           & 1.11                                                      \\ 
\hline
SMOG Index                                                             & 23.32                                                    & Module                                                              & 1.09                                                      \\ 
\hline
Git                                                                    & 20.20                                                    & totalPrize                                                          & 1.05                                                      \\ 
\hline
Mobile                                                                 & 18.65                                                    & documentation                                                       & 1.01                                                      \\ 
\hline
APP                                                                    & 17.73                                                    & Amazon                                                              & 1.00                                                      \\ 
\hline
Repository                                                             & 12.16                                                    & HTTP                                                                & 0.85                                                      \\ 
\hline
LIX                                                                    & 8.25                                                     & Develop                                                             & 0.83                                                      \\ 
\hline
Forums                                                                 & 7.93                                                     & Google                                                              & 0.82                                                      \\ 
\hline
Gunning Fog Index                                                      & 7.75                                                     & FTP                                                                 & 0.80                                                      \\ 
\hline
Social                                                                 & 7.32                                                     & Notification                                                        & 0.54                                                      \\ 
\hline
Screen                                                                 & 7.04                                                     & \begin{tabular}[c]{@{}l@{}} Automated Reading \\Index \end{tabular} & 0.54                                                      \\ 
\hline
Certain                                                                & 6.22                                                     & Function                                                            & 0.46                                                      \\ 
\hline
wordcount                                                              & 6.17                                                     & Architecture                                                        & 0.44                                                      \\ 
\hline
\begin{tabular}[c]{@{}l@{}} Flesch Reading \\Ease Score \end{tabular}  & 5.45                                                     & \begin{tabular}[c]{@{}l@{}} Coleman Liau \\Index \end{tabular}      & 0.38                                                      \\ 
\hline
Email                                                                  & 5.08                                                     & Security                                                            & 0.33                                                      \\ 
\hline
NASA                                                                   & 4.82                                                     & Apple                                                               & 0.23                                                      \\ 
\hline
numTech                                                                & 4.04                                                     & popPlat                                                             & 0.23                                                      \\ 
\hline
\begin{tabular}[c]{@{}l@{}} Flesch Kincaid \\Grade Level \end{tabular} & 3.50                                                     & Design                                                              & 0.13                                                      \\ 
\hline
Data                                                                   & 3.36                                                     & Design                                                              & 0.13                                                      \\ 
\hline
Chat                                                                   & 2.99                                                     & maintenance                                                         & 0.08                                                      \\ 
\hline
linkcount                                                              & 2.87                                                     & UML                                                                 & 0.08                                                      \\ 
\hline
Workflow                                                               & 2.66                                                     & Update                                                              & 0.08                                                      \\ 
\hline
Workflow                                                               & 2.66                                                     & Bitbucket                                                           & 0.07                                                      \\ 
\hline
Links                                                                  & 2.57                                                     & Mockup                                                              & 0.06                                                      \\ 
\hline
GUI                                                                    & 2.57                                                     & Dashboard                                                           & 0.05                                                      \\ 
\hline
IBM                                                                    & 2.43                                                     & Vehicle                                                             & 0.04                                                      \\ 
\hline
Chart                                                                  & 2.33                                                     & Blogs                                                               & 0.04                                                      \\ 
\hline
Nouns                                                                  & 2.22                                                     & numPlat                                                             & 0.03                                                      \\ 
\hline
requirement                                                            & 2.14                                                     & popTech                                                             & 0.03                                                      \\ 
\hline
Wireframe                                                              & 1.99                                                     & Links                                                               & 0.03                                                      \\ 
\hline
Microsoft                                                              & 1.98                                                     & \multicolumn{2}{l!{\color{black}\vrule}}{\multirow{3}{*}{}}                                                                     \\ 
\hline
bug fix                                                                & 1.84                                                     & \multicolumn{2}{l!{\color{black}\vrule}}{}                                                                                      \\ 
\hline
UI                                                                     & 1.64                                                     & \multicolumn{2}{l!{\color{black}\vrule}}{}                                                                                      \\
\hline
\end{tabular}
\end{table}

The study indicated that tasks with more headers as well as external resources tend to be successful by engaging more developer, and that the Textual category was key in predicting a task’s success. Specifically, the number of headers and Readability were among top features in the proposed model. Finally, Software Engineering and Application categories significantly influence the success of a task, Table 8.     

Table~\ref{tab:ml_comparison} provides a comparative summary of the performance and characteristics of three machine learning models—Logistic Regression (LR), Random Forest (RF), and Neural Network (NN)—across the Price1, Price2, and Classification experiments. The table highlights each model’s strengths and limitations in handling specific features and tasks. Logistic Regression served as a baseline model, demonstrating reasonable performance for simpler relationships. Random Forest improved robustness and non-linear pattern recognition but struggled with high-dimensional data. Neural Network, while computationally intensive, achieved the highest accuracy and recall in the Classification experiment, showcasing its ability to handle intricate data patterns effectively.

\begin{table}[h!]
\centering
\caption{Comparison of ML Models Across Experiments}
\label{tab:ml_comparison}
\begin{tabular}{|p{2.5cm}|p{2.5cm}|p{8cm}|}
\hline
\textbf{Experiment}      & \textbf{Model} & \textbf{Summary of Performance and Characteristics} \\ \hline
Price1                  & Logistic Regression (LR) & 
Baseline model with satisfactory performance for simpler predictors. Provided insights into linear relationships. \textbf{Limitations:} Struggled with non-linearities, reducing accuracy for complex tasks. \\ \hline
Price1                  & Random Forest (RF)       & 
Enhanced generalization through ensemble learning. Captured non-linear patterns effectively. \textbf{Limitations:} Challenges with high-dimensional data led to slight overfitting. \\ \hline
Price1                  & Neural Network (NN)      & 
Not explicitly evaluated in Price1 but could excel in capturing complex non-linear patterns. \\ \hline
Price2                  & Logistic Regression (LR) & 
Utilized domain-specific features like GitHub and UI attributes. Moderate improvement over Price1. \textbf{Limitations:} Limited in handling complex feature interactions. \\ \hline
Price2                  & Random Forest (RF)       & 
Significant improvement by leveraging domain-specific features. Robust for high-dimensional and categorical data. \textbf{Limitations:} Increased computational cost with additional features. \\ \hline
Price2                  & Neural Network (NN)      & 
Not explicitly evaluated but suitable for modeling non-linear interactions. \\ \hline
Classification          & Logistic Regression (LR) & 
Baseline model for binary classification (finished vs. canceled tasks). Achieved \textbf{76\% accuracy}, precision, and recall. \textbf{Limitations:} Limited in capturing non-linear patterns. \\ \hline
Classification          & Random Forest (RF)       & 
Improved over LR with \textbf{80\% accuracy}. Robust to noise; provided feature importance analysis. \textbf{Limitations:} Performance slightly lower than NN. \\ \hline
Classification          & Neural Network (NN)      & 
Highest performance: \textbf{Accuracy:} 81\%, \textbf{Precision:} 80\%, \textbf{Recall:} 79\%. \textbf{Strengths:} Captured intricate non-linear relationships and feature interactions. Excelled in high-dimensional data handling. \textbf{Limitations:} High computational cost and risk of overfitting (mitigated with dropout regularization). \\ \hline
\end{tabular}
\end{table}

% \section{Results}

% This section presents the results of the regression models and classification models used to analyze software crowdsourcing task designs. The evaluation metrics for regression models include the coefficient of determination (R²), Akaike Information Criterion (AIC), and Mean Squared Error (MSE), while classification models are assessed based on accuracy, precision, recall, and F1-score.

% \subsection{Regression Models: Price1 and Price2}

% Multiple linear regression analysis was used to develop two models for predicting software crowdsourcing task prices: \textbf{Price1}, which relies on text descriptor features, and \textbf{Price2}, which incorporates domain-specific features. The results are summarized in Tables~\ref{tab:price1_results} and~\ref{tab:price2_results}.

% \textbf{Price1} includes seven predictors: user stories, technologies, platforms, days, sentences, headers, and nouns. The regression results, as shown in Table~\ref{tab:price1_results}, indicate a moderate fit (\( R^2 = 48\% \), AIC = 67043.11), with an MSE of 25. These results suggest that the number of user stories, platforms, and technologies significantly influence task pricing, while the number of days has an inverse relationship with price, indicating that shorter timelines increase cost.

% \begin{table}
% \centering
% \caption{Multiple Linear Regression (Price1)}
% \label{tab:price1_results}
% \begin{tabular}{lllll} 
% \hline
% \textbf{Variable} & \textbf{B} & \textbf{STD} & \textbf{t-value} & \textbf{P-value} \\ 
% \hline
% Intercept         & 1238.3595  & 69.7994      & 17.742           & 0.00            \\ 
% User Stories      & 22.7250    & 0.790        & 28.753           & 0.000           \\ 
% Technologies      & 95.9684    & 9.813        & -5.483           & 0.000           \\ 
% Platforms         & 178.8948   & 21.009       & 8.515            & 0.000           \\ 
% Days              & -35.3231   & 0.804        & -43.920          & 0.000           \\ 
% Sentences         & -2.5939    & 0.924        & -2.806           & 0.005           \\ 
% Headers           & 6.7013     & 1.518        & 4.416            & 0.000           \\ 
% Nouns             & 1.7389     & 0.214        & 8.112            & 0.000           \\ 
% \hline
% \end{tabular}
% \end{table}

% \textbf{Price2} expands the predictors to include 32 features across seven categories: Application, Design, Software Engineering, Organization, External, Textual, and Readability. The regression results, detailed in Table~\ref{tab:price2_results}, demonstrate a significantly improved fit (\( R^2 = 77\% \), AIC = 612131.42), with a lower MSE of 12. These results highlight the critical role of domain-specific features, such as application type, design characteristics, and external links, in accurately predicting task pricing.

% \begin{table}
% \centering
% \caption{Multiple Linear Regression (Price2)}
% \label{tab:price2_results}
% \begin{tabular}{lllll} 
% \hline
% \textbf{Variable} & \textbf{B} & \textbf{STD} & \textbf{t-value} & \textbf{P-value} \\ 
% \hline
% Intercept         & 1139.2730  & 35.0759      & 29.954           & 0.00            \\ 
% Sentences         & -2.4226    & 1.497        & -2.690           & 0.007           \\ 
% Headers           & 6.7050     & 0.213        & 4.479            & 0.000           \\ 
% Nouns             & 1.3387     & 9.282        & 6.281            & 0.000           \\ 
% Links             & 39.8744    & 0.885        & 4.296            & 0.000           \\ 
% User Stories      & 18.2030    & 19.571       & 20.568           & 0.000           \\ 
% Platforms         & 134.4831   & 6.067        & 6.872            & 0.000           \\ 
% Technologies      & 30.5884    & 0.829        & 5.042            & 0.000           \\ 
% Days              & -30.6089   & 9.447        & -36.915          & 0.000           \\ 
% \hline
% \end{tabular}
% \end{table}

% \begin{table}
% \centering
% \caption{Performance Metrics of Regression Models}
% \label{tab:regression_metrics}
% \begin{tabular}{lll} 
% \hline
% \textbf{Model} & \textbf{R²} & \textbf{Mean Squared Error (MSE)} \\ 
% \hline
% Price1         & 0.48        & 25                               \\ 
% Price2         & 0.77        & 12                               \\ 
% \hline
% \end{tabular}
% \end{table}

% As summarized in Table~\ref{tab:regression_metrics}, the MSE for Price2 is significantly lower than that of Price1, reflecting its enhanced ability to capture complex task pricing relationships.

% \subsection{Classification Models}

% To answer RQ2 (Which actionable factors in software crowdsourcing task design affect success?), classification models Logistic Regression (LR), Random Forest (RF), and Neural Network (NN) were applied. These models used features from eight categories, including Application, Design, Software Engineering, and Readability, to classify completed and canceled tasks. The evaluation metrics, including accuracy, precision, recall, and F1-score, are reported in Table~\ref{tab:classification_metrics}.

% \begin{table}
% \centering
% \caption{Performance Accuracy of Task Design Models}
% \label{tab:classification_metrics}
% \begin{tabular}{lllll} 
% \hline
% Algorithm           & Accuracy & Precision & Recall & F1-Score \\ 
% \hline
% Logistic Regression & 76\%     & 76\%      & 76\%   & 75\%     \\ 
% Random Forest       & 80\%     & 79\%      & 79\%   & 79\%     \\ 
% Neural Network      & 81\%     & 80\%      & 79\%   & 80\%     \\
% \hline
% \end{tabular}
% \end{table}

% Neural Network achieved the highest accuracy (81\%), precision (80\%), and recall (79\%), confirming its ability to classify tasks effectively. Random Forest performed slightly lower, with 80\% accuracy, while Logistic Regression served as a baseline model with 76\% accuracy. These results demonstrate that the proposed task design factors significantly predict software crowdsourcing task success.

\section{Discussion}

 Different design factors were studied to understand their contribution to software crowdsourcing task price estimation and to provide actionable insights for crowdsourcing managers regarding what contributes to task success.
The results indicate that software crowdsourcing task prices can be estimated through an enhanced approach comparable to prior work in CCP. Specifically, pricing was estimated using a multiple regression model, which exhibits significantly better results than the previous model, allowing for more validation on a larger dataset. The drivers derived from this study were mapped to different design categories (Table 3).
The proposed approach overcomes software crowdsourcing task design limitations and guides crowdsourcing platforms through an automated technique. The main contributions are summarized as follows:
1.	Validation with Large Dataset: Recent and large real datasets extracted from TopCoder, the largest software crowdsourcing platform, were employed to validate the proposed approach. The current analysis uses software crowdsourcing data with more than 4,500 software tasks.
2.	Improvement with Domain-Specific Features: The second model, Price2, improves the results of Price1 by introducing domain-specific features, providing more context for the task.
3.	Availability of Pricing Factors: The proposed pricing factors are always available before the development process.
4.	Systematic Extraction of Design Factors: A systematic approach was used to extract design factors that help determine software crowdsourcing task success.
Throughout this study, the process of leveraging machine learning techniques and a domain-specific approach to analyze and extract design factors from software requirements was demonstrated. The proposed approach showed improvement over prior work, with the extracted design factors contributing to task price estimation and outperforming existing methods. Additionally, the pricing drivers better represented the tasks than only textual drivers, allowing for more detailed task characteristics and context.
For the second part of the analysis, eight categories with 40 features helped classify software tasks into successful and canceled tasks. The method captured an accuracy of 81\% using a neural network. The findings were consistent with previous work in [32]. Since the predictors were numerical features, ANOVA was applied to rank them in the study. Table 6 illustrates the top 20 features that contribute to the model’s success, including Days, GitLab, GitHub, Software, Headers, Web, SMOG Index, Git, Mobile, APP, Repository, Forums, Gunning Fog Index, Social, Screen, Certain, Wordcount, Flesch Reading Ease Score, Email, and NASA.
The results suggest that the use of more headers, detailed tasks, low uncertainty in describing the text, and external resources in task design contribute to task success. This aligns with previous research in [32], which argued that narrative-based tasks with less linguistic uncertainty were more engaging for participants and received more submissions. This finding highlights the importance of providing clear tasks with external resources to better clarify and complete the information provided, thus improving readability.
Crowdsourcing platform managers should increase the visibility of these guidelines in their design of software tasks. Another design category influencing task success is the readability in describing a task’s presentation quality. Tasks with high readability scores (SMOG Index, Gunning Fog Index, Flesch Reading Ease Score) were more successful. Writing tasks with higher readability indicates successful design, a finding consistent with previous studies [30].
The analysis also identified social and certain features as contributors to software task success. Social measures the level of positivity and engagement in the task description, while certain measures the level of ambiguity in the given tasks. As suggested in [6], software tasks with less ambiguity tend to be more successful than tasks with more. This calls for strategies for crowdsourcing platforms to ensure task clarity.
Tasks related to widely used technologies receive more attention in the TopCoder community. For instance, web applications such as HTML, CSS, and JavaScript tend to attract the community. Other factors influencing community attraction include the number of technologies, the number of days, and requestor types such as NASA, IBM, Microsoft, and Google. Future studies are needed to explore the effects of crowdsourcing different tasks that vary in their durations to find an optimal approach to crowdsourcing.
Finally, it is possible to develop an automated technique to identify types of software crowdsourcing tasks and applied design strategies. Such a technique would improve the current metrics, which are mainly focused on the textual description of a task in platforms like Stack Overflow (e.g., title length, post length, and tags), allowing for a more generalized approach to crowdsourcing. Although this study employed six readability indices—Flesch Reading Ease, Flesch-Kincaid Grade Level, Automated Reading Index (ARI), Coleman-Liau Index, Gunning Fog Index, and SMOG—to measure task comprehensibility, our findings suggest that only a subset (e.g., Flesch Reading Ease, SMOG, and Gunning Fog Index) were consistently among the top predictors of task success (see Table 8). Therefore, practitioners may choose to prioritize these key indices when designing or auditing tasks for clarity and engagement, without necessarily applying all six in real-world settings.


\section{Practical Application of Design Factors}

To demonstrate how the ranked factors in Table~8 can be applied in real-world scenarios, consider the following software development use case:

A project manager on a crowdsourcing platform intends to post a backend development task involving the integration of a payment gateway. Based on the top-ranked factors identified in this study, the manager first ensures that the task description is highly readable, achieving favorable scores on the SMOG and Flesch-Kincaid Grade Level indices. Clear language reduces developer confusion and increases submission quality.

Next, the manager includes external resources—such as technical API documentation, sample code snippets, and database schemas—to enhance task clarity. Recognizing the importance of fair and motivating compensation, the manager uses the Price2 model developed in this study to determine a reward estimate based on task features, rather than relying solely on platform norms.

Informed by the identified success factors, the task is also broken down into smaller subtasks with intermediate milestones to manage complexity and encourage participation. This structured and data-informed approach increases the likelihood of timely and successful task completion, demonstrating the practical value of the proposed design framework.

 
\section{Implications for Software Crowdsourcing Practitioners}

In this section, several factors identified as effective for the success of software crowdsourcing task design are discussed. Additionally, the utilization of these factors as guidelines for software crowdsourcing platforms when designing tasks is shown.

\textbf{External Resources:} Regarding the impact of external resources, software tasks that include resources such as GitLab, Forums, and GitHub tend to be highly engaging for developers on the platform. In fact, according to the results of a prior survey, most respondents agreed that having external links helped provide more information, increasing their chances of success. Future studies are needed to quantify the effectiveness of different external resources concerning the type of task.

\textbf{Presentation Quality:} Findings confirm the importance of providing a clear and comprehensive task description. Improving clarity across multiple sections of the task description is suggested for a more engaging experience. For example, writing a task description that includes different sections explaining the purpose of the task and detailed requirements should be adopted. Additionally, positive sentiments in the task description can enhance task effectiveness.

\textbf{Task Domain:} The study characterized the task domain into different factors, including requester type and task-specific domain. For instance, requester reputation was found to be an important factor for task success. Table 4 shows that keywords like NASA and IBM were among the features determining the success of a task.

Finally, tasks involving web and mobile applications and user experience were among the major specific domains attracting crowd attention. This finding may relate to the popularity of these domains on the TopCoder platform. Future studies are needed to confirm this finding on different crowdsourcing platforms.

\section{Limitations}
This study has several limitations that should be acknowledged. First, the findings are based exclusively on data collected from the TopCoder platform. As such, applying the proposed approach to other crowdsourcing platforms may yield different results due to variations in task structure, user behavior, and platform dynamics. Future research should replicate and extend this methodology across diverse platforms to assess its generalizability.

Second, while the study identifies key task design factors influencing task success, it does not consider individual developer attributes such as motivation, skills, experience, or cultural background—all of which may affect outcomes. Incorporating qualitative approaches, such as interviews or surveys, could offer deeper insights into developer decision-making and behavior.

Third, although the proposed pricing models (Price1 and Price2) provide data-driven estimates based on task features, it is important to recognize that many software crowdsourcing platforms determine pricing according to prevailing market benchmarks. This practice reflects platform maturity and helps maintain competitiveness and worker engagement. Therefore, the proposed models should be viewed as complementary tools that enhance traditional pricing strategies by offering predictive insights—particularly valuable for new platforms, novel task categories, or when historical pricing data is limited.



\section{Conclusion}

This research provides a framework to help software crowdsourcing platforms understand the design features that affect software task success. Guidelines were presented that platforms can follow to increase task success. Platforms are advised to consider task presentation quality by implementing task assessment metrics before task development begins. Additionally, external resources and forums were specifically associated with successful tasks. Future studies should investigate what improvements can be made to forums to enhance member engagement and productivity. For example, points could be awarded to members who support beginners on the platforms.

To bolster the generalizability of the approach and findings, future studies are needed to empirically investigate the effectiveness of each design category on task success. For instance, how does crowdsourcing Google tasks influence crowd behavior? Does posting tasks from companies like Google or IBM affect the success rate of other tasks? Finally, new tools for the large-scale application of this approach might be developed to manage platform heterogeneity in crowdsourcing.


% \section{Results}\label{sec2}

% Sample body text. Sample body text. Sample body text. Sample body text. Sample body text. Sample body text. Sample body text. Sample body text.

% \section{This is an example for first level head---section head}\label{sec3}

% \subsection{This is an example for second level head---subsection head}\label{subsec2}

% \subsubsection{This is an example for third level head---subsubsection head}\label{subsubsec2}

% Sample body text. Sample body text. Sample body text. Sample body text. Sample body text. Sample body text. Sample body text. Sample body text. 

% \section{Equations}\label{sec4}

% Equations in \LaTeX\ can either be inline or on-a-line by itself (``display equations''). For
% inline equations use the \verb+$...$+ commands. E.g.: The equation
% $H\psi = E \psi$ is written via the command \verb+$H \psi = E \psi$+.

% For display equations (with auto generated equation numbers)
% one can use the equation or align environments:
% \begin{equation}
% \|\tilde{X}(k)\|^2 \leq\frac{\sum\limits_{i=1}^{p}\left\|\tilde{Y}_i(k)\right\|^2+\sum\limits_{j=1}^{q}\left\|\tilde{Z}_j(k)\right\|^2 }{p+q}.\label{eq1}
% \end{equation}
% where,
% \begin{align}
% D_\mu &=  \partial_\mu - ig \frac{\lambda^a}{2} A^a_\mu \nonumber \\
% F^a_{\mu\nu} &= \partial_\mu A^a_\nu - \partial_\nu A^a_\mu + g f^{abc} A^b_\mu A^a_\nu \label{eq2}
% \end{align}
% Notice the use of \verb+\nonumber+ in the align environment at the end
% of each line, except the last, so as not to produce equation numbers on
% lines where no equation numbers are required. The \verb+\label{}+ command
% should only be used at the last line of an align environment where
% \verb+\nonumber+ is not used.
% \begin{equation}
% Y_\infty = \left( \frac{m}{\textrm{GeV}} \right)^{-3}
%     \left[ 1 + \frac{3 \ln(m/\textrm{GeV})}{15}
%     + \frac{\ln(c_2/5)}{15} \right]
% \end{equation}
% The class file also supports the use of \verb+\mathbb{}+, \verb+\mathscr{}+ and
% \verb+\mathcal{}+ commands. As such \verb+\mathbb{R}+, \verb+\mathscr{R}+
% and \verb+\mathcal{R}+ produces $\mathbb{R}$, $\mathscr{R}$ and $\mathcal{R}$
% respectively (refer Subsubsection~\ref{subsubsec2}).

% \section{Tables}\label{sec5}

% Tables can be inserted via the normal table and tabular environment. To put
% footnotes inside tables you should use \verb+\footnotetext[]{...}+ tag.
% The footnote appears just below the table itself (refer Tables~\ref{tab1} and \ref{tab2}). 
% For the corresponding footnotemark use \verb+\footnotemark[...]+

% \begin{table}[h]
% \caption{Caption text}\label{tab1}%
% \begin{tabular}{@{}llll@{}}
% \toprule
% Column 1 & Column 2  & Column 3 & Column 4\\
% \midrule
% row 1    & data 1   & data 2  & data 3  \\
% row 2    & data 4   & data 5\footnotemark[1]  & data 6  \\
% row 3    & data 7   & data 8  & data 9\footnotemark[2]  \\
% \botrule
% \end{tabular}
% \footnotetext{Source: This is an example of table footnote. This is an example of table footnote.}
% \footnotetext[1]{Example for a first table footnote. This is an example of table footnote.}
% \footnotetext[2]{Example for a second table footnote. This is an example of table footnote.}
% \end{table}

% \noindent
% The input format for the above table is as follows:

% %%=============================================%%
% %% For presentation purpose, we have included  %%
% %% \bigskip command. Please ignore this.       %%
% %%=============================================%%
% \bigskip
% \begin{verbatim}
% \begin{table}[<placement-specifier>]
% \caption{<table-caption>}\label{<table-label>}%
% \begin{tabular}{@{}llll@{}}
% \toprule
% Column 1 & Column 2 & Column 3 & Column 4\\
% \midrule
% row 1 & data 1 & data 2	 & data 3 \\
% row 2 & data 4 & data 5\footnotemark[1] & data 6 \\
% row 3 & data 7 & data 8	 & data 9\footnotemark[2]\\
% \botrule
% \end{tabular}
% \footnotetext{Source: This is an example of table footnote. 
% This is an example of table footnote.}
% \footnotetext[1]{Example for a first table footnote.
% This is an example of table footnote.}
% \footnotetext[2]{Example for a second table footnote. 
% This is an example of table footnote.}
% \end{table}
% \end{verbatim}
% \bigskip
% %%=============================================%%
% %% For presentation purpose, we have included  %%
% %% \bigskip command. Please ignore this.       %%
% %%=============================================%%

% \begin{table}[h]
% \caption{Example of a lengthy table which is set to full textwidth}\label{tab2}
% \begin{tabular*}{\textwidth}{@{\extracolsep\fill}lcccccc}
% \toprule%
% & \multicolumn{3}{@{}c@{}}{Element 1\footnotemark[1]} & \multicolumn{3}{@{}c@{}}{Element 2\footnotemark[2]} \\\cmidrule{2-4}\cmidrule{5-7}%
% Project & Energy & $\sigma_{calc}$ & $\sigma_{expt}$ & Energy & $\sigma_{calc}$ & $\sigma_{expt}$ \\
% \midrule
% Element 3  & 990 A & 1168 & $1547\pm12$ & 780 A & 1166 & $1239\pm100$\\
% Element 4  & 500 A & 961  & $922\pm10$  & 900 A & 1268 & $1092\pm40$\\
% \botrule
% \end{tabular*}
% \footnotetext{Note: This is an example of table footnote. This is an example of table footnote this is an example of table footnote this is an example of~table footnote this is an example of table footnote.}
% \footnotetext[1]{Example for a first table footnote.}
% \footnotetext[2]{Example for a second table footnote.}
% \end{table}

% In case of double column layout, tables which do not fit in single column width should be set to full text width. For this, you need to use \verb+\begin{table*}+ \verb+...+ \verb+\end{table*}+ instead of \verb+\begin{table}+ \verb+...+ \verb+\end{table}+ environment. Lengthy tables which do not fit in textwidth should be set as rotated table. For this, you need to use \verb+\begin{sidewaystable}+ \verb+...+ \verb+\end{sidewaystable}+ instead of \verb+\begin{table*}+ \verb+...+ \verb+\end{table*}+ environment. This environment puts tables rotated to single column width. For tables rotated to double column width, use \verb+\begin{sidewaystable*}+ \verb+...+ \verb+\end{sidewaystable*}+.

% \begin{sidewaystable}
% \caption{Tables which are too long to fit, should be written using the ``sidewaystable'' environment as shown here}\label{tab3}
% \begin{tabular*}{\textheight}{@{\extracolsep\fill}lcccccc}
% \toprule%
% & \multicolumn{3}{@{}c@{}}{Element 1\footnotemark[1]}& \multicolumn{3}{@{}c@{}}{Element\footnotemark[2]} \\\cmidrule{2-4}\cmidrule{5-7}%
% Projectile & Energy	& $\sigma_{calc}$ & $\sigma_{expt}$ & Energy & $\sigma_{calc}$ & $\sigma_{expt}$ \\
% \midrule
% Element 3 & 990 A & 1168 & $1547\pm12$ & 780 A & 1166 & $1239\pm100$ \\
% Element 4 & 500 A & 961  & $922\pm10$  & 900 A & 1268 & $1092\pm40$ \\
% Element 5 & 990 A & 1168 & $1547\pm12$ & 780 A & 1166 & $1239\pm100$ \\
% Element 6 & 500 A & 961  & $922\pm10$  & 900 A & 1268 & $1092\pm40$ \\
% \botrule
% \end{tabular*}
% \footnotetext{Note: This is an example of table footnote this is an example of table footnote this is an example of table footnote this is an example of~table footnote this is an example of table footnote.}
% \footnotetext[1]{This is an example of table footnote.}
% \end{sidewaystable}

% \section{Figures}\label{sec6}

% As per the \LaTeX\ standards you need to use eps images for \LaTeX\ compilation and \verb+pdf/jpg/png+ images for \verb+PDFLaTeX+ compilation. This is one of the major difference between \LaTeX\ and \verb+PDFLaTeX+. Each image should be from a single input .eps/vector image file. Avoid using subfigures. The command for inserting images for \LaTeX\ and \verb+PDFLaTeX+ can be generalized. The package used to insert images in \verb+LaTeX/PDFLaTeX+ is the graphicx package. Figures can be inserted via the normal figure environment as shown in the below example:

% %%=============================================%%
% %% For presentation purpose, we have included  %%
% %% \bigskip command. Please ignore this.       %%
% %%=============================================%%
% \bigskip
% \begin{verbatim}
% \begin{figure}[<placement-specifier>]
% \centering
% \includegraphics{<eps-file>}
% \caption{<figure-caption>}\label{<figure-label>}
% \end{figure}
% \end{verbatim}
% \bigskip
% %%=============================================%%
% %% For presentation purpose, we have included  %%
% %% \bigskip command. Please ignore this.       %%
% %%=============================================%%

% \begin{figure}[h]
% \centering
% \includegraphics[width=0.9\textwidth]{fig.eps}
% \caption{This is a widefig. This is an example of long caption this is an example of long caption  this is an example of long caption this is an example of long caption}\label{fig1}
% \end{figure}

% In case of double column layout, the above format puts figure captions/images to single column width. To get spanned images, we need to provide \verb+\begin{figure*}+ \verb+...+ \verb+\end{figure*}+.

% For sample purpose, we have included the width of images in the optional argument of \verb+\includegraphics+ tag. Please ignore this. 

% \section{Algorithms, Program codes and Listings}\label{sec7}

% Packages \verb+algorithm+, \verb+algorithmicx+ and \verb+algpseudocode+ are used for setting algorithms in \LaTeX\ using the format:

% %%=============================================%%
% %% For presentation purpose, we have included  %%
% %% \bigskip command. Please ignore this.       %%
% %%=============================================%%
% \bigskip
% \begin{verbatim}
% \begin{algorithm}
% \caption{<alg-caption>}\label{<alg-label>}
% \begin{algorithmic}[1]
% . . .
% \end{algorithmic}
% \end{algorithm}
% \end{verbatim}
% \bigskip
% %%=============================================%%
% %% For presentation purpose, we have included  %%
% %% \bigskip command. Please ignore this.       %%
% %%=============================================%%

% You may refer above listed package documentations for more details before setting \verb+algorithm+ environment. For program codes, the ``verbatim'' package is required and the command to be used is \verb+\begin{verbatim}+ \verb+...+ \verb+\end{verbatim}+. 

% Similarly, for \verb+listings+, use the \verb+listings+ package. \verb+\begin{lstlisting}+ \verb+...+ \verb+\end{lstlisting}+ is used to set environments similar to \verb+verbatim+ environment. Refer to the \verb+lstlisting+ package documentation for more details.

% A fast exponentiation procedure:

% \lstset{texcl=true,basicstyle=\small\sf,commentstyle=\small\rm,mathescape=true,escapeinside={(*}{*)}}
% \begin{lstlisting}
% begin
%   for $i:=1$ to $10$ step $1$ do
%       expt($2,i$);  
%       newline() od                (*\textrm{Comments will be set flush to the right margin}*)
% where
% proc expt($x,n$) $\equiv$
%   $z:=1$;
%   do if $n=0$ then exit fi;
%      do if odd($n$) then exit fi;                 
%         comment: (*\textrm{This is a comment statement;}*)
%         $n:=n/2$; $x:=x*x$ od;
%      { $n>0$ };
%      $n:=n-1$; $z:=z*x$ od;
%   print($z$). 
% end
% \end{lstlisting}

% \begin{algorithm}
% \caption{Calculate $y = x^n$}\label{algo1}
% \begin{algorithmic}[1]
% \Require $n \geq 0 \vee x \neq 0$
% \Ensure $y = x^n$ 
% \State $y \Leftarrow 1$
% \If{$n < 0$}\label{algln2}
%         \State $X \Leftarrow 1 / x$
%         \State $N \Leftarrow -n$
% \Else
%         \State $X \Leftarrow x$
%         \State $N \Leftarrow n$
% \EndIf
% \While{$N \neq 0$}
%         \If{$N$ is even}
%             \State $X \Leftarrow X \times X$
%             \State $N \Leftarrow N / 2$
%         \Else[$N$ is odd]
%             \State $y \Leftarrow y \times X$
%             \State $N \Leftarrow N - 1$
%         \EndIf
% \EndWhile
% \end{algorithmic}
% \end{algorithm}

% %%=============================================%%
% %% For presentation purpose, we have included  %%
% %% \bigskip command. Please ignore this.       %%
% %%=============================================%%
% \bigskip
% \begin{minipage}{\hsize}%
% \lstset{frame=single,framexleftmargin=-1pt,framexrightmargin=-17pt,framesep=12pt,linewidth=0.98\textwidth,language=pascal}% Set your language (you can change the language for each code-block optionally)
% %%% Start your code-block
% \begin{lstlisting}
% for i:=maxint to 0 do
% begin
% { do nothing }
% end;
% Write('Case insensitive ');
% Write('Pascal keywords.');
% \end{lstlisting}
% \end{minipage}

% \section{Cross referencing}\label{sec8}

% Environments such as figure, table, equation and align can have a label
% declared via the \verb+\label{#label}+ command. For figures and table
% environments use the \verb+\label{}+ command inside or just
% below the \verb+\caption{}+ command. You can then use the
% \verb+\ref{#label}+ command to cross-reference them. As an example, consider
% the label declared for Figure~\ref{fig1} which is
% \verb+\label{fig1}+. To cross-reference it, use the command 
% \verb+Figure \ref{fig1}+, for which it comes up as
% ``Figure~\ref{fig1}''. 

% To reference line numbers in an algorithm, consider the label declared for the line number 2 of Algorithm~\ref{algo1} is \verb+\label{algln2}+. To cross-reference it, use the command \verb+\ref{algln2}+ for which it comes up as line~\ref{algln2} of Algorithm~\ref{algo1}.

% \subsection{Details on reference citations}\label{subsec7}

% Standard \LaTeX\ permits only numerical citations. To support both numerical and author-year citations this template uses \verb+natbib+ \LaTeX\ package. For style guidance please refer to the template user manual.

% Here is an example for \verb+\cite{...}+: \cite{bib1}. Another example for \verb+\citep{...}+: \citep{bib2}. For author-year citation mode, \verb+\cite{...}+ prints Jones et al. (1990) and \verb+\citep{...}+ prints (Jones et al., 1990).

% All cited bib entries are printed at the end of this article: \cite{bib3}, \cite{bib4}, \cite{bib5}, \cite{bib6}, \cite{bib7}, \cite{bib8}, \cite{bib9}, \cite{bib10}, \cite{bib11}, \cite{bib12} and \cite{bib13}.


% \section{Examples for theorem like environments}\label{sec10}

% For theorem like environments, we require \verb+amsthm+ package. There are three types of predefined theorem styles exists---\verb+thmstyleone+, \verb+thmstyletwo+ and \verb+thmstylethree+ 

% %%=============================================%%
% %% For presentation purpose, we have included  %%
% %% \bigskip command. Please ignore this.       %%
% %%=============================================%%
% \bigskip
% \begin{tabular}{|l|p{19pc}|}
% \hline
% \verb+thmstyleone+ & Numbered, theorem head in bold font and theorem text in italic style \\\hline
% \verb+thmstyletwo+ & Numbered, theorem head in roman font and theorem text in italic style \\\hline
% \verb+thmstylethree+ & Numbered, theorem head in bold font and theorem text in roman style \\\hline
% \end{tabular}
% \bigskip
% %%=============================================%%
% %% For presentation purpose, we have included  %%
% %% \bigskip command. Please ignore this.       %%
% %%=============================================%%

% For mathematics journals, theorem styles can be included as shown in the following examples:

% \begin{theorem}[Theorem subhead]\label{thm1}
% Example theorem text. Example theorem text. Example theorem text. Example theorem text. Example theorem text. 
% Example theorem text. Example theorem text. Example theorem text. Example theorem text. Example theorem text. 
% Example theorem text. 
% \end{theorem}

% Sample body text. Sample body text. Sample body text. Sample body text. Sample body text. Sample body text. Sample body text. Sample body text.

% \begin{proposition}
% Example proposition text. Example proposition text. Example proposition text. Example proposition text. Example proposition text. 
% Example proposition text. Example proposition text. Example proposition text. Example proposition text. Example proposition text. 
% \end{proposition}

% Sample body text. Sample body text. Sample body text. Sample body text. Sample body text. Sample body text. Sample body text. Sample body text.

% \begin{example}
% Phasellus adipiscing semper elit. Proin fermentum massa
% ac quam. Sed diam turpis, molestie vitae, placerat a, molestie nec, leo. Maecenas lacinia. Nam ipsum ligula, eleifend
% at, accumsan nec, suscipit a, ipsum. Morbi blandit ligula feugiat magna. Nunc eleifend consequat lorem. 
% \end{example}

% Sample body text. Sample body text. Sample body text. Sample body text. Sample body text. Sample body text. Sample body text. Sample body text.

% \begin{remark}
% Phasellus adipiscing semper elit. Proin fermentum massa
% ac quam. Sed diam turpis, molestie vitae, placerat a, molestie nec, leo. Maecenas lacinia. Nam ipsum ligula, eleifend
% at, accumsan nec, suscipit a, ipsum. Morbi blandit ligula feugiat magna. Nunc eleifend consequat lorem. 
% \end{remark}

% Sample body text. Sample body text. Sample body text. Sample body text. Sample body text. Sample body text. Sample body text. Sample body text.

% \begin{definition}[Definition sub head]
% Example definition text. Example definition text. Example definition text. Example definition text. Example definition text. Example definition text. Example definition text. Example definition text. 
% \end{definition}

% Additionally a predefined ``proof'' environment is available: \verb+\begin{proof}+ \verb+...+ \verb+\end{proof}+. This prints a ``Proof'' head in italic font style and the ``body text'' in roman font style with an open square at the end of each proof environment. 

% \begin{proof}
% Example for proof text. Example for proof text. Example for proof text. Example for proof text. Example for proof text. Example for proof text. Example for proof text. Example for proof text. Example for proof text. Example for proof text. 
% \end{proof}

% Sample body text. Sample body text. Sample body text. Sample body text. Sample body text. Sample body text. Sample body text. Sample body text.

% \begin{proof}[Proof of Theorem~{\upshape\ref{thm1}}]
% Example for proof text. Example for proof text. Example for proof text. Example for proof text. Example for proof text. Example for proof text. Example for proof text. Example for proof text. Example for proof text. Example for proof text. 
% \end{proof}

% \noindent
% For a quote environment, use \verb+\begin{quote}...\end{quote}+
% \begin{quote}
% Quoted text example. Aliquam porttitor quam a lacus. Praesent vel arcu ut tortor cursus volutpat. In vitae pede quis diam bibendum placerat. Fusce elementum
% convallis neque. Sed dolor orci, scelerisque ac, dapibus nec, ultricies ut, mi. Duis nec dui quis leo sagittis commodo.
% \end{quote}

% Sample body text. Sample body text. Sample body text. Sample body text. Sample body text (refer Figure~\ref{fig1}). Sample body text. Sample body text. Sample body text (refer Table~\ref{tab3}). 

% \section{Methods}\label{sec11}

% Topical subheadings are allowed. Authors must ensure that their Methods section includes adequate experimental and characterization data necessary for others in the field to reproduce their work. Authors are encouraged to include RIIDs where appropriate. 

% \textbf{Ethical approval declarations} (only required where applicable) Any article reporting experiment/s carried out on (i)~live vertebrate (or higher invertebrates), (ii)~humans or (iii)~human samples must include an unambiguous statement within the methods section that meets the following requirements: 

% \begin{enumerate}[1.]
% \item Approval: a statement which confirms that all experimental protocols were approved by a named institutional and/or licensing committee. Please identify the approving body in the methods section

% \item Accordance: a statement explicitly saying that the methods were carried out in accordance with the relevant guidelines and regulations

% \item Informed consent (for experiments involving humans or human tissue samples): include a statement confirming that informed consent was obtained from all participants and/or their legal guardian/s
% \end{enumerate}

% If your manuscript includes potentially identifying patient/participant information, or if it describes human transplantation research, or if it reports results of a clinical trial then  additional information will be required. Please visit (\url{https://www.nature.com/nature-research/editorial-policies}) for Nature Portfolio journals, (\url{https://www.springer.com/gp/authors-editors/journal-author/journal-author-helpdesk/publishing-ethics/14214}) for Springer Nature journals, or (\url{https://www.biomedcentral.com/getpublished/editorial-policies\#ethics+and+consent}) for BMC.

% \section{Discussion}\label{sec12}

% Discussions should be brief and focused. In some disciplines use of Discussion or `Conclusion' is interchangeable. It is not mandatory to use both. Some journals prefer a section `Results and Discussion' followed by a section `Conclusion'. Please refer to Journal-level guidance for any specific requirements. 

% \section{Conclusion}\label{sec13}

% Conclusions may be used to restate your hypothesis or research question, restate your major findings, explain the relevance and the added value of your work, highlight any limitations of your study, describe future directions for research and recommendations. 

% In some disciplines use of Discussion or 'Conclusion' is interchangeable. It is not mandatory to use both. Please refer to Journal-level guidance for any specific requirements. 

% \backmatter

% \bmhead{Supplementary information}

% If your article has accompanying supplementary file/s please state so here. 

% Authors reporting data from electrophoretic gels and blots should supply the full unprocessed scans for key as part of their Supplementary information. This may be requested by the editorial team/s if it is missing.

% Please refer to Journal-level guidance for any specific requirements.

% \bmhead{Acknowledgements}

% Acknowledgements are not compulsory. Where included they should be brief. Grant or contribution numbers may be acknowledged.

% Please refer to Journal-level guidance for any specific requirements.

\section*{Declarations}

% Some journals require declarations to be submitted in a standardised format. Please check the Instructions for Authors of the journal to which you are submitting to see if you need to complete this section. If yes, your manuscript must contain the following sections under the heading `Declarations':

\begin{itemize}
\item Funding:  This research received no external funding.
\item Conflict of interest: The author declares no conflict of interest.  
\item Data availability: The datasets analyzed during the current study were obtained from Topcoder.com. Due to licensing restrictions, the data are not publicly available. However, they can be accessed from the author, Turki Alelyani, upon reasonable request and with permission from Topcoder. The data were used under license for this study, and thus, sharing directly through public repositories is not permitted. For further information or access requests, please contact the author.








% \item Ethics approval and consent to participate
% \item Consent for publication
% \item Data availability 
% \item Materials availability
% \item Code availability 
% \item Author contribution
\end{itemize}

% \noindent
% If any of the sections are not relevant to your manuscript, please include the heading and write `Not applicable' for that section. 

%%===================================================%%
%% For presentation purpose, we have included        %%
%% \bigskip command. Please ignore this.             %%
%%===================================================%%
% \bigskip
% \begin{flushleft}%
% Editorial Policies for:

% \bigskip\noindent
% Springer journals and proceedings: \url{https://www.springer.com/gp/editorial-policies}

% \bigskip\noindent
% Nature Portfolio journals: \url{https://www.nature.com/nature-research/editorial-policies}

% \bigskip\noindent
% \textit{Scientific Reports}: \url{https://www.nature.com/srep/journal-policies/editorial-policies}

% \bigskip\noindent
% BMC journals: \url{https://www.biomedcentral.com/getpublished/editorial-policies}
% \end{flushleft}

% \begin{appendices}

% \section{Section title of first appendix}\label{secA1}

% An appendix contains supplementary information that is not an essential part of the text itself but which may be helpful in providing a more comprehensive understanding of the research problem or it is information that is too cumbersome to be included in the body of the paper.

%%=============================================%%
%% For submissions to Nature Portfolio Journals %%
%% please use the heading ``Extended Data''.   %%
%%=============================================%%

%%=============================================================%%
%% Sample for another appendix section			       %%
%%=============================================================%%

%% \section{Example of another appendix section}\label{secA2}%
%% Appendices may be used for helpful, supporting or essential material that would otherwise 
%% clutter, break up or be distracting to the text. Appendices can consist of sections, figures, 
%% tables and equations etc.

% \end{appendices}

%%===========================================================================================%%
%% If you are submitting to one of the Nature Portfolio journals, using the eJP submission   %%
%% system, please include the references within the manuscript file itself. You may do this  %%
%% by copying the reference list from your .bbl file, paste it into the main manuscript .tex %%
%% file, and delete the associated \verb+\bibliography+ commands.                            %%
%%===========================================================================================%%

\bibliography{sn-bibliography}% common bib file
%% if required, the content of .bbl file can be included here once bbl is generated
%%\input sn-article.bbl

\begin{thebibliography}{00}




\bibitem{b1} Begel, J. D. Herbsleb, and M.-A. D. Storey, “The future of collaborative software development.” in CSCW (Companion), S.E. Poltrock, C. Simone, J. Grudin, G. Mark, and J. Riedl, Eds. ACM, 2012 pp. 17–18.

\bibitem{b2} Bhatti, Shahzad Sarwar, Xiaofeng Gao, and Guihai Chen. "General framework, opportunities and challenges for crowdsourcing techniques: A comprehensive survey." Journal of Systems and Software 167 (2020): 110611

\bibitem{b3}  Franzoni, Chiara, and Henry Sauermann. "Crowd science: The organization of scientific research in open collaborative projects." Research policy 43.1 (2014): 1-20

\bibitem{b4} Levine, Diane E., John Mason, and Jennifer Hadley. "Software development and quality assurance." Computer security handbook (2012): 39-1

\bibitem{b5} Cohn, Mike. User stories applied: For agile software development. Addison-Wesley Professional, 2004

\bibitem{b6} Turki Alelyani, Paul T. Grogan, Yla Tausczik and Ye Yang "Software Crowdsourcing Design: An Experiment on the Relationship between Task Design and Crowdsourcing Performance”. “The 22nd International Conference on Human- Computer Interaction, 2020, Copenhagen, Denmark.

\bibitem{b7}Dubey, Alpana, et al. "Dynamics of software development crowdsourcing." 2016 IEEE 11th International Conference on Global Software Engineering (ICGSE). IEEE, 2016.

\bibitem{b8} F. Xia, S. Liu, and J. Zhang, “How Social Subsystem and Technical Subsystem
Risks Influence Crowdsourcing Performance,” PACIS 2015 Proc., Jan. 2015.

\bibitem{b9} Alelyani, Turki. Understanding Design Factors in Software Crowdsourcing. Stevens Institute of Technology, 2020.

\bibitem{b10} J. Rogstadius, V. Kostakos, A. Kittur, B. Smus, J. Laredo, and M. Vukovic, “An Assessment of Intrinsic and Extrinsic Motivation on Task Performance in Crowdsourcing Markets.”

\bibitem{b11}K.-J. Stol and B. Fitzgerald, “Two’s Company, Three’s a Crowd: A Case Study of Crowdsourcing Software Development,” 2014.



\bibitem{b12}P. Kucherbaev, S. Tranquillini, A. Finnerty, and G. Convertino, Keep it Simple: Reward and Task Design in Crowdsourcing, 2013.
\bibitem{b13} T. Alelyani and Y. Yang, “Software crowdsourcing reliability: an empirical study on developers behavior,” in Proceedings of the 2nd International Workshop on Software Analytics - SWAN 2016, 2016, pp. 36–42.

\bibitem{b14} N. Archak, Money, Glory and Cheap Talk: Analyzing Strategic Behavior of Contestants in Simultaneous Crowdsourcing Contests on TopCoder.com, 2010


\bibitem{b15} S. Moussawi and M. Koufaris, “Working on Low-Paid Micro-Task Crowdsourcing Platforms: An Existence, Relatedness and Growth View,” ICIS 2015 Proc., Dec. 2015.

\bibitem{b16} Goodman, J., and Gabriele Paolacci. "Questioning the turk: Conducting high quality research with Amazon mechanical turk." Workshop held at the Association for Consumer Research North American Conference. 2014

\bibitem{b17} Pee, Loo Geok, E. Koh, and Mark Goh. "Trait motivations of crowdsourcing and task choice: A distal-proximal perspective." International Journal of Information Management 40 (2018): 28-41.


\bibitem{b18}Vaz, Luis, Igor Steinmacher, and Sabrina Marczak. "An empirical study on task documentation in software crowdsourcing on topcoder." 2019 ACM/IEEE 14th International Conference on Global Software Engineering (ICGSE). IEEE, 2019.

\bibitem{b19}Das Sarma, Akash, Aditya Parameswaran, and Jennifer Widom. "Towards globally optimal crowdsourcing quality management: The uniform worker setting." Proceedings of the 2016 international conference on management of data. 2016

\bibitem{b20} T. Alelyani, K. Mao, and Y. Yang, “Context-Centric Pricing,” in Proceedings of the 13th International Conference on Predictive Models and Data Analytics in Software Engineering - PROMISE, 2017, pp. 63–72.

\bibitem{b21} K. Mao, Y. Yang, Q. Wang, Y. Jia, and M. Harman, “Developer Recommendation for Crowdsourced Software Development Tasks,” in 2015 IEEE Symposium on Service-Oriented System Engineering, 2015, pp. 347–356.

\bibitem{b22}Sherif, Karma, Robert W. Zmud, and Glenn J. Browne. "Managing peer-to-peer conflicts in disruptive information technology innovations: The case of software reuse." MIS quarterly (2006): 339-356.

\bibitem{b23} F. Xia, S. Liu, and J. Zhang, “How Social Subsystem and Technical Subsystem
Risks Influence Crowdsourcing Performance,” PACIS 2015 Proc., Jan. 2015.


\bibitem{b24} R.  Senter  and  E.  A.  Smith.  Automated  readability  index.  Technical  report,DTIC Document, 1967

\bibitem{b25}J. P. Kincaid,  R. P. Fishburne Jr,  R. L. Rogers,  and B. S. Chissom. Deriva-tion  of  new  readability  formulas  (automated  readability  index,  fog  count  andflesch reading ease formula) for navy enlisted personnel. Technical report, DTICDocument, 1975

\bibitem{b26}M. Coleman and T. L. Liau. A computer readability formula designed for ma-chine scoring. Journal of Applied Psychology, 60(2):283, 1975

\bibitem{27} G. H. Mc Laughlin. Smog grading-a new readability formula. Journal of reading,12(8):639646, 1969

\bibitem{b28} McCallum, Andrew Kachites. ”MALLET: A Machine Learning for Language Toolkit.” http://mallet.cs.umass.edu. 2002.

\bibitem{b29} Menzies, Tim, and Andrian Marcus. ”Automated severity assessment of softwaredefect reports.” Software Maintenance, 2008. ICSM 2008. IEEE International Con-ference on. IEEE, 2008.

\bibitem{b30}Asaduzzaman, Muhammad, et al. ”Answering questions about unanswered questions of stack overflow.” Proceedings of the 10th Working Conference on Mining Software Repositories. IEEE Press, 2013

\bibitem{b31}Ikeda, Kazushi, and Michael S. Bernstein. "Pay it backward: Per-task payments on crowdsourcing platforms reduce productivity." Proceedings of the 2016 CHI Conference on Human Factors in Computing Systems. 2016.

\end{thebibliography}




\end{document}
