diff --git a/_quarto.yml b/_quarto.yml index 13ba2d9..2f49402 100644 --- a/_quarto.yml +++ b/_quarto.yml @@ -77,6 +77,7 @@ website: - indicator_templates/quarto/2_academic_impact/extra-academic_collaboration.qmd - indicator_templates/quarto/2_academic_impact/interdisciplinarity.qmd - indicator_templates/quarto/2_academic_impact/novelty.qmd + - indicator_templates/quarto/2_academic_impact/quality.qmd - indicator_templates/quarto/2_academic_impact/use_of_code_in_research.qmd - indicator_templates/quarto/2_academic_impact/use_of_data_in_research.qmd diff --git a/indicator_templates/quarto/2_academic_impact/quality.qmd b/indicator_templates/quarto/2_academic_impact/quality.qmd new file mode 100644 index 0000000..ef2a142 --- /dev/null +++ b/indicator_templates/quarto/2_academic_impact/quality.qmd @@ -0,0 +1,36 @@ +--- +author: + - name: V.A Traag + orcid: 0000-0003-3170-3879 + affiliations: + - ref: cwts + +affiliations: +- id: cwts + name: Leiden University + department: Centre for Science and Technology Studies + city: Leiden + country: the Netherlands +--- + +# Quality {#quality .unnumbered} + +
+ +## History + +| Version | Revision date | Revision | Author | +|---------|---------------|-------------|------------| +| 1.0 | 2024-12-06 | First draft | V.A. Traag | + +
+ +## Description + +Quality is a very complicated concept, and in the context of academic work, is very challenging to measure. To start with, it should be clarified what object is being considered, which could range from data to peer review. In most such cases, quality cannot be defined on the basis of any easily measurable data, and rather require some form of manual assessment. + +For the most traditional academic output, a scholarly publication, such a manual assessment is typically provided through peer review [@bornmann_scientific_2011]. Peer review is much discussed in science studies, and there are discussions about its reliability [@cole_chance_1981] and its biases [@lee_bias_2013], but also about its positive effects [@goodman_manuscript_1994] and complementaries [@goyal_causal_2024]. + +Quality is typically considered to be a multidimensional concept [@aksnes2019], composed of various other concepts. For instance, in peer review of manuscripts submitted to journals, it is common to assess the novelty and the rigour of the manuscript. Yet even if quality is considered a multidimensional concept, in practice, quality is sometimes still considered to be unidimensional. For example, in the [UK REF](https://www.ref.ac.uk/) research articles are assigned a number of stars, varying from "recognised nationally" (1 star) to "world-leading" (4 stars). + +In the context of exercises such as the UK REF there have also been discussions about the possibility to use citations as a proxy for quality. Indeed, there are substantial correlations between peer review results and citations, but this depends on the level of aggregation. At the individual paper level the correlation is typically low, yet at higher levels, such as institutional, the correlations are substantially higher [@traag_metrics_2023]. Overall, as summarised in the reputable "Metrics Tide" report [@wilsdon_metric_2015, viii], "Metrics should support, not supplant, expert judgement.", and this is particularly relevant at the individual paper level. diff --git a/references.bib b/references.bib index 8f322b5..31072a3 100644 --- a/references.bib +++ b/references.bib @@ -152,6 +152,19 @@ @book{bellis2009 langid = {en} } +@article{bornmann_scientific_2011, + title = {Scientific peer review}, + volume = {45}, + issn = {00664200}, + doi = {10.1002/aris.2011.1440450112}, + number = {1}, + urldate = {2023-03-14}, + journal = {Annual Review of Information Science and Technology}, + author = {Bornmann, Lutz}, + year = {2011}, + pages = {197--245} +} + @article{bornmann2016, title = {Normalization of Mendeley reader impact on the reader-and paper-side: A comparison of the mean discipline normalized reader score (MDNRS) with the mean normalized reader score (MNRS) and bare reader counts}, author = {Bornmann, Lutz and Haunschild, Robin}, @@ -213,6 +226,7 @@ @article{brembs2019 url = {http://dx.doi.org/10.1371/journal.pbio.3000117} } + @techreport{brown2016, title = {The GRIM test: A simple technique detects numerous anomalies in the reporting of results in psychology}, author = {Brown, Nicholas J. L. and Heathers, James A.}, @@ -237,7 +251,6 @@ @article{bryan2021 note = {Publisher: MIT Press One Rogers Street, Cambridge, MA 02142-1209, USA journals-info {\ldots}} } - @article{budi2022, title = {Understanding the meanings of citations using sentiment, role, and citation function classifications}, author = {Budi, Indra and Yaniasih, Yaniasih}, @@ -340,6 +353,7 @@ @misc{codeof langid = {en} } + @article{cohen2002, title = {Links and Impacts: The Influence of Public Research on Industrial R&D}, author = {Cohen, Wesley M. and Nelson, Richard R. and Walsh, John P.}, @@ -355,6 +369,7 @@ @article{cohen2002 langid = {en} } + @article{colavizza2020, title = {The citation advantage of linking publications to research data}, author = {Colavizza, Giovanni and Hrynaszkiewicz, Iain and Staden, Isla and Whitaker, Kirstie and McGillivray, Barbara}, @@ -371,6 +386,18 @@ @article{colavizza2020 langid = {en} } +@article{cole_chance_1981, + title = {Chance and consensus in peer review}, + volume = {214}, + issn = {0036-8075}, + doi = {10.1126/science.7302566}, + number = {4523}, + journal = {Science}, + author = {Cole, Stephen and Cole, J R and Simon, G A}, + year = {1981}, + pmid = {7302566}, + pages = {881--886} +} @article{cole_societal_2024, title = {The societal impact of {Open} {Science}: a scoping review}, @@ -388,7 +415,6 @@ @article{cole_societal_2024 pages = {240286} } - @book{cost-ben2018, title = {Cost-benefit analysis for FAIR research data: cost of not having FAIR research data}, year = {2018}, @@ -850,6 +876,19 @@ @article{goodman_manuscript_1994 pages = {11--21} } +@article{goodman_manuscript_1994, + title = {Manuscript {Quality} before and after {Peer} {Review} and {Editing} at {Annals} of {Internal} {Medicine}}, + volume = {121}, + issn = {0003-4819}, + doi = {10.7326/0003-4819-121-1-199407010-00003}, + number = {1}, + journal = {Ann. Intern. Med.}, + author = {Goodman, Steven N and Berlin, Jesse and Fletcher, Suzanne W and Fletcher, Robert H}, + month = jul, + year = {1994}, + pages = {11} +} + @article{goodman2016, title = {What does research reproducibility mean?}, author = {Goodman, Steven N. and Fanelli, Daniele and Ioannidis, John P. A.}, @@ -880,6 +919,17 @@ @article{gordon2021 langid = {en} } +@misc{goyal_causal_2024, + title = {Causal {Effect} of {Group} {Diversity} on {Redundancy} and {Coverage} in {Peer}-{Reviewing}}, + doi = {10.48550/arXiv.2411.11437}, + abstract = {A large host of scientific journals and conferences solicit peer reviews from multiple reviewers for the same submission, aiming to gather a broader range of perspectives and mitigate individual biases. In this work, we reflect on the role of diversity in the slate of reviewers assigned to evaluate a submitted paper as a factor in diversifying perspectives and improving the utility of the peer-review process. We propose two measures for assessing review utility: review coverage—reviews should cover most contents of the paper—and review redundancy—reviews should add information not already present in other reviews. We hypothesize that reviews from diverse reviewers will exhibit high coverage and low redundancy. We conduct a causal study of different measures of reviewer diversity on review coverage and redundancy using observational data from a peer-reviewed conference with approximately 5,000 submitted papers. Our study reveals disparate effects of different diversity measures on review coverage and redundancy. Our study finds that assigning a group of reviewers that are topically diverse, have different seniority levels, or have distinct publication networks leads to broader coverage of the paper or review criteria, but we find no evidence of an increase in coverage for reviewer slates with reviewers from diverse organizations or geographical locations. Reviewers from different organizations, seniority levels, topics, or publications networks (all except geographical diversity) lead to a decrease in redundancy in reviews. Furthermore, publication network-based diversity alone also helps bring in varying perspectives (that is, low redundancy), even within specific review criteria. Our study adopts a group decision-making perspective for reviewer assignments in peer review and suggests dimensions of diversity that can help guide the reviewer assignment process.}, + language = {en}, + publisher = {arXiv}, + author = {Goyal, Navita and Stelmakh, Ivan and Shah, Nihar and III, Hal Daumé}, + month = nov, + year = {2024} +} + @article{grimme, title = {The State of Open Monographs}, author = {Grimme, Sara and Holland, Cathy and Potter, Peter and Taylor, Mike and Watkinson, Charles}, @@ -1003,6 +1053,7 @@ @misc{hunermund_causal_2023 year = {2023} } + @inproceedings{hunter2015, title = {Formal Acknowledgement of Citizen Scientists{\textquoteright} Contributions via Dynamic Data Citations}, author = {Hunter, Jane and Hsu, Chih-Hsiang}, @@ -1107,7 +1158,6 @@ @book{jung2023 url = {https://cran.r-project.org/web/packages/scrutiny/index.html} } - @article{keller2014, title = {Re-use of public sector information in cultural heritage institutions}, author = {Keller, Paul and Margoni, Thomas and Rybicka, Katarzyna and Tarkowski, Alek}, @@ -1342,6 +1392,20 @@ @article{laugksch2000a langid = {en} } +@article{lee_bias_2013, + title = {Bias in peer review}, + volume = {64}, + issn = {15322882}, + doi = {10.1002/asi.22784}, + number = {1}, + urldate = {2023-03-14}, + journal = {Journal of the American Society for Information Science and Technology}, + author = {Lee, Carole J. and Sugimoto, Cassidy R. and Zhang, Guo and Cronin, Blaise}, + month = jan, + year = {2013}, + pages = {2--17} +} + @article{lee2015, title = {Open access target validation is a more efficient way to accelerate drug discovery}, author = {Lee, Wen Hwa}, @@ -1947,6 +2011,7 @@ @article{radicchi2008 url = {http://www.pnas.org/cgi/content/abstract/105/45/17268} } + @article{ràfols2020, title = {{\textquoteleft}Measuring{\textquoteright}interdisciplinarity: from indicators to indicating}, author = {{Ràfols}, Ismael}, @@ -2009,7 +2074,6 @@ @article{robinson-garcia2017 url = {https://www.sciencedirect.com/science/article/pii/S1751157717300834} } - @article{robinson-garcia2020, title = {Open Access uptake by universities worldwide}, author = {Robinson-Garcia, Nicolas and Costas, Rodrigo and van Leeuwen, Thed N.}, @@ -2273,6 +2337,15 @@ @misc{traag_causal_2022 year = {2022} } +@misc{traag_metrics_2023, + title = {Metrics and peer review agreement at the institutional level}, + doi = {10.48550/arXiv.2006.14830}, + publisher = {arXiv}, + author = {Traag, V. A. and Malgarini, M. and Sarlo, S.}, + month = mar, + year = {2023} +} + @article{traag2021, title = {Inferring the causal effect of journals on citations}, author = {Traag, V. A.}, @@ -2384,6 +2457,7 @@ @article{wang2020 note = {Publisher: MIT Press One Rogers Street, Cambridge, MA 02142-1209, USA journals-info {\ldots}} } + @article{westreich2013, title = {The Table 2 Fallacy: Presenting and Interpreting Confounder and Modifier Coefficients}, author = {Westreich, Daniel and Greenland, Sander}, @@ -2404,6 +2478,7 @@ @misc{whatper langid = {en} } + @article{wilkinson2016, title = {The FAIR Guiding Principles for scientific data management and stewardship}, author = {Wilkinson, Mark D. and Dumontier, Michel and Aalbersberg, IJsbrand Jan and Appleton, Gabrielle and Axton, Myles and Baak, Arie and Blomberg, Niklas and Boiten, Jan-Willem and da Silva Santos, Luiz Bonino and Bourne, Philip E. and Bouwman, Jildau and Brookes, Anthony J. and Clark, Tim and Crosas, {Mercè} and Dillo, Ingrid and Dumon, Olivier and Edmunds, Scott and Evelo, Chris T. and Finkers, Richard and Gonzalez-Beltran, Alejandra and Gray, Alasdair J. G. and Groth, Paul and Goble, Carole and Grethe, Jeffrey S. and Heringa, Jaap and {{\textquoteright}t Hoen}, Peter A. C. and Hooft, Rob and Kuhn, Tobias and Kok, Ruben and Kok, Joost and Lusher, Scott J. and Martone, Maryann E. and Mons, Albert and Packer, Abel L. and Persson, Bengt and Rocca-Serra, Philippe and Roos, Marco and van Schaik, Rene and Sansone, Susanna-Assunta and Schultes, Erik and Sengstag, Thierry and Slater, Ted and Strawn, George and Swertz, Morris A. and Thompson, Mark and van der Lei, Johan and van Mulligen, Erik and Velterop, Jan and Waagmeester, Andra and Wittenburg, Peter and Wolstencroft, Katherine and Zhao, Jun and Mons, Barend}, @@ -2443,6 +2518,17 @@ @article{wilner langid = {en-us} } + +@techreport{wilsdon_metric_2015, + title = {Metric {Tide}: {Report} of the {Independent} {Review} of the {Role} of {Metrics} in {Research} {Assessment} and {Management}}, + institution = {Higher Education Funding Council for England}, + author = {Wilsdon, James and Allen, Liz and Belfiore, Eleonora and Campbell, Philip and Curry, Stephen and Hill, Steven and Jones, Richard and Kain, Roger and Kerridge, Simon and Thelwall, Mike and Tinkler, Jane and Viney, Ian and Wouters, Paul and Hill, Jude and Johnson, Ben}, + year = {2015}, + doi = {10.13140/RG.2.1.4929.1363}, + pages = {163} +} + + @book{wood2021, title = {CORVIDS}, author = {Wood, Katherine}, @@ -2482,6 +2568,7 @@ @article{wuchty2007 langid = {en} } + @article{yarkoni2019, title = {The Generalizability Crisis}, author = {Yarkoni, Tal}, @@ -2492,7 +2579,6 @@ @article{yarkoni2019 url = {https://psyarxiv.com/jqw35/} } - @article{zahedi2017, title = {Mendeley readership as a filtering tool to identify highly cited publications}, author = {Zahedi, Zohreh and Costas, Rodrigo and Wouters, Paul}, @@ -2507,6 +2593,8 @@ @article{zahedi2017 note = {Publisher: Wiley} } + + @article{zahedi2020, title = {Do Online Readerships Offer Useful Assessment Tools? Discussion Around the Practical Applications of Mendeley Readership for Scholarly Assessment}, author = {Zahedi, Zohreh and Costas, Rodrigo},