@techreport{BrodeurMikolaCooketal.2024, type = {Working Paper}, author = {Brodeur, Abel and Mikola, Derek and Cook, Nikolai and Brailey, Thomas and Briggs, Ryan and Gendre, Alexandra de and Dupraz, Yannick and Fiala, Lenka and Gabani, Jacopo and Gauriot, Romain and Haddad, Joanne and Lima, Goncalo and Ankel-Peters, J{\"o}rg and Dreber, Anna and Campbell, Douglas and Kattan, Lamis and Fages, Diego Marino and Mierisch, Fabian and Sun, Pu and Wright, Taylor and Connolly, Marie and Hoces de la Guardia, Fernando and Johannesson, Magnus and Miguel, Edward and Vilhuber, Lars and Abarca, Alejandro and Acharya, Mahesh and Adjisse, Sossou Simplice and Akhtar, Ahwaz and Lizardi, Eduardo Alberto Ramirez and Albrecht, Sabina and Andersen, Synve Nygaard and Andlib, Zubaria and Arrora, Falak and Ash, Thomas and Bacher, Etienne and Bachler, Sebastian and Bacon, F{\´e}lix and Bagues, Manuel and Balogh, Timea and Batmanov, Alisher and Barschkett, Mara and Basdil, B. Kaan and Dower, Jaromneda and Castek, Ondrej and Caviglia-Harris, Jill and Strand, Gabriella Chauca and Chen, Shi and Chzhen, Asya and Chung, Jong and Collins, Jason and Coppock, Alexander and Cordeau, Hugo and Couillard, Ben and Crechet, Jonathan and Crippa, Lorenzo and Cui, Jeanne and Czymara, Christian and Daarstad, Haley and Dao, Danh Chi and Dao, Dong and Schmandt, Marco David and Linde, Astrid de and Melo, Lucas De and Deer, Lachlan and Vera, Micole De and Dimitrova, Velichka and Dollbaum, Jan Fabian and Dollbaum, Jan Matti and Donnelly, Michael and Huynh, Luu Duc Toan and Dumbalska, Tsvetomira and Duncan, Jamie and Duong, Kiet Tuan and Duprey, Thibaut and Dworschak, Christoph and Ellingsrud, Sigmund and Elminejad, Ali and Eissa, Yasmine and Erhart, Andrea and Etingin-Frati, Giulian and Fatemi-Pour, Elaheh and Federice, Alexa and Feld, Jan and Fenig, Guidon and Firouzjaeiangalougah, Mojtaba and Fleisje, Erlend and Fortier-Chouinard, Alexandre and Engel, Julia Francesca and Fries, Tilman and Fortier, Reid and Fr{\´e}chet, Nadjim and Galipeau, Thomas and Gallegos, Sebasti{\´a}n and Gangji, Areez and Gao, Xiaoying and Garnache, Clo{\´e} and G{\´a}sp{\´a}r, Attila and Gavrilova, Evelina and Ghosh, Arijit and Gibney, Garreth and Gibson, Grant and Godager, Geir and Goff, Leonard and Gong, Da and Gonz{\´a}lez, Javier and Gretton, Jeremy and Griffa, Cristina and Grigoryeva, Idaliya and Grtting, Maja and Guntermann, Eric and Guo, Jiaqi and Gugushvili, Alexi and Habibnia, Hooman and H{\"a}ffner, Sonja and Hall, Jonathan D. and Hammar, Olle and Kordt, Amund Hanson and Hashimoto, Barry and Hartley, Jonathan S. and Hausladen, Carina I. and Havr{\´a}nek, Tom{\´a}š and Hazen, Jacob and He, Harry and Hepplewhite, Matthew and Herrera-Rodriguez, Mario and Heuer, Felix and Heyes, Anthony and Ho, Anson T. Y. and Holmes, Jonathan and Holzknecht, Armando and Hsu, Yu-Hsiang Dexter and Hu, Shiang-Hung and Huang, Yu-Shiuan and Huebener, Mathias and Huber, Christoph and Huynh, Kim P. and Irsova, Zuzana and Isler, Ozan and Jakobsson, Niklas and Frith, Michael James and Jananji, Rapha{\"e}l and Jayalath, Tharaka A. and Jetter, Michael and John, Jenny and Forshaw, Rachel Joy and Juan, Felipe and Kadriu, Valon and Karim, Sunny and Kelly, Edmund and Dang, Duy Khanh Hoang and Khushboo, Tazia and Kim, Jin and Kjellsson, Gustav and Kjelsrud, Anders and Kotsadam, Andreas and Korpershoek, Jori and Krashinsky, Lewis and Kundu, Suranjana and Kustov, Alexander and Lalayev, Nurlan and Langlois, Audr{\´e}e and Laufer, Jill and Lee-Whiting, Blake and Leibing, Andreas and Lenz, Gabriel and Levin, Joel and Li, Peng and Li, Tongzhe and Lin, Yuchen and Listo, Ariel and Liu, Dan and Lu, Xuewen and Lukmanova, Elvina and Luscombe, Alex and Lusher, Lester R. and Lyu, Ke and Ma, Hai and M{\"a}der, Nicolas and Makate, Clifton and Malmberg, Alice and Maitra, Adit and Mandas, Marco and Marcus, Jan and Margaryan, Shushanik and M{\´a}rk, Lili and Martignano, Andres and Marsh, Abigail and Masetto, Isabella and McCanny, Anthony and McManus, Emma and McWay, Ryan and Metson, Lennard and Kinge, Jonas Minet and Mishra, Sumit and Mohnen, Myra and M{\"o}ller, Jakob and Montambeault, Rosalie and Montpetit, S{\´e}bastien and Morin, Louis-Philippe and Morris, Todd and Moser, Scott and Motoki, Fabio and Muehlenbachs, Lucija and Musulan, Andreea and Musumeci, Marco and Nabin, Munirul and Nchare, Karim and Neubauer, Florian and Nguyen, Quan M. P. and Nguyen, Tuan and Nguyen-Tien, Viet and Niazi, Ali and Nikolaishvili, Giorgi and Nordstrom, Ardyn and N{\"u}, Patrick and Odermatt, Angela and Olson, Matt and ien, Henning and {\"O}lkers, Tim and Vert, Miquel Oliver i. and Oral, Emre and Oswald, Christian and Ousman, Ali and {\"O}zak, {\"O}mer and Pandey, Shubham and Pavlov, Alexandre and Pelli, Martino and Penheiro, Romeo and Park, RyuGyung and Martel, Eva P{\´e}rez and Petrovičov{\´a}, Tereza and Phan, Linh and Prettyman, Alexa and Proch{\´a}zka, Jakub and Putri, Aqila and Quandt, Julian and Qiu, Kangyu and Nguyen, Loan Quynh Thi and Rahman, Andaleeb and Rea, Carson H. and Reiremo, Adam and Ren{\´e}e, La{\"e}titia and Richardson, Joseph and Rivers, Nicholas and Rodrigues, Bruno and Roelofs, William and Roemer, Tobias and Rogeberg, Ole and Rose, Julian and Roskos-Ewoldsen, Andrew and Rosmer, Paul and Sabada, Barbara and Saberian, Soodeh and Salamanca, Nicolas and Sator, Georg and Sawyer, Antoine and Scates, Daniel and Schl{\"u}ter, Elmar and Sells, Cameron and Sen, Sharmi and Sethi, Ritika and Shcherbiak, Anna and Sogaolu, Moyosore and Soosalu, Matt and Srensen, Erik and Sovani, Manali and Spencer, Noah and Staubli, Stefan and Stans, Renske and Stewart, Anya and Stips, Felix and Stockley, Kieran and Strobel, Stephenson and Struby, Ethan and Tang, John and Tanrisever, Idil and Yang, Thomas Tao and Tastan, Ipek and Tatić, Dejan and Tatlow, Benjamin and Seuyong, F{\´e}raud Tchuisseu and Th{\´e}riault, R{\´e}mi and Thivierge, Vincent and Tian, Wenjie and Toma, Filip-Mihai and Totarelli, Maddalena and Tran, Van-Anh and Truong, Hung and Tsoy, Nikita and Tuzcuoglu, Kerem and Ubfal, Diego and Villalobos, Laura and Walterskirchen, Julian and Wang, Joseph Taoyi and Wattal, Vasudha and Webb, Matthew D. and Weber, Bryan and Weisser, Reinhard and Weng, Wei-Chien and Westheide, Christian and White, Kimberly and Winter, Jacob and Wochner, Timo and Woerman, Matt and Wong, Jared and Woodard, Ritchie and Wroński, Marcin and Yazbeck, Myra and Yang, Gustav Chung and Yap, Luther and Yassin, Kareman and Ye, Hao and Yoon, Jin Young and Yurris, Chris and Zahra, Tahreen and Zaneva, Mirela and Zayat, Aline and Zhang, Jonathan and Zhao, Ziwei and Yaolang, Zhong}, title = {Mass reproducibility and replicability}, series = {I4R discussion paper series}, journal = {I4R discussion paper series}, number = {107}, publisher = {Institute for Replication}, address = {Essen}, issn = {2752-1931}, pages = {250}, year = {2024}, abstract = {This study pushes our understanding of research reliability by reproducing and replicating claims from 110 papers in leading economic and political science journals. The analysis involves computational reproducibility checks and robustness assessments. It reveals several patterns. First, we uncover a high rate of fully computationally reproducible results (over 85\%). Second, excluding minor issues like missing packages or broken pathways, we uncover coding errors for about 25\% of studies, with some studies containing multiple errors. Third, we test the robustness of the results to 5,511 re-analyses. We find a robustness reproducibility of about 70\%. Robustness reproducibility rates are relatively higher for re-analyses that introduce new data and lower for re-analyses that change the sample or the definition of the dependent variable. Fourth, 52\% of re-analysis effect size estimates are smaller than the original published estimates and the average statistical significance of a re-analysis is 77\% of the original. Lastly, we rely on six teams of researchers working independently to answer eight additional research questions on the determinants of robustness reproducibility. Most teams find a negative relationship between replicators' experience and reproducibility, while finding no relationship between reproducibility and the provision of intermediate or even raw data combined with the necessary cleaning codes.}, language = {en} } @article{KhiderEmileGeayMcKayetal.2019, author = {Khider, D. and Emile-Geay, J. and McKay, N. P. and Gil, Y. and Garijo, D. and Ratnakar, V and Alonso-Garcia, M. and Bertrand, S. and Bothe, O. and Brewer, P. and Bunn, A. and Chevalier, M. and Comas-Bru, L. and Csank, A. and Dassie, E. and DeLong, K. and Felis, T. and Francus, P. and Frappier, A. and Gray, W. and Goring, S. and Jonkers, L. and Kahle, M. and Kaufman, D. and Kehrwald, N. M. and Martrat, B. and McGregor, H. and Richey, J. and Schmittner, A. and Scroxton, N. and Sutherland, E. and Thirumalai, Kaustubh and Allen, K. and Arnaud, F. and Axford, Y. and Barrows, T. and Bazin, L. and Birch, S. E. Pilaar and Bradley, E. and Bregy, J. and Capron, E. and Cartapanis, O. and Chiang, H-W and Cobb, K. M. and Debret, M. and Dommain, R{\´e}ne and Du, J. and Dyez, K. and Emerick, S. and Erb, M. P. and Falster, G. and Finsinger, W. and Fortier, D. and Gauthier, Nicolas and George, S. and Grimm, E. and Hertzberg, J. and Hibbert, F. and Hillman, A. and Hobbs, W. and Huber, M. and Hughes, A. L. C. and Jaccard, S. and Ruan, J. and Kienast, M. and Konecky, B. and Le Roux, G. and Lyubchich, V and Novello, V. F. and Olaka, L. and Partin, J. W. and Pearce, C. and Phipps, S. J. and Pignol, C. and Piotrowska, N. and Poli, M-S and Prokopenko, A. and Schwanck, F. and Stepanek, C. and Swann, G. E. A. and Telford, R. and Thomas, E. and Thomas, Z. and Truebe, S. and von Gunten, L. and Waite, A. and Weitzel, N. and Wilhelm, B. and Williams, J. and Winstrup, M. and Zhao, N. and Zhou, Y.}, title = {PaCTS 1.0: A Crowdsourced Reporting Standard for Paleoclimate Data}, series = {Paleoceanography and paleoclimatology}, volume = {34}, journal = {Paleoceanography and paleoclimatology}, number = {10}, publisher = {American Geophysical Union}, address = {Washington}, issn = {2572-4517}, doi = {10.1029/2019PA003632}, pages = {1570 -- 1596}, year = {2019}, abstract = {The progress of science is tied to the standardization of measurements, instruments, and data. This is especially true in the Big Data age, where analyzing large data volumes critically hinges on the data being standardized. Accordingly, the lack of community-sanctioned data standards in paleoclimatology has largely precluded the benefits of Big Data advances in the field. Building upon recent efforts to standardize the format and terminology of paleoclimate data, this article describes the Paleoclimate Community reporTing Standard (PaCTS), a crowdsourced reporting standard for such data. PaCTS captures which information should be included when reporting paleoclimate data, with the goal of maximizing the reuse value of paleoclimate data sets, particularly for synthesis work and comparison to climate model simulations. Initiated by the LinkedEarth project, the process to elicit a reporting standard involved an international workshop in 2016, various forms of digital community engagement over the next few years, and grassroots working groups. Participants in this process identified important properties across paleoclimate archives, in addition to the reporting of uncertainties and chronologies; they also identified archive-specific properties and distinguished reporting standards for new versus legacy data sets. This work shows that at least 135 respondents overwhelmingly support a drastic increase in the amount of metadata accompanying paleoclimate data sets. Since such goals are at odds with present practices, we discuss a transparent path toward implementing or revising these recommendations in the near future, using both bottom-up and top-down approaches.}, language = {en} }