{"id":"https://openalex.org/W4366588833","doi":"https://doi.org/10.1145/3544548.3581555","title":"Faulty or Ready? Handling Failures in Deep-Learning Computer Vision Models until Deployment: A Study of Practices, Challenges, and Needs","display_name":"Faulty or Ready? Handling Failures in Deep-Learning Computer Vision Models until Deployment: A Study of Practices, Challenges, and Needs","publication_year":2023,"publication_date":"2023-04-19","ids":{"openalex":"https://openalex.org/W4366588833","doi":"https://doi.org/10.1145/3544548.3581555"},"language":"en","primary_location":{"id":"doi:10.1145/3544548.3581555","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3544548.3581555","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3544548.3581555","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 2023 CHI Conference on Human Factors in Computing Systems","raw_type":"proceedings-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"gold","oa_url":"https://dl.acm.org/doi/pdf/10.1145/3544548.3581555","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5088069213","display_name":"Agathe Balayn","orcid":"https://orcid.org/0000-0003-2725-5305"},"institutions":[{"id":"https://openalex.org/I98358874","display_name":"Delft University of Technology","ror":"https://ror.org/02e2c7k09","country_code":"NL","type":"education","lineage":["https://openalex.org/I98358874"]}],"countries":["NL"],"is_corresponding":false,"raw_author_name":"Agathe Balayn","raw_affiliation_strings":["Software Technology, Delft University of Technology, Netherlands"],"raw_orcid":"https://orcid.org/0000-0003-2725-5305","affiliations":[{"raw_affiliation_string":"Software Technology, Delft University of Technology, Netherlands","institution_ids":["https://openalex.org/I98358874"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5020739145","display_name":"Natasa Rikalo","orcid":"https://orcid.org/0000-0001-8261-4652"},"institutions":[{"id":"https://openalex.org/I98358874","display_name":"Delft University of Technology","ror":"https://ror.org/02e2c7k09","country_code":"NL","type":"education","lineage":["https://openalex.org/I98358874"]}],"countries":["NL"],"is_corresponding":false,"raw_author_name":"Natasa Rikalo","raw_affiliation_strings":["Sustainable Design Engineering, TU Delft, Netherlands"],"raw_orcid":"https://orcid.org/0000-0001-8261-4652","affiliations":[{"raw_affiliation_string":"Sustainable Design Engineering, TU Delft, Netherlands","institution_ids":["https://openalex.org/I98358874"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5029123550","display_name":"Jie Yang","orcid":"https://orcid.org/0000-0002-0350-0313"},"institutions":[{"id":"https://openalex.org/I98358874","display_name":"Delft University of Technology","ror":"https://ror.org/02e2c7k09","country_code":"NL","type":"education","lineage":["https://openalex.org/I98358874"]}],"countries":["NL"],"is_corresponding":false,"raw_author_name":"Jie Yang","raw_affiliation_strings":["Delft University of Technology, Netherlands"],"raw_orcid":"https://orcid.org/0000-0002-0350-0313","affiliations":[{"raw_affiliation_string":"Delft University of Technology, Netherlands","institution_ids":["https://openalex.org/I98358874"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5028101938","display_name":"Alessandro Bozzon","orcid":"https://orcid.org/0000-0002-3300-2913"},"institutions":[{"id":"https://openalex.org/I98358874","display_name":"Delft University of Technology","ror":"https://ror.org/02e2c7k09","country_code":"NL","type":"education","lineage":["https://openalex.org/I98358874"]}],"countries":["NL"],"is_corresponding":false,"raw_author_name":"Alessandro Bozzon","raw_affiliation_strings":["Industrial Design Engineering, Delft University of Technology, Netherlands"],"raw_orcid":"https://orcid.org/0000-0002-3300-2913","affiliations":[{"raw_affiliation_string":"Industrial Design Engineering, Delft University of Technology, Netherlands","institution_ids":["https://openalex.org/I98358874"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":4,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":1.6316,"has_fulltext":true,"cited_by_count":10,"citation_normalized_percentile":{"value":0.86539611,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":90,"max":98},"biblio":{"volume":null,"issue":null,"first_page":"1","last_page":"20"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11689","display_name":"Adversarial Robustness in Machine Learning","score":0.9987999796867371,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11689","display_name":"Adversarial Robustness in Machine Learning","score":0.9987999796867371,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12026","display_name":"Explainable Artificial Intelligence (XAI)","score":0.9922000169754028,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10883","display_name":"Ethics and Social Impacts of AI","score":0.9907000064849854,"subfield":{"id":"https://openalex.org/subfields/3311","display_name":"Safety Research"},"field":{"id":"https://openalex.org/fields/33","display_name":"Social Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/software-deployment","display_name":"Software deployment","score":0.8331129550933838},{"id":"https://openalex.org/keywords/workflow","display_name":"Workflow","score":0.8218557834625244},{"id":"https://openalex.org/keywords/exploit","display_name":"Exploit","score":0.8063567876815796},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.7701934576034546},{"id":"https://openalex.org/keywords/identification","display_name":"Identification (biology)","score":0.6340014934539795},{"id":"https://openalex.org/keywords/process","display_name":"Process (computing)","score":0.6249485015869141},{"id":"https://openalex.org/keywords/empirical-research","display_name":"Empirical research","score":0.4719112515449524},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.4555280804634094},{"id":"https://openalex.org/keywords/deep-learning","display_name":"Deep learning","score":0.4292127788066864},{"id":"https://openalex.org/keywords/data-science","display_name":"Data science","score":0.4206918776035309},{"id":"https://openalex.org/keywords/software-engineering","display_name":"Software engineering","score":0.3608103394508362},{"id":"https://openalex.org/keywords/computer-security","display_name":"Computer security","score":0.17536437511444092}],"concepts":[{"id":"https://openalex.org/C105339364","wikidata":"https://www.wikidata.org/wiki/Q2297740","display_name":"Software deployment","level":2,"score":0.8331129550933838},{"id":"https://openalex.org/C177212765","wikidata":"https://www.wikidata.org/wiki/Q627335","display_name":"Workflow","level":2,"score":0.8218557834625244},{"id":"https://openalex.org/C165696696","wikidata":"https://www.wikidata.org/wiki/Q11287","display_name":"Exploit","level":2,"score":0.8063567876815796},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7701934576034546},{"id":"https://openalex.org/C116834253","wikidata":"https://www.wikidata.org/wiki/Q2039217","display_name":"Identification (biology)","level":2,"score":0.6340014934539795},{"id":"https://openalex.org/C98045186","wikidata":"https://www.wikidata.org/wiki/Q205663","display_name":"Process (computing)","level":2,"score":0.6249485015869141},{"id":"https://openalex.org/C120936955","wikidata":"https://www.wikidata.org/wiki/Q2155640","display_name":"Empirical research","level":2,"score":0.4719112515449524},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.4555280804634094},{"id":"https://openalex.org/C108583219","wikidata":"https://www.wikidata.org/wiki/Q197536","display_name":"Deep learning","level":2,"score":0.4292127788066864},{"id":"https://openalex.org/C2522767166","wikidata":"https://www.wikidata.org/wiki/Q2374463","display_name":"Data science","level":1,"score":0.4206918776035309},{"id":"https://openalex.org/C115903868","wikidata":"https://www.wikidata.org/wiki/Q80993","display_name":"Software engineering","level":1,"score":0.3608103394508362},{"id":"https://openalex.org/C38652104","wikidata":"https://www.wikidata.org/wiki/Q3510521","display_name":"Computer security","level":1,"score":0.17536437511444092},{"id":"https://openalex.org/C77088390","wikidata":"https://www.wikidata.org/wiki/Q8513","display_name":"Database","level":1,"score":0.0},{"id":"https://openalex.org/C111919701","wikidata":"https://www.wikidata.org/wiki/Q9135","display_name":"Operating system","level":1,"score":0.0},{"id":"https://openalex.org/C138885662","wikidata":"https://www.wikidata.org/wiki/Q5891","display_name":"Philosophy","level":0,"score":0.0},{"id":"https://openalex.org/C111472728","wikidata":"https://www.wikidata.org/wiki/Q9471","display_name":"Epistemology","level":1,"score":0.0},{"id":"https://openalex.org/C86803240","wikidata":"https://www.wikidata.org/wiki/Q420","display_name":"Biology","level":0,"score":0.0},{"id":"https://openalex.org/C59822182","wikidata":"https://www.wikidata.org/wiki/Q441","display_name":"Botany","level":1,"score":0.0}],"mesh":[],"locations_count":2,"locations":[{"id":"doi:10.1145/3544548.3581555","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3544548.3581555","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3544548.3581555","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 2023 CHI Conference on Human Factors in Computing Systems","raw_type":"proceedings-article"},{"id":"pmh:tud:oai:tudelft.nl:uuid:2a445dad-39b5-4fa1-b0b6-e2541e68aa70","is_oa":true,"landing_page_url":"http://resolver.tudelft.nl/uuid:2a445dad-39b5-4fa1-b0b6-e2541e68aa70","pdf_url":"https://repository.tudelft.nl/file/File_32d730ad-07dd-4b50-9c18-e678e4c5b663","source":{"id":"https://openalex.org/S4306401843","display_name":"Data Archiving and Networked Services (DANS)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I1322597698","host_organization_name":"Royal Netherlands Academy of Arts and Sciences","host_organization_lineage":["https://openalex.org/I1322597698"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"CHI 2023 - Proceedings of the 2023 CHI Conference on Human Factors in Computing Systems","raw_type":"info:eu-repo/semantics/conferencepaper"}],"best_oa_location":{"id":"doi:10.1145/3544548.3581555","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3544548.3581555","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3544548.3581555","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 2023 CHI Conference on Human Factors in Computing Systems","raw_type":"proceedings-article"},"sustainable_development_goals":[{"id":"https://metadata.un.org/sdg/17","display_name":"Partnerships for the goals","score":0.5199999809265137}],"awards":[],"funders":[],"has_content":{"pdf":true,"grobid_xml":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4366588833.pdf","grobid_xml":"https://content.openalex.org/works/W4366588833.grobid-xml"},"referenced_works_count":114,"referenced_works":["https://openalex.org/W1999043044","https://openalex.org/W2031342017","https://openalex.org/W2032947860","https://openalex.org/W2039179155","https://openalex.org/W2053075547","https://openalex.org/W2059362837","https://openalex.org/W2064135357","https://openalex.org/W2066327708","https://openalex.org/W2087419855","https://openalex.org/W2090266317","https://openalex.org/W2113302524","https://openalex.org/W2137406659","https://openalex.org/W2146141698","https://openalex.org/W2165254944","https://openalex.org/W2282821441","https://openalex.org/W2284072287","https://openalex.org/W2512274390","https://openalex.org/W2565907704","https://openalex.org/W2728625256","https://openalex.org/W2756202949","https://openalex.org/W2789444109","https://openalex.org/W2804746362","https://openalex.org/W2807910285","https://openalex.org/W2886614482","https://openalex.org/W2897042519","https://openalex.org/W2898589728","https://openalex.org/W2898868990","https://openalex.org/W2914202940","https://openalex.org/W2922234936","https://openalex.org/W2923421605","https://openalex.org/W2942073295","https://openalex.org/W2948038809","https://openalex.org/W2953027140","https://openalex.org/W2956281901","https://openalex.org/W2963588812","https://openalex.org/W2963622213","https://openalex.org/W2963855133","https://openalex.org/W2964759459","https://openalex.org/W2968594320","https://openalex.org/W2970837303","https://openalex.org/W2982167689","https://openalex.org/W2988450373","https://openalex.org/W2997618083","https://openalex.org/W2999134777","https://openalex.org/W2999765337","https://openalex.org/W3005780259","https://openalex.org/W3007157104","https://openalex.org/W3010300896","https://openalex.org/W3012736183","https://openalex.org/W3013004221","https://openalex.org/W3013149459","https://openalex.org/W3014011525","https://openalex.org/W3014120508","https://openalex.org/W3016099278","https://openalex.org/W3019489177","https://openalex.org/W3029022390","https://openalex.org/W3029504795","https://openalex.org/W3030790076","https://openalex.org/W3032086959","https://openalex.org/W3100279624","https://openalex.org/W3104847483","https://openalex.org/W3105396370","https://openalex.org/W3106151957","https://openalex.org/W3112486745","https://openalex.org/W3115483167","https://openalex.org/W3121784008","https://openalex.org/W3122778363","https://openalex.org/W3123497755","https://openalex.org/W3127021193","https://openalex.org/W3128442340","https://openalex.org/W3134080822","https://openalex.org/W3135057764","https://openalex.org/W3152877557","https://openalex.org/W3155456206","https://openalex.org/W3156354433","https://openalex.org/W3156666346","https://openalex.org/W3157320130","https://openalex.org/W3157506384","https://openalex.org/W3158471737","https://openalex.org/W3158735461","https://openalex.org/W3159177113","https://openalex.org/W3160037564","https://openalex.org/W3160343688","https://openalex.org/W3160349922","https://openalex.org/W3162977957","https://openalex.org/W3175034780","https://openalex.org/W3176051226","https://openalex.org/W3176115515","https://openalex.org/W3184924454","https://openalex.org/W3185555828","https://openalex.org/W3204393347","https://openalex.org/W3206153464","https://openalex.org/W3206213635","https://openalex.org/W3206271843","https://openalex.org/W3206420877","https://openalex.org/W3212368439","https://openalex.org/W3216607306","https://openalex.org/W4205433266","https://openalex.org/W4211180451","https://openalex.org/W4214851743","https://openalex.org/W4221061784","https://openalex.org/W4224322655","https://openalex.org/W4224980442","https://openalex.org/W4225008476","https://openalex.org/W4225127292","https://openalex.org/W4226099958","https://openalex.org/W4229494842","https://openalex.org/W4286800222","https://openalex.org/W4287328644","https://openalex.org/W4288083705","https://openalex.org/W4288083797","https://openalex.org/W4288086173","https://openalex.org/W4308784155","https://openalex.org/W6807892083"],"related_works":["https://openalex.org/W17155033","https://openalex.org/W3207760230","https://openalex.org/W1496222301","https://openalex.org/W4312814274","https://openalex.org/W1590307681","https://openalex.org/W2536018345","https://openalex.org/W4285370786","https://openalex.org/W2296488620","https://openalex.org/W2358353312","https://openalex.org/W4366307084"],"abstract_inverted_index":{"Handling":[0],"failures":[1,37,106],"in":[2,38,65,94],"computer":[3],"vision":[4],"systems":[5],"that":[6,84,100,110],"rely":[7,115],"on":[8,116],"deep":[9],"learning":[10],"models":[11],"remains":[12],"a":[13,76,87,131,139],"challenge.":[14],"While":[15],"an":[16,43],"increasing":[17],"number":[18],"of":[19,52,90,105,133,141,148],"methods":[20],"for":[21,36],"bug":[22],"identification":[23],"and":[24,50,56,60,63,107,109,120,144],"correction":[25],"are":[26,97],"proposed,":[27],"little":[28],"is":[29,86],"known":[30],"about":[31],"how":[32],"practitioners":[33,71,101],"actually":[34],"search":[35],"these":[39],"models.":[40],"We":[41,68,82],"perform":[42],"empirical":[44],"study":[45],"to":[46,129,153],"understand":[47],"the":[48,54,61],"goals":[49],"needs":[51],"practitioners,":[53],"workflows":[55,93],"artifacts":[57],"they":[58,111],"use,":[59],"challenges":[62],"limitations":[64],"their":[66],"process.":[67],"interview":[69],"18":[70],"by":[72],"probing":[73],"them":[74],"with":[75],"carefully":[77],"crafted":[78],"failure":[79,91,155],"handling":[80,92,156],"scenario.":[81],"observe":[83],"there":[85],"great":[88],"diversity":[89],"which":[95],"cooperations":[96],"often":[98],"necessary,":[99],"overlook":[102],"certain":[103],"types":[104],"bugs,":[108],"generally":[112],"do":[113],"not":[114],"potentially":[117],"relevant":[118],"approaches":[119],"tools":[121],"originally":[122],"stemming":[123],"from":[124],"research.":[125],"These":[126],"insights":[127],"allow":[128],"draw":[130],"list":[132],"research":[134],"opportunities,":[135],"such":[136],"as":[137,158,160],"creating":[138],"library":[140],"best":[142],"practices":[143],"more":[145],"representative":[146],"formalisations":[147],"practitioners\u2019":[149],"goals,":[150],"developing":[151],"interfaces":[152],"exploit":[154],"artifacts,":[157],"well":[159],"providing":[161],"specialized":[162],"training.":[163]},"counts_by_year":[{"year":2026,"cited_by_count":1},{"year":2025,"cited_by_count":4},{"year":2024,"cited_by_count":1},{"year":2023,"cited_by_count":4}],"updated_date":"2026-06-11T09:08:48.828518","created_date":"2025-10-10T00:00:00"}