{"id":"https://openalex.org/W6967130704","doi":"https://doi.org/10.48550/arxiv.2503.07572","title":"Optimizing Test-Time Compute via Meta Reinforcement Fine-Tuning","display_name":"Optimizing Test-Time Compute via Meta Reinforcement Fine-Tuning","publication_year":2025,"publication_date":"2025-03-10","ids":{"openalex":"https://openalex.org/W6967130704","doi":"https://doi.org/10.48550/arxiv.2503.07572"},"language":"en","primary_location":{"id":"doi:10.48550/arxiv.2503.07572","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2503.07572","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"type":"preprint","indexed_in":["datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://doi.org/10.48550/arxiv.2503.07572","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":null,"display_name":"Qu, Yuxiao","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Qu, Yuxiao","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":null,"display_name":"Yang, Matthew Y. R.","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Yang, Matthew Y. R.","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":null,"display_name":"Setlur, Amrith","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Setlur, Amrith","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":null,"display_name":"Tunstall, Lewis","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Tunstall, Lewis","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":null,"display_name":"Beeching, Edward Emanuel","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Beeching, Edward Emanuel","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":null,"display_name":"Salakhutdinov, Ruslan","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Salakhutdinov, Ruslan","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":null,"display_name":"Kumar, Aviral","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Kumar, Aviral","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":true,"primary_topic":{"id":"https://openalex.org/T10462","display_name":"Reinforcement Learning in Robotics","score":0.09380000084638596,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10462","display_name":"Reinforcement Learning in Robotics","score":0.09380000084638596,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.08269999921321869,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11596","display_name":"Constraint Satisfaction and Optimization","score":0.07760000228881836,"subfield":{"id":"https://openalex.org/subfields/1705","display_name":"Computer Networks and Communications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/regret","display_name":"Regret","score":0.845300018787384},{"id":"https://openalex.org/keywords/reinforcement-learning","display_name":"Reinforcement learning","score":0.6996999979019165},{"id":"https://openalex.org/keywords/perspective","display_name":"Perspective (graphical)","score":0.6068000197410583},{"id":"https://openalex.org/keywords/outcome","display_name":"Outcome (game theory)","score":0.580299973487854},{"id":"https://openalex.org/keywords/security-token","display_name":"Security token","score":0.503000020980835},{"id":"https://openalex.org/keywords/measure","display_name":"Measure (data warehouse)","score":0.4505000114440918},{"id":"https://openalex.org/keywords/class","display_name":"Class (philosophy)","score":0.4323999881744385},{"id":"https://openalex.org/keywords/key","display_name":"Key (lock)","score":0.4242999851703644},{"id":"https://openalex.org/keywords/scale","display_name":"Scale (ratio)","score":0.421999990940094},{"id":"https://openalex.org/keywords/block","display_name":"Block (permutation group theory)","score":0.38830000162124634}],"concepts":[{"id":"https://openalex.org/C50817715","wikidata":"https://www.wikidata.org/wiki/Q79895177","display_name":"Regret","level":2,"score":0.845300018787384},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7009999752044678},{"id":"https://openalex.org/C97541855","wikidata":"https://www.wikidata.org/wiki/Q830687","display_name":"Reinforcement learning","level":2,"score":0.6996999979019165},{"id":"https://openalex.org/C12713177","wikidata":"https://www.wikidata.org/wiki/Q1900281","display_name":"Perspective (graphical)","level":2,"score":0.6068000197410583},{"id":"https://openalex.org/C148220186","wikidata":"https://www.wikidata.org/wiki/Q7111912","display_name":"Outcome (game theory)","level":2,"score":0.580299973487854},{"id":"https://openalex.org/C48145219","wikidata":"https://www.wikidata.org/wiki/Q1335365","display_name":"Security token","level":2,"score":0.503000020980835},{"id":"https://openalex.org/C2780009758","wikidata":"https://www.wikidata.org/wiki/Q6804172","display_name":"Measure (data warehouse)","level":2,"score":0.4505000114440918},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.43290001153945923},{"id":"https://openalex.org/C2777212361","wikidata":"https://www.wikidata.org/wiki/Q5127848","display_name":"Class (philosophy)","level":2,"score":0.4323999881744385},{"id":"https://openalex.org/C26517878","wikidata":"https://www.wikidata.org/wiki/Q228039","display_name":"Key (lock)","level":2,"score":0.4242999851703644},{"id":"https://openalex.org/C2778755073","wikidata":"https://www.wikidata.org/wiki/Q10858537","display_name":"Scale (ratio)","level":2,"score":0.421999990940094},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.399399995803833},{"id":"https://openalex.org/C2777210771","wikidata":"https://www.wikidata.org/wiki/Q4927124","display_name":"Block (permutation group theory)","level":2,"score":0.38830000162124634},{"id":"https://openalex.org/C126255220","wikidata":"https://www.wikidata.org/wiki/Q141495","display_name":"Mathematical optimization","level":1,"score":0.35740000009536743},{"id":"https://openalex.org/C12725497","wikidata":"https://www.wikidata.org/wiki/Q810247","display_name":"Baseline (sea)","level":2,"score":0.3345000147819519},{"id":"https://openalex.org/C158154518","wikidata":"https://www.wikidata.org/wiki/Q7310970","display_name":"Relevance (law)","level":2,"score":0.3118000030517578},{"id":"https://openalex.org/C149629883","wikidata":"https://www.wikidata.org/wiki/Q660926","display_name":"Fraction (chemistry)","level":2,"score":0.30140000581741333},{"id":"https://openalex.org/C204323151","wikidata":"https://www.wikidata.org/wiki/Q905424","display_name":"Range (aeronautics)","level":2,"score":0.29840001463890076},{"id":"https://openalex.org/C2776848632","wikidata":"https://www.wikidata.org/wiki/Q853463","display_name":"Clipping (morphology)","level":2,"score":0.29750001430511475},{"id":"https://openalex.org/C73602740","wikidata":"https://www.wikidata.org/wiki/Q7795822","display_name":"Thompson sampling","level":3,"score":0.28360000252723694},{"id":"https://openalex.org/C59656382","wikidata":"https://www.wikidata.org/wiki/Q191536","display_name":"Conjunction (astronomy)","level":2,"score":0.2768999934196472},{"id":"https://openalex.org/C2777267654","wikidata":"https://www.wikidata.org/wiki/Q3519023","display_name":"Test (biology)","level":2,"score":0.24240000545978546},{"id":"https://openalex.org/C140779682","wikidata":"https://www.wikidata.org/wiki/Q210868","display_name":"Sampling (signal processing)","level":3,"score":0.23739999532699585},{"id":"https://openalex.org/C28761237","wikidata":"https://www.wikidata.org/wiki/Q7805321","display_name":"Time horizon","level":2,"score":0.2361000031232834},{"id":"https://openalex.org/C8505890","wikidata":"https://www.wikidata.org/wiki/Q605095","display_name":"Budget constraint","level":2,"score":0.22439999878406525},{"id":"https://openalex.org/C177264268","wikidata":"https://www.wikidata.org/wiki/Q1514741","display_name":"Set (abstract data type)","level":2,"score":0.2232999950647354},{"id":"https://openalex.org/C2776214188","wikidata":"https://www.wikidata.org/wiki/Q408386","display_name":"Inference","level":2,"score":0.22030000388622284},{"id":"https://openalex.org/C2164484","wikidata":"https://www.wikidata.org/wiki/Q5170150","display_name":"Core (optical fiber)","level":2,"score":0.21459999680519104},{"id":"https://openalex.org/C192209626","wikidata":"https://www.wikidata.org/wiki/Q190909","display_name":"Focus (optics)","level":2,"score":0.21369999647140503},{"id":"https://openalex.org/C67203356","wikidata":"https://www.wikidata.org/wiki/Q1321905","display_name":"Reinforcement","level":2,"score":0.2094999998807907},{"id":"https://openalex.org/C2781002164","wikidata":"https://www.wikidata.org/wiki/Q6822311","display_name":"Meta learning (computer science)","level":3,"score":0.2087000012397766},{"id":"https://openalex.org/C6177178","wikidata":"https://www.wikidata.org/wiki/Q10998070","display_name":"Discounting","level":2,"score":0.20350000262260437},{"id":"https://openalex.org/C21200559","wikidata":"https://www.wikidata.org/wiki/Q7451068","display_name":"Sensitivity (control systems)","level":2,"score":0.20309999585151672},{"id":"https://openalex.org/C48677424","wikidata":"https://www.wikidata.org/wiki/Q6888088","display_name":"Mode (computer interface)","level":2,"score":0.2003999948501587},{"id":"https://openalex.org/C2779545769","wikidata":"https://www.wikidata.org/wiki/Q5135364","display_name":"Closeness","level":2,"score":0.19910000264644623},{"id":"https://openalex.org/C2986053828","wikidata":"https://www.wikidata.org/wiki/Q355217","display_name":"Time budget","level":2,"score":0.19750000536441803},{"id":"https://openalex.org/C22367795","wikidata":"https://www.wikidata.org/wiki/Q7625208","display_name":"Structured prediction","level":2,"score":0.19359999895095825},{"id":"https://openalex.org/C87007009","wikidata":"https://www.wikidata.org/wiki/Q210832","display_name":"Statistical hypothesis testing","level":2,"score":0.19269999861717224},{"id":"https://openalex.org/C175154964","wikidata":"https://www.wikidata.org/wiki/Q380077","display_name":"Task analysis","level":3,"score":0.19040000438690186},{"id":"https://openalex.org/C136197465","wikidata":"https://www.wikidata.org/wiki/Q1729295","display_name":"Variety (cybernetics)","level":2,"score":0.1889999955892563}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.48550/arxiv.2503.07572","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2503.07572","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"doi:10.48550/arxiv.2503.07572","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2503.07572","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":[],"abstract_inverted_index":{"Training":[0],"models":[1,163],"to":[2,45,56,87,108,121,129,232,251],"effectively":[3],"use":[4,109],"test-time":[5,39,66,81,126,228],"compute":[6,67],"is":[7,188],"crucial":[8],"for":[9,226,247],"improving":[10],"the":[11,48,62,89,94,123,147,155,181,189,197,202,205],"reasoning":[12,249],"performance":[13,238],"of":[14,64,98,112,125,207,223],"LLMs.":[15],"Current":[16],"methods":[17,225],"mostly":[18],"do":[19,34,164,170],"so":[20,171],"via":[21],"fine-tuning":[22,224],"on":[23,79],"search":[24],"traces":[25],"or":[26,218],"running":[27],"RL":[28,131],"with":[29,180],"0/1":[30,183],"outcome":[31,182],"reward,":[32],"but":[33],"these":[35,42,58,211],"approaches":[36,43],"efficiently":[37],"utilize":[38],"compute?":[40],"Would":[41],"continue":[44],"scale":[46],"as":[47,68,96,118],"budget":[49],"improves?":[50],"In":[51],"this":[52],"paper,":[53],"we":[54,159,213],"try":[55],"answer":[57],"questions.":[59],"We":[60],"formalize":[61],"problem":[63],"optimizing":[65,227],"a":[69,76,110,119,174,220,233,241],"meta-reinforcement":[70],"learning":[71],"(RL)":[72],"problem,":[73],"which":[74],"provides":[75],"principled":[77],"perspective":[78,84],"spending":[80],"compute.":[82,127,229],"This":[83,186],"enables":[85],"us":[86,107],"view":[88],"long":[90],"output":[91,116,198],"stream":[92],"from":[93],"LLM":[95],"consisting":[97],"several":[99],"episodes":[100],"run":[101],"at":[102],"test":[103],"time":[104],"and":[105,137,152,239],"leads":[106,231],"notion":[111],"cumulative":[113,142],"regret":[114,143],"over":[115,139],"tokens":[117],"way":[120],"measure":[122],"efficacy":[124],"Akin":[128],"how":[130],"algorithms":[132],"can":[133,169],"best":[134,148],"tradeoff":[135],"exploration":[136,151],"exploitation":[138,153],"training,":[140],"minimizing":[141],"would":[144],"also":[145],"provide":[146],"balance":[149],"between":[150],"in":[154,178,196,204,237,244],"token":[156,245],"stream.":[157],"While":[158],"show":[160],"that":[161],"state-of-the-art":[162],"not":[165],"minimize":[166],"regret,":[167],"one":[168],"by":[172,192,201],"maximizing":[173],"dense":[175],"reward":[176,184],"bonus":[177,187],"conjunction":[179],"RL.":[185,253],"''progress''":[190],"made":[191],"each":[193],"subsequent":[194],"block":[195],"stream,":[199],"quantified":[200],"change":[203],"likelihood":[206],"eventual":[208],"success.":[209],"Using":[210],"insights,":[212],"develop":[214],"Meta":[215],"Reinforcement":[216],"Fine-Tuning,":[217],"MRT,":[219],"new":[221],"class":[222],"MRT":[230],"2-3x":[234],"relative":[235],"gain":[236,243],"roughly":[240],"1.5x":[242],"efficiency":[246],"math":[248],"compared":[250],"outcome-reward":[252]},"counts_by_year":[],"updated_date":"2026-07-01T06:00:48.157686","created_date":"2025-10-10T00:00:00"}