Beyond the Prompt: Local Optimization and Provider Caching for Cost-Efficient LLM Workloads

{"tonst":[0],"is":[1,108,112],"an":[2,9,98],"MIT-licensed,":[3],"local-first":[4],"wrapper":[5],"that":[6],"sits":[7],"between":[8],"application":[10],"and":[11,31,66,74,82,97,125,154],"any":[12],"paid":[13],"LLM":[14],"API.":[15],"Before":[16],"a":[17,57,69,87,93,118,132],"request":[18,34],"leaves":[19],"the":[20,33,36,79,106,126,140,148],"calling":[21],"machine,":[22],"it":[23],"redacts":[24],"personal":[25],"data":[26],"(PII),":[27],"trims":[28],"redundant":[29],"tokens,":[30],"structures":[32],"so":[35],"provider's":[37],"own":[38,128],"prompt-caching":[39],"mechanism":[40],"can":[41],"discount":[42],"repeat":[43],"calls.":[44],"This":[45],"whitepaper":[46],"reports":[47],"real,":[48,133],"measured":[49],"results":[50],"rather":[51],"than":[52],"simulated":[53],"ones":[54],"wherever":[55],"possible:":[56],"360-iteration":[58],"benchmark":[59,152],"across":[60],"six":[61],"industry":[62],"domains":[63],"(token":[64],"reduction":[65,91],"PII-redaction":[67],"recall),":[68],"200-call":[70],"concurrency":[71],"stress":[72],"test,":[73],"live-billed":[75],"caching":[76],"tests":[77],"against":[78],"real":[80,119],"Anthropic":[81],"Gemini":[83],"APIs":[84],"—":[85,130,144],"including":[86,131],"53%":[88],"net":[89],"cost":[90,100],"on":[92,102],"realistic":[94],"mixed":[95],"workload":[96],"84.9%":[99],"saving":[101],"cache-read":[103],"calls":[104],"once":[105],"cache":[107],"warm.":[109],"Every":[110],"figure":[111],"labeled":[113],"as":[114],"either":[115],"\\"measured\\"":[116],"(from":[117],"run)":[120],"or":[121],"\\"estimate\\"":[122],"(mocked":[123],"pricing),":[124],"paper's":[127],"limitations":[129],"non-zero":[134],"free-text":[135],"PII":[136],"leak":[137],"rate":[138],"under":[139],"current":[141],"redaction":[142],"configuration":[143],"are":[145],"reported":[146],"alongside":[147],"results.":[149],"Source":[150],"code,":[151],"scripts,":[153],"full":[155],"methodology":[156],"notes:":[157],"https://github.com/v-nightwolf/tonst":[158]}