Repository navigation
datastore grows in size; StorageMax not respected since 0.39.0 #11096
Description
Checklist
- This is a bug report, not a question. Ask questions on discuss.ipfs.tech.
- I have searched on the issue tracker for my bug.
- I am running the latest kubo version or have an issue updating.
Installation method
dist.ipfs.tech or ipfs-update
Version
Kubo version: 0.39.0
Repo version: 18
System version: amd64/linux
Golang version: go1.25.4
Config
{
"API": {
"HTTPHeaders": {
"Access-Control-Allow-Methods": [
"PUT",
"POST"
],
"Access-Control-Allow-Origin": [
"http://localhost:5001",
"http://127.0.0.1:5001",
]
}
},
"Addresses": {
"API": [
"/ip4/0.0.0.0/tcp/5001",
"/ip6/::/tcp/5001"
],
"Announce": [
"/ip4/185.40.199.126/tcp/4001",
"/ip4/185.40.199.126/udp/4001/webrtc-direct",
"/ip4/185.40.199.126/udp/4001/quic-v1",
"/ip4/185.40.199.126/udp/4001/quic-v1/webtransport",
"/ip6/2a00:c90::56:2/tcp/4001",
"/ip6/2a00:c90::56:2/udp/4001/webrtc-direct",
"/ip6/2a00:c90::56:2/udp/4001/quic-v1",
"/ip6/2a00:c90::56:2/udp/4001/quic-v1/webtransport"
],
"AppendAnnounce": null,
"Gateway": [
"/ip4/0.0.0.0/tcp/8080",
"/ip6/::/tcp/8080"
],
"NoAnnounce": [
"/ip4/10.0.0.0/ipcidr/8",
"/ip4/100.64.0.0/ipcidr/10",
"/ip4/169.254.0.0/ipcidr/16",
"/ip4/172.16.0.0/ipcidr/12",
"/ip4/192.0.0.0/ipcidr/24",
"/ip4/192.0.2.0/ipcidr/24",
"/ip4/192.168.0.0/ipcidr/16",
"/ip4/198.18.0.0/ipcidr/15",
"/ip4/198.51.100.0/ipcidr/24",
"/ip4/203.0.113.0/ipcidr/24",
"/ip4/240.0.0.0/ipcidr/4",
"/ip6/100::/ipcidr/64",
"/ip6/2001:2::/ipcidr/48",
"/ip6/2001:db8::/ipcidr/32",
"/ip6/fc00::/ipcidr/7",
"/ip6/fe80::/ipcidr/10"
],
"Swarm": [
"/ip4/0.0.0.0/tcp/4001",
"/ip4/0.0.0.0/udp/4001/quic-v1",
"/ip4/0.0.0.0/udp/4001/quic-v1/webtransport",
"/ip6/::/tcp/4001",
"/ip6/::/udp/4001/quic-v1",
"/ip6/::/udp/4001/quic-v1/webtransport"
]
},
"AutoConf": {},
"AutoNAT": {},
"Bootstrap": [
"auto"
],
"DNS": {
"Resolvers": {
".": "auto"
}
},
"Datastore": {
"BloomFilterSize": 0,
"GCPeriod": "1h",
"HashOnRead": false,
"Spec": {
"mounts": [
{
"child": {
"path": "blocks",
"shardFunc": "/repo/flatfs/shard/v1/next-to-last/2",
"sync": true,
"type": "flatfs"
},
"mountpoint": "/blocks",
"prefix": "flatfs.datastore",
"type": "measure"
},
{
"child": {
"compression": "none",
"path": "datastore",
"type": "levelds"
},
"mountpoint": "/",
"prefix": "leveldb.datastore",
"type": "measure"
}
],
"type": "mount"
},
"StorageGCWatermark": 90,
"StorageMax": "5GB"
},
"Discovery": {
"MDNS": {
"Enabled": false
}
},
"Experimental": {
"FilestoreEnabled": false,
"GraphsyncEnabled": false,
"Libp2pStreamMounting": false,
"P2pHttpProxy": false,
"StrategicProviding": false,
"UrlstoreEnabled": false
},
"Gateway": {
"APICommands": [],
"HTTPHeaders": {},
"NoDNSLink": false,
"NoFetch": false,
"PathPrefixes": [],
"PublicGateways": {
"localhost": null
},
"RootRedirect": "",
"Writable": false
},
"Identity": {
"PeerID": "<redacted>",
"PrivKey": "<redacted>"
},
"Internal": {},
"Ipns": {
"DelegatedPublishers": [
"auto"
],
"RecordLifetime": "",
"RepublishPeriod": "",
"ResolveCacheSize": 128
},
"Migration": {
"DownloadSources": [],
"Keep": ""
},
"Mounts": {
"FuseAllowOther": false,
"IPFS": "/ipfs",
"IPNS": "/ipns"
},
"Peering": {
"Peers": [
{
"Addrs": [
"/ip4/146.59.1.158/tcp/4001",
"/ip4/146.59.1.158/udp/4001/webrtc-direct",
"/ip4/146.59.1.158/udp/4001/quic-v1",
"/ip4/146.59.1.158/udp/4001/quic-v1/webtransport",
"/ip6/2001:41d0:602:3d9e::1/tcp/4001",
"/ip6/2001:41d0:602:3d9e::1/udp/4001/webrtc-direct",
"/ip6/2001:41d0:602:3d9e::1/udp/4001/quic-v1",
"/ip6/2001:41d0:602:3d9e::1/udp/4001/quic-v1/webtransport",
"/ip4/10.35.17.9/tcp/4001",
"/ip4/10.35.17.9/udp/4001/webrtc-direct",
"/ip4/10.35.17.9/udp/4001/quic-v1",
"/ip4/10.35.17.9/udp/4001/quic-v1/webtransport",
"/ip6/fde8:87d3:9415:9832::9/tcp/4001",
"/ip6/fde8:87d3:9415:9832::9/udp/4001/webrtc-direct",
"/ip6/fde8:87d3:9415:9832::9/udp/4001/quic-v1",
"/ip6/fde8:87d3:9415:9832::9/udp/4001/quic-v1/webtransport"
],
"ID": "12D3KooWSMyfy3sqRyd6Q1VA85DSedLrkaoQKSx551BEZHjwkx1k"
},
{
"Addrs": [
"/ip4/185.236.108.204/tcp/4001",
"/ip4/185.236.108.204/udp/4001/webrtc-direct",
"/ip4/185.236.108.204/udp/4001/quic-v1",
"/ip4/185.236.108.204/udp/4001/quic-v1/webtransport",
"/ip4/10.35.18.2/tcp/4001",
"/ip4/10.35.18.2/udp/4001/webrtc-direct",
"/ip4/10.35.18.2/udp/4001/quic-v1",
"/ip4/10.35.18.2/udp/4001/quic-v1/webtransport",
"/ip6/fde8:87d3:9415:9833::2/tcp/4001",
"/ip6/fde8:87d3:9415:9833::2/udp/4001/webrtc-direct",
"/ip6/fde8:87d3:9415:9833::2/udp/4001/quic-v1",
"/ip6/fde8:87d3:9415:9833::2/udp/4001/quic-v1/webtransport"
],
"ID": "12D3KooWMB9trMK6nwymDYZiTh2niKncoZydSf5A9AkKyeDcdws9"
}
]
},
"Pinning": {
"RemoteServices": {}
},
"Plugins": {
"Plugins": null
},
"Pubsub": {
"DisableSigning": false,
"Router": ""
},
"Routing": {
"AcceleratedDHTClient": false,
"DelegatedRouters": [
"auto"
],
"Routers": null
},
"Swarm": {
"AddrFilters": [
"/ip4/10.0.0.0/ipcidr/8",
"/ip4/100.64.0.0/ipcidr/10",
"/ip4/169.254.0.0/ipcidr/16",
"/ip4/172.16.0.0/ipcidr/12",
"/ip4/192.0.0.0/ipcidr/24",
"/ip4/192.0.2.0/ipcidr/24",
"/ip4/192.168.0.0/ipcidr/16",
"/ip4/198.18.0.0/ipcidr/15",
"/ip4/198.51.100.0/ipcidr/24",
"/ip4/203.0.113.0/ipcidr/24",
"/ip4/240.0.0.0/ipcidr/4",
"/ip6/100::/ipcidr/64",
"/ip6/2001:2::/ipcidr/48",
"/ip6/2001:db8::/ipcidr/32",
"/ip6/fc00::/ipcidr/7",
"/ip6/fe80::/ipcidr/10"
],
"ConnMgr": {},
"DisableBandwidthMetrics": false,
"DisableNatPortMap": true,
"RelayClient": {},
"RelayService": {},
"ResourceMgr": {},
"Transports": {
"Multiplexers": {},
"Network": {},
"Security": {}
}
}
}Description
After update to 0.39.0 the datastore started to gain weight quite quickly.
Initially I had Datastore.StorageMax set to default 10G. I changed it to 5GB and restarted kubo. A restart is purging the datastore. It grew again to 8.4G, which is way beyond 5GB. I'm running "ipfs repo gc" daily from cron. Even when run manually, it changes nothing.
Activity
hsanjuan commentedon Dec 9, 2025
Triage:
StorageMaxis no limit, it's just used to decide if gc should be triggered when--enable-gcis used (false by default). See: https://github.com/ipfs/kubo/blob/master/docs/config.md#datastorestoragemaxThe increased disk usage rate might come from Provide.Sweep (https://github.com/ipfs/kubo/blob/master/docs/config.md#providedhtsweepenabled) but it seems like too much. So:
etam commentedon Dec 9, 2025
I set
Provide.DHT.SweepEnabled: false. Let's wait for results...etam commentedon Dec 15, 2025
Yep. The disk usage stays flat.
github-actions commentedon Dec 22, 2025
Oops, seems like we needed more information for this issue, please comment with more details or this issue will be closed in 7 days.
etam commentedon Dec 22, 2025
What information is needed?
github-actions commentedon Dec 29, 2025
Oops, seems like we needed more information for this issue, please comment with more details or this issue will be closed in 7 days.
etam commentedon Dec 29, 2025
What information is needed?
15 remaining items
Rashkae2 commentedon Jan 30, 2026
How large do you want me to wait for? My Datastore usually shrinks to less than 2GB when Kubo is Started, and now sitting at 3.3 after almost 30 hrs uptime. (I might not be able to let this computer run continuously this week.)
Rashkae2 commentedon Jan 30, 2026
The ipfs diag datastore commands does not appear to be correct. I get:
ipfs diag datastore count /
Error: expected 0 argument(s), got 3
USAGE
ipfs diag - Generate diagnostic reports.
ipfs diag
SUBCOMMANDS
ipfs diag cmds - List commands run on this IPFS node.
ipfs diag profile - Collect a performance profile for debugging.
ipfs diag sys - Print system diagnostic information.
For more information about each command, use:
'ipfs diag --help'
(I also tried on an offline 0.39-rc1 node, and got the same results.)
guillaumemichel commentedon Jan 30, 2026
@Rashkae2 yes, this should be good enough!
Oh right!
ipfs diag datastoreis a new command that will be shipped in v0.40.0.If you are able to reproduce this issue from
masterand runipfs diag datastorecommands, it would be awesome.Otherwise running
ipfs provide stat --allandipfs provide stat --lan --allbefore and after node restart should already provide some useful information.Rashkae2 commentedon Jan 30, 2026
Before Restart:
After Restart!!
guillaumemichel commentedon Feb 3, 2026
Thanks for the update @Rashkae2 !
It seems to show that the issue doesn't come from LAN DHT provider nor provide queue.
So far, I have been unable to reproduce the issue on the latest
master.It would be great if you can checkout the latest
master, runmake install, try to reproduce the problem and runipfs diag datastore countcommands to show the datastore state. Alternatively, you could try again after the v0.40.0 release (or rc).guillaumemichel commentedon Feb 5, 2026
I have been trying to reproduce the issue. I have a much lower number of advertised CIDs (7k vs 148k)
I can confirm that the datastore filesize seems to increase over time, and then shrink (probably leveldb compaction).
Here is the script I ran to monitor the datastore size when kubo was running
datastore_monitor.log
I noticed that the size of the datastore shrank when I shut down kubo. Hence the observed datastore key counts look normal:
It seems that leveldb compaction is for something in the datastore size variation, but it may not be the only reason to reach such a large datastore state.
docs: clarify LevelDB compaction limitations and StorageMax scope
lidel commentedon Feb 9, 2026
Thank you @guillaumemichel, this felt like useful path of inquiry, I did some digging into this last week. Here's what I found (sorry for chaotic notes, did not had time to clean up):
Reproduction
Built kubo from current master (
c6702eaf8), created a fresh LevelDB repo with 50k small unpinned blocks viaipfs block put, and setProvide.DHT.Intervalto2mso I could watch multiple provide cycles without waiting 22 hours.Did the same with a Pebble repo (flatfs for blocks, pebbleds for metadata) to compare.
The double-buffer keystore swap
The main source of write churn is the
ResettableKeystorein go-libp2p-kad-dht. It keeps two copies of the CID keystore under/provider/keystore/0and/provider/keystore/1, and on everysyncKeystore()call (once at startup, then everyProvide.DHT.Interval), it:So for N CIDs, each cycle does roughly 2N deletes + N writes in LevelDB. With 50k CIDs that's ~150k operations per cycle. Each delete becomes a tombstone in the SSTable files that sits there until background compaction gets around to it.
Code:
kubo/core/node/provider.go:514-565callsResetCids()in a loop, which triggers the swap ingo-libp2p-kad-dht/provider/keystore/resettable_keystore.go:294-336.LevelDB numbers
Measured
datastore/directory size at each stage:LevelDB does compact, but it only reclaimed about 1 MB during a 5-minute window, while each 2m cycle dumps another round of tombstones. The net result is growth that outpaces compaction.
Pebble numbers (same test, flatfs + pebbleds)
Pebble reclaimed 6 MB (21%) vs LevelDB's 1 MB (2.8%) in the same observation window. Its tombstone-aware compaction keeps growth bounded. LevelDB's growth outpaces its compaction.
Other findings
ipfs provide clearonly clears the in-memory queue. No datastore writes, no tombstones, no effect on disk./providers/. After 47 min with the 2m interval the Pebble test had 581k such records. Separate growth vector, but adds up on long-running nodes.Close(), but not enough to offset what accumulated it seems?What could help
Quick and easy:
docs/config.mdthatStorageMaxdoesn't limitdatastore/growth, and thatpebbledshandles this workload better than LevelDB.As for actual fixes, one thought is to introduce concept of compaction somehow, perhaps:
Compact()methods -- neither go-ds-leveldb nor go-ds-pebble expose compaction. AddingCompact(ctx context.Context) errorto both (wrappingDB.CompactRange/db.Compact) is a prerequisite for the rest. Perhaps we could define aCompactorinterface in go-datastore? Something to discuss.ipfs repo compactcommand -- offline manual trigger, likeipfs diag datastore countbut callsCompact(). Lets operators reclaim space without waiting for a restart.Compact()beforeClose()inFSRepo.Close(). Adds shutdown latency, may break various orchestrations that force-kill if process does not shut down fast.Second, probably unlikely as it mitigates rather than fixes leveldb problem:
ResettableKeystoredesign generates 2N tombstones per cycle. Reworking this somehow to an in-place diff approach (only delete CIDs that were actually removed, only add new ones) would cut write amplification significantly. Bigger change in go-libp2p-kad-dht, probably not feasible / not worth it given switching to pebble fixes problem and does not require big refactor.Third, change default Datastore of
ipfs initremoval the way Pebble does, so touching any existing defaults in LevelDB may harm/change existing performance profile (could do more harm than good for existing users.)
Unsure if there is an easy win here beyond documenting the current leveldb behavior.
We could switch from flatfs for blocks and leveldb for everything else → flatfs + pebbleds, but don't want to float the boat if we dont have to.
Thoughts?
guillaumemichel commentedon Feb 10, 2026
This isn't an option in the current state of Kubo. If we ever switch to reference counting CIDs to advertise, then it becomes easy to implement.
docs: clarify LevelDB compaction limitations and StorageMax scope (#1…
guillaumemichel commentedon Feb 13, 2026
See potential fix in #11198 and libp2p/go-libp2p-kad-dht#1233
Rashkae2 commentedon Feb 15, 2026
I won't have a long running node sample for a few more days at least, but 40-rc1 seems completely different from 0.39. In 18 hours uptime, datasore actually shrunk (from 2.0GB to 1.8GB) instead of suddenly growing.