Documentation
¶
Overview ¶
Package storage provides DeleteRowsWithCallback for FTS support
Package storage provides data persistence for xxldb ¶
Package storage provides data persistence for xxldb ¶
Package storage provides migration from V1 to V2 storage format ¶
Package storage provides page-based storage for XxLdb ¶
Package storage provides page cache for XxLdb ¶
Package storage provides data persistence for xxldb ¶
Package storage provides data persistence for xxldb ¶
Package storage provides table file management for XxLdb ¶
Package storage provides data persistence for xxldb ¶
Package storage provides UpdateRowsWithCallback for FTS support ¶
Package storage provides WAL (Write-Ahead Logging) support ¶
Package storage provides data persistence for xxldb
Index ¶
- Constants
- Variables
- func DecodeRow(data []byte, colDefs []types.ColumnDef) (rowID uint64, row []types.Value, err error)
- func DeleteRowFromPage(page *PageV2, index int) error
- func EncodeRow(rowID uint64, row []types.Value) []byte
- func EstimateRowSize(row []types.Value) int
- func GetCellData(page *PageV2, index int) ([]byte, error)
- func GetEncryptionSalt(path string) ([]byte, error)
- func GetRowFromPage(page *PageV2, index int, colDefs []types.ColumnDef) (rowID uint64, row []types.Value, err error)
- func InsertRowToPage(page *PageV2, rowID uint64, row []types.Value) (int, error)
- func IsDatabaseEncrypted(path string) bool
- func IsTempFile(name string) bool
- type BatchFS
- type BatchFileSystem
- type Config
- type DirEntry
- type FTSIndexInfo
- type File
- type FileInfo
- type FileMode
- type FileSystem
- type LocalFS
- func (fs *LocalFS) Abs(path string) (string, error)
- func (fs *LocalFS) Base(path string) string
- func (fs *LocalFS) Close() error
- func (fs *LocalFS) Create(name string) (File, error)
- func (fs *LocalFS) Dir(path string) string
- func (fs *LocalFS) Exists(name string) bool
- func (fs *LocalFS) Join(elem ...string) string
- func (fs *LocalFS) MkdirAll(path string, perm FileMode) error
- func (fs *LocalFS) Open(name string) (File, error)
- func (fs *LocalFS) OpenFile(name string, flag int, perm FileMode) (File, error)
- func (fs *LocalFS) ReadDir(dirname string) ([]DirEntry, error)
- func (fs *LocalFS) ReadFile(name string) ([]byte, error)
- func (fs *LocalFS) Remove(name string) error
- func (fs *LocalFS) RemoveAll(path string) error
- func (fs *LocalFS) Rename(oldpath, newpath string) error
- func (fs *LocalFS) SafeWriteFile(path string, data []byte, perm FileMode) error
- func (fs *LocalFS) Stat(name string) (FileInfo, error)
- func (fs *LocalFS) TempFile(dir, pattern string) (File, error)
- func (fs *LocalFS) Walk(root string, walkFn WalkFunc) error
- func (fs *LocalFS) WriteFile(name string, data []byte, perm FileMode) error
- type Migration
- type PageCache
- func (c *PageCache) Clear()
- func (c *PageCache) ClearDirty(pageID uint64)
- func (c *PageCache) Flush(writeFunc func(*PageV2) error) error
- func (c *PageCache) FlushPage(pageID uint64, writeFunc func(*PageV2) error) error
- func (c *PageCache) Get(pageID uint64) *PageV2
- func (c *PageCache) GetDirtyPages() []uint64
- func (c *PageCache) IsDirty(pageID uint64) bool
- func (c *PageCache) MarkDirty(pageID uint64)
- func (c *PageCache) Put(page *PageV2) error
- func (c *PageCache) Remove(pageID uint64)
- func (c *PageCache) Resize(maxPages int) error
- func (c *PageCache) Size() int
- func (c *PageCache) Stats() map[string]interface{}
- type PageCacheConfig
- type PageHeaderV2
- type PageType
- type PageV2
- func (p *PageV2) AddCellPointer(cellOffset uint16)
- func (p *PageV2) AvailableSpace() uint16
- func (p *PageV2) CalculateChecksum() uint32
- func (p *PageV2) CanInsert(rowSize int) bool
- func (p *PageV2) FromBytes(data []byte) error
- func (p *PageV2) GetCellPointer(index int) uint16
- func (p *PageV2) SetCellPointer(index int, cellOffset uint16)
- func (p *PageV2) ToBytes() []byte
- func (p *PageV2) UpdateChecksum()
- func (p *PageV2) UsageRatio() float64
- func (p *PageV2) VerifyChecksum() bool
- type ResumeInfo
- type RowIterator
- type RowWithID
- type SFTPConfig
- type SFTPFS
- func (fs *SFTPFS) Abs(path string) (string, error)
- func (fs *SFTPFS) Base(path string) string
- func (fs *SFTPFS) Batch(fn func(BatchFS) error) error
- func (fs *SFTPFS) Close() error
- func (fs *SFTPFS) Create(name string) (File, error)
- func (fs *SFTPFS) Dir(path string) string
- func (fs *SFTPFS) Exists(name string) bool
- func (fs *SFTPFS) Join(elem ...string) string
- func (fs *SFTPFS) MkdirAll(path string, perm FileMode) error
- func (fs *SFTPFS) Open(name string) (File, error)
- func (fs *SFTPFS) OpenFile(name string, flag int, perm FileMode) (File, error)
- func (fs *SFTPFS) ReadDir(dirname string) ([]DirEntry, error)
- func (fs *SFTPFS) ReadFile(name string) ([]byte, error)
- func (fs *SFTPFS) Remove(name string) error
- func (fs *SFTPFS) RemoveAll(path string) error
- func (fs *SFTPFS) Rename(oldpath, newpath string) error
- func (fs *SFTPFS) SafeWriteFile(path string, data []byte, perm FileMode) error
- func (fs *SFTPFS) Stat(name string) (FileInfo, error)
- func (fs *SFTPFS) TempFile(dir, pattern string) (File, error)
- func (fs *SFTPFS) Walk(root string, walkFn WalkFunc) error
- func (fs *SFTPFS) WriteFile(name string, data []byte, perm FileMode) error
- type SMBConfig
- type SMBFS
- func (fs *SMBFS) Abs(path string) (string, error)
- func (fs *SMBFS) Base(path string) string
- func (fs *SMBFS) Batch(fn func(BatchFS) error) error
- func (fs *SMBFS) Close() error
- func (fs *SMBFS) Create(name string) (File, error)
- func (fs *SMBFS) Dir(path string) string
- func (fs *SMBFS) Exists(name string) bool
- func (fs *SMBFS) Join(elem ...string) string
- func (fs *SMBFS) MkdirAll(path string, perm FileMode) error
- func (fs *SMBFS) Open(name string) (File, error)
- func (fs *SMBFS) OpenFile(name string, flag int, perm FileMode) (File, error)
- func (fs *SMBFS) ReadDir(dirname string) ([]DirEntry, error)
- func (fs *SMBFS) ReadFile(name string) ([]byte, error)
- func (fs *SMBFS) Remove(name string) error
- func (fs *SMBFS) RemoveAll(path string) error
- func (fs *SMBFS) Rename(oldpath, newpath string) error
- func (fs *SMBFS) SafeWriteFile(path string, data []byte, perm FileMode) error
- func (fs *SMBFS) Stat(name string) (FileInfo, error)
- func (fs *SMBFS) TempFile(dir, pattern string) (File, error)
- func (fs *SMBFS) Walk(root string, walkFn WalkFunc) error
- func (fs *SMBFS) WithContext(ctx context.Context) *SMBFSContext
- func (fs *SMBFS) WriteFile(name string, data []byte, perm FileMode) error
- type SMBFSContext
- type Storage
- func (s *Storage) AutoMigrate() error
- func (s *Storage) Backup(backupPath string) error
- func (s *Storage) BeginTxn() uint64
- func (s *Storage) ChangePassword(oldPassword, newPassword string) error
- func (s *Storage) Checkpoint() error
- func (s *Storage) Close() error
- func (s *Storage) CloseAllTableFiles() error
- func (s *Storage) CloseTableFile(tableName string) error
- func (s *Storage) CommitTxn(txnID uint64) error
- func (s *Storage) CreateFTSIndex(tableName, columnName, indexName string) error
- func (s *Storage) CreateTable(info *types.TableInfo) error
- func (s *Storage) DeleteBlob(blobID uint64) error
- func (s *Storage) DeleteRows(tableName string, condition func([]types.Value) bool) (int64, error)
- func (s *Storage) DeleteRowsWithCallback(tableName string, condition func([]types.Value) bool, ...) (int64, error)
- func (s *Storage) DropFTSIndex(tableName, columnName string) error
- func (s *Storage) DropTable(name string) error
- func (s *Storage) ExportFile(blobID uint64, filePath string) error
- func (s *Storage) FlushAll() error
- func (s *Storage) ForceSave() error
- func (s *Storage) GetAuthConfig() map[string]interface{}
- func (s *Storage) GetConfig() Config
- func (s *Storage) GetDataPath() string
- func (s *Storage) GetEncryptionSalt() []byte
- func (s *Storage) GetFTSIndexPath(tableName, columnName string) string
- func (s *Storage) GetFTSIndexes() map[string]*FTSIndexInfo
- func (s *Storage) GetPageCacheStats() map[string]interface{}
- func (s *Storage) GetRows(tableName string) ([][]types.Value, error)
- func (s *Storage) GetRowsWithIDs(tableName string) ([]RowWithID, error)
- func (s *Storage) GetRowsWithoutBlobRefs(tableName string) ([][]types.Value, error)
- func (s *Storage) GetSequence(name string) int64
- func (s *Storage) GetTableFile(tableName string) (*TableFile, error)
- func (s *Storage) GetTableInfo(name string) (*types.TableInfo, error)
- func (s *Storage) HasFTSIndex(tableName, columnName string) bool
- func (s *Storage) ImportFile(filePath string) (uint64, error)
- func (s *Storage) InsertRow(tableName string, row []types.Value) (uint64, int64, error)
- func (s *Storage) IsEncrypted() bool
- func (s *Storage) ListTables() []string
- func (s *Storage) LoadAndVerifyMetadata() error
- func (s *Storage) ReadBlob(blobID uint64) ([]byte, error)
- func (s *Storage) Reader(blobID uint64) (io.ReadCloser, error)
- func (s *Storage) RenameTable(oldName, newName string) error
- func (s *Storage) RollbackTxn(txnID uint64) error
- func (s *Storage) SetAuthConfig(authConfig map[string]interface{}) error
- func (s *Storage) SetConfig(config Config)
- func (s *Storage) SetEncryption(password string) error
- func (s *Storage) SetEncryptionWithSalt(password string, salt []byte) error
- func (s *Storage) SetSequence(name string, value int64)
- func (s *Storage) SetSkipSave(skip bool)
- func (s *Storage) Stats() map[string]interface{}
- func (s *Storage) TruncateTable(name string) error
- func (s *Storage) UpdateRows(tableName string, updates map[int]types.Value, ...) (int64, error)
- func (s *Storage) UpdateRowsWithCallback(tableName string, updateFunc func([]types.Value) map[int]types.Value, ...) (int64, error)
- func (s *Storage) UpdateRowsWithFunc(tableName string, updateFunc func([]types.Value) map[int]types.Value, ...) (int64, error)
- func (s *Storage) WriteBlob(data []byte) (uint64, error)
- func (s *Storage) Writer() (io.WriteCloser, uint64, error)
- type TableFile
- func (tf *TableFile) CacheSize() int
- func (tf *TableFile) CacheStats() map[string]interface{}
- func (tf *TableFile) Clear() error
- func (tf *TableFile) Close() error
- func (tf *TableFile) DeleteRow(rowID uint64) error
- func (tf *TableFile) FilePath() string
- func (tf *TableFile) Flush() error
- func (tf *TableFile) GetAllRows() ([][]types.Value, error)
- func (tf *TableFile) GetAllRowsWithIDs() ([]RowWithID, error)
- func (tf *TableFile) GetColumnDefs() []types.ColumnDef
- func (tf *TableFile) GetFirstDataPage() uint64
- func (tf *TableFile) GetLastDataPage() uint64
- func (tf *TableFile) GetRow(rowID uint64) ([]types.Value, error)
- func (tf *TableFile) GetRowCount() uint64
- func (tf *TableFile) InsertRow(row []types.Value) (uint64, error)
- func (tf *TableFile) PageCount() uint32
- func (tf *TableFile) PageSize() int
- func (tf *TableFile) ReadPageDebug(pageID uint64) (*PageV2, error)
- func (tf *TableFile) ScanRows() *RowIterator
- func (tf *TableFile) UpdateRow(rowID uint64, row []types.Value) error
- type TableFileHeader
- type TableInfo
- type Transaction
- type TransferInfo
- type TransferState
- func (ts *TransferState) BeginTransfer(id, source, target, temp string, totalSize int64)
- func (ts *TransferState) CanRetry(id string, maxRetries int) bool
- func (ts *TransferState) CleanupIncomplete(fs FileSystem) []error
- func (ts *TransferState) CleanupOldTransfers(maxAge time.Duration) int
- func (ts *TransferState) CompleteTransfer(id string)
- func (ts *TransferState) Count() int
- func (ts *TransferState) CountByStatus(status TransferStatus) int
- func (ts *TransferState) FailTransfer(id string, err error)
- func (ts *TransferState) GetPendingTransfers() []*TransferInfo
- func (ts *TransferState) GetResumeInfo(id string) *ResumeInfo
- func (ts *TransferState) GetTransfer(id string) *TransferInfo
- func (ts *TransferState) IncrementRetry(id string) int
- func (ts *TransferState) UpdateProgress(id string, transferred int64)
- type TransferStatus
- type WAL
- type WALHeader
- type WALRecord
- type WALType
- type WalkFunc
- type WebDAVConfig
- type WebDAVFS
- func (fs *WebDAVFS) Abs(path string) (string, error)
- func (fs *WebDAVFS) Base(path string) string
- func (fs *WebDAVFS) Batch(fn func(BatchFS) error) error
- func (fs *WebDAVFS) Close() error
- func (fs *WebDAVFS) Create(name string) (File, error)
- func (fs *WebDAVFS) Dir(path string) string
- func (fs *WebDAVFS) Exists(name string) bool
- func (fs *WebDAVFS) Join(elem ...string) string
- func (fs *WebDAVFS) MkdirAll(path string, perm FileMode) error
- func (fs *WebDAVFS) Open(name string) (File, error)
- func (fs *WebDAVFS) OpenFile(name string, flag int, perm FileMode) (File, error)
- func (fs *WebDAVFS) ReadDir(dirname string) ([]DirEntry, error)
- func (fs *WebDAVFS) ReadFile(name string) ([]byte, error)
- func (fs *WebDAVFS) Remove(name string) error
- func (fs *WebDAVFS) RemoveAll(path string) error
- func (fs *WebDAVFS) Rename(oldpath, newpath string) error
- func (fs *WebDAVFS) SafeWriteFile(path string, data []byte, perm FileMode) error
- func (fs *WebDAVFS) Stat(name string) (FileInfo, error)
- func (fs *WebDAVFS) TempFile(dir, pattern string) (File, error)
- func (fs *WebDAVFS) Walk(root string, walkFn WalkFunc) error
- func (fs *WebDAVFS) WriteFile(name string, data []byte, perm FileMode) error
Constants ¶
const ( FlagO_RDONLY int = 0 FlagO_WRONLY int = 1 FlagO_RDWR int = 2 FlagO_APPEND int = 0400 FlagO_CREATE int = 0100 FlagO_EXCL int = 0200 FlagO_SYNC int = 01000 FlagO_TRUNC int = 01000 )
OpenFile flags
const ( VersionV1 = 2 // Old format: all data in meta file VersionV2 = 3 // New format: per-table page files )
Storage version constants
const ( PageSize16KB = 16384 PageHeaderSizeV2 = 32 MaxPageContentV2 = PageSize16KB - PageHeaderSizeV2 // 16352 bytes )
Page size constants (16KB)
const ( PageFlagEncrypted uint8 = 1 << iota PageFlagCompressed PageFlagDirty )
Page flags
const ( ColNullMarker = 0xFF ColBlobRefMarker = 0xFE )
Column encoding constants
const ( // PageSize is the size of a data page PageSize = 4096 // PageHeaderSize is the size of the page header PageHeaderSize = 24 // MaxPageContent is the maximum content size per page MaxPageContent = PageSize - PageHeaderSize // MetaFileName is the metadata file name MetaFileName = "xxldb.meta" // WALFileName is the WAL file name WALFileName = "xxldb.wal" // CheckpointFileName is the checkpoint file name CheckpointFileName = "xxldb.chk" // DataDirName is the data directory name DataDirName = "data" // BlobDirName is the blob directory name BlobDirName = "blobs" // MagicNumber for file identification MagicNumber = 0x58584C44 // "XXLD" // CurrentVersion is the current storage version CurrentVersion = 2 )
const ( TableFileMagic = "XXLDTBL\x00" TableFileVersion = 1 TableFileExt = ".db" )
Table file constants
Variables ¶
var (
ErrEncryptionKeyRequired = fmt.Errorf("encryption key required - database is encrypted")
)
Errors
Functions ¶
func DeleteRowFromPage ¶
DeleteRowFromPage marks a row as deleted (by setting fragmented space)
func EstimateRowSize ¶
EstimateRowSize estimates the encoded size of a row
func GetCellData ¶
GetCellData returns the raw cell data for a given index This is a helper function to avoid code duplication
func GetEncryptionSalt ¶
GetEncryptionSalt reads the salt from an encrypted database Note: TXDEF encryption doesn't use salt, so this returns nil for compatibility
func GetRowFromPage ¶
func GetRowFromPage(page *PageV2, index int, colDefs []types.ColumnDef) (rowID uint64, row []types.Value, err error)
GetRowFromPage retrieves a row from a page by cell index Cell data is stored from the end of Content, growing toward the beginning. Cell pointers are stored from the beginning of Content, growing toward the end. Pointers are in decreasing order: pointer[0] is largest (first inserted), pointer[n-1] is smallest (last inserted). For each cell: data is at Content[pointer[index]:pointer[index-1]] or Content[pointer[0]:MaxContent] for index=0.
func InsertRowToPage ¶
InsertRowToPage inserts a row into a page Returns the row offset within the page, or error if page is full
func IsDatabaseEncrypted ¶
IsDatabaseEncrypted checks if a database at the given path is encrypted
func IsTempFile ¶
IsTempFile checks if a file is a temporary file (starts with .tmp.)
Types ¶
type BatchFS ¶
type BatchFS interface {
// File operations
Create(name string) (File, error)
Open(name string) (File, error)
OpenFile(name string, flag int, perm FileMode) (File, error)
// Directory operations
MkdirAll(path string, perm FileMode) error
ReadDir(dirname string) ([]DirEntry, error)
// File info
Stat(name string) (FileInfo, error)
Exists(name string) bool
Remove(name string) error
Rename(oldpath, newpath string) error
// Read/Write helpers
ReadFile(name string) ([]byte, error)
WriteFile(name string, data []byte, perm FileMode) error
// Path operations
Join(elem ...string) string
Dir(path string) string
Base(path string) string
// Cleanup
RemoveAll(path string) error
}
BatchFS provides the operations available within a batch context Operations are performed on a maintained connection
type BatchFileSystem ¶
type BatchFileSystem interface {
FileSystem
// Batch executes multiple operations within a single connection
// The provided function receives a connected filesystem that can be used
// for multiple operations. The connection is managed by the implementation.
// Example:
// fs.Batch(func(bfs BatchFS) error {
// bfs.MkdirAll("dir", 0755)
// bfs.WriteFile("dir/file.txt", data, 0644)
// return nil
// })
Batch(fn func(BatchFS) error) error
}
BatchFileSystem extends FileSystem with batch operation support Batch allows multiple operations to be performed within a single connection This is useful for remote filesystems (SFTP, SMB, WebDAV) to avoid connection overhead for each operation
type Config ¶
type Config struct {
SyncInterval time.Duration // File sync interval
CheckpointInt time.Duration // Checkpoint interval
BufferSize int // Page buffer size
AutoCheckpoint bool // Enable auto checkpoint
BlobThreshold int64 // Size threshold for external blob storage (default: 64KB, 0 = always inline)
}
Config holds storage configuration
type FTSIndexInfo ¶
type FTSIndexInfo struct {
TableName string `json:"table_name"`
ColumnName string `json:"column_name"`
IndexName string `json:"index_name"`
}
FTSIndexInfo stores full-text index metadata
type File ¶
type File interface {
io.Reader
io.Writer
io.Closer
io.Seeker
Name() string
Stat() (FileInfo, error)
Sync() error
Truncate(size int64) error
Readdir(n int) ([]FileInfo, error)
}
File represents an open file
type FileInfo ¶
type FileInfo interface {
Name() string // base name of the file
Size() int64 // length in bytes
Mode() FileMode // file mode bits
ModTime() time.Time // modification time
IsDir() bool // abbreviation for Mode().IsDir()
}
FileInfo represents file metadata
type FileMode ¶
type FileMode uint32
FileMode represents file mode bits
const ( ModeDir FileMode = 1 << (32 - 1 - iota) // d: is a directory ModeAppend // a: append-only ModeExclusive // l: exclusive use ModeTemporary // T: temporary file ModeSymlink // L: symbolic link ModeDevice // D: device file ModeNamedPipe // p: named pipe (FIFO) ModeSocket // S: Unix domain socket ModeSetuid // u: setuid ModeSetgid // g: setgid ModeCharDevice // c: Unix character device ModeSticky // t: sticky ModeType = ModeDir | ModeSymlink | ModeNamedPipe | ModeSocket | ModeDevice ModePerm FileMode = 0777 // Unix permission bits )
type FileSystem ¶
type FileSystem interface {
// File operations
Create(name string) (File, error)
Open(name string) (File, error)
OpenFile(name string, flag int, perm FileMode) (File, error)
// Directory operations
MkdirAll(path string, perm FileMode) error
ReadDir(dirname string) ([]DirEntry, error)
// File info
Stat(name string) (FileInfo, error)
Exists(name string) bool
Remove(name string) error
Rename(oldpath, newpath string) error
// Read/Write helpers
ReadFile(name string) ([]byte, error)
WriteFile(name string, data []byte, perm FileMode) error
// Path operations
Join(elem ...string) string
Dir(path string) string
Base(path string) string
Abs(path string) (string, error)
// Temp file for atomic writes
TempFile(dir, pattern string) (File, error)
// Cleanup
RemoveAll(path string) error
// Walk directory tree
Walk(root string, walkFn WalkFunc) error
// Close closes the filesystem connection
Close() error
}
FileSystem defines file system operations needed by Storage This abstraction allows for different backends (local, SFTP, etc.)
type LocalFS ¶
type LocalFS struct {
// contains filtered or unexported fields
}
LocalFS implements FileSystem using the local os package
func NewLocalFS ¶
NewLocalFS creates a new local filesystem
func (*LocalFS) SafeWriteFile ¶
SafeWriteFile writes data safely using temp file + atomic rename
type Migration ¶
type Migration struct {
// contains filtered or unexported fields
}
Migration handles data migration from V1 to V2
func NewMigration ¶
NewMigration creates a new migration handler
func (*Migration) NeedsMigration ¶
NeedsMigration checks if the database needs migration from V1 to V2
type PageCache ¶
type PageCache struct {
// contains filtered or unexported fields
}
PageCache implements an LRU page cache
func NewPageCache ¶
func NewPageCache(config PageCacheConfig) *PageCache
NewPageCache creates a new page cache
func (*PageCache) ClearDirty ¶
ClearDirty clears the dirty flag for a page
func (*PageCache) GetDirtyPages ¶
GetDirtyPages returns a list of dirty page IDs
func (*PageCache) Put ¶
Put adds a page to the cache If cache is full, evicts the least recently used page
type PageCacheConfig ¶
type PageCacheConfig struct {
MaxPages int // Maximum number of pages to cache
WriteBehind bool // Enable write-behind (dirty pages written asynchronously)
WriteBatch int // Number of dirty pages to write in a batch
}
PageCacheConfig holds configuration for page cache
func DefaultPageCacheConfig ¶
func DefaultPageCacheConfig() PageCacheConfig
DefaultPageCacheConfig returns default cache configuration
type PageHeaderV2 ¶
type PageHeaderV2 struct {
// Basic info (12 bytes)
PageID uint64 // 8 bytes - Unique page ID
PageType PageType // 1 byte - Page type
Flags uint8 // 1 byte - Flags (encrypted, compressed, dirty)
Version uint16 // 2 bytes - Format version
// Space management (8 bytes)
FreeSpace uint16 // 2 bytes - Free space offset from end
CellCount uint16 // 2 bytes - Number of cells/rows
Fragmented uint16 // 2 bytes - Fragmented space size
Reserved uint16 // 2 bytes - Reserved
// Link structure (8 bytes)
RightPage uint64 // 8 bytes - Right sibling page ID (for B-tree or data page list)
// Checksum (4 bytes)
Checksum uint32 // 4 bytes - CRC32 checksum
}
PageHeaderV2 is the header for a 16KB page Total: 32 bytes
type PageV2 ¶
type PageV2 struct {
Header PageHeaderV2
Content [MaxPageContentV2]byte
}
PageV2 represents a 16KB page
func (*PageV2) AddCellPointer ¶
AddCellPointer adds a new cell pointer
func (*PageV2) AvailableSpace ¶
AvailableSpace returns the available space in the page
func (*PageV2) CalculateChecksum ¶
CalculateChecksum calculates the CRC32 checksum of the page content
func (*PageV2) GetCellPointer ¶
GetCellPointer returns the offset of a cell by index
func (*PageV2) SetCellPointer ¶
SetCellPointer sets the offset of a cell by index
func (*PageV2) UpdateChecksum ¶
func (p *PageV2) UpdateChecksum()
UpdateChecksum updates the checksum field
func (*PageV2) UsageRatio ¶
UsageRatio returns the page usage ratio (0.0 to 1.0)
func (*PageV2) VerifyChecksum ¶
VerifyChecksum verifies the page checksum
type ResumeInfo ¶
ResumeInfo contains information needed to resume a transfer
type RowIterator ¶
type RowIterator struct {
// contains filtered or unexported fields
}
RowIterator iterates over rows in a table
func (*RowIterator) PageIDDebug ¶
func (it *RowIterator) PageIDDebug() uint64
PageIDDebug returns the current page ID for debugging
type SFTPConfig ¶
type SFTPConfig struct {
Host string
Port int
Username string
Password string
PrivateKey string // Path to private key file
PrivateKeyPEM string // Private key content (PEM format)
Passphrase string // Key passphrase
Timeout time.Duration
MaxRetries int
RetryDelay time.Duration
ConnectTimeout time.Duration
}
SFTPConfig holds SFTP connection configuration
type SFTPFS ¶
type SFTPFS struct {
// contains filtered or unexported fields
}
SFTPFS implements FileSystem using SFTP with short connections Each operation connects, executes, and disconnects Use Batch() for multiple operations on a single connection
func NewSFTPFS ¶
func NewSFTPFS(basePath string, config SFTPConfig) (*SFTPFS, error)
NewSFTPFS creates a new SFTP filesystem (short connection mode)
func (*SFTPFS) SafeWriteFile ¶
SafeWriteFile writes data safely using temp file + atomic rename
type SMBConfig ¶
type SMBConfig struct {
Host string
Port int
Username string
Password string
Domain string // Optional domain
DBPath string // Path within the share
Timeout time.Duration
MaxRetries int
RetryDelay time.Duration
}
SMBConfig holds SMB connection configuration
type SMBFS ¶
type SMBFS struct {
// contains filtered or unexported fields
}
SMBFS implements FileSystem using SMB/CIFS with short connections Each operation connects, executes, and disconnects Use Batch() for multiple operations on a single connection
func (*SMBFS) SafeWriteFile ¶
SafeWriteFile writes data safely using temp file + atomic rename
func (*SMBFS) WithContext ¶
func (fs *SMBFS) WithContext(ctx context.Context) *SMBFSContext
WithContext returns a context-aware wrapper
type SMBFSContext ¶
type SMBFSContext struct {
// contains filtered or unexported fields
}
SMBFSContext wraps SMBFS with context
type Storage ¶
type Storage struct {
// contains filtered or unexported fields
}
Storage handles data persistence
func NewStorage ¶
NewStorage creates a new storage instance
func NewStorageWithFS ¶
func NewStorageWithFS(path string, inMemory bool, fs FileSystem) (*Storage, error)
NewStorageWithFS creates a new storage instance with custom filesystem
func (*Storage) AutoMigrate ¶
AutoMigrate automatically detects and performs migration if needed
func (*Storage) ChangePassword ¶
ChangePassword changes the encryption password
func (*Storage) CloseAllTableFiles ¶
CloseAllTableFiles closes all table files
func (*Storage) CloseTableFile ¶
CloseTableFile closes a table file
func (*Storage) CreateFTSIndex ¶
CreateFTSIndex creates a full-text search index
func (*Storage) CreateTable ¶
CreateTable creates a new table
func (*Storage) DeleteBlob ¶
DeleteBlob deletes a blob from storage
func (*Storage) DeleteRows ¶
DeleteRows deletes rows matching a condition
func (*Storage) DeleteRowsWithCallback ¶
func (s *Storage) DeleteRowsWithCallback(tableName string, condition func([]types.Value) bool, callback func(uint64, []types.Value)) (int64, error)
DeleteRowsWithCallback deletes rows matching a condition and calls a callback for each deleted row The callback receives the row ID and row data, useful for maintaining FTS indexes
func (*Storage) DropFTSIndex ¶
DropFTSIndex drops a full-text search index
func (*Storage) ExportFile ¶
ExportFile exports a blob to a file
func (*Storage) GetAuthConfig ¶
GetAuthConfig returns the stored auth configuration
func (*Storage) GetDataPath ¶
GetDataPath returns the data path
func (*Storage) GetEncryptionSalt ¶
GetEncryptionSalt returns the salt used for encryption
func (*Storage) GetFTSIndexPath ¶
GetFTSIndexPath returns the file path for persisting FTS index
func (*Storage) GetFTSIndexes ¶
func (s *Storage) GetFTSIndexes() map[string]*FTSIndexInfo
GetFTSIndexes returns all FTS indexes
func (*Storage) GetPageCacheStats ¶
GetPageCacheStats returns page cache statistics
func (*Storage) GetRowsWithIDs ¶
GetRowsWithIDs gets all rows from a table with their IDs
func (*Storage) GetRowsWithoutBlobRefs ¶
GetRowsWithoutBlobRefs gets all rows without resolving blob references This is more efficient when you only need to check row structure or non-blob columns
func (*Storage) GetSequence ¶
GetSequence gets the current value of a sequence
func (*Storage) GetTableFile ¶
GetTableFile returns a TableFile for the given table, creating if necessary
func (*Storage) GetTableInfo ¶
GetTableInfo gets table information
func (*Storage) HasFTSIndex ¶
HasFTSIndex checks if an FTS index exists
func (*Storage) ImportFile ¶
ImportFile imports a file into the database
func (*Storage) IsEncrypted ¶
IsEncrypted returns whether the storage is encrypted
func (*Storage) LoadAndVerifyMetadata ¶
LoadAndVerifyMetadata loads and verifies metadata (used to verify encryption password)
func (*Storage) Reader ¶
func (s *Storage) Reader(blobID uint64) (io.ReadCloser, error)
Reader returns an io.Reader for a blob
func (*Storage) RenameTable ¶
RenameTable renames a table
func (*Storage) RollbackTxn ¶
RollbackTxn rolls back a transaction
func (*Storage) SetAuthConfig ¶
SetAuthConfig updates the auth configuration and saves metadata
func (*Storage) SetEncryption ¶
SetEncryption enables or disables encryption
func (*Storage) SetEncryptionWithSalt ¶
SetEncryptionWithSalt enables encryption with existing salt (for opening encrypted DB)
func (*Storage) SetSequence ¶
SetSequence sets the value of a sequence
func (*Storage) SetSkipSave ¶
SetSkipSave enables or disables skip-save mode for bulk imports When enabled, saveMetadata() is a no-op until disabled
func (*Storage) TruncateTable ¶
TruncateTable removes all rows from a table but keeps the table structure
func (*Storage) UpdateRows ¶
func (s *Storage) UpdateRows(tableName string, updates map[int]types.Value, condition func([]types.Value) bool) (int64, error)
UpdateRows updates rows matching a condition
func (*Storage) UpdateRowsWithCallback ¶
func (s *Storage) UpdateRowsWithCallback(tableName string, updateFunc func([]types.Value) map[int]types.Value, condition func([]types.Value) bool, callback func(uint64, []types.Value)) (int64, error)
UpdateRowsWithCallback updates rows matching a condition and calls a callback with the updated data The callback receives the row ID and the new row data, useful for maintaining FTS indexes
func (*Storage) UpdateRowsWithFunc ¶
func (s *Storage) UpdateRowsWithFunc(tableName string, updateFunc func([]types.Value) map[int]types.Value, condition func([]types.Value) bool) (int64, error)
UpdateRowsWithFunc updates rows matching a condition using a function to compute updates This allows expressions like "SET counter = counter + 1" to work correctly
type TableFile ¶
type TableFile struct {
// contains filtered or unexported fields
}
TableFile manages a table's data file
func CreateTableFile ¶
func CreateTableFile(storage *Storage, tableName string, tableID uint64, columns []types.ColumnDef) (*TableFile, error)
CreateTableFile creates a new table file
func OpenTableFile ¶
OpenTableFile opens an existing table file
func (*TableFile) CacheStats ¶
CacheStats returns cache statistics (for debugging)
func (*TableFile) GetAllRows ¶
GetAllRows returns all rows in the table
func (*TableFile) GetAllRowsWithIDs ¶
GetAllRowsWithIDs returns all rows with their IDs
func (*TableFile) GetColumnDefs ¶
GetColumnDefs returns the column definitions
func (*TableFile) GetFirstDataPage ¶
GetFirstDataPage returns the first data page ID (for debugging)
func (*TableFile) GetLastDataPage ¶
GetLastDataPage returns the last data page ID (for debugging)
func (*TableFile) GetRowCount ¶
GetRowCount returns the number of rows in the table
func (*TableFile) InsertRow ¶
InsertRow inserts a row into the table Note: This method does NOT acquire locks internally. The caller (Storage) is responsible for locking.
func (*TableFile) ReadPageDebug ¶
ReadPageDebug reads a page for debugging (public wrapper for readPage)
func (*TableFile) ScanRows ¶
func (tf *TableFile) ScanRows() *RowIterator
ScanRows returns an iterator over all rows
type TableFileHeader ¶
type TableFileHeader struct {
Magic [8]byte // "XXLDTBL\x00"
Version uint16 // File format version
TableID uint64 // Table ID (matches metadata)
PageSize uint16 // Page size (16384)
PageCount uint32 // Total pages in file
RootPage uint64 // B-tree root page (0 if no index)
FirstDataPage uint64 // First data page ID
LastDataPage uint64 // Last data page ID
FreePageList uint64 // Free page list head
RowCount uint64 // Total rows in table
Reserved [120]byte
ColumnDefs []types.ColumnDef // Column definitions (JSON serialized)
Checksum uint32 // Header checksum
}
TableFileHeader is the header of a table data file Stored in page 0 (first 16KB)
type Transaction ¶
type Transaction struct {
// contains filtered or unexported fields
}
Transaction represents a database transaction
func (*Transaction) Rollback ¶
func (t *Transaction) Rollback() error
Rollback rolls back the transaction
type TransferInfo ¶
type TransferInfo struct {
ID string `json:"id"`
SourcePath string `json:"source_path,omitempty"`
TargetPath string `json:"target_path"`
TempPath string `json:"temp_path"`
TotalSize int64 `json:"total_size"`
Transferred int64 `json:"transferred"`
Checksum string `json:"checksum,omitempty"`
Status TransferStatus `json:"status"`
StartedAt time.Time `json:"started_at"`
UpdatedAt time.Time `json:"updated_at"`
Error string `json:"error,omitempty"`
Retries int `json:"retries"`
}
TransferInfo tracks file transfer progress for recovery
type TransferState ¶
type TransferState struct {
// contains filtered or unexported fields
}
TransferState tracks file transfer progress for recovery
func NewTransferState ¶
func NewTransferState(fs FileSystem, stateFile string) *TransferState
NewTransferState creates a transfer state tracker
func (*TransferState) BeginTransfer ¶
func (ts *TransferState) BeginTransfer(id, source, target, temp string, totalSize int64)
BeginTransfer starts tracking a new transfer
func (*TransferState) CanRetry ¶
func (ts *TransferState) CanRetry(id string, maxRetries int) bool
CanRetry checks if a transfer can be retried
func (*TransferState) CleanupIncomplete ¶
func (ts *TransferState) CleanupIncomplete(fs FileSystem) []error
CleanupIncomplete removes temporary files from failed/incomplete transfers
func (*TransferState) CleanupOldTransfers ¶
func (ts *TransferState) CleanupOldTransfers(maxAge time.Duration) int
CleanupOldTransfers removes transfers older than the specified duration
func (*TransferState) CompleteTransfer ¶
func (ts *TransferState) CompleteTransfer(id string)
CompleteTransfer marks transfer as complete
func (*TransferState) Count ¶
func (ts *TransferState) Count() int
Count returns the number of tracked transfers
func (*TransferState) CountByStatus ¶
func (ts *TransferState) CountByStatus(status TransferStatus) int
CountByStatus returns the number of transfers with a specific status
func (*TransferState) FailTransfer ¶
func (ts *TransferState) FailTransfer(id string, err error)
FailTransfer marks transfer as failed
func (*TransferState) GetPendingTransfers ¶
func (ts *TransferState) GetPendingTransfers() []*TransferInfo
GetPendingTransfers returns incomplete transfers for recovery
func (*TransferState) GetResumeInfo ¶
func (ts *TransferState) GetResumeInfo(id string) *ResumeInfo
GetResumeInfo returns resume information for a pending transfer
func (*TransferState) GetTransfer ¶
func (ts *TransferState) GetTransfer(id string) *TransferInfo
GetTransfer returns transfer info by ID
func (*TransferState) IncrementRetry ¶
func (ts *TransferState) IncrementRetry(id string) int
IncrementRetry increments retry count for a transfer
func (*TransferState) UpdateProgress ¶
func (ts *TransferState) UpdateProgress(id string, transferred int64)
UpdateProgress updates transfer progress
type TransferStatus ¶
type TransferStatus string
TransferStatus represents the status of a transfer
const ( TransferStatusPending TransferStatus = "pending" TransferStatusUploading TransferStatus = "uploading" TransferStatusComplete TransferStatus = "complete" TransferStatusFailed TransferStatus = "failed" )
type WAL ¶
type WAL struct {
// contains filtered or unexported fields
}
WAL manages Write-Ahead Logging
func (*WAL) SetEncryptor ¶
SetEncryptor sets the encryptor for WAL encryption
type WALRecord ¶
type WALRecord struct {
LSN uint64 `json:"lsn"`
TxnID uint64 `json:"txn_id"`
Type WALType `json:"type"`
TableID uint64 `json:"table_id"`
RowID uint64 `json:"row_id"`
Timestamp int64 `json:"timestamp"`
Data interface{} `json:"data,omitempty"`
OldData interface{} `json:"old_data,omitempty"`
}
WALRecord represents a WAL record
type WALType ¶
type WALType uint8
WALType represents the type of WAL record
type WebDAVConfig ¶
type WebDAVConfig struct {
URL string // WebDAV server URL (e.g., "https://cloud.example.com/dav")
Username string
Password string
DBPath string // Path on WebDAV server
Timeout time.Duration
MaxRetries int
RetryDelay time.Duration
}
WebDAVConfig holds WebDAV connection configuration
type WebDAVFS ¶
type WebDAVFS struct {
// contains filtered or unexported fields
}
WebDAVFS implements FileSystem using WebDAV (short connection mode) Each operation is a separate HTTP request Use Batch() to indicate multiple operations will be performed (for optimization hints)
func NewWebDAVFS ¶
func NewWebDAVFS(basePath string, config WebDAVConfig) (*WebDAVFS, error)
NewWebDAVFS creates a new WebDAV filesystem
func (*WebDAVFS) Batch ¶
Batch executes multiple operations For WebDAV, this is primarily for API consistency since each operation is already independent However, it can be useful for grouping related operations
func (*WebDAVFS) SafeWriteFile ¶
SafeWriteFile writes data safely using temp file + atomic rename