storage

package
v0.0.0-...-079bc26 Latest Latest
Warning

This package is not in the latest version of its module.

Go to latest
Published: Apr 8, 2026 License: MIT Imports: 25 Imported by: 0

Documentation

Overview

Package storage provides DeleteRowsWithCallback for FTS support

Package storage provides data persistence for xxldb

Package storage provides data persistence for xxldb

Package storage provides migration from V1 to V2 storage format

Package storage provides page-based storage for XxLdb

Package storage provides page cache for XxLdb

Package storage provides data persistence for xxldb

Package storage provides data persistence for xxldb

Package storage provides table file management for XxLdb

Package storage provides data persistence for xxldb

Package storage provides UpdateRowsWithCallback for FTS support

Package storage provides WAL (Write-Ahead Logging) support

Package storage provides data persistence for xxldb

Index

Constants

View Source
const (
	FlagO_RDONLY int = 0
	FlagO_WRONLY int = 1
	FlagO_RDWR   int = 2
	FlagO_APPEND int = 0400
	FlagO_CREATE int = 0100
	FlagO_EXCL   int = 0200
	FlagO_SYNC   int = 01000
	FlagO_TRUNC  int = 01000
)

OpenFile flags

View Source
const (
	VersionV1 = 2 // Old format: all data in meta file
	VersionV2 = 3 // New format: per-table page files
)

Storage version constants

View Source
const (
	PageSize16KB     = 16384
	PageHeaderSizeV2 = 32
	MaxPageContentV2 = PageSize16KB - PageHeaderSizeV2 // 16352 bytes
)

Page size constants (16KB)

View Source
const (
	PageFlagEncrypted uint8 = 1 << iota
	PageFlagCompressed
	PageFlagDirty
)

Page flags

View Source
const (
	ColNullMarker    = 0xFF
	ColBlobRefMarker = 0xFE
)

Column encoding constants

View Source
const (
	// PageSize is the size of a data page
	PageSize = 4096

	// PageHeaderSize is the size of the page header
	PageHeaderSize = 24

	// MaxPageContent is the maximum content size per page
	MaxPageContent = PageSize - PageHeaderSize

	// MetaFileName is the metadata file name
	MetaFileName = "xxldb.meta"

	// WALFileName is the WAL file name
	WALFileName = "xxldb.wal"

	// CheckpointFileName is the checkpoint file name
	CheckpointFileName = "xxldb.chk"

	// DataDirName is the data directory name
	DataDirName = "data"

	// BlobDirName is the blob directory name
	BlobDirName = "blobs"

	// MagicNumber for file identification
	MagicNumber = 0x58584C44 // "XXLD"

	// CurrentVersion is the current storage version
	CurrentVersion = 2
)
View Source
const (
	TableFileMagic   = "XXLDTBL\x00"
	TableFileVersion = 1
	TableFileExt     = ".db"
)

Table file constants

Variables

View Source
var (
	ErrEncryptionKeyRequired = fmt.Errorf("encryption key required - database is encrypted")
)

Errors

Functions

func DecodeRow

func DecodeRow(data []byte, colDefs []types.ColumnDef) (rowID uint64, row []types.Value, err error)

DecodeRow decodes a row from bytes

func DeleteRowFromPage

func DeleteRowFromPage(page *PageV2, index int) error

DeleteRowFromPage marks a row as deleted (by setting fragmented space)

func EncodeRow

func EncodeRow(rowID uint64, row []types.Value) []byte

EncodeRow encodes a row into bytes

func EstimateRowSize

func EstimateRowSize(row []types.Value) int

EstimateRowSize estimates the encoded size of a row

func GetCellData

func GetCellData(page *PageV2, index int) ([]byte, error)

GetCellData returns the raw cell data for a given index This is a helper function to avoid code duplication

func GetEncryptionSalt

func GetEncryptionSalt(path string) ([]byte, error)

GetEncryptionSalt reads the salt from an encrypted database Note: TXDEF encryption doesn't use salt, so this returns nil for compatibility

func GetRowFromPage

func GetRowFromPage(page *PageV2, index int, colDefs []types.ColumnDef) (rowID uint64, row []types.Value, err error)

GetRowFromPage retrieves a row from a page by cell index Cell data is stored from the end of Content, growing toward the beginning. Cell pointers are stored from the beginning of Content, growing toward the end. Pointers are in decreasing order: pointer[0] is largest (first inserted), pointer[n-1] is smallest (last inserted). For each cell: data is at Content[pointer[index]:pointer[index-1]] or Content[pointer[0]:MaxContent] for index=0.

func InsertRowToPage

func InsertRowToPage(page *PageV2, rowID uint64, row []types.Value) (int, error)

InsertRowToPage inserts a row into a page Returns the row offset within the page, or error if page is full

func IsDatabaseEncrypted

func IsDatabaseEncrypted(path string) bool

IsDatabaseEncrypted checks if a database at the given path is encrypted

func IsTempFile

func IsTempFile(name string) bool

IsTempFile checks if a file is a temporary file (starts with .tmp.)

Types

type BatchFS

type BatchFS interface {
	// File operations
	Create(name string) (File, error)
	Open(name string) (File, error)
	OpenFile(name string, flag int, perm FileMode) (File, error)

	// Directory operations
	MkdirAll(path string, perm FileMode) error
	ReadDir(dirname string) ([]DirEntry, error)

	// File info
	Stat(name string) (FileInfo, error)
	Exists(name string) bool
	Remove(name string) error
	Rename(oldpath, newpath string) error

	// Read/Write helpers
	ReadFile(name string) ([]byte, error)
	WriteFile(name string, data []byte, perm FileMode) error

	// Path operations
	Join(elem ...string) string
	Dir(path string) string
	Base(path string) string

	// Cleanup
	RemoveAll(path string) error
}

BatchFS provides the operations available within a batch context Operations are performed on a maintained connection

type BatchFileSystem

type BatchFileSystem interface {
	FileSystem

	// Batch executes multiple operations within a single connection
	// The provided function receives a connected filesystem that can be used
	// for multiple operations. The connection is managed by the implementation.
	// Example:
	//   fs.Batch(func(bfs BatchFS) error {
	//       bfs.MkdirAll("dir", 0755)
	//       bfs.WriteFile("dir/file.txt", data, 0644)
	//       return nil
	//   })
	Batch(fn func(BatchFS) error) error
}

BatchFileSystem extends FileSystem with batch operation support Batch allows multiple operations to be performed within a single connection This is useful for remote filesystems (SFTP, SMB, WebDAV) to avoid connection overhead for each operation

type Config

type Config struct {
	SyncInterval   time.Duration // File sync interval
	CheckpointInt  time.Duration // Checkpoint interval
	BufferSize     int           // Page buffer size
	AutoCheckpoint bool          // Enable auto checkpoint
	BlobThreshold  int64         // Size threshold for external blob storage (default: 64KB, 0 = always inline)
}

Config holds storage configuration

func DefaultConfig

func DefaultConfig() Config

DefaultConfig returns the default configuration

type DirEntry

type DirEntry interface {
	Name() string
	IsDir() bool
	Type() FileMode
	Info() (FileInfo, error)
}

DirEntry represents a directory entry

type FTSIndexInfo

type FTSIndexInfo struct {
	TableName  string `json:"table_name"`
	ColumnName string `json:"column_name"`
	IndexName  string `json:"index_name"`
}

FTSIndexInfo stores full-text index metadata

type File

type File interface {
	io.Reader
	io.Writer
	io.Closer
	io.Seeker

	Name() string
	Stat() (FileInfo, error)
	Sync() error
	Truncate(size int64) error
	Readdir(n int) ([]FileInfo, error)
}

File represents an open file

type FileInfo

type FileInfo interface {
	Name() string       // base name of the file
	Size() int64        // length in bytes
	Mode() FileMode     // file mode bits
	ModTime() time.Time // modification time
	IsDir() bool        // abbreviation for Mode().IsDir()
}

FileInfo represents file metadata

type FileMode

type FileMode uint32

FileMode represents file mode bits

const (
	ModeDir        FileMode = 1 << (32 - 1 - iota) // d: is a directory
	ModeAppend                                     // a: append-only
	ModeExclusive                                  // l: exclusive use
	ModeTemporary                                  // T: temporary file
	ModeSymlink                                    // L: symbolic link
	ModeDevice                                     // D: device file
	ModeNamedPipe                                  // p: named pipe (FIFO)
	ModeSocket                                     // S: Unix domain socket
	ModeSetuid                                     // u: setuid
	ModeSetgid                                     // g: setgid
	ModeCharDevice                                 // c: Unix character device
	ModeSticky                                     // t: sticky

	ModeType = ModeDir | ModeSymlink | ModeNamedPipe | ModeSocket | ModeDevice

	ModePerm FileMode = 0777 // Unix permission bits
)
const (
	PermDir  FileMode = 0755
	PermFile FileMode = 0644
)

FileMode constants

type FileSystem

type FileSystem interface {
	// File operations
	Create(name string) (File, error)
	Open(name string) (File, error)
	OpenFile(name string, flag int, perm FileMode) (File, error)

	// Directory operations
	MkdirAll(path string, perm FileMode) error
	ReadDir(dirname string) ([]DirEntry, error)

	// File info
	Stat(name string) (FileInfo, error)
	Exists(name string) bool
	Remove(name string) error
	Rename(oldpath, newpath string) error

	// Read/Write helpers
	ReadFile(name string) ([]byte, error)
	WriteFile(name string, data []byte, perm FileMode) error

	// Path operations
	Join(elem ...string) string
	Dir(path string) string
	Base(path string) string
	Abs(path string) (string, error)

	// Temp file for atomic writes
	TempFile(dir, pattern string) (File, error)

	// Cleanup
	RemoveAll(path string) error

	// Walk directory tree
	Walk(root string, walkFn WalkFunc) error

	// Close closes the filesystem connection
	Close() error
}

FileSystem defines file system operations needed by Storage This abstraction allows for different backends (local, SFTP, etc.)

type LocalFS

type LocalFS struct {
	// contains filtered or unexported fields
}

LocalFS implements FileSystem using the local os package

func NewLocalFS

func NewLocalFS(basePath string) *LocalFS

NewLocalFS creates a new local filesystem

func (*LocalFS) Abs

func (fs *LocalFS) Abs(path string) (string, error)

Abs returns an absolute path

func (*LocalFS) Base

func (fs *LocalFS) Base(path string) string

Base returns the base name of a path

func (*LocalFS) Close

func (fs *LocalFS) Close() error

Close closes the filesystem (no-op for local filesystem)

func (*LocalFS) Create

func (fs *LocalFS) Create(name string) (File, error)

Create creates a new file

func (*LocalFS) Dir

func (fs *LocalFS) Dir(path string) string

Dir returns the directory part of a path

func (*LocalFS) Exists

func (fs *LocalFS) Exists(name string) bool

Exists checks if a file exists

func (*LocalFS) Join

func (fs *LocalFS) Join(elem ...string) string

Join joins path elements

func (*LocalFS) MkdirAll

func (fs *LocalFS) MkdirAll(path string, perm FileMode) error

MkdirAll creates a directory tree

func (*LocalFS) Open

func (fs *LocalFS) Open(name string) (File, error)

Open opens an existing file

func (*LocalFS) OpenFile

func (fs *LocalFS) OpenFile(name string, flag int, perm FileMode) (File, error)

OpenFile opens a file with specified flags and permissions

func (*LocalFS) ReadDir

func (fs *LocalFS) ReadDir(dirname string) ([]DirEntry, error)

ReadDir reads directory contents

func (*LocalFS) ReadFile

func (fs *LocalFS) ReadFile(name string) ([]byte, error)

ReadFile reads entire file content

func (*LocalFS) Remove

func (fs *LocalFS) Remove(name string) error

Remove removes a file

func (*LocalFS) RemoveAll

func (fs *LocalFS) RemoveAll(path string) error

RemoveAll removes a directory tree

func (*LocalFS) Rename

func (fs *LocalFS) Rename(oldpath, newpath string) error

Rename renames a file

func (*LocalFS) SafeWriteFile

func (fs *LocalFS) SafeWriteFile(path string, data []byte, perm FileMode) error

SafeWriteFile writes data safely using temp file + atomic rename

func (*LocalFS) Stat

func (fs *LocalFS) Stat(name string) (FileInfo, error)

Stat returns file info

func (*LocalFS) TempFile

func (fs *LocalFS) TempFile(dir, pattern string) (File, error)

TempFile creates a temporary file

func (*LocalFS) Walk

func (fs *LocalFS) Walk(root string, walkFn WalkFunc) error

Walk walks the file tree

func (*LocalFS) WriteFile

func (fs *LocalFS) WriteFile(name string, data []byte, perm FileMode) error

WriteFile writes data to a file

type Migration

type Migration struct {
	// contains filtered or unexported fields
}

Migration handles data migration from V1 to V2

func NewMigration

func NewMigration(storage *Storage) *Migration

NewMigration creates a new migration handler

func (*Migration) Migrate

func (m *Migration) Migrate() error

Migrate performs the migration from V1 to V2

func (*Migration) NeedsMigration

func (m *Migration) NeedsMigration() bool

NeedsMigration checks if the database needs migration from V1 to V2

type PageCache

type PageCache struct {
	// contains filtered or unexported fields
}

PageCache implements an LRU page cache

func NewPageCache

func NewPageCache(config PageCacheConfig) *PageCache

NewPageCache creates a new page cache

func (*PageCache) Clear

func (c *PageCache) Clear()

Clear clears all pages from the cache

func (*PageCache) ClearDirty

func (c *PageCache) ClearDirty(pageID uint64)

ClearDirty clears the dirty flag for a page

func (*PageCache) Flush

func (c *PageCache) Flush(writeFunc func(*PageV2) error) error

Flush flushes all dirty pages using the provided write function

func (*PageCache) FlushPage

func (c *PageCache) FlushPage(pageID uint64, writeFunc func(*PageV2) error) error

FlushPage flushes a specific page

func (*PageCache) Get

func (c *PageCache) Get(pageID uint64) *PageV2

Get retrieves a page from the cache Returns nil if page is not in cache

func (*PageCache) GetDirtyPages

func (c *PageCache) GetDirtyPages() []uint64

GetDirtyPages returns a list of dirty page IDs

func (*PageCache) IsDirty

func (c *PageCache) IsDirty(pageID uint64) bool

IsDirty checks if a page is dirty

func (*PageCache) MarkDirty

func (c *PageCache) MarkDirty(pageID uint64)

MarkDirty marks a page as dirty (modified)

func (*PageCache) Put

func (c *PageCache) Put(page *PageV2) error

Put adds a page to the cache If cache is full, evicts the least recently used page

func (*PageCache) Remove

func (c *PageCache) Remove(pageID uint64)

Remove removes a page from the cache

func (*PageCache) Resize

func (c *PageCache) Resize(maxPages int) error

Resize changes the maximum cache size

func (*PageCache) Size

func (c *PageCache) Size() int

Size returns the number of pages in the cache

func (*PageCache) Stats

func (c *PageCache) Stats() map[string]interface{}

Stats returns cache statistics

type PageCacheConfig

type PageCacheConfig struct {
	MaxPages    int  // Maximum number of pages to cache
	WriteBehind bool // Enable write-behind (dirty pages written asynchronously)
	WriteBatch  int  // Number of dirty pages to write in a batch
}

PageCacheConfig holds configuration for page cache

func DefaultPageCacheConfig

func DefaultPageCacheConfig() PageCacheConfig

DefaultPageCacheConfig returns default cache configuration

type PageHeaderV2

type PageHeaderV2 struct {
	// Basic info (12 bytes)
	PageID   uint64   // 8 bytes - Unique page ID
	PageType PageType // 1 byte - Page type
	Flags    uint8    // 1 byte - Flags (encrypted, compressed, dirty)
	Version  uint16   // 2 bytes - Format version

	// Space management (8 bytes)
	FreeSpace  uint16 // 2 bytes - Free space offset from end
	CellCount  uint16 // 2 bytes - Number of cells/rows
	Fragmented uint16 // 2 bytes - Fragmented space size
	Reserved   uint16 // 2 bytes - Reserved

	// Link structure (8 bytes)
	RightPage uint64 // 8 bytes - Right sibling page ID (for B-tree or data page list)

	// Checksum (4 bytes)
	Checksum uint32 // 4 bytes - CRC32 checksum
}

PageHeaderV2 is the header for a 16KB page Total: 32 bytes

type PageType

type PageType uint8

PageType represents the type of a page

const (
	PageTypeFree      PageType = 0 // Free/unused page
	PageTypeData      PageType = 1 // Data page (stores row records)
	PageTypeOverflow  PageType = 2 // Overflow page (large row data)
	PageTypeBTree     PageType = 3 // B-tree index page
	PageTypeBTreeLeaf PageType = 4 // B-tree leaf page
)

func (PageType) String

func (pt PageType) String() string

String returns the string representation of page type

type PageV2

type PageV2 struct {
	Header  PageHeaderV2
	Content [MaxPageContentV2]byte
}

PageV2 represents a 16KB page

func NewPageV2

func NewPageV2(pageID uint64, pageType PageType) *PageV2

NewPageV2 creates a new page with the given ID and type

func (*PageV2) AddCellPointer

func (p *PageV2) AddCellPointer(cellOffset uint16)

AddCellPointer adds a new cell pointer

func (*PageV2) AvailableSpace

func (p *PageV2) AvailableSpace() uint16

AvailableSpace returns the available space in the page

func (*PageV2) CalculateChecksum

func (p *PageV2) CalculateChecksum() uint32

CalculateChecksum calculates the CRC32 checksum of the page content

func (*PageV2) CanInsert

func (p *PageV2) CanInsert(rowSize int) bool

CanInsert checks if a row of the given size can be inserted

func (*PageV2) FromBytes

func (p *PageV2) FromBytes(data []byte) error

FromBytes deserializes the page from bytes

func (*PageV2) GetCellPointer

func (p *PageV2) GetCellPointer(index int) uint16

GetCellPointer returns the offset of a cell by index

func (*PageV2) SetCellPointer

func (p *PageV2) SetCellPointer(index int, cellOffset uint16)

SetCellPointer sets the offset of a cell by index

func (*PageV2) ToBytes

func (p *PageV2) ToBytes() []byte

ToBytes serializes the page to bytes

func (*PageV2) UpdateChecksum

func (p *PageV2) UpdateChecksum()

UpdateChecksum updates the checksum field

func (*PageV2) UsageRatio

func (p *PageV2) UsageRatio() float64

UsageRatio returns the page usage ratio (0.0 to 1.0)

func (*PageV2) VerifyChecksum

func (p *PageV2) VerifyChecksum() bool

VerifyChecksum verifies the page checksum

type ResumeInfo

type ResumeInfo struct {
	Offset   int64
	TempPath string
}

ResumeInfo contains information needed to resume a transfer

type RowIterator

type RowIterator struct {
	// contains filtered or unexported fields
}

RowIterator iterates over rows in a table

func (*RowIterator) Err

func (it *RowIterator) Err() error

Err returns any error encountered

func (*RowIterator) Next

func (it *RowIterator) Next() bool

Next advances to the next row

func (*RowIterator) PageIDDebug

func (it *RowIterator) PageIDDebug() uint64

PageIDDebug returns the current page ID for debugging

func (*RowIterator) Row

func (it *RowIterator) Row() (uint64, []types.Value, error)

Row returns the current row

type RowWithID

type RowWithID struct {
	ID  uint64
	Row []types.Value
}

RowWithID represents a row with its ID

type SFTPConfig

type SFTPConfig struct {
	Host           string
	Port           int
	Username       string
	Password       string
	PrivateKey     string // Path to private key file
	PrivateKeyPEM  string // Private key content (PEM format)
	Passphrase     string // Key passphrase
	Timeout        time.Duration
	MaxRetries     int
	RetryDelay     time.Duration
	ConnectTimeout time.Duration
}

SFTPConfig holds SFTP connection configuration

type SFTPFS

type SFTPFS struct {
	// contains filtered or unexported fields
}

SFTPFS implements FileSystem using SFTP with short connections Each operation connects, executes, and disconnects Use Batch() for multiple operations on a single connection

func NewSFTPFS

func NewSFTPFS(basePath string, config SFTPConfig) (*SFTPFS, error)

NewSFTPFS creates a new SFTP filesystem (short connection mode)

func (*SFTPFS) Abs

func (fs *SFTPFS) Abs(path string) (string, error)

Abs returns an absolute path

func (*SFTPFS) Base

func (fs *SFTPFS) Base(path string) string

Base returns the base name of a path

func (*SFTPFS) Batch

func (fs *SFTPFS) Batch(fn func(BatchFS) error) error

Batch executes multiple operations within a single connection

func (*SFTPFS) Close

func (fs *SFTPFS) Close() error

Close closes the filesystem

func (*SFTPFS) Create

func (fs *SFTPFS) Create(name string) (File, error)

Create creates a new file

func (*SFTPFS) Dir

func (fs *SFTPFS) Dir(path string) string

Dir returns the directory part of a path

func (*SFTPFS) Exists

func (fs *SFTPFS) Exists(name string) bool

Exists checks if a file exists

func (*SFTPFS) Join

func (fs *SFTPFS) Join(elem ...string) string

Join joins path elements

func (*SFTPFS) MkdirAll

func (fs *SFTPFS) MkdirAll(path string, perm FileMode) error

MkdirAll creates a directory tree

func (*SFTPFS) Open

func (fs *SFTPFS) Open(name string) (File, error)

Open opens an existing file

func (*SFTPFS) OpenFile

func (fs *SFTPFS) OpenFile(name string, flag int, perm FileMode) (File, error)

OpenFile opens a file with specified flags

func (*SFTPFS) ReadDir

func (fs *SFTPFS) ReadDir(dirname string) ([]DirEntry, error)

ReadDir reads directory contents

func (*SFTPFS) ReadFile

func (fs *SFTPFS) ReadFile(name string) ([]byte, error)

ReadFile reads entire file content

func (*SFTPFS) Remove

func (fs *SFTPFS) Remove(name string) error

Remove removes a file

func (*SFTPFS) RemoveAll

func (fs *SFTPFS) RemoveAll(path string) error

RemoveAll removes a directory tree

func (*SFTPFS) Rename

func (fs *SFTPFS) Rename(oldpath, newpath string) error

Rename renames a file

func (*SFTPFS) SafeWriteFile

func (fs *SFTPFS) SafeWriteFile(path string, data []byte, perm FileMode) error

SafeWriteFile writes data safely using temp file + atomic rename

func (*SFTPFS) Stat

func (fs *SFTPFS) Stat(name string) (FileInfo, error)

Stat returns file info

func (*SFTPFS) TempFile

func (fs *SFTPFS) TempFile(dir, pattern string) (File, error)

TempFile creates a temporary file

func (*SFTPFS) Walk

func (fs *SFTPFS) Walk(root string, walkFn WalkFunc) error

Walk walks the file tree

func (*SFTPFS) WriteFile

func (fs *SFTPFS) WriteFile(name string, data []byte, perm FileMode) error

WriteFile writes data to a file

type SMBConfig

type SMBConfig struct {
	Host       string
	Port       int
	Share      string // Share name (e.g., "shared")
	Username   string
	Password   string
	Domain     string // Optional domain
	DBPath     string // Path within the share
	Timeout    time.Duration
	MaxRetries int
	RetryDelay time.Duration
}

SMBConfig holds SMB connection configuration

type SMBFS

type SMBFS struct {
	// contains filtered or unexported fields
}

SMBFS implements FileSystem using SMB/CIFS with short connections Each operation connects, executes, and disconnects Use Batch() for multiple operations on a single connection

func NewSMBFS

func NewSMBFS(basePath string, config SMBConfig) (*SMBFS, error)

NewSMBFS creates a new SMB filesystem (short connection mode)

func (*SMBFS) Abs

func (fs *SMBFS) Abs(path string) (string, error)

Abs returns an absolute path

func (*SMBFS) Base

func (fs *SMBFS) Base(path string) string

Base returns the base name of a path

func (*SMBFS) Batch

func (fs *SMBFS) Batch(fn func(BatchFS) error) error

Batch executes multiple operations within a single connection

func (*SMBFS) Close

func (fs *SMBFS) Close() error

Close closes the filesystem

func (*SMBFS) Create

func (fs *SMBFS) Create(name string) (File, error)

Create creates a new file

func (*SMBFS) Dir

func (fs *SMBFS) Dir(path string) string

Dir returns the directory part of a path

func (*SMBFS) Exists

func (fs *SMBFS) Exists(name string) bool

Exists checks if a file exists

func (*SMBFS) Join

func (fs *SMBFS) Join(elem ...string) string

Join joins path elements

func (*SMBFS) MkdirAll

func (fs *SMBFS) MkdirAll(path string, perm FileMode) error

MkdirAll creates a directory tree

func (*SMBFS) Open

func (fs *SMBFS) Open(name string) (File, error)

Open opens an existing file

func (*SMBFS) OpenFile

func (fs *SMBFS) OpenFile(name string, flag int, perm FileMode) (File, error)

OpenFile opens a file with specified flags

func (*SMBFS) ReadDir

func (fs *SMBFS) ReadDir(dirname string) ([]DirEntry, error)

ReadDir reads directory contents

func (*SMBFS) ReadFile

func (fs *SMBFS) ReadFile(name string) ([]byte, error)

ReadFile reads entire file content

func (*SMBFS) Remove

func (fs *SMBFS) Remove(name string) error

Remove removes a file

func (*SMBFS) RemoveAll

func (fs *SMBFS) RemoveAll(path string) error

RemoveAll removes a directory tree

func (*SMBFS) Rename

func (fs *SMBFS) Rename(oldpath, newpath string) error

Rename renames a file

func (*SMBFS) SafeWriteFile

func (fs *SMBFS) SafeWriteFile(path string, data []byte, perm FileMode) error

SafeWriteFile writes data safely using temp file + atomic rename

func (*SMBFS) Stat

func (fs *SMBFS) Stat(name string) (FileInfo, error)

Stat returns file info

func (*SMBFS) TempFile

func (fs *SMBFS) TempFile(dir, pattern string) (File, error)

TempFile creates a temporary file

func (*SMBFS) Walk

func (fs *SMBFS) Walk(root string, walkFn WalkFunc) error

Walk walks the file tree

func (*SMBFS) WithContext

func (fs *SMBFS) WithContext(ctx context.Context) *SMBFSContext

WithContext returns a context-aware wrapper

func (*SMBFS) WriteFile

func (fs *SMBFS) WriteFile(name string, data []byte, perm FileMode) error

WriteFile writes data to a file

type SMBFSContext

type SMBFSContext struct {
	// contains filtered or unexported fields
}

SMBFSContext wraps SMBFS with context

type Storage

type Storage struct {
	// contains filtered or unexported fields
}

Storage handles data persistence

func NewStorage

func NewStorage(path string, inMemory bool) (*Storage, error)

NewStorage creates a new storage instance

func NewStorageWithFS

func NewStorageWithFS(path string, inMemory bool, fs FileSystem) (*Storage, error)

NewStorageWithFS creates a new storage instance with custom filesystem

func (*Storage) AutoMigrate

func (s *Storage) AutoMigrate() error

AutoMigrate automatically detects and performs migration if needed

func (*Storage) Backup

func (s *Storage) Backup(backupPath string) error

Backup creates a backup of the database

func (*Storage) BeginTxn

func (s *Storage) BeginTxn() uint64

BeginTxn begins a transaction

func (*Storage) ChangePassword

func (s *Storage) ChangePassword(oldPassword, newPassword string) error

ChangePassword changes the encryption password

func (*Storage) Checkpoint

func (s *Storage) Checkpoint() error

Checkpoint creates a checkpoint

func (*Storage) Close

func (s *Storage) Close() error

Close closes the storage

func (*Storage) CloseAllTableFiles

func (s *Storage) CloseAllTableFiles() error

CloseAllTableFiles closes all table files

func (*Storage) CloseTableFile

func (s *Storage) CloseTableFile(tableName string) error

CloseTableFile closes a table file

func (*Storage) CommitTxn

func (s *Storage) CommitTxn(txnID uint64) error

CommitTxn commits a transaction

func (*Storage) CreateFTSIndex

func (s *Storage) CreateFTSIndex(tableName, columnName, indexName string) error

CreateFTSIndex creates a full-text search index

func (*Storage) CreateTable

func (s *Storage) CreateTable(info *types.TableInfo) error

CreateTable creates a new table

func (*Storage) DeleteBlob

func (s *Storage) DeleteBlob(blobID uint64) error

DeleteBlob deletes a blob from storage

func (*Storage) DeleteRows

func (s *Storage) DeleteRows(tableName string, condition func([]types.Value) bool) (int64, error)

DeleteRows deletes rows matching a condition

func (*Storage) DeleteRowsWithCallback

func (s *Storage) DeleteRowsWithCallback(tableName string, condition func([]types.Value) bool, callback func(uint64, []types.Value)) (int64, error)

DeleteRowsWithCallback deletes rows matching a condition and calls a callback for each deleted row The callback receives the row ID and row data, useful for maintaining FTS indexes

func (*Storage) DropFTSIndex

func (s *Storage) DropFTSIndex(tableName, columnName string) error

DropFTSIndex drops a full-text search index

func (*Storage) DropTable

func (s *Storage) DropTable(name string) error

DropTable drops a table

func (*Storage) ExportFile

func (s *Storage) ExportFile(blobID uint64, filePath string) error

ExportFile exports a blob to a file

func (*Storage) FlushAll

func (s *Storage) FlushAll() error

FlushAll flushes all table files and the page cache

func (*Storage) ForceSave

func (s *Storage) ForceSave() error

ForceSave forces a save even if skipSave is enabled, and truncates WAL

func (*Storage) GetAuthConfig

func (s *Storage) GetAuthConfig() map[string]interface{}

GetAuthConfig returns the stored auth configuration

func (*Storage) GetConfig

func (s *Storage) GetConfig() Config

GetConfig returns the current storage configuration

func (*Storage) GetDataPath

func (s *Storage) GetDataPath() string

GetDataPath returns the data path

func (*Storage) GetEncryptionSalt

func (s *Storage) GetEncryptionSalt() []byte

GetEncryptionSalt returns the salt used for encryption

func (*Storage) GetFTSIndexPath

func (s *Storage) GetFTSIndexPath(tableName, columnName string) string

GetFTSIndexPath returns the file path for persisting FTS index

func (*Storage) GetFTSIndexes

func (s *Storage) GetFTSIndexes() map[string]*FTSIndexInfo

GetFTSIndexes returns all FTS indexes

func (*Storage) GetPageCacheStats

func (s *Storage) GetPageCacheStats() map[string]interface{}

GetPageCacheStats returns page cache statistics

func (*Storage) GetRows

func (s *Storage) GetRows(tableName string) ([][]types.Value, error)

GetRows gets all rows from a table

func (*Storage) GetRowsWithIDs

func (s *Storage) GetRowsWithIDs(tableName string) ([]RowWithID, error)

GetRowsWithIDs gets all rows from a table with their IDs

func (*Storage) GetRowsWithoutBlobRefs

func (s *Storage) GetRowsWithoutBlobRefs(tableName string) ([][]types.Value, error)

GetRowsWithoutBlobRefs gets all rows without resolving blob references This is more efficient when you only need to check row structure or non-blob columns

func (*Storage) GetSequence

func (s *Storage) GetSequence(name string) int64

GetSequence gets the current value of a sequence

func (*Storage) GetTableFile

func (s *Storage) GetTableFile(tableName string) (*TableFile, error)

GetTableFile returns a TableFile for the given table, creating if necessary

func (*Storage) GetTableInfo

func (s *Storage) GetTableInfo(name string) (*types.TableInfo, error)

GetTableInfo gets table information

func (*Storage) HasFTSIndex

func (s *Storage) HasFTSIndex(tableName, columnName string) bool

HasFTSIndex checks if an FTS index exists

func (*Storage) ImportFile

func (s *Storage) ImportFile(filePath string) (uint64, error)

ImportFile imports a file into the database

func (*Storage) InsertRow

func (s *Storage) InsertRow(tableName string, row []types.Value) (uint64, int64, error)

InsertRow inserts a row into a table

func (*Storage) IsEncrypted

func (s *Storage) IsEncrypted() bool

IsEncrypted returns whether the storage is encrypted

func (*Storage) ListTables

func (s *Storage) ListTables() []string

ListTables lists all tables

func (*Storage) LoadAndVerifyMetadata

func (s *Storage) LoadAndVerifyMetadata() error

LoadAndVerifyMetadata loads and verifies metadata (used to verify encryption password)

func (*Storage) ReadBlob

func (s *Storage) ReadBlob(blobID uint64) ([]byte, error)

ReadBlob reads a blob from storage

func (*Storage) Reader

func (s *Storage) Reader(blobID uint64) (io.ReadCloser, error)

Reader returns an io.Reader for a blob

func (*Storage) RenameTable

func (s *Storage) RenameTable(oldName, newName string) error

RenameTable renames a table

func (*Storage) RollbackTxn

func (s *Storage) RollbackTxn(txnID uint64) error

RollbackTxn rolls back a transaction

func (*Storage) SetAuthConfig

func (s *Storage) SetAuthConfig(authConfig map[string]interface{}) error

SetAuthConfig updates the auth configuration and saves metadata

func (*Storage) SetConfig

func (s *Storage) SetConfig(config Config)

SetConfig updates the storage configuration

func (*Storage) SetEncryption

func (s *Storage) SetEncryption(password string) error

SetEncryption enables or disables encryption

func (*Storage) SetEncryptionWithSalt

func (s *Storage) SetEncryptionWithSalt(password string, salt []byte) error

SetEncryptionWithSalt enables encryption with existing salt (for opening encrypted DB)

func (*Storage) SetSequence

func (s *Storage) SetSequence(name string, value int64)

SetSequence sets the value of a sequence

func (*Storage) SetSkipSave

func (s *Storage) SetSkipSave(skip bool)

SetSkipSave enables or disables skip-save mode for bulk imports When enabled, saveMetadata() is a no-op until disabled

func (*Storage) Stats

func (s *Storage) Stats() map[string]interface{}

Stats returns storage statistics

func (*Storage) TruncateTable

func (s *Storage) TruncateTable(name string) error

TruncateTable removes all rows from a table but keeps the table structure

func (*Storage) UpdateRows

func (s *Storage) UpdateRows(tableName string, updates map[int]types.Value, condition func([]types.Value) bool) (int64, error)

UpdateRows updates rows matching a condition

func (*Storage) UpdateRowsWithCallback

func (s *Storage) UpdateRowsWithCallback(tableName string, updateFunc func([]types.Value) map[int]types.Value, condition func([]types.Value) bool, callback func(uint64, []types.Value)) (int64, error)

UpdateRowsWithCallback updates rows matching a condition and calls a callback with the updated data The callback receives the row ID and the new row data, useful for maintaining FTS indexes

func (*Storage) UpdateRowsWithFunc

func (s *Storage) UpdateRowsWithFunc(tableName string, updateFunc func([]types.Value) map[int]types.Value, condition func([]types.Value) bool) (int64, error)

UpdateRowsWithFunc updates rows matching a condition using a function to compute updates This allows expressions like "SET counter = counter + 1" to work correctly

func (*Storage) WriteBlob

func (s *Storage) WriteBlob(data []byte) (uint64, error)

WriteBlob writes a blob to storage

func (*Storage) Writer

func (s *Storage) Writer() (io.WriteCloser, uint64, error)

Writer returns an io.Writer for a blob

type TableFile

type TableFile struct {
	// contains filtered or unexported fields
}

TableFile manages a table's data file

func CreateTableFile

func CreateTableFile(storage *Storage, tableName string, tableID uint64, columns []types.ColumnDef) (*TableFile, error)

CreateTableFile creates a new table file

func OpenTableFile

func OpenTableFile(storage *Storage, tableName string) (*TableFile, error)

OpenTableFile opens an existing table file

func (*TableFile) CacheSize

func (tf *TableFile) CacheSize() int

CacheSize returns the number of pages in cache (for debugging)

func (*TableFile) CacheStats

func (tf *TableFile) CacheStats() map[string]interface{}

CacheStats returns cache statistics (for debugging)

func (*TableFile) Clear

func (tf *TableFile) Clear() error

Clear removes all rows from the table

func (*TableFile) Close

func (tf *TableFile) Close() error

Close closes the table file

func (*TableFile) DeleteRow

func (tf *TableFile) DeleteRow(rowID uint64) error

DeleteRow deletes a row by row ID

func (*TableFile) FilePath

func (tf *TableFile) FilePath() string

FilePath returns the file path

func (*TableFile) Flush

func (tf *TableFile) Flush() error

Flush flushes all dirty pages to disk

func (*TableFile) GetAllRows

func (tf *TableFile) GetAllRows() ([][]types.Value, error)

GetAllRows returns all rows in the table

func (*TableFile) GetAllRowsWithIDs

func (tf *TableFile) GetAllRowsWithIDs() ([]RowWithID, error)

GetAllRowsWithIDs returns all rows with their IDs

func (*TableFile) GetColumnDefs

func (tf *TableFile) GetColumnDefs() []types.ColumnDef

GetColumnDefs returns the column definitions

func (*TableFile) GetFirstDataPage

func (tf *TableFile) GetFirstDataPage() uint64

GetFirstDataPage returns the first data page ID (for debugging)

func (*TableFile) GetLastDataPage

func (tf *TableFile) GetLastDataPage() uint64

GetLastDataPage returns the last data page ID (for debugging)

func (*TableFile) GetRow

func (tf *TableFile) GetRow(rowID uint64) ([]types.Value, error)

GetRow retrieves a row by row ID

func (*TableFile) GetRowCount

func (tf *TableFile) GetRowCount() uint64

GetRowCount returns the number of rows in the table

func (*TableFile) InsertRow

func (tf *TableFile) InsertRow(row []types.Value) (uint64, error)

InsertRow inserts a row into the table Note: This method does NOT acquire locks internally. The caller (Storage) is responsible for locking.

func (*TableFile) PageCount

func (tf *TableFile) PageCount() uint32

PageCount returns the number of pages

func (*TableFile) PageSize

func (tf *TableFile) PageSize() int

PageSize returns the page size

func (*TableFile) ReadPageDebug

func (tf *TableFile) ReadPageDebug(pageID uint64) (*PageV2, error)

ReadPageDebug reads a page for debugging (public wrapper for readPage)

func (*TableFile) ScanRows

func (tf *TableFile) ScanRows() *RowIterator

ScanRows returns an iterator over all rows

func (*TableFile) UpdateRow

func (tf *TableFile) UpdateRow(rowID uint64, row []types.Value) error

UpdateRow updates a row by row ID

type TableFileHeader

type TableFileHeader struct {
	Magic         [8]byte // "XXLDTBL\x00"
	Version       uint16  // File format version
	TableID       uint64  // Table ID (matches metadata)
	PageSize      uint16  // Page size (16384)
	PageCount     uint32  // Total pages in file
	RootPage      uint64  // B-tree root page (0 if no index)
	FirstDataPage uint64  // First data page ID
	LastDataPage  uint64  // Last data page ID
	FreePageList  uint64  // Free page list head
	RowCount      uint64  // Total rows in table
	Reserved      [120]byte
	ColumnDefs    []types.ColumnDef // Column definitions (JSON serialized)
	Checksum      uint32            // Header checksum
}

TableFileHeader is the header of a table data file Stored in page 0 (first 16KB)

type TableInfo

type TableInfo struct {
	types.TableInfo
	DataFile string `json:"data_file"`
}

TableInfo stores table metadata (extends types.TableInfo)

type Transaction

type Transaction struct {
	// contains filtered or unexported fields
}

Transaction represents a database transaction

func (*Transaction) Commit

func (t *Transaction) Commit() error

Commit commits the transaction

func (*Transaction) ID

func (t *Transaction) ID() uint64

ID returns the transaction ID

func (*Transaction) Rollback

func (t *Transaction) Rollback() error

Rollback rolls back the transaction

type TransferInfo

type TransferInfo struct {
	ID          string         `json:"id"`
	SourcePath  string         `json:"source_path,omitempty"`
	TargetPath  string         `json:"target_path"`
	TempPath    string         `json:"temp_path"`
	TotalSize   int64          `json:"total_size"`
	Transferred int64          `json:"transferred"`
	Checksum    string         `json:"checksum,omitempty"`
	Status      TransferStatus `json:"status"`
	StartedAt   time.Time      `json:"started_at"`
	UpdatedAt   time.Time      `json:"updated_at"`
	Error       string         `json:"error,omitempty"`
	Retries     int            `json:"retries"`
}

TransferInfo tracks file transfer progress for recovery

type TransferState

type TransferState struct {
	// contains filtered or unexported fields
}

TransferState tracks file transfer progress for recovery

func NewTransferState

func NewTransferState(fs FileSystem, stateFile string) *TransferState

NewTransferState creates a transfer state tracker

func (*TransferState) BeginTransfer

func (ts *TransferState) BeginTransfer(id, source, target, temp string, totalSize int64)

BeginTransfer starts tracking a new transfer

func (*TransferState) CanRetry

func (ts *TransferState) CanRetry(id string, maxRetries int) bool

CanRetry checks if a transfer can be retried

func (*TransferState) CleanupIncomplete

func (ts *TransferState) CleanupIncomplete(fs FileSystem) []error

CleanupIncomplete removes temporary files from failed/incomplete transfers

func (*TransferState) CleanupOldTransfers

func (ts *TransferState) CleanupOldTransfers(maxAge time.Duration) int

CleanupOldTransfers removes transfers older than the specified duration

func (*TransferState) CompleteTransfer

func (ts *TransferState) CompleteTransfer(id string)

CompleteTransfer marks transfer as complete

func (*TransferState) Count

func (ts *TransferState) Count() int

Count returns the number of tracked transfers

func (*TransferState) CountByStatus

func (ts *TransferState) CountByStatus(status TransferStatus) int

CountByStatus returns the number of transfers with a specific status

func (*TransferState) FailTransfer

func (ts *TransferState) FailTransfer(id string, err error)

FailTransfer marks transfer as failed

func (*TransferState) GetPendingTransfers

func (ts *TransferState) GetPendingTransfers() []*TransferInfo

GetPendingTransfers returns incomplete transfers for recovery

func (*TransferState) GetResumeInfo

func (ts *TransferState) GetResumeInfo(id string) *ResumeInfo

GetResumeInfo returns resume information for a pending transfer

func (*TransferState) GetTransfer

func (ts *TransferState) GetTransfer(id string) *TransferInfo

GetTransfer returns transfer info by ID

func (*TransferState) IncrementRetry

func (ts *TransferState) IncrementRetry(id string) int

IncrementRetry increments retry count for a transfer

func (*TransferState) UpdateProgress

func (ts *TransferState) UpdateProgress(id string, transferred int64)

UpdateProgress updates transfer progress

type TransferStatus

type TransferStatus string

TransferStatus represents the status of a transfer

const (
	TransferStatusPending   TransferStatus = "pending"
	TransferStatusUploading TransferStatus = "uploading"
	TransferStatusComplete  TransferStatus = "complete"
	TransferStatusFailed    TransferStatus = "failed"
)

type WAL

type WAL struct {
	// contains filtered or unexported fields
}

WAL manages Write-Ahead Logging

func NewWAL

func NewWAL(path string) (*WAL, error)

NewWAL creates a new WAL instance

func (*WAL) Close

func (w *WAL) Close() error

Close closes the WAL file

func (*WAL) CurrentLSN

func (w *WAL) CurrentLSN() uint64

CurrentLSN returns the current LSN

func (*WAL) ReadAll

func (w *WAL) ReadAll() ([]WALRecord, error)

ReadAll reads all records from the WAL

func (*WAL) SetEncryptor

func (w *WAL) SetEncryptor(encryptor *crypto.Encryptor)

SetEncryptor sets the encryptor for WAL encryption

func (*WAL) Truncate

func (w *WAL) Truncate() error

Truncate truncates the WAL file

func (*WAL) Write

func (w *WAL) Write(record WALRecord) (uint64, error)

Write writes a record to the WAL

type WALHeader

type WALHeader struct {
	LSN      uint64
	Length   uint32
	Checksum uint32
}

WALHeader is the header for each WAL record on disk

type WALRecord

type WALRecord struct {
	LSN       uint64      `json:"lsn"`
	TxnID     uint64      `json:"txn_id"`
	Type      WALType     `json:"type"`
	TableID   uint64      `json:"table_id"`
	RowID     uint64      `json:"row_id"`
	Timestamp int64       `json:"timestamp"`
	Data      interface{} `json:"data,omitempty"`
	OldData   interface{} `json:"old_data,omitempty"`
}

WALRecord represents a WAL record

type WALType

type WALType uint8

WALType represents the type of WAL record

const (
	WALTypeBegin WALType = iota
	WALTypeInsert
	WALTypeUpdate
	WALTypeDelete
	WALTypeCreateTable
	WALTypeDropTable
	WALTypeCommit
	WALTypeRollback
	WALTypeCheckpoint
	WALTypeRenameTable
	WALTypeTruncateTable
)

WAL record types

func (WALType) String

func (t WALType) String() string

String returns the string representation

type WalkFunc

type WalkFunc func(path string, info FileInfo, err error) error

WalkFunc is the type of function called by Walk to visit each file or directory

type WebDAVConfig

type WebDAVConfig struct {
	URL        string // WebDAV server URL (e.g., "https://cloud.example.com/dav")
	Username   string
	Password   string
	DBPath     string // Path on WebDAV server
	Timeout    time.Duration
	MaxRetries int
	RetryDelay time.Duration
}

WebDAVConfig holds WebDAV connection configuration

type WebDAVFS

type WebDAVFS struct {
	// contains filtered or unexported fields
}

WebDAVFS implements FileSystem using WebDAV (short connection mode) Each operation is a separate HTTP request Use Batch() to indicate multiple operations will be performed (for optimization hints)

func NewWebDAVFS

func NewWebDAVFS(basePath string, config WebDAVConfig) (*WebDAVFS, error)

NewWebDAVFS creates a new WebDAV filesystem

func (*WebDAVFS) Abs

func (fs *WebDAVFS) Abs(path string) (string, error)

Abs returns an absolute path

func (*WebDAVFS) Base

func (fs *WebDAVFS) Base(path string) string

Base returns the base name of a path

func (*WebDAVFS) Batch

func (fs *WebDAVFS) Batch(fn func(BatchFS) error) error

Batch executes multiple operations For WebDAV, this is primarily for API consistency since each operation is already independent However, it can be useful for grouping related operations

func (*WebDAVFS) Close

func (fs *WebDAVFS) Close() error

Close closes the filesystem (no persistent connection)

func (*WebDAVFS) Create

func (fs *WebDAVFS) Create(name string) (File, error)

Create creates a new file

func (*WebDAVFS) Dir

func (fs *WebDAVFS) Dir(path string) string

Dir returns the directory part of a path

func (*WebDAVFS) Exists

func (fs *WebDAVFS) Exists(name string) bool

Exists checks if a file exists

func (*WebDAVFS) Join

func (fs *WebDAVFS) Join(elem ...string) string

Join joins path elements

func (*WebDAVFS) MkdirAll

func (fs *WebDAVFS) MkdirAll(path string, perm FileMode) error

MkdirAll creates a directory tree

func (*WebDAVFS) Open

func (fs *WebDAVFS) Open(name string) (File, error)

Open opens an existing file

func (*WebDAVFS) OpenFile

func (fs *WebDAVFS) OpenFile(name string, flag int, perm FileMode) (File, error)

OpenFile opens a file with specified flags and permissions

func (*WebDAVFS) ReadDir

func (fs *WebDAVFS) ReadDir(dirname string) ([]DirEntry, error)

ReadDir reads directory contents

func (*WebDAVFS) ReadFile

func (fs *WebDAVFS) ReadFile(name string) ([]byte, error)

ReadFile reads entire file content

func (*WebDAVFS) Remove

func (fs *WebDAVFS) Remove(name string) error

Remove removes a file

func (*WebDAVFS) RemoveAll

func (fs *WebDAVFS) RemoveAll(path string) error

RemoveAll removes a directory tree

func (*WebDAVFS) Rename

func (fs *WebDAVFS) Rename(oldpath, newpath string) error

Rename renames a file

func (*WebDAVFS) SafeWriteFile

func (fs *WebDAVFS) SafeWriteFile(path string, data []byte, perm FileMode) error

SafeWriteFile writes data safely using temp file + atomic rename

func (*WebDAVFS) Stat

func (fs *WebDAVFS) Stat(name string) (FileInfo, error)

Stat returns file info

func (*WebDAVFS) TempFile

func (fs *WebDAVFS) TempFile(dir, pattern string) (File, error)

TempFile creates a temporary file

func (*WebDAVFS) Walk

func (fs *WebDAVFS) Walk(root string, walkFn WalkFunc) error

Walk walks the file tree

func (*WebDAVFS) WriteFile

func (fs *WebDAVFS) WriteFile(name string, data []byte, perm FileMode) error

WriteFile writes data to a file

Directories

Path Synopsis
Package crypto provides TXDEF encryption for xxldb
Package crypto provides TXDEF encryption for xxldb

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
⌂ 换网址