Documentation
¶
Index ¶
- Variables
- func CSUUID(uuid string) (primitive.Binary, error)
- func Compile[T any](plan *Plan, backend Backend[T]) (T, error)
- func Execute[R any](ctx context.Context, plan *Plan, executor Executor[R]) (R, error)
- func Map(d bson.D) bson.M
- func SetSQLHandler(fn func(sql string, query *Query) *Query)
- type AnyBackend
- type AnyExecutor
- type Backend
- type BinaryExpr
- type ColumnExpr
- type Connector
- type Executor
- type Expr
- type FilterPredicate
- type FuncExpr
- type Join
- type ListExpr
- type LiteralExpr
- type LogicalExpr
- type MongoBackend
- func (b *MongoBackend) Build(plan *Plan, target any) (any, error)
- func (b *MongoBackend) Client() *mongo.Client
- func (b *MongoBackend) Close(ctx context.Context) error
- func (b *MongoBackend) Compile(plan *Plan) (*Query, error)
- func (b *MongoBackend) CompileAny(plan *Plan) (any, error)
- func (b *MongoBackend) CompileWithQuery(plan *Plan, q *Query) (*Query, error)
- func (b *MongoBackend) Connect(ctx context.Context) error
- func (b *MongoBackend) Database() *mongo.Database
- func (b *MongoBackend) Execute(ctx context.Context, plan *Plan) (*MongoResult, error)
- func (b *MongoBackend) ExecuteAny(ctx context.Context, plan *Plan) (any, error)
- func (b *MongoBackend) ExecuteQuery(ctx context.Context, q *Query) (*MongoResult, error)
- func (b *MongoBackend) Name() string
- type MongoConfig
- type MongoOption
- type MongoQuery
- type MongoResult
- type OrderDirection
- type OrderItem
- type PartitionStyle
- type Plan
- type Projection
- type Query
- type QueryBuilder
- type RadixS3Client
- func (c *RadixS3Client) Delete(key string) bool
- func (c *RadixS3Client) Get(key string) ([]byte, bool)
- func (c *RadixS3Client) GetObject(ctx context.Context, params *s3.GetObjectInput, optFns ...func(*s3.Options)) (*s3.GetObjectOutput, error)
- func (c *RadixS3Client) Keys() []string
- func (c *RadixS3Client) Len() int
- func (c *RadixS3Client) ListObjectsV2(ctx context.Context, params *s3.ListObjectsV2Input, ...) (*s3.ListObjectsV2Output, error)
- func (c *RadixS3Client) PutObject(ctx context.Context, params *s3.PutObjectInput, optFns ...func(*s3.Options)) (*s3.PutObjectOutput, error)
- func (c *RadixS3Client) Set(key string, data []byte) *RadixS3Client
- type RangeExpr
- type S3
- type S3Backend
- func (b *S3Backend) Client() S3ClientAPI
- func (b *S3Backend) Compile(plan *Plan) (*S3Query, error)
- func (b *S3Backend) CompileAny(plan *Plan) (any, error)
- func (b *S3Backend) Connect(ctx context.Context) error
- func (b *S3Backend) Downloader() *manager.Downloader
- func (b *S3Backend) Execute(ctx context.Context, plan *Plan) ([]*S3Object, error)
- func (b *S3Backend) ExecuteAny(ctx context.Context, plan *Plan) (any, error)
- func (b *S3Backend) GetBytes(ctx context.Context, plan *Plan) iter.Seq2[[]byte, error]
- func (b *S3Backend) Name() string
- func (b *S3Backend) Stream(ctx context.Context, plan *Plan) iter.Seq2[[]byte, error]
- func (b *S3Backend) StreamObjects(ctx context.Context, plan *Plan) iter.Seq2[*S3Object, error]
- func (b *S3Backend) Upload(ctx context.Context, key string, data []byte) (*manager.UploadOutput, error)
- func (b *S3Backend) Uploader() *manager.Uploader
- type S3ClientAPI
- type S3Config
- type S3Object
- type S3Option
- func WithAnonymous(anonymous bool) S3Option
- func WithAutoDetectPartitions(auto bool) S3Option
- func WithBasePrefix(prefix string) S3Option
- func WithBucket(bucket string) S3Option
- func WithCredentials(accessKeyID, secretAccessKey, sessionToken string) S3Option
- func WithDefaultPartitionKeys(keys ...string) S3Option
- func WithDownloader(downloader *manager.Downloader) S3Option
- func WithEndpoint(endpoint string) S3Option
- func WithHTTPTimeout(timeout time.Duration) S3Option
- func WithMaxDateRangePartitions(max int) S3Option
- func WithPartitionKeys(table string, keys ...string) S3Option
- func WithPartitionStyle(style PartitionStyle) S3Option
- func WithPrefetchConcurrency(concurrency int) S3Option
- func WithRegion(region string) S3Option
- func WithS3Client(client S3ClientAPI) S3Option
- func WithUploader(uploader *manager.Uploader) S3Option
- func WithUsePathStyle(usePathStyle bool) S3Option
- type S3Query
- func (q *S3Query) GetBytes(ctx context.Context, client S3ClientAPI, downloader *manager.Downloader) iter.Seq2[[]byte, error]
- func (q *S3Query) HasResidualFilters() bool
- func (q *S3Query) S3URIs() []string
- func (q *S3Query) Stream(ctx context.Context, client S3ClientAPI, downloader *manager.Downloader) iter.Seq2[*S3Object, error]
- func (q *S3Query) String() string
- type StarExpr
- type Streamer
- type SubqueryExpr
- type TableRef
Constants ¶
This section is empty.
Variables ¶
var SQLHandler func(sql string, query *Query) *Query
SQLHandler is a pluggable callback used by ParseRequest when handling the "sql" query parameter.
Functions ¶
func SetSQLHandler ¶
SetSQLHandler registers the SQL handler callback.
Types ¶
type AnyBackend ¶
AnyBackend provides a type-erased backend interface for dynamic backend dispatch.
type AnyExecutor ¶
AnyExecutor provides a type-erased execution interface for dynamic query dispatch.
type Backend ¶
type Backend[T any] interface { // Name returns the identifier of the backend (e.g. "mongodb", "s3"). Name() string // Compile transforms the abstract query plan into the backend's target representation. Compile(plan *Plan) (T, error) }
Backend defines the adapter contract that database and storage backends implement to compile an abstract Plan into their specific query or execution plan.
type BinaryExpr ¶
BinaryExpr represents a binary operation: Left Operator Right. Supported operators include: "=", "!=", ">", ">=", "<", "<=", "LIKE", "IN".
func (*BinaryExpr) String ¶
func (b *BinaryExpr) String() string
type ColumnExpr ¶
type ColumnExpr struct {
Qualifier string // Table qualifier or alias, if any
Name string // Column name
}
ColumnExpr represents a column reference (e.g., "id", "users.id", "theme.nl").
func (*ColumnExpr) FullName ¶
func (c *ColumnExpr) FullName() string
FullName returns the compliant column name, combining qualifier and name when appropriate.
func (*ColumnExpr) String ¶
func (c *ColumnExpr) String() string
type Connector ¶
Connector defines the lifecycle interface for backends that maintain connections to external systems.
type Executor ¶
Executor defines the generic execution contract for backends that run queries and return typed results.
type Expr ¶
type Expr interface {
String() string
// contains filtered or unexported methods
}
Expr represents an abstract expression node in the plan AST.
type FilterPredicate ¶
FilterPredicate represents a residual filter to apply in-memory or post-download.
func (FilterPredicate) String ¶
func (p FilterPredicate) String() string
type Join ¶
type Join struct {
Type string // "JOIN", "LEFT JOIN", "INNER JOIN", "RIGHT JOIN"
Table TableRef // Joined table reference
On Expr // The ON condition expression
LeftField string // Extracted left column name in the join condition
RightField string // Extracted right column name in the join condition
}
Join represents a JOIN clause between tables.
type ListExpr ¶
type ListExpr struct {
Elements []Expr
}
ListExpr represents a tuple or list of expressions, e.g. in IN ('a', 'b').
type LiteralExpr ¶
type LiteralExpr struct {
Value any
}
LiteralExpr represents a literal constant value (string, int64, float64, bool, etc.).
func (*LiteralExpr) String ¶
func (l *LiteralExpr) String() string
type LogicalExpr ¶
LogicalExpr represents a logical operation: AND, OR, NOT.
func (*LogicalExpr) String ¶
func (l *LogicalExpr) String() string
type MongoBackend ¶
type MongoBackend struct {
// contains filtered or unexported fields
}
MongoBackend is an adapter that translates an abstract Plan into a MongoDB Query configuration and manages live connections to MongoDB for query execution.
func NewMongo ¶
func NewMongo(opts ...MongoOption) *MongoBackend
NewMongo creates a new MongoBackend adapter instance with optional connection settings.
func NewMongoBackend ¶
func NewMongoBackend(opts ...MongoOption) *MongoBackend
NewMongoBackend creates a new MongoBackend adapter instance with optional connection settings.
func (*MongoBackend) Build ¶
func (b *MongoBackend) Build(plan *Plan, target any) (any, error)
Build implements QueryBuilder for populating an existing target structure.
func (*MongoBackend) Client ¶
func (b *MongoBackend) Client() *mongo.Client
Client returns the underlying *mongo.Client, or nil if not connected.
func (*MongoBackend) Close ¶
func (b *MongoBackend) Close(ctx context.Context) error
Close disconnects the MongoDB client if it was opened by this backend.
func (*MongoBackend) Compile ¶
func (b *MongoBackend) Compile(plan *Plan) (*Query, error)
Compile compiles an abstract Plan into a MongoDB Query.
func (*MongoBackend) CompileAny ¶
func (b *MongoBackend) CompileAny(plan *Plan) (any, error)
CompileAny implements AnyBackend.
func (*MongoBackend) CompileWithQuery ¶
func (b *MongoBackend) CompileWithQuery(plan *Plan, q *Query) (*Query, error)
CompileWithQuery compiles an abstract Plan populating an existing Query object.
func (*MongoBackend) Connect ¶
func (b *MongoBackend) Connect(ctx context.Context) error
Connect initializes the MongoDB client and database connection if not already connected.
func (*MongoBackend) Database ¶
func (b *MongoBackend) Database() *mongo.Database
Database returns the underlying *mongo.Database, or nil if not connected.
func (*MongoBackend) Execute ¶
func (b *MongoBackend) Execute(ctx context.Context, plan *Plan) (*MongoResult, error)
Execute compiles and executes the plan against the configured MongoDB database.
func (*MongoBackend) ExecuteAny ¶
ExecuteAny implements AnyExecutor for dynamic execution dispatch.
func (*MongoBackend) ExecuteQuery ¶
func (b *MongoBackend) ExecuteQuery(ctx context.Context, q *Query) (*MongoResult, error)
ExecuteQuery executes a pre-compiled Query against the configured MongoDB database.
func (*MongoBackend) Name ¶
func (b *MongoBackend) Name() string
Name returns the backend identifier.
type MongoConfig ¶
type MongoConfig struct {
URI string
Database string
Timeout time.Duration
Client *mongo.Client
DB *mongo.Database
}
MongoConfig holds connection parameters and client references for MongoDB.
type MongoOption ¶
type MongoOption func(*MongoConfig)
MongoOption is a functional configuration option for MongoBackend.
func WithMongoClient ¶
func WithMongoClient(client *mongo.Client) MongoOption
WithMongoClient sets a pre-existing mongo.Client instance.
func WithMongoDB ¶
func WithMongoDB(db *mongo.Database) MongoOption
WithMongoDB sets a pre-existing mongo.Database instance.
func WithMongoDatabase ¶
func WithMongoDatabase(name string) MongoOption
WithMongoDatabase sets the target database name.
func WithMongoTimeout ¶
func WithMongoTimeout(timeout time.Duration) MongoOption
WithMongoTimeout sets the connection/operation timeout.
func WithMongoURI ¶
func WithMongoURI(uri string) MongoOption
WithMongoURI sets the connection URI string for MongoDB.
type MongoResult ¶
type MongoResult struct {
Operation string
Cursor *mongo.Cursor
SingleResult *mongo.SingleResult
Count int64
DistinctValues []any
}
MongoResult wraps the result of executing a Query against MongoDB.
func (*MongoResult) All ¶
func (r *MongoResult) All(ctx context.Context, target any) error
All decodes all documents into the provided target slice pointer (e.g. &[]bson.M or &[]MyStruct).
type OrderDirection ¶
type OrderDirection int
OrderDirection specifies sorting direction.
const ( Asc OrderDirection = iota Desc )
func (OrderDirection) String ¶
func (d OrderDirection) String() string
type OrderItem ¶
type OrderItem struct {
Field string
Direction OrderDirection
}
OrderItem specifies an ORDER BY item with a field and direction.
type PartitionStyle ¶
type PartitionStyle int
PartitionStyle defines how partition keys are formatted into S3 path segments.
const ( // HivePartitioning formats partitions as key=value/, e.g. "department=Engineering/". // This is the standard in AWS Athena, Glue, Spark, Presto, DuckDB. HivePartitioning PartitionStyle = iota // FlatPartitioning formats partitions directly as value/, e.g. "Engineering/". FlatPartitioning )
type Plan ¶
type Plan struct {
Raw string // The original raw SQL string
Table TableRef // Primary table or collection
Joins []Join // Any table joins
Projections []Projection // Selected expressions, columns, or aggregates
Distinct bool // Whether the query specifies DISTINCT
Where Expr // Filter expression tree
GroupBy []string // Grouping column names
Having Expr // Post-aggregation having filter expression tree
OrderBy []OrderItem // Sort ordering specifications
Limit *int64 // Row limit, if specified
Offset *int64 // Row offset / skip, if specified
}
Plan is an abstract, storage-agnostic intermediate representation of a parsed SQL query. It captures the intent of the SQL query without tying it to any specific database or storage engine (e.g. MongoDB, S3, SQL databases).
type Projection ¶
type Projection struct {
Expr Expr // Expression being selected (ColumnExpr, FuncExpr, LiteralExpr, etc.)
Alias string // Optional column alias
Distinct bool // Whether this projection is DISTINCT
IsStar bool // Whether this represents a wildcard (*)
}
Projection represents a projected item in the SELECT list.
type Query ¶
type Query struct {
Context context.Context // Context for the query execution
Comment string // Optional comment for query logging/debugging
Operation string // Type of MongoDB operation (find, aggregate, etc.)
Collection string // Target MongoDB collection
Filter bson.D // Query filter criteria
Projection bson.D // Field projection specification
Sort bson.D // Sort order specification
Limit *int64 // Maximum number of documents to return
Offset *int64 // Number of documents to skip
Pipeline mongo.Pipeline // Aggregation pipeline stages
Payload bson.D // Additional query parameters
}
Query represents a MongoDB query configuration that can be built from SQL statements. It contains all the necessary components to execute various types of MongoDB operations, including find, aggregate, and distinct queries.
func NewQuery ¶
func NewQuery() *Query
NewQuery creates a new Query instance with initialized slices and default values.
func (*Query) ParseRequest ¶
ParseRequest processes an HTTP request's query parameters and updates the Query object.
func (*Query) VerifyCollection ¶
VerifyCollection checks if the query has a target collection specified.
func (*Query) VerifyContext ¶
VerifyContext checks if the query has a valid context.
func (*Query) VerifyOperation ¶
VerifyOperation checks if the query has a specified operation type.
type QueryBuilder ¶
QueryBuilder defines an optional interface for backends that support mutating or populating an existing target structure.
type RadixS3Client ¶
type RadixS3Client struct {
// contains filtered or unexported fields
}
RadixS3Client is an in-memory S3 client mock powered by an immutable radix tree. It implements S3ClientAPI (ListObjectsV2APIClient and DownloadAPIClient) and manager.UploadAPIClient, providing prefix-matched object retrieval that mirrors AWS S3 semantics.
func NewRadixS3Client ¶
func NewRadixS3Client() *RadixS3Client
NewRadixS3Client creates a new empty in-memory S3 client backed by an immutable radix tree.
func (*RadixS3Client) Delete ¶
func (c *RadixS3Client) Delete(key string) bool
Delete removes an object by key.
func (*RadixS3Client) Get ¶
func (c *RadixS3Client) Get(key string) ([]byte, bool)
Get retrieves an object payload by key.
func (*RadixS3Client) GetObject ¶
func (c *RadixS3Client) GetObject( ctx context.Context, params *s3.GetObjectInput, optFns ...func(*s3.Options), ) (*s3.GetObjectOutput, error)
GetObject implements manager.DownloadAPIClient by fetching the object payload from the radix tree.
func (*RadixS3Client) Keys ¶
func (c *RadixS3Client) Keys() []string
Keys returns all object keys stored in the radix tree in lexicographical order.
func (*RadixS3Client) Len ¶
func (c *RadixS3Client) Len() int
Len returns the total number of objects in the radix tree.
func (*RadixS3Client) ListObjectsV2 ¶
func (c *RadixS3Client) ListObjectsV2( ctx context.Context, params *s3.ListObjectsV2Input, optFns ...func(*s3.Options), ) (*s3.ListObjectsV2Output, error)
ListObjectsV2 implements s3.ListObjectsV2APIClient by walking the radix tree at the specified prefix.
func (*RadixS3Client) PutObject ¶
func (c *RadixS3Client) PutObject( ctx context.Context, params *s3.PutObjectInput, optFns ...func(*s3.Options), ) (*s3.PutObjectOutput, error)
PutObject implements manager.UploadAPIClient by storing the object payload into the radix tree.
func (*RadixS3Client) Set ¶
func (c *RadixS3Client) Set(key string, data []byte) *RadixS3Client
Set inserts or updates an object payload at the specified key.
type S3Backend ¶
type S3Backend struct {
// contains filtered or unexported fields
}
S3Backend is an adapter that translates an abstract Plan into an S3Query with prefix pruning and provides streaming execution against AWS S3 compatible object stores.
func NewS3Backend ¶
NewS3Backend creates a new S3Backend adapter configured with the provided options.
func (*S3Backend) Client ¶
func (b *S3Backend) Client() S3ClientAPI
Client returns the underlying S3ClientAPI instance.
func (*S3Backend) CompileAny ¶
CompileAny implements AnyBackend.
func (*S3Backend) Connect ¶
Connect initializes the AWS S3 client, downloader, and uploader if not already configured.
func (*S3Backend) Downloader ¶
func (b *S3Backend) Downloader() *manager.Downloader
Downloader returns the underlying manager.Downloader instance.
func (*S3Backend) Execute ¶
Execute downloads and collects all matching objects for the plan, satisfying Executor[[]*S3Object].
func (*S3Backend) ExecuteAny ¶
ExecuteAny implements AnyExecutor for dynamic query execution dispatch.
func (*S3Backend) Stream ¶
Stream compiles the plan and streams raw object byte slices, satisfying the Streamer interface.
func (*S3Backend) StreamObjects ¶
StreamObjects compiles the plan and streams matching S3Object items using push iterators.
type S3ClientAPI ¶
type S3ClientAPI interface {
s3.ListObjectsV2APIClient
manager.DownloadAPIClient
}
S3ClientAPI specifies the AWS S3 client operations used for object listing and downloading.
type S3Config ¶
type S3Config struct {
Bucket string // Target S3 bucket name (e.g. "my-lake-bucket")
BasePrefix string // Base path prefix prepended to all table paths (e.g. "data/" or "")
PartitionStyle PartitionStyle // HivePartitioning (default) or FlatPartitioning
PartitionKeys map[string][]string // Per-table ordered partition key hierarchy (e.g. {"orders": {"year", "month"}})
DefaultPartitionKeys []string // Default partition keys for tables not in PartitionKeys
AutoDetectPartitions bool // If true, detects equality/IN conditions as partition keys when none configured
MaxDateRangePartitions int // Maximum days to enumerate for date range partition pruning (default 365)
PrefetchConcurrency int // Concurrent prefetch downloads in Stream pipeline (default 16)
// Live Connection Configuration
Endpoint string // Optional custom S3 endpoint (e.g. MinIO, LocalStack)
Region string // AWS Region (e.g. "us-east-1", "eu-central-1")
AccessKeyID string // AWS Access Key ID
SecretAccessKey string // AWS Secret Access Key
SessionToken string // Optional AWS Session Token
Anonymous bool // Connect with anonymous credentials
UsePathStyle bool // Enable S3 path-style addressing
HTTPTimeout time.Duration // Response header timeout for S3 requests
Client S3ClientAPI // Pre-configured S3 client (e.g. *s3.Client)
Downloader *manager.Downloader // S3 download manager
Uploader *manager.Uploader // S3 upload manager
}
S3Config holds configuration options for the S3 backend adapter and live S3 connections.
type S3Object ¶
S3Object represents a downloaded S3 object with its key, bucket, and binary payload.
type S3Option ¶
type S3Option func(*S3Config)
S3Option is a functional configuration option for S3Backend.
func WithAnonymous ¶
WithAnonymous sets anonymous credentials for public S3 buckets.
func WithAutoDetectPartitions ¶
WithAutoDetectPartitions enables or disables partition key auto-detection.
func WithBasePrefix ¶
WithBasePrefix sets a common base prefix for all table paths in S3.
func WithCredentials ¶
WithCredentials sets static AWS credentials.
func WithDefaultPartitionKeys ¶
WithDefaultPartitionKeys sets default partition columns for any table without explicit configuration.
func WithDownloader ¶
func WithDownloader(downloader *manager.Downloader) S3Option
WithDownloader sets a pre-configured S3 download manager.
func WithEndpoint ¶
WithEndpoint sets a custom S3 base endpoint (e.g. for MinIO, LocalStack, Ceph).
func WithHTTPTimeout ¶
WithHTTPTimeout sets a custom response header timeout for HTTP requests.
func WithMaxDateRangePartitions ¶
WithMaxDateRangePartitions sets the maximum number of daily partition prefixes to expand for date ranges.
func WithPartitionKeys ¶
WithPartitionKeys specifies the ordered partition column hierarchy for a given table.
func WithPartitionStyle ¶
func WithPartitionStyle(style PartitionStyle) S3Option
WithPartitionStyle sets the partition path format (HivePartitioning or FlatPartitioning).
func WithPrefetchConcurrency ¶
WithPrefetchConcurrency sets the concurrent prefetch download pipeline capacity in Stream.
func WithS3Client ¶
func WithS3Client(client S3ClientAPI) S3Option
WithS3Client sets a pre-configured S3ClientAPI (e.g. *s3.Client).
func WithUploader ¶
WithUploader sets a pre-configured S3 upload manager.
func WithUsePathStyle ¶
WithUsePathStyle enables or disables path-style S3 URLs (bucket in path rather than subdomain).
type S3Query ¶
type S3Query struct {
Bucket string // S3 bucket name
Table string // Target table name
TableBase string // Base path for table in S3 (e.g. "symm/" or "data/symm/")
Prefixes []string // S3 key prefixes that must be listed and downloaded
Columns []string // Projected column names (empty if SELECT *)
IsStar bool // True if SELECT *
Predicates []FilterPredicate // Residual filters to apply to downloaded data
Limit *int64 // Limit, if specified
Offset *int64 // Offset, if specified
Distinct bool // Whether SELECT DISTINCT
PartitionKeys []string // Partition keys recognized for this table
PartitionStyle PartitionStyle // HivePartitioning or FlatPartitioning
KeyConstraints map[string][]string // Equality/IN constraint values for any partition key in WHERE
OrderBy []OrderItem // OrderBy items from query
PrefetchConcurrency int // Sliding-window prefetch download concurrency (default 16)
}
S3Query represents the compiled plan for querying data stored in S3. It specifies the exact S3 key prefixes that need to be downloaded, along with projected columns and any residual filters that must be evaluated on the records.
func (*S3Query) GetBytes ¶
func (q *S3Query) GetBytes( ctx context.Context, client S3ClientAPI, downloader *manager.Downloader, ) iter.Seq2[[]byte, error]
GetBytes streams raw byte buffers for all matching objects in this query.
func (*S3Query) HasResidualFilters ¶
HasResidualFilters returns true if there are filter conditions that could not be fully resolved by prefix pruning.
func (*S3Query) Stream ¶
func (q *S3Query) Stream( ctx context.Context, client S3ClientAPI, downloader *manager.Downloader, ) iter.Seq2[*S3Object, error]
Stream streams matching objects from S3 for this compiled query in exact sequence order across prefixes, filtering out non-matching middle partition keys, applying OFFSET/LIMIT to metadata before downloading, and using a bounded sliding-window concurrent prefetch pipeline.
type StarExpr ¶
type StarExpr struct {
Qualifier string
}
StarExpr represents a wildcard selector (* or table.*).
type SubqueryExpr ¶
type SubqueryExpr struct {
Plan *Plan
}
SubqueryExpr represents a nested subquery.
func (*SubqueryExpr) String ¶
func (s *SubqueryExpr) String() string