Documentation
¶
Overview ¶
Package flatjsonl implements a CLI tool to scan JSONL files and transform them to flat tables.
Index ¶
- Constants
- func Format(v interface{}) string
- func KeyFromPath(path []string) string
- func Main()
- func PrepareRegex(reg string) string
- type CSVWriter
- type Config
- type DuckDBCLIWriter
- type FastWalker
- type Flags
- type Input
- type KeyHierarchy
- type LoopReader
- type PGDumpWriter
- type ParquetWriter
- type Processor
- type RawWriter
- type Reader
- type SQLite3CLIWriter
- type SQLiteWriter
- type Type
- type URL
- type Value
- type WriteReceiver
- type Writer
Constants ¶
const ( TypeArray = Type("array") TypeObject = Type("object") TypeString = Type("string") TypeInt = Type("int") TypeFloat = Type("float") TypeBool = Type("bool") TypeNull = Type("null") TypeJSON = Type("json") TypeAbsent = Type("") )
Type enumeration.
const NopFile = "<nop>"
NopFile indicates a no op file.
Variables ¶
This section is empty.
Functions ¶
func KeyFromPath ¶
KeyFromPath joins path elements into a dot-separated scalar key.
func PrepareRegex ¶ added in v0.8.3
PrepareRegex converts * syntax to regex.
Types ¶
type CSVWriter ¶
type CSVWriter struct {
// contains filtered or unexported fields
}
CSVWriter writes rows to CSV file.
func NewCSVWriter ¶
NewCSVWriter creates an instance of CSVWriter.
func (*CSVWriter) ReceiveRow ¶
ReceiveRow receives rows.
type Config ¶
type Config struct {
MatchLinePrefix string `json:"matchLinePrefix" yaml:"matchLinePrefix"`
IncludeKeys []string `json:"includeKeys" yaml:"includeKeys"`
IncludeKeysRegex []string `json:"includeKeysRegex" yaml:"includeKeysRegex"`
ExcludeKeys []string `json:"excludeKeys" yaml:"excludeKeys" description:"List of keys remove from columns."`
ExcludeKeysRegex []string `json:"excludeKeysRegex" yaml:"excludeKeysRegex" description:"List of key regex to remove keys from columns."`
ReplaceKeys map[string]string `json:"replaceKeys" yaml:"replaceKeys"`
ReplaceKeysRegex map[string]string `json:"replaceKeysRegex" yaml:"replaceKeysRegex"`
ParseTime map[string]string `json:"parseTime" yaml:"parseTime" description:"Map of key to time format, RAW format means no processing of original value."`
OutputTimeFormat string `` /* 140-byte string literal not displayed */
OutputTimezone string `json:"outputTZ" yaml:"outputTZ" example:"UTC"`
ConcatDelimiter *string `` /* 152-byte string literal not displayed */
Transpose map[string]string `json:"transpose" yaml:"transpose" description:"Map of key prefixes to transposed table names."`
ExtractValuesRegex map[string]extract `` /* 186-byte string literal not displayed */
KeepJSON []string `json:"keepJSON" yaml:"keepJSON" description:"List of keys to keep as JSON literals."`
KeepJSONRegex []string `json:"keepJSONRegex" yaml:"keepJSONRegex" description:"List of key patterns to keep as JSON literals."`
AllowCardinality []string `json:"allowCardinality" yaml:"allowCardinality" description:"List of keys to allow high cardinality of child keys."`
}
Config describes processing options.
type DuckDBCLIWriter ¶ added in v0.10.3
type DuckDBCLIWriter struct {
// contains filtered or unexported fields
}
DuckDBCLIWriter writes data to DuckDB DB with CLI CSV import.
func NewDuckDBCLIWriter ¶ added in v0.10.3
func NewDuckDBCLIWriter(fn string, tableName string, nullValue string) (*DuckDBCLIWriter, error)
NewDuckDBCLIWriter creates DuckDB CLI writer.
func (*DuckDBCLIWriter) Close ¶ added in v0.10.3
func (w *DuckDBCLIWriter) Close() error
Close flushes all the remainders and closes resources.
func (*DuckDBCLIWriter) ReceiveRow ¶ added in v0.10.3
func (w *DuckDBCLIWriter) ReceiveRow(seq int64, values []Value) error
ReceiveRow collects data values.
func (*DuckDBCLIWriter) SetupKeys ¶ added in v0.10.3
func (w *DuckDBCLIWriter) SetupKeys(keys []flKey) error
SetupKeys inits writer with list of known keys.
type FastWalker ¶ added in v0.3.0
type FastWalker struct {
// These callbacks are invoked during JSON traversal.
// Common arguments:
// * seq is a sequence number of parent line,
// * flatPath is a dot-separated path to the current element,
// * parserPool is a length of parent prefix in flatPath,
// * path holds a list of segments, it is nil if WantPath is false.
FnObjectStop func(seq int64, flatPath []byte, pl int, path []string) (stop bool)
FnArrayStop func(seq int64, flatPath []byte, pl int, path []string) (stop bool)
FnNumber func(seq int64, flatPath []byte, pl int, path []string, value float64, raw []byte)
FnString func(seq int64, flatPath []byte, pl int, path []string, value []byte) []extractor
FnBool func(seq int64, flatPath []byte, pl int, path []string, value bool)
FnNull func(seq int64, flatPath []byte, pl int, path []string)
WantPath bool
ExtractStrings bool
KeepJSON map[string]bool
KeepJSONRegex []*regexp.Regexp
// contains filtered or unexported fields
}
FastWalker walks JSON with fastjson.
func (*FastWalker) WalkFastJSON ¶ added in v0.3.0
func (fv *FastWalker) WalkFastJSON(seq int64, flatPath []byte, pl int, path []string, v *fastjson.Value)
WalkFastJSON iterates fastjson.Value JSON structure.
type Flags ¶
type Flags struct {
Verbosity int
ProgressInterval time.Duration
InPlace bool
Input string
Output string
CSV string
CSVNull string
Parquet string
ParquetCompression string
DuckDB string
SQLite string
SQLiteInstance *sql.DB
SQLiteCLI bool
SQLMaxCols int
SQLTable string
PGDump string
Raw string
RawDelim string
MaxLines int
OffsetLines int
MaxLinesKeys int
FieldLimit int
ChildrenLimitObject int
ChildrenLimitArray int
KeyLimit int
BufSize int
Config string
GetKey string
ReplaceKeys bool
StripKeys bool
ExtractStrings bool
SkipZeroCols bool
AddSequence bool
MatchLinePrefix string
CaseSensitiveKeys bool
ShowKeysFlat bool
ShowKeysHier bool
ShowKeysInfo bool
ShowJSONSchema bool
Concurrency int
MemLimit int
}
Flags contains command-line flags.
func (*Flags) LoadGeoIPDB ¶ added in v0.10.0
LoadGeoIPDB loads a GeoIP database from the specified file.
func (*Flags) LoadNetrieDB ¶ added in v0.10.0
LoadNetrieDB loads a netrie database.
type Input ¶ added in v0.5.0
type Input struct {
FileName string
Reader interface {
io.Reader
Size() int64
Reset()
Compression() string
}
}
Input can be either a file name or a reader.
type KeyHierarchy ¶ added in v0.4.0
type KeyHierarchy struct {
Name string
Sub map[string]KeyHierarchy
}
KeyHierarchy collects structural relations.
func (*KeyHierarchy) Add ¶ added in v0.4.0
func (k *KeyHierarchy) Add(path []string)
Add registers path to KeyHierarchy.
func (KeyHierarchy) Hierarchy ¶ added in v0.4.0
func (k KeyHierarchy) Hierarchy() interface{}
Hierarchy exposes keys as tree hierarchy.
type LoopReader ¶ added in v0.5.5
type LoopReader struct {
BytesLimit int
// contains filtered or unexported fields
}
LoopReader repeats bytes buffer until the limit is hit.
func LoopReaderFromFile ¶ added in v0.5.5
func LoopReaderFromFile(fn string, bytesLimit int) (*LoopReader, error)
LoopReaderFromFile creates LoopReader from a file.
func (*LoopReader) Compression ¶ added in v0.5.5
func (l *LoopReader) Compression() string
Compression implements Input.
type PGDumpWriter ¶ added in v0.5.6
type PGDumpWriter struct {
// contains filtered or unexported fields
}
PGDumpWriter creates PostgreSQL dump file. .
func NewPGDumpWriter ¶ added in v0.5.6
func NewPGDumpWriter(fn string, tableName string, p *Processor) (*PGDumpWriter, error)
NewPGDumpWriter creates an instance of PGDumpWriter.
func (*PGDumpWriter) Close ¶ added in v0.5.6
func (c *PGDumpWriter) Close() error
Close flushes CSV and closes output file.
func (*PGDumpWriter) ReceiveRow ¶ added in v0.5.6
func (c *PGDumpWriter) ReceiveRow(seq int64, values []Value) error
ReceiveRow receives rows.
func (*PGDumpWriter) SetupKeys ¶ added in v0.5.6
func (c *PGDumpWriter) SetupKeys(keys []flKey) error
SetupKeys creates tables.
type ParquetWriter ¶ added in v0.10.2
type ParquetWriter struct {
// contains filtered or unexported fields
}
ParquetWriter writes rows to Parquet file.
func NewParquetWriter ¶ added in v0.10.2
func NewParquetWriter(fn string, compression string, p *Processor) (*ParquetWriter, error)
NewParquetWriter creates an instance of ParquetWriter.
func (*ParquetWriter) Close ¶ added in v0.10.2
func (c *ParquetWriter) Close() error
Close flushes rows and closes file.
func (*ParquetWriter) ReceiveRow ¶ added in v0.10.2
func (c *ParquetWriter) ReceiveRow(seq int64, values []Value) error
ReceiveRow receives rows.
func (*ParquetWriter) SetupKeys ¶ added in v0.10.2
func (c *ParquetWriter) SetupKeys(keys []flKey) (err error)
SetupKeys initializes writer.
type Processor ¶
type Processor struct {
Log func(args ...any)
Stdout io.Writer
// contains filtered or unexported fields
}
Processor reads JSONL files with Reader and passes flat rows to Writer.
func NewProcessor ¶
NewProcessor creates an instance of Processor.
func (*Processor) PrepareKeys ¶ added in v0.5.0
PrepareKeys runs first pass of reading if necessary to scan the keys.
func (*Processor) ProgressStatus ¶ added in v0.10.6
ProgressStatus returns the latest progress status string and the time it was updated.
func (*Processor) WriteOutput ¶ added in v0.5.0
WriteOutput runs second pass of reading to create the output.
type RawWriter ¶ added in v0.3.19
type RawWriter struct {
// contains filtered or unexported fields
}
RawWriter writes rows to RAW file.
func NewRawWriter ¶ added in v0.3.19
NewRawWriter creates an instance of RawWriter.
func (*RawWriter) ReceiveRow ¶ added in v0.3.19
ReceiveRow receives rows.
type Reader ¶
type Reader struct {
AddSequence bool
MaxLines int64
OffsetLines int64
OnError func(err error)
Progress *progress.Progress
Buf []byte
Concurrency int
Processor *Processor
Sequence int64
MatchPrefix *regexp.Regexp
ExtractStrings bool
// contains filtered or unexported fields
}
Reader scans lines and decodes JSON in them.
type SQLite3CLIWriter ¶ added in v0.9.7
type SQLite3CLIWriter struct {
// contains filtered or unexported fields
}
SQLite3CLIWriter writes data to SQLite3 DB with CLI CSV import.
func NewSQLite3CLIWriter ¶ added in v0.9.7
func NewSQLite3CLIWriter(fn string, tableName string, nullValue string) (*SQLite3CLIWriter, error)
NewSQLite3CLIWriter creates SQLite3 CLI writer.
func (*SQLite3CLIWriter) Close ¶ added in v0.9.7
func (w *SQLite3CLIWriter) Close() error
Close flushes all the remainders and closes resources.
func (*SQLite3CLIWriter) ReceiveRow ¶ added in v0.9.7
func (w *SQLite3CLIWriter) ReceiveRow(seq int64, values []Value) error
ReceiveRow collects data values.
func (*SQLite3CLIWriter) SetupKeys ¶ added in v0.9.7
func (w *SQLite3CLIWriter) SetupKeys(keys []flKey) error
SetupKeys inits writer with list of known keys.
type SQLiteWriter ¶
type SQLiteWriter struct {
// contains filtered or unexported fields
}
SQLiteWriter inserts rows into SQLite database.
func NewSQLiteWriter ¶
func NewSQLiteWriter(fn string, tableName string, p *Processor) (*SQLiteWriter, error)
NewSQLiteWriter creates an instance of SQLiteWriter.
func (*SQLiteWriter) Close ¶
func (c *SQLiteWriter) Close() error
Close commits outstanding transaction and closes database instance.
func (*SQLiteWriter) ReceiveRow ¶
func (c *SQLiteWriter) ReceiveRow(seq int64, values []Value) error
ReceiveRow receives rows.
func (*SQLiteWriter) SetupKeys ¶ added in v0.4.0
func (c *SQLiteWriter) SetupKeys(keys []flKey) error
SetupKeys creates tables.
type URL ¶ added in v0.7.0
type URL struct {
Scheme string `json:"scheme,omitempty"`
User string `json:"user,omitempty"`
Pass string `json:"pass,omitempty"`
Host string `json:"host,omitempty"`
Port string `json:"port,omitempty"`
Query url.Values `json:"query,omitempty"`
Path []string `json:"path,omitempty"`
Fragment string `json:"fragment,omitempty"`
}
URL is a JSON representation of URL.
type WriteReceiver ¶
type WriteReceiver interface {
SetupKeys(keys []flKey) error
ReceiveRow(seq int64, values []Value) error
Close() error
}
WriteReceiver can receive a row for processing.
type Writer ¶
Writer dispatches rows to multiple receivers.
func (*Writer) Close ¶
Close tries to close all receivers and returns combined error in case of failures.
func (*Writer) HasReceivers ¶
HasReceivers is true if there are receivers.
func (*Writer) ReceiveRow ¶
ReceiveRow passes row to all receivers.