diff --git a/package.json b/package.json index 46713d13c..92728a339 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "memfs-monorepo", - "version": "4.76.0", + "version": "4.77.0", "private": true, "description": "In-memory file-system with Node's fs API.", "keywords": [ diff --git a/packages/fs-core/package.json b/packages/fs-core/package.json index 5fb24d2b3..80b48dc5e 100644 --- a/packages/fs-core/package.json +++ b/packages/fs-core/package.json @@ -3,7 +3,7 @@ "publishConfig": { "access": "public" }, - "version": "4.76.0", + "version": "4.77.0", "description": "Core filesystem primitives: Node, Link, File, Superblock", "author": { "name": "streamich", diff --git a/packages/fs-fsa-to-node/package.json b/packages/fs-fsa-to-node/package.json index 129ec668e..33c3647be 100644 --- a/packages/fs-fsa-to-node/package.json +++ b/packages/fs-fsa-to-node/package.json @@ -3,7 +3,7 @@ "publishConfig": { "access": "public" }, - "version": "4.76.0", + "version": "4.77.0", "description": "Adapter to convert File System Access API to Node.js fs API", "author": { "name": "streamich", diff --git a/packages/fs-fsa/package.json b/packages/fs-fsa/package.json index 41d34b8af..a2aa01b15 100644 --- a/packages/fs-fsa/package.json +++ b/packages/fs-fsa/package.json @@ -3,7 +3,7 @@ "publishConfig": { "access": "public" }, - "version": "4.76.0", + "version": "4.77.0", "description": "File System Access API implementation backed by core filesystem primitives", "author": { "name": "streamich", diff --git a/packages/fs-node-builtins/package.json b/packages/fs-node-builtins/package.json index 9f7467bd3..265ceff01 100644 --- a/packages/fs-node-builtins/package.json +++ b/packages/fs-node-builtins/package.json @@ -3,7 +3,7 @@ "publishConfig": { "access": "public" }, - "version": "4.76.0", + "version": "4.77.0", "description": "Node.js standard library dependencies for fs-related packages", "author": { "name": "streamich", diff --git a/packages/fs-node-to-fsa/package.json b/packages/fs-node-to-fsa/package.json index 867b18c0e..06039331e 100644 --- a/packages/fs-node-to-fsa/package.json +++ b/packages/fs-node-to-fsa/package.json @@ -3,7 +3,7 @@ "publishConfig": { "access": "public" }, - "version": "4.76.0", + "version": "4.77.0", "description": "Adapter to convert Node.js fs API to File System Access API", "author": { "name": "streamich", diff --git a/packages/fs-node-utils/package.json b/packages/fs-node-utils/package.json index 6a753305d..cf927b89a 100644 --- a/packages/fs-node-utils/package.json +++ b/packages/fs-node-utils/package.json @@ -3,7 +3,7 @@ "publishConfig": { "access": "public" }, - "version": "4.76.0", + "version": "4.77.0", "description": "Utility functions for Node.js fs module", "author": { "name": "streamich", @@ -84,6 +84,6 @@ }, "dependencies": { "@jsonjoy.com/fs-node-builtins": "workspace:*", - "glob-to-regex.js": "^1.0.1" + "glob-to-regex.js": "^1.3.1" } } diff --git a/packages/fs-node-utils/src/types/FsCallbackApi.ts b/packages/fs-node-utils/src/types/FsCallbackApi.ts index 32d916ba8..6961eccb4 100644 --- a/packages/fs-node-utils/src/types/FsCallbackApi.ts +++ b/packages/fs-node-utils/src/types/FsCallbackApi.ts @@ -43,8 +43,8 @@ export interface FsCallbackApi { }; futimes: (fd: number, atime: misc.TTime, mtime: misc.TTime, callback: misc.TCallback) => void; glob: { - (pattern: string, callback: misc.TCallback): void; - (pattern: string, options: opts.IGlobOptions, callback: misc.TCallback): void; + (pattern: string | string[], callback: misc.TCallback<(string | misc.IDirent)[]>): void; + (pattern: string | string[], options: opts.IGlobOptions, callback: misc.TCallback<(string | misc.IDirent)[]>): void; }; lchmod: (path: misc.PathLike, mode: misc.TMode, callback: misc.TCallback) => void; lchown: (path: misc.PathLike, uid: number, gid: number, callback: misc.TCallback) => void; diff --git a/packages/fs-node-utils/src/types/FsPromisesApi.ts b/packages/fs-node-utils/src/types/FsPromisesApi.ts index b17ed64e3..bd1568c5c 100644 --- a/packages/fs-node-utils/src/types/FsPromisesApi.ts +++ b/packages/fs-node-utils/src/types/FsPromisesApi.ts @@ -38,5 +38,5 @@ export interface FsPromisesApi { options?: opts.IWatchOptions, ) => AsyncIterableIterator<{ eventType: string; filename: string | Buffer }>; writeFile: (id: misc.TFileHandle, data: misc.TPromisesData, options?: opts.IWriteFileOptions) => Promise; - glob: (pattern: string, options?: opts.IGlobOptions) => Promise; + glob: (pattern: string | string[], options?: opts.IGlobOptions) => AsyncGenerator; } diff --git a/packages/fs-node-utils/src/types/FsSynchronousApi.ts b/packages/fs-node-utils/src/types/FsSynchronousApi.ts index 16e466aae..6ab2b596c 100644 --- a/packages/fs-node-utils/src/types/FsSynchronousApi.ts +++ b/packages/fs-node-utils/src/types/FsSynchronousApi.ts @@ -21,7 +21,7 @@ export interface FsSynchronousApi { fsyncSync: (fd: number) => void; ftruncateSync: (fd: number, len?: number) => void; futimesSync: (fd: number, atime: misc.TTime, mtime: misc.TTime) => void; - globSync: (pattern: string, options?: opts.IGlobOptions) => string[]; + globSync: (pattern: string | string[], options?: opts.IGlobOptions) => (string | misc.IDirent)[]; lchmodSync: (path: misc.PathLike, mode: misc.TMode) => void; lchownSync: (path: misc.PathLike, uid: number, gid: number) => void; lutimesSync: (path: misc.PathLike, atime: number | string | Date, time: number | string | Date) => void; diff --git a/packages/fs-node-utils/src/types/options.ts b/packages/fs-node-utils/src/types/options.ts index 6139f9fe8..696b7c9bc 100644 --- a/packages/fs-node-utils/src/types/options.ts +++ b/packages/fs-node-utils/src/types/options.ts @@ -1,4 +1,4 @@ -import type { IFileHandle, TEncodingExtended, TFlags, TMode } from './misc'; +import type { IDirent, IFileHandle, TEncodingExtended, TFlags, TMode } from './misc'; export interface IOptions { encoding?: BufferEncoding | TEncodingExtended; @@ -228,12 +228,14 @@ export interface IOpenAsBlobOptions { export interface IGlobOptions { /** Current working directory. */ cwd?: string | URL; - /** Exclude patterns. */ - exclude?: string | string[]; - /** Maximum search depth. */ + /** Glob patterns to drop, or a predicate consulted while descending. */ + exclude?: string[] | ((entry: string | IDirent) => boolean); + /** Maximum search depth. Not a Node option. */ maxdepth?: number; - /** Whether to include symbolic links. */ + /** Whether to report each match as a `Dirent` instead of a path. */ withFileTypes?: boolean; + /** Whether to descend symbolic links to directories, stopping at a cycle. */ + followSymlinks?: boolean; } export interface IOpendirOptions extends IOptions { diff --git a/packages/fs-node/package.json b/packages/fs-node/package.json index ce7a0ca54..83fba9f65 100644 --- a/packages/fs-node/package.json +++ b/packages/fs-node/package.json @@ -3,7 +3,7 @@ "publishConfig": { "access": "public" }, - "version": "4.76.0", + "version": "4.77.0", "description": "In-memory filesystem with Node.js fs-compatible API", "author": { "name": "streamich", @@ -83,7 +83,7 @@ "@jsonjoy.com/fs-node-utils": "workspace:*", "@jsonjoy.com/fs-print": "workspace:*", "@jsonjoy.com/fs-snapshot": "workspace:*", - "glob-to-regex.js": "^1.0.0", + "glob-to-regex.js": "^1.3.1", "thingies": "^2.5.0" } } diff --git a/packages/fs-node/src/FsPromises.ts b/packages/fs-node/src/FsPromises.ts index 42971608f..de547a39c 100644 --- a/packages/fs-node/src/FsPromises.ts +++ b/packages/fs-node/src/FsPromises.ts @@ -1,4 +1,5 @@ import { isReadableStream, promisify, streamToBuffer } from './util'; +import { glob } from './glob'; import { constants } from '@jsonjoy.com/fs-node-utils'; import type * as opts from '@jsonjoy.com/fs-node-utils/lib/types/options'; import type * as misc from '@jsonjoy.com/fs-node-utils/lib/types/misc'; @@ -166,7 +167,8 @@ export class FsPromises implements FsPromisesApi { public readonly opendir = promisify(this.fs, 'opendir'); public readonly statfs = promisify(this.fs, 'statfs'); public readonly lutimes = promisify(this.fs, 'lutimes'); - public readonly glob = promisify(this.fs, 'glob'); + public readonly glob = (pattern: string | string[], options?: opts.IGlobOptions) => + glob(this.fs as any, pattern, options); public readonly access = promisify(this.fs, 'access'); public readonly chmod = promisify(this.fs, 'chmod'); public readonly chown = promisify(this.fs, 'chown'); diff --git a/packages/fs-node/src/__tests__/glob.test.ts b/packages/fs-node/src/__tests__/glob.test.ts index 2524d9082..b1fc26fd4 100644 --- a/packages/fs-node/src/__tests__/glob.test.ts +++ b/packages/fs-node/src/__tests__/glob.test.ts @@ -66,6 +66,17 @@ describe('glob APIs', () => { expect(results).toEqual(['file1.js']); }); + it('maxdepth counts directory levels below cwd for an absolute pattern', () => { + const { vol } = setup(); + expect(vol.globSync('/test/*.js', { maxdepth: 0 })).toEqual(['/test/file1.js']); + expect(vol.globSync('/test/**/*.js', { cwd: '/test', maxdepth: 0 })).toEqual(['/test/file1.js']); + expect(vol.globSync('/test/**/*.js', { cwd: '/', maxdepth: 1 })).toEqual(['/test/file1.js']); + expect(vol.globSync('/test/**/*.js', { cwd: '/', maxdepth: 2 }).sort()).toEqual([ + '/test/file1.js', + '/test/subdir/nested.js', + ]); + }); + it('should return empty array for non-matching pattern', () => { const { vol } = setup(); const results = vol.globSync('*.xyz', { cwd: '/test' }); @@ -91,6 +102,38 @@ describe('glob APIs', () => { }); }); + describe('matcher parity with minimatch', () => { + it('!(...) rejects the whole remainder', () => { + const { vol } = setup(); + expect(vol.globSync('!(file1).js', { cwd: '/test' })).toEqual([]); + expect(vol.globSync('!(file2).js', { cwd: '/test' })).toEqual(['file1.js']); + }); + + it('a leading ] is a member of a bracket expression', () => { + const { vol } = setup(); + expect(vol.globSync('[]f]ile1.js', { cwd: '/test' })).toEqual(['file1.js']); + }); + + it('expands a numeric brace range', () => { + const { vol } = setup(); + expect(vol.globSync('file{1..2}.*', { cwd: '/test' }).sort()).toEqual(['file1.js', 'file2.ts']); + }); + + it('a wildcard skips dotfiles but a literal or class dot matches one', () => { + const { vol } = setup(); + vol.writeFileSync('/test/.hidden.js', ''); + expect(vol.globSync('*.js', { cwd: '/test' })).toEqual(['file1.js']); + expect(vol.globSync('.*.js', { cwd: '/test' })).toEqual(['.hidden.js']); + expect(vol.globSync('[.]hidden.js', { cwd: '/test' })).toEqual(['.hidden.js']); + }); + + it('exclude takes globstar and extglob patterns', () => { + const { vol } = setup(); + expect(vol.globSync('**/*.js', { cwd: '/test', exclude: ['subdir/**'] })).toEqual(['file1.js']); + expect(vol.globSync('**/*.js', { cwd: '/test', exclude: ['**/!(nested).js'] })).toEqual(['subdir/nested.js']); + }); + }); + describe('glob (callback)', () => { it('should call callback with matching files', done => { const { vol } = setup(); @@ -121,22 +164,26 @@ describe('glob APIs', () => { }); describe('promises.glob', () => { - it('should return promise resolving to matching files', async () => { + const drain = async (iterator: AsyncIterable): Promise => { + const files: unknown[] = []; + for await (const file of iterator) files.push(file); + return files; + }; + + it('should yield the matching files', async () => { const { vol } = setup(); - const files = await vol.promises.glob('*.js', { cwd: '/test' }); - expect(files).toEqual(['file1.js']); + expect(await drain(vol.promises.glob('*.js', { cwd: '/test' }))).toEqual(['file1.js']); }); it('should work with recursive patterns', async () => { const { vol } = setup(); - const files = await vol.promises.glob('**/*.js', { cwd: '/test' }); + const files = await drain(vol.promises.glob('**/*.js', { cwd: '/test' })); expect(files.sort()).toEqual(['file1.js', 'subdir/nested.js']); }); it('should work without options', async () => { const { vol } = setup(); - const files = await vol.promises.glob('/test/*.js'); - expect(files).toEqual(['/test/file1.js']); + expect(await drain(vol.promises.glob('/test/*.js'))).toEqual(['/test/file1.js']); }); }); }); diff --git a/packages/fs-node/src/glob.ts b/packages/fs-node/src/glob.ts index add44e543..8c546bd3a 100644 --- a/packages/fs-node/src/glob.ts +++ b/packages/fs-node/src/glob.ts @@ -1,108 +1,595 @@ import { posix } from '@jsonjoy.com/fs-node-builtins/lib/path'; -import { toRegex } from 'glob-to-regex.js'; -import { IGlobOptions } from '@jsonjoy.com/fs-node-utils/lib/types/options'; -import { pathJoin, pathRelative } from '@jsonjoy.com/fs-node-utils'; +import { URL } from '@jsonjoy.com/fs-node-builtins/lib/url'; +import { invalidArgType, invalidUrlScheme, withCode } from '@jsonjoy.com/fs-node-utils/lib/argErrors'; +import { + validateBoolean, + validateObject, + validateString, + validateStringArray, +} from '@jsonjoy.com/fs-node-utils/lib/validators'; +import { expandBraces, toRegex } from 'glob-to-regex.js'; import { pathToFilename } from './util'; import Dirent from './Dirent'; -const pathResolve = posix.resolve; +const { basename, dirname, isAbsolute, join, relative, resolve } = posix; -/** - * Check if a path matches a glob pattern - */ -function matchesPattern(path: string, pattern: string): boolean { - const regex = toRegex(pattern); - return regex.test(path); -} +const internalAssertionError = (): Error => + withCode( + new Error( + 'This is caused by either a bug in Node.js or incorrect usage of Node.js internals.\n' + + 'Please open an issue with this stack trace at https://github.com/nodejs/node/issues\n', + ), + 'ERR_INTERNAL_ASSERTION', + ); + +const validateStringArrayOrFunction = (value: unknown, name: string): void => { + if (Array.isArray(value)) { + for (let i = 0; i < value.length; i++) + if (typeof value[i] !== 'string') throw invalidArgType(name + '[' + i + ']', 'of type string', value[i]); + return; + } + if (typeof value !== 'function') throw invalidArgType(name, 'of type function or string[]', value); +}; + +const toPathIfFileURL = (value: unknown): unknown => { + if (!(value instanceof URL)) return value; + if (value.protocol !== 'file:') throw invalidUrlScheme('file'); + return pathToFilename(value as any); +}; + +type Part = string | typeof GLOBSTAR | RegExp; + +const GLOBSTAR = Symbol('globstar **'); +const MAGIC = /[*?[]|[!+@]\(/; +const OPTS = { extglob: true, dot: false }; + +const parsePart = (part: string): Part => { + if (part === '**') return GLOBSTAR; + if (!MAGIC.test(part)) return part; + return toRegex(part, OPTS); +}; /** - * Check if a path should be excluded based on exclude patterns + * - drops `.` + * - drops empty components + * - collapses adjacent `**` + * - resolves a `..` that follows an ordinary component */ -function isExcluded(path: string, exclude: string | string[] | ((path: string) => boolean) | undefined): boolean { - if (!exclude) return false; +const firstPhasePreProcess = (globParts: string[][]): string[][] => { + let didSomething = false; + do { + didSomething = false; + for (let n = 0; n < globParts.length; n++) { + const parts = globParts[n]; + let gs = -1; + while (-1 !== (gs = parts.indexOf('**', gs + 1))) { + let gss = gs; + while (parts[gss + 1] === '**') gss++; + if (gss > gs) parts.splice(gs + 1, gss - gs); + const next = parts[gs + 1]; + const p = parts[gs + 2]; + const p2 = parts[gs + 3]; + if (next !== '..') continue; + if (!p || p === '.' || p === '..' || !p2 || p2 === '.' || p2 === '..') continue; + didSomething = true; + parts.splice(gs, 1); + const other = parts.slice(0); + other[gs] = '**'; + globParts.push(other); + gs--; + } + for (let i = 1; i < parts.length - 1; i++) { + const p = parts[i]; + if (i === 1 && p === '' && parts[0] === '') continue; + if (p === '.' || p === '') { + didSomething = true; + parts.splice(i, 1); + i--; + } + } + if (parts[0] === '.' && parts.length === 2 && (parts[1] === '.' || parts[1] === '')) { + didSomething = true; + parts.pop(); + } + let dd = 0; + while (-1 !== (dd = parts.indexOf('..', dd + 1))) { + const p = parts[dd - 1]; + if (p && p !== '.' && p !== '..' && p !== '**') { + didSomething = true; + const needDot = dd === 1 && parts[dd + 1] === '**'; + parts.splice(dd - 1, 2, ...(needDot ? ['.'] : [])); + if (!parts.length) parts.push(''); + dd -= 2; + } + } + } + } while (didSomething); + return globParts; +}; + +const partsMatch = (a: string[], b: string[]): string[] | false => { + let ai = 0; + let bi = 0; + const result: string[] = []; + let which = ''; + while (ai < a.length && bi < b.length) { + if (a[ai] === b[bi]) { + result.push(which === 'b' ? b[bi] : a[ai]); + ai++; + bi++; + } else if (a[ai] === '**' && b[bi] === a[ai + 1]) { + result.push(a[ai]); + ai++; + } else if (b[bi] === '**' && a[ai] === b[bi + 1]) { + result.push(b[bi]); + bi++; + } else if (a[ai] === '*' && b[bi] && !b[bi].startsWith('.') && b[bi] !== '**') { + if (which === 'b') return false; + which = 'a'; + result.push(a[ai]); + ai++; + bi++; + } else if (b[bi] === '*' && a[ai] && !a[ai].startsWith('.') && a[ai] !== '**') { + if (which === 'a') return false; + which = 'b'; + result.push(b[bi]); + ai++; + bi++; + } else return false; + } + return a.length === b.length && result; +}; + +const secondPhasePreProcess = (globParts: string[][]): string[][] => { + for (let i = 0; i < globParts.length - 1; i++) { + for (let j = i + 1; j < globParts.length; j++) { + const matched = partsMatch(globParts[i], globParts[j]); + if (matched) { + globParts[i] = []; + globParts[j] = matched; + break; + } + } + } + return globParts.filter(parts => parts.length); +}; + +const compile = (pattern: string): [set: Part[][], globParts: string[][]] => { + const source = pattern.replace(/\\/g, '/'); + if (!source) return [[], []]; + const globSet = [...new Set(expandBraces(source))]; + const globParts = secondPhasePreProcess(firstPhasePreProcess(globSet.map(one => one.split(/\/+/)))); + return [globParts.map(parts => parts.map(parsePart)), globParts]; +}; + +const createExcludeMatcher = (patterns: string[], root: string): ((value: string) => boolean) => { + const regexes: RegExp[] = []; + for (let i = 0; i < patterns.length; i++) regexes.push(toRegex(resolve(root, patterns[i].replace(/\\/g, '/')), OPTS)); + return (value: string): boolean => { + for (let i = 0; i < regexes.length; i++) if (regexes[i].test(value)) return true; + return false; + }; +}; + +class Pattern { + public readonly last: number; - if (typeof exclude === 'function') { - return exclude(path); + constructor( + protected readonly parts: Part[], + protected readonly globStrings: string[], + public readonly indexes: Set, + public readonly symlinks: Set, + public readonly realpaths: Set = new Set(), + ) { + this.last = parts.length - 1; } - const patterns = Array.isArray(exclude) ? exclude : [exclude]; - return patterns.some(pattern => matchesPattern(path, pattern)); + public isLast(isDirectory: boolean): boolean { + return ( + this.indexes.has(this.last) || + (this.at(-1) === '' && isDirectory && this.indexes.has(this.last - 1) && this.at(-2) === GLOBSTAR) + ); + } + + public isFirst(): boolean { + return this.indexes.has(0); + } + + public get hasSeenSymlinks(): boolean { + for (const index of this.indexes) if (!this.symlinks.has(index)) return true; + return false; + } + + public at(index: number): Part | undefined { + const parts = this.parts; + return parts[index < 0 ? parts.length + index : index]; + } + + public child(indexes: Set, symlinks = new Set(), realpaths = this.realpaths): Pattern { + return new Pattern(this.parts, this.globStrings, indexes, symlinks, realpaths); + } + + public test(index: number, path: string): boolean { + const part = this.parts[index]; + if (part === GLOBSTAR) return true; + if (typeof part === 'string') return part === path; + return part !== undefined && part.test(path); + } + + public cacheKey(index: number): string { + const globStrings = this.globStrings; + const length = globStrings.length; + let key = ''; + for (let i = index; i < length; i++) { + key += globStrings[i]; + if (i !== length - 1) key += '/'; + } + return key; + } } -/** - * Walk directory tree and collect matching paths - */ -function walkDirectory(fs: any, dir: string, patterns: string[], options: IGlobOptions, currentDepth = 0): string[] { - const results: string[] = []; - const maxDepth = options.maxdepth ?? Infinity; - const baseCwd = options.cwd ? pathToFilename(options.cwd as any) : process.cwd(); +interface GlobFs { + lstatSync(path: string): { mode: number }; + statSync(path: string): { mode: number }; + realpathSync(path: string): unknown; + readdirSync(path: string, options: { withFileTypes: true }): unknown[]; +} + +const rethrowUnlessFsError = (error: unknown): void => { + if (!(error as { code?: string })?.code) throw error; +}; - if (currentDepth > maxDepth) { - return results; +const direntOf = (name: string, parentPath: string, mode: number): Dirent => { + const dirent = new Dirent(); + dirent.name = name; + dirent.parentPath = parentPath; + dirent.path = parentPath; + (dirent as any).mode = mode; + return dirent; +}; + +class Cache { + protected readonly stats = new Map(); + protected readonly followStats = new Map(); + protected readonly realpaths = new Map(); + protected readonly readdirs = new Map(); + protected readonly seenKeys = new Map>(); + + constructor(protected readonly fs: GlobFs) {} + + public statSync(path: string): Dirent | null { + const cached = this.stats.get(path); + if (cached !== undefined) return cached; + let value: Dirent | null = null; + try { + value = direntOf(basename(path), dirname(path), this.fs.lstatSync(path).mode); + } catch (error) { + rethrowUnlessFsError(error); + } + this.stats.set(path, value); + return value; } - try { - const entries = fs.readdirSync(dir, { withFileTypes: true }) as Dirent[]; + public followStatSync(path: string): Dirent | null { + const cached = this.followStats.get(path); + if (cached !== undefined) return cached; + let value: Dirent | null = null; + try { + value = direntOf(basename(path), dirname(path), this.fs.statSync(path).mode); + } catch (error) { + rethrowUnlessFsError(error); + } + this.followStats.set(path, value); + return value; + } - for (const entry of entries) { - const fullPath = pathJoin(dir, entry.name.toString()); - const relativePath = pathRelative(baseCwd, fullPath); + public realpathSync(path: string): string | null { + const cached = this.realpaths.get(path); + if (cached !== undefined) return cached; + let value: string | null = null; + try { + value = String(this.fs.realpathSync(path)); + } catch (error) { + rethrowUnlessFsError(error); + } + this.realpaths.set(path, value); + return value; + } - // Skip if excluded - if (isExcluded(relativePath, options.exclude)) { - continue; - } + public readdirSync(path: string): Dirent[] { + const cached = this.readdirs.get(path); + if (cached !== undefined) return cached; + let value: Dirent[] = []; + try { + value = this.fs.readdirSync(path, { withFileTypes: true }) as Dirent[]; + } catch (error) { + rethrowUnlessFsError(error); + } + this.readdirs.set(path, value); + return value; + } - // Check if this path matches any pattern - const matches = patterns.some(pattern => matchesPattern(relativePath, pattern)); - if (matches) { - results.push(relativePath); - } + public addToStatCache(path: string, value: Dirent): void { + this.stats.set(path, value); + } - // Recurse into directories - if (entry.isDirectory() && currentDepth < maxDepth) { - const subResults = walkDirectory(fs, fullPath, patterns, options, currentDepth + 1); - results.push(...subResults); - } + public add(path: string, pattern: Pattern): boolean { + let keys = this.seenKeys.get(path); + if (!keys) { + keys = new Set(); + this.seenKeys.set(path, keys); } - } catch (err) { - // Skip directories we can't read + const originalSize = keys.size; + for (const index of pattern.indexes) keys.add(pattern.cacheKey(index)); + return keys.size !== originalSize + pattern.indexes.size; } - return results; + public seen(path: string, pattern: Pattern, index: number): boolean { + return !!this.seenKeys.get(path)?.has(pattern.cacheKey(index)); + } } -/** - * Main glob implementation - */ -export function globSync(fs: any, pattern: string, options: IGlobOptions = {}): string[] { - const cwd = options.cwd ? pathToFilename(options.cwd as any) : process.cwd(); - const resolvedCwd = pathResolve(cwd); - - const globOptions: IGlobOptions = { - cwd: resolvedCwd, - exclude: options.exclude, - maxdepth: options.maxdepth, - withFileTypes: options.withFileTypes || false, - }; +export interface GlobOptions { + /** Directory relative patterns are resolved against and matches are reported from. */ + cwd?: string | URL; + /** Glob patterns to drop, or a predicate consulted while descending. */ + exclude?: string[] | ((entry: string | Dirent) => boolean); + /** Report each match as a `Dirent` instead of a path. */ + withFileTypes?: boolean; + /** Descend symlinks to directories, stopping at a cycle. */ + followSymlinks?: boolean; + /** memfs extension, absent from Node: deepest directory level to descend from `cwd`. */ + maxdepth?: number; +} - let results: string[] = []; +const depthOf = (root: string, fullpath: string): number => { + const rel = relative(root, fullpath); + if (!rel) return 0; + const parts = rel.split('/'); + let depth = 0; + for (let i = 0; i < parts.length; i++) depth += parts[i] === '..' ? -1 : 1; + return depth; +}; - // Handle absolute patterns - if (posix.isAbsolute(pattern)) { - const dir = posix.dirname(pattern); - const patternBasename = posix.basename(pattern); - const dirResults = walkDirectory(fs, dir, [patternBasename], { ...globOptions, cwd: dir }); - results.push(...dirResults.map(r => posix.resolve(dir, r))); - } else { - // Handle relative patterns - normalize by stripping leading './' - const normalizedPattern = pattern.replace(/^\.\//, ''); - const dirResults = walkDirectory(fs, resolvedCwd, [normalizedPattern], globOptions); - results.push(...dirResults); +const EMPTY_OPTIONS: GlobOptions = {}; + +class Glob { + protected readonly root: unknown; + protected readonly withFileTypes: boolean; + protected readonly followSymlinks: boolean = false; + protected readonly maxdepth: number; + protected readonly patterns: Pattern[]; + protected readonly exclude: ((entry: string | Dirent) => boolean) | undefined; + protected readonly isExcluded: (value: string) => boolean = () => false; + protected readonly cache: Cache; + protected readonly results = new Set(); + protected readonly queue: { path: string; patterns: Pattern[] }[] = []; + protected readonly subpatterns = new Map(); + + constructor( + protected readonly fs: GlobFs, + pattern: unknown, + options: GlobOptions = EMPTY_OPTIONS, + ) { + validateObject(options, 'options'); + const { exclude, cwd, followSymlinks, withFileTypes, maxdepth } = options; + this.cache = new Cache(fs); + this.root = toPathIfFileURL(cwd) ?? '.'; + if (followSymlinks != null) { + validateBoolean(followSymlinks, 'options.followSymlinks'); + this.followSymlinks = followSymlinks; + } + this.withFileTypes = !!withFileTypes; + this.maxdepth = maxdepth ?? Infinity; + if (exclude != null) { + validateStringArrayOrFunction(exclude, 'options.exclude'); + if (Array.isArray(exclude)) { + if (typeof this.root !== 'string') throw internalAssertionError(); + this.isExcluded = createExcludeMatcher(exclude, this.root); + } else this.exclude = exclude as (entry: string | Dirent) => boolean; + } + let patterns: string[]; + if (typeof pattern === 'object') { + validateStringArray(pattern, 'patterns'); + patterns = pattern as string[]; + } else { + validateString(pattern, 'patterns'); + patterns = [pattern as string]; + } + const compiled: Pattern[] = []; + for (let i = 0; i < patterns.length; i++) { + const [set, globParts] = compile(patterns[i]); + for (let j = 0; j < set.length; j++) compiled.push(new Pattern(set[j], globParts[j], new Set([0]), new Set())); + } + this.patterns = compiled; } - // Remove duplicates and sort - results = [...new Set(results)].sort(); + public *walk(): Generator { + const queue = this.queue; + queue.push({ path: '.', patterns: this.patterns }); + while (queue.length > 0) { + const item = queue.pop()!; + for (let i = 0; i < item.patterns.length; i++) yield* this.walkPattern(item.path, item.patterns[i]); + this.subpatterns.forEach((patterns, path) => queue.push({ path, patterns })); + this.subpatterns.clear(); + } + } + protected result(path: string): string | Dirent { + if (!this.withFileTypes) return path; + const full = isAbsolute(path) ? path : join(this.root as string, path); + const stat = this.cache.statSync(full); + return direntOf(basename(full), dirname(full), stat ? (stat as any).mode : 0); + } + + protected isDirectory(path: string, stat: Dirent | null, pattern: Pattern): boolean { + if (stat?.isDirectory()) return true; + if (!stat?.isSymbolicLink()) return false; + if (this.followSymlinks) return !!this.cache.followStatSync(path)?.isDirectory(); + return pattern.hasSeenSymlinks; + } + + protected nextRealpaths(path: string, isDirectory: boolean, pattern: Pattern): Set { + if (!this.followSymlinks || !isDirectory) return pattern.realpaths; + const real = this.cache.realpathSync(path); + if (real === null) return pattern.realpaths; + const realpaths = new Set(pattern.realpaths); + realpaths.add(real); + return realpaths; + } + + protected isCyclic(path: string, isDirectory: boolean, pattern: Pattern): boolean { + if (!this.followSymlinks || !isDirectory) return false; + const real = this.cache.realpathSync(path); + return real !== null && pattern.realpaths.has(real); + } + + protected addSubpattern(path: string, pattern: Pattern): void { + if (this.isExcluded(path)) return; + const fullpath = resolve(this.root as string, path); + if (this.isExcluded(fullpath + '/') && this.cache.statSync(fullpath)?.isDirectory()) return; + const exclude = this.exclude; + if (exclude) { + if (this.withFileTypes) { + const stat = this.cache.statSync(path); + if (stat !== null && exclude(stat)) return; + } else if (exclude(path)) return; + } + const existing = this.subpatterns.get(path); + if (existing) existing.push(pattern); + else this.subpatterns.set(path, [pattern]); + } + + protected accept(path: string): boolean { + if (this.results.has(path)) return false; + if (this.isExcluded(resolve(this.root as string, path))) return false; + this.results.add(path); + return true; + } + + protected *walkPattern(path: string, pattern: Pattern): Generator { + if (this.cache.add(path, pattern)) return; + const cache = this.cache; + const fullpath = resolve(this.root as string, path); + const stat = cache.statSync(fullpath); + const last = pattern.last; + const isDirectory = this.isDirectory(fullpath, stat, pattern); + const isLast = pattern.isLast(isDirectory); + const isFirst = pattern.isFirst(); + if (this.isExcluded(fullpath)) return; + if (isFirst && pattern.at(0) === '') { + this.addSubpattern('/', pattern.child(new Set([1]))); + return; + } + if (isFirst && pattern.at(0) === '..') { + this.addSubpattern('../', pattern.child(new Set([1]))); + return; + } + if (isFirst && pattern.at(0) === '.') { + this.addSubpattern('.', pattern.child(new Set([1]))); + return; + } + if (isLast && typeof pattern.at(-1) === 'string') { + const p = pattern.at(-1) as string; + const leaf = cache.statSync(join(fullpath, p)); + const match = join(path, p); + if (leaf && (p || isDirectory) && this.accept(match)) yield this.result(match); + if (pattern.indexes.size === 1 && pattern.indexes.has(last)) return; + } else if (isLast && pattern.at(-1) === GLOBSTAR && (path !== '.' || pattern.at(0) === '.' || (!last && stat))) { + // `**` reports the directory it starts from; "." only for `**` alone or a pattern that starts with "." + if (this.accept(path)) yield this.result(path); + } + if (!isDirectory || this.isCyclic(fullpath, isDirectory, pattern)) return; + if (this.maxdepth !== Infinity && depthOf(this.root as string, fullpath) > this.maxdepth) return; + const nextRealpaths = this.nextRealpaths(fullpath, isDirectory, pattern); + let children: Dirent[]; + const firstPattern = pattern.indexes.size === 1 && pattern.at(pattern.indexes.values().next().value!); + if (typeof firstPattern === 'string') { + const child = cache.statSync(join(fullpath, firstPattern)); + if (!child) return; + child.name = firstPattern; + children = [child]; + } else children = cache.readdirSync(fullpath); + for (let i = 0; i < children.length; i++) { + const entry = children[i]; + const name = entry.name as string; + const entryPath = join(path, name); + const entryFullpath = join(fullpath, name); + cache.addToStatCache(entryFullpath, entry); + const entryIsDirectory = + entry.isDirectory() || + (this.followSymlinks && entry.isSymbolicLink() && !!cache.followStatSync(entryFullpath)?.isDirectory()); + const subPatterns = new Set(); + const nSymlinks = new Set(); + for (const index of pattern.indexes) { + if (cache.seen(entryPath, pattern, index) || cache.seen(entryPath, pattern, index + 1)) return; + const current = pattern.at(index); + const nextIndex = index + 1; + const next = pattern.at(nextIndex); + const fromSymlink = !this.followSymlinks && pattern.symlinks.has(index); + if (current === GLOBSTAR) { + const isDot = name.charCodeAt(0) === 46; + const nextMatches = pattern.test(nextIndex, name); + let nextNonGlobIndex = nextIndex; + while (pattern.at(nextNonGlobIndex) === GLOBSTAR) nextNonGlobIndex++; + const matchesDot = isDot && pattern.test(nextNonGlobIndex, name); + if ((isDot && !matchesDot) || (this.exclude && this.exclude(this.withFileTypes ? entry : name))) continue; + if (!fromSymlink && entryIsDirectory) subPatterns.add(index); + else if (!fromSymlink && index === last && this.accept(entryPath)) yield this.result(entryPath); + if (nextMatches && nextIndex === last && !isLast) { + if (this.accept(entryPath)) yield this.result(entryPath); + } else if (nextMatches && entryIsDirectory) subPatterns.add(index + 2); + if ((nextMatches || pattern.at(0) === '.') && (entryIsDirectory || entry.isSymbolicLink()) && !fromSymlink) + subPatterns.add(nextIndex); + if (!this.followSymlinks && entry.isSymbolicLink()) nSymlinks.add(index); + if (next === '..' && entryIsDirectory) { + // `**/..` reaches both this directory and its parent + const parent = join(path, '..'); + if (nextIndex < last) { + if (!this.subpatterns.has(path) && !cache.seen(path, pattern, nextIndex + 1)) + this.subpatterns.set(path, [pattern.child(new Set([nextIndex + 1]))]); + if (!this.subpatterns.has(parent) && !cache.seen(parent, pattern, nextIndex + 1)) + this.subpatterns.set(parent, [pattern.child(new Set([nextIndex + 1]))]); + } else { + if (!cache.seen(path, pattern, nextIndex)) { + cache.add(path, pattern.child(new Set([nextIndex]))); + if (this.accept(path)) yield this.result(path); + } + if (!cache.seen(path, pattern, nextIndex) || !cache.seen(parent, pattern, nextIndex)) { + cache.add(parent, pattern.child(new Set([nextIndex]))); + if (this.accept(parent)) yield this.result(parent); + } + } + } + } + if (typeof current === 'string') { + if (pattern.test(index, name) && index !== last) subPatterns.add(nextIndex); + else if (current === '.' && pattern.test(nextIndex, name)) { + if (nextIndex === last) { + if (this.accept(entryPath)) yield this.result(entryPath); + } else subPatterns.add(nextIndex + 1); + } + } + if (typeof current === 'object' && pattern.test(index, name)) { + if (index === last) { + if (this.accept(entryPath)) yield this.result(entryPath); + } else if (entryIsDirectory) subPatterns.add(nextIndex); + } + } + if (subPatterns.size > 0) this.addSubpattern(entryPath, pattern.child(subPatterns, nSymlinks, nextRealpaths)); + } + } +} + +export const globWalk = (fs: GlobFs, pattern: unknown, options?: GlobOptions): Generator => + new Glob(fs, pattern, options).walk(); + +export const globSync = (fs: GlobFs, pattern: unknown, options?: GlobOptions): (string | Dirent)[] => { + const results: (string | Dirent)[] = []; + for (const match of globWalk(fs, pattern, options)) results.push(match); return results; +}; + +export async function* glob(fs: GlobFs, pattern: unknown, options?: GlobOptions): AsyncGenerator { + yield* globWalk(fs, pattern, options); } diff --git a/packages/fs-node/src/types/FsPromisesApi.ts b/packages/fs-node/src/types/FsPromisesApi.ts index bba3296bf..b84d4afdb 100644 --- a/packages/fs-node/src/types/FsPromisesApi.ts +++ b/packages/fs-node/src/types/FsPromisesApi.ts @@ -38,5 +38,5 @@ export interface FsPromisesApi { options?: opts.IWatchOptions, ) => AsyncIterableIterator<{ eventType: string; filename: string | Buffer }>; writeFile: (id: misc.TFileHandle, data: misc.TPromisesData, options?: opts.IWriteFileOptions) => Promise; - glob: (pattern: string, options?: opts.IGlobOptions) => Promise; + glob: (pattern: string | string[], options?: opts.IGlobOptions) => AsyncGenerator; } diff --git a/packages/fs-node/src/volume.ts b/packages/fs-node/src/volume.ts index af24aa41b..a3d216159 100644 --- a/packages/fs-node/src/volume.ts +++ b/packages/fs-node/src/volume.ts @@ -22,6 +22,7 @@ import { import Stats from './Stats'; import Dirent from './Dirent'; import StatFs from './StatFs'; +import { globSync, globWalk } from './glob'; import { Buffer, bufferAllocUnsafe, bufferFrom } from '@jsonjoy.com/fs-node-builtins/lib/internal/buffer'; import setTimeoutUnref, { TSetTimeout } from '@jsonjoy.com/fs-node-utils/lib/setTimeoutUnref'; import { Readable, Writable } from '@jsonjoy.com/fs-node-builtins/lib/stream'; @@ -1587,19 +1588,24 @@ export class Volume implements FsCallbackApi, FsSynchronousApi { return new Blob([buffer as BlobPart], { type }); }; - public glob: FsCallbackApi['glob'] = (pattern: string, ...args: any[]) => { - const [options, callback] = args.length === 1 ? [{}, args[0]] : [args[0], args[1]]; - this.wrapAsync(this._globSync, [pattern, options || {}], callback); - }; - - public globSync: FsSynchronousApi['globSync'] = (pattern: string, options: opts.IGlobOptions = {}) => { - return this._globSync(pattern, options); - }; - - private readonly _globSync = (pattern: string, options: opts.IGlobOptions = {}): string[] => { - const { globSync } = require('./glob'); - return globSync(this, pattern, options); - }; + public glob: FsCallbackApi['glob'] = (pattern: string, options?: any, callback?: any) => { + if (typeof options === 'function') { + callback = options; + options = undefined; + } + const cb = validateCallback(callback); + const walk = globWalk(this, pattern, options); + Promise.resolve() + .then(() => { + const results: (string | Dirent)[] = []; + for (const match of walk) results.push(match); + return results; + }) + .then(results => cb(null, results), cb); + }; + + public globSync: FsSynchronousApi['globSync'] = (pattern: string, options?: opts.IGlobOptions) => + globSync(this, pattern, options as any); /** @param errorPath Empty for the synchronous form, whose errors carry no `err.path`. */ private readonly _opendir = ( diff --git a/packages/fs-print/package.json b/packages/fs-print/package.json index 6cb2da38a..4063d7ec0 100644 --- a/packages/fs-print/package.json +++ b/packages/fs-print/package.json @@ -3,7 +3,7 @@ "publishConfig": { "access": "public" }, - "version": "4.76.0", + "version": "4.77.0", "description": "File system tree printer - print a file system structure as a tree", "author": { "name": "streamich", diff --git a/packages/fs-snapshot/package.json b/packages/fs-snapshot/package.json index 58ce3be9a..996020197 100644 --- a/packages/fs-snapshot/package.json +++ b/packages/fs-snapshot/package.json @@ -3,7 +3,7 @@ "publishConfig": { "access": "public" }, - "version": "4.76.0", + "version": "4.77.0", "description": "File system snapshot - serialize and deserialize file system trees to binary or JSON", "author": { "name": "streamich", diff --git a/packages/memfs/package.json b/packages/memfs/package.json index e7939bf0f..a4babbaae 100644 --- a/packages/memfs/package.json +++ b/packages/memfs/package.json @@ -1,6 +1,6 @@ { "name": "memfs", - "version": "4.76.0", + "version": "4.77.0", "publishConfig": { "access": "public" }, @@ -136,7 +136,7 @@ "@jsonjoy.com/fs-snapshot": "workspace:*", "@jsonjoy.com/json-pack": "^1.11.0", "@jsonjoy.com/util": "^1.9.0", - "glob-to-regex.js": "^1.0.1", + "glob-to-regex.js": "^1.3.1", "thingies": "^2.5.0", "tree-dump": "^1.0.3", "tslib": "^2.0.0" diff --git a/yarn.lock b/yarn.lock index 8d9757d6e..f2333f02f 100644 --- a/yarn.lock +++ b/yarn.lock @@ -912,7 +912,7 @@ __metadata: "@jsonjoy.com/fs-node-builtins": "workspace:*" "@types/jest": "npm:^29.0.0" "@types/node": "npm:^20.0.0" - glob-to-regex.js: "npm:^1.0.1" + glob-to-regex.js: "npm:^1.3.1" jest: "npm:^29.0.0" rimraf: "npm:^5.0.0" ts-jest: "npm:^29.4.2" @@ -934,7 +934,7 @@ __metadata: "@jsonjoy.com/fs-snapshot": "workspace:*" "@types/jest": "npm:^29.0.0" "@types/node": "npm:^20.0.0" - glob-to-regex.js: "npm:^1.0.0" + glob-to-regex.js: "npm:^1.3.1" jest: "npm:^29.0.0" rimraf: "npm:^5.0.0" thingies: "npm:^2.5.0" @@ -3500,12 +3500,12 @@ __metadata: languageName: node linkType: hard -"glob-to-regex.js@npm:^1.0.0, glob-to-regex.js@npm:^1.0.1": - version: 1.2.0 - resolution: "glob-to-regex.js@npm:1.2.0" +"glob-to-regex.js@npm:^1.3.1": + version: 1.3.1 + resolution: "glob-to-regex.js@npm:1.3.1" peerDependencies: tslib: 2 - checksum: 10c0/011c81ae2a4d7ac5fd617038209fd9639d54c76211cc88fe8dd85d1a0850bc683a63cf5b1eae370141fca7dd2c834dfb9684dfdd8bf7472f2c1e4ef6ab6e34f9 + checksum: 10c0/0a4d8824c911a119b2cc7cbc2dd85287e899a5a4d16c26fee2cb0cac6d599a8273d29d04af47192116574212b63ada07f7af1c1369ba8417c75e1c3727b1c266 languageName: node linkType: hard @@ -4952,7 +4952,7 @@ __metadata: assert: "npm:^2.0.0" buffer: "npm:^6.0.3" events: "npm:^3.3.0" - glob-to-regex.js: "npm:^1.0.1" + glob-to-regex.js: "npm:^1.3.1" html-webpack-plugin: "npm:^5.5.3" husky: "npm:^8.0.1" isomorphic-git: "npm:^1.24.2"