123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203 |
- /* global emit */
- const async = require('async')
- const fs = require('fs')
- const os = require('os')
- const path = require('path')
- const {GitRepository} = require('atom')
- const {Minimatch} = require('minimatch')
- const childProcess = require('child_process')
- const { rgPath } = require('vscode-ripgrep')
- const PathsChunkSize = 100
- // Use the unpacked path if the ripgrep binary is in asar archive.
- const realRgPath = rgPath.replace(/\bapp\.asar\b/, 'app.asar.unpacked')
- // Define the maximum number of concurrent crawling processes based on the number of CPUs
- // with a maximum value of 8 and minimum of 1.
- const MaxConcurrentCrawls = Math.min(Math.max(os.cpus().length - 1, 8), 1)
- const emittedPaths = new Set()
- class PathLoader {
- constructor (rootPath, ignoreVcsIgnores, traverseSymlinkDirectories, ignoredNames, useRipGrep) {
- this.rootPath = rootPath
- this.ignoreVcsIgnores = ignoreVcsIgnores
- this.traverseSymlinkDirectories = traverseSymlinkDirectories
- this.ignoredNames = ignoredNames
- this.useRipGrep = useRipGrep
- this.paths = []
- this.inodes = new Set()
- this.repo = null
- if (ignoreVcsIgnores && !this.useRipGrep) {
- const repo = GitRepository.open(this.rootPath, {refreshOnWindowFocus: false})
- if ((repo && repo.relativize(path.join(this.rootPath, 'test'))) === 'test') {
- this.repo = repo
- }
- }
- }
- load (done) {
- if (this.useRipGrep) {
- this.loadFromRipGrep().then(done)
- return
- }
- this.loadPath(this.rootPath, true, () => {
- this.flushPaths()
- if (this.repo != null) this.repo.destroy()
- done()
- })
- }
- async loadFromRipGrep () {
- return new Promise((resolve) => {
- const args = ['--files', '--hidden', '--sort', 'path']
- if (!this.ignoreVcsIgnores) {
- args.push('--no-ignore')
- }
- if (this.traverseSymlinkDirectories) {
- args.push('--follow')
- }
- for (let ignoredName of this.ignoredNames) {
- args.push('-g', '!' + ignoredName.pattern)
- }
- if (this.ignoreVcsIgnores) {
- if (!args.includes('!.git')) args.push('-g', '!.git')
- if (!args.includes('!.hg')) args.push('-g', '!.hg')
- }
- let output = ''
- const result = childProcess.spawn(realRgPath, args, {cwd: this.rootPath})
- result.stdout.on('data', chunk => {
- const files = (output + chunk).split('\n')
- output = files.pop()
- for (const file of files) {
- this.pathLoaded(path.join(this.rootPath, file))
- }
- })
- result.stderr.on('data', () => {
- // intentionally ignoring errors for now
- })
- result.on('close', () => {
- this.flushPaths()
- resolve()
- })
- })
- }
- isIgnored (loadedPath) {
- const relativePath = path.relative(this.rootPath, loadedPath)
- if (this.repo && this.repo.isPathIgnored(relativePath)) {
- return true
- } else {
- for (let ignoredName of this.ignoredNames) {
- if (ignoredName.match(relativePath)) return true
- }
- }
- }
- pathLoaded (loadedPath, done) {
- if (!emittedPaths.has(loadedPath)) {
- this.paths.push(loadedPath)
- emittedPaths.add(loadedPath)
- }
- if (this.paths.length === PathsChunkSize) {
- this.flushPaths()
- }
- done && done()
- }
- flushPaths () {
- emit('load-paths:paths-found', this.paths)
- this.paths = []
- }
- loadPath (pathToLoad, root, done) {
- if (this.isIgnored(pathToLoad) && !root) return done()
- fs.lstat(pathToLoad, (error, stats) => {
- if (error != null) { return done() }
- if (stats.isSymbolicLink()) {
- fs.stat(pathToLoad, (error, stats) => {
- if (error != null) return done()
- if (this.inodes.has(stats.ino)) {
- return done()
- } else {
- this.inodes.add(stats.ino)
- }
- if (stats.isFile()) {
- this.pathLoaded(pathToLoad, done)
- } else if (stats.isDirectory()) {
- if (this.traverseSymlinkDirectories) {
- this.loadFolder(pathToLoad, done)
- } else {
- done()
- }
- } else {
- done()
- }
- })
- } else {
- this.inodes.add(stats.ino)
- if (stats.isDirectory()) {
- this.loadFolder(pathToLoad, done)
- } else if (stats.isFile()) {
- this.pathLoaded(pathToLoad, done)
- } else {
- done()
- }
- }
- })
- }
- loadFolder (folderPath, done) {
- fs.readdir(folderPath, (_, children = []) => {
- async.each(
- children,
- (childName, next) => {
- this.loadPath(path.join(folderPath, childName), false, next)
- },
- done
- )
- })
- }
- }
- module.exports = function (rootPaths, followSymlinks, ignoreVcsIgnores, ignores, useRipGrep) {
- const ignoredNames = []
- for (let ignore of ignores) {
- if (ignore) {
- try {
- ignoredNames.push(new Minimatch(ignore, {matchBase: true, dot: true}))
- } catch (error) {
- console.warn(`Error parsing ignore pattern (${ignore}): ${error.message}`)
- }
- }
- }
- async.eachLimit(
- rootPaths,
- MaxConcurrentCrawls,
- (rootPath, next) =>
- new PathLoader(
- rootPath,
- ignoreVcsIgnores,
- followSymlinks,
- ignoredNames,
- useRipGrep
- ).load(next)
- ,
- this.async()
- )
- }
|