load-paths-handler.js 5.2 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203
  1. /* global emit */
  2. const async = require('async')
  3. const fs = require('fs')
  4. const os = require('os')
  5. const path = require('path')
  6. const {GitRepository} = require('atom')
  7. const {Minimatch} = require('minimatch')
  8. const childProcess = require('child_process')
  9. const { rgPath } = require('vscode-ripgrep')
  10. const PathsChunkSize = 100
  11. // Use the unpacked path if the ripgrep binary is in asar archive.
  12. const realRgPath = rgPath.replace(/\bapp\.asar\b/, 'app.asar.unpacked')
  13. // Define the maximum number of concurrent crawling processes based on the number of CPUs
  14. // with a maximum value of 8 and minimum of 1.
  15. const MaxConcurrentCrawls = Math.min(Math.max(os.cpus().length - 1, 8), 1)
  16. const emittedPaths = new Set()
  17. class PathLoader {
  18. constructor (rootPath, ignoreVcsIgnores, traverseSymlinkDirectories, ignoredNames, useRipGrep) {
  19. this.rootPath = rootPath
  20. this.ignoreVcsIgnores = ignoreVcsIgnores
  21. this.traverseSymlinkDirectories = traverseSymlinkDirectories
  22. this.ignoredNames = ignoredNames
  23. this.useRipGrep = useRipGrep
  24. this.paths = []
  25. this.inodes = new Set()
  26. this.repo = null
  27. if (ignoreVcsIgnores && !this.useRipGrep) {
  28. const repo = GitRepository.open(this.rootPath, {refreshOnWindowFocus: false})
  29. if ((repo && repo.relativize(path.join(this.rootPath, 'test'))) === 'test') {
  30. this.repo = repo
  31. }
  32. }
  33. }
  34. load (done) {
  35. if (this.useRipGrep) {
  36. this.loadFromRipGrep().then(done)
  37. return
  38. }
  39. this.loadPath(this.rootPath, true, () => {
  40. this.flushPaths()
  41. if (this.repo != null) this.repo.destroy()
  42. done()
  43. })
  44. }
  45. async loadFromRipGrep () {
  46. return new Promise((resolve) => {
  47. const args = ['--files', '--hidden', '--sort', 'path']
  48. if (!this.ignoreVcsIgnores) {
  49. args.push('--no-ignore')
  50. }
  51. if (this.traverseSymlinkDirectories) {
  52. args.push('--follow')
  53. }
  54. for (let ignoredName of this.ignoredNames) {
  55. args.push('-g', '!' + ignoredName.pattern)
  56. }
  57. if (this.ignoreVcsIgnores) {
  58. if (!args.includes('!.git')) args.push('-g', '!.git')
  59. if (!args.includes('!.hg')) args.push('-g', '!.hg')
  60. }
  61. let output = ''
  62. const result = childProcess.spawn(realRgPath, args, {cwd: this.rootPath})
  63. result.stdout.on('data', chunk => {
  64. const files = (output + chunk).split('\n')
  65. output = files.pop()
  66. for (const file of files) {
  67. this.pathLoaded(path.join(this.rootPath, file))
  68. }
  69. })
  70. result.stderr.on('data', () => {
  71. // intentionally ignoring errors for now
  72. })
  73. result.on('close', () => {
  74. this.flushPaths()
  75. resolve()
  76. })
  77. })
  78. }
  79. isIgnored (loadedPath) {
  80. const relativePath = path.relative(this.rootPath, loadedPath)
  81. if (this.repo && this.repo.isPathIgnored(relativePath)) {
  82. return true
  83. } else {
  84. for (let ignoredName of this.ignoredNames) {
  85. if (ignoredName.match(relativePath)) return true
  86. }
  87. }
  88. }
  89. pathLoaded (loadedPath, done) {
  90. if (!emittedPaths.has(loadedPath)) {
  91. this.paths.push(loadedPath)
  92. emittedPaths.add(loadedPath)
  93. }
  94. if (this.paths.length === PathsChunkSize) {
  95. this.flushPaths()
  96. }
  97. done && done()
  98. }
  99. flushPaths () {
  100. emit('load-paths:paths-found', this.paths)
  101. this.paths = []
  102. }
  103. loadPath (pathToLoad, root, done) {
  104. if (this.isIgnored(pathToLoad) && !root) return done()
  105. fs.lstat(pathToLoad, (error, stats) => {
  106. if (error != null) { return done() }
  107. if (stats.isSymbolicLink()) {
  108. fs.stat(pathToLoad, (error, stats) => {
  109. if (error != null) return done()
  110. if (this.inodes.has(stats.ino)) {
  111. return done()
  112. } else {
  113. this.inodes.add(stats.ino)
  114. }
  115. if (stats.isFile()) {
  116. this.pathLoaded(pathToLoad, done)
  117. } else if (stats.isDirectory()) {
  118. if (this.traverseSymlinkDirectories) {
  119. this.loadFolder(pathToLoad, done)
  120. } else {
  121. done()
  122. }
  123. } else {
  124. done()
  125. }
  126. })
  127. } else {
  128. this.inodes.add(stats.ino)
  129. if (stats.isDirectory()) {
  130. this.loadFolder(pathToLoad, done)
  131. } else if (stats.isFile()) {
  132. this.pathLoaded(pathToLoad, done)
  133. } else {
  134. done()
  135. }
  136. }
  137. })
  138. }
  139. loadFolder (folderPath, done) {
  140. fs.readdir(folderPath, (_, children = []) => {
  141. async.each(
  142. children,
  143. (childName, next) => {
  144. this.loadPath(path.join(folderPath, childName), false, next)
  145. },
  146. done
  147. )
  148. })
  149. }
  150. }
  151. module.exports = function (rootPaths, followSymlinks, ignoreVcsIgnores, ignores, useRipGrep) {
  152. const ignoredNames = []
  153. for (let ignore of ignores) {
  154. if (ignore) {
  155. try {
  156. ignoredNames.push(new Minimatch(ignore, {matchBase: true, dot: true}))
  157. } catch (error) {
  158. console.warn(`Error parsing ignore pattern (${ignore}): ${error.message}`)
  159. }
  160. }
  161. }
  162. async.eachLimit(
  163. rootPaths,
  164. MaxConcurrentCrawls,
  165. (rootPath, next) =>
  166. new PathLoader(
  167. rootPath,
  168. ignoreVcsIgnores,
  169. followSymlinks,
  170. ignoredNames,
  171. useRipGrep
  172. ).load(next)
  173. ,
  174. this.async()
  175. )
  176. }