Port IsSourceFileFromExternalLibrary, fix node18 emit (#1234)

Co-authored-by: Jake Bailey <[email protected]>

Author: maschwenk

internal/ast/utilities.go

@@ -1611,7 +1611,7 @@ func IsEffectiveExternalModule(node *SourceFile, compilerOptions *core.CompilerO
 }
 
 func isCommonJSContainingModuleKind(kind core.ModuleKind) bool {
-	return kind == core.ModuleKindCommonJS || kind == core.ModuleKindNode16 || kind == core.ModuleKindNodeNext
+	return kind == core.ModuleKindCommonJS || core.ModuleKindNode16 <= kind && kind <= core.ModuleKindNodeNext
 }
 
 func IsExternalModuleIndicator(node *Statement) bool {

internal/compiler/emitHost.go

@@ -124,3 +124,7 @@ func (host *emitHost) GetEmitResolver(file *ast.SourceFile, skipDiagnostics bool
 	defer done()
 	return checker.GetEmitResolver(file, skipDiagnostics)
 }
+
+func (host *emitHost) IsSourceFileFromExternalLibrary(file *ast.SourceFile) bool {
+	return host.program.IsSourceFileFromExternalLibrary(file)
+}

internal/compiler/emitter.go

@@ -2,7 +2,6 @@ package compiler
 
 import (
 	"encoding/base64"
-	"strings"
 
 	"github.com/microsoft/typescript-go/internal/ast"
 	"github.com/microsoft/typescript-go/internal/core"
@@ -305,10 +304,7 @@ func sourceFileMayBeEmitted(sourceFile *ast.SourceFile, host printer.EmitHost, f
 		return false
 	}
 
-	// !!! Source file from node_modules are not emitted. In Strada, this depends on module resolution and uses
-	// `sourceFilesFoundSearchingNodeModules` in `createProgram`. For now, we will just check for `/node_modules/` in
-	// the file name.
-	if strings.Contains(sourceFile.FileName(), "/node_modules/") {
+	if host.IsSourceFileFromExternalLibrary(sourceFile) {
 		return false
 	}
 

internal/compiler/fileloader.go

@@ -49,7 +49,8 @@ type processedFiles struct {
 	jsxRuntimeImportSpecifiers    map[tspath.Path]*jsxRuntimeImportSpecifier
 	importHelpersImportSpecifiers map[tspath.Path]*ast.Node
 	// List of present unsupported extensions
-	unsupportedExtensions []string
+	unsupportedExtensions                []string
+	sourceFilesFoundSearchingNodeModules collections.Set[tspath.Path]
 }
 
 type jsxRuntimeImportSpecifier struct {
@@ -111,6 +112,7 @@ func processAllProgramFiles(
 	var jsxRuntimeImportSpecifiers map[tspath.Path]*jsxRuntimeImportSpecifier
 	var importHelpersImportSpecifiers map[tspath.Path]*ast.Node
 	var unsupportedExtensions []string
+	var sourceFilesFoundSearchingNodeModules collections.Set[tspath.Path]
 
 	loader.parseTasks.collect(&loader, loader.rootTasks, func(task *parseTask, _ []tspath.Path) {
 		if task.isRedirected {
@@ -153,22 +155,26 @@ func processAllProgramFiles(
 		if slices.Contains(tspath.SupportedJSExtensionsFlat, extension) {
 			unsupportedExtensions = core.AppendIfUnique(unsupportedExtensions, extension)
 		}
+		if task.fromExternalLibrary {
+			sourceFilesFoundSearchingNodeModules.Add(path)
+		}
 	})
 	loader.sortLibs(libFiles)
 
 	allFiles := append(libFiles, files...)
 
 	return processedFiles{
-		resolver:                      loader.resolver,
-		files:                         allFiles,
-		filesByPath:                   filesByPath,
-		projectReferenceFileMapper:    loader.projectReferenceFileMapper,
-		resolvedModules:               resolvedModules,
-		typeResolutionsInFile:         typeResolutionsInFile,
-		sourceFileMetaDatas:           sourceFileMetaDatas,
-		jsxRuntimeImportSpecifiers:    jsxRuntimeImportSpecifiers,
-		importHelpersImportSpecifiers: importHelpersImportSpecifiers,
-		unsupportedExtensions:         unsupportedExtensions,
+		resolver:                             loader.resolver,
+		files:                                allFiles,
+		filesByPath:                          filesByPath,
+		projectReferenceFileMapper:           loader.projectReferenceFileMapper,
+		resolvedModules:                      resolvedModules,
+		typeResolutionsInFile:                typeResolutionsInFile,
+		sourceFileMetaDatas:                  sourceFileMetaDatas,
+		jsxRuntimeImportSpecifiers:           jsxRuntimeImportSpecifiers,
+		importHelpersImportSpecifiers:        importHelpersImportSpecifiers,
+		unsupportedExtensions:                unsupportedExtensions,
+		sourceFilesFoundSearchingNodeModules: sourceFilesFoundSearchingNodeModules,
 	}
 }
 
@@ -180,7 +186,7 @@ func (p *fileLoader) addRootTasks(files []string, isLib bool) {
 	for _, fileName := range files {
 		absPath := tspath.GetNormalizedAbsolutePath(fileName, p.opts.Host.GetCurrentDirectory())
 		if core.Tristate.IsTrue(p.opts.Config.CompilerOptions().AllowNonTsExtensions) || slices.Contains(p.supportedExtensions, tspath.TryGetExtensionFromPath(absPath)) {
-			p.rootTasks = append(p.rootTasks, &parseTask{normalizedFilePath: absPath, isLib: isLib})
+			p.rootTasks = append(p.rootTasks, &parseTask{normalizedFilePath: absPath, isLib: isLib, root: true})
 		}
 	}
 }
@@ -327,38 +333,38 @@ func (p *fileLoader) resolveTripleslashPathReference(moduleName string, containi
 	}
 }
 
-func (p *fileLoader) resolveTypeReferenceDirectives(file *ast.SourceFile, meta ast.SourceFileMetaData) (
-	toParse []resolvedRef,
-	typeResolutionsInFile module.ModeAwareCache[*module.ResolvedTypeReferenceDirective],
-) {
-	if len(file.TypeReferenceDirectives) != 0 {
-		toParse = make([]resolvedRef, 0, len(file.TypeReferenceDirectives))
-		typeResolutionsInFile = make(module.ModeAwareCache[*module.ResolvedTypeReferenceDirective], len(file.TypeReferenceDirectives))
-		for _, ref := range file.TypeReferenceDirectives {
-			redirect := p.projectReferenceFileMapper.getRedirectForResolution(file)
-			resolutionMode := getModeForTypeReferenceDirectiveInFile(ref, file, meta, module.GetCompilerOptionsWithRedirect(p.opts.Config.CompilerOptions(), redirect))
-			resolved := p.resolver.ResolveTypeReferenceDirective(ref.FileName, file.FileName(), resolutionMode, redirect)
-			typeResolutionsInFile[module.ModeAwareCacheKey{Name: ref.FileName, Mode: resolutionMode}] = resolved
-			if resolved.IsResolved() {
-				toParse = append(toParse, resolvedRef{
-					fileName:      resolved.ResolvedFileName,
-					increaseDepth: resolved.IsExternalLibraryImport,
-					elideOnDepth:  false,
-				})
-			}
+func (p *fileLoader) resolveTypeReferenceDirectives(t *parseTask) {
+	file := t.file
+	if len(file.TypeReferenceDirectives) == 0 {
+		return
+	}
+	meta := t.metadata
+
+	typeResolutionsInFile := make(module.ModeAwareCache[*module.ResolvedTypeReferenceDirective], len(file.TypeReferenceDirectives))
+	for _, ref := range file.TypeReferenceDirectives {
+		redirect := p.projectReferenceFileMapper.getRedirectForResolution(file)
+		resolutionMode := getModeForTypeReferenceDirectiveInFile(ref, file, meta, module.GetCompilerOptionsWithRedirect(p.opts.Config.CompilerOptions(), redirect))
+		resolved := p.resolver.ResolveTypeReferenceDirective(ref.FileName, file.FileName(), resolutionMode, redirect)
+		typeResolutionsInFile[module.ModeAwareCacheKey{Name: ref.FileName, Mode: resolutionMode}] = resolved
+		if resolved.IsResolved() {
+			t.addSubTask(resolvedRef{
+				fileName:              resolved.ResolvedFileName,
+				increaseDepth:         resolved.IsExternalLibraryImport,
+				elideOnDepth:          false,
+				isFromExternalLibrary: resolved.IsExternalLibraryImport,
+			}, false)
 		}
 	}
-	return toParse, typeResolutionsInFile
+
+	t.typeResolutionsInFile = typeResolutionsInFile
 }
 
 const externalHelpersModuleNameText = "tslib" // TODO(jakebailey): dedupe
 
-func (p *fileLoader) resolveImportsAndModuleAugmentations(file *ast.SourceFile, meta ast.SourceFileMetaData) (
-	toParse []resolvedRef,
-	resolutionsInFile module.ModeAwareCache[*module.ResolvedModule],
-	importHelpersImportSpecifier *ast.Node,
-	jsxRuntimeImportSpecifier_ *jsxRuntimeImportSpecifier,
-) {
+func (p *fileLoader) resolveImportsAndModuleAugmentations(t *parseTask) {
+	file := t.file
+	meta := t.metadata
+
 	moduleNames := make([]*ast.Node, 0, len(file.Imports())+len(file.ModuleAugmentations)+2)
 
 	isJavaScriptFile := ast.IsSourceFileJS(file)
@@ -370,14 +376,14 @@ func (p *fileLoader) resolveImportsAndModuleAugmentations(file *ast.SourceFile,
 		if optionsForFile.ImportHelpers.IsTrue() {
 			specifier := p.createSyntheticImport(externalHelpersModuleNameText, file)
 			moduleNames = append(moduleNames, specifier)
-			importHelpersImportSpecifier = specifier
+			t.importHelpersImportSpecifier = specifier
 		}
 
 		jsxImport := ast.GetJSXRuntimeImport(ast.GetJSXImplicitImportBase(optionsForFile, file), optionsForFile)
 		if jsxImport != "" {
 			specifier := p.createSyntheticImport(jsxImport, file)
 			moduleNames = append(moduleNames, specifier)
-			jsxRuntimeImportSpecifier_ = &jsxRuntimeImportSpecifier{
+			t.jsxRuntimeImportSpecifier = &jsxRuntimeImportSpecifier{
 				moduleReference: jsxImport,
 				specifier:       specifier,
 			}
@@ -395,8 +401,7 @@ func (p *fileLoader) resolveImportsAndModuleAugmentations(file *ast.SourceFile,
 	}
 
 	if len(moduleNames) != 0 {
-		toParse = make([]resolvedRef, 0, len(moduleNames))
-		resolutionsInFile = make(module.ModeAwareCache[*module.ResolvedModule], len(moduleNames))
+		resolutionsInFile := make(module.ModeAwareCache[*module.ResolvedModule], len(moduleNames))
 
 		for index, entry := range moduleNames {
 			moduleName := entry.Text()
@@ -433,16 +438,17 @@ func (p *fileLoader) resolveImportsAndModuleAugmentations(file *ast.SourceFile,
 				(importIndex < 0 || (importIndex < len(file.Imports()) && (ast.IsInJSFile(file.Imports()[importIndex]) || file.Imports()[importIndex].Flags&ast.NodeFlagsJSDoc == 0)))
 
 			if shouldAddFile {
-				toParse = append(toParse, resolvedRef{
-					fileName:      resolvedFileName,
-					increaseDepth: resolvedModule.IsExternalLibraryImport,
-					elideOnDepth:  isJsFileFromNodeModules,
-				})
+				t.addSubTask(resolvedRef{
+					fileName:              resolvedFileName,
+					increaseDepth:         resolvedModule.IsExternalLibraryImport,
+					elideOnDepth:          isJsFileFromNodeModules,
+					isFromExternalLibrary: resolvedModule.IsExternalLibraryImport,
+				}, false)
 			}
 		}
-	}
 
-	return toParse, resolutionsInFile, importHelpersImportSpecifier, jsxRuntimeImportSpecifier_
+		t.resolutionsInFile = resolutionsInFile
+	}
 }
 
 // Returns a DiagnosticMessage if we won't include a resolved module due to its extension.

internal/compiler/fileloadertask.go

@@ -17,6 +17,9 @@ type fileLoaderWorkerTask[T any] interface {
 	getSubTasks() []T
 	shouldIncreaseDepth() bool
 	shouldElideOnDepth() bool
+	isRoot() bool
+	isFromExternalLibrary() bool
+	markFromExternalLibrary()
 }
 
 type fileLoaderWorker[K fileLoaderWorkerTask[K]] struct {
@@ -32,42 +35,55 @@ type queuedTask[K fileLoaderWorkerTask[K]] struct {
 }
 
 func (w *fileLoaderWorker[K]) runAndWait(loader *fileLoader, tasks []K) {
-	w.start(loader, tasks, 0)
+	w.start(loader, tasks, 0, false)
 	w.wg.RunAndWait()
 }
 
-func (w *fileLoaderWorker[K]) start(loader *fileLoader, tasks []K, depth int) {
+func (w *fileLoaderWorker[K]) start(loader *fileLoader, tasks []K, depth int, isFromExternalLibrary bool) {
 	for i, task := range tasks {
+		taskIsFromExternalLibrary := isFromExternalLibrary || task.isFromExternalLibrary()
 		newTask := &queuedTask[K]{task: task, lowestDepth: math.MaxInt}
 		loadedTask, loaded := w.tasksByFileName.LoadOrStore(task.FileName(), newTask)
 		task = loadedTask.task
 		if loaded {
 			tasks[i] = task
 		}
 
-		currentDepth := depth
-		if task.shouldIncreaseDepth() {
-			currentDepth++
-		}
-
-		if task.shouldElideOnDepth() && currentDepth > w.maxDepth {
-			continue
-		}
-
 		w.wg.Queue(func() {
 			loadedTask.mu.Lock()
 			defer loadedTask.mu.Unlock()
 
-			if !task.isLoaded() {
-				task.load(loader)
+			currentDepth := depth
+			if task.shouldIncreaseDepth() {
+				currentDepth++
 			}
 
+			startSubtasks := false
+
 			if currentDepth < loadedTask.lowestDepth {
 				// If we're seeing this task at a lower depth than before,
 				// reprocess its subtasks to ensure they are loaded.
 				loadedTask.lowestDepth = currentDepth
-				subTasks := task.getSubTasks()
-				w.start(loader, subTasks, currentDepth)
+				startSubtasks = true
+			}
+
+			if !task.isRoot() && taskIsFromExternalLibrary && !task.isFromExternalLibrary() {
+				// If we're seeing this task now as an external library,
+				// reprocess its subtasks to ensure they are also marked as external.
+				task.markFromExternalLibrary()
+				startSubtasks = true
+			}
+
+			if task.shouldElideOnDepth() && currentDepth > w.maxDepth {
+				return
+			}
+
+			if !task.isLoaded() {
+				task.load(loader)
+			}
+
+			if startSubtasks {
+				w.start(loader, task.getSubTasks(), loadedTask.lowestDepth, task.isFromExternalLibrary())
 			}
 		})
 	}