diff --git a/internal/collections/manytomanyset.go b/internal/collections/manytomanyset.go deleted file mode 100644 index 9338beda10..0000000000 --- a/internal/collections/manytomanyset.go +++ /dev/null @@ -1,49 +0,0 @@ -package collections - -import "fmt" - -type ManyToManySet[K comparable, V comparable] struct { - keyToValueSet map[K]*Set[V] - valueToKeySet map[V]*Set[K] -} - -func (m *ManyToManySet[K, V]) GetKeys(value V) (*Set[K], bool) { - keys, present := m.valueToKeySet[value] - return keys, present -} - -func (m *ManyToManySet[K, V]) GetValues(key K) (*Set[V], bool) { - values, present := m.keyToValueSet[key] - return values, present -} - -func (m *ManyToManySet[K, V]) Len() int { - return len(m.keyToValueSet) -} - -func (m *ManyToManySet[K, V]) Keys() map[K]*Set[V] { - return m.keyToValueSet -} - -func (m *ManyToManySet[K, V]) Set(key K, valueSet *Set[V]) { - _, hasExisting := m.keyToValueSet[key] - if hasExisting { - panic("ManyToManySet.Set: key already exists: " + fmt.Sprintf("%v", key)) - } - if m.keyToValueSet == nil { - m.keyToValueSet = make(map[K]*Set[V]) - } - m.keyToValueSet[key] = valueSet - for value := range valueSet.Keys() { - // Add to valueToKeySet - keySetForValue, exists := m.valueToKeySet[value] - if !exists { - if m.valueToKeySet == nil { - m.valueToKeySet = make(map[V]*Set[K]) - } - keySetForValue = &Set[K]{} - m.valueToKeySet[value] = keySetForValue - } - keySetForValue.Add(key) - } -} diff --git a/internal/collections/syncmanytomanyset.go b/internal/collections/syncmanytomanyset.go new file mode 100644 index 0000000000..739eca2650 --- /dev/null +++ b/internal/collections/syncmanytomanyset.go @@ -0,0 +1,34 @@ +package collections + +import "fmt" + +type SyncManyToManySet[K comparable, V comparable] struct { + keyToValueSet SyncMap[K, *Set[V]] + valueToKeySet SyncMap[V, *SyncSet[K]] +} + +func (m *SyncManyToManySet[K, V]) GetKeys(value V) (*SyncSet[K], bool) { + keys, present := m.valueToKeySet.Load(value) + return keys, present +} + +func (m *SyncManyToManySet[K, V]) GetValues(key K) (*Set[V], bool) { + values, present := m.keyToValueSet.Load(key) + return values, present +} + +func (m *SyncManyToManySet[K, V]) Keys() *SyncMap[K, *Set[V]] { + return &m.keyToValueSet +} + +func (m *SyncManyToManySet[K, V]) Store(key K, valueSet *Set[V]) { + _, hasExisting := m.keyToValueSet.LoadOrStore(key, valueSet) + if hasExisting { + panic("ManyToManySet.Set: key already exists: " + fmt.Sprintf("%v", key)) + } + for value := range valueSet.Keys() { + // Add to valueToKeySet + keySetForValue, _ := m.valueToKeySet.LoadOrStore(value, &SyncSet[K]{}) + keySetForValue.Add(key) + } +} diff --git a/internal/collections/syncmap.go b/internal/collections/syncmap.go index f1101cf2bd..b28178fe62 100644 --- a/internal/collections/syncmap.go +++ b/internal/collections/syncmap.go @@ -1,6 +1,9 @@ package collections -import "sync" +import ( + "iter" + "sync" +) type SyncMap[K comparable, V any] struct { m sync.Map @@ -57,3 +60,14 @@ func (s *SyncMap[K, V]) ToMap() map[K]V { }) return m } + +func (s *SyncMap[K, V]) Keys() iter.Seq[K] { + return func(yield func(K) bool) { + s.m.Range(func(key, value any) bool { + if !yield(key.(K)) { + return false + } + return true + }) + } +} diff --git a/internal/collections/syncset.go b/internal/collections/syncset.go index 0ee9324885..1b9be611c0 100644 --- a/internal/collections/syncset.go +++ b/internal/collections/syncset.go @@ -1,5 +1,7 @@ package collections +import "iter" + type SyncSet[T comparable] struct { m SyncMap[T, struct{}] } @@ -23,6 +25,18 @@ func (s *SyncSet[T]) Range(fn func(key T) bool) { }) } +// Size returns the approximate number of items in the map. +// Note that this is not a precise count, as the map may be modified +// concurrently while this method is running. +func (s *SyncSet[T]) Size() int { + count := 0 + s.m.Range(func(_ T, _ struct{}) bool { + count++ + return true + }) + return count +} + func (s *SyncSet[T]) ToSlice() []T { var arr []T arr = make([]T, 0, s.m.Size()) @@ -32,3 +46,14 @@ func (s *SyncSet[T]) ToSlice() []T { }) return arr } + +func (s *SyncSet[T]) Keys() iter.Seq[T] { + return func(yield func(T) bool) { + s.m.Range(func(key T, value struct{}) bool { + if !yield(key) { + return false + } + return true + }) + } +} diff --git a/internal/execute/testsys_test.go b/internal/execute/testsys_test.go index 41a499bb94..67ff726cec 100644 --- a/internal/execute/testsys_test.go +++ b/internal/execute/testsys_test.go @@ -183,8 +183,8 @@ func (s *testSys) baselineProgram(baseline *strings.Builder, program *incrementa baseline.WriteString("SemanticDiagnostics::\n") testingData := program.GetTestingData(program.GetProgram()) for _, file := range program.GetProgram().GetSourceFiles() { - if diagnostics, ok := testingData.SemanticDiagnosticsPerFile[file.Path()]; ok { - if oldDiagnostics, ok := testingData.OldProgramSemanticDiagnosticsPerFile[file.Path()]; !ok || oldDiagnostics != diagnostics { + if diagnostics, ok := testingData.SemanticDiagnosticsPerFile.Load(file.Path()); ok { + if oldDiagnostics, ok := testingData.OldProgramSemanticDiagnosticsPerFile.Load(file.Path()); !ok || oldDiagnostics != diagnostics { baseline.WriteString("*refresh* " + file.FileName() + "\n") } } else { diff --git a/internal/incremental/affectedfileshandler.go b/internal/incremental/affectedfileshandler.go index 1851e35331..acecac6a8d 100644 --- a/internal/incremental/affectedfileshandler.go +++ b/internal/incremental/affectedfileshandler.go @@ -2,6 +2,7 @@ package incremental import ( "context" + "iter" "maps" "slices" "sync" @@ -41,8 +42,8 @@ func (h *affectedFilesHandler) getDtsMayChange(affectedFilePath tspath.Path, aff func (h *affectedFilesHandler) isChangedSignature(path tspath.Path) bool { newSignature, _ := h.updatedSignatures.Load(path) - oldSignature := h.program.snapshot.fileInfos[path].signature - return newSignature != oldSignature + oldInfo, _ := h.program.snapshot.fileInfos.Load(path) + return newSignature != oldInfo.signature } func (h *affectedFilesHandler) removeSemanticDiagnosticsOf(path tspath.Path) { @@ -81,7 +82,7 @@ func (h *affectedFilesHandler) updateShapeSignature(file *ast.SourceFile, useFil return false } - info := h.program.snapshot.fileInfos[file.Path()] + info, _ := h.program.snapshot.fileInfos.Load(file.Path()) prevSignature := info.signature var latestSignature string var updateKind SignatureUpdateKind @@ -110,7 +111,7 @@ func (h *affectedFilesHandler) getFilesAffectedBy(path tspath.Path) []*ast.Sourc return []*ast.SourceFile{file} } - if info := h.program.snapshot.fileInfos[file.Path()]; info.affectsGlobalScope { + if info, _ := h.program.snapshot.fileInfos.Load(file.Path()); info.affectsGlobalScope { h.hasAllFilesExcludingDefaultLibraryFile.Store(true) h.program.snapshot.getAllFilesExcludingDefaultLibraryFile(h.program.program, file) } @@ -139,10 +140,10 @@ func (h *affectedFilesHandler) getFilesAffectedBy(path tspath.Path) []*ast.Sourc } // Gets the files referenced by the the file path -func (h *affectedFilesHandler) getReferencedByPaths(file tspath.Path) map[tspath.Path]struct{} { +func (h *affectedFilesHandler) getReferencedByPaths(file tspath.Path) iter.Seq[tspath.Path] { keys, ok := h.program.snapshot.referencedMap.GetKeys(file) if !ok { - return nil + return func(yield func(tspath.Path) bool) {} } return keys.Keys() } @@ -154,8 +155,7 @@ func (h *affectedFilesHandler) forEachFileReferencedBy(file *ast.SourceFile, fn seenFileNamesMap := map[tspath.Path]*ast.SourceFile{} // Start with the paths this file was referenced by seenFileNamesMap[file.Path()] = file - references := h.getReferencedByPaths(file.Path()) - queue := slices.Collect(maps.Keys(references)) + queue := slices.Collect(h.getReferencedByPaths(file.Path())) for len(queue) > 0 { currentPath := queue[len(queue)-1] queue = queue[:len(queue)-1] @@ -290,7 +290,7 @@ func (h *affectedFilesHandler) handleDtsMayChangeOfFileAndExportsOfFile(dtsMayCh } func (h *affectedFilesHandler) handleDtsMayChangeOfGlobalScope(dtsMayChange dtsMayChange, filePath tspath.Path, invalidateJsFiles bool) bool { - if info, ok := h.program.snapshot.fileInfos[filePath]; !ok || !info.affectsGlobalScope { + if info, ok := h.program.snapshot.fileInfos.Load(filePath); !ok || !info.affectsGlobalScope { return false } // Every file needs to be handled @@ -331,7 +331,9 @@ func (h *affectedFilesHandler) updateSnapshot() { return } h.updatedSignatures.Range(func(filePath tspath.Path, signature string) bool { - h.program.snapshot.fileInfos[filePath].signature = signature + if info, ok := h.program.snapshot.fileInfos.Load(filePath); ok { + info.signature = signature + } return true }) if h.updatedSignatureKinds != nil { @@ -341,7 +343,7 @@ func (h *affectedFilesHandler) updateSnapshot() { }) } h.filesToRemoveDiagnostics.Range(func(file tspath.Path) bool { - delete(h.program.snapshot.semanticDiagnosticsPerFile, file) + h.program.snapshot.semanticDiagnosticsPerFile.Delete(file) return true }) for _, change := range h.dtsMayChange { @@ -349,25 +351,26 @@ func (h *affectedFilesHandler) updateSnapshot() { h.program.snapshot.addFileToAffectedFilesPendingEmit(filePath, emitKind) } } - h.program.snapshot.changedFilesSet = &collections.Set[tspath.Path]{} - h.program.snapshot.buildInfoEmitPending = true + h.program.snapshot.changedFilesSet = collections.SyncSet[tspath.Path]{} + h.program.snapshot.buildInfoEmitPending.Store(true) } func collectAllAffectedFiles(ctx context.Context, program *Program) { - if program.snapshot.changedFilesSet.Len() == 0 { + if program.snapshot.changedFilesSet.Size() == 0 { return } handler := affectedFilesHandler{ctx: ctx, program: program, updatedSignatureKinds: core.IfElse(program.updatedSignatureKinds == nil, nil, &collections.SyncMap[tspath.Path, SignatureUpdateKind]{})} wg := core.NewWorkGroup(handler.program.program.SingleThreaded()) var result collections.SyncSet[*ast.SourceFile] - for file := range program.snapshot.changedFilesSet.Keys() { + program.snapshot.changedFilesSet.Range(func(file tspath.Path) bool { wg.Queue(func() { for _, affectedFile := range handler.getFilesAffectedBy(file) { result.Add(affectedFile) } }) - } + return true + }) wg.RunAndWait() if ctx.Err() != nil { diff --git a/internal/incremental/buildInfo.go b/internal/incremental/buildInfo.go index 9afba79405..bf537e0084 100644 --- a/internal/incremental/buildInfo.go +++ b/internal/incremental/buildInfo.go @@ -328,12 +328,13 @@ func (b *BuildInfoEmitSignature) noEmitSignature() bool { return b.Signature == "" && !b.DiffersOnlyInDtsMap && !b.DiffersInOptions } -func (b *BuildInfoEmitSignature) toEmitSignature(path tspath.Path, emitSignatures map[tspath.Path]*emitSignature) *emitSignature { +func (b *BuildInfoEmitSignature) toEmitSignature(path tspath.Path, emitSignatures *collections.SyncMap[tspath.Path, *emitSignature]) *emitSignature { var signature string var signatureWithDifferentOptions []string if b.DiffersOnlyInDtsMap { signatureWithDifferentOptions = make([]string, 0, 1) - signatureWithDifferentOptions = append(signatureWithDifferentOptions, emitSignatures[path].signature) + info, _ := emitSignatures.Load(path) + signatureWithDifferentOptions = append(signatureWithDifferentOptions, info.signature) } else if b.DiffersInOptions { signatureWithDifferentOptions = make([]string, 0, 1) signatureWithDifferentOptions = append(signatureWithDifferentOptions, b.Signature) diff --git a/internal/incremental/tosnapshot.go b/internal/incremental/buildinfotosnapshot.go similarity index 75% rename from internal/incremental/tosnapshot.go rename to internal/incremental/buildinfotosnapshot.go index 28dcf4fa1a..684faf499e 100644 --- a/internal/incremental/tosnapshot.go +++ b/internal/incremental/buildinfotosnapshot.go @@ -93,36 +93,34 @@ func (t *toSnapshot) setCompilerOptions() { } func (t *toSnapshot) setFileInfoAndEmitSignatures() { - t.snapshot.fileInfos = make(map[tspath.Path]*fileInfo, len(t.buildInfo.FileInfos)) - t.snapshot.createEmitSignaturesMap() + isComposite := t.snapshot.options.Composite.IsTrue() for index, buildInfoFileInfo := range t.buildInfo.FileInfos { path := t.toFilePath(BuildInfoFileId(index + 1)) info := buildInfoFileInfo.GetFileInfo() - t.snapshot.fileInfos[path] = info + t.snapshot.fileInfos.Store(path, info) // Add default emit signature as file's signature - if info.signature != "" && t.snapshot.emitSignatures != nil { - t.snapshot.emitSignatures[path] = &emitSignature{signature: info.signature} + if info.signature != "" && isComposite { + t.snapshot.emitSignatures.Store(path, &emitSignature{signature: info.signature}) } } // Fix up emit signatures for _, value := range t.buildInfo.EmitSignatures { if value.noEmitSignature() { - delete(t.snapshot.emitSignatures, t.toFilePath(value.FileId)) + t.snapshot.emitSignatures.Delete(t.toFilePath(value.FileId)) } else { path := t.toFilePath(value.FileId) - t.snapshot.emitSignatures[path] = value.toEmitSignature(path, t.snapshot.emitSignatures) + t.snapshot.emitSignatures.Store(path, value.toEmitSignature(path, &t.snapshot.emitSignatures)) } } } func (t *toSnapshot) setReferencedMap() { for _, entry := range t.buildInfo.ReferencedMap { - t.snapshot.referencedMap.Set(t.toFilePath(entry.FileId), t.toFilePathSet(entry.FileIdListId)) + t.snapshot.referencedMap.Store(t.toFilePath(entry.FileId), t.toFilePathSet(entry.FileIdListId)) } } func (t *toSnapshot) setChangeFileSet() { - t.snapshot.changedFilesSet = collections.NewSetWithSizeHint[tspath.Path](len(t.buildInfo.ChangeFileSet)) for _, fileId := range t.buildInfo.ChangeFileSet { filePath := t.toFilePath(fileId) t.snapshot.changedFilesSet.Add(filePath) @@ -130,29 +128,28 @@ func (t *toSnapshot) setChangeFileSet() { } func (t *toSnapshot) setSemanticDiagnostics() { - t.snapshot.semanticDiagnosticsPerFile = make(map[tspath.Path]*diagnosticsOrBuildInfoDiagnosticsWithFileName, len(t.snapshot.fileInfos)) - for path := range t.snapshot.fileInfos { + t.snapshot.fileInfos.Range(func(path tspath.Path, info *fileInfo) bool { // Initialize to have no diagnostics if its not changed file if !t.snapshot.changedFilesSet.Has(path) { - t.snapshot.semanticDiagnosticsPerFile[path] = &diagnosticsOrBuildInfoDiagnosticsWithFileName{} + t.snapshot.semanticDiagnosticsPerFile.Store(path, &diagnosticsOrBuildInfoDiagnosticsWithFileName{}) } - } + return true + }) for _, diagnostic := range t.buildInfo.SemanticDiagnosticsPerFile { if diagnostic.FileId != 0 { filePath := t.toFilePath(diagnostic.FileId) - delete(t.snapshot.semanticDiagnosticsPerFile, filePath) // does not have cached diagnostics + t.snapshot.semanticDiagnosticsPerFile.Delete(filePath) // does not have cached diagnostics } else { filePath := t.toFilePath(diagnostic.Diagnostics.FileId) - t.snapshot.semanticDiagnosticsPerFile[filePath] = t.toDiagnosticsOrBuildInfoDiagnosticsWithFileName(diagnostic.Diagnostics) + t.snapshot.semanticDiagnosticsPerFile.Store(filePath, t.toDiagnosticsOrBuildInfoDiagnosticsWithFileName(diagnostic.Diagnostics)) } } } func (t *toSnapshot) setEmitDiagnostics() { - t.snapshot.emitDiagnosticsPerFile = make(map[tspath.Path]*diagnosticsOrBuildInfoDiagnosticsWithFileName, len(t.snapshot.fileInfos)) for _, diagnostic := range t.buildInfo.EmitDiagnosticsPerFile { filePath := t.toFilePath(diagnostic.FileId) - t.snapshot.emitDiagnosticsPerFile[filePath] = t.toDiagnosticsOrBuildInfoDiagnosticsWithFileName(diagnostic) + t.snapshot.emitDiagnosticsPerFile.Store(filePath, t.toDiagnosticsOrBuildInfoDiagnosticsWithFileName(diagnostic)) } } @@ -161,8 +158,7 @@ func (t *toSnapshot) setAffectedFilesPendingEmit() { return } ownOptionsEmitKind := GetFileEmitKind(t.snapshot.options) - t.snapshot.affectedFilesPendingEmit = make(map[tspath.Path]FileEmitKind, len(t.buildInfo.AffectedFilesPendingEmit)) for _, pendingEmit := range t.buildInfo.AffectedFilesPendingEmit { - t.snapshot.affectedFilesPendingEmit[t.toFilePath(pendingEmit.FileId)] = core.IfElse(pendingEmit.EmitKind == 0, ownOptionsEmitKind, pendingEmit.EmitKind) + t.snapshot.affectedFilesPendingEmit.Store(t.toFilePath(pendingEmit.FileId), core.IfElse(pendingEmit.EmitKind == 0, ownOptionsEmitKind, pendingEmit.EmitKind)) } } diff --git a/internal/incremental/emitfileshandler.go b/internal/incremental/emitfileshandler.go index dce81e60ec..6f2937815b 100644 --- a/internal/incremental/emitfileshandler.go +++ b/internal/incremental/emitfileshandler.go @@ -51,76 +51,76 @@ func (h *emitFilesHandler) emitAllAffectedFiles(options compiler.EmitOptions) *c // Emit all affected files var results []*compiler.EmitResult - if len(h.program.snapshot.affectedFilesPendingEmit) != 0 { - wg := core.NewWorkGroup(h.program.program.SingleThreaded()) - for path, emitKind := range h.program.snapshot.affectedFilesPendingEmit { - affectedFile := h.program.program.GetSourceFileByPath(path) - if affectedFile == nil || !h.program.program.SourceFileMayBeEmitted(affectedFile, false) { - h.deletedPendingKinds.Add(path) - continue - } - pendingKind := h.getPendingEmitKindForEmitOptions(emitKind, options) - if pendingKind != 0 { - wg.Queue(func() { - // Determine if we can do partial emit - var emitOnly compiler.EmitOnly - if (pendingKind & FileEmitKindAllJs) != 0 { - emitOnly = compiler.EmitOnlyJs - } - if (pendingKind & FileEmitKindAllDts) != 0 { - if emitOnly == compiler.EmitOnlyJs { - emitOnly = compiler.EmitAll - } else { - emitOnly = compiler.EmitOnlyDts - } - } - var result *compiler.EmitResult - if !h.isForDtsErrors { - result = h.program.program.Emit(h.ctx, h.getEmitOptions(compiler.EmitOptions{ - TargetSourceFile: affectedFile, - EmitOnly: emitOnly, - WriteFile: options.WriteFile, - })) + wg := core.NewWorkGroup(h.program.program.SingleThreaded()) + h.program.snapshot.affectedFilesPendingEmit.Range(func(path tspath.Path, emitKind FileEmitKind) bool { + affectedFile := h.program.program.GetSourceFileByPath(path) + if affectedFile == nil || !h.program.program.SourceFileMayBeEmitted(affectedFile, false) { + h.deletedPendingKinds.Add(path) + return true + } + pendingKind := h.getPendingEmitKindForEmitOptions(emitKind, options) + if pendingKind != 0 { + wg.Queue(func() { + // Determine if we can do partial emit + var emitOnly compiler.EmitOnly + if (pendingKind & FileEmitKindAllJs) != 0 { + emitOnly = compiler.EmitOnlyJs + } + if (pendingKind & FileEmitKindAllDts) != 0 { + if emitOnly == compiler.EmitOnlyJs { + emitOnly = compiler.EmitAll } else { - result = &compiler.EmitResult{ - EmitSkipped: true, - Diagnostics: h.program.program.GetDeclarationDiagnostics(h.ctx, affectedFile), - } + emitOnly = compiler.EmitOnlyDts } + } + var result *compiler.EmitResult + if !h.isForDtsErrors { + result = h.program.program.Emit(h.ctx, h.getEmitOptions(compiler.EmitOptions{ + TargetSourceFile: affectedFile, + EmitOnly: emitOnly, + WriteFile: options.WriteFile, + })) + } else { + result = &compiler.EmitResult{ + EmitSkipped: true, + Diagnostics: h.program.program.GetDeclarationDiagnostics(h.ctx, affectedFile), + } + } - // Update the pendingEmit for the file - h.emitUpdates.Store(path, &emitUpdate{pendingKind: getPendingEmitKind(emitKind, pendingKind), result: result}) - }) - } - } - wg.RunAndWait() - if h.ctx.Err() != nil { - return nil + // Update the pendingEmit for the file + h.emitUpdates.Store(path, &emitUpdate{pendingKind: getPendingEmitKind(emitKind, pendingKind), result: result}) + }) } + return true + }) + wg.RunAndWait() + if h.ctx.Err() != nil { + return nil } // Get updated errors that were not included in affected files emit - for path, diagnostics := range h.program.snapshot.emitDiagnosticsPerFile { + h.program.snapshot.emitDiagnosticsPerFile.Range(func(path tspath.Path, diagnostics *diagnosticsOrBuildInfoDiagnosticsWithFileName) bool { if _, ok := h.emitUpdates.Load(path); !ok { affectedFile := h.program.program.GetSourceFileByPath(path) if affectedFile == nil || !h.program.program.SourceFileMayBeEmitted(affectedFile, false) { h.deletedPendingKinds.Add(path) - continue + return true } - pendingKind := h.program.snapshot.affectedFilesPendingEmit[path] + pendingKind, _ := h.program.snapshot.affectedFilesPendingEmit.Load(path) h.emitUpdates.Store(path, &emitUpdate{pendingKind: pendingKind, result: &compiler.EmitResult{ EmitSkipped: true, Diagnostics: diagnostics.getDiagnostics(h.program.program, affectedFile), }}) } - } + return true + }) results = h.updateSnapshot() // Combine results and update buildInfo if h.isForDtsErrors && options.TargetSourceFile != nil { // Result from cache - diagnostics := h.program.snapshot.emitDiagnosticsPerFile[options.TargetSourceFile.Path()] + diagnostics, _ := h.program.snapshot.emitDiagnosticsPerFile.Load(options.TargetSourceFile.Path()) return &compiler.EmitResult{ EmitSkipped: true, Diagnostics: diagnostics.getDiagnostics(h.program.program, options.TargetSourceFile), @@ -149,7 +149,7 @@ func (h *emitFilesHandler) getEmitOptions(options compiler.EmitOptions) compiler WriteFile: func(fileName string, text string, writeByteOrderMark bool, data *compiler.WriteFileData) error { if tspath.IsDeclarationFileName(fileName) { var emitSignature string - info := h.program.snapshot.fileInfos[options.TargetSourceFile.Path()] + info, _ := h.program.snapshot.fileInfos.Load(options.TargetSourceFile.Path()) if info.signature == info.version { signature := h.program.snapshot.computeSignatureWithDiagnostics(options.TargetSourceFile, text, data) // With d.ts diagnostics they are also part of the signature so emitSignature will be different from it since its just hash of d.ts @@ -184,7 +184,7 @@ func (h *emitFilesHandler) skipDtsOutputOfComposite(file *ast.SourceFile, output return false } var oldSignature string - oldSignatureFormat, ok := h.program.snapshot.emitSignatures[file.Path()] + oldSignatureFormat, ok := h.program.snapshot.emitSignatures.Load(file.Path()) if ok { if oldSignatureFormat.signature != "" { oldSignature = oldSignatureFormat.signature @@ -215,49 +215,43 @@ func (h *emitFilesHandler) skipDtsOutputOfComposite(file *ast.SourceFile, output func (h *emitFilesHandler) updateSnapshot() []*compiler.EmitResult { h.signatures.Range(func(file tspath.Path, signature string) bool { - info := h.program.snapshot.fileInfos[file] + info, _ := h.program.snapshot.fileInfos.Load(file) info.signature = signature if h.program.updatedSignatureKinds != nil { h.program.updatedSignatureKinds[file] = SignatureUpdateKindStoredAtEmit } - h.program.snapshot.buildInfoEmitPending = true + h.program.snapshot.buildInfoEmitPending.Store(true) return true }) h.emitSignatures.Range(func(file tspath.Path, signature *emitSignature) bool { - if h.program.snapshot.emitSignatures == nil { - h.program.snapshot.emitSignatures = make(map[tspath.Path]*emitSignature) - } - h.program.snapshot.emitSignatures[file] = signature - h.program.snapshot.buildInfoEmitPending = true + h.program.snapshot.emitSignatures.Store(file, signature) + h.program.snapshot.buildInfoEmitPending.Store(true) return true }) latestChangedDtsFiles := h.latestChangedDtsFiles.ToSlice() slices.Sort(latestChangedDtsFiles) if latestChangedDtsFile := core.LastOrNil(latestChangedDtsFiles); latestChangedDtsFile != "" { h.program.snapshot.latestChangedDtsFile = latestChangedDtsFile - h.program.snapshot.buildInfoEmitPending = true + h.program.snapshot.buildInfoEmitPending.Store(true) } for file := range h.deletedPendingKinds.Keys() { - delete(h.program.snapshot.affectedFilesPendingEmit, file) - h.program.snapshot.buildInfoEmitPending = true + h.program.snapshot.affectedFilesPendingEmit.Delete(file) + h.program.snapshot.buildInfoEmitPending.Store(true) } var results []*compiler.EmitResult h.emitUpdates.Range(func(file tspath.Path, update *emitUpdate) bool { if update.pendingKind == 0 { - delete(h.program.snapshot.affectedFilesPendingEmit, file) + h.program.snapshot.affectedFilesPendingEmit.Delete(file) } else { - h.program.snapshot.affectedFilesPendingEmit[file] = update.pendingKind + h.program.snapshot.affectedFilesPendingEmit.Store(file, update.pendingKind) } if update.result != nil { results = append(results, update.result) if len(update.result.Diagnostics) != 0 { - if h.program.snapshot.emitDiagnosticsPerFile == nil { - h.program.snapshot.emitDiagnosticsPerFile = make(map[tspath.Path]*diagnosticsOrBuildInfoDiagnosticsWithFileName) - } - h.program.snapshot.emitDiagnosticsPerFile[file] = &diagnosticsOrBuildInfoDiagnosticsWithFileName{diagnostics: update.result.Diagnostics} + h.program.snapshot.emitDiagnosticsPerFile.Store(file, &diagnosticsOrBuildInfoDiagnosticsWithFileName{diagnostics: update.result.Diagnostics}) } } - h.program.snapshot.buildInfoEmitPending = true + h.program.snapshot.buildInfoEmitPending.Store(true) return true }) return results diff --git a/internal/incremental/program.go b/internal/incremental/program.go index 6352e782a6..4271f9c9d7 100644 --- a/internal/incremental/program.go +++ b/internal/incremental/program.go @@ -7,6 +7,7 @@ import ( "github.com/go-json-experiment/json" "github.com/microsoft/typescript-go/internal/ast" + "github.com/microsoft/typescript-go/internal/collections" "github.com/microsoft/typescript-go/internal/compiler" "github.com/microsoft/typescript-go/internal/core" "github.com/microsoft/typescript-go/internal/diagnostics" @@ -25,7 +26,7 @@ const ( type Program struct { snapshot *snapshot program *compiler.Program - semanticDiagnosticsPerFile map[tspath.Path]*diagnosticsOrBuildInfoDiagnosticsWithFileName + semanticDiagnosticsPerFile *collections.SyncMap[tspath.Path, *diagnosticsOrBuildInfoDiagnosticsWithFileName] updatedSignatureKinds map[tspath.Path]SignatureUpdateKind } @@ -33,13 +34,15 @@ var _ compiler.ProgramLike = (*Program)(nil) func NewProgram(program *compiler.Program, oldProgram *Program, testing bool) *Program { incrementalProgram := &Program{ - snapshot: newSnapshotForProgram(program, oldProgram, testing), + snapshot: programToSnapshot(program, oldProgram, testing), program: program, } if testing { if oldProgram != nil { - incrementalProgram.semanticDiagnosticsPerFile = oldProgram.snapshot.semanticDiagnosticsPerFile + incrementalProgram.semanticDiagnosticsPerFile = &oldProgram.snapshot.semanticDiagnosticsPerFile + } else { + incrementalProgram.semanticDiagnosticsPerFile = &collections.SyncMap[tspath.Path, *diagnosticsOrBuildInfoDiagnosticsWithFileName]{} } incrementalProgram.updatedSignatureKinds = make(map[tspath.Path]SignatureUpdateKind) } @@ -47,14 +50,14 @@ func NewProgram(program *compiler.Program, oldProgram *Program, testing bool) *P } type TestingData struct { - SemanticDiagnosticsPerFile map[tspath.Path]*diagnosticsOrBuildInfoDiagnosticsWithFileName - OldProgramSemanticDiagnosticsPerFile map[tspath.Path]*diagnosticsOrBuildInfoDiagnosticsWithFileName + SemanticDiagnosticsPerFile *collections.SyncMap[tspath.Path, *diagnosticsOrBuildInfoDiagnosticsWithFileName] + OldProgramSemanticDiagnosticsPerFile *collections.SyncMap[tspath.Path, *diagnosticsOrBuildInfoDiagnosticsWithFileName] UpdatedSignatureKinds map[tspath.Path]SignatureUpdateKind } func (p *Program) GetTestingData(program *compiler.Program) TestingData { return TestingData{ - SemanticDiagnosticsPerFile: p.snapshot.semanticDiagnosticsPerFile, + SemanticDiagnosticsPerFile: &p.snapshot.semanticDiagnosticsPerFile, OldProgramSemanticDiagnosticsPerFile: p.semanticDiagnosticsPerFile, UpdatedSignatureKinds: p.updatedSignatureKinds, } @@ -131,7 +134,7 @@ func (p *Program) GetSemanticDiagnostics(ctx context.Context, file *ast.SourceFi // Return result from cache if file != nil { - cachedDiagnostics, ok := p.snapshot.semanticDiagnosticsPerFile[file.Path()] + cachedDiagnostics, ok := p.snapshot.semanticDiagnosticsPerFile.Load(file.Path()) if !ok { panic("After handling all the affected files, there shouldnt be more changes") } @@ -140,7 +143,7 @@ func (p *Program) GetSemanticDiagnostics(ctx context.Context, file *ast.SourceFi var diagnostics []*ast.Diagnostic for _, file := range p.program.GetSourceFiles() { - cachedDiagnostics, ok := p.snapshot.semanticDiagnosticsPerFile[file.Path()] + cachedDiagnostics, ok := p.snapshot.semanticDiagnosticsPerFile.Load(file.Path()) if !ok { panic("After handling all the affected files, there shouldnt be more changes") } @@ -198,21 +201,21 @@ func (p *Program) collectSemanticDiagnosticsOfAffectedFiles(ctx context.Context, return } - if len(p.snapshot.semanticDiagnosticsPerFile) == len(p.program.GetSourceFiles()) { + if p.snapshot.semanticDiagnosticsPerFile.Size() == len(p.program.GetSourceFiles()) { // If we have all the files, return } var affectedFiles []*ast.SourceFile if file != nil { - _, ok := p.snapshot.semanticDiagnosticsPerFile[file.Path()] + _, ok := p.snapshot.semanticDiagnosticsPerFile.Load(file.Path()) if ok { return } affectedFiles = []*ast.SourceFile{file} } else { for _, file := range p.program.GetSourceFiles() { - if _, ok := p.snapshot.semanticDiagnosticsPerFile[file.Path()]; !ok { + if _, ok := p.snapshot.semanticDiagnosticsPerFile.Load(file.Path()); !ok { affectedFiles = append(affectedFiles, file) } } @@ -227,12 +230,12 @@ func (p *Program) collectSemanticDiagnosticsOfAffectedFiles(ctx context.Context, // Commit changes to snapshot for file, diagnostics := range diagnosticsPerFile { - p.snapshot.semanticDiagnosticsPerFile[file.Path()] = &diagnosticsOrBuildInfoDiagnosticsWithFileName{diagnostics: diagnostics} + p.snapshot.semanticDiagnosticsPerFile.Store(file.Path(), &diagnosticsOrBuildInfoDiagnosticsWithFileName{diagnostics: diagnostics}) } - if len(p.snapshot.semanticDiagnosticsPerFile) == len(p.program.GetSourceFiles()) && p.snapshot.checkPending && !p.snapshot.options.NoCheck.IsTrue() { + if p.snapshot.semanticDiagnosticsPerFile.Size() == len(p.program.GetSourceFiles()) && p.snapshot.checkPending && !p.snapshot.options.NoCheck.IsTrue() { p.snapshot.checkPending = false } - p.snapshot.buildInfoEmitPending = true + p.snapshot.buildInfoEmitPending.Store(true) } func (p *Program) emitBuildInfo(ctx context.Context, options compiler.EmitOptions) *compiler.EmitResult { @@ -246,10 +249,10 @@ func (p *Program) emitBuildInfo(ctx context.Context, options compiler.EmitOption if p.snapshot.hasErrors == core.TSUnknown { p.snapshot.hasErrors = p.ensureHasErrorsForState(ctx, p.program) if p.snapshot.hasErrors != p.snapshot.hasErrorsFromOldState { - p.snapshot.buildInfoEmitPending = true + p.snapshot.buildInfoEmitPending.Store(true) } } - if !p.snapshot.buildInfoEmitPending { + if !p.snapshot.buildInfoEmitPending.Load() { return nil } if ctx.Err() != nil { @@ -275,7 +278,7 @@ func (p *Program) emitBuildInfo(ctx context.Context, options compiler.EmitOption }, } } - p.snapshot.buildInfoEmitPending = false + p.snapshot.buildInfoEmitPending.Store(false) var emittedFiles []string if p.snapshot.options.ListEmittedFiles.IsTrue() { @@ -290,8 +293,8 @@ func (p *Program) emitBuildInfo(ctx context.Context, options compiler.EmitOption func (p *Program) ensureHasErrorsForState(ctx context.Context, program *compiler.Program) core.Tristate { // Check semantic and emit diagnostics first as we dont need to ask program about it if slices.ContainsFunc(program.GetSourceFiles(), func(file *ast.SourceFile) bool { - semanticDiagnostics := p.snapshot.semanticDiagnosticsPerFile[file.Path()] - if semanticDiagnostics == nil { + semanticDiagnostics, ok := p.snapshot.semanticDiagnosticsPerFile.Load(file.Path()) + if !ok { // Missing semantic diagnostics in cache will be encoded in incremental buildInfo return p.snapshot.options.IsIncremental() } @@ -299,7 +302,7 @@ func (p *Program) ensureHasErrorsForState(ctx context.Context, program *compiler // cached semantic diagnostics will be encoded in buildInfo return true } - if _, ok := p.snapshot.emitDiagnosticsPerFile[file.Path()]; ok { + if _, ok := p.snapshot.emitDiagnosticsPerFile.Load(file.Path()); ok { // emit diagnostics will be encoded in buildInfo; return true } diff --git a/internal/incremental/programtosnapshot.go b/internal/incremental/programtosnapshot.go new file mode 100644 index 0000000000..9c4f78d8f6 --- /dev/null +++ b/internal/incremental/programtosnapshot.go @@ -0,0 +1,300 @@ +package incremental + +import ( + "context" + + "github.com/microsoft/typescript-go/internal/ast" + "github.com/microsoft/typescript-go/internal/checker" + "github.com/microsoft/typescript-go/internal/collections" + "github.com/microsoft/typescript-go/internal/compiler" + "github.com/microsoft/typescript-go/internal/core" + "github.com/microsoft/typescript-go/internal/tsoptions" + "github.com/microsoft/typescript-go/internal/tspath" +) + +func programToSnapshot(program *compiler.Program, oldProgram *Program, hashWithText bool) *snapshot { + if oldProgram != nil && oldProgram.program == program { + return oldProgram.snapshot + } + + to := &toProgramSnapshot{ + program: program, + oldProgram: oldProgram, + snapshot: &snapshot{ + options: program.Options(), + hashWithText: hashWithText, + }, + } + + to.reuseFromOldProgram() + to.computeProgramFileChanges() + to.handleFileDelete() + to.handlePendingEmit() + to.handlePendingCheck() + return to.snapshot +} + +type toProgramSnapshot struct { + program *compiler.Program + oldProgram *Program + snapshot *snapshot + globalFileRemoved bool +} + +func (t *toProgramSnapshot) reuseFromOldProgram() { + if t.oldProgram != nil && t.snapshot.options.Composite.IsTrue() { + t.snapshot.latestChangedDtsFile = t.oldProgram.snapshot.latestChangedDtsFile + } + if t.snapshot.options.NoCheck.IsTrue() { + t.snapshot.checkPending = true + } + + if t.oldProgram != nil { + // Copy old snapshot's changed files set + t.oldProgram.snapshot.changedFilesSet.Range(func(key tspath.Path) bool { + t.snapshot.changedFilesSet.Add(key) + return true + }) + t.oldProgram.snapshot.affectedFilesPendingEmit.Range(func(key tspath.Path, emitKind FileEmitKind) bool { + t.snapshot.affectedFilesPendingEmit.Store(key, emitKind) + return true + }) + t.snapshot.buildInfoEmitPending.Store(t.oldProgram.snapshot.buildInfoEmitPending.Load()) + t.snapshot.hasErrorsFromOldState = t.oldProgram.snapshot.hasErrors + } else { + t.snapshot.buildInfoEmitPending.Store(t.snapshot.options.IsIncremental()) + } +} + +func (t *toProgramSnapshot) computeProgramFileChanges() { + canCopySemanticDiagnostics := t.oldProgram != nil && + !tsoptions.CompilerOptionsAffectSemanticDiagnostics(t.oldProgram.snapshot.options, t.program.Options()) + // We can only reuse emit signatures (i.e. .d.ts signatures) if the .d.ts file is unchanged, + // which will eg be depedent on change in options like declarationDir and outDir options are unchanged. + // We need to look in oldState.compilerOptions, rather than oldCompilerOptions (i.e.we need to disregard useOldState) because + // oldCompilerOptions can be undefined if there was change in say module from None to some other option + // which would make useOldState as false since we can now use reference maps that are needed to track what to emit, what to check etc + // but that option change does not affect d.ts file name so emitSignatures should still be reused. + canCopyEmitSignatures := t.snapshot.options.Composite.IsTrue() && + t.oldProgram != nil && + !tsoptions.CompilerOptionsAffectDeclarationPath(t.oldProgram.snapshot.options, t.program.Options()) + copyDeclarationFileDiagnostics := canCopySemanticDiagnostics && + t.snapshot.options.SkipLibCheck.IsTrue() == t.oldProgram.snapshot.options.SkipLibCheck.IsTrue() + copyLibFileDiagnostics := copyDeclarationFileDiagnostics && + t.snapshot.options.SkipDefaultLibCheck.IsTrue() == t.oldProgram.snapshot.options.SkipDefaultLibCheck.IsTrue() + + files := t.program.GetSourceFiles() + wg := core.NewWorkGroup(t.program.SingleThreaded()) + for _, file := range files { + wg.Queue(func() { + version := t.snapshot.computeHash(file.Text()) + impliedNodeFormat := t.program.GetSourceFileMetaData(file.Path()).ImpliedNodeFormat + affectsGlobalScope := fileAffectsGlobalScope(file) + var signature string + newReferences := getReferencedFiles(t.program, file) + if newReferences != nil { + t.snapshot.referencedMap.Store(file.Path(), newReferences) + } + if t.oldProgram != nil { + if oldFileInfo, ok := t.oldProgram.snapshot.fileInfos.Load(file.Path()); ok { + signature = oldFileInfo.signature + if oldFileInfo.version != version || oldFileInfo.affectsGlobalScope != affectsGlobalScope || oldFileInfo.impliedNodeFormat != impliedNodeFormat { + t.snapshot.addFileToChangeSet(file.Path()) + } else if oldReferences, _ := t.oldProgram.snapshot.referencedMap.GetValues(file.Path()); !newReferences.Equals(oldReferences) { + // Referenced files changed + t.snapshot.addFileToChangeSet(file.Path()) + } else if newReferences != nil { + for refPath := range newReferences.Keys() { + if t.program.GetSourceFileByPath(refPath) == nil { + if _, ok := t.oldProgram.snapshot.fileInfos.Load(refPath); ok { + // Referenced file was deleted in the new program + t.snapshot.addFileToChangeSet(file.Path()) + break + } + } + } + } + } else { + t.snapshot.addFileToChangeSet(file.Path()) + } + if !t.snapshot.changedFilesSet.Has(file.Path()) { + if emitDiagnostics, ok := t.oldProgram.snapshot.emitDiagnosticsPerFile.Load(file.Path()); ok { + t.snapshot.emitDiagnosticsPerFile.Store(file.Path(), emitDiagnostics) + } + if canCopySemanticDiagnostics { + if (!file.IsDeclarationFile || copyDeclarationFileDiagnostics) && + (!t.program.IsSourceFileDefaultLibrary(file.Path()) || copyLibFileDiagnostics) { + // Unchanged file copy diagnostics + if diagnostics, ok := t.oldProgram.snapshot.semanticDiagnosticsPerFile.Load(file.Path()); ok { + t.snapshot.semanticDiagnosticsPerFile.Store(file.Path(), diagnostics) + } + } + } + } + if canCopyEmitSignatures { + if oldEmitSignature, ok := t.oldProgram.snapshot.emitSignatures.Load(file.Path()); ok { + t.snapshot.emitSignatures.Store(file.Path(), oldEmitSignature.getNewEmitSignature(t.oldProgram.snapshot.options, t.snapshot.options)) + } + } + } else { + t.snapshot.addFileToAffectedFilesPendingEmit(file.Path(), GetFileEmitKind(t.snapshot.options)) + signature = version + } + t.snapshot.fileInfos.Store(file.Path(), &fileInfo{ + version: version, + signature: signature, + affectsGlobalScope: affectsGlobalScope, + impliedNodeFormat: impliedNodeFormat, + }) + }) + } + wg.RunAndWait() +} + +func (t *toProgramSnapshot) handleFileDelete() { + if t.oldProgram != nil { + // If the global file is removed, add all files as changed + t.oldProgram.snapshot.fileInfos.Range(func(filePath tspath.Path, oldInfo *fileInfo) bool { + if _, ok := t.snapshot.fileInfos.Load(filePath); !ok { + if oldInfo.affectsGlobalScope { + for _, file := range t.snapshot.getAllFilesExcludingDefaultLibraryFile(t.program, nil) { + t.snapshot.addFileToChangeSet(file.Path()) + } + t.globalFileRemoved = true + } else { + t.snapshot.buildInfoEmitPending.Store(true) + } + return false + } + return true + }) + } +} + +func (t *toProgramSnapshot) handlePendingEmit() { + if t.oldProgram != nil && !t.globalFileRemoved { + // If options affect emit, then we need to do complete emit per compiler options + // otherwise only the js or dts that needs to emitted because its different from previously emitted options + var pendingEmitKind FileEmitKind + if tsoptions.CompilerOptionsAffectEmit(t.oldProgram.snapshot.options, t.snapshot.options) { + pendingEmitKind = GetFileEmitKind(t.snapshot.options) + } else { + pendingEmitKind = getPendingEmitKindWithOptions(t.snapshot.options, t.oldProgram.snapshot.options) + } + if pendingEmitKind != FileEmitKindNone { + // Add all files to affectedFilesPendingEmit since emit changed + for _, file := range t.program.GetSourceFiles() { + // Add to affectedFilesPending emit only if not changed since any changed file will do full emit + if !t.snapshot.changedFilesSet.Has(file.Path()) { + t.snapshot.addFileToAffectedFilesPendingEmit(file.Path(), pendingEmitKind) + } + } + t.snapshot.buildInfoEmitPending.Store(true) + } + } +} + +func (t *toProgramSnapshot) handlePendingCheck() { + if t.oldProgram != nil && + t.snapshot.semanticDiagnosticsPerFile.Size() != len(t.program.GetSourceFiles()) && + t.oldProgram.snapshot.checkPending != t.snapshot.checkPending { + t.snapshot.buildInfoEmitPending.Store(true) + } +} + +func fileAffectsGlobalScope(file *ast.SourceFile) bool { + // if file contains anything that augments to global scope we need to build them as if + // they are global files as well as module + if core.Some(file.ModuleAugmentations, func(augmentation *ast.ModuleName) bool { + return ast.IsGlobalScopeAugmentation(augmentation.Parent) + }) { + return true + } + + if ast.IsExternalOrCommonJSModule(file) || ast.IsJsonSourceFile(file) { + return false + } + + // For script files that contains only ambient external modules, although they are not actually external module files, + // they can only be consumed via importing elements from them. Regular script files cannot consume them. Therefore, + // there are no point to rebuild all script files if these special files have changed. However, if any statement + // in the file is not ambient external module, we treat it as a regular script file. + return file.Statements != nil && + file.Statements.Nodes != nil && + core.Some(file.Statements.Nodes, func(stmt *ast.Node) bool { + return !ast.IsModuleWithStringLiteralName(stmt) + }) +} + +func addReferencedFilesFromSymbol(file *ast.SourceFile, referencedFiles *collections.Set[tspath.Path], symbol *ast.Symbol) { + if symbol == nil { + return + } + for _, declaration := range symbol.Declarations { + fileOfDecl := ast.GetSourceFileOfNode(declaration) + if fileOfDecl == nil { + continue + } + if file != fileOfDecl { + referencedFiles.Add(fileOfDecl.Path()) + } + } +} + +// Get the module source file and all augmenting files from the import name node from file +func addReferencedFilesFromImportLiteral(file *ast.SourceFile, referencedFiles *collections.Set[tspath.Path], checker *checker.Checker, importName *ast.LiteralLikeNode) { + symbol := checker.GetSymbolAtLocation(importName) + addReferencedFilesFromSymbol(file, referencedFiles, symbol) +} + +// Gets the path to reference file from file name, it could be resolvedPath if present otherwise path +func addReferencedFileFromFileName(program *compiler.Program, fileName string, referencedFiles *collections.Set[tspath.Path], sourceFileDirectory string) { + if redirect := program.GetParseFileRedirect(fileName); redirect != "" { + referencedFiles.Add(tspath.ToPath(redirect, program.GetCurrentDirectory(), program.UseCaseSensitiveFileNames())) + } else { + referencedFiles.Add(tspath.ToPath(fileName, sourceFileDirectory, program.UseCaseSensitiveFileNames())) + } +} + +// Gets the referenced files for a file from the program with values for the keys as referenced file's path to be true +func getReferencedFiles(program *compiler.Program, file *ast.SourceFile) *collections.Set[tspath.Path] { + referencedFiles := collections.Set[tspath.Path]{} + + // We need to use a set here since the code can contain the same import twice, + // but that will only be one dependency. + // To avoid invernal conversion, the key of the referencedFiles map must be of type Path + checker, done := program.GetTypeCheckerForFile(context.TODO(), file) + defer done() + for _, importName := range file.Imports() { + addReferencedFilesFromImportLiteral(file, &referencedFiles, checker, importName) + } + + sourceFileDirectory := tspath.GetDirectoryPath(file.FileName()) + // Handle triple slash references + for _, referencedFile := range file.ReferencedFiles { + addReferencedFileFromFileName(program, referencedFile.FileName, &referencedFiles, sourceFileDirectory) + } + + // Handle type reference directives + if typeRefsInFile, ok := program.GetResolvedTypeReferenceDirectives()[file.Path()]; ok { + for _, typeRef := range typeRefsInFile { + if typeRef.ResolvedFileName != "" { + addReferencedFileFromFileName(program, typeRef.ResolvedFileName, &referencedFiles, sourceFileDirectory) + } + } + } + + // Add module augmentation as references + for _, moduleName := range file.ModuleAugmentations { + if !ast.IsStringLiteral(moduleName) { + continue + } + addReferencedFilesFromImportLiteral(file, &referencedFiles, checker, moduleName) + } + + // From ambient modules + for _, ambientModule := range checker.GetAmbientModules() { + addReferencedFilesFromSymbol(file, &referencedFiles, ambientModule) + } + return core.IfElse(referencedFiles.Len() > 0, &referencedFiles, nil) +} diff --git a/internal/incremental/snapshot.go b/internal/incremental/snapshot.go index 5e93fcd998..f0808597fc 100644 --- a/internal/incremental/snapshot.go +++ b/internal/incremental/snapshot.go @@ -1,20 +1,17 @@ package incremental import ( - "context" "encoding/hex" "fmt" - "maps" "strings" "sync" + "sync/atomic" "github.com/microsoft/typescript-go/internal/ast" - "github.com/microsoft/typescript-go/internal/checker" "github.com/microsoft/typescript-go/internal/collections" "github.com/microsoft/typescript-go/internal/compiler" "github.com/microsoft/typescript-go/internal/core" "github.com/microsoft/typescript-go/internal/diagnostics" - "github.com/microsoft/typescript-go/internal/tsoptions" "github.com/microsoft/typescript-go/internal/tspath" "github.com/zeebo/xxh3" ) @@ -188,22 +185,22 @@ type snapshot struct { // These are the fields that get serialized // Information of the file eg. its version, signature etc - fileInfos map[tspath.Path]*fileInfo + fileInfos collections.SyncMap[tspath.Path, *fileInfo] options *core.CompilerOptions // Contains the map of ReferencedSet=Referenced files of the file if module emit is enabled - referencedMap collections.ManyToManySet[tspath.Path, tspath.Path] + referencedMap collections.SyncManyToManySet[tspath.Path, tspath.Path] // Cache of semantic diagnostics for files with their Path being the key - semanticDiagnosticsPerFile map[tspath.Path]*diagnosticsOrBuildInfoDiagnosticsWithFileName + semanticDiagnosticsPerFile collections.SyncMap[tspath.Path, *diagnosticsOrBuildInfoDiagnosticsWithFileName] // Cache of dts emit diagnostics for files with their Path being the key - emitDiagnosticsPerFile map[tspath.Path]*diagnosticsOrBuildInfoDiagnosticsWithFileName + emitDiagnosticsPerFile collections.SyncMap[tspath.Path, *diagnosticsOrBuildInfoDiagnosticsWithFileName] // The map has key by source file's path that has been changed - changedFilesSet *collections.Set[tspath.Path] + changedFilesSet collections.SyncSet[tspath.Path] // Files pending to be emitted - affectedFilesPendingEmit map[tspath.Path]FileEmitKind + affectedFilesPendingEmit collections.SyncMap[tspath.Path, FileEmitKind] // Name of the file whose dts was the latest to change latestChangedDtsFile string // Hash of d.ts emitted for the file, use to track when emit of d.ts changes - emitSignatures map[tspath.Path]*emitSignature + emitSignatures collections.SyncMap[tspath.Path, *emitSignature] // Recorded if program had errors hasErrors core.Tristate // If semantic diagnsotic check is pending @@ -212,7 +209,7 @@ type snapshot struct { // Additional fields that are not serialized but needed to track state // true if build info emit is pending - buildInfoEmitPending bool + buildInfoEmitPending atomic.Bool hasErrorsFromOldState core.Tristate allFilesExcludingDefaultLibraryFileOnce sync.Once // Cache of all files excluding default library file for the current program @@ -222,25 +219,16 @@ type snapshot struct { hashWithText bool } -func (s *snapshot) createEmitSignaturesMap() { - if s.emitSignatures == nil && s.options.Composite.IsTrue() { - s.emitSignatures = make(map[tspath.Path]*emitSignature) - } -} - func (s *snapshot) addFileToChangeSet(filePath tspath.Path) { s.changedFilesSet.Add(filePath) - s.buildInfoEmitPending = true + s.buildInfoEmitPending.Store(true) } func (s *snapshot) addFileToAffectedFilesPendingEmit(filePath tspath.Path, emitKind FileEmitKind) { - existingKind := s.affectedFilesPendingEmit[filePath] - if s.affectedFilesPendingEmit == nil { - s.affectedFilesPendingEmit = make(map[tspath.Path]FileEmitKind) - } - s.affectedFilesPendingEmit[filePath] = existingKind | emitKind - delete(s.emitDiagnosticsPerFile, filePath) - s.buildInfoEmitPending = true + existingKind, _ := s.affectedFilesPendingEmit.Load(filePath) + s.affectedFilesPendingEmit.Store(filePath, existingKind|emitKind) + s.emitDiagnosticsPerFile.Delete(filePath) + s.buildInfoEmitPending.Store(true) } func (s *snapshot) getAllFilesExcludingDefaultLibraryFile(program *compiler.Program, firstSourceFile *ast.SourceFile) []*ast.SourceFile { @@ -314,254 +302,3 @@ func (s *snapshot) computeHash(text string) string { } return hash } - -func newSnapshotForProgram(program *compiler.Program, oldProgram *Program, hashWithText bool) *snapshot { - if oldProgram != nil && oldProgram.program == program { - return oldProgram.snapshot - } - files := program.GetSourceFiles() - snapshot := &snapshot{ - options: program.Options(), - semanticDiagnosticsPerFile: make(map[tspath.Path]*diagnosticsOrBuildInfoDiagnosticsWithFileName, len(files)), - hashWithText: hashWithText, - } - if oldProgram != nil && snapshot.options.Composite.IsTrue() { - snapshot.latestChangedDtsFile = oldProgram.snapshot.latestChangedDtsFile - } - if snapshot.options.NoCheck.IsTrue() { - snapshot.checkPending = true - } - - if oldProgram != nil { - // Copy old snapshot's changed files set - snapshot.changedFilesSet = oldProgram.snapshot.changedFilesSet.Clone() - if len(oldProgram.snapshot.affectedFilesPendingEmit) != 0 { - snapshot.affectedFilesPendingEmit = maps.Clone(oldProgram.snapshot.affectedFilesPendingEmit) - } - snapshot.buildInfoEmitPending = oldProgram.snapshot.buildInfoEmitPending - snapshot.hasErrorsFromOldState = oldProgram.snapshot.hasErrors - } else { - snapshot.changedFilesSet = &collections.Set[tspath.Path]{} - snapshot.buildInfoEmitPending = snapshot.options.IsIncremental() - } - - canCopySemanticDiagnostics := oldProgram != nil && - !tsoptions.CompilerOptionsAffectSemanticDiagnostics(oldProgram.snapshot.options, program.Options()) - // We can only reuse emit signatures (i.e. .d.ts signatures) if the .d.ts file is unchanged, - // which will eg be depedent on change in options like declarationDir and outDir options are unchanged. - // We need to look in oldState.compilerOptions, rather than oldCompilerOptions (i.e.we need to disregard useOldState) because - // oldCompilerOptions can be undefined if there was change in say module from None to some other option - // which would make useOldState as false since we can now use reference maps that are needed to track what to emit, what to check etc - // but that option change does not affect d.ts file name so emitSignatures should still be reused. - canCopyEmitSignatures := snapshot.options.Composite.IsTrue() && - oldProgram != nil && - oldProgram.snapshot.emitSignatures != nil && - !tsoptions.CompilerOptionsAffectDeclarationPath(oldProgram.snapshot.options, program.Options()) - copyDeclarationFileDiagnostics := canCopySemanticDiagnostics && - snapshot.options.SkipLibCheck.IsTrue() == oldProgram.snapshot.options.SkipLibCheck.IsTrue() - copyLibFileDiagnostics := copyDeclarationFileDiagnostics && - snapshot.options.SkipDefaultLibCheck.IsTrue() == oldProgram.snapshot.options.SkipDefaultLibCheck.IsTrue() - snapshot.fileInfos = make(map[tspath.Path]*fileInfo, len(files)) - for _, file := range files { - version := snapshot.computeHash(file.Text()) - impliedNodeFormat := program.GetSourceFileMetaData(file.Path()).ImpliedNodeFormat - affectsGlobalScope := fileAffectsGlobalScope(file) - var signature string - newReferences := getReferencedFiles(program, file) - if newReferences != nil { - snapshot.referencedMap.Set(file.Path(), newReferences) - } - if oldProgram != nil { - if oldFileInfo, ok := oldProgram.snapshot.fileInfos[file.Path()]; ok { - signature = oldFileInfo.signature - if oldFileInfo.version != version || oldFileInfo.affectsGlobalScope != affectsGlobalScope || oldFileInfo.impliedNodeFormat != impliedNodeFormat { - snapshot.addFileToChangeSet(file.Path()) - } else if oldReferences, _ := oldProgram.snapshot.referencedMap.GetValues(file.Path()); !newReferences.Equals(oldReferences) { - // Referenced files changed - snapshot.addFileToChangeSet(file.Path()) - } else if newReferences != nil { - for refPath := range newReferences.Keys() { - if program.GetSourceFileByPath(refPath) == nil && oldProgram.snapshot.fileInfos[refPath] != nil { - // Referenced file was deleted in the new program - snapshot.addFileToChangeSet(file.Path()) - break - } - } - } - } else { - snapshot.addFileToChangeSet(file.Path()) - } - if !snapshot.changedFilesSet.Has(file.Path()) { - if emitDiagnostics, ok := oldProgram.snapshot.emitDiagnosticsPerFile[file.Path()]; ok { - if snapshot.emitDiagnosticsPerFile == nil { - snapshot.emitDiagnosticsPerFile = make(map[tspath.Path]*diagnosticsOrBuildInfoDiagnosticsWithFileName, len(files)) - } - snapshot.emitDiagnosticsPerFile[file.Path()] = emitDiagnostics - } - if canCopySemanticDiagnostics { - if (!file.IsDeclarationFile || copyDeclarationFileDiagnostics) && - (!program.IsSourceFileDefaultLibrary(file.Path()) || copyLibFileDiagnostics) { - // Unchanged file copy diagnostics - if diagnostics, ok := oldProgram.snapshot.semanticDiagnosticsPerFile[file.Path()]; ok { - snapshot.semanticDiagnosticsPerFile[file.Path()] = diagnostics - } - } - } - } - if canCopyEmitSignatures { - if oldEmitSignature, ok := oldProgram.snapshot.emitSignatures[file.Path()]; ok { - snapshot.createEmitSignaturesMap() - snapshot.emitSignatures[file.Path()] = oldEmitSignature.getNewEmitSignature(oldProgram.snapshot.options, snapshot.options) - } - } - } else { - snapshot.addFileToAffectedFilesPendingEmit(file.Path(), GetFileEmitKind(snapshot.options)) - signature = version - } - snapshot.fileInfos[file.Path()] = &fileInfo{ - version: version, - signature: signature, - affectsGlobalScope: affectsGlobalScope, - impliedNodeFormat: impliedNodeFormat, - } - } - if oldProgram != nil { - // If the global file is removed, add all files as changed - allFilesExcludingDefaultLibraryFileAddedToChangeSet := false - for filePath, oldInfo := range oldProgram.snapshot.fileInfos { - if _, ok := snapshot.fileInfos[filePath]; !ok { - if oldInfo.affectsGlobalScope { - for _, file := range snapshot.getAllFilesExcludingDefaultLibraryFile(program, nil) { - snapshot.addFileToChangeSet(file.Path()) - } - allFilesExcludingDefaultLibraryFileAddedToChangeSet = true - } else { - snapshot.buildInfoEmitPending = true - } - break - } - } - if !allFilesExcludingDefaultLibraryFileAddedToChangeSet { - // If options affect emit, then we need to do complete emit per compiler options - // otherwise only the js or dts that needs to emitted because its different from previously emitted options - var pendingEmitKind FileEmitKind - if tsoptions.CompilerOptionsAffectEmit(oldProgram.snapshot.options, snapshot.options) { - pendingEmitKind = GetFileEmitKind(snapshot.options) - } else { - pendingEmitKind = getPendingEmitKindWithOptions(snapshot.options, oldProgram.snapshot.options) - } - if pendingEmitKind != FileEmitKindNone { - // Add all files to affectedFilesPendingEmit since emit changed - for _, file := range files { - // Add to affectedFilesPending emit only if not changed since any changed file will do full emit - if !snapshot.changedFilesSet.Has(file.Path()) { - snapshot.addFileToAffectedFilesPendingEmit(file.Path(), pendingEmitKind) - } - } - snapshot.buildInfoEmitPending = true - } - } - if len(snapshot.semanticDiagnosticsPerFile) != len(snapshot.fileInfos) && - oldProgram.snapshot.checkPending != snapshot.checkPending { - snapshot.buildInfoEmitPending = true - } - } - return snapshot -} - -func fileAffectsGlobalScope(file *ast.SourceFile) bool { - // if file contains anything that augments to global scope we need to build them as if - // they are global files as well as module - if core.Some(file.ModuleAugmentations, func(augmentation *ast.ModuleName) bool { - return ast.IsGlobalScopeAugmentation(augmentation.Parent) - }) { - return true - } - - if ast.IsExternalOrCommonJSModule(file) || ast.IsJsonSourceFile(file) { - return false - } - - // For script files that contains only ambient external modules, although they are not actually external module files, - // they can only be consumed via importing elements from them. Regular script files cannot consume them. Therefore, - // there are no point to rebuild all script files if these special files have changed. However, if any statement - // in the file is not ambient external module, we treat it as a regular script file. - return file.Statements != nil && - file.Statements.Nodes != nil && - core.Some(file.Statements.Nodes, func(stmt *ast.Node) bool { - return !ast.IsModuleWithStringLiteralName(stmt) - }) -} - -func addReferencedFilesFromSymbol(file *ast.SourceFile, referencedFiles *collections.Set[tspath.Path], symbol *ast.Symbol) { - if symbol == nil { - return - } - for _, declaration := range symbol.Declarations { - fileOfDecl := ast.GetSourceFileOfNode(declaration) - if fileOfDecl == nil { - continue - } - if file != fileOfDecl { - referencedFiles.Add(fileOfDecl.Path()) - } - } -} - -// Get the module source file and all augmenting files from the import name node from file -func addReferencedFilesFromImportLiteral(file *ast.SourceFile, referencedFiles *collections.Set[tspath.Path], checker *checker.Checker, importName *ast.LiteralLikeNode) { - symbol := checker.GetSymbolAtLocation(importName) - addReferencedFilesFromSymbol(file, referencedFiles, symbol) -} - -// Gets the path to reference file from file name, it could be resolvedPath if present otherwise path -func addReferencedFileFromFileName(program *compiler.Program, fileName string, referencedFiles *collections.Set[tspath.Path], sourceFileDirectory string) { - if redirect := program.GetParseFileRedirect(fileName); redirect != "" { - referencedFiles.Add(tspath.ToPath(redirect, program.GetCurrentDirectory(), program.UseCaseSensitiveFileNames())) - } else { - referencedFiles.Add(tspath.ToPath(fileName, sourceFileDirectory, program.UseCaseSensitiveFileNames())) - } -} - -// Gets the referenced files for a file from the program with values for the keys as referenced file's path to be true -func getReferencedFiles(program *compiler.Program, file *ast.SourceFile) *collections.Set[tspath.Path] { - referencedFiles := collections.Set[tspath.Path]{} - - // We need to use a set here since the code can contain the same import twice, - // but that will only be one dependency. - // To avoid invernal conversion, the key of the referencedFiles map must be of type Path - checker, done := program.GetTypeCheckerForFile(context.TODO(), file) - defer done() - for _, importName := range file.Imports() { - addReferencedFilesFromImportLiteral(file, &referencedFiles, checker, importName) - } - - sourceFileDirectory := tspath.GetDirectoryPath(file.FileName()) - // Handle triple slash references - for _, referencedFile := range file.ReferencedFiles { - addReferencedFileFromFileName(program, referencedFile.FileName, &referencedFiles, sourceFileDirectory) - } - - // Handle type reference directives - if typeRefsInFile, ok := program.GetResolvedTypeReferenceDirectives()[file.Path()]; ok { - for _, typeRef := range typeRefsInFile { - if typeRef.ResolvedFileName != "" { - addReferencedFileFromFileName(program, typeRef.ResolvedFileName, &referencedFiles, sourceFileDirectory) - } - } - } - - // Add module augmentation as references - for _, moduleName := range file.ModuleAugmentations { - if !ast.IsStringLiteral(moduleName) { - continue - } - addReferencedFilesFromImportLiteral(file, &referencedFiles, checker, moduleName) - } - - // From ambient modules - for _, ambientModule := range checker.GetAmbientModules() { - addReferencedFilesFromSymbol(file, &referencedFiles, ambientModule) - } - return core.IfElse(referencedFiles.Len() > 0, &referencedFiles, nil) -} diff --git a/internal/incremental/tobuildinfo.go b/internal/incremental/snapshottobuildinfo.go similarity index 93% rename from internal/incremental/tobuildinfo.go rename to internal/incremental/snapshottobuildinfo.go index 22345723a4..169d53f43f 100644 --- a/internal/incremental/tobuildinfo.go +++ b/internal/incremental/snapshottobuildinfo.go @@ -171,7 +171,7 @@ func (t *toBuildInfo) toBuildInfoDiagnosticsOfFile(filePath tspath.Path, diags * func (t *toBuildInfo) setFileInfoAndEmitSignatures() { t.buildInfo.FileInfos = core.Map(t.program.GetSourceFiles(), func(file *ast.SourceFile) *BuildInfoFileInfo { - info := t.snapshot.fileInfos[file.Path()] + info, _ := t.snapshot.fileInfos.Load(file.Path()) fileId := t.toFileId(file.Path()) // tryAddRoot(key, fileId); if t.buildInfo.FileNames[fileId-1] != t.relativeToBuildInfo(string(file.Path())) { @@ -179,8 +179,7 @@ func (t *toBuildInfo) setFileInfoAndEmitSignatures() { } if t.snapshot.options.Composite.IsTrue() { if !ast.IsJsonSourceFile(file) && t.program.SourceFileMayBeEmitted(file, false) { - emitSignature := t.snapshot.emitSignatures[file.Path()] - if emitSignature == nil { + if emitSignature, loaded := t.snapshot.emitSignatures.Load(file.Path()); !loaded { t.buildInfo.EmitSignatures = append(t.buildInfo.EmitSignatures, &BuildInfoEmitSignature{ FileId: fileId, }) @@ -225,7 +224,7 @@ func (t *toBuildInfo) setCompilerOptions() { } func (t *toBuildInfo) setReferencedMap() { - keys := slices.Collect(maps.Keys(t.snapshot.referencedMap.Keys())) + keys := slices.Collect(t.snapshot.referencedMap.Keys().Keys()) slices.Sort(keys) t.buildInfo.ReferencedMap = core.Map(keys, func(filePath tspath.Path) *BuildInfoReferenceMapEntry { references, _ := t.snapshot.referencedMap.GetValues(filePath) @@ -237,14 +236,14 @@ func (t *toBuildInfo) setReferencedMap() { } func (t *toBuildInfo) setChangeFileSet() { - files := slices.Collect(maps.Keys(t.snapshot.changedFilesSet.Keys())) + files := slices.Collect(t.snapshot.changedFilesSet.Keys()) slices.Sort(files) t.buildInfo.ChangeFileSet = core.Map(files, t.toFileId) } func (t *toBuildInfo) setSemanticDiagnostics() { for _, file := range t.program.GetSourceFiles() { - value, ok := t.snapshot.semanticDiagnosticsPerFile[file.Path()] + value, ok := t.snapshot.semanticDiagnosticsPerFile.Load(file.Path()) if !ok { if !t.snapshot.changedFilesSet.Has(file.Path()) { t.buildInfo.SemanticDiagnosticsPerFile = append(t.buildInfo.SemanticDiagnosticsPerFile, &BuildInfoSemanticDiagnostic{ @@ -263,18 +262,16 @@ func (t *toBuildInfo) setSemanticDiagnostics() { } func (t *toBuildInfo) setEmitDiagnostics() { - files := slices.Collect(maps.Keys(t.snapshot.emitDiagnosticsPerFile)) + files := slices.Collect(t.snapshot.emitDiagnosticsPerFile.Keys()) slices.Sort(files) t.buildInfo.EmitDiagnosticsPerFile = core.Map(files, func(filePath tspath.Path) *BuildInfoDiagnosticsOfFile { - return t.toBuildInfoDiagnosticsOfFile(filePath, t.snapshot.emitDiagnosticsPerFile[filePath]) + value, _ := t.snapshot.emitDiagnosticsPerFile.Load(filePath) + return t.toBuildInfoDiagnosticsOfFile(filePath, value) }) } func (t *toBuildInfo) setAffectedFilesPendingEmit() { - if len(t.snapshot.affectedFilesPendingEmit) == 0 { - return - } - files := slices.Collect(maps.Keys(t.snapshot.affectedFilesPendingEmit)) + files := slices.Collect(t.snapshot.affectedFilesPendingEmit.Keys()) slices.Sort(files) fullEmitKind := GetFileEmitKind(t.snapshot.options) for _, filePath := range files { @@ -282,7 +279,7 @@ func (t *toBuildInfo) setAffectedFilesPendingEmit() { if file == nil || !t.program.SourceFileMayBeEmitted(file, false) { continue } - pendingEmit := t.snapshot.affectedFilesPendingEmit[filePath] + pendingEmit, _ := t.snapshot.affectedFilesPendingEmit.Load(filePath) t.buildInfo.AffectedFilesPendingEmit = append(t.buildInfo.AffectedFilesPendingEmit, &BuildInfoFilePendingEmit{ FileId: t.toFileId(filePath), EmitKind: core.IfElse(pendingEmit == fullEmitKind, 0, pendingEmit),