diff --git a/src/services/services.ts b/src/services/services.ts
index 32fd03a331f3f..c0d386291a5a6 100644
--- a/src/services/services.ts
+++ b/src/services/services.ts
@@ -1405,8 +1405,10 @@ namespace ts {
 
         function dispose(): void {
             if (program) {
+                // Use paths to ensure we are using correct key and paths as document registry could bre created with different current directory than host
+                const key = documentRegistry.getKeyForCompilationSettings(program.getCompilerOptions());
                 forEach(program.getSourceFiles(), f =>
-                    documentRegistry.releaseDocument(f.fileName, program.getCompilerOptions()));
+                    documentRegistry.releaseDocumentWithKey(f.resolvedPath, key));
                 program = undefined!; // TODO: GH#18217
             }
             host = undefined!;
diff --git a/src/testRunner/unittests/tsserver/dynamicFiles.ts b/src/testRunner/unittests/tsserver/dynamicFiles.ts
index 214fd020d65b9..6b551a6453d78 100644
--- a/src/testRunner/unittests/tsserver/dynamicFiles.ts
+++ b/src/testRunner/unittests/tsserver/dynamicFiles.ts
@@ -97,6 +97,30 @@ var x = 10;`
             checkProjectActualFiles(service.configuredProjects.get(config.path)!, [untitled.path, libFile.path, config.path]);
             checkProjectActualFiles(service.inferredProjects[0], [untitledFile, libFile.path]);
         });
+
+        it("opening and closing untitled files when projectRootPath is different from currentDirectory", () => {
+            const config: File = {
+                path: `${tscWatch.projectRoot}/tsconfig.json`,
+                content: "{}"
+            };
+            const file: File = {
+                path: `${tscWatch.projectRoot}/file.ts`,
+                content: "const y = 10"
+            };
+            const host = createServerHost([config, file, libFile], { useCaseSensitiveFileNames: true });
+            const service = createProjectService(host, /*parameters*/ undefined, { useInferredProjectPerProjectRoot: true });
+            service.openClientFile(untitledFile, "const x = 10;", /*scriptKind*/ undefined, tscWatch.projectRoot);
+            checkNumberOfProjects(service, { inferredProjects: 1 });
+            checkProjectActualFiles(service.inferredProjects[0], [untitledFile, libFile.path]);
+            verifyDynamic(service, `${tscWatch.projectRoot}/${untitledFile}`);
+
+            // Close untitled file
+            service.closeClientFile(untitledFile);
+
+            // Open file from configured project which should collect inferredProject
+            service.openClientFile(file.path);
+            checkNumberOfProjects(service, { configuredProjects: 1 });
+        });
     });
 
     describe("unittests:: tsserver:: dynamicFiles:: ", () => {