From f473b57ff8fb2f32e0397702d4c6b75bf2b83ac5 Mon Sep 17 00:00:00 2001 From: Wim Tibackx Date: Fri, 17 May 2024 21:20:03 +0200 Subject: [PATCH 01/10] fix: reflect current pnpm version (in step with lockfile) and document usage of symlinks, which requires extra steps on Windows. --- CONTRIBUTING.md | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 659fe2184..f9f601c99 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -5,10 +5,16 @@ I want to think you first for considering contributing to ZenStack 🙏🏻. It' ## Prerequisites - [Node.js](https://nodejs.org/): v18 or above -- [pnpm](https://pnpm.io/): latest version +- [pnpm](https://pnpm.io/): v8.x ## Get started +1. (Windows only) Your environment should support symlinks, by enabling "Developer mode" in `Settings => System => For developers` (Windows 10/11 only) and setting the `core.symlinks` setting in git to `true`. For more info [refer to this StackOverflow answer](https://stackoverflow.com/questions/5917249/git-symbolic-links-in-windows/59761201#59761201). + + ```pwsh + git config --global core.symlinks true + ``` + 1. Make a fork of the repository Make sure all branches are included. From 8e645d5d58e1715d2fa374a8464a5e5afcbd295f Mon Sep 17 00:00:00 2001 From: Wim Tibackx Date: Fri, 17 May 2024 21:37:42 +0200 Subject: [PATCH 02/10] fix: enforce line endings on which the repo is currently standardized. Running `pnpm build` on windows builds generated files with CRLF line endings. Adding the `eol=lf` entry in the `.gitattributes` file enforces LF line endings when files are staged, so line ending changes on those generated files are negated / ignored. Furthermore, this ensures consistency on newly added files. --- .gitattributes | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 .gitattributes diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 000000000..2766c310e --- /dev/null +++ b/.gitattributes @@ -0,0 +1,3 @@ +* text=auto eol=lf + +*.bat text=auto eol=crlf \ No newline at end of file From 12f4a5b6edf4bdf684f6026e0f56a057c92d0a4e Mon Sep 17 00:00:00 2001 From: Wim Tibackx Date: Fri, 17 May 2024 21:50:20 +0200 Subject: [PATCH 03/10] fix: use escaped double quotes when we need quotes in package.json scripts; and remove single quotes when we don't need them. This improves windows compatibility. --- packages/language/package.json | 2 +- packages/plugins/trpc/package.json | 2 +- packages/runtime/package.json | 2 +- packages/schema/package.json | 2 +- packages/sdk/package.json | 2 +- packages/server/package.json | 2 +- packages/testtools/package.json | 2 +- 7 files changed, 7 insertions(+), 7 deletions(-) diff --git a/packages/language/package.json b/packages/language/package.json index 01ae8ec10..9ed8397d9 100644 --- a/packages/language/package.json +++ b/packages/language/package.json @@ -9,7 +9,7 @@ "generate": "langium generate && npx ts-node script/generate-plist.ts", "watch": "concurrently \"langium generate --watch\" \"tsc --watch\"", "lint": "eslint src --ext ts", - "build": "pnpm lint --max-warnings=0 && pnpm clean && pnpm generate && tsc && copyfiles -F ./README.md ./LICENSE ./package.json 'syntaxes/**/*' dist && pnpm pack dist --pack-destination ../../../.build", + "build": "pnpm lint --max-warnings=0 && pnpm clean && pnpm generate && tsc && copyfiles -F ./README.md ./LICENSE ./package.json \"syntaxes/**/*\" dist && pnpm pack dist --pack-destination ../../../.build", "prepublishOnly": "pnpm build" }, "publishConfig": { diff --git a/packages/plugins/trpc/package.json b/packages/plugins/trpc/package.json index 7027f2d0a..9e4f1a4c5 100644 --- a/packages/plugins/trpc/package.json +++ b/packages/plugins/trpc/package.json @@ -10,7 +10,7 @@ }, "scripts": { "clean": "rimraf dist", - "build": "pnpm lint --max-warnings=0 && pnpm clean && tsc && copyfiles ./package.json ./README.md ./LICENSE 'res/**/*' dist && pnpm pack dist --pack-destination ../../../../.build", + "build": "pnpm lint --max-warnings=0 && pnpm clean && tsc && copyfiles ./package.json ./README.md ./LICENSE \"res/**/*\" dist && pnpm pack dist --pack-destination ../../../../.build", "watch": "tsc --watch", "lint": "eslint src --ext ts", "test": "jest", diff --git a/packages/runtime/package.json b/packages/runtime/package.json index 12aa58df1..6609dadc8 100644 --- a/packages/runtime/package.json +++ b/packages/runtime/package.json @@ -9,7 +9,7 @@ }, "scripts": { "clean": "rimraf dist", - "build": "pnpm lint --max-warnings=0 && pnpm clean && tsc && tsup-node --config ./tsup-browser.config.ts && tsup-node --config ./tsup-cross.config.ts && copyfiles ./package.json ./README.md ../../LICENSE dist && copyfiles -u1 'res/**/*' dist && pnpm pack dist --pack-destination '../../../.build'", + "build": "pnpm lint --max-warnings=0 && pnpm clean && tsc && tsup-node --config ./tsup-browser.config.ts && tsup-node --config ./tsup-cross.config.ts && copyfiles ./package.json ./README.md ../../LICENSE dist && copyfiles -u1 \"res/**/*\" dist && pnpm pack dist --pack-destination ../../../.build", "watch": "concurrently \"tsc --watch\" \"tsup-node --config ./tsup-browser.config.ts --watch\" \"tsup-node --config ./tsup-cross.config.ts --watch\"", "lint": "eslint src --ext ts", "prepublishOnly": "pnpm build" diff --git a/packages/schema/package.json b/packages/schema/package.json index a27f41f3b..90d3d8edd 100644 --- a/packages/schema/package.json +++ b/packages/schema/package.json @@ -80,7 +80,7 @@ "vscode:prepublish": "pnpm bundle", "vscode:package": "pnpm bundle && vsce package --no-dependencies", "clean": "rimraf dist", - "build": "pnpm clean && pnpm lint --max-warnings=0 && tsc && copyfiles -F \"bin/*\" dist && copyfiles ./README-global.md ./LICENSE ./package.json dist && renamer --replace \"README.md\" dist/README-global.md && copyfiles -u 1 \"src/res/*\" dist && node build/post-build.js && pnpm pack dist --pack-destination '../../../.build'", + "build": "pnpm clean && pnpm lint --max-warnings=0 && tsc && copyfiles -F \"bin/*\" dist && copyfiles ./README-global.md ./LICENSE ./package.json dist && renamer --replace \"README.md\" dist/README-global.md && copyfiles -u 1 \"src/res/*\" dist && node build/post-build.js && pnpm pack dist --pack-destination ../../../.build", "bundle": "rimraf bundle && pnpm lint --max-warnings=0 && node build/bundle.js --minify", "watch": "tsc --watch", "lint": "eslint src tests --ext ts", diff --git a/packages/sdk/package.json b/packages/sdk/package.json index 436a715a6..274039a58 100644 --- a/packages/sdk/package.json +++ b/packages/sdk/package.json @@ -6,7 +6,7 @@ "scripts": { "clean": "rimraf dist", "lint": "eslint src --ext ts", - "build": "pnpm lint --max-warnings=0 && pnpm clean && tsc && copyfiles ./package.json ./LICENSE ./README.md dist && pnpm pack dist --pack-destination '../../../.build'", + "build": "pnpm lint --max-warnings=0 && pnpm clean && tsc && copyfiles ./package.json ./LICENSE ./README.md dist && pnpm pack dist --pack-destination ../../../.build", "watch": "tsc --watch", "prepublishOnly": "pnpm build" }, diff --git a/packages/server/package.json b/packages/server/package.json index 1802f0771..7dab9fe75 100644 --- a/packages/server/package.json +++ b/packages/server/package.json @@ -6,7 +6,7 @@ "homepage": "https://zenstack.dev", "scripts": { "clean": "rimraf dist", - "build": "pnpm lint --max-warnings=0 && pnpm clean && tsc && copyfiles ./package.json ./README.md ./LICENSE dist && pnpm pack dist --pack-destination '../../../.build'", + "build": "pnpm lint --max-warnings=0 && pnpm clean && tsc && copyfiles ./package.json ./README.md ./LICENSE dist && pnpm pack dist --pack-destination ../../../.build", "watch": "tsc --watch", "lint": "eslint src --ext ts", "test": "jest", diff --git a/packages/testtools/package.json b/packages/testtools/package.json index 2c3bc1ae1..324296dda 100644 --- a/packages/testtools/package.json +++ b/packages/testtools/package.json @@ -11,7 +11,7 @@ "scripts": { "clean": "rimraf dist", "lint": "eslint src --ext ts", - "build": "pnpm lint && pnpm clean && tsc && copyfiles ./package.json ./LICENSE ./README.md dist && pnpm pack dist --pack-destination '../../../.build'", + "build": "pnpm lint && pnpm clean && tsc && copyfiles ./package.json ./LICENSE ./README.md dist && pnpm pack dist --pack-destination ../../../.build", "watch": "tsc --watch", "prepublishOnly": "pnpm build" }, From b31d0cfa5d250049df92fb36e1547b82480a8733 Mon Sep 17 00:00:00 2001 From: Wim Tibackx Date: Fri, 17 May 2024 22:06:10 +0200 Subject: [PATCH 04/10] fix: execute the fitting gradlew variant depending on OS --- packages/ide/jetbrains/package.json | 9 ++++++--- pnpm-lock.yaml | 8 ++++++++ 2 files changed, 14 insertions(+), 3 deletions(-) diff --git a/packages/ide/jetbrains/package.json b/packages/ide/jetbrains/package.json index 7f18e5e4a..5392f2b67 100644 --- a/packages/ide/jetbrains/package.json +++ b/packages/ide/jetbrains/package.json @@ -6,12 +6,15 @@ "homepage": "https://zenstack.dev", "private": true, "scripts": { - "build": "./gradlew buildPlugin" + "build": "run-script-os", + "build:win32": "gradlew.bat buildPlugin", + "build:default": "./gradlew buildPlugin" }, "author": "ZenStack Team", "license": "MIT", "devDependencies": { - "zenstack": "workspace:*", - "@zenstackhq/language": "workspace:*" + "@zenstackhq/language": "workspace:*", + "run-script-os": "^1.1.6", + "zenstack": "workspace:*" } } diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index a8e8c7318..8bb3f206a 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -65,6 +65,9 @@ importers: '@zenstackhq/language': specifier: workspace:* version: link:../../language/dist + run-script-os: + specifier: ^1.1.6 + version: 1.1.6 zenstack: specifier: workspace:* version: link:../../schema/dist @@ -13062,6 +13065,11 @@ packages: dependencies: queue-microtask: 1.2.3 + /run-script-os@1.1.6: + resolution: {integrity: sha512-ql6P2LzhBTTDfzKts+Qo4H94VUKpxKDFz6QxxwaUZN0mwvi7L3lpOI7BqPCq7lgDh3XLl0dpeXwfcVIitlrYrw==} + hasBin: true + dev: true + /rxjs@7.8.1: resolution: {integrity: sha512-AA3TVj+0A2iuIoQkWEK/tqFjBq2j+6PO6Y0zJcvzLAFhEFIO3HL0vls9hWLncZbAAbK0mar7oZ4V079I/qPMxg==} dependencies: From 3b096c56cfd9b1552ddee33e6aedabccde969a7c Mon Sep 17 00:00:00 2001 From: Wim Tibackx Date: Sat, 18 May 2024 16:19:03 +0200 Subject: [PATCH 05/10] feat: document the use of a postgres database for tests, and allow configuring connection details through environment variables to allow slightly more flexibility. --- CONTRIBUTING.md | 8 ++++++++ packages/testtools/src/db.ts | 23 ++++++++++++++++++----- 2 files changed, 26 insertions(+), 5 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index f9f601c99..e31698560 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -7,6 +7,14 @@ I want to think you first for considering contributing to ZenStack 🙏🏻. It' - [Node.js](https://nodejs.org/): v18 or above - [pnpm](https://pnpm.io/): v8.x +If you want to run the tests, you should be aware that some of the integration tests run against postgres. These tests will attempt to set up and subsequently their own database, so you'll need to provide a connection details for a postgres user with at least those permissions. To provide connection details, you can configure the following environment variables or provide them when executing `pnpm test` commands. + +- `ZENSTACK_TEST_DB_USER`: The postgres username, for a user with permission to create/drop databases. Default: `postgres`. +- `ZENSTACK_TEST_DB_PASS`: Password for said user. Default: `abc123`. +- `ZENSTACK_TEST_DB_NAME`: Default database to connect onto. This database isn't used any further, so it's recommended to just use the default `postgres` database. Default: `postgres`. +- `ZENSTACK_TEST_DB_HOST`: Hostname or IP to connect onto. Default: `localhost`. +- `ZENSTACK_TEST_DB_PORT`: Port number to connect onto. Default: `5432`. + ## Get started 1. (Windows only) Your environment should support symlinks, by enabling "Developer mode" in `Settings => System => For developers` (Windows 10/11 only) and setting the `core.symlinks` setting in git to `true`. For more info [refer to this StackOverflow answer](https://stackoverflow.com/questions/5917249/git-symbolic-links-in-windows/59761201#59761201). diff --git a/packages/testtools/src/db.ts b/packages/testtools/src/db.ts index 8de49a7c1..b242a236b 100644 --- a/packages/testtools/src/db.ts +++ b/packages/testtools/src/db.ts @@ -1,16 +1,29 @@ import { Pool } from 'pg'; -const USERNAME = 'postgres'; -const PASSWORD = 'abc123'; +const USERNAME = process.env.ZENSTACK_TEST_DB_USER || 'postgres'; +const PASSWORD = process.env.ZENSTACK_TEST_DB_PASS || 'abc123'; +const CONNECTION_DB = process.env.ZENSTACK_TEST_DB_NAME || 'postgres'; +const HOST = process.env.ZENSTACK_TEST_DB_HOST || 'localhost'; +const PORT = (process.env.ZENSTACK_TEST_DB_PORT ? parseInt(process.env.ZENSTACK_TEST_DB_PORT) : null) || 5432; + +function connect() { + return new Pool({ + user: USERNAME, + password: PASSWORD, + database: CONNECTION_DB, + host: HOST, + port: PORT + }); +} export async function createPostgresDb(db: string) { - const pool = new Pool({ user: USERNAME, password: PASSWORD }); + const pool = connect(); await pool.query(`DROP DATABASE IF EXISTS "${db}";`); await pool.query(`CREATE DATABASE "${db}";`); - return `postgresql://${USERNAME}:${PASSWORD}@localhost:5432/${db}`; + return `postgresql://${USERNAME}:${PASSWORD}@${HOST}:${PORT}/${db}`; } export async function dropPostgresDb(db: string) { - const pool = new Pool({ user: USERNAME, password: PASSWORD }); + const pool = connect(); await pool.query(`DROP DATABASE IF EXISTS "${db}";`); } From 77034e2f98ce7ddab786c26fe7194d1b396c5bff Mon Sep 17 00:00:00 2001 From: Wim Tibackx Date: Sat, 18 May 2024 16:19:35 +0200 Subject: [PATCH 06/10] fix: minimal path normalization to handle Windows paths --- packages/testtools/src/schema.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/testtools/src/schema.ts b/packages/testtools/src/schema.ts index 4495ddf14..3d9605de1 100644 --- a/packages/testtools/src/schema.ts +++ b/packages/testtools/src/schema.ts @@ -172,7 +172,7 @@ export async function loadSchema(schema: string, options?: SchemaLoadOptions) { process.chdir(projectDir); // copy project structure from scaffold (prepared by test-setup.ts) - fs.cpSync(path.join(workspaceRoot, '.test/scaffold'), projectDir, { recursive: true, force: true }); + fs.cpSync(path.normalize(path.join(workspaceRoot, '.test/scaffold')), path.normalize(projectDir), { recursive: true, force: true }); // install local deps const localInstallDeps = [ From 07f3dbac9cd4f9d232e727aa42023c0757a74896 Mon Sep 17 00:00:00 2001 From: Wim Tibackx Date: Sat, 18 May 2024 18:16:38 +0200 Subject: [PATCH 07/10] fix: close the postgres connection pool --- packages/testtools/src/db.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/packages/testtools/src/db.ts b/packages/testtools/src/db.ts index b242a236b..16142d527 100644 --- a/packages/testtools/src/db.ts +++ b/packages/testtools/src/db.ts @@ -20,10 +20,12 @@ export async function createPostgresDb(db: string) { const pool = connect(); await pool.query(`DROP DATABASE IF EXISTS "${db}";`); await pool.query(`CREATE DATABASE "${db}";`); + await pool.end(); return `postgresql://${USERNAME}:${PASSWORD}@${HOST}:${PORT}/${db}`; } export async function dropPostgresDb(db: string) { const pool = connect(); await pool.query(`DROP DATABASE IF EXISTS "${db}";`); + await pool.end(); } From 9516d5406f897020571546125680ab6d0986a585 Mon Sep 17 00:00:00 2001 From: Wim Tibackx Date: Sat, 18 May 2024 19:05:25 +0200 Subject: [PATCH 08/10] fix: seems like we can just generally use nodes' path.normalize? --- packages/testtools/src/schema.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/testtools/src/schema.ts b/packages/testtools/src/schema.ts index 3d9605de1..00093fab6 100644 --- a/packages/testtools/src/schema.ts +++ b/packages/testtools/src/schema.ts @@ -62,7 +62,7 @@ export function installPackage(pkg: string, dev = false) { } export function normalizePath(p: string) { - return p ? p.split(path.sep).join(path.posix.sep) : p; + return path.normalize(p); } export function getWorkspaceRoot(start: string) { @@ -172,7 +172,7 @@ export async function loadSchema(schema: string, options?: SchemaLoadOptions) { process.chdir(projectDir); // copy project structure from scaffold (prepared by test-setup.ts) - fs.cpSync(path.normalize(path.join(workspaceRoot, '.test/scaffold')), path.normalize(projectDir), { recursive: true, force: true }); + fs.cpSync(path.join(workspaceRoot, '.test/scaffold'), projectDir, { recursive: true, force: true }); // install local deps const localInstallDeps = [ From 67eeeeb31f1f4daaca7ce52520529ef91af2af3a Mon Sep 17 00:00:00 2001 From: Wim Tibackx Date: Fri, 24 May 2024 15:30:06 +0200 Subject: [PATCH 09/10] fix: improve path handling in tests on windows --- packages/plugins/swr/tests/swr.test.ts | 6 ++++-- packages/testtools/src/schema.ts | 9 +++++++-- 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/packages/plugins/swr/tests/swr.test.ts b/packages/plugins/swr/tests/swr.test.ts index 7c26f18bd..c2a539879 100644 --- a/packages/plugins/swr/tests/swr.test.ts +++ b/packages/plugins/swr/tests/swr.test.ts @@ -106,11 +106,13 @@ ${sharedModel} const { name: projectDir } = tmp.dirSync(); fs.writeFileSync(path.join(projectDir, 'swr'), 'hello'); + const swrPath = normalizePath(path.join(__dirname, '..', 'dist')); + await expect( loadSchema( ` plugin swr { - provider = '${normalizePath(path.resolve(__dirname, '../dist'))}' + provider = '${swrPath}' output = '$projectRoot/swr' } @@ -122,7 +124,7 @@ ${sharedModel} password String @omit } `, - { pushDb: false, projectDir, extraDependencies: [`${normalizePath(path.join(__dirname, '../dist'))}`] } + { pushDb: false, projectDir, extraDependencies: [`${swrPath}`] } ) ).rejects.toThrow('already exists and is not a directory'); }); diff --git a/packages/testtools/src/schema.ts b/packages/testtools/src/schema.ts index 00093fab6..5e5b492d2 100644 --- a/packages/testtools/src/schema.ts +++ b/packages/testtools/src/schema.ts @@ -61,8 +61,14 @@ export function installPackage(pkg: string, dev = false) { run(`npm install ${dev ? '-D' : ''} --no-audit --no-fund ${pkg}`); } +/** + * Normalizes a path for use within zenstack model files, that is: replaces the default path separators for the OS with POSIX path separators. + * + * @param p A filesystem path + * @returns The filesystem path adjusted to posix path separators (`/`) + */ export function normalizePath(p: string) { - return path.normalize(p); + return p ? p.split(path.sep).join(path.posix.sep) : p; } export function getWorkspaceRoot(start: string) { @@ -163,7 +169,6 @@ export async function loadSchema(schema: string, options?: SchemaLoadOptions) { } const workspaceRoot = getWorkspaceRoot(__dirname); - if (!workspaceRoot) { throw new Error('Could not find workspace root'); } From 38623d5890934a23a4f2ccdb24de138da3ef7423 Mon Sep 17 00:00:00 2001 From: Wim Tibackx Date: Fri, 24 May 2024 15:55:39 +0200 Subject: [PATCH 10/10] fix: revert any path changes --- packages/plugins/swr/tests/swr.test.ts | 6 ++---- packages/testtools/src/schema.ts | 7 +------ 2 files changed, 3 insertions(+), 10 deletions(-) diff --git a/packages/plugins/swr/tests/swr.test.ts b/packages/plugins/swr/tests/swr.test.ts index c2a539879..7c26f18bd 100644 --- a/packages/plugins/swr/tests/swr.test.ts +++ b/packages/plugins/swr/tests/swr.test.ts @@ -106,13 +106,11 @@ ${sharedModel} const { name: projectDir } = tmp.dirSync(); fs.writeFileSync(path.join(projectDir, 'swr'), 'hello'); - const swrPath = normalizePath(path.join(__dirname, '..', 'dist')); - await expect( loadSchema( ` plugin swr { - provider = '${swrPath}' + provider = '${normalizePath(path.resolve(__dirname, '../dist'))}' output = '$projectRoot/swr' } @@ -124,7 +122,7 @@ ${sharedModel} password String @omit } `, - { pushDb: false, projectDir, extraDependencies: [`${swrPath}`] } + { pushDb: false, projectDir, extraDependencies: [`${normalizePath(path.join(__dirname, '../dist'))}`] } ) ).rejects.toThrow('already exists and is not a directory'); }); diff --git a/packages/testtools/src/schema.ts b/packages/testtools/src/schema.ts index 5e5b492d2..4495ddf14 100644 --- a/packages/testtools/src/schema.ts +++ b/packages/testtools/src/schema.ts @@ -61,12 +61,6 @@ export function installPackage(pkg: string, dev = false) { run(`npm install ${dev ? '-D' : ''} --no-audit --no-fund ${pkg}`); } -/** - * Normalizes a path for use within zenstack model files, that is: replaces the default path separators for the OS with POSIX path separators. - * - * @param p A filesystem path - * @returns The filesystem path adjusted to posix path separators (`/`) - */ export function normalizePath(p: string) { return p ? p.split(path.sep).join(path.posix.sep) : p; } @@ -169,6 +163,7 @@ export async function loadSchema(schema: string, options?: SchemaLoadOptions) { } const workspaceRoot = getWorkspaceRoot(__dirname); + if (!workspaceRoot) { throw new Error('Could not find workspace root'); }