master merge

.github/workflows/jetbrains-update-plugin-platform.yml

@@ -20,6 +20,6 @@ jobs:
             pluginName: JetBrains Gateway Plugin
             pluginId: gateway-plugin
             xpath: "(/html/body/table[preceding::h2/text()='com.jetbrains.gateway'][1]/tbody/tr/td[contains(text(),'-CUSTOM-SNAPSHOT') and starts-with(text(),'MAJOR_VERSION_PLACEHOLDER') and not(contains(text(),'-NIGHTLY'))]/text())[1]"
-            gradlePropertiesPath: components/ide/jetbrains/gateway-plugin/gradle.properties
+            gradlePropertiesPath: components/ide/jetbrains/gateway-plugin/gradle-latest.properties
         secrets:
             slackWebhook: ${{ secrets.IDE_SLACK_WEBHOOK }}

.gitpod.yml

@@ -52,7 +52,7 @@ tasks:
         read -r -p "Press enter to continue Java gradle task"
       fi
       leeway exec --package components/supervisor-api/java:lib --package components/gitpod-protocol/java:lib -- ./gradlew build
-      leeway exec --package components/ide/jetbrains/backend-plugin:plugin-latest --package components/ide/jetbrains/gateway-plugin:publish --parallel -- ./gradlew buildPlugin
+      leeway exec --package components/ide/jetbrains/backend-plugin:plugin-latest --package components/ide/jetbrains/gateway-plugin:publish-latest --parallel -- ./gradlew buildPlugin
   - name: TypeScript
     before: scripts/branch-namespace.sh
     init: yarn --network-timeout 100000 && yarn build

.werft/installer-tests.ts

@@ -1,7 +1,7 @@
 import * as fs from "fs";
 import * as https from "https";
 import { join } from "path";
-import { exec } from "./util/shell";
+import { exec, execStream } from "./util/shell";
 import { Werft } from "./util/werft";
 import { deleteReplicatedLicense } from "./jobs/build/self-hosted-upgrade-tests";
 
@@ -282,11 +282,15 @@ if (config === undefined) {
 }
 
 installerTests(TEST_CONFIGURATIONS[testConfig]).catch((err) => {
+    console.error(err);
+
     if(deleteOnFail == "true") {
-        cleanup();
+        cleanup().finally(() => {
+            process.exit(1);
+        });
+    } else {
+        process.exit(1);
     }
-    console.error(err);
-    process.exit(1);
 });
 
 function logJobConfig(): void {
@@ -332,7 +336,7 @@ export async function installerTests(config: TestConfig) {
     werft.phase(majorPhase, phaseMessage);
     for (let phase of config.PHASES) {
         const phaseSteps = INFRA_PHASES[phase];
-        const ret = callMakeTargets(phaseSteps.phase, phaseSteps.description, phaseSteps.makeTarget);
+        const ret = await callMakeTargets(phaseSteps.phase, phaseSteps.description, phaseSteps.makeTarget);
         if (ret) {
             // there is not point in continuing if one stage fails for infra setup
             const err: Error = new Error("Cluster creation failed");
@@ -359,7 +363,7 @@ export async function installerTests(config: TestConfig) {
         // runIntegrationTests()
 
         const upgradePhase = INFRA_PHASES["KOTS_UPGRADE"];
-        const ret = callMakeTargets(upgradePhase.phase, upgradePhase.description, upgradePhase.makeTarget);
+        const ret = await callMakeTargets(upgradePhase.phase, upgradePhase.description, upgradePhase.makeTarget);
         if (ret) {
             sendFailureSlackAlert(
                 upgradePhase.description,
@@ -375,7 +379,7 @@ export async function installerTests(config: TestConfig) {
     if (skipTests === "true") {
         console.log("Skipping integration tests");
     } else {
-        runIntegrationTests();
+        await runIntegrationTests();
     }
 
     // if the preview flag is set to true, the script will print the result and exits
@@ -413,11 +417,11 @@ export async function installerTests(config: TestConfig) {
         werft.done("print-output");
     } else {
         // if we are not doing preview, we delete the infrastructure
-        cleanup();
+        await cleanup();
     }
 }
 
-function runIntegrationTests() {
+async function runIntegrationTests() {
     werft.phase(`run-${testSuite}-integration-tests`, `Run all ${testSuite} integration tests`);
 
     const componentTests = TestMap[testSuite.toLowerCase()]
@@ -431,7 +435,7 @@ function runIntegrationTests() {
     console.log(`Running ${testSuite} tests`)
     for (let test in componentTests) {
         const testPhase = componentTests[test];
-        const ret = callMakeTargets(testPhase.phase, testPhase.description, testPhase.makeTarget);
+        const ret = await callMakeTargets(testPhase.phase, testPhase.description, testPhase.makeTarget);
         if (ret) {
             exec(
                 `werft log result -d "failed test" url "${testPhase.description}(Phase ${testPhase.phase}) failed. Please refer logs."`,
@@ -449,33 +453,38 @@ function runIntegrationTests() {
     werft.done("run-integration-tests");
 }
 
-function callMakeTargets(phase: string, description: string, makeTarget: string, failable: boolean = false) {
-    werft.log(phase, `Calling ${makeTarget}`);
+/**
+ * Run a make target within a werft slice.
+ *
+ * @return The make target return code
+ */
+async function callMakeTargets(slice: string, description: string, makeTarget: string, failable: boolean = false): Promise<number> {
+    werft.log(slice, `Calling ${makeTarget}`);
     // exporting cloud env var is important for the make targets
     const env = `export TF_VAR_cluster_version=${k8s_version} cloud=${cloud} TF_VAR_domain=${baseDomain} TF_VAR_gcp_zone=${gcpDnsZone}`;
 
-    const response = exec(
+    const code = await execStream(
         `${env} && make -C ${makefilePath} ${makeTarget}`,
         {
-            slice: phase,
+            slice: slice,
             dontCheckRc: true,
         },
     );
 
-    if (response.code) {
-        console.error(`Error: ${response.stderr}`);
+    if (code !== 0) {
+        console.error(`Error: make target ${makeTarget} exited with code ${code}`);
 
         if (failable) {
-            werft.fail(phase, "Operation failed");
-            return response.code;
+            werft.fail(slice, "Operation failed");
+            return code;
         }
-        werft.log(phase, `'${description}' failed`);
+        werft.log(slice, `'${description}' failed`);
     } else {
-        werft.log(phase, `'${description}' succeeded`);
-        werft.done(phase);
+        werft.log(slice, `'${description}' succeeded`);
+        werft.done(slice);
     }
 
-    return response.code;
+    return code;
 }
 
 function randomize(options: string[]): string {
@@ -526,40 +535,43 @@ function randOsVersion(): string {
     return randomize(options);
 }
 
-function cleanup() {
+async function cleanup(): Promise<void> {
     const phase = INFRA_PHASES["DESTROY"];
     werft.phase(phase.phase, phase.description);
 
-    const ret = callMakeTargets(phase.phase, phase.description, phase.makeTarget);
-
-    // if the destroy command fail, we check if any resources are pending to be removed
-    // if nothing is yet to be cleaned, we return with success
-    // else we list the rest of the resources to be cleaned up
-    if (ret) {
-        const existingState = exec(`make -C ${makefilePath} list-state`, { slice: "get-uncleaned-resources" });
-
-        if (existingState.code) {
-            console.error(`Error: Failed to check for the left over resources`);
-        }
-
-        const itemsTobeCleaned = existingState.stdout.toString().split("\n").slice(1, -1);
+    try {
+        const ret = await callMakeTargets("cleanup", phase.description, phase.makeTarget);
 
-        if (itemsTobeCleaned.length == 0) {
-            console.log("Eventhough it was not a clean run, all resources has been cleaned. Nothing to do");
+        // if the destroy command fail, we check if any resources are pending to be removed
+        // if nothing is yet to be cleaned, we return with success
+        // else we list the rest of the resources to be cleaned up
+        if (ret === 0) {
             werft.done(phase.phase);
-            return;
-        }
+        } else {
+            const existingState = exec(`make -C ${makefilePath} list-state`, { slice: "get-uncleaned-resources" });
 
-        console.log(`Cleanup the following resources manually: ${itemsTobeCleaned}`);
+            if (existingState.code) {
+                console.error(`Error: Failed to check for the left over resources`);
+            }
 
-        sendFailureSlackAlert(phase.description, new Error("Cleanup job failed"), slackHook.get("self-hosted-jobs"));
-    }
+            const itemsTobeCleaned = existingState.stdout.toString().split("\n").slice(1, -1);
 
-    werft.done(phase.phase);
+            if (itemsTobeCleaned.length === 0) {
+                console.log("Eventhough it was not a clean run, all resources has been cleaned. Nothing to do");
+                werft.done(phase.phase);
+                return;
+            }
 
-    deleteReplicatedLicense(werft, process.env["TF_VAR_TEST_ID"]);
+            console.log(`Cleanup the following resources manually: ${itemsTobeCleaned}`);
 
-    return ret;
+            werft.failSlice(phase.phase, new Error("Failed to cleanup resources"));
+
+            await sendFailureSlackAlert(phase.description, new Error("Cleanup job failed"), slackHook.get("self-hosted-jobs"));
+        }
+    }
+    finally {
+        await deleteReplicatedLicense(werft, process.env["TF_VAR_TEST_ID"])
+    }
 }
 
 export function sendFailureSlackAlert(phase: string, err: Error, hook: string): Promise<void> {

.werft/jobs/build/installer/installer.ts

@@ -274,6 +274,7 @@ EOF`);
         exec(`yq w -i ${this.options.installerConfigPath} experimental.webapp.usage.billInstancesAfter "2022-08-11T08:05:32.499Z"`, { slice: slice })
         exec(`yq w -i ${this.options.installerConfigPath} experimental.webapp.usage.defaultSpendingLimit.forUsers 500`, { slice: slice })
         exec(`yq w -i ${this.options.installerConfigPath} experimental.webapp.usage.defaultSpendingLimit.forTeams 0`, { slice: slice })
+        exec(`yq w -i ${this.options.installerConfigPath} experimental.webapp.usage.defaultSpendingLimit.minForUsersOnStripe 1000`, { slice: slice })
         exec(`yq w -i ${this.options.installerConfigPath} experimental.webapp.usage.creditsPerMinuteByWorkspaceClass['default'] 0.1666666667`, { slice: slice })
         exec(`yq w -i ${this.options.installerConfigPath} experimental.webapp.usage.creditsPerMinuteByWorkspaceClass['gitpodio-internal-xl'] 0.3333333333`, { slice: slice })
     }

.werft/platform-trigger-werft-cleanup.sh

@@ -44,11 +44,14 @@ function cordon-node-if-almost-full {
         if [[ "${node}" =~ "builds-static" ]]; then
           echo "Cleaning up static node [${node}]"
           while ! is_node_empty "${node}";do
-            echo "Node is not empty yet. Sleeping for 15 seconds."
+            echo "Node is not empty yet. Sleeping for 15 seconds." | werft log slice "$slice_id"
             sleep 15
           done
 
-          gcloud compute instances delete "${node}" --zone="${zone}" -q
+          kubectl drain "${node}" --delete-emptydir-data --force --ignore-daemonsets=true --grace-period=120 | werft log slice "$slice_id"
+
+          gcloud compute instances delete "${node}" --zone="${zone}" -q | werft log slice "$slice_id"
+          kubectl uncordon "${node}" | werft log slice "$slice_id"
         fi
     else
         echo "${disk_used_pct} is less than the trehold of ${DISK_USED_THRESHOLD}. Skipping node" | werft log slice "$slice_id"

.werft/util/shell.ts

@@ -62,6 +62,75 @@ export function exec(cmd: string, options?: ExecOptions): ChildProcess | shell.S
     }
 }
 
+/**
+ * Execute a command and stream output logs.
+ *
+ * If a slice is given logs are streamed using the werft log syntax; else they're streamed directly
+ * to stderr/stdout.
+ *
+ * @return The process exit code
+ */
+export async function execStream(command: string, options: ExecOptions ): Promise<number> {
+    const werft = getGlobalWerftInstance();
+
+    options = options || {};
+
+    if (options.slice) {
+        options.silent = true;
+    }
+
+    const child = shell.exec(command, {...options, async: true});
+
+    // Collect output from a subprocess file and print newline terminated log messages.
+    //
+    // The event handlers attached to the child process stdout/stderr aren't guaranteed to be newline
+    // terminated; this can have odd interactions with `werft.logOutput` which appends newlines.
+    // to log messages. To ensure faithful reproduction of the underlying command output we
+    // perform our own buffer management to emit newline delimited logs accurately.
+    const bufferedLog = (slice: string, buffer: string, data: string) => {
+        buffer += data;
+        const lastIndex = buffer.lastIndexOf("\n");
+        if (lastIndex >= 0) {
+            // Extract the substring till the last newline in the buffer, and trim off the newline
+            // as werft.logOutput will append a newline to the log message.
+            let msg = buffer.slice(0, lastIndex + 1).trimEnd();
+            werft.logOutput(slice, msg);
+
+            buffer = buffer.slice(lastIndex + 1);
+        }
+
+    };
+
+    let stdoutBuffer: string = '';
+    child.stdout.on('data', (data) => {
+        if (options.slice) bufferedLog(options.slice, stdoutBuffer, data);
+    });
+
+    let stderrBuffer: string = '';
+    child.stderr.on('data', (data) => {
+        if (options.slice) bufferedLog(options.slice, stderrBuffer, data);
+    });
+
+    const code = await new Promise<number>((resolve, reject) => {
+        child.on('close', (code, _signal) => {
+            if (options.slice) {
+                // The child process stdout and stderr buffers may not be fully flushed as the child process
+                // may emit logs that aren't terminated with a newline; flush those buffers now.
+                if (stdoutBuffer.length > 0) werft.logOutput(options.slice, stdoutBuffer.trimEnd());
+                if (stderrBuffer.length > 0) werft.logOutput(options.slice, stderrBuffer.trimEnd());
+            }
+
+            if (code === 0 || options.dontCheckRc) {
+                resolve(code)
+            } else {
+                reject(new Error(`Process exited non-zero exit code ${code}`))
+            }
+        });
+    });
+
+    return code;
+}
+
 // gitTag tags the current state and pushes that tag to the repo origin
 export const gitTag = (tag) => {
     shell.mkdir("/root/.ssh");

CHANGELOG.md

@@ -1,6 +1,14 @@
 # Change Log
 
 ## October 2022
+- Fix an issue with a workspace not starting with a self-signed cluster ([#13821](https://github.com/gitpod-io/gitpod/pull/13821)) - [@utam0k](https://github.com/utam0k)
+- VSCode Browser: Fix portsView address open twice in some browsers ([#13844](https://github.com/gitpod-io/gitpod/pull/13844)) - [@mustard-mh](https://github.com/mustard-mh)
+- Fixed the auto-port-forwarding on JetBrains EAP IDEs ([#13747](https://github.com/gitpod-io/gitpod/pull/13747)) - [@felladrin](https://github.com/felladrin)
+- Support multi-line environment variables in SSH ([#13822](https://github.com/gitpod-io/gitpod/pull/13822)) - [@iQQBot](https://github.com/iQQBot)
+- VSCode Browser/Desktop: Responsive portsView ([#13838](https://github.com/gitpod-io/gitpod/pull/13838)) - [@mustard-mh](https://github.com/mustard-mh)
+- Update GoLand IDE image to version 222.4345.24. ([#13836](https://github.com/gitpod-io/gitpod/pull/13836)) - [@roboquat](https://github.com/roboquat)
+- [terraform/aks] Separate workloads into different pools, expose terraform variables for max node pool counts ([#13143](https://github.com/gitpod-io/gitpod/pull/13143)) - [@adrienthebo](https://github.com/adrienthebo)
+- Update PyCharm IDE image to version 222.4345.23. ([#13797](https://github.com/gitpod-io/gitpod/pull/13797)) - [@roboquat](https://github.com/roboquat)
 - Update PhpStorm IDE image to version 222.4345.15. ([#13759](https://github.com/gitpod-io/gitpod/pull/13759)) - [@roboquat](https://github.com/roboquat)
 - Update WebStorm IDE image to version 222.4345.14. ([#13757](https://github.com/gitpod-io/gitpod/pull/13757)) - [@roboquat](https://github.com/roboquat)
 - Update RubyMine IDE image to version 222.4345.14. ([#13758](https://github.com/gitpod-io/gitpod/pull/13758)) - [@roboquat](https://github.com/roboquat)

WORKSPACE.yaml

@@ -8,11 +8,11 @@ defaultArgs:
   jbMarketplacePublishTrigger: "false"
   publishToJBMarketplace: true
   localAppVersion: unknown
-  codeCommit: 4e566342849b494651383099088327dfeab91c86
+  codeCommit: 239fa626a44652cebf293a994a3330e90e873daa
   codeQuality: stable
   noVerifyJBPlugin: false
   intellijDownloadUrl: "https://download.jetbrains.com/idea/ideaIU-2022.2.3.tar.gz"
-  golandDownloadUrl: "https://download.jetbrains.com/go/goland-2022.2.3.tar.gz"
+  golandDownloadUrl: "https://download.jetbrains.com/go/goland-2022.2.4.tar.gz"
   pycharmDownloadUrl: "https://download.jetbrains.com/python/pycharm-professional-2022.2.3.tar.gz"
   phpstormDownloadUrl: "https://download.jetbrains.com/webide/PhpStorm-2022.2.3.tar.gz"
   rubymineDownloadUrl: "https://download.jetbrains.com/ruby/RubyMine-2022.2.3.tar.gz"

components/BUILD.yaml

@@ -98,7 +98,8 @@ packages:
       - components/local-app-api/typescript-grpcweb:publish
       - components/supervisor-api/typescript-grpc:publish
       - components/supervisor-api/typescript-grpcweb:publish
-      - components/ide/jetbrains/gateway-plugin:publish
+      - components/ide/jetbrains/gateway-plugin:publish-stable
+      - components/ide/jetbrains/gateway-plugin:publish-latest
       - components/public-api/typescript:publish
   - name: all-apps
     type: generic