Merge pull request #1783 from RedisInsight/be/feature/RI-4231-migrate_tutorials_json_to_arrays

#RI-4231 BE move to array-based structure

Author: arthosofteq

redisinsight/api/config/default.ts

@@ -110,14 +110,14 @@ export default {
   },
   guides: {
     updateUrl: process.env.GUIDES_UPDATE_URL
-      || 'https://github.com/RedisInsight/Guides/releases/download/release',
+      || 'https://github.com/RedisInsight/Guides/releases/download/2.x.x',
     zip: process.env.GUIDES_ZIP || dataZipFileName,
     buildInfo: process.env.GUIDES_CHECKSUM || buildInfoFileName,
     devMode: !!process.env.GUIDES_DEV_PATH,
   },
   tutorials: {
     updateUrl: process.env.TUTORIALS_UPDATE_URL
-      || 'https://github.com/RedisInsight/Tutorials/releases/download/release',
+      || 'https://github.com/RedisInsight/Tutorials/releases/download/2.x.x',
     zip: process.env.TUTORIALS_ZIP || dataZipFileName,
     buildInfo: process.env.TUTORIALS_CHECKSUM || buildInfoFileName,
     devMode: !!process.env.TUTORIALS_DEV_PATH,

redisinsight/api/package.json

@@ -21,7 +21,7 @@
     "format": "prettier --write \"src/**/*.ts\"",
     "lint": "eslint --ext .ts .",
     "start": "nest start",
-    "start:dev": "cross-env NODE_ENV=development SERVER_STATIC_CONTENT=1 nest start --watch",
+    "start:dev": "cross-env NODE_ENV=development SERVER_STATIC_CONTENT=1 nest start --watch --preserveWatchOutput",
     "start:debug": "nest start --debug --watch",
     "start:stage": "cross-env NODE_ENV=staging SERVER_STATIC_CONTENT=true node dist/src/main",
     "start:prod": "cross-env NODE_ENV=production node dist/src/main",

redisinsight/api/src/modules/custom-tutorial/custom-tutorial.controller.ts

@@ -17,6 +17,7 @@ import { CreateClientCertificateDto } from 'src/modules/certificate/dto/create.c
 import { UseClientCertificateDto } from 'src/modules/certificate/dto/use.client-certificate.dto';
 import { CreateBasicSshOptionsDto } from 'src/modules/ssh/dto/create.basic-ssh-options.dto';
 import { CreateCertSshOptionsDto } from 'src/modules/ssh/dto/create.cert-ssh-options.dto';
+import { RootCustomTutorialManifest } from 'src/modules/custom-tutorial/models/custom-tutorial.manifest';
 
 @ApiExtraModels(
   CreateCaCertificateDto, UseCaCertificateDto,
@@ -45,7 +46,7 @@ export class CustomTutorialController {
   })
   async create(
     @Body() dto: UploadCustomTutorialDto,
-  ): Promise<Record<string, any>> {
+  ): Promise<RootCustomTutorialManifest> {
     return this.service.create(dto);
   }
 
@@ -59,7 +60,7 @@ export class CustomTutorialController {
       },
     ],
   })
-  async getGlobalManifest(): Promise<any> {
+  async getGlobalManifest(): Promise<RootCustomTutorialManifest[]> {
     return this.service.getGlobalManifest();
   }
 

redisinsight/api/src/modules/custom-tutorial/custom-tutorial.service.ts

@@ -1,4 +1,5 @@
 import {
+  BadRequestException,
   Injectable, Logger, NotFoundException,
 } from '@nestjs/common';
 import { v4 as uuidv4 } from 'uuid';
@@ -13,7 +14,7 @@ import {
 } from 'src/modules/custom-tutorial/providers/custom-tutorial.manifest.provider';
 import {
   CustomTutorialManifestType,
-  ICustomTutorialManifest,
+  RootCustomTutorialManifest,
 } from 'src/modules/custom-tutorial/models/custom-tutorial.manifest';
 import { wrapHttpError } from 'src/common/utils';
 
@@ -32,9 +33,17 @@ export class CustomTutorialService {
    * Currently from zip file only
    * @param dto
    */
-  public async create(dto: UploadCustomTutorialDto): Promise<Record<string, any>> {
+  public async create(dto: UploadCustomTutorialDto): Promise<RootCustomTutorialManifest> {
     try {
-      const tmpPath = await this.customTutorialFsProvider.unzipToTmpFolder(dto.file);
+      let tmpPath = '';
+
+      if (dto.file) {
+        tmpPath = await this.customTutorialFsProvider.unzipFromMemoryStoredFile(dto.file);
+      } else if (dto.link) {
+        tmpPath = await this.customTutorialFsProvider.unzipFromExternalLink(dto.link);
+      } else {
+        throw new BadRequestException('File or external link should be provided');
+      }
 
       // todo: validate
 
@@ -59,8 +68,8 @@ export class CustomTutorialService {
    * Get global manifest for all custom tutorials
    * In the future will be removed with some kind of partial load
    */
-  public async getGlobalManifest(): Promise<Record<string, ICustomTutorialManifest>> {
-    const children = {};
+  public async getGlobalManifest(): Promise<RootCustomTutorialManifest[]> {
+    const children = [];
 
     try {
       const tutorials = await this.customTutorialRepository.list();
@@ -73,26 +82,26 @@ export class CustomTutorialService {
 
       manifests.forEach((manifest) => {
         if (manifest) {
-          children[manifest.id] = manifest;
+          children.push(manifest);
         }
       });
     } catch (e) {
       this.logger.warn('Unable to generate entire custom tutorials manifest', e);
     }
 
-    return {
-      'custom-tutorials': {
+    return [
+      {
         type: CustomTutorialManifestType.Group,
         id: 'custom-tutorials',
-        label: 'My Tutorials',
+        label: 'MY TUTORIALS',
         _actions: [CustomTutorialActions.CREATE],
         args: {
           withBorder: true,
           initialIsOpen: true,
         },
         children,
       },
-    };
+    ];
   }
 
   public async get(id: string): Promise<CustomTutorial> {

redisinsight/api/src/modules/custom-tutorial/dto/upload.custom-tutorial.dto.ts

@@ -1,20 +1,30 @@
-import { ApiProperty } from '@nestjs/swagger';
+import { ApiProperty, ApiPropertyOptional } from '@nestjs/swagger';
 import { Expose } from 'class-transformer';
-import { IsNotEmpty, IsString } from 'class-validator';
+import { IsNotEmpty, IsOptional, IsString } from 'class-validator';
 import {
   HasMimeType, IsFile, MaxFileSize, MemoryStoredFile,
 } from 'nestjs-form-data';
 
 export class UploadCustomTutorialDto {
-  @ApiProperty({
+  @ApiPropertyOptional({
     type: 'string',
     format: 'binary',
     description: 'ZIP archive with tutorial static files',
   })
+  @IsOptional()
   @IsFile()
   @HasMimeType(['application/zip'])
   @MaxFileSize(10 * 1024 * 1024)
-  file: MemoryStoredFile;
+  file?: MemoryStoredFile;
+
+  @ApiPropertyOptional({
+    type: 'string',
+    description: 'External link to zip archive',
+  })
+  @IsOptional()
+  @IsString()
+  @IsNotEmpty()
+  link?: string;
 
   @ApiProperty({
     description: 'Name to show for custom tutorials',

redisinsight/api/src/modules/custom-tutorial/models/custom-tutorial.manifest.ts

@@ -1,7 +1,9 @@
 import { CustomTutorialActions } from 'src/modules/custom-tutorial/models/custom-tutorial';
-import { ApiProperty } from '@nestjs/swagger';
-import { Expose } from 'class-transformer';
-import { IsEnum, IsNotEmpty } from 'class-validator';
+import { ApiProperty, ApiPropertyOptional } from '@nestjs/swagger';
+import { Expose, Type } from 'class-transformer';
+import {
+  IsArray, IsBoolean, IsEnum, IsNotEmpty, IsOptional, IsString, ValidateNested
+} from 'class-validator';
 
 export enum CustomTutorialManifestType {
   CodeButton = 'code-button',
@@ -19,6 +21,27 @@ export interface ICustomTutorialManifest {
   _path?: string,
 }
 
+export class CustomTutorialManifestArgs {
+  @ApiPropertyOptional({ type: Boolean })
+  @IsOptional()
+  @Expose()
+  @IsString()
+  @IsNotEmpty()
+  path?: string;
+
+  @ApiPropertyOptional({ type: Boolean })
+  @IsOptional()
+  @Expose()
+  @IsBoolean()
+  initialIsOpen?: boolean;
+
+  @ApiPropertyOptional({ type: Boolean })
+  @IsOptional()
+  @Expose()
+  @IsBoolean()
+  withBorder?: boolean;
+}
+
 export class CustomTutorialManifest {
   @ApiProperty({ type: String })
   @Expose()
@@ -35,5 +58,34 @@ export class CustomTutorialManifest {
   @IsNotEmpty()
   label: string;
 
-  children: Record<string, CustomTutorialManifest>
+  @ApiPropertyOptional({ type: CustomTutorialManifestArgs })
+  @IsOptional()
+  @Expose()
+  @ValidateNested()
+  @Type(() => CustomTutorialManifestArgs)
+  args?: CustomTutorialManifestArgs;
+
+  @ApiPropertyOptional({ type: CustomTutorialManifest })
+  @IsOptional()
+  @Expose()
+  @ValidateNested({ each: true })
+  @IsArray()
+  @Type(() => CustomTutorialManifest)
+  children?: CustomTutorialManifest[];
+}
+
+export class RootCustomTutorialManifest extends CustomTutorialManifest {
+  @ApiPropertyOptional({ enum: CustomTutorialActions })
+  @IsOptional()
+  @Expose()
+  @IsArray()
+  @IsEnum(CustomTutorialActions, { each: true })
+  _actions?: CustomTutorialActions[];
+
+  @ApiPropertyOptional({ type: String })
+  @IsOptional()
+  @Expose()
+  @IsString()
+  @IsNotEmpty()
+  _path?: string;
 }

redisinsight/api/src/modules/custom-tutorial/models/custom-tutorial.ts

@@ -7,6 +7,7 @@ const PATH_CONFIG = config.get('dir_path');
 export enum CustomTutorialActions {
   CREATE = 'create',
   DELETE = 'delete',
+  SYNC = 'sync',
 }
 
 export class CustomTutorial {
@@ -26,7 +27,13 @@ export class CustomTutorial {
   createdAt: Date;
 
   get actions(): CustomTutorialActions[] {
-    return [CustomTutorialActions.DELETE];
+    const actions = [CustomTutorialActions.DELETE];
+
+    if (this.link) {
+      actions.push(CustomTutorialActions.SYNC);
+    }
+
+    return actions;
   }
 
   get path(): string {

redisinsight/api/src/modules/custom-tutorial/providers/custom-tutorial.fs.provider.ts

@@ -5,6 +5,8 @@ import { v4 as uuidv4 } from 'uuid';
 import * as fs from 'fs-extra';
 import config from 'src/utils/config';
 import * as AdmZip from 'adm-zip';
+import axios from 'axios';
+import { wrapHttpError } from 'src/common/utils';
 
 const PATH_CONFIG = config.get('dir_path');
 
@@ -15,13 +17,13 @@ export class CustomTutorialFsProvider {
   private logger = new Logger('CustomTutorialFsProvider');
 
   /**
-   * Unzip custom tutorials to temporary folder
-   * @param file
+   * Unzip custom tutorials archive to temporary folder
+   * @param zip
    */
-  public async unzipToTmpFolder(file: MemoryStoredFile): Promise<string> {
+  public async unzipToTmpFolder(zip: AdmZip): Promise<string> {
     try {
       const path = await CustomTutorialFsProvider.prepareTmpFolder();
-      const zip = new AdmZip(file.buffer);
+
       await fs.remove(path);
       await zip.extractAllTo(path, true);
 
@@ -32,6 +34,31 @@ export class CustomTutorialFsProvider {
     }
   }
 
+  /**
+   * Unzip archive from multipart/form-data file input
+   * @param file
+   */
+  public async unzipFromMemoryStoredFile(file: MemoryStoredFile): Promise<string> {
+    return this.unzipToTmpFolder(new AdmZip(file.buffer));
+  }
+
+  /**
+   * Download zip archive from external source and unzip it to temporary directory
+   * @param link
+   */
+  public async unzipFromExternalLink(link: string): Promise<string> {
+    try {
+      const { data } = await axios.get(link, {
+        responseType: 'arraybuffer',
+      });
+
+      return this.unzipToTmpFolder(new AdmZip(data));
+    } catch (e) {
+      this.logger.error('Unable fetch zip file from external source', e);
+      throw wrapHttpError(e);
+    }
+  }
+
   /**
    * Move custom tutorial from tmp folder to proper path to serve static files
    * force - default false, will remove existing folder