feat: code tokenization

Author: Mikadows

src/code-quality/tokenizer/CodeTokenizer.ts

@@ -0,0 +1,115 @@
+import { CodeTokens } from './CodeTokens';
+import { PythonIdentifiers } from './identifiers/PythonIdentifiers';
+import { CommonLangIdentifiers } from './identifiers/CommonLangIdentifiers';
+
+export class CodeTokenizer {
+  private pyIdentifiers = new PythonIdentifiers();
+
+  public tokenizeCode(code: string, language: string): CodeTokens[] {
+    switch (language) {
+      case 'python':
+        return this.tokenizePython(code, this.pyIdentifiers);
+      default:
+        return [];
+    }
+  }
+
+  private tokenizePython(
+    code: string,
+    languageIdentifier: CommonLangIdentifiers,
+  ): CodeTokens[] {
+    const lines = code.split('\n');
+    let tokenizedCode: CodeTokens[] = [];
+
+    lines.forEach((line) => {
+      const trimedLine = line.trim();
+
+      // Function detection
+      if (
+        trimedLine.startsWith(languageIdentifier.functionIdentifier()) &&
+        trimedLine.endsWith(languageIdentifier.endLoopAndCondIdentifier())
+      ) {
+        tokenizedCode.push(CodeTokens.FUNC);
+        const numberOfParams =
+          CodeTokenizer.numberOfFunctionParameters(trimedLine);
+        tokenizedCode = CodeTokenizer.pushTokensInTab(
+          tokenizedCode,
+          CodeTokens.FUNCPARAM,
+          numberOfParams,
+        );
+
+        // For line
+      } else if (
+        trimedLine.startsWith(languageIdentifier.forIdentifier()) &&
+        trimedLine.endsWith(languageIdentifier.endLoopAndCondIdentifier())
+      ) {
+        tokenizedCode.push(CodeTokens.FOR);
+        // While line
+      } else if (
+        trimedLine.startsWith(languageIdentifier.whileIdentifier()) &&
+        trimedLine.endsWith(languageIdentifier.endLoopAndCondIdentifier())
+      ) {
+        tokenizedCode.push(CodeTokens.WHILE);
+        // If line
+      } else if (
+        trimedLine.startsWith(languageIdentifier.ifIdentifier()) &&
+        trimedLine.endsWith(languageIdentifier.endLoopAndCondIdentifier())
+      ) {
+        tokenizedCode.push(CodeTokens.IF);
+        const andOccurrences = CodeTokenizer.numberTokenPresentInLine(
+          languageIdentifier.andIdentifier(),
+          trimedLine,
+        );
+        tokenizedCode = CodeTokenizer.pushTokensInTab(
+          tokenizedCode,
+          CodeTokens.AND,
+          andOccurrences,
+        );
+
+        const orOccurrences = CodeTokenizer.numberTokenPresentInLine(
+          languageIdentifier.orIdentifier(),
+          trimedLine,
+        );
+        tokenizedCode = CodeTokenizer.pushTokensInTab(
+          tokenizedCode,
+          CodeTokens.OR,
+          orOccurrences,
+        );
+
+        // Case line
+      } else if (
+        trimedLine.startsWith(languageIdentifier.caseIdentifier()) &&
+        trimedLine.endsWith(languageIdentifier.endLoopAndCondIdentifier())
+      ) {
+        tokenizedCode.push(CodeTokens.CASE);
+        // Other lines
+      } else {
+        tokenizedCode.push(CodeTokens.LINE);
+      }
+    });
+
+    console.log(tokenizedCode);
+
+    return tokenizedCode;
+  }
+
+  private static numberOfFunctionParameters(line: string): number {
+    const attributes = line.substring(line.indexOf('('), line.indexOf(')'));
+    return this.numberTokenPresentInLine(',', attributes) + 1;
+  }
+
+  private static numberTokenPresentInLine(token: string, line: string): number {
+    return line.split(token).length - 1;
+  }
+
+  private static pushTokensInTab(
+    tab: CodeTokens[],
+    token: CodeTokens,
+    occurrences: number,
+  ): CodeTokens[] {
+    for (let i = 0; i < occurrences; i += 1) {
+      tab.push(token);
+    }
+    return tab;
+  }
+}

src/code-quality/tokenizer/CodeTokens.ts

@@ -0,0 +1,15 @@
+export enum CodeTokens {
+  // functions
+  FUNC = 'FUNCTION',
+  FUNCPARAM = 'FUNCPARAM',
+  // loops
+  FOR = 'FOR',
+  WHILE = 'WHILE',
+  // conditions
+  IF = 'IF',
+  CASE = 'CASE',
+  OR = 'OR',
+  AND = 'AND',
+  // common line
+  LINE = 'LINE',
+}

src/code-quality/tokenizer/identifiers/CommonLangIdentifiers.ts

@@ -0,0 +1,33 @@
+export class CommonLangIdentifiers {
+  functionIdentifier(): string {
+    return '';
+  }
+
+  forIdentifier(): string {
+    return 'for';
+  }
+
+  whileIdentifier(): string {
+    return 'while';
+  }
+
+  ifIdentifier(): string {
+    return 'if';
+  }
+
+  caseIdentifier(): string {
+    return 'case';
+  }
+
+  orIdentifier(): string {
+    return '||';
+  }
+
+  andIdentifier(): string {
+    return '&&';
+  }
+
+  endLoopAndCondIdentifier(): string {
+    return '{';
+  }
+}

src/code-quality/tokenizer/identifiers/PythonIdentifiers.ts

@@ -0,0 +1,15 @@
+import { CommonLangIdentifiers } from './CommonLangIdentifiers';
+
+export class PythonIdentifiers extends CommonLangIdentifiers {
+  functionIdentifier(): string {
+    return 'def';
+  }
+
+  caseIdentifier(): string {
+    return '';
+  }
+
+  endLoopAndCondIdentifier(): string {
+    return ':';
+  }
+}

src/submissions/submissions.service.ts

@@ -18,6 +18,7 @@ import { FindSubmissionDTO } from './dto/find-submission.dto';
 import { InsertSubmissionDTO } from './dto/insert-submission-dto';
 import { JobStatusDTO } from './dto/job-status.dto';
 import { Submission } from './submission.entity';
+import { CodeTokenizer } from '../code-quality/tokenizer/CodeTokenizer';
 
 @Injectable()
 export class SubmissionsService {
@@ -31,6 +32,8 @@ export class SubmissionsService {
 
   private hashService = new HashService();
 
+  private codeTokenizer = new CodeTokenizer();
+
   async create(
     insertSubmissionDTO: InsertSubmissionDTO,
     lintScore: number,
@@ -57,6 +60,7 @@ export class SubmissionsService {
       insertSubmissionDTO.language,
     );
     // submission.self = submission;
+    this.codeTokenizer.tokenizeCode(submission.code, submission.language);
 
     return submission.save();
   }