feat: implement cpp tokenization

Author: Mikadows

src/code-quality/tokenizer/CodeTokenizer.ts

@@ -2,20 +2,23 @@ import { CodeTokens } from './CodeTokens';
 import { PythonIdentifiers } from './identifiers/PythonIdentifiers';
 import { CommonLangIdentifiers } from './identifiers/CommonLangIdentifiers';
 import { GolangIdentifiers } from './identifiers/GolangIdentifiers';
+import { CppIdentifiers } from './identifiers/CppIdentifiers';
 
 export class CodeTokenizer {
   private pyIdentifiers = new PythonIdentifiers();
 
   private golangIdentifiers = new GolangIdentifiers();
 
+  private cppIdentifiers = new CppIdentifiers();
+
   public tokenize(code: string, language: string): CodeTokens[] {
     switch (language) {
       case 'python':
         return this.tokenizeCode(code, this.pyIdentifiers);
       case 'go':
         return this.tokenizeCode(code, this.golangIdentifiers);
       case 'cpp':
-        return [];
+        return this.tokenizeCode(code, this.cppIdentifiers);
       default:
         return [];
     }
@@ -32,10 +35,7 @@ export class CodeTokenizer {
       const trimmedLine = line.trim();
 
       if (trimmedLine !== '' && trimmedLine !== '}')
-        if (
-          trimmedLine.startsWith(languageIdentifier.functionIdentifier()) &&
-          trimmedLine.endsWith(languageIdentifier.endLoopAndCondIdentifier())
-        ) {
+        if (languageIdentifier.isFunction(trimmedLine)) {
           // Function detection
           tokenizedCode.push(CodeTokens.FUNC);
           const numberOfParams =

src/code-quality/tokenizer/identifiers/CommonLangIdentifiers.ts

@@ -30,4 +30,11 @@ export class CommonLangIdentifiers {
   endLoopAndCondIdentifier(): string {
     return '{';
   }
+
+  isFunction(line: string): boolean {
+    return (
+      line.startsWith(this.functionIdentifier()) &&
+      line.endsWith(this.endLoopAndCondIdentifier())
+    );
+  }
 }

src/code-quality/tokenizer/identifiers/CppIdentifiers.ts

@@ -0,0 +1,14 @@
+import { CommonLangIdentifiers } from './CommonLangIdentifiers';
+
+export class CppIdentifiers extends CommonLangIdentifiers {
+  isFunction(line: string): boolean {
+    return (
+      (line.startsWith('void') ||
+        line.startsWith('char') ||
+        line.startsWith('int') ||
+        line.startsWith('float') ||
+        line.startsWith('double')) &&
+      line.endsWith('{')
+    );
+  }
+}