Added initial implementation for transformer

transformers/from_scratch/src/sentence.py

@@ -0,0 +1,25 @@
+import re
+from typing import List, Optional
+
+
+class Sentence:
+    """Generate a list of tokens from a list of input sentences """
+    BOS = "BOS"  # Beginning of sentence
+    EOS = "EOS"  # End of Sentence
+    PAD = "PAD"
+
+    def __init__(self, sentence_list: Optional[List[str]] = None):
+        self.token_2_index = {self.BOS: 0, self.EOS: 1, self.PAD: 2}
+        self.index_to_token = {v: k for k, v in self.token_2_index.items()}
+        # Check whether there is a list of input sentences
+        if not sentence_list:
+            return  # then return to initial state
+
+        # Call token method if there is a list of sentences
+        # Tokenize and substitute BOS, EAS and PAD
+        for sentence in sentence_list:
+            self.add_tokens(self.tokenize(sentence))
+
+    def add_tokens(self, param):
+        pass
+

transformers/src/vocabulary.py

@@ -4,15 +4,15 @@
 
 
 class Vocabulary:
-    """Generates a list of tokens from imput string """
+    """Generates a list of tokens from input string """
     BOS = "BOS"
     EOS = "EOS"
     PAD = "PAD"
 
     def __init__(self, list_of_sentences: Optional[List[str]]) -> None:
         self.token2index = {self.BOS: 0, self.EOS: 1, self.PAD: 2}
         self.index2token = {v: k for k, v in self.token2index.items()}
-        # check whethere there is a list of sentences?
+        # check whether there is a list of sentences?
         if not list_of_sentences:
             # then return
             return