Solving conflicts, merging v2 into master.

README.md

@@ -49,13 +49,13 @@ python nspm/data_gen.py --input data/art_30 --output data/art_30
 
 ### The Learner module
 
-Now go back to the initial directory and launch `learner.py` to train the model. Currently the epochs and batch_size is not parametrized for that you can change the epoch is train.py and batch size in data_gen.py (recommended batch size for large 64, medium 32 and small like art_30 is 16) also epochs varies with batch size for art 30 its 40.
+Now go back to the initial directory and launch `learner.py` to train the model. 
 
 ```bash
 python nspm/learner.py --input data/art_30 --output data/art_30
 ```
 
-This command will create a model checkpoints in `data/art_30`.
+This command will create a model checkpoints in `data/art_30` and some pickle files in `data/art_30/pickle_objects`.
 
 ### The Interpreter module
 
@@ -64,6 +64,13 @@ Predict the SPARQL query for a given question it will store the detailed output
 ```bash
 python nspm/interpreter.py --input data/art_30 --output data/art_30 --query "yuncken freeman has architected in how many cities?"
 ```
+or, if you want to use NSpM with [airml](https://github.com/sahandilshan/airML) to install pre-trained models, follow these steps,
+1. Install airML latest version from [here](https://pypi.org/project/airML/)
+2. Navigate to the table.kns [here](https://github.com/sahandilshan/KBox/blob/dev/kns/2.0/table.kns) and check if your model is listed in that file.
+3. Then copy the name of that model and use it with the `interpreter.py` as follows
+```bash
+python interpreter.py --airml http://nspm.org/art --output data/art_30 --inputstr "yuncken freeman has architected in how many cities?"
+```
 
 ## Use cases & integrations
 

nspm/interpreter.py

@@ -24,6 +24,9 @@
 from prepare_dataset import preprocess_sentence
 from generator_utils import decode, fix_URI
 
+from airML import airML
+import json
+
 
 def evaluate(sentence, config, neural_mt):
 
@@ -153,16 +156,50 @@ def interpret(input_dir, query):
   outputfile.close()
 
 
+def install_model(url):
+  output = airML.install(url, format='nspm')
+  output = json.loads(output)
+  if output['status_code'] == 200:
+      print(output['message'])
+  else:
+      raise Exception(output['message'])
+
+
+def locate_model(url):
+  install_model(url)
+  output = airML.locate(url, format='nspm')
+  output = json.loads(output)
+  if output['status_code'] == 200:
+      print(output['message'])
+      model_dir = output['results'][0]
+      return model_dir
+  else:
+      raise Exception(output['message'])
+
+
 if __name__ == '__main__':
   parser = argparse.ArgumentParser()
   requiredNamed = parser.add_argument_group('required named arguments')
   requiredNamed.add_argument(
-      '--input', dest='input', metavar='inputDirectory', help='dataset directory', required=True)
+        '--input', dest='input', metavar='inputDirectory', help='dataset directory', required=False)
   requiredNamed.add_argument(
-          '--query', dest='query', metavar='query', help='Input query in natural language', required=True)
+        '--airml', dest='airml', metavar='airmlURL', help='name of the knowledge base', required=False)
+  requiredNamed.add_argument(
+        '--query', dest='query', metavar='query', help='Input query in natural language', required=True)
+
 
   args = parser.parse_args()
-  input_dir = args.input
+
+  if args.input is not None:
+    model_dir = args.input
+    input_dir = args.input
+  elif args.airml is not None:
+    airml_url = args.airml
+    model_dir = locate_model(airml_url)
+    input_dir = model_dir
+  else:
+    print('--input or --airml argument should be provided to load the model.')
+
   query = args.query
 
   interpret(input_dir, query)

nspm/prepare_dataset.py

@@ -76,3 +76,23 @@ def convert(lang, tensor):
   for t in tensor:
     if t!=0:
       print ("%d ----> %s" % (t, lang.index_word[t]))
+
+
+def merging_datafile(input_dir,output_dir):
+    input_diren=input_dir+'/data.en'
+    input_dirspq=input_dir+'/data.sparql'
+    output_dir+='/data.txt'
+    file1 = open(input_diren,'r',encoding="utf8")
+    Lines1 = file1.readlines()
+    file2 = open(input_dirspq,'r',encoding="utf8")
+    Lines2 = file2.readlines()
+    s=[]
+    for i in range(len(Lines1)):
+        s.append(Lines1[i].replace('\n'," ")+"\t "+Lines2[i])
+
+    filef = open(output_dir,'w',encoding="utf8")
+    filef.writelines(s)
+    file1.close()
+    file2.close()
+    filef.close()
+    return output_dir

requirements.txt

@@ -23,3 +23,4 @@ tensorflow==2.4.0
 termcolor==1.1.0
 tqdm==4.56.0
 Werkzeug==1.0.1
+airML==0.0.3

test/interpreter_airml_test.py

@@ -0,0 +1,34 @@
+import subprocess
+import unittest
+
+
+class TestAirML(unittest.TestCase):
+    def test_airml_input_args_with_valid_kn(self):
+        process = subprocess.Popen(
+            ['python3', 'interpreter.py', "--airml", "http://nspm.org/art", "--output", "test", "--inputstr",
+             '"yuncken freeman has architected in how many cities?"'], stdout=subprocess.PIPE)
+        output, err = process.communicate()
+        output = output.decode("utf-8")
+        self.assertTrue("http://nspm.org/art KB installed." in output)
+        self.assertTrue("Predicted translation:" in output)
+
+    def test_airml_input_args_with_invalid_kn(self):
+        process = subprocess.Popen(
+            ['python3', 'interpreter.py', "--airml", "http://nspm.org/arts", "--output", "test", "--inputstr",
+             '"yuncken freeman has architected in how many cities?"'], stdout=subprocess.PIPE)
+        output, err = process.communicate()
+        output = output.decode("utf-8")
+        self.assertTrue("Predicted translation:" not in output)
+
+    def test_airml_without_input_arg(self):
+        process = subprocess.Popen(
+            ['python3', 'interpreter.py', "--output", "test", "--inputstr",
+             '"yuncken freeman has architected in how many cities?"'], stdout=subprocess.PIPE)
+        output, err = process.communicate()
+        output = output.decode("utf-8")
+        self.assertTrue("--input or --airml argument should be provided to load the model." in output)
+        self.assertTrue("Predicted translation:" not in output)
+
+
+if __name__ == '__main__':
+    unittest.main()