ordered option + stringify all numbers

Author: bmccann

evaluate.py

@@ -12,6 +12,7 @@
     parser.add_argument('source_file', help='source file for the prediction')
     parser.add_argument('db_file', help='source database for the prediction')
     parser.add_argument('pred_file', help='predictions by the model')
+    parser.add_argument('--ordered', action='store_true', help='whether the exact match should consider the order of conditions')
     args = parser.parse_args()
 
     engine = DBEngine(args.db_file)
@@ -21,13 +22,13 @@
         for ls, lp in tqdm(zip(fs, fp), total=count_lines(args.source_file)):
             eg = json.loads(ls)
             ep = json.loads(lp)
-            qg = Query.from_dict(eg['sql'])
+            qg = Query.from_dict(eg['sql'], ordered=args.ordered)
             gold = engine.execute_query(eg['table_id'], qg, lower=True)
             pred = ep.get('error', None)
             qp = None
             if not ep.get('error', None):
                 try:
-                    qp = Query.from_dict(ep['query'])
+                    qp = Query.from_dict(ep['query'], ordered=args.ordered)
                     pred = engine.execute_query(eg['table_id'], qp, lower=True)
                 except Exception as e:
                     pred = repr(e)

lib/query.py

@@ -13,15 +13,20 @@ class Query:
     cond_ops = ['=', '>', '<', 'OP']
     syms = ['SELECT', 'WHERE', 'AND', 'COL', 'TABLE', 'CAPTION', 'PAGE', 'SECTION', 'OP', 'COND', 'QUESTION', 'AGG', 'AGGOPS', 'CONDOPS']
 
-    def __init__(self, sel_index, agg_index, conditions=tuple()):
+    def __init__(self, sel_index, agg_index, conditions=tuple(), ordered=False):
         self.sel_index = sel_index
         self.agg_index = agg_index
         self.conditions = list(conditions)
+        self.ordered = ordered
 
     def __eq__(self, other):
         if isinstance(other, self.__class__):
             indices = self.sel_index == other.sel_index and self.agg_index == other.agg_index
-            conds = [(col, op, cond.lower() if isinstance(cond, str) else cond) for col, op, cond in self.conditions] == [(col, op, cond.lower() if isinstance(cond, str) else cond) for col, op, cond in other.conditions]
+            if other.ordered:
+                conds = [(col, op, str(cond).lower()) for col, op, cond in self.conditions] == [(col, op, str(cond).lower()) for col, op, cond in other.conditions]
+            else:
+                conds = set([(col, op, str(cond).lower()) for col, op, cond in self.conditions]) == set([(col, op, str(cond).lower()) for col, op, cond in other.conditions])
+
             return indices and conds
         return NotImplemented
 
@@ -52,8 +57,8 @@ def lower(self):
         return self.__class__(self.sel_index, self.agg_index, conds)
 
     @classmethod
-    def from_dict(cls, d):
-        return cls(sel_index=d['sel'], agg_index=d['agg'], conditions=d['conds'])
+    def from_dict(cls, d, ordered=False):
+        return cls(sel_index=d['sel'], agg_index=d['agg'], conditions=d['conds'], ordered=ordered)
 
     @classmethod
     def from_tokenized_dict(cls, d):