Fixed the single image inference issue

Author: digbose92

Visual_Attention/bert_features_extract.py

@@ -29,7 +29,7 @@ def extract_bert_features(json_file,dataroot_folder,choice="yes_no",split="train
     h5f.close()
 
 if __name__ == "__main__":
-    json_file="/proj/digbose92/VQA/VisualQuestion_VQA/Visual_All/data/v2_OpenEnded_mscoco_train2014_1000_questions.json"
+    json_file="/proj/digbose92/VQA/VisualQuestion_VQA/Visual_All/data/v2_OpenEnded_mscoco_train2014_yes_no_questions.json"
     dataroot_folder="/data/digbose92/VQA/COCO/train_hdf5_COCO"
     extract_bert_features(json_file,dataroot_folder)
 

Visual_Attention/fusion_models.py

@@ -1,7 +1,6 @@
 import torch
 import torch.nn as nn
 import torch.nn.functional as F
-
 class mfh_baseline(nn.Module):
     def __init__(self,QUEST_EMBED,VIS_EMBED,MFB_FACTOR_NUM=5,MFB_OUT_DIM=1000,MFB_DROPOUT_RATIO=0.1,NUM_OUTPUT_UNITS=2):
         super(mfh_baseline, self).__init__()
@@ -15,30 +14,42 @@ def __init__(self,QUEST_EMBED,VIS_EMBED,MFB_FACTOR_NUM=5,MFB_OUT_DIM=1000,MFB_DR
         self.Linear_imgproj2 = nn.Linear(VIS_EMBED, self.JOINT_EMB_SIZE)
         #self.Linear_predict = nn.Linear(MFB_OUT_DIM * 2, NUM_OUTPUT_UNITS)
         #self.Dropout1 = nn.Dropout(p=opt.LSTM_DROPOUT_RATIO)
-        self.Dropout2 = nn.Dropout(MFB_DROPOUT_RATIO)
+        #self.Dropout2 = nn.Dropout(MFB_DROPOUT_RATIO)
 
     def forward(self, q_feat, img_feat):
 
         mfb_q_o2_proj = self.Linear_dataproj1(q_feat)                       # data_out (N, 5000)
         mfb_i_o2_proj = self.Linear_imgproj1(img_feat.float())              # img_feature (N, 5000)
         mfb_iq_o2_eltwise = torch.mul(mfb_q_o2_proj, mfb_i_o2_proj)
-        mfb_iq_o2_drop = self.Dropout2(mfb_iq_o2_eltwise)
-        mfb_iq_o2_resh = mfb_iq_o2_drop.view(-1, 1, self.MFB_OUT_DIM, self.MFB_FACTOR_NUM)  # N x 1 x 1000 x 5
-        mfb_o2_out = torch.squeeze(torch.sum(mfb_iq_o2_resh, 3))                            # N x 1000
-        mfb_o2_out = torch.sqrt(F.relu(mfb_o2_out)) - torch.sqrt(F.relu(-mfb_o2_out))       # signed sqrt
+        mfb_iq_o2_drop = mfb_iq_o2_eltwise
+        #mfb_iq_o2_drop = self.Dropout2(mfb_iq_o2_eltwise)
+        mfb_iq_o2_resh = mfb_iq_o2_drop.view(-1, 1, self.MFB_OUT_DIM, self.MFB_FACTOR_NUM)
+        if(mfb_iq_o2_resh.size(0)>1):                                                                                             # N x 1 x 1000 x 5
+            mfb_o2_out = torch.squeeze(torch.sum(mfb_iq_o2_resh, 3)) 
+        else:
+            mfb_o2_out = torch.sum(mfb_iq_o2_resh, 3).view(1,mfb_iq_o2_resh.size(2))                    # N x 1000
+        mfb_o2_out = torch.sqrt(F.relu(mfb_o2_out)) - torch.sqrt(F.relu(-mfb_o2_out))
+        #print(mfb_o2_out.size())       # signed sqrt
         mfb_o2_out = F.normalize(mfb_o2_out)
+        
 
         mfb_q_o3_proj = self.Linear_dataproj2(q_feat)                   # data_out (N, 5000)
         mfb_i_o3_proj = self.Linear_imgproj2(img_feat.float())          # img_feature (N, 5000)
         mfb_iq_o3_eltwise = torch.mul(mfb_q_o3_proj, mfb_i_o3_proj)
         mfb_iq_o3_eltwise = torch.mul(mfb_iq_o3_eltwise, mfb_iq_o2_drop)
-        mfb_iq_o3_drop = self.Dropout2(mfb_iq_o3_eltwise)
+        mfb_iq_o3_drop = mfb_iq_o3_eltwise
+        #mfb_iq_o3_drop = self.Dropout2(mfb_iq_o3_eltwise)
         mfb_iq_o3_resh = mfb_iq_o3_drop.view(-1, 1, self.MFB_OUT_DIM, self.MFB_FACTOR_NUM)
-        mfb_o3_out = torch.squeeze(torch.sum(mfb_iq_o3_resh, 3))                            # N x 1000
+
+        #mfb_o3_out = torch.squeeze(torch.sum(mfb_iq_o3_resh, 3))                            # N x 1000
+        if(mfb_iq_o3_resh.size(0)>1):                                                                                             # N x 1 x 1000 x 5
+            mfb_o3_out = torch.squeeze(torch.sum(mfb_iq_o3_resh, 3)) 
+        else:
+            mfb_o3_out = torch.sum(mfb_iq_o3_resh, 3).view(1,mfb_iq_o3_resh.size(2)) 
         mfb_o3_out = torch.sqrt(F.relu(mfb_o3_out)) - torch.sqrt(F.relu(-mfb_o3_out))
         mfb_o3_out = F.normalize(mfb_o3_out)
 
-        mfb_o23_out = torch.cat((mfb_o2_out, mfb_o3_out), 1)        #200,2000     
+        mfb_o23_out = torch.cat((mfb_o2_out, mfb_o3_out), 1)#200,2000     
         #prediction = self.Linear_predict(mfb_o23_out)               
         #prediction = F.log_softmax(prediction)
 

Visual_Attention/grad_cam.py

@@ -37,41 +37,48 @@ def compute_score_with_logits(logits, labels):
 def evaluate_attention_model(args):
 
     class_data=pd.read_csv(args.class_metadata_file)
-    class_label_map={0:"no",1:"yes"}
+    #class_label_map={0:"no",1:"yes"}
 
-    #class_label_map=class_data['Label_names'].tolist()
+    class_label_map=class_data['Label_names'].tolist()
 
     print('Loading model checkpoint')
     attention_model_checkpoint=torch.load(args.model_path)
-    
     new_state_dict = OrderedDict()
     for k, v in attention_model_checkpoint.items():
         name = k[7:] # remove `module.`
         new_state_dict[name] = v
     print('Model checkpoint loaded')
+    #new_state_dict["classifier.main.2.bias"]=new_state_dict.pop("classifier.main.3.bias")
+    #new_state_dict["classifier.main.2.weight_g"]=new_state_dict.pop("classifier.main.3.weight_g")
+    #new_state_dict["classifier.main.2.weight_v"]=new_state_dict.pop("classifier.main.3.weight_v")
 
     print(new_state_dict.keys())
     print('Loading Dictionary')
     dictionary=Dictionary.load_from_file(args.pickle_path)
 
     train_dataset=Dataset_VQA(img_root_dir=args.image_root_dir,feats_data_path=args.feats_data_path,dictionary=dictionary,choice='train',dataroot=args.data_root,arch_choice=args.arch_choice,layer_option=args.layer_option)
     print('Loading the attention model')
-    attention_model = attention_mfh(train_dataset, num_hid=args.num_hid, dropout= args.dropout, norm=args.norm,\
+    attention_model = attention_baseline(train_dataset, num_hid=args.num_hid, dropout= args.dropout, norm=args.norm,\
                                activation=args.activation, drop_L=args.dropout_L, drop_G=args.dropout_G,\
-                               drop_W=args.dropout_W, drop_C=args.dropout_C,mfb_out_dim=args.mfb_out_dim)
+                               drop_W=args.dropout_W, drop_C=args.dropout_C)
+
+    #attention_model=attention_mfh(train_dataset, num_hid=args.num_hid, dropout= args.dropout, norm=args.norm,\
+                               #activation=args.activation, drop_L=args.dropout_L, drop_G=args.dropout_G,\
+                               #drop_W=args.dropout_W, drop_C=args.dropout_C,mfb_out_dim=args.mfb_out_dim)
     attention_model.load_state_dict(new_state_dict)
     attention_model.eval()
 
     torch.manual_seed(args.seed)
-    torch.cuda.manual_seed(args.seed)
+    torch.cuda.manual_seed_all(args.seed)
+    #torch.cuda.manual_seed(args.seed)
     torch.cuda.set_device(args.device)
     attention_model.to(args.device)
     if(args.image_model is None):
         """use extracted features as a Dataset and Dataloader
         """
         print('Using validation features')
-        dataset_temp=Dataset_VQA(img_root_dir=args.image_root_dir,feats_data_path=args.feats_data_path,dictionary=dictionary,bert_option=args.bert_option,rcnn_pkl_path=args.rcnn_path,choice=args.choice,dataroot=args.data_root,arch_choice=args.arch_choice,layer_option=args.layer_option)
-        loader=DataLoader(dataset_temp, batch_size=args.batch_size, shuffle=False, num_workers=10)
+        dataset_temp=Dataset_VQA(img_root_dir=args.image_root_dir,feats_data_path=args.feats_data_path,dictionary=dictionary,bert_option=args.bert_option,rcnn_pkl_path=None,choice=args.choice,dataroot=args.data_root,arch_choice=args.arch_choice,layer_option=args.layer_option)
+        loader=DataLoader(dataset_temp, batch_size=args.batch_size, shuffle=False, num_workers=1)
         print('Length of validation dataloader:', len(loader))
         upper_bound = 0
         num_data = 0
@@ -82,25 +89,26 @@ def evaluate_attention_model(args):
         predicted_class_labels=[]
         question_set=[]
         question_id=[]
-        for data in tqdm(loader):
-
-            feat,quest,quest_sent,quest_id,target = data
-            feat = feat.to(args.device)
-            quest = quest.to(args.device)
-            target = target.to(args.device)
-            
-            question_id=question_id+quest_id.tolist()
-            pred = attention_model(feat, quest, target)
-            question_set=question_set+list(quest_sent)
-            loss = instance_bce_with_logits(pred, target)
-            V_loss += loss.item() * feat.size(0)
-            score_temp, logits, class_labels= compute_score_with_logits(pred, target.data)
-            actual_class_labels=actual_class_labels+list(class_labels.cpu().numpy())
-            predicted_class_labels=predicted_class_labels+list(logits.cpu().numpy())
-            batch_score=score_temp.sum()
-            score += batch_score
-            upper_bound += (target.max(1)[0]).sum()
-            num_data += pred.size(0)
+        count=0
+        for data in tqdm(loader):         
+                feat,quest,quest_sent,quest_id,target = data
+                feat = feat.to(args.device)
+                quest = quest.to(args.device)
+                target = target.to(args.device)
+                
+                question_id=question_id+quest_id.tolist()
+                pred = attention_model(feat, quest)
+                question_set=question_set+list(quest_sent)
+                loss = instance_bce_with_logits(pred, target)
+                V_loss += loss.item() * feat.size(0)
+                score_temp, logits, class_labels= compute_score_with_logits(pred, target.data)
+                actual_class_labels=actual_class_labels+list(class_labels.cpu().numpy())
+                predicted_class_labels=predicted_class_labels+list(logits.cpu().numpy())
+                batch_score=score_temp.sum()
+                score += batch_score
+                upper_bound += (target.max(1)[0]).sum()
+                num_data += pred.size(0)
+            #count=count+1
 
 
 
@@ -112,10 +120,10 @@ def evaluate_attention_model(args):
         for index,val in tqdm(enumerate(question_id)):
             temp={"answer":class_predicted_name[index],"question_id":val}
             list_set.append(temp)
-        with open('validation_results.json', 'w') as fout:
+        with open('validation_results_resnet_152_attention_baseline_num_hid_512_batch_size_512.json', 'w') as fout:
             json.dump(list_set , fout)
-        #predicted_df=pd.DataFrame({'Questions':question_set,'Actual_Answers':class_actual_name,'Predicted_Answers':class_predicted_name})
-        #predicted_df.to_csv('Validation_Stats.csv')
+        predicted_df=pd.DataFrame({'Question_id':question_id,'Questions':question_set,'Actual_Answers':class_actual_name,'Predicted_Answers':class_predicted_name})
+        predicted_df.to_csv('Validation_Stats_resnet_152_attention_baseline_num_hid_512_batch_size_512.csv')
         score = score / len(loader.dataset)
         V_loss /= len(loader.dataset)
         upper_bound = upper_bound / len(loader.dataset)
@@ -136,10 +144,10 @@ def evaluate_attention_model(args):
     parser.add_argument('--feats_data_path', type=str, default="/data/digbose92/VQA/COCO/train_hdf5_COCO/")
     parser.add_argument('--data_root', type=str, default="/proj/digbose92/VQA/VisualQuestion_VQA/common_resources")
     parser.add_argument('--npy_file', type=str, default="../../VisualQuestion_VQA/Visual_All/data/glove6b_init_300d.npy")
-    parser.add_argument('--model_path', type=str, default="results_GRU_uni/results_rcnn_hid_1280_mfh_YES_NO_ADAM/model.pth")
+    parser.add_argument('--model_path', type=str, default="results_GRU_uni/results_resnet_152_hid_512_YES_NO_ADAM/model.pth")
     parser.add_argument('--image_model', type=str, default=None)
-    parser.add_argument('--batch_size', type=int, default=32)
-    parser.add_argument('--num_hid', type=int, default=1280) # they used 1024
+    parser.add_argument('--batch_size', type=int, default=512)
+    parser.add_argument('--num_hid', type=int, default=512) # they used 1024
     parser.add_argument('--dropout', type=float, default=0.3)
     parser.add_argument('--dropout_L', type=float, default=0.1)
     parser.add_argument('--dropout_G', type=float, default=0.2)

Visual_Attention/inference_attention_model.py

@@ -25,7 +25,7 @@ def __init__(self, w_emb, q_emb, v_att, q_net, v_net, classifier):
         self.v_net = v_net
         self.classifier = classifier
 
-    def forward(self, v, q, labels):
+    def forward(self, v, q):
         """Forward
 
         v: [batch, num_objs, obj_dim]
@@ -86,7 +86,7 @@ def __init__(self, w_emb, q_emb, v_att, q_net, v_net, mfh_net, classifier):
         self.mfh_net = mfh_net
         self.classifier = classifier
 
-    def forward(self, v, q, labels):
+    def forward(self, v, q):
         """Forward
 
         v: [batch, num_objs, obj_dim]
@@ -112,6 +112,7 @@ def forward(self, v, q, labels):
         logits = self.classifier(joint_repr)
         return logits
 
+
 class VQA_Model_MFH_classifier(nn.Module):
     def __init__(self, w_emb, q_emb, v_att, q_net, v_net, mfh_net):
         super(VQA_Model_MFH_classifier, self).__init__()
@@ -150,10 +151,44 @@ def forward(self, v, q, labels):
         #logits = self.classifier(joint_repr)
         return logits
 
+class VQA_Model_MFH_BERT_fusion(nn.Module):
+    def __init__(self, bert_emb, v_att, q_net, v_net, mfh_net,classifier):
+        super(VQA_Model_MFH_BERT_fusion, self).__init__()
+        self.bert_emb = bert_emb
+        self.v_att = v_att
+        self.q_net = q_net
+        self.v_net = v_net
+        self.mfh_net = mfh_net
+        self.classifier = classifier
+
+    def forward(self, v, q, labels):
+        """Forward
+
+        v: [batch, num_objs, obj_dim]
+        q: [batch_size, seq_length]
+
+        return: logits, not probs
+        """   
+        q_emb = self.bert_emb(q)
+        #print(q_emb.size())
+
+        att = self.v_att(v, q_emb) # [batch, 1, v_dim]
+        v_emb = (att * v).sum(1) # [batch, v_dim]
+
+        q_repr = self.q_net(q_emb)
+        v_repr = self.v_net(v_emb)
+        #joint_repr=self.mfh_net(q_repr,v_repr)
+        joint_repr=self.mfh_net(q_repr,v_repr)
+        #joint_repr = q_repr * v_repr
+
+        #invoke MFH for fusion of q_repr and v_repr
 
+        logits = self.classifier(joint_repr)
+        return logits
 
 ############# ATTENTION BASELINE ############
 def attention_baseline(dataset, num_hid, dropout, norm, activation, drop_L , drop_G, drop_W, drop_C, bidirect_val=False):
+    print('Here in the attention baseline')
     w_emb = WordEmbedding(dataset.dictionary.ntoken, emb_dim=300, dropout=drop_W)
     q_emb = QuestionEmbedding(in_dim=300, num_hid=num_hid, nlayers=1, bidirect=bidirect_val, dropout=drop_G, rnn_type='GRU')
     #bert_emb=BertEmbedding(in_dim=7168,num_hid=num_hid)
@@ -225,6 +260,24 @@ def attention_mfh_classifier(dataset, num_hid, dropout, norm, activation, drop_L
     return(VQA_Model_MFH_classifier(w_emb,q_emb,v_att,q_net,v_net,mfh_net))
 
 
+###### ATTENTION + BERT + MFH FUSION #############
+def attention_bert_mfh_fusion(dataset, num_hid, dropout, norm, activation, drop_L , drop_G, drop_W, drop_C, mfb_out_dim, bidirect_val=False):
+    #w_emb = WordEmbedding(dataset.dictionary.ntoken, emb_dim=300, dropout=drop_W)
+    #q_emb = QuestionEmbedding(in_dim=300, num_hid=num_hid, nlayers=1, bidirect=bidirect_val, dropout=drop_G, rnn_type='GRU')
+
+    bert_emb=BertEmbedding(in_dim=3072,num_hid=num_hid)
+    v_att = Base_Att(v_dim= dataset.v_dim, q_dim= num_hid, num_hid= num_hid, dropout= dropout, bidirect=bidirect_val,norm= norm, act= activation)
+    if(bidirect_val is False):
+        q_net = FCNet([num_hid, num_hid], dropout= drop_L, norm= norm, act= activation)
+        #v_net = FCNet([dataset.v_dim, num_hid], dropout= drop_L, norm= norm, act= activation)
+    else:
+        q_net = FCNet([2*num_hid, num_hid], dropout= drop_L, norm= norm, act= activation)
+        
+    v_net = FCNet([dataset.v_dim, num_hid], dropout= drop_L, norm= norm, act= activation)
+    mfh_net=mfh_baseline(QUEST_EMBED=num_hid,VIS_EMBED=num_hid,MFB_OUT_DIM=mfb_out_dim)
+    classifier = SimpleClassifier(in_dim=2*mfb_out_dim, hid_dim=2 * num_hid, out_dim=dataset.num_ans_candidates, dropout=drop_C, norm= norm, act= activation)
+    return(VQA_Model_MFH_BERT_fusion(bert_emb,v_att,q_net,v_net,mfh_net,classifier))
+
 
 def weights_init_xn(m):
     if isinstance(m, nn.Linear):