More results and Delete client models

Author: gferrate

orchestrator/orchestrator.py

@@ -85,6 +85,15 @@ def main(op_mode, communication_rounds):
         logging.info('Iteration {}...'.format(i))
         send_iteration_to_frontend(i)
 
+        logging.info('Deleting client models...')
+        url = 'http://{}:{}/del_client_models'.format(
+            hosts['secure_aggregator']['host'],
+            hosts['secure_aggregator']['port']
+        )
+        res = requests.post(url)
+        check_response_ok(res)
+        logging.info('Done')
+
         logging.info('Sending /train_model request to clients...')
         ch.perform_requests_and_wait('train_model')
         #logging.info('Performed clients: {}'.format(performed_clients))

results/1_clients_iid_1_frame.tar.gz results/1_client_1_frame/1_clients_iid_1_frame.tar.gz

@@ -1,6 +1,6 @@
+
 def extract_results(fn):
     data = {}
-    last_epoch = -1
     clustering = False
     with open(fn, 'r') as f:
         for l in f:
@@ -11,15 +11,19 @@ def extract_results(fn):
                 epoch = int(s[0].split('[')[1].split(']')[0])
                 top_1 = float(s[-2].split(' ')[1].split(' ')[0])
                 top_3 = float(s[-1].split(' ')[1].split(' ')[0])
+                time = float(s[1].split(' ')[1].split(' ')[0])
                 if clustering:
                     data.setdefault(epoch, {}).setdefault('test-top1-clustering', []).append(top_1)
                     data[epoch].setdefault('test-top3-clustering', []).append(top_3)
                 else:
                     data.setdefault(epoch, {}).setdefault('test-top1', []).append(top_1)
                     data[epoch].setdefault('test-top3', []).append(top_3)
-                if last_epoch != epoch:
-                    print('Extracting epoch', epoch)
-                    last_epoch = epoch
+            if l.startswith('Test') or l.startswith('Train'):
+                s = l.split('\t')
+                epoch = int(s[0].split('[')[1].split(']')[0])
+                time = float(s[1].split(' ')[1].split(' ')[0])
+                data.setdefault(epoch, {}).setdefault('time', 0.0)
+                data[epoch]['time'] += time
     return data
 
 def avg(_list):

results/1_clients_non-iid_1_frame.tar.gz results/1_client_1_frame/1_clients_non-iid_1_frame.tar.gz

@@ -1 +1 @@
-results = {0: {'test-top1': 17.462779156327542, 'test-top3': 39.73325062034739}, 1: {'test-top1': 31.674937965260547, 'test-top3': 57.411083746898264}, 2: {'test-top1': 30.43424317617866, 'test-top3': 55.27364205955335}, 3: {'test-top1': 34.026054590570716, 'test-top3': 57.44485794044665}, 4: {'test-top1': 38.65177853598015, 'test-top3': 60.79955893300248}, 5: {'test-top1': 37.68196836228288, 'test-top3': 60.16060111662531}, 6: {'test-top1': 40.578301488833745, 'test-top3': 62.80328101736973}, 7: {'test-top1': 35.774745037220846, 'test-top3': 57.41315136476427}, 8: {'test-top1': 41.339950372208435, 'test-top3': 63.39261104218362}, 9: {'test-top1': 35.57692307692308, 'test-top3': 59.23697270471464}, 10: {'test-top1': 40.421836228287845, 'test-top3': 60.803004962779156}, 11: {'test-top1': 36.32754342431762, 'test-top3': 60.45147518610422}, 12: {'test-top1': 31.477116004962777, 'test-top3': 56.06010483870968}, 13: {'test-top1': 38.22994230769231, 'test-top3': 61.11042183622829}, 14: {'test-top1': 37.96526054590571, 'test-top3': 59.4044665012407}, 15: {'test-top1': 40.898125310173704, 'test-top3': 62.81637717121588}, 16: {'test-top1': 40.259856699751865, 'test-top3': 61.74937965260546}, 17: {'test-top1': 39.83802047146402, 'test-top3': 62.20774751861042}, 18: {'test-top1': 41.177970843672455, 'test-top3': 62.697821960297766}, 19: {'test-top1': 41.43300248138958, 'test-top3': 62.70402543424317}, 20: {'test-top1': 39.312103598014886, 'test-top3': 61.38888895781638}, 21: {'test-top1': 39.10876736972705, 'test-top3': 62.74193548387097}, 22: {'test-top1': 40.607940446650126, 'test-top3': 62.25737531017369}, 23: {'test-top1': 40.12062282878412, 'test-top3': 62.83498759305211}, 24: {'test-top1': 41.67149131513648, 'test-top3': 62.23325062034739}, 25: {'test-top1': 40.607940446650126, 'test-top3': 63.49875930521092}, 26: {'test-top1': 42.42831513647643, 'test-top3': 63.455334987593055}, 27: {'test-top1': 38.50565198511166, 'test-top3': 60.7678523573201}, 28: {'test-top1': 41.30893300248139, 'test-top3': 62.03473945409429}, 29: {'test-top1': 39.88144478908189, 'test-top3': 61.178660049627794}, 30: {'test-top1': 40.17162965260546, 'test-top3': 61.22139516129032}, 31: {'test-top1': 39.565067617866006, 'test-top3': 61.525365384615384}, 32: {'test-top1': 40.42252543424318, 'test-top3': 61.364764267990076}, 33: {'test-top1': 41.79900744416874, 'test-top3': 62.03405024813895}, 34: {'test-top1': 40.53349875930521, 'test-top3': 62.86531575682382}, 35: {'test-top1': 40.71960297766749, 'test-top3': 61.31237965260546}, 36: {'test-top1': 40.931210297766754, 'test-top3': 62.54273511166253}, 37: {'test-top1': 39.88833746898263, 'test-top3': 62.03473945409429}, 38: {'test-top1': 39.844913151364764, 'test-top3': 62.52757071960298}, 39: {'test-top1': 39.62434491315137, 'test-top3': 61.74937965260546}, 40: {'test-top1': 40.58450496277916, 'test-top3': 61.38888895781638}, 41: {'test-top1': 41.53363647642679, 'test-top3': 62.10228846153846}, 42: {'test-top1': 43.15481141439206, 'test-top3': 63.31816935483871}, 43: {'test-top1': 41.36752109181142, 'test-top3': 63.03280955334987}, 44: {'test-top1': 39.59057071960298, 'test-top3': 61.12834305210918}, 45: {'test-top1': 42.69024007444169, 'test-top3': 64.26178660049628}, 46: {'test-top1': 39.6712158808933, 'test-top3': 61.59980707196029}, 47: {'test-top1': 41.07044416873449, 'test-top3': 61.934794665012404}, 48: {'test-top1': 41.068375930521086, 'test-top3': 63.03901302729528}, 49: {'test-top1': 41.15177853598015, 'test-top3': 63.026606079404466}, 50: {'test-top1': 41.15177853598015, 'test-top3': 63.026606079404466, 'test-top1-clustering': 41.15177853598015, 'test-top3-clustering': 63.026606079404466}}
+results = {0: {'time': 54.089000000001406, 'test-top1': 17.462779156327542, 'test-top3': 39.73325062034739}, 1: {'time': 54.33400000000157, 'test-top1': 31.674937965260547, 'test-top3': 57.411083746898264}, 2: {'time': 54.29600000000146, 'test-top1': 30.43424317617866, 'test-top3': 55.27364205955335}, 3: {'time': 54.38400000000163, 'test-top1': 34.026054590570716, 'test-top3': 57.44485794044665}, 4: {'time': 54.39200000000173, 'test-top1': 38.65177853598015, 'test-top3': 60.79955893300248}, 5: {'time': 54.254000000001525, 'test-top1': 37.68196836228288, 'test-top3': 60.16060111662531}, 6: {'time': 54.44000000000157, 'test-top1': 40.578301488833745, 'test-top3': 62.80328101736973}, 7: {'time': 54.431000000001596, 'test-top1': 35.774745037220846, 'test-top3': 57.41315136476427}, 8: {'time': 54.38100000000166, 'test-top1': 41.339950372208435, 'test-top3': 63.39261104218362}, 9: {'time': 54.331000000001545, 'test-top1': 35.57692307692308, 'test-top3': 59.23697270471464}, 10: {'time': 54.33600000000168, 'test-top1': 40.421836228287845, 'test-top3': 60.803004962779156}, 11: {'time': 54.37300000000157, 'test-top1': 36.32754342431762, 'test-top3': 60.45147518610422}, 12: {'time': 54.43800000000148, 'test-top1': 31.477116004962777, 'test-top3': 56.06010483870968}, 13: {'time': 54.05300000000153, 'test-top1': 38.22994230769231, 'test-top3': 61.11042183622829}, 14: {'time': 54.35700000000159, 'test-top1': 37.96526054590571, 'test-top3': 59.4044665012407}, 15: {'time': 54.379000000001604, 'test-top1': 40.898125310173704, 'test-top3': 62.81637717121588}, 16: {'time': 54.43200000000163, 'test-top1': 40.259856699751865, 'test-top3': 61.74937965260546}, 17: {'time': 54.33900000000156, 'test-top1': 39.83802047146402, 'test-top3': 62.20774751861042}, 18: {'time': 54.48800000000155, 'test-top1': 41.177970843672455, 'test-top3': 62.697821960297766}, 19: {'time': 54.28000000000167, 'test-top1': 41.43300248138958, 'test-top3': 62.70402543424317}, 20: {'time': 54.24200000000154, 'test-top1': 39.312103598014886, 'test-top3': 61.38888895781638}, 21: {'time': 54.25100000000166, 'test-top1': 39.10876736972705, 'test-top3': 62.74193548387097}, 22: {'time': 54.401000000001616, 'test-top1': 40.607940446650126, 'test-top3': 62.25737531017369}, 23: {'time': 54.32900000000156, 'test-top1': 40.12062282878412, 'test-top3': 62.83498759305211}, 24: {'time': 54.36400000000154, 'test-top1': 41.67149131513648, 'test-top3': 62.23325062034739}, 25: {'time': 54.43000000000164, 'test-top1': 40.607940446650126, 'test-top3': 63.49875930521092}, 26: {'time': 54.412000000001576, 'test-top1': 42.42831513647643, 'test-top3': 63.455334987593055}, 27: {'time': 54.432000000001636, 'test-top1': 38.50565198511166, 'test-top3': 60.7678523573201}, 28: {'time': 54.5070000000016, 'test-top1': 41.30893300248139, 'test-top3': 62.03473945409429}, 29: {'time': 54.518000000001535, 'test-top1': 39.88144478908189, 'test-top3': 61.178660049627794}, 30: {'time': 54.432000000001665, 'test-top1': 40.17162965260546, 'test-top3': 61.22139516129032}, 31: {'time': 54.214000000001555, 'test-top1': 39.565067617866006, 'test-top3': 61.525365384615384}, 32: {'time': 54.36600000000168, 'test-top1': 40.42252543424318, 'test-top3': 61.364764267990076}, 33: {'time': 54.35100000000157, 'test-top1': 41.79900744416874, 'test-top3': 62.03405024813895}, 34: {'time': 54.45900000000149, 'test-top1': 40.53349875930521, 'test-top3': 62.86531575682382}, 35: {'time': 54.38100000000156, 'test-top1': 40.71960297766749, 'test-top3': 61.31237965260546}, 36: {'time': 54.23700000000167, 'test-top1': 40.931210297766754, 'test-top3': 62.54273511166253}, 37: {'time': 54.102000000001546, 'test-top1': 39.88833746898263, 'test-top3': 62.03473945409429}, 38: {'time': 54.35300000000153, 'test-top1': 39.844913151364764, 'test-top3': 62.52757071960298}, 39: {'time': 54.33900000000159, 'test-top1': 39.62434491315137, 'test-top3': 61.74937965260546}, 40: {'time': 54.44600000000157, 'test-top1': 40.58450496277916, 'test-top3': 61.38888895781638}, 41: {'time': 54.43500000000168, 'test-top1': 41.53363647642679, 'test-top3': 62.10228846153846}, 42: {'time': 54.28200000000162, 'test-top1': 43.15481141439206, 'test-top3': 63.31816935483871}, 43: {'time': 53.94600000000138, 'test-top1': 41.36752109181142, 'test-top3': 63.03280955334987}, 44: {'time': 54.36000000000159, 'test-top1': 39.59057071960298, 'test-top3': 61.12834305210918}, 45: {'time': 54.36200000000148, 'test-top1': 42.69024007444169, 'test-top3': 64.26178660049628}, 46: {'time': 54.305000000001634, 'test-top1': 39.6712158808933, 'test-top3': 61.59980707196029}, 47: {'time': 54.27000000000154, 'test-top1': 41.07044416873449, 'test-top3': 61.934794665012404}, 48: {'time': 54.300000000001454, 'test-top1': 41.068375930521086, 'test-top3': 63.03901302729528}, 49: {'time': 54.197000000001516, 'test-top1': 41.15177853598015, 'test-top3': 63.026606079404466}, 50: {'test-top1': 41.15177853598015, 'test-top3': 63.026606079404466, 'time': 9.674000000000255, 'test-top1-clustering': 41.15177853598015, 'test-top3-clustering': 63.026606079404466}}

results/5_clients_7_frames/1_clients_iid_7_frames.tar.gz results/1_client_7_frames/1_clients_iid_7_frames.tar.gz

@@ -26,7 +26,7 @@
     y_values = list(sorted(y_values, key=lambda t: t[0])) # Order by epoch asc
     y_values = [x[1] for x in y_values]
     plt.plot(X_AXIS, y_values, linewidth=1, label=legend[t], marker='o', markersize=MARKER_SIZE)
-    plt.legend(loc="lower right")
+    plt.legend(loc="upper right")
 
 # IID results:
 from raw_5_clients_iid_1_frame import results
@@ -44,7 +44,7 @@
              linestyle='solid',
              marker="^",
              markersize=MARKER_SIZE)
-    plt.legend(loc="lower right")
+    plt.legend(loc="upper right")
 
 # NON-IID results:
 from raw_5_clients_non_iid_1_frame import results
@@ -62,7 +62,7 @@
              linestyle='solid',
              marker="s",
              markersize=MARKER_SIZE)
-    plt.legend(loc="lower right")
+    plt.legend(loc="upper right")
 
 # Limits
 plt.axis(xmin=0, xmax=epochs+1, ymin=0, ymax=100)

results/5_clients_1_frame/extract_results.py

@@ -0,0 +1,80 @@
+import matplotlib
+matplotlib.use('agg')
+import matplotlib.pyplot as plt
+
+
+MARKER_SIZE=2
+
+# Init
+plt.figure()
+
+#epochs = 50
+## Add vertical Lines
+#for i in range(epochs):
+#    plt.axvline(
+#        i+1, linewidth=0.5, color='black', linestyle='dotted', alpha=0.5)
+
+# No FL results
+from raw_non_fedlearn_1_frame import results
+
+timings = [(e, x['time']) for e, x in results.items()][:-1]
+timings = list(sorted(timings, key=lambda t: t[0])) # Order by epoch asc
+timings = [x[1] for x in timings]
+last = 0
+for i in range(len(timings)):
+    timings[i] += last
+    last = timings[i]
+
+types = ['test-top1', 'test-top3']
+legend = {'test-top1': 'no-fl-top-1', 'test-top3': 'no-fl-top-3'}
+for t in types:
+    y_values = [(e, x[t]) for e, x in results.items()][:-1]
+    y_values = list(sorted(y_values, key=lambda t: t[0])) # Order by epoch asc
+    y_values = [x[1] for x in y_values]
+    plt.plot(timings, y_values, linewidth=1, label=legend[t], marker='o', markersize=MARKER_SIZE)
+    plt.legend(loc="upper right")
+
+# IID results:
+from raw_5_clients_iid_1_frame import results
+types = ['test-top1', 'test-top3']
+legend = {'test-top1': '5-clients-iid-top-1',
+          'test-top3': '5-clients-iid-top-3'}
+timings = [x['elapsed_time'] for x in results]
+for t in types:
+    y_values = [x['test_result'][t] for x in results]
+    plt.plot(timings,
+             y_values,
+             linewidth=1,
+             label=legend[t],
+             linestyle='solid',
+             marker="^",
+             markersize=MARKER_SIZE)
+    plt.legend(loc="upper right")
+
+# NON-IID results:
+from raw_5_clients_non_iid_1_frame import results
+types = ['test-top1', 'test-top3']
+legend = {'test-top1': '5-clients-non-iid-top-1',
+          'test-top3': '5-clients-non-iid-top-3'}
+timings = [x['elapsed_time'] for x in results]
+for t in types:
+    y_values = [x['test_result'][t] for x in results]
+    plt.plot(timings,
+             y_values,
+             linewidth=1,
+             label=legend[t],
+             linestyle='solid',
+             marker="s",
+             markersize=MARKER_SIZE)
+    plt.legend(loc="upper right")
+
+# Limits
+plt.axis(ymin=0, ymax=100)
+
+# Labels
+plt.title('1 input frame comparison')
+plt.ylabel('Accuracy')
+plt.xlabel('Time (seconds)')
+
+# Save
+plt.savefig('result_xtime.png', dpi=400)

results/5_clients_1_frame/raw_non_fedlearn_1_frame.py

@@ -63,6 +63,13 @@ def get_pub_key():
     return jsonify({'pub_key': rsa.export_public_key()})
 
 
+@assert_idle_state
+@app.route('/del_client_models', methods=['POST'])
+def del_client_models():
+    sec_agg.delete_client_models()
+    return jsonify({'ok': 1})
+
+
 @assert_idle_state
 @app.route('/client_model', methods=['POST'])
 def get_client_model():