More results

Author: gferrate

.gitignore

@@ -56,7 +56,6 @@ coverage.xml
 *.pot
 
 # Django stuff:
-*.log
 local_settings.py
 db.sqlite3
 db.sqlite3-journal
@@ -137,3 +136,5 @@ dmypy.json
 
 *.pyc
 *.swp
+
+logs/*.log

results/5_clients_1_frame/1_frames_non_fedlearn.log

@@ -0,0 +1,43 @@
+def extract_results(fn):
+    data = {}
+    last_epoch = -1
+    clustering = False
+    with open(fn, 'r') as f:
+        for l in f:
+            if 'clustering...' in l:
+                clustering = True
+            if l.startswith('Test'):
+                s = l.split('\t')
+                epoch = int(s[0].split('[')[1].split(']')[0])
+                top_1 = float(s[-2].split(' ')[1].split(' ')[0])
+                top_3 = float(s[-1].split(' ')[1].split(' ')[0])
+                if clustering:
+                    data.setdefault(epoch, {}).setdefault('test-top1-clustering', []).append(top_1)
+                    data[epoch].setdefault('test-top3-clustering', []).append(top_3)
+                else:
+                    data.setdefault(epoch, {}).setdefault('test-top1', []).append(top_1)
+                    data[epoch].setdefault('test-top3', []).append(top_3)
+                if last_epoch != epoch:
+                    print('Extracting epoch', epoch)
+                    last_epoch = epoch
+    return data
+
+def avg(_list):
+    return sum(_list) / len(_list)
+
+def average_epochs(data):
+    for epoch, d in data.items():
+        d['test-top1'] = avg(d['test-top1'])
+        d['test-top3'] = avg(d['test-top3'])
+        if 'test-top1-clustering' in d:
+            d['test-top1-clustering'] = avg(d['test-top1-clustering'])
+            d['test-top3-clustering'] = avg(d['test-top3-clustering'])
+    return data
+
+if __name__ == '__main__':
+    fn = '1_frames_non_fedlearn.log'
+    data = extract_results(fn)
+    average_epochs(data)
+    print(data)
+
+

results/5_clients_1_frame/5_clients_non-iid_1_frame.tar.gz

@@ -0,0 +1 @@
+results = [{'msg': 'Model aggregation done!\nGlobal model written to persistent storage.', 'test_result': {'test-top1': 4.541224641727155, 'test-top3': 11.061480240709722, 'test_cluster-top1': 4.541224641727155, 'test_cluster-top3': 11.061480240709722}, 'elapsed_time': 59.37310552597046}, {'msg': 'Model aggregation done!\nGlobal model written to persistent storage.', 'test_result': {'test-top1': 4.255847136919163, 'test-top3': 14.852037967615857, 'test_cluster-top1': 4.255847136919163, 'test_cluster-top3': 14.852037967615857}, 'elapsed_time': 118.3054575920105}, {'msg': 'Model aggregation done!\nGlobal model written to persistent storage.', 'test_result': {'test-top1': 6.50784788138222, 'test-top3': 18.67361498852286, 'test_cluster-top1': 6.50784788138222, 'test_cluster-top3': 18.67361498852286}, 'elapsed_time': 177.7696177959442}, {'msg': 'Model aggregation done!\nGlobal model written to persistent storage.', 'test_result': {'test-top1': 11.700477697127614, 'test-top3': 24.424592096283888, 'test_cluster-top1': 11.700477697127614, 'test_cluster-top3': 24.424592096283888}, 'elapsed_time': 237.16842317581177}, {'msg': 'Model aggregation done!\nGlobal model written to persistent storage.', 'test_result': {'test-top1': 13.356907996773993, 'test-top3': 28.922389726409826, 'test_cluster-top1': 13.356907996773993, 'test_cluster-top3': 28.922389726409826}, 'elapsed_time': 297.6419303417206}, {'msg': 'Model aggregation done!\nGlobal model written to persistent storage.', 'test_result': {'test-top1': 14.039332464793102, 'test-top3': 28.07246107078603, 'test_cluster-top1': 14.039332464793102, 'test_cluster-top3': 28.07246107078603}, 'elapsed_time': 357.37578535079956}, {'msg': 'Model aggregation done!\nGlobal model written to persistent storage.', 'test_result': {'test-top1': 12.693095105155408, 'test-top3': 33.227867731837115, 'test_cluster-top1': 12.693095105155408, 'test_cluster-top3': 33.227867731837115}, 'elapsed_time': 417.51899886131287}, {'msg': 'Model aggregation done!\nGlobal model written to persistent storage.', 'test_result': {'test-top1': 15.143619331224022, 'test-top3': 34.41900862336373, 'test_cluster-top1': 15.143619331224022, 'test_cluster-top3': 34.41900862336373}, 'elapsed_time': 477.49101209640503}, {'msg': 'Model aggregation done!\nGlobal model written to persistent storage.', 'test_result': {'test-top1': 9.516719398993152, 'test-top3': 31.71412618459473, 'test_cluster-top1': 9.516719398993152, 'test_cluster-top3': 31.71412618459473}, 'elapsed_time': 536.6495304107666}, {'msg': 'Model aggregation done!\nGlobal model written to persistent storage.', 'test_result': {'test-top1': 12.717910540356101, 'test-top3': 32.1608040201005, 'test_cluster-top1': 12.717910540356101, 'test_cluster-top3': 32.1608040201005}, 'elapsed_time': 596.4128613471985}, {'msg': 'Model aggregation done!\nGlobal model written to persistent storage.', 'test_result': {'test-top1': 9.864135492276196, 'test-top3': 26.98678578075563, 'test_cluster-top1': 9.864135492276196, 'test_cluster-top3': 26.98678578075563}, 'elapsed_time': 658.2025229930878}, {'msg': 'Model aggregation done!\nGlobal model written to persistent storage.', 'test_result': {'test-top1': 11.154538122712328, 'test-top3': 30.001861157640054, 'test_cluster-top1': 11.154538122712328, 'test_cluster-top3': 30.001861157640054}, 'elapsed_time': 717.4545338153839}, {'msg': 'Model aggregation done!\nGlobal model written to persistent storage.', 'test_result': {'test-top1': 12.19678640114151, 'test-top3': 32.50201625032352, 'test_cluster-top1': 12.19678640114151, 'test_cluster-top3': 32.50201625032352}, 'elapsed_time': 777.8643741607666}, {'msg': 'Model aggregation done!\nGlobal model written to persistent storage.', 'test_result': {'test-top1': 7.34536881940567, 'test-top3': 24.666542589490664, 'test_cluster-top1': 7.34536881940567, 'test_cluster-top3': 24.666542589490664}, 'elapsed_time': 837.7167854309082}, {'msg': 'Model aggregation done!\nGlobal model written to persistent storage.', 'test_result': {'test-top1': 9.913766362677585, 'test-top3': 30.262423227247346, 'test_cluster-top1': 9.913766362677585, 'test_cluster-top3': 30.262423227247346}, 'elapsed_time': 897.2649583816528}, {'msg': 'Model aggregation done!\nGlobal model written to persistent storage.', 'test_result': {'test-top1': 10.627210124697562, 'test-top3': 23.94069110987034, 'test_cluster-top1': 10.627210124697562, 'test_cluster-top3': 23.94069110987034}, 'elapsed_time': 957.111136674881}, {'msg': 'Model aggregation done!\nGlobal model written to persistent storage.', 'test_result': {'test-top1': 12.252621130343073, 'test-top3': 28.878962712915342, 'test_cluster-top1': 12.252621130343073, 'test_cluster-top3': 28.878962712915342}, 'elapsed_time': 1017.3406517505646}, {'msg': 'Model aggregation done!\nGlobal model written to persistent storage.', 'test_result': {'test-top1': 12.004466778336125, 'test-top3': 25.671567715118805, 'test_cluster-top1': 12.004466778336125, 'test_cluster-top3': 25.671567715118805}, 'elapsed_time': 1076.5560207366943}, {'msg': 'Model aggregation done!\nGlobal model written to persistent storage.', 'test_result': {'test-top1': 15.652335752838265, 'test-top3': 29.75370680492313, 'test_cluster-top1': 15.652335752838265, 'test_cluster-top3': 29.75370680492313}, 'elapsed_time': 1136.7740137577057}, {'msg': 'Model aggregation done!\nGlobal model written to persistent storage.', 'test_result': {'test-top1': 10.248774737886965, 'test-top3': 24.64793101309014, 'test_cluster-top1': 10.248774737886965, 'test_cluster-top3': 24.64793101309014}, 'elapsed_time': 1195.482326745987}, {'msg': 'Model aggregation done!\nGlobal model written to persistent storage.', 'test_result': {'test-top1': 14.969911284819158, 'test-top3': 30.609839318637125, 'test_cluster-top1': 14.969911284819158, 'test_cluster-top3': 30.609839318637125}, 'elapsed_time': 1255.3089566230774}, {'msg': 'Model aggregation done!\nGlobal model written to persistent storage.', 'test_result': {'test-top1': 10.869160617904337, 'test-top3': 24.436999813884235, 'test_cluster-top1': 10.869160617904337, 'test_cluster-top3': 24.436999813884235}, 'elapsed_time': 1314.9909992218018}, {'msg': 'Model aggregation done!\nGlobal model written to persistent storage.', 'test_result': {'test-top1': 17.712016874495937, 'test-top3': 31.39152552887896, 'test_cluster-top1': 17.712016874495937, 'test_cluster-top3': 31.39152552887896}, 'elapsed_time': 1374.4790575504303}, {'msg': 'Model aggregation done!\nGlobal model written to persistent storage.', 'test_result': {'test-top1': 12.277436565543768, 'test-top3': 27.048824368757366, 'test_cluster-top1': 12.277436565543768, 'test_cluster-top3': 27.048824368757366}, 'elapsed_time': 1434.1551711559296}, {'msg': 'Model aggregation done!\nGlobal model written to persistent storage.', 'test_result': {'test-top1': 16.458837396860847, 'test-top3': 30.169365344298107, 'test_cluster-top1': 16.458837396860847, 'test_cluster-top3': 30.169365344298107}, 'elapsed_time': 1493.718698978424}, {'msg': 'Model aggregation done!\nGlobal model written to persistent storage.', 'test_result': {'test-top1': 12.010670637136299, 'test-top3': 26.068614678329922, 'test_cluster-top1': 12.010670637136299, 'test_cluster-top3': 26.068614678329922}, 'elapsed_time': 1553.0630447864532}, {'msg': 'Model aggregation done!\nGlobal model written to persistent storage.', 'test_result': {'test-top1': 16.967553818475093, 'test-top3': 31.64588373589955, 'test_cluster-top1': 16.967553818475093, 'test_cluster-top3': 31.64588373589955}, 'elapsed_time': 1612.3575139045715}, {'msg': 'Model aggregation done!\nGlobal model written to persistent storage.', 'test_result': {'test-top1': 13.704324089583722, 'test-top3': 29.369067560022334, 'test_cluster-top1': 13.704324089583722, 'test_cluster-top3': 29.369067560022334}, 'elapsed_time': 1671.8282525539398}, {'msg': 'Model aggregation done!\nGlobal model written to persistent storage.', 'test_result': {'test-top1': 17.20330045288169, 'test-top3': 31.143371175452064, 'test_cluster-top1': 17.20330045288169, 'test_cluster-top3': 31.143371175452064}, 'elapsed_time': 1731.3699855804443}, {'msg': 'Model aggregation done!\nGlobal model written to persistent storage.', 'test_result': {'test-top1': 14.62249519200943, 'test-top3': 30.34927725044978, 'test_cluster-top1': 14.62249519200943, 'test_cluster-top3': 30.34927725044978}, 'elapsed_time': 1791.5872287750244}, {'msg': 'Model aggregation done!\nGlobal model written to persistent storage.', 'test_result': {'test-top1': 16.01215956324834, 'test-top3': 29.375271417875872, 'test_cluster-top1': 16.01215956324834, 'test_cluster-top3': 29.375271417875872}, 'elapsed_time': 1850.8100917339325}, {'msg': 'Model aggregation done!\nGlobal model written to persistent storage.', 'test_result': {'test-top1': 16.384391091258763, 'test-top3': 32.917674793721694, 'test_cluster-top1': 16.384391091258763, 'test_cluster-top3': 32.917674793721694}, 'elapsed_time': 1910.2514789104462}, {'msg': 'Model aggregation done!\nGlobal model written to persistent storage.', 'test_result': {'test-top1': 16.75041876046901, 'test-top3': 29.766114519446916, 'test_cluster-top1': 16.75041876046901, 'test_cluster-top3': 29.766114519446916}, 'elapsed_time': 1969.7298474311829}, {'msg': 'Model aggregation done!\nGlobal model written to persistent storage.', 'test_result': {'test-top1': 16.11142130405112, 'test-top3': 32.29108505490415, 'test_cluster-top1': 16.11142130405112, 'test_cluster-top3': 32.29108505490415}, 'elapsed_time': 2029.595244884491}, {'msg': 'Model aggregation done!\nGlobal model written to persistent storage.', 'test_result': {'test-top1': 18.748061294124945, 'test-top3': 33.804826602146534, 'test_cluster-top1': 18.748061294124945, 'test_cluster-top3': 33.804826602146534}, 'elapsed_time': 2089.4417345523834}, {'msg': 'Model aggregation done!\nGlobal model written to persistent storage.', 'test_result': {'test-top1': 14.517029592406477, 'test-top3': 29.883987840436752, 'test_cluster-top1': 14.517029592406477, 'test_cluster-top3': 29.883987840436752}, 'elapsed_time': 2149.2716057300568}, {'msg': 'Model aggregation done!\nGlobal model written to persistent storage.', 'test_result': {'test-top1': 18.239344872510703, 'test-top3': 33.52565295613872, 'test_cluster-top1': 18.239344872510703, 'test_cluster-top3': 33.52565295613872}, 'elapsed_time': 2209.2456431388855}, {'msg': 'Model aggregation done!\nGlobal model written to persistent storage.', 'test_result': {'test-top1': 14.926484273217941, 'test-top3': 29.375271418822507, 'test_cluster-top1': 14.926484273217941, 'test_cluster-top3': 29.375271418822507}, 'elapsed_time': 2269.3379595279694}, {'msg': 'Model aggregation done!\nGlobal model written to persistent storage.', 'test_result': {'test-top1': 16.94894224207457, 'test-top3': 33.77380730814566, 'test_cluster-top1': 16.94894224207457, 'test_cluster-top3': 33.77380730814566}, 'elapsed_time': 2328.8460552692413}, {'msg': 'Model aggregation done!\nGlobal model written to persistent storage.', 'test_result': {'test-top1': 14.051740182393448, 'test-top3': 29.071282337613997, 'test_cluster-top1': 14.051740182393448, 'test_cluster-top3': 29.071282337613997}, 'elapsed_time': 2388.9179491996765}, {'msg': 'Model aggregation done!\nGlobal model written to persistent storage.', 'test_result': {'test-top1': 15.925305540045908, 'test-top3': 34.02816551895279, 'test_cluster-top1': 15.925305540045908, 'test_cluster-top3': 34.02816551895279}, 'elapsed_time': 2448.8549942970276}, {'msg': 'Model aggregation done!\nGlobal model written to persistent storage.', 'test_result': {'test-top1': 16.105217445250947, 'test-top3': 32.19182331410137, 'test_cluster-top1': 16.105217445250947, 'test_cluster-top3': 32.19182331410137}, 'elapsed_time': 2508.3097388744354}, {'msg': 'Model aggregation done!\nGlobal model written to persistent storage.', 'test_result': {'test-top1': 17.277746758483776, 'test-top3': 35.55431478379552, 'test_cluster-top1': 17.277746758483776, 'test_cluster-top3': 35.55431478379552}, 'elapsed_time': 2568.142520904541}, {'msg': 'Model aggregation done!\nGlobal model written to persistent storage.', 'test_result': {'test-top1': 14.95750356721881, 'test-top3': 30.485762144053602, 'test_cluster-top1': 14.95750356721881, 'test_cluster-top3': 30.485762144053602}, 'elapsed_time': 2627.968149662018}, {'msg': 'Model aggregation done!\nGlobal model written to persistent storage.', 'test_result': {'test-top1': 17.073019418078044, 'test-top3': 34.61753210473263, 'test_cluster-top1': 17.073019418078044, 'test_cluster-top3': 34.61753210473263}, 'elapsed_time': 2688.1428141593933}, {'msg': 'Model aggregation done!\nGlobal model written to persistent storage.', 'test_result': {'test-top1': 14.976115143619332, 'test-top3': 28.76108939760531, 'test_cluster-top1': 14.976115143619332, 'test_cluster-top3': 28.76108939760531}, 'elapsed_time': 2748.251120567322}, {'msg': 'Model aggregation done!\nGlobal model written to persistent storage.', 'test_result': {'test-top1': 18.45647993051678, 'test-top3': 35.678391958379045, 'test_cluster-top1': 18.45647993051678, 'test_cluster-top3': 35.678391958379045}, 'elapsed_time': 2807.9178018569946}, {'msg': 'Model aggregation done!\nGlobal model written to persistent storage.', 'test_result': {'test-top1': 15.931509398846082, 'test-top3': 29.393882995223027, 'test_cluster-top1': 15.931509398846082, 'test_cluster-top3': 29.393882995223027}, 'elapsed_time': 2866.948897600174}, {'msg': 'Model aggregation done!\nGlobal model written to persistent storage.', 'test_result': {'test-top1': 19.47391277374527, 'test-top3': 36.18710838141324, 'test_cluster-top1': 19.47391277374527, 'test_cluster-top3': 36.18710838141324}, 'elapsed_time': 2927.6191284656525}, {'msg': 'Model aggregation done!\nGlobal model written to persistent storage.', 'test_result': {'test-top1': 14.628699050809603, 'test-top3': 29.331844407221293, 'test_cluster-top1': 14.628699050809603, 'test_cluster-top3': 29.331844407221293}, 'elapsed_time': 2987.014724254608}]

results/5_clients_1_frame/extract_results.py

@@ -0,0 +1 @@
+results = {0: {'test-top1': 17.462779156327542, 'test-top3': 39.73325062034739}, 1: {'test-top1': 31.674937965260547, 'test-top3': 57.411083746898264}, 2: {'test-top1': 30.43424317617866, 'test-top3': 55.27364205955335}, 3: {'test-top1': 34.026054590570716, 'test-top3': 57.44485794044665}, 4: {'test-top1': 38.65177853598015, 'test-top3': 60.79955893300248}, 5: {'test-top1': 37.68196836228288, 'test-top3': 60.16060111662531}, 6: {'test-top1': 40.578301488833745, 'test-top3': 62.80328101736973}, 7: {'test-top1': 35.774745037220846, 'test-top3': 57.41315136476427}, 8: {'test-top1': 41.339950372208435, 'test-top3': 63.39261104218362}, 9: {'test-top1': 35.57692307692308, 'test-top3': 59.23697270471464}, 10: {'test-top1': 40.421836228287845, 'test-top3': 60.803004962779156}, 11: {'test-top1': 36.32754342431762, 'test-top3': 60.45147518610422}, 12: {'test-top1': 31.477116004962777, 'test-top3': 56.06010483870968}, 13: {'test-top1': 38.22994230769231, 'test-top3': 61.11042183622829}, 14: {'test-top1': 37.96526054590571, 'test-top3': 59.4044665012407}, 15: {'test-top1': 40.898125310173704, 'test-top3': 62.81637717121588}, 16: {'test-top1': 40.259856699751865, 'test-top3': 61.74937965260546}, 17: {'test-top1': 39.83802047146402, 'test-top3': 62.20774751861042}, 18: {'test-top1': 41.177970843672455, 'test-top3': 62.697821960297766}, 19: {'test-top1': 41.43300248138958, 'test-top3': 62.70402543424317}, 20: {'test-top1': 39.312103598014886, 'test-top3': 61.38888895781638}, 21: {'test-top1': 39.10876736972705, 'test-top3': 62.74193548387097}, 22: {'test-top1': 40.607940446650126, 'test-top3': 62.25737531017369}, 23: {'test-top1': 40.12062282878412, 'test-top3': 62.83498759305211}, 24: {'test-top1': 41.67149131513648, 'test-top3': 62.23325062034739}, 25: {'test-top1': 40.607940446650126, 'test-top3': 63.49875930521092}, 26: {'test-top1': 42.42831513647643, 'test-top3': 63.455334987593055}, 27: {'test-top1': 38.50565198511166, 'test-top3': 60.7678523573201}, 28: {'test-top1': 41.30893300248139, 'test-top3': 62.03473945409429}, 29: {'test-top1': 39.88144478908189, 'test-top3': 61.178660049627794}, 30: {'test-top1': 40.17162965260546, 'test-top3': 61.22139516129032}, 31: {'test-top1': 39.565067617866006, 'test-top3': 61.525365384615384}, 32: {'test-top1': 40.42252543424318, 'test-top3': 61.364764267990076}, 33: {'test-top1': 41.79900744416874, 'test-top3': 62.03405024813895}, 34: {'test-top1': 40.53349875930521, 'test-top3': 62.86531575682382}, 35: {'test-top1': 40.71960297766749, 'test-top3': 61.31237965260546}, 36: {'test-top1': 40.931210297766754, 'test-top3': 62.54273511166253}, 37: {'test-top1': 39.88833746898263, 'test-top3': 62.03473945409429}, 38: {'test-top1': 39.844913151364764, 'test-top3': 62.52757071960298}, 39: {'test-top1': 39.62434491315137, 'test-top3': 61.74937965260546}, 40: {'test-top1': 40.58450496277916, 'test-top3': 61.38888895781638}, 41: {'test-top1': 41.53363647642679, 'test-top3': 62.10228846153846}, 42: {'test-top1': 43.15481141439206, 'test-top3': 63.31816935483871}, 43: {'test-top1': 41.36752109181142, 'test-top3': 63.03280955334987}, 44: {'test-top1': 39.59057071960298, 'test-top3': 61.12834305210918}, 45: {'test-top1': 42.69024007444169, 'test-top3': 64.26178660049628}, 46: {'test-top1': 39.6712158808933, 'test-top3': 61.59980707196029}, 47: {'test-top1': 41.07044416873449, 'test-top3': 61.934794665012404}, 48: {'test-top1': 41.068375930521086, 'test-top3': 63.03901302729528}, 49: {'test-top1': 41.15177853598015, 'test-top3': 63.026606079404466}, 50: {'test-top1': 41.15177853598015, 'test-top3': 63.026606079404466, 'test-top1-clustering': 41.15177853598015, 'test-top3-clustering': 63.026606079404466}}