Added intermitent profiling (#541)

* Added profiling

* Don't collect data at peak times and cleanup after 200 samples have been collected

* Cleanup profiler after we've collected the data we need.

server.py

@@ -28,6 +28,7 @@
 from server.stats.game_stats_service import (
     AchievementService, EventService, GameStatsService
 )
+from server.timing import at_interval
 
 if __name__ == '__main__':
     args = docopt(__doc__, version='FAF Server')
@@ -74,6 +75,33 @@ def signal_handler(_sig, _frame):
 
         players_online = PlayerService(database)
 
+        if config.PROFILING_INTERVAL > 0:
+            logger.warning("Profiling enabled! This will create additional load.")
+            import cProfile
+            pr = cProfile.Profile()
+            profiled_count = 0
+            max_count = 300
+
+            @at_interval(config.PROFILING_INTERVAL, loop=loop)
+            async def run_profiler():
+                global profiled_count
+                global pr
+
+                if len(players_online) > 1000:
+                    return
+                elif profiled_count >= max_count:
+                    pr = None
+                    return
+
+                logger.info("Starting profiler")
+                pr.enable()
+                await asyncio.sleep(2)
+                pr.disable()
+                profiled_count += 1
+
+                logging.info("Done profiling %i/%i", profiled_count, max_count)
+                pr.dump_stats("profile.txt")
+
         twilio_nts = None
         if TWILIO_ACCOUNT_SID:
             twilio_nts = TwilioNTS()

server/config.py

@@ -10,6 +10,7 @@
 
 # Environment
 LOG_LEVEL = logging.getLevelName(os.getenv('LOG_LEVEL', 'DEBUG'))
+PROFILING_INTERVAL = int(os.getenv('PROFILING_INTERVAL', -1))
 
 # Credit to Axle for parameter changes, see: http://forums.faforever.com/viewtopic.php?f=45&t=11698#p119599
 # Optimum values for ladder here, using them for global as well.