Skip to content

Commit f461006

Browse files
JipyJipy
Jipy
authored and
Jipy
committed
Optmizations, need to rework the HttpClient in PokemonGoClient, the HttpClient class is terrible in multithreaded environments.
Scanner up to 10 times faster now.
1 parent 1367bad commit f461006

18 files changed

+183
-43
lines changed

‎.gitignore

+2-1
Original file line numberDiff line numberDiff line change
@@ -188,4 +188,5 @@ $RECYCLE.BIN/
188188

189189
#####
190190
# End of core ignore list, below put you custom 'per project' settings (patterns or path)
191-
#####
191+
#####
192+
*.json

‎Api/ClientExtensions/Map.cs

+16-12
Original file line numberDiff line numberDiff line change
@@ -15,28 +15,32 @@ namespace MandraSoft.PokemonGo.Api.ClientExtensions
1515
static public class Map
1616
{
1717

18-
static internal Request GetMapRequest(this PokemonGoClient client, IList<ulong> CellIds = null)
18+
static internal Request GetMapRequest(this PokemonGoClient client, IList<ulong> CellIds = null,double? latitude=null,double? longitude = null)
1919
{
20+
double requestedLat = latitude.HasValue ? latitude.Value : client.Latitude;
21+
double requestedLng = longitude.HasValue ? longitude.Value : client.Longitude;
2022
var customRequest = new GetMapObjectsMessage()
2123
{
22-
Latitude = client.Latitude,
23-
Longitude = client.Longitude
24+
Latitude = requestedLat,
25+
Longitude = requestedLng
2426
};
2527
if (CellIds == null)
26-
CellIds = S2Helper.GetNearbyCellIds(client.Latitude, client.Longitude);
28+
CellIds = S2Helper.GetNearbyCellIds(requestedLat, requestedLng);
2729
customRequest.CellId.Add(CellIds);
2830
customRequest.SinceTimestampMs.AddRange(client.MapManager.GetLastUpdatedTimestamp(CellIds));
2931
return new Request() { RequestType = RequestType.GetMapObjects, RequestMessage = customRequest.ToByteString() };
3032
}
31-
static public async Task UpdateMapObjects(this PokemonGoClient client,IList<ulong> CellIds = null)
33+
static public async Task UpdateMapObjects(this PokemonGoClient client,double? latitude = null,double? longitude = null,IList<ulong> CellIds = null)
3234
{
33-
34-
var res = await client._httpClient.GetResponses(client, true, client._apiUrl,
35-
client.GetMapRequest(CellIds),
36-
client.GetHatchedEggRequest(),
37-
client.GetInventoryRequest(),
38-
client.GetCheckAwardedBadgeRequest(),
39-
client.GetDownloadSettingsRequest());
35+
using (var httpClient = client.GetHttpClient())
36+
{
37+
var res = await httpClient.GetResponses(client, true, client._apiUrl,
38+
client.GetMapRequest(CellIds, latitude, longitude),
39+
client.GetHatchedEggRequest(),
40+
client.GetInventoryRequest(),
41+
client.GetCheckAwardedBadgeRequest(),
42+
client.GetDownloadSettingsRequest());
43+
}
4044
}
4145

4246
static internal Request GetPlayerUpdateRequest(this PokemonGoClient client)

‎Api/Extensions/HttpClientExtensions.cs

+57-2
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@
33
using POGOProtos.Networking.Requests;
44
using System;
55
using System.Collections.Generic;
6+
using System.Diagnostics;
67
using System.Linq;
78
using System.Net.Http;
89
using System.Text;
@@ -13,9 +14,19 @@ namespace MandraSoft.PokemonGo.Api.Extensions
1314
public static class HttpClientExtensions
1415
{
1516
private static bool Relogging = false;
17+
static int _Count = 0;
18+
static long _TotalElapsedNetwork = 0;
19+
static long _TotalElapsedHandling = 0;
1620
public static async Task<List<IMessage>> GetResponses(this HttpClient client, PokemonGoClient pogoClient, bool withAuthTicket, string url, params Request[] requests)
1721
{
22+
#if _INSTRUMENTING
23+
Stopwatch sw = new Stopwatch();
24+
sw.Start();
25+
#endif
1826
var response = await GetEnvelope(client, pogoClient, withAuthTicket, url, requests);
27+
#if _INSTRUMENTING
28+
_TotalElapsedNetwork += sw.ElapsedMilliseconds;
29+
#endif
1930
int retryCount = 0;
2031
while (response.Returns.Count != requests.Length)
2132
{
@@ -40,6 +51,10 @@ public static async Task<List<IMessage>> GetResponses(this HttpClient client, Po
4051
}
4152
if (retryCount > 5) throw new Exception("Client in a weird state");
4253
}
54+
#if _INSTRUMENTING
55+
sw.Reset();
56+
sw.Start();
57+
#endif
4358
var result = new List<IMessage>();
4459
for (var i = 0; i < requests.Length; i++)
4560
{
@@ -53,26 +68,66 @@ public static async Task<List<IMessage>> GetResponses(this HttpClient client, Po
5368
else result.Add(new POGOProtos.Networking.Responses.EchoResponse());
5469
}
5570
await pogoClient.HandleGenericResponses(result);
71+
#if _INSTRUMENTING
72+
_TotalElapsedHandling += sw.ElapsedMilliseconds;
73+
_Count++;
74+
if (_Count % 100 == 0)
75+
{
76+
Console.WriteLine("Average network time : " + (double)_TotalElapsedNetwork / (double)_Count + "ms");
77+
Console.WriteLine(" AuthTicket : " + (double)_TotalAuthTicket / (double)_Count + "ms");
78+
Console.WriteLine(" Request Serialization : " + (double)_TotalSerializationRequest / (double)_Count + "ms");
79+
Console.WriteLine(" Post Time : " + (double)_TotalPostTime / (double)_Count + "ms");
80+
Console.WriteLine(" ReadResponse : " + (double)_TotalReadResponse / (double)_Count + "ms");
81+
Console.WriteLine(" Deserialization : " + (double)_TotalDeserialization / (double)_Count + "ms");
82+
Console.WriteLine("Average handling time : " + (double)_TotalElapsedHandling / (double)_Count + "ms");
83+
}
84+
#endif
5685
return result;
5786
}
58-
87+
static private long _TotalAuthTicket = 0;
88+
static private long _TotalSerializationRequest = 0;
89+
static private long _TotalPostTime = 0;
90+
static private long _TotalReadResponse = 0;
91+
static private long _TotalDeserialization = 0;
5992
public static async Task<POGOProtos.Networking.Envelopes.ResponseEnvelope> GetEnvelope(this HttpClient client, PokemonGoClient pogoClient, bool withAuthTicket, string url, params Request[] requests)
6093
{
94+
#if _INSTRUMENTING
95+
Stopwatch sw = new Stopwatch();
96+
sw.Start();
97+
#endif
6198
//Check Session closed.
6299
if (withAuthTicket && pogoClient._authTicket.ExpireTimestampMs < (ulong)DateTime.UtcNow.ToUnixTime())
63100
{
64101
await pogoClient.LoginPtc();
65102
await pogoClient.SetServer();
66103
url = pogoClient._apiUrl;
67104
}
105+
#if _INSTRUMENTING
106+
_TotalAuthTicket += sw.ElapsedMilliseconds;
107+
sw.Restart();
108+
#endif
68109
var response = new POGOProtos.Networking.Envelopes.ResponseEnvelope();
69110
var requestEnvloppe = await pogoClient.GetRequest(withAuthTicket, requests);
70111
var data = requestEnvloppe.ToByteString();
112+
#if _INSTRUMENTING
113+
_TotalSerializationRequest += sw.ElapsedMilliseconds;
114+
sw.Restart();
115+
#endif
71116
var result = await client.PostAsync(url, new ByteArrayContent(data.ToByteArray()));
72-
117+
#if _INSTRUMENTING
118+
_TotalPostTime += sw.ElapsedMilliseconds;
119+
sw.Restart();
120+
#endif
73121
var responseData = await result.Content.ReadAsByteArrayAsync();
122+
#if _INSTRUMENTING
123+
_TotalReadResponse += sw.ElapsedMilliseconds;
124+
sw.Restart();
125+
#endif
74126
var codedStream = new CodedInputStream(responseData);
75127
response.MergeFrom(codedStream);
128+
#if _INSTRUMENTING
129+
_TotalDeserialization += sw.ElapsedMilliseconds;
130+
#endif
76131
return response;
77132
}
78133
}

‎Api/Helpers/RetryHandler.cs

+1-1
Original file line numberDiff line numberDiff line change
@@ -37,7 +37,7 @@ protected override async Task<HttpResponseMessage> SendAsync(
3737
System.Console.WriteLine($"retry request {request.RequestUri}");
3838
if (i < MaxRetries)
3939
{
40-
await Task.Delay(1000);
40+
await Task.Delay(2000);
4141
continue;
4242
}
4343
throw;

‎Api/Managers/MapsCellsManager.cs

+8-1
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,7 @@
1616
using System.Net.Http.Headers;
1717
using System.Threading;
1818
using System.Threading.Tasks;
19+
using System.Diagnostics;
1920

2021
namespace MandraSoft.PokemonGo.Api.Managers
2122
{
@@ -55,12 +56,14 @@ public MapsCellsManager()
5556
private SemaphoreSlim _semaphorePurge = new SemaphoreSlim(1, 1);
5657

5758
private DateTime _lastPurge = DateTime.MinValue;
59+
5860
/// <summary>
5961
/// Called by the GenericResponseHandler each time a response of type GetMapObjectsResponse is received.
6062
/// </summary>
6163
/// <param name="resp"></param>
6264
internal void UpdateMapCells(GetMapObjectsResponse resp)
6365
{
66+
6467
// Clearing expired Pokemons
6568
// Only 1 thread need to do it hence the _semaphore, not blocking if a thread is already doing it
6669
// the others will just go on.
@@ -87,7 +90,11 @@ internal void UpdateMapCells(GetMapObjectsResponse resp)
8790
}
8891
}
8992
// Making sure it's a success before going on with the treatment.
90-
if (resp.Status != MapObjectsStatus.Success) return;
93+
if (resp.Status != MapObjectsStatus.Success)
94+
{
95+
Console.WriteLine("Map response error");
96+
return;
97+
}
9198
// Converting POGProto to Poco ObjectList
9299
// Because when i tried to keep the state as POGProtos I kept having exception when adding/removing from RepeatedFields
93100
// Anyway all the serialization overhead was completely useless and a resource sink.

‎Api/PokemonGoClient.cs

+20-10
Original file line numberDiff line numberDiff line change
@@ -33,6 +33,9 @@ public class PokemonGoClient : IDisposable
3333
public delegate Task WalkCallback(PokemonGoClient client);
3434
public string Name { get; set; }
3535

36+
public string LoginUser { get; set; } = Configuration.Login;
37+
public string Password { get; set; } = Configuration.Password;
38+
3639
internal HttpClient _httpClient;
3740
internal string _token,_apiUrl;
3841
private AuthType _provider;
@@ -63,8 +66,12 @@ public PokemonGoClient(double lat, double lng) : this()
6366
public PokemonGoClient()
6467
{
6568
InitManagers();
66-
ClientCompressionHandler handler = new ClientCompressionHandler(new GZipCompressor(), new DeflateCompressor());
67-
_httpClient = new HttpClient(new RetryHandler(handler));
69+
_httpClient = GetHttpClient();
70+
}
71+
public HttpClient GetHttpClient()
72+
{
73+
ClientCompressionHandler handler = new ClientCompressionHandler(new GZipCompressor(), new DeflateCompressor());
74+
var _httpClient = new HttpClient(new RetryHandler(handler));
6875
_httpClient.DefaultRequestHeaders.TryAddWithoutValidation("User-Agent", "Niantic App");
6976
//"Dalvik/2.1.0 (Linux; U; Android 5.1.1; SM-G900F Build/LMY48G)");
7077
_httpClient.DefaultRequestHeaders.ExpectContinue = false;
@@ -73,6 +80,7 @@ public PokemonGoClient()
7380
_httpClient.DefaultRequestHeaders.TryAddWithoutValidation("Content-Type",
7481
"application/x-www-form-urlencoded");
7582
_httpClient.Timeout = new TimeSpan(0, 0, 10);
83+
return _httpClient;
7684
}
7785
public void InitManagers()
7886
{
@@ -90,33 +98,35 @@ public async Task Login()
9098
if (Configuration.AuthType == AuthType.PTC)
9199
{
92100
_provider = AuthType.PTC;
93-
_token = await PtcLogin.GetAccessToken(Configuration.Login, Configuration.Password);
101+
_token = await PtcLogin.GetAccessToken(LoginUser,Password);
94102
}
95103
else if (Configuration.AuthType == AuthType.Google)
96104
{
97105
_provider = AuthType.Google;
98-
_token = await GoogleLogin.LoginGoogle(Configuration.Login, Configuration.Password);
106+
_token = await GoogleLogin.LoginGoogle(LoginUser,Password);
99107
}
100108
}
101109
public async Task LoginPtc(string login,string password)
102110
{
103-
_token = await PtcLogin.GetAccessToken(login,password);
104-
_provider = AuthType.PTC;
111+
LoginUser = login;
112+
Password = password;
113+
await LoginPtc();
105114
}
106115
public async Task LoginPtc()
107116
{
108-
_token = await PtcLogin.GetAccessToken(Configuration.Login, Configuration.Password);
117+
_token = await PtcLogin.GetAccessToken(LoginUser,Password);
109118
_provider = AuthType.PTC;
110119
}
111120
public async Task LoginGoogle()
112121
{
113-
_token = await GoogleLogin.LoginGoogle(Configuration.Login, Configuration.Password);
122+
_token = await GoogleLogin.LoginGoogle(LoginUser,Password);
114123
_provider = AuthType.Google;
115124
}
116125
public async Task LoginGoogle(string username, string password)
117126
{
118-
_token = await GoogleLogin.LoginGoogle(username, password);
119-
_provider = AuthType.Google;
127+
LoginUser = username;
128+
Password = password;
129+
await LoginGoogle();
120130
}
121131

122132
public TimeSpan GetWalkingDuration(double lat, double lng, TravelingSpeed speed = TravelingSpeed.Walk)

‎Bot/Program.cs

+2-4
Original file line numberDiff line numberDiff line change
@@ -73,15 +73,13 @@ static public async Task TestHugeCellIds(PokemonGoClient client)
7373
{
7474
var cellIds = covering.Take(61).ToList();
7575
var origin = cellIds[30].ToLatLng();
76-
client.SetCoordinates(origin.LatDegrees, origin.LngDegrees);
77-
await client.UpdateMapObjects(cellIds.Select(x => x.Id).ToList());
76+
await client.UpdateMapObjects(origin.LatDegrees, origin.LngDegrees,cellIds.Select(x => x.Id).ToList());
7877
covering.RemoveRange(0, 61);
7978
}
8079
if (covering.Count % 2 != 1)
8180
covering.Add(covering.Last().Next);
8281
var origin2 = covering[(covering.Count - 1) / 2].ToLatLng();
83-
client.SetCoordinates(origin2.LatDegrees, origin2.LngDegrees);
84-
await client.UpdateMapObjects(covering.Select(x => x.Id).ToList());
82+
await client.UpdateMapObjects(origin2.LatDegrees, origin2.LngDegrees,covering.Select(x => x.Id).ToList());
8583

8684
}
8785
private static async Task ExecuteFarmingPokestops(PokemonGoClient client,int maxToDo = int.MaxValue)

‎Tools/PokeScanner/App.config

+8-4
Original file line numberDiff line numberDiff line change
@@ -1,13 +1,13 @@
11
<?xml version="1.0" encoding="utf-8"?>
22
<configuration>
33
<appSettings>
4-
<add key="WebUri" value="http://localhost/beta/" />
5-
<add key="Login" value="ptcMandrasoft" />
6-
<add key="Password" value="azerty" />
4+
<add key="WebUri" value="http://pokemongo.mandrasoft.fr/" />
5+
<add key="Login" value="login" />
6+
<add key="Password" value="password" />
77
<add key="AuthType" value="PTC" />
88
<add key="BoundsToScan" value="48.899352, 2.260842, 48.818289, 2.454376" /> <!-- 2 points to make a Rectangle (Lat,lnt,lat,lng)-->
99
<add key="JobsToLaunch" value="50" />
10-
<add key="WebDelay" value="15"/> <!-- In Seconds. -->
10+
<add key="WebDelay" value="15" /> <!-- In Seconds. Time before each post to the website. If you use it for real Time keep it at less than a minute. Otherwise just keep it low enough so that the packages are 2000/3000k pokemons on average. -->
1111
</appSettings>
1212
<runtime>
1313
<assemblyBinding xmlns="urn:schemas-microsoft-com:asm.v1">
@@ -19,6 +19,10 @@
1919
<assemblyIdentity name="Newtonsoft.Json" publicKeyToken="30ad4fe6b2a6aeed" culture="neutral" />
2020
<bindingRedirect oldVersion="0.0.0.0-9.0.0.0" newVersion="9.0.0.0" />
2121
</dependentAssembly>
22+
<dependentAssembly>
23+
<assemblyIdentity name="EntityFramework.MappingAPI" publicKeyToken="7ee2e825d201459e" culture="neutral" />
24+
<bindingRedirect oldVersion="0.0.0.0-6.1.0.9" newVersion="6.1.0.9" />
25+
</dependentAssembly>
2226
</assemblyBinding>
2327
</runtime>
2428
</configuration>

0 commit comments

Comments
 (0)