Skip to content

Commit 7b16bf1

Browse files
esterTionesterTion
committed
bang dream tools: optimize memory usage
1 parent 0feb02a commit 7b16bf1

File tree

3 files changed

+52
-199
lines changed

3 files changed

+52
-199
lines changed

‎bang/SuiteMaster_gen.proto

-157
Original file line numberDiff line numberDiff line change
@@ -25,20 +25,6 @@ message uint_MasterTimerWordingCollection {
2525
message MasterTimerWordingCollectionMap {
2626
repeated uint_MasterTimerWordingCollection entries = 1; // dictionary
2727
}
28-
message MasterMissionLabel {
29-
optional uint32 labelId = 1; // normal type
30-
optional string missionTabType = 2; // normal type
31-
optional string tabLabel = 3; // normal type
32-
optional uint64 startAt = 4; // normal type
33-
optional uint64 endAt = 5; // normal type
34-
}
35-
message uint_MasterMissionLabel {
36-
optional uint32 uint_MasterMissionLabel_key = 1; // normal type
37-
optional MasterMissionLabel uint_MasterMissionLabel_value = 2; // sub class
38-
}
39-
message MasterMissionLabelMap {
40-
repeated uint_MasterMissionLabel entries = 1; // dictionary
41-
}
4228
message MasterMatchingBonus {
4329
optional uint32 matchingBonusId = 1; // normal type
4430
optional string matchingBonusName = 2; // normal type
@@ -62,23 +48,6 @@ message MasterMatchingBonusReward {
6248
message MasterMatchingBonusRewardList {
6349
repeated MasterMatchingBonusReward entries = 1; // array
6450
}
65-
message MasterMissionGroup {
66-
optional uint32 missionGroupId = 1; // normal type
67-
optional string missionTabType = 2; // normal type
68-
optional string missionResetType = 3; // normal type
69-
optional string assetBundleName = 4; // normal type
70-
optional string url = 5; // normal type
71-
optional uint64 startAt = 6; // normal type
72-
optional uint64 endAt = 7; // normal type
73-
optional uint64 rewardEndAt = 8; // normal type
74-
}
75-
message uint_MasterMissionGroup {
76-
optional uint32 uint_MasterMissionGroup_key = 1; // normal type
77-
optional MasterMissionGroup uint_MasterMissionGroup_value = 2; // sub class
78-
}
79-
message MasterMissionGroupMap {
80-
repeated uint_MasterMissionGroup entries = 1; // dictionary
81-
}
8251
message MasterBandStoryIntroduction {
8352
optional uint32 bandStoryChapterId = 1; // normal type
8453
optional string introduction = 2; // normal type
@@ -1223,71 +1192,6 @@ message MasterSituationSkillExpTable {
12231192
message MasterSituationSkillExpTableList {
12241193
repeated MasterSituationSkillExpTable entries = 1; // array
12251194
}
1226-
message MasterMission {
1227-
optional uint32 missionId = 1; // normal type
1228-
optional string missionType = 2; // normal type
1229-
optional string transitionMethod = 3; // normal type
1230-
optional uint32 transitionId = 4; // normal type
1231-
optional uint64 startAt = 5; // normal type
1232-
optional uint64 endAt = 6; // normal type
1233-
optional string missionTabType = 7; // normal type
1234-
optional uint32 seq = 8; // normal type
1235-
optional string url = 9; // normal type
1236-
optional uint32 eventId = 10; // normal type
1237-
optional bool isSpecial = 11; // normal type
1238-
optional uint32 bandId = 12; // normal type
1239-
optional string characterAttribute = 13; // normal type
1240-
repeated uint32 areaItemCategoryIds = 14; // array
1241-
optional string resetType = 15; // normal type
1242-
optional uint32 missionGroupId = 16; // normal type
1243-
}
1244-
message uint_MasterMission {
1245-
optional uint32 uint_MasterMission_key = 1; // normal type
1246-
optional MasterMission uint_MasterMission_value = 2; // sub class
1247-
}
1248-
message MasterMissionMap {
1249-
repeated uint_MasterMission entries = 1; // dictionary
1250-
}
1251-
message MasterMissionDetail {
1252-
optional uint32 missionId = 1; // normal type
1253-
optional uint32 seq = 2; // normal type
1254-
optional string title = 3; // normal type
1255-
optional string description = 4; // normal type
1256-
optional uint32 maxProgress = 5; // normal type
1257-
optional string missionConditionTypes = 6; // normal type
1258-
repeated uint32 values = 7; // array
1259-
optional string hint = 8; // normal type
1260-
optional string releaseConditionDescription = 9; // normal type
1261-
optional uint32 releaseRank = 10; // normal type
1262-
}
1263-
message MasterMissionDetailList {
1264-
repeated MasterMissionDetail entries = 1; // array
1265-
}
1266-
message uint_MasterMissionDetailList {
1267-
optional uint32 uint_MasterMissionDetailList_key = 1; // normal type
1268-
optional MasterMissionDetailList uint_MasterMissionDetailList_value = 2; // sub class
1269-
}
1270-
message MasterMissionDetailMap {
1271-
repeated uint_MasterMissionDetailList entries = 1; // dictionary
1272-
}
1273-
message MasterMissionReward {
1274-
optional uint32 missionId = 1; // normal type
1275-
optional uint32 seq = 2; // normal type
1276-
optional uint32 missionRewardId = 3; // normal type
1277-
optional string resourceType = 4; // normal type
1278-
optional uint32 resourceId = 5; // normal type
1279-
optional uint32 quantity = 6; // normal type
1280-
}
1281-
message MasterMissionRewardList {
1282-
repeated MasterMissionReward entries = 1; // array
1283-
}
1284-
message uint_MasterMissionRewardList {
1285-
optional uint32 uint_MasterMissionRewardList_key = 1; // normal type
1286-
optional MasterMissionRewardList uint_MasterMissionRewardList_value = 2; // sub class
1287-
}
1288-
message MasterMissionRewardMap {
1289-
repeated uint_MasterMissionRewardList entries = 1; // dictionary
1290-
}
12911195
message MasterSkillActivateEffect {
12921196
optional uint32 skillId = 1; // normal type
12931197
optional uint32 skillLevel = 2; // normal type
@@ -1867,16 +1771,9 @@ message MasterMusicVideo {
18671771
optional string thumbAssetBundleName = 8; // normal type
18681772
optional string description = 9; // normal type
18691773
}
1870-
message uint_MasterMusicVideo {
1871-
optional uint32 uint_MasterMusicVideo_key = 1; // normal type
1872-
optional MasterMusicVideo uint_MasterMusicVideo_value = 2; // sub class
1873-
}
18741774
message MasterMusicVideoList {
18751775
repeated MasterMusicVideo entries = 1; // array
18761776
}
1877-
message MasterMusicVideoMap {
1878-
repeated uint_MasterMusicVideo entries = 1; // dictionary
1879-
}
18801777
message uint_MasterMusicVideoList {
18811778
optional uint32 uint_MasterMusicVideoList_key = 1; // normal type
18821779
optional MasterMusicVideoList uint_MasterMusicVideoList_value = 2; // sub class
@@ -1995,30 +1892,6 @@ message uint_MasterCollaborationSkin {
19951892
message MasterCollaborationSkinMap {
19961893
repeated uint_MasterCollaborationSkin entries = 1; // dictionary
19971894
}
1998-
message MasterMissionPlatformOverview {
1999-
optional uint32 missionId = 1; // normal type
2000-
optional uint32 seq = 2; // normal type
2001-
optional string platform = 3; // normal type
2002-
optional string missionTitle = 4; // normal type
2003-
optional string missionDescription = 5; // normal type
2004-
}
2005-
message MasterMissionPlatformOverviewList {
2006-
repeated MasterMissionPlatformOverview entries = 1; // array
2007-
}
2008-
message uint_MasterMissionPlatformOverviewList {
2009-
optional uint32 uint_MasterMissionPlatformOverviewList_key = 1; // normal type
2010-
optional MasterMissionPlatformOverviewList uint_MasterMissionPlatformOverviewList_value = 2; // sub class
2011-
}
2012-
message MasterMissionPlatformOverviewMap {
2013-
repeated uint_MasterMissionPlatformOverviewList entries = 1; // dictionary
2014-
}
2015-
message string_MasterMissionPlatformOverviewMap {
2016-
optional string string_MasterMissionPlatformOverviewMap_key = 1; // normal type
2017-
optional MasterMissionPlatformOverviewMap string_MasterMissionPlatformOverviewMap_value = 2; // sub class
2018-
}
2019-
message MasterMissionPlatformOverviewPlatformMap {
2020-
repeated string_MasterMissionPlatformOverviewMap entries = 1; // dictionary
2021-
}
20221895
message MasterNewYearCard {
20231896
optional uint32 newYearCardId = 1; // normal type
20241897
optional uint64 selectStartAt = 2; // normal type
@@ -3296,27 +3169,6 @@ message MasterBirthdayIntroduction {
32963169
message MasterBirthdayIntroductionList {
32973170
repeated MasterBirthdayIntroduction entries = 1; // list
32983171
}
3299-
message MasterAprilFool2021Page {
3300-
optional uint32 aprilFool2021PageId = 1; // normal type
3301-
optional string assetBundleName = 2; // normal type
3302-
optional string bgmFileName = 3; // normal type
3303-
optional uint64 startAt = 4; // normal type
3304-
optional uint64 endAt = 5; // normal type
3305-
}
3306-
message MasterAprilFool2021PageList {
3307-
repeated MasterAprilFool2021Page entries = 1; // list
3308-
}
3309-
message MasterAprilFool2021Region {
3310-
optional uint32 aprilFool2021RegionId = 1; // normal type
3311-
optional string region = 2; // normal type
3312-
optional string storyType = 3; // normal type
3313-
optional uint32 seq = 4; // normal type
3314-
optional uint32 characterId = 5; // normal type
3315-
optional uint32 genericStoryId = 6; // normal type
3316-
}
3317-
message MasterAprilFool2021RegionList {
3318-
repeated MasterAprilFool2021Region entries = 1; // list
3319-
}
33203172
message MasterFestivalRotationMusic {
33213173
optional uint32 id = 1; // normal type
33223174
optional uint32 eventId = 2; // normal type
@@ -3626,10 +3478,8 @@ message MasterSpecialLotteryCharacterMap {
36263478
message SuiteMasterGetResponse {
36273479
optional MasterSubscriptionCountBonusRewardList masterSubscriptionCountBonusRewardList = 307; // sub class
36283480
optional MasterTimerWordingCollectionMap masterTimerWordingCollectionMap = 309; // sub class
3629-
optional MasterMissionLabelMap masterMissionLabelMap = 310; // sub class
36303481
optional MasterMatchingBonusMap masterMatchingBonusMap = 313; // sub class
36313482
optional MasterMatchingBonusRewardList masterMatchingBonusRewardList = 314; // sub class
3632-
optional MasterMissionGroupMap masterMissionGroupMap = 315; // sub class
36333483
optional MasterBandStoryIntroductionList masterBandStoryIntroductionList = 311; // sub class
36343484
optional MasterEventStoryIntroductionList masterEventStoryIntroductionList = 312; // sub class
36353485
optional MasterIngameV2TalkingMap masterIngameV2TalkingMap = 316; // sub class
@@ -3689,9 +3539,6 @@ message SuiteMasterGetResponse {
36893539
optional MasterCommonConfigMap masterCommonConfigMap = 50; // sub class
36903540
optional MasterSkillList masterSkillList = 51; // sub class
36913541
optional MasterSituationSkillExpTableList masterSituationSkillExpTableList = 52; // sub class
3692-
optional MasterMissionMap masterMissionMap = 53; // sub class
3693-
optional MasterMissionDetailMap masterMissionDetailMap = 54; // sub class
3694-
optional MasterMissionRewardMap masterMissionRewardMap = 55; // sub class
36953542
optional MasterSkillActivateEffectList masterSkillActivateEffectList = 56; // sub class
36963543
optional MasterSkillOnceEffectList masterSkillOnceEffectList = 57; // sub class
36973544
optional MasterLiveBoostRecoveryItemMap masterLiveBoostRecoveryItemMap = 58; // sub class
@@ -3721,15 +3568,13 @@ message SuiteMasterGetResponse {
37213568
optional MasterSingleFrameCartoonCharacterMap masterSingleFrameCartoonCharacterMap = 83; // sub class
37223569
optional MasterFourFrameCartoonMap masterFourFrameCartoonMap = 84; // sub class
37233570
optional MasterFourFrameCartoonCharacterMap masterFourFrameCartoonCharacterMap = 85; // sub class
3724-
optional MasterMusicVideoMap masterMusicVideoMap = 86; // sub class
37253571
optional MasterMusicVideoListMap masterMusicVideoListMap = 109; // sub class
37263572
optional MasterPurchaseMenuList masterPurchaseMenuList = 110; // sub class
37273573
optional MasterBandStoryChapterMap masterBandStoryChapterMap = 87; // sub class
37283574
optional MasterNewMusicIntroductionMap masterNewMusicIntroductionMap = 89; // sub class
37293575
optional MasterSituationFirstAppearanceMap masterSituationFirstAppearanceMap = 90; // sub class
37303576
optional MasterCollaborationSkinMap masterCollaborationSkinMap = 91; // sub class
37313577
optional MasterGenericStoryMap masterOpeningAreaGenericStoryMap = 92; // sub class
3732-
optional MasterMissionPlatformOverviewPlatformMap masterMissionPlatformOverviewMap = 93; // sub class
37333578
optional MasterNewYearCardMap masterNewYearCardMap = 94; // sub class
37343579
optional MasterMusicJacketMap masterMusicJacketMap = 95; // sub class
37353580
optional MasterBackstageTalkSetMap masterBackstageTalkSetMap = 96; // sub class
@@ -3798,8 +3643,6 @@ message SuiteMasterGetResponse {
37983643
optional MasterPanelMissionMap masterPanelMissionMap = 700; // sub class
37993644
optional MasterPanelMissionValueList masterPanelMissionValueList = 701; // sub class
38003645
optional MasterBirthdayIntroductionList masterBirthdayIntroductionList = 703; // sub class
3801-
optional MasterAprilFool2021PageList masterAprilFool2021PageList = 704; // sub class
3802-
optional MasterAprilFool2021RegionList masterAprilFool2021RegionList = 705; // sub class
38033646
optional MasterFestivalRotationMusicList masterFestivalRotationMusicList = 807; // sub class
38043647
optional MasterFestivalStageList masterFestivalStageList = 808; // sub class
38053648
optional MasterFestivalStageDescriptionList masterFestivalStageDescriptionList = 813; // sub class

‎bang/main.php

+26-24
Original file line numberDiff line numberDiff line change
@@ -61,7 +61,7 @@ function setValue(&$arr, $key, &$val, $isArr) {
6161
}
6262
}
6363

64-
function parseProtoBuf($messageName, $end, &$pb, &$proto) {
64+
function parseProtoBuf($messageName, $end, &$pb, &$proto, $messageCallback = null, $level = 1) {
6565
$message = [];
6666
while ($pb->position() < $end){
6767
$id = readVarInt32($pb);
@@ -92,8 +92,10 @@ function parseProtoBuf($messageName, $end, &$pb, &$proto) {
9292
case 2:
9393
$length = readVarInt32($pb);
9494
if (isset($proto[$typeName])) {
95-
$sub = parseProtoBuf($typeName, $pb->position()+$length, $pb, $proto);
96-
setValue($message, $name, $sub, $isRepeated);
95+
$sub = parseProtoBuf($typeName, $pb->position()+$length, $pb, $proto, $messageCallback, $level + 1);
96+
if ($messageCallback === null || call_user_func($messageCallback, $name, $sub, $level)) {
97+
setValue($message, $name, $sub, $isRepeated);
98+
}
9799
} else if ($length > 0) {
98100
$data = $pb->readData($length);
99101
if ($typeName == 'string') {
@@ -153,7 +155,8 @@ function processDict(&$arr) {
153155
function prettifyJSON($in, Stream $out = NULL, $returnData = true) {
154156
if ($out == NULL) $out = new MemoryStream('');
155157

156-
$a = json_decode($in);
158+
$a = $in;
159+
if (gettype($a) == 'string') $a = json_decode($a);
157160
$a = json_encode($a, JSON_UNESCAPED_UNICODE+JSON_UNESCAPED_SLASHES+JSON_PRETTY_PRINT);
158161
$a = preg_replace("/([^\]\}]),\n +/", "$1, ", $a);
159162
$a = preg_replace('/("[^"]+?":) /', '$1', $a);
@@ -255,31 +258,30 @@ function prettifyJSON($in, Stream $out = NULL, $returnData = true) {
255258
$masterData = new FileStream('master.dat');
256259
//$masterData = new FileStream('master');
257260

258-
ini_set("memory_limit",256 *1024*1024);
259-
$MasterProto = parseProto('SuiteMaster_gen.proto');
260-
$master = parseProtoBuf('SuiteMasterGetResponse', $masterData->size, $masterData, $MasterProto);
261-
processDict($master);
262-
unset($masterData, $MasterProto);
263-
264-
$count = count(array_keys($master));
265-
_log("dumping master ${count} entries");
266-
267261
chdir('data');
268262
exec('git rm *.json --cached');
269263
chdir(__DIR__);
270264
foreach (glob('data/*.json') as $file) {$file!='data/AssetBundleInfo.json'&&unlink($file);}
271265

272-
$i = 0;
273-
foreach ($master as $part=>&$data) {
274-
//echo (++$i)."/$count $part\n";
275-
$json = json_encode($data, JSON_UNESCAPED_SLASHES+JSON_UNESCAPED_UNICODE);
276-
fclose(fopen("data/${part}.json", 'w'));
277-
prettifyJSON($json, new FileStream("data/${part}.json"), false);
278-
unset($json);
279-
}
266+
$MasterProto = parseProto('SuiteMaster_gen.proto');
267+
_log("dumping master");
268+
parseProtoBuf('SuiteMasterGetResponse', $masterData->size, $masterData, $MasterProto, function ($name, $sub, $level) {
269+
if ($level > 1) return true;
270+
$sub = ['a'=>$sub];
271+
processDict($sub);
272+
$sub = $sub['a'];
273+
if ($name === 'masterCharacterSituationMap') {
274+
global $situationMap;
275+
$situationMap = $sub;
276+
} else if ($name === 'masterCharacterInfoMap') {
277+
global $charaInfo;
278+
$charaInfo = $sub;
279+
}
280+
file_put_contents("data/${name}.json", prettifyJSON($sub));
281+
return false;
282+
});
283+
unset($masterData, $MasterProto);
280284

281-
$situationMap = &$master['masterCharacterSituationMap'];
282-
$charaInfo = &$master['masterCharacterInfoMap'];
283285
foreach ($situationMap as &$entry) {
284286
$id = substr($entry['resourceSetName'], 3);
285287
$names[$id] = str_repeat('★',$entry['rarity']).'['.$entry['prefix'].']'.$charaInfo[$entry['characterId']]['characterName'];
@@ -318,7 +320,7 @@ function prettifyJSON($in, Stream $out = NULL, $returnData = true) {
318320
foreach ($bundleInfo['bundles'] as &$item) {
319321
unset($item['version']);
320322
}
321-
file_put_contents('data/AssetBundleInfo.json', prettifyJSON(json_encode($bundleInfo, JSON_UNESCAPED_SLASHES+JSON_UNESCAPED_UNICODE)));
323+
file_put_contents('data/AssetBundleInfo.json', prettifyJSON($bundleInfo));
322324
unset($bundleInfoData, $bundleInfoProto, $bundleInfo);
323325
file_put_contents('data/!dataVersion.txt', $dataVer."\n");
324326
$commit[] = 'data: '.$dataVer;

0 commit comments

Comments
 (0)