Skip to content

Commit 83df238

Browse files
markzegarellipwseg
markzegarelli
and
pwseg
authored
Catalog Script Refactor (#5058)
* trigger a build * Refactor....complete --------- Co-authored-by: pwseg <[email protected]>
1 parent 3fbedd0 commit 83df238

12 files changed

+669
-491
lines changed

‎Makefile

+2-2
Original file line numberDiff line numberDiff line change
@@ -56,7 +56,7 @@ serve: package
5656
@docker run -p 4000:80 segment-docs:latest
5757

5858
catalog:
59-
@node scripts/catalog_papi.js
59+
@node scripts/catalog/index.js
6060

6161
# make the list of beta connections
6262
.PHONY: beta
@@ -163,7 +163,7 @@ docker-build:
163163

164164
.PHONY: private-destination
165165
private_destination:
166-
@node scripts/private-destination.js
166+
@node scripts/catalog/addPrivateDestination.js
167167
#.PHONY: docs
168168
#docs: node_modules
169169
# $(BIN)/webpack --mode=production

‎netlify.toml

+1-1
Original file line numberDiff line numberDiff line change
@@ -14,7 +14,7 @@
1414

1515
[context.branch-deploy]
1616
command = "yarn build"
17-
# ignore = "./scripts/ignore.sh"
17+
# ignore = "./scripts/ignore.sh"
1818

1919
[context.develop]
2020
command = "yarn develop"

‎scripts/catalog/addPrivateDestination.js

+35
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,35 @@
1+
const path = require('path');
2+
const fs = require('fs');
3+
const { prompt } = require('enquirer');
4+
const {
5+
getDestinationData,
6+
checkExistingStatus
7+
} = require('./utilities_private_destinations.js');
8+
9+
const addPrivateDestination = async () => {
10+
let ids = await checkExistingStatus();
11+
ids.sort();
12+
13+
const DEST_ID = await prompt({
14+
type: 'input',
15+
name: 'id',
16+
message: 'Enter the destination ID'
17+
});
18+
19+
20+
if (ids.includes(DEST_ID.id)) {
21+
console.log("This destination is already captured.");
22+
return;
23+
} else {
24+
ids.push(DEST_ID.id);
25+
fs.writeFileSync(path.resolve(__dirname, `../../src/_data/catalog/destinations_private.yml`), '');
26+
}
27+
ids.sort();
28+
29+
for (const element in ids) {
30+
let currentId = ids[element];
31+
await getDestinationData(currentId);
32+
}
33+
};
34+
35+
addPrivateDestination();

‎scripts/catalog/catalog_papi.js

+95
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,95 @@
1+
const path = require('path');
2+
const fs = require('fs');
3+
const yaml = require('js-yaml');
4+
const {
5+
slugify,
6+
getCatalog,
7+
getConnectionModes,
8+
isCatalogItemHidden,
9+
sanitize,
10+
doesCatalogItemExist
11+
} = require('./utilities.js');
12+
13+
require('dotenv').config();
14+
15+
const PAPI_URL = "https://api.segmentapis.com";
16+
17+
const regionalSupport = yaml.load(fs.readFileSync(path.resolve(__dirname, `../src/_data/regional-support.yml`)));
18+
19+
// This file keeps a list of known test sources that show up in the system.
20+
// Because we don't have a status value for sources, they end up showing in our catalog.
21+
// We use this below to prevent them from being written to yaml.
22+
const testSources = yaml.load(fs.readFileSync(path.resolve(__dirname, `../src/_data/catalog/test_sources.yml`)));
23+
24+
25+
26+
const updateWarehouses = async () => {
27+
let warehouses = [];
28+
let nextPageToken = "MA==";
29+
let warehousesUpdated = [];
30+
31+
while (nextPageToken !== undefined) {
32+
const res = await getCatalog(`${PAPI_URL}/catalog/warehouses/`, nextPageToken);
33+
warehouses = warehouses.concat(res.data.warehousesCatalog);
34+
nextPageToken = res.data.pagination.next;
35+
}
36+
37+
warehouses.sort((a, b) => {
38+
if (a.name.toLowerCase() < b.name.toLowerCase()) {
39+
return -1;
40+
}
41+
if (a.name.toLowerCase() > b.name.toLowerCase()) {
42+
return 1;
43+
}
44+
return 0;
45+
});
46+
47+
const regionalWarehouseEndpoints = regionalSupport.warehouses.endpoint;
48+
const regionalWarehouseRegions = regionalSupport.warehouses.region;
49+
50+
warehouses.forEach(warehouse => {
51+
let slug = slugify(warehouse.slug);
52+
let endpoints = ['us'];
53+
let regions = ['us'];
54+
let url = `connections/storage/catalog/${slug}`;
55+
56+
if (regionalWarehouseEndpoints.includes(slug)) {
57+
endpoints.push('eu');
58+
}
59+
60+
if (regionalWarehouseRegions.includes(slug)) {
61+
regions.push('eu');
62+
}
63+
64+
let updatedWarehouse = {
65+
id: warehouse.id,
66+
display_name: warehouse.name,
67+
url,
68+
slug,
69+
endpoints,
70+
regions
71+
};
72+
73+
warehousesUpdated.push(updatedWarehouse);
74+
});
75+
76+
const options = {
77+
noArrayIndent: true
78+
};
79+
const todayDate = new Date().toISOString().slice(0, 10);
80+
81+
// Create regional support YAML file
82+
let output = "# AUTOGENERATED LIST OF CONNECTIONS THAT SUPPORT REGIONAL\n";
83+
output += "# Last updated " + todayDate + " \n";
84+
output += yaml.dump({
85+
warehouses: warehousesUpdated
86+
}, options);
87+
fs.writeFileSync(path.resolve(__dirname, `../src/_data/catalog/regional-supported.yml`), output);
88+
89+
console.log("warehouses done");
90+
};
91+
92+
// Execute the update functions
93+
updateWarehouses();
94+
updateSources();
95+
updateDestinations();

‎scripts/catalog/index.js

+17
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,17 @@
1+
const {updateSources} = require('./updateSources.js');
2+
const {updateDestinations} = require('./updateDestinations.js');
3+
const {updateWarehouses} = require('./updateWarehouses.js');
4+
const {updatePrivateDestinations} = require('./updatePrivateDestinations.js');
5+
6+
updateSources();
7+
updateWarehouses();
8+
9+
// Wait for the main catalog to update before updating the private destinations
10+
async function destinations() {
11+
await updateDestinations()
12+
updatePrivateDestinations();
13+
}
14+
15+
16+
17+
destinations();

‎scripts/catalog/updateDestinations.js

+195
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,195 @@
1+
const path = require('path');
2+
const fs = require('fs');
3+
const yaml = require('js-yaml');
4+
const {
5+
slugify,
6+
getCatalog,
7+
getConnectionModes,
8+
isCatalogItemHidden,
9+
sanitize,
10+
doesCatalogItemExist
11+
} = require('./utilities.js');
12+
13+
require('dotenv').config();
14+
15+
const PAPI_URL = "https://api.segmentapis.com";
16+
17+
18+
const updateDestinations = async () => {
19+
let destinations = [];
20+
let destinationsUpdated = [];
21+
let destinationCategories = [];
22+
let categories = new Set();
23+
let nextPageToken = "MA==";
24+
25+
// Get all destinations from the Public API
26+
while (nextPageToken !== undefined) {
27+
const res = await getCatalog(`${PAPI_URL}/catalog/destinations/`, nextPageToken);
28+
destinations = destinations.concat(res.data.destinationsCatalog);
29+
nextPageToken = res.data.pagination.next;
30+
}
31+
32+
// Sort the destinations alphabetically
33+
destinations.sort((a, b) => {
34+
if (a.name.toLowerCase() < b.name.toLowerCase()) {
35+
return -1;
36+
}
37+
if (a.name.toLowerCase() > b.name.toLowerCase()) {
38+
return 1;
39+
}
40+
return 0;
41+
});
42+
43+
// Loop through all destinations and create a new object with the data we want
44+
destinations.forEach(destination => {
45+
let endpoints = [];
46+
let regions = [];
47+
48+
let slug = slugify(destination.name, "destinations");
49+
50+
if (typeof destination.supportedRegions != "undefined") {
51+
regions = destination.supportedRegions;
52+
} else {
53+
regions.push('us-west-2', 'eu-west-1');
54+
}
55+
56+
if (typeof destination.regionEndpoints != "undefined") {
57+
endpoints = destination.regionEndpoints;
58+
} else {
59+
endpoints.push('US');
60+
}
61+
62+
let url = `connections/destinations/catalog/${slug}`;
63+
64+
let tempCategories = [destination.categories];
65+
tempCategories = tempCategories.filter(category => category != '');
66+
tempCategories = tempCategories.flat();
67+
68+
let connection_modes = getConnectionModes({
69+
components: destination.components,
70+
platforms: destination.supportedPlatforms,
71+
browserUnbundling: destination.supportedFeatures.browserUnbundling,
72+
browserUnbundlingPublic: destination.supportedFeatures.browserUnbundlingPublic,
73+
methods: destination.supportedMethods
74+
});
75+
76+
let settings = destination.options;
77+
78+
settings.sort((a, b) => {
79+
if (a.name.toLowerCase() < b.name.toLowerCase()) {
80+
return -1;
81+
}
82+
if (a.name.toLowerCase() > b.name.toLowerCase()) {
83+
return 1;
84+
}
85+
return 0;
86+
});
87+
88+
settings.forEach(setting => {
89+
setting.description = sanitize(setting.description);
90+
});
91+
92+
let actions = destination.actions;
93+
let presets = destination.presets;
94+
95+
const clone = (obj) => Object.assign({}, obj);
96+
const renameKey = (object, key, newKey) => {
97+
const clonedObj = clone(object);
98+
const targetKey = clonedObj[key];
99+
delete clonedObj[key];
100+
101+
clonedObj[newKey] = targetKey;
102+
return clonedObj;
103+
};
104+
105+
// I honestly don't remember why I did this.
106+
// I think someone wanted to mention support for the Screen method to whatever destination that is
107+
destination.supportedMethods.screen = false;
108+
if (destination.id == '63e42b47479274407b671071') {
109+
destination.supportedMethods.screen = true;
110+
}
111+
112+
// Pageview is renamed to Page
113+
destination.supportedMethods = renameKey(destination.supportedMethods, 'pageview', 'page');
114+
115+
// All updated destination information gets added to this object
116+
let updatedDestination = {
117+
id: destination.id,
118+
display_name: destination.name,
119+
name: destination.name,
120+
slug,
121+
hidden: isCatalogItemHidden(url),
122+
endpoints,
123+
regions,
124+
url,
125+
previous_names: destination.previousNames,
126+
website: destination.website,
127+
status: destination.status,
128+
categories: tempCategories,
129+
logo: {
130+
url: destination.logos.default
131+
},
132+
mark: {
133+
url: destination.logos.mark
134+
},
135+
methods: destination.supportedMethods,
136+
platforms: destination.supportedPlatforms,
137+
components: destination.components,
138+
browserUnbundlingSupported: destination.supportedFeatures.browserUnbundling,
139+
browserUnbundlingPublic: destination.supportedFeatures.browserUnbundlingPublic,
140+
replay: destination.supportedFeatures.replay,
141+
connection_modes,
142+
settings,
143+
actions,
144+
presets
145+
};
146+
147+
// Add the updated destination to the destinationsUpdated array
148+
destinationsUpdated.push(updatedDestination);
149+
doesCatalogItemExist(updatedDestination);
150+
tempCategories.reduce((s, e) => s.add(e), categories);
151+
});
152+
153+
const destinationArray = Array.from(categories);
154+
destinationArray.forEach(category => {
155+
destinationCategories.push({
156+
display_name: category,
157+
slug: slugify(category)
158+
});
159+
destinationCategories.sort((a, b) => {
160+
if (a.display_name.toLowerCase() < b.display_name.toLowerCase()) {
161+
return -1;
162+
}
163+
if (a.display_name.toLowerCase() > b.display_name.toLowerCase()) {
164+
return 1;
165+
}
166+
return 0;
167+
});
168+
});
169+
170+
const options = {
171+
noArrayIndent: true
172+
};
173+
const todayDate = new Date().toISOString().slice(0, 10);
174+
175+
// Create destination catalog YAML file
176+
let output = "# AUTOGENERATED FROM PUBLIC API. DO NOT EDIT\n";
177+
output += "# destination data last updated " + todayDate + " \n";
178+
output += yaml.dump({
179+
items: destinationsUpdated
180+
}, options);
181+
fs.writeFileSync(path.resolve(__dirname, `../../src/_data/catalog/destinations.yml`), output);
182+
183+
// Create destination-category mapping YAML file
184+
output = "# AUTOGENERATED FROM PUBLIC API. DO NOT EDIT\n";
185+
output += "# destination categories last updated " + todayDate + " \n";
186+
output += yaml.dump({
187+
items: destinationCategories
188+
}, options);
189+
fs.writeFileSync(path.resolve(__dirname, `../../src/_data/catalog/destination_categories.yml`), output);
190+
191+
console.log("destinations done");
192+
};
193+
194+
195+
exports.updateDestinations = updateDestinations;

‎scripts/catalog/updatePrivateDestinations.js

+17
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,17 @@
1+
const {
2+
getDestinationData,
3+
checkExistingStatus
4+
} = require('./utilities_private_destinations.js');
5+
6+
const updatePrivateDestinations = async () => {
7+
let ids = await checkExistingStatus();
8+
ids.sort();
9+
10+
for (const element in ids) {
11+
let currentId = ids[element];
12+
await getDestinationData(currentId);
13+
}
14+
console.log("private destinations done");
15+
};
16+
17+
exports.updatePrivateDestinations = updatePrivateDestinations;

0 commit comments

Comments
 (0)