feat: enhance parser_sitemap.js for pdf (#2252)

* chore: update gpt translate

* feat: enhance parser_sitemap.js for pdf

Author: Chasen-Zhang

scripts/parser_sitemap.js

@@ -1,51 +1,90 @@
+// https://www.princexml.com/ download princexml first
 const fs = require("fs");
-const xml2js = require("xml2js");
 const path = require("path");
+const https = require("https");
+const xml2js = require("xml2js");
+
 const site = process.env.site;
+
+if (!site || !["cn", "en"].includes(site)) {
+  process.exit(1);
+}
+
 const domain = site === "cn" ? "cn" : "com";
+const sitemapUrl = `https://docs.databend.${domain}/sitemap.xml`;
 const sitemapPath = path.join(__dirname, `sitemap-${site}.xml`);
-
 const outputFilePath = path.join("./", "pdf", `docs.databend.${site}-sql.txt`);
 
+const excludeUrls = [
+  `https://docs.databend.${domain}/`,
+  `https://docs.databend.${domain}/download/`,
+  `https://docs.databend.${domain}/search`,
+];
+
 if (!fs.existsSync(path.dirname(outputFilePath))) {
   fs.mkdirSync(path.dirname(outputFilePath), { recursive: true });
 }
 
 if (fs.existsSync(outputFilePath)) {
-  console.log(`file ${outputFilePath} existed, delete it...`);
+  console.log(`File ${outputFilePath} existed, delete it...`);
   fs.unlinkSync(outputFilePath);
 }
 
-const excludeUrls = [
-  `https://docs.databend.${domain}/`,
-  `https://docs.databend.${domain}/download/`,
-  `https://docs.databend.${domain}/search`,
-];
-
-try {
-  const xmlData = fs.readFileSync(sitemapPath, "utf8");
-
-  xml2js.parseString(xmlData, (err, result) => {
-    if (err) {
-      console.error("XML pasrder errors", err);
-      return;
+function downloadSitemap(url, destPath) {
+  return new Promise((resolve, reject) => {
+    if (fs.existsSync(destPath)) {
+      return resolve();
     }
 
-    const urls = result.urlset.url
-      .map((entry) => entry.loc[0]?.trim())
-      .filter((url) => !excludeUrls.includes(url))
-      .sort((a, b) => {
-        const aHasGuides = a.includes("/guides/");
-        const bHasGuides = b.includes("/guides/");
-        if (aHasGuides && !bHasGuides) return -1; // a comes first
-        if (!aHasGuides && bHasGuides) return 1; // b comes first
-        return 0; // maintain original order for others
-      });
-
-    fs.writeFileSync(outputFilePath, urls.join("\n"), "utf8");
+    const file = fs.createWriteStream(destPath);
+    https
+      .get(url, (response) => {
+        if (response.statusCode !== 200) {
+          reject(new Error(`Failed, error code：${response.statusCode}`));
+          return;
+        }
 
-    console.log(`${urls.length}, ${outputFilePath}`);
+        response.pipe(file);
+        file.on("finish", () => {
+          file.close(resolve);
+        });
+      })
+      .on("error", (err) => {
+        fs.unlink(destPath, () => reject(err));
+      });
   });
-} catch (error) {
-  console.error("file read errors:", error);
 }
+
+(async () => {
+  try {
+    console.log(`sitemap：${site} (${sitemapUrl})`);
+
+    await downloadSitemap(sitemapUrl, sitemapPath);
+    console.log(`Finished：${sitemapPath}`);
+
+    const xmlData = fs.readFileSync(sitemapPath, "utf8");
+
+    xml2js.parseString(xmlData, (err, result) => {
+      if (err) {
+        console.error("❌", err);
+        return;
+      }
+
+      const urls = result.urlset.url
+        .map((entry) => entry.loc[0]?.trim())
+        .filter((url) => !excludeUrls.includes(url))
+        .sort((a, b) => {
+          const aHasGuides = a.includes("/guides/");
+          const bHasGuides = b.includes("/guides/");
+          if (aHasGuides && !bHasGuides) return -1;
+          if (!aHasGuides && bHasGuides) return 1;
+          return 0;
+        });
+
+      fs.writeFileSync(outputFilePath, urls.join("\n"), "utf8");
+      console.log(`✅ Successed ${urls.length}, ${outputFilePath}`);
+    });
+  } catch (error) {
+    console.error("❌ Failed:", error);
+  }
+})();