Babel compile the script so it's all bluebird long stack promises

Author: deployable

.babelrc

@@ -0,0 +1,14 @@
+{
+  "presets": [
+    [ "env", { "targets": { "node": "6" } } ]
+  ],
+  "plugins": [
+    [
+      "transform-async-to-module-method",
+      {
+        "module": "bluebird",
+        "method": "coroutine"
+      }
+    ]
+  ]
+}

package.json

@@ -24,6 +24,9 @@
   },
   "homepage": "https://github.com/deployable/docker-apt-cacher-ng#readme",
   "devDependencies": {
+    "babel-cli": "^6.26.0",
+    "babel-plugin-transform-async-to-module-method": "^6.24.1",
+    "babel-preset-env": "^1.6.1",
     "chai": "^3.5.0",
     "cheerio": "^1.0.0-rc.2",
     "coffeescript": "^1.12.4",

src/fetch-mirrors.es2017

@@ -0,0 +1,147 @@
+const Promise = require('bluebird')
+Promise.config({ longStackTraces: true, warnings: true })
+const cheerio = require('cheerio')
+const needle = require('needle')
+const debug = require('debug')('mh:docker:kind:mirrors')
+const fs = Promise.promisifyAll(require('fs'))
+const readline = require('readline')
+const path = require('path')
+
+class Fetch {
+
+  static async fetchAndCheerio(url){
+    debug('retriving url: %s', url)
+    let response = await needle('get', url)
+    debug('retrieved response:', response.body)
+    return cheerio.load(response.body)
+  }
+
+  static /*async*/ fetchCentos(){
+    return new Promise((resolve,reject) => {
+      let response = needle.get('https://www.centos.org/download/full-mirrorlist.csv')
+      const line_reader = require('readline').createInterface({
+        input: response
+      })
+
+      const mirrors = []
+
+      line_reader.on('line', line => {
+        // poor mans csv parsing, remove leading and trailing `"`s
+        // then splut the string on `","`. 
+        // Only works when every field is quoted with `"`
+        line.replace(/^"/,'').replace(/"$/,'')
+        let fields = line.split(/","/)
+        if ( fields.length > 4 && fields[4] ) {
+          //debug('fields', fields)
+          mirrors.push(fields[4])
+        } 
+        else {
+          if ( 
+            fields.length === 1 || 
+            ( fields[5] && fields[5].includes('ftp') )
+          ) return
+          console.error('centos - bad line', line)
+        }
+      })
+
+      line_reader.on('close', ()=>{
+        debug('centos mirrors', mirrors.join("\n"))
+        resolve(mirrors)
+      })
+
+      line_reader.on('error', ()=> reject(error))
+    })
+  }
+
+  static async fetchEpel(){
+    let $ = await this.fetchAndCheerio('https://admin.fedoraproject.org/mirrormanager/mirrors/EPEL')
+    let $rows = $('.container table').first().find('tr')
+    let mirrors = []
+    $rows.each((row,el) => {
+      let mirror_info = $(el).children('td').slice(3,4).contents()
+      let mode = null
+      mirror_info.each((mirror_data, el) => {
+        if ( el.type === 'text' ) {
+          if ( /Fedora EPEL/.exec(el.data) ) mode = 'epel'
+          if ( /Fedora Linux/.exec(el.data) ) mode = 'fedora'
+        }
+        if ( mode === 'epel' && el.type === 'tag' && el.name === 'a' && $(el).text() === 'http' ) {
+          mirrors.push($(el).attr('href'))
+        }
+      })
+    })
+    debug('epel mirrors', mirrors.join('\n'))
+    return mirrors 
+  }
+
+  static async fetchFedora(){
+    let $ = await this.fetchAndCheerio('https://admin.fedoraproject.org/mirrormanager/mirrors/Fedora')
+    let $rows = $('.container table').first().find('tr')
+    let mirrors = []
+    /*
+    $rows.map((row,el) => {
+      return $(el).children('td').slice(3,4).find('a').each((linki, el)=> {
+        if ( $(el).text() === 'http' ) mirrors.push($(el).attr('href'))
+      })
+    })
+    */
+    $rows.each((row,el) => {
+      let mirror_info = $(el).children('td').slice(3,4).contents()
+      let mode = null
+      mirror_info.each((mirror_data, el) => {
+        if ( el.type === 'text' ) {
+          if ( /Fedora EPEL/.exec(el.data) ) mode = 'epel'
+          if ( /Fedora Linux/.exec(el.data) ) mode = 'fedora'
+        } 
+        if ( mode === 'fedora' && el.type === 'tag' && el.name === 'a' && $(el).text() === 'http' ) {
+          mirrors.push($(el).attr('href'))  
+        }
+      })
+    })
+    debug('fedora mirrors', mirrors.join('\n'))
+    return mirrors 
+  }
+
+  static async fetchApache(){
+    let $ = await this.fetchAndCheerio('https://www.apache.org/mirrors/dist.html')
+    let $rows = $('table tr')
+    let mirrors = []
+    $rows.each((rowi, el) => {
+      let $el = $(el)
+      let cols = $el.find('td')
+      debug('apache row %s: size %s: ', rowi, cols.length, $(cols).text())
+      if ( cols.length === 5 ) {
+        let $mirror_col = $( $(cols).get(0) )
+        let $scheme_col = $( $(cols).get(1) )
+        let is_http = $scheme_col.text().includes('http')
+        let mirror_link = $mirror_col.find('a').first().attr('href')
+        debug('is_https: %s  mirror: %s', is_http, mirror_link)
+        if ( is_http === true ) mirrors.push(mirror_link)
+      }
+    })
+    return mirrors
+  }
+
+  static async writeMirror( file, promise ){
+    let file_path = path.resolve( __dirname, '..', 'files', file )
+    if ( typeof promise === 'function') promise = promise()
+    let mirror_data = await promise
+    debug('writeMirror has got the mirror data for file "%s"', file)
+    return fs.writeFileAsync(file_path, mirror_data.join('\n'))
+  }
+
+  static async go(){
+    try {
+      await this.writeMirror('centos_mirrors', ()=>this.fetchCentos())
+      await this.writeMirror('fedora_mirrors', ()=>this.fetchFedora())
+      await this.writeMirror('epel_mirrors', ()=>this.fetchEpel())
+      await this.writeMirror('apache_mirrors', ()=>this.fetchApache())
+    }
+    catch (error) {
+      console.log(error)
+    }
+
+  }
+}
+
+Fetch.go()