|
| 1 | +// DC Affordable Housing loader. Pulls every record from the DCGIS |
| 2 | +// ArcGIS feature service (~900 projects) and maps onto the projects |
| 3 | +// schema. |
| 4 | +// |
| 5 | +// node scripts/load-dc.mjs |
| 6 | +// |
| 7 | +// This is one of the richer datasets we ingest: it has an AMI-bucket |
| 8 | +// breakdown that maps cleanly onto our HUD-style income tiers, plus |
| 9 | +// ward (council district), construction end date, and total/market/ |
| 10 | +// affordable unit counts. ArcGIS REST returns x,y as lng,lat. |
| 11 | + |
| 12 | +import { config as loadDotenv } from "dotenv"; |
| 13 | +import pg from "pg"; |
| 14 | + |
| 15 | +loadDotenv({ path: ".env.local" }); |
| 16 | +loadDotenv({ path: ".env" }); |
| 17 | + |
| 18 | +if (!process.env.DATABASE_URL) { |
| 19 | + console.error("error: DATABASE_URL is not set."); |
| 20 | + process.exit(1); |
| 21 | +} |
| 22 | + |
| 23 | +const ENDPOINT = |
| 24 | + "https://maps2.dcgis.dc.gov/dcgis/rest/services/DCGIS_DATA/Property_and_Land_WebMercator/FeatureServer/62/query"; |
| 25 | +const CITY_ID = "dc"; |
| 26 | + |
| 27 | +const { Pool } = pg; |
| 28 | +const pool = new Pool({ |
| 29 | + connectionString: process.env.DATABASE_URL, |
| 30 | + ssl: process.env.DATABASE_URL.includes("supabase") |
| 31 | + ? { rejectUnauthorized: false } |
| 32 | + : false, |
| 33 | +}); |
| 34 | + |
| 35 | +function num(v) { |
| 36 | + if (v == null) return 0; |
| 37 | + const n = parseFloat(v); |
| 38 | + return Number.isFinite(n) ? Math.round(n) : 0; |
| 39 | +} |
| 40 | + |
| 41 | +async function fetchAll() { |
| 42 | + // ArcGIS REST pages at 2000 per request; loop until exceededTransferLimit |
| 43 | + // goes away. DC has ~922 features so usually one page does it. |
| 44 | + const out = []; |
| 45 | + let offset = 0; |
| 46 | + while (true) { |
| 47 | + const params = new URLSearchParams({ |
| 48 | + where: "1=1", |
| 49 | + outFields: "*", |
| 50 | + outSR: "4326", |
| 51 | + f: "json", |
| 52 | + resultOffset: String(offset), |
| 53 | + resultRecordCount: "2000", |
| 54 | + }); |
| 55 | + const resp = await fetch(`${ENDPOINT}?${params}`); |
| 56 | + if (!resp.ok) throw new Error(`arcgis ${resp.status}: ${resp.statusText}`); |
| 57 | + const json = await resp.json(); |
| 58 | + const feats = json.features ?? []; |
| 59 | + out.push(...feats); |
| 60 | + if (!json.exceededTransferLimit || feats.length === 0) break; |
| 61 | + offset += feats.length; |
| 62 | + } |
| 63 | + return out; |
| 64 | +} |
| 65 | + |
| 66 | +// MAR_WARD is "Ward 8" / "Ward 1" — strip to the digit so it matches |
| 67 | +// council_members.district format. |
| 68 | +function wardToDistrict(w) { |
| 69 | + if (!w) return null; |
| 70 | + const m = String(w).match(/(\d+)/); |
| 71 | + return m ? m[1] : null; |
| 72 | +} |
| 73 | + |
| 74 | +function toProject(feat) { |
| 75 | + const a = feat.attributes || {}; |
| 76 | + const g = feat.geometry || {}; |
| 77 | + const lat = parseFloat(a.LATITUDE) || g.y; |
| 78 | + const lng = parseFloat(a.LONGITUDE) || g.x; |
| 79 | + if (!Number.isFinite(lat) || !Number.isFinite(lng)) return null; |
| 80 | + if (!a.PROJECT_NAME && !a.ADDRESS) return null; |
| 81 | + |
| 82 | + // AMI buckets map onto HUD's standard 30/50/60/80/120 cutoffs. |
| 83 | + // DC uses 0-30 (extremely low), 31-50 (very low), 51-60 (low), 61-80 |
| 84 | + // (low+), 81+ (anything above 80% AMI — we put in moderate). |
| 85 | + const xLow = num(a.AFFORDABLE_UNITS_AT_0_30_AMI); |
| 86 | + const vLow = num(a.AFFORDABLE_UNITS_AT_31_50_AMI); |
| 87 | + const low = num(a.AFFORDABLE_UNITS_AT_51_60_AMI) + num(a.AFFORDABLE_UNITS_AT_61_80_AMI); |
| 88 | + const moderate = num(a.AFFORDABLE_UNITS_AT_81_AMI); |
| 89 | + |
| 90 | + const totalAffordable = num(a.TOTAL_AFFORDABLE_UNITS); |
| 91 | + const totalAll = num(a.UNITS_TOTAL) || totalAffordable; |
| 92 | + // Whatever isn't bucketed gets dumped into "other" so totals match. |
| 93 | + const bucketed = xLow + vLow + low + moderate; |
| 94 | + const other = Math.max(0, totalAffordable - bucketed); |
| 95 | + |
| 96 | + // STATUS_PUBLIC is "Completed 2015 to Date" or "Under Construction" etc. |
| 97 | + const status = (a.STATUS_PUBLIC || "").toLowerCase(); |
| 98 | + let constructionType = null; |
| 99 | + if (status.includes("preserv") || num(a.AFFORDABLE_UNITS_PRESERVED) > num(a.AFFORDABLE_UNITS_PRODUCTION)) { |
| 100 | + constructionType = "Preservation"; |
| 101 | + } else if (num(a.AFFORDABLE_UNITS_PRODUCTION) > 0 || num(a.UNITS_NET_NEW) > 0) { |
| 102 | + constructionType = "New Construction"; |
| 103 | + } |
| 104 | + |
| 105 | + // CONSTRUCTION_END_DATE is unix ms. |
| 106 | + let completionDate = null; |
| 107 | + if (typeof a.CONSTRUCTION_END_DATE === "number" && a.CONSTRUCTION_END_DATE > 0) { |
| 108 | + completionDate = new Date(a.CONSTRUCTION_END_DATE).toISOString().slice(0, 10); |
| 109 | + } |
| 110 | + |
| 111 | + return { |
| 112 | + city_id: CITY_ID, |
| 113 | + external_id: String(a.OBJECTID), |
| 114 | + name: (a.PROJECT_NAME || a.ADDRESS || "(unnamed)").trim(), |
| 115 | + address: a.ADDRESS || a.FULLADDRESS || null, |
| 116 | + borough: a.PLANNING_AREA || null, |
| 117 | + neighborhood: a.PLANNING_AREA || null, |
| 118 | + postcode: null, |
| 119 | + council_district: wardToDistrict(a.MAR_WARD), |
| 120 | + community_board: null, |
| 121 | + construction_type: constructionType, |
| 122 | + extended_affordability: false, |
| 123 | + prevailing_wage: false, |
| 124 | + start_date: null, |
| 125 | + completion_date: completionDate, |
| 126 | + buildings_count: 1, |
| 127 | + lat, |
| 128 | + lng, |
| 129 | + units_total: totalAll, |
| 130 | + units_counted: totalAffordable, |
| 131 | + units_rental: totalAffordable, |
| 132 | + units_homeownership: 0, |
| 133 | + units_extremely_low: xLow, |
| 134 | + units_very_low: vLow, |
| 135 | + units_low: low, |
| 136 | + units_moderate: moderate, |
| 137 | + units_middle: 0, |
| 138 | + units_other_income: other, |
| 139 | + units_studio: 0, |
| 140 | + units_1br: 0, |
| 141 | + units_2br: 0, |
| 142 | + units_3br: 0, |
| 143 | + units_4plus_br: 0, |
| 144 | + }; |
| 145 | +} |
| 146 | + |
| 147 | +const UPSERT_SQL = ` |
| 148 | +INSERT INTO projects ( |
| 149 | + city_id, external_id, name, address, borough, neighborhood, postcode, |
| 150 | + council_district, community_board, construction_type, |
| 151 | + extended_affordability, prevailing_wage, |
| 152 | + start_date, completion_date, geom, |
| 153 | + units_total, units_counted, units_rental, units_homeownership, |
| 154 | + units_extremely_low, units_very_low, units_low, units_moderate, |
| 155 | + units_middle, units_other_income, |
| 156 | + units_studio, units_1br, units_2br, units_3br, units_4plus_br, |
| 157 | + buildings_count, imported_at |
| 158 | +) VALUES ( |
| 159 | + $1,$2,$3,$4,$5,$6,$7, |
| 160 | + $8,$9,$10, |
| 161 | + $11,$12, |
| 162 | + $13,$14, ST_SetSRID(ST_MakePoint($15,$16),4326)::geography, |
| 163 | + $17,$18,$19,$20, |
| 164 | + $21,$22,$23,$24, |
| 165 | + $25,$26, |
| 166 | + $27,$28,$29,$30,$31, |
| 167 | + $32, NOW() |
| 168 | +) |
| 169 | +ON CONFLICT (city_id, external_id) DO UPDATE SET |
| 170 | + name = EXCLUDED.name, |
| 171 | + address = EXCLUDED.address, |
| 172 | + borough = EXCLUDED.borough, |
| 173 | + neighborhood = EXCLUDED.neighborhood, |
| 174 | + council_district = EXCLUDED.council_district, |
| 175 | + construction_type = EXCLUDED.construction_type, |
| 176 | + completion_date = EXCLUDED.completion_date, |
| 177 | + geom = EXCLUDED.geom, |
| 178 | + units_total = EXCLUDED.units_total, |
| 179 | + units_counted = EXCLUDED.units_counted, |
| 180 | + units_rental = EXCLUDED.units_rental, |
| 181 | + units_extremely_low = EXCLUDED.units_extremely_low, |
| 182 | + units_very_low = EXCLUDED.units_very_low, |
| 183 | + units_low = EXCLUDED.units_low, |
| 184 | + units_moderate = EXCLUDED.units_moderate, |
| 185 | + units_other_income = EXCLUDED.units_other_income, |
| 186 | + imported_at = NOW(); |
| 187 | +`; |
| 188 | + |
| 189 | +async function main() { |
| 190 | + const client = await pool.connect(); |
| 191 | + try { |
| 192 | + console.log("fetching DC affordable housing inventory..."); |
| 193 | + const features = await fetchAll(); |
| 194 | + console.log(`got ${features.length} features`); |
| 195 | + const projects = features.map(toProject).filter(Boolean); |
| 196 | + console.log(`upserting ${projects.length} projects with valid coordinates...`); |
| 197 | + |
| 198 | + await client.query("BEGIN"); |
| 199 | + for (const p of projects) { |
| 200 | + await client.query(UPSERT_SQL, [ |
| 201 | + p.city_id, p.external_id, p.name, p.address, p.borough, p.neighborhood, p.postcode, |
| 202 | + p.council_district, p.community_board, p.construction_type, |
| 203 | + p.extended_affordability, p.prevailing_wage, |
| 204 | + p.start_date, p.completion_date, p.lng, p.lat, |
| 205 | + p.units_total, p.units_counted, p.units_rental, p.units_homeownership, |
| 206 | + p.units_extremely_low, p.units_very_low, p.units_low, p.units_moderate, |
| 207 | + p.units_middle, p.units_other_income, |
| 208 | + p.units_studio, p.units_1br, p.units_2br, p.units_3br, p.units_4plus_br, |
| 209 | + p.buildings_count, |
| 210 | + ]); |
| 211 | + } |
| 212 | + await client.query("UPDATE cities SET fetched_at = NOW() WHERE id = $1", [CITY_ID]); |
| 213 | + await client.query("COMMIT"); |
| 214 | + console.log("done."); |
| 215 | + } catch (e) { |
| 216 | + await client.query("ROLLBACK").catch(() => {}); |
| 217 | + throw e; |
| 218 | + } finally { |
| 219 | + client.release(); |
| 220 | + await pool.end(); |
| 221 | + } |
| 222 | +} |
| 223 | + |
| 224 | +main().catch((e) => { |
| 225 | + console.error(e); |
| 226 | + process.exit(1); |
| 227 | +}); |
0 commit comments