diff --git a/package.json b/package.json index fd7d113..d94578c 100644 --- a/package.json +++ b/package.json @@ -21,6 +21,7 @@ "@slack/web-api": "^6.7.1", "async-await-queue": "^1.2.0", "axios": "^0.26.0", + "j2m": "^1.1.0", "probot": "14.0.0-beta.11" }, "devDependencies": { diff --git a/src/modules/fetch_project_list/index.ts b/src/modules/fetch_project_list/index.ts new file mode 100644 index 0000000..c94d975 --- /dev/null +++ b/src/modules/fetch_project_list/index.ts @@ -0,0 +1,438 @@ +import {ProbotOctokit} from "probot"; +import {getProjectItemsForExport} from "../../shared/graphql_queries"; +import {logger} from "../../shared/logger"; +import fs from "fs"; +import {FieldValue, ProjectItem, SearchResult, Issue} from "./types"; +// @ts-ignore +import J2M from 'j2m'; + +const searchCutOffArr = [ + // `created:>2025-07-01`, // test set + // `31093` // test set + + // prod set + `created:>2025-04-01`, + `created:2025-01-01..2025-04-01`, + `created:2024-07-01..2025-01-01`, + 'created:2022-06-01..2024-07-01', + `created:<2022-06-01`, +] + +const repoWhitelist = [ + "cloud", + "neon", + "company_projects", + "analytics", + "neonctl", + "autoscaling" +]; + +const projectsBlacklist = [ + '@erikgrinaker\'s untitled project', + 'CIv2 Project', + 'Deals', + 'PD: AI XP team', + '[TEMPLATE] tasks', + 'Infra Initiatives', + 'Postgres team tasks', + 'PD: API Platform team tasks', + 'PD: QA tasks', + 'SOC2', + 'Triage: Product Delivery', + '@MihaiBojin', + 'Developer Productivity team tasks', + 'PD: Workflow tasks', + 'PD: Growth', + 'PD: Identity tasks', +] + +const projectsAsTeams = [ + 'PCI-DSS', + 'Data team tasks', + 'RE: Support Tools', + 'Product Design Team Tasks', + 'PD: FE Infra tasks', + 'PD: Workflow tasks', + 'Pl: Compute team tasks', + 'Pl: Proxy team tasks', + 'PD: DBaaS tasks', + 'Security', + 'PD: Docs tasks', + 'Pl: Storage team tasks', + 'PD: Billing tasks', + 'Pl: Control-plane team tasks', + 'Pl: Autoscaling team tasks' +] + +const projectsAsLabels = [ + 'COGS', + 'Project Trust', + 'Support Escalations', + 'PD: Azure tasks', + 'HIPAA', + 'Incident Follow-ups' +] + +const priorityMap = { + 'P0':'Blocker', + 'P1':'Critical', + 'P2':'Major', + 'P3':'Minor', + 'πŸŒ‹ Urgent':'Critical', + 'πŸ” High':'Major', + 'πŸ• Medium':'Minor', + '🏝 Low':'Trivial', + 'πŸŒ‹ P0 - High':'Blocker', + 'πŸ” P1 - Moderate':'Critical', + 'πŸ• P2 - Low Priority':'Major', + '🏝 P3 - Negligible':'Minor', +} + +// const statusesWhiteList = [ +// 'selected', +// 'in progress', +// 'needs refinement', +// 'triage', +// 'blocked', +// 'inbox' +// ] + +// const statusesWhiteListRegex = /selected|in\sprogress|needs\srefinement|triage|blocked|inbox/i + +const getFieldValue = (field: FieldValue) => (field.text || field.date || field.number || field.name || ''); + +const tryGetFieldValue = (search: string, fieldsData: ProjectItem["fieldValues"]["nodes"]) => { + const field = fieldsData.find(fieldItem => fieldItem.field && fieldItem.field.name.toLowerCase().includes(search.toLowerCase())); + + return field ? getFieldValue(field) : ''; +} + +const getIssueId = (issue: Pick) => { + return `${issue.repository.name}#${issue.number}`; +} + +const prepareLabels = (issue: Issue) => { + const ghLabels = issue.labels.nodes + .filter((labelNode) => labelNode.name && !labelNode.name.startsWith("c/") && !labelNode.name.startsWith("team/")) + .map((labelNode) => labelNode.name); + const projectToLabels = issue.projectItems.nodes.filter(prItem => ( + !prItem.isArchived && projectsAsLabels.includes(prItem.project.title) + )).map(prItem => `project/${prItem.project.title.replace(/\s/g, '_')}`); + + return [ + ...ghLabels, + ...projectToLabels + ] +} + +const prepareBody = (issue: Issue, projectItems: ProjectItem[], linksMap: Record)=> { + // todo: replace links + console.log(issue) + console.log(projectItems) + + const projectItemsFieldsText = "## Fields from Neon Github Projects\n\n" + + projectItems.map((prItem) => { + let res = `||Field||Value||\n`; + res = res + prItem.fieldValues.nodes + .filter((fieldValue) => fieldValue.field && fieldValue.field.name && !fieldValue.field.name.includes('πŸ”„') && fieldValue.field.name !== "Title") + .map((fieldValue) => ( + `|${fieldValue.field.name}|${getFieldValue(fieldValue)}|` + )).join('\n') + + return `### ${prItem.project.title}\n${res}`; + }).join('\n') + + const reporterLink = `[${issue.author.login}](${issue.author.url})`; + const assigneesArr = issue.assignees.nodes.map(user => (`[${user.login}](${user.url})`)) + + const originalAssignees = assigneesArr.length ? `GH Assignees: ${assigneesArr.join(', ')}\n\n` : '\n'; + + // replacing links to look nicer in body + // const mathFullLinks = + + const content = `[Original Github Issue from Neon](${issue.url}), created at ${issue.createdAt} by ${reporterLink}\n` + + originalAssignees + + issue.body + + "\n\n" + + projectItemsFieldsText + + const wiki = J2M.toJ(content); + return escape(wiki); + // return '' +} + +const prepareComponents = (issue:Issue) => { + return issue.labels.nodes + .filter((labelNode) => labelNode.name && labelNode.name.startsWith("c/")) + .map((labelNode) => labelNode.name.substring(2)); +} + +const prepareIssueType = (issue: Issue) => { + // todo + + if (issue.repository.name === "company_projects") { + return "Major Initiative" + } + + return issue.issueType ? issue.issueType.name : 'Task'; +} + +// get project item that is most likely to make progress +const getMainProject = (teamProjectItems: ProjectItem[], labelProjectItems: ProjectItem[]) => { + if (!teamProjectItems.length && !labelProjectItems.length) { + return; + } + + const projectItems = teamProjectItems.length ? teamProjectItems : labelProjectItems; + + return projectItems.find((prItem) => (tryGetFieldValue("status", prItem.fieldValues.nodes).toLowerCase() === 'in progress')) + || projectItems.find((prItem) => (tryGetFieldValue("status", prItem.fieldValues.nodes).toLowerCase() === 'selected')) + || projectItems.find((prItem) => (tryGetFieldValue("status", prItem.fieldValues.nodes).toLowerCase() === 'needs refinement')) + || projectItems.find((prItem) => (tryGetFieldValue("status", prItem.fieldValues.nodes).toLowerCase() === 'blocked')) + || projectItems.find((prItem) => (tryGetFieldValue("status", prItem.fieldValues.nodes).toLowerCase() === 'inbox')) + || projectItems[0] +} + +const escape = (str: string) => `"${str}"`; + +const MAX_LABELS = 11; +const MAX_TEAMS = 5; +const MAX_COMPONENTS = 5; +const MAX_ASSIGNEES = 1; +const MAX_LINKED = 5; + +export const fetch_project_list = async (octokit: ProbotOctokit) => { + console.log("start fetching"); + + const stats = { + maxLabelsLength: 0, + maxComponentsLength: 0, + maxProjectsItemsLength: 0, + maxTeams: 0, + maxAssignees: 0, + maxRelatesTo: 0, + } + + const exportData = [ + [ + "url", + "id", + "title", + "body", + "issueType", + "author", + "parent", + ...new Array(MAX_LINKED).fill('relates to'), + ...new Array(MAX_ASSIGNEES).fill('assignee'), + ...new Array(MAX_COMPONENTS).fill('component'), + ...new Array(MAX_TEAMS).fill('team'), + ...new Array(MAX_LABELS).fill('label'), + ...new Array(MAX_TEAMS).fill('Statuses in team projects'), + ...new Array(MAX_TEAMS).fill('Statuses in label projects'), + "Selected project Name", + "Selected status", + "Selected priority", + "Selected size", + ] + ]; + const exportExceptions = [ + [ + "url", + "title", + "reason" + ] + ] + + const addException = (issue: Issue, reason: string) => { + exportExceptions.push([ + issue.url, + issue.title, + reason + ]); + } + + console.time('fetch_project_list') + let i = 0; + for (let searchSlice of searchCutOffArr) { + const searchQuery = `org:neondatabase is:issue state:open ${searchSlice}`; + let pageInfo: { + hasNextPage: boolean, + endCursor: string, + } = { + hasNextPage: true, + endCursor: '', + } + while (pageInfo.hasNextPage) { + console.log("counter", i); + try { + const res: { search: SearchResult, totalCount: number } = await octokit.graphql(getProjectItemsForExport, { + q: searchQuery, + cursor: pageInfo.endCursor, + }); + logger('info', `Search query: ${searchQuery}, total count: ${res.search.issueCount}`); + const { search } = res; + // logger('info', `processing page from cursor ${pageInfo.endCursor}, itemsCount: ${search.nodes.length}`); + + pageInfo.endCursor = search.pageInfo.endCursor + pageInfo.hasNextPage = search.pageInfo.hasNextPage + if (res.totalCount > 1000) { + logger('error', `Search slice returned more than 1000 results, query: "${searchQuery}"`); + } + + for (let issue of search.nodes) { + i++; + // logger('info', `it. #${i} Processing issue ${issue.repository.name}#${issue.number} ${issue.title}`) + if (!(repoWhitelist).includes(issue.repository.name)) { + addException(issue, `Does not belong to whitelisted repositories, repo name: ${issue.repository.name}`) + continue; + } + + if (!issue.projectItems.nodes.length) { + addException(issue, 'Does not belong to any projects') + continue; + } + + // list of projects which fields get to issue body + const whitelistedProjectItems = issue.projectItems.nodes + .filter((prItem) => ( + !prItem.project.closedAt // don't care about closed projects + && !prItem.isArchived // don't care about archived items + && !projectsBlacklist.includes(prItem.project.title) // exclude blacklisted projects + // && // todo: exclude blacklisted statuses + )) + + if (!whitelistedProjectItems.length) { + const projectsStr = issue.projectItems.nodes + .map((prItem) => { + if (prItem.project.closedAt) { + `"${prItem.project.title}": project closed at ${prItem.project.closedAt}` + } + if (prItem.isArchived) { + return `"${prItem.project.title}": issue archived` + } + if (projectsBlacklist.includes(prItem.project.title)) { + return `"${prItem.project.title}": project is not whitelisted` + } + return `"${prItem.project.title}": unknown reason, check me` + }).join('; ') + addException(issue, `Does not belong to any allowed projects, details: ${projectsStr}`) + continue; + } + stats.maxProjectsItemsLength = Math.max(whitelistedProjectItems.length, stats.maxProjectsItemsLength) + + const teamsProjectItems = whitelistedProjectItems + .filter((prItem) => ( + projectsAsTeams.includes(prItem.project.title) // only for teams we want to migrate as Assigned Team + )) + + const labelProjectItems = whitelistedProjectItems.filter((prItem) => ( + projectsAsLabels.includes(prItem.project.title) // only for teams we want to migrate as Assigned Team + )) + + // get project item with the best status (In Progress, Selected, Blocked, Need Refinement, Inbox) + const mainProjectItem: ProjectItem | undefined = getMainProject(teamsProjectItems, labelProjectItems); + + // get components from labels + const components = prepareComponents(issue); + stats.maxComponentsLength = Math.max(components.length, stats.maxComponentsLength) + components.length = MAX_COMPONENTS + + // sanitise labels + const labels = prepareLabels(issue); + stats.maxLabelsLength = Math.max(labels.length, stats.maxLabelsLength) + labels.length = MAX_LABELS + + const teams = teamsProjectItems.map((prItem) => (prItem.project.title)); + stats.maxTeams = Math.max(teams.length, stats.maxTeams); + teams.length = MAX_TEAMS + + const assignees = issue.assignees.nodes.map((item) => (item.login)); + stats.maxAssignees = Math.max(assignees.length, stats.maxAssignees); + assignees.length = MAX_ASSIGNEES; + + const teamsStatuses = teamsProjectItems.map((prItem) => (`${prItem.project.title}/${tryGetFieldValue('status', prItem.fieldValues.nodes) || 'unknown'}`)); + teamsStatuses.length = MAX_TEAMS; + const labelStatuses = labelProjectItems.map((prItem) => (`${prItem.project.title}/${tryGetFieldValue('status', prItem.fieldValues.nodes) || 'unknown'}`)); + labelStatuses.length = MAX_TEAMS; + + const id = getIssueId(issue); + const body = prepareBody(issue, whitelistedProjectItems, {}); + const title = escape(`${issue.title}, ${id}`); + const author = issue.author.login // todo: map to databricks emails pending Crystal + const parent = issue.parent ? getIssueId(issue.parent) : ''; + const relatesTo = issue.trackedInIssues.nodes.map(item => getIssueId(item)); + stats.maxRelatesTo = Math.max(relatesTo.length, stats.maxRelatesTo); + relatesTo.length = MAX_LINKED; + + const selectedPriority = mainProjectItem ? priorityMap[tryGetFieldValue("priority", mainProjectItem.fieldValues.nodes) || 'unknown'] || 'unknown' : ''; + + + const issueType = prepareIssueType(issue); + + exportData.push( + [ + // "url", + issue.url, + // "repo#number", + id, + // "title", + title, + // "body", + body, + // "issueType", + issueType, + // "author", + author, + // "parent", + parent, + // relates to + ...relatesTo, + // assignee + ...assignees, + // ...new Array(5).fill('component'), + ...components, + // ...new Array(5).fill('team'), + ...teams, + // ...new Array(5).fill('label'), + ...labels, + // ...new Array(MAX_TEAMS).fill('Statuses in teams projects'), + ...teamsStatuses, + // ...new Array(MAX_TEAMS).fill('Statuses in label projects'), + ...labelStatuses, + // "Selected project Name", + mainProjectItem ? mainProjectItem.project.title : '', + // "Selected status", + mainProjectItem ? tryGetFieldValue("status", mainProjectItem.fieldValues.nodes) : '', + // "Selected priority", + selectedPriority, + // "Selected size", + mainProjectItem ? tryGetFieldValue("size", mainProjectItem.fieldValues.nodes) : '', + ] + ) + } + + logger('info', search); + } catch (e) { + logger('error', e) + } + } + } + const content = exportData.map(item => item.join('ΓΏ')).join('\n') + const exceptionsContent = exportExceptions.map(item => item.join('ΓΏ')).join('\n') + + console.log("stats") + console.log(stats) + console.log("Processed issues:", i); + console.log(`will migrate: ${exportData.length - 1}, will not migrate: ${exportExceptions.length - 1}`); + console.timeEnd('fetch_project_list') + + try { + const timestamp = new Date().toISOString(); + fs.writeFileSync(`./${timestamp}_Dump_all.csv`, content); + fs.writeFileSync(`./${timestamp}_Exceptions_all.csv`, exceptionsContent); + console.log("written") + // file written successfully + } catch (err) { + console.error(err); + } +} \ No newline at end of file diff --git a/src/modules/fetch_project_list/types.ts b/src/modules/fetch_project_list/types.ts new file mode 100644 index 0000000..6271976 --- /dev/null +++ b/src/modules/fetch_project_list/types.ts @@ -0,0 +1,80 @@ +export type PageInfo = { + endCursor: string, + hasNextPage: boolean, + startCursor: string, + hasPreviousPage: boolean, +} + +type Repository = { + name: string +} + +export type FieldValue = { + field: { + id: string + name: string + } + text?: string + date?: string + name?: string + number?: string +} & ({text: string} | {date: string} | { name: string } |{number: number}) + +export type ProjectItem = { + id: string + isArchived: boolean + type: string + updatedAt: string + project: { + id: string + title: string + closedAt: string + } + fieldValues: { + nodes: Array + } +} + +export type Issue = { + number: number + title: string + body: string + url: string + createdAt: string + updatedAt: string + closedAt: string + repository: Repository + issueType: { + name: string + }, + author: { + login: string + url: string + } + trackedInIssues: { + totalCount: number + nodes: Array> + } + parent: Pick + assignees: { + totalCount: number + nodes: Array<{ + login: string + url: string + }> + } + labels: { + nodes: Array<{ + name: string + }> + } + projectItems: { + nodes: Array + } +} + +export type SearchResult = { + pageInfo: PageInfo, + issueCount: number, + nodes: Array +} diff --git a/src/shared/graphql_queries.ts b/src/shared/graphql_queries.ts index cc9a3ba..8d69d29 100644 --- a/src/shared/graphql_queries.ts +++ b/src/shared/graphql_queries.ts @@ -419,4 +419,126 @@ query($id: ID!){ } } } -` \ No newline at end of file +` + +export const getProjectItemsForExport = ` +query ($q: String!, $cursor: String!){ + search(query: $q, type: ISSUE, first: 100, after: $cursor){ + pageInfo { + endCursor, + hasNextPage, + startCursor, + hasPreviousPage, + } + issueCount + nodes { + ... on Issue { + number, + title, + body, + url, + createdAt, + updatedAt, + closedAt, + issueType { + ... on IssueType { + name + } + } + trackedInIssues(last: 5) { + totalCount + nodes { + ... on Issue { + repository { + ... on Repository { + name + } + } + number + } + } + } + parent { + ... on Issue { + number + repository { + ... on Repository { + name + } + } + } + } + repository { + ... on Repository { + name + } + } + author { + ... on Actor { + login + url + } + } + assignees (first: 10) { + totalCount + nodes { + ... on User { + login + url + } + } + } + labels(first: 20) { + nodes { + ... on Label { + name + } + } + } + projectItems(first: 10, includeArchived: false) { + ... on ProjectV2ItemConnection { + nodes{ + ... on ProjectV2Item { + id, + isArchived, + type, + updatedAt, + project { + ... on ProjectV2 { + id + title + closedAt + } + } + fieldValues(first: 15) { + nodes { + ... on ProjectV2ItemFieldValueCommon { + field { + ... on ProjectV2FieldCommon { + id + name + } + } + } + ... on ProjectV2ItemFieldDateValue { + date + } + ... on ProjectV2ItemFieldSingleSelectValue { + name + } + ... on ProjectV2ItemFieldNumberValue { + number + } + ... on ProjectV2ItemFieldTextValue { + text + } + } + }, + } + } + } + } + } + } + } +}` \ No newline at end of file diff --git a/yarn.lock b/yarn.lock index 52a0b38..e0ca6ab 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1261,6 +1261,11 @@ colorette@^2.0.7: resolved "https://registry.yarnpkg.com/colorette/-/colorette-2.0.20.tgz#9eb793e6833067f7235902fcd3b09917a000a95a" integrity sha512-IfEDxwoWIjkeXL1eXcDiow4UbKjhLdq6/EuSVR9GMN7KVH3r9gQ83e73hsz1Nd1T3ijd5xv1wcWRYO+D6kCI2w== +colors@^1.1.2: + version "1.4.0" + resolved "https://registry.yarnpkg.com/colors/-/colors-1.4.0.tgz#c50491479d4c1bdaed2c9ced32cf7c7dc2360f78" + integrity sha512-a+UqTh4kgZg/SlGvfbzDHpgRu7AAQOmmqRHJnxhRZICKFUT91brVhNNt58CMWU9PsBbv3PDCZUHbVxuDiH2mtA== + combined-stream@^1.0.6, combined-stream@^1.0.8: version "1.0.8" resolved "https://registry.yarnpkg.com/combined-stream/-/combined-stream-1.0.8.tgz#c3d45a8b34fd730631a110a8a2520682b31d5a7f" @@ -1791,6 +1796,14 @@ is-stream@^1.1.0: resolved "https://registry.yarnpkg.com/is-stream/-/is-stream-1.1.0.tgz#12d4a3dd4e68e0b79ceb8dbc84173ae80d91ca44" integrity sha1-EtSj3U5o4Lec6428hBc66A2RykQ= +j2m@^1.1.0: + version "1.1.0" + resolved "https://registry.yarnpkg.com/j2m/-/j2m-1.1.0.tgz#bcd2fe36902eaf012eb9613a39a2a9e14e458df4" + integrity sha512-48wXbhVo5fUsqxAhpEPZIP8HJaHGJGK38XwZCyjbS5MuEdkCR786U4U4Hk5g34e60Kf/jF8y+TphVTHYe/mSWA== + dependencies: + colors "^1.1.2" + minimist "^1.2.0" + joycon@^3.1.1: version "3.1.1" resolved "https://registry.yarnpkg.com/joycon/-/joycon-3.1.1.tgz#bce8596d6ae808f8b68168f5fc69280996894f03" @@ -1896,7 +1909,7 @@ mime@^2.4.6: resolved "https://registry.yarnpkg.com/mime/-/mime-2.6.0.tgz#a2a682a95cd4d0cb1d6257e28f83da7e35800367" integrity sha512-USPkMeET31rOMiarsBNIHZKLGgvKc/LrjofAnBlOttf5ajRvqiRA8QsenbcooctK6d6Ts6aqZXBA+XbkKthiQg== -minimist@^1.2.6: +minimist@^1.2.0, minimist@^1.2.6: version "1.2.8" resolved "https://registry.yarnpkg.com/minimist/-/minimist-1.2.8.tgz#c1a464e7693302e082a075cee0c057741ac4772c" integrity sha512-2yyAR8qBkN3YuheJanUpWC5U3bb5osDywNB8RzDVlDwDHbocAJveqqj1u8+SVD7jkWT4yvsHCpWqqWqAxb0zCA==