Merge pull request #709 from ibi-group/dev

Feature Release: September 2 2022

Author: binh-dam-ibigroup

.github/issue_template.md

@@ -1,4 +1,4 @@
-_**NOTE:** This issue system is intended for reporting bugs and tracking progress in software development. Although this software is licensed with an open-source license, any issue opened here may not be responded to in a timely manner. [Conveyal](https://www.conveyal.com) is unable to provide technical support for custom deployments of this software unless your company has a support contract with us. Please remove this note when creating the issue._
+_**NOTE:** This issue system is intended for reporting bugs and tracking progress in software development. Although this software is licensed with an open-source license, any issue opened here may not be dealt with in a timely manner. [IBI Group](https://www.ibigroup.com/) is able to provide technical support for custom deployments of this software. Please contact [Ritesh Warade](mailto:[email protected]?subject=Data%20Tools%20inquiry%20via%20GitHub&body=Name:%20%0D%0AAgency/Company:%20%0D%0ABest%20date/time%20for%20a%20demo/discussion:%20%0D%0ADescription%20of%20needs:%20) if your company or organization is interested in opening a support contract with us. Please remove this note when creating the issue._
 
 ## Observed behavior (please include a screenshot if possible)
 

.github/pull_request_template.md

@@ -7,7 +7,6 @@
 - [ ] All tests and CI builds passing
 - [ ] The description lists all relevant PRs included in this release _(remove this if not merging to master)_
 - [ ] e2e tests are all passing _(remove this if not merging to master)_
-- [ ] Code coverage does not significantly worsen (ideally it improves) _(remove this if not merging to master)_
 
 ### Description
 

.github/workflows/node-ci.yml

@@ -0,0 +1,107 @@
+name: Node.js CI
+
+on: [push, pull_request]
+
+jobs:
+  test-build-release:
+
+    runs-on: ubuntu-latest
+    # Add postgres for end-to-end
+    services:
+      postgres:
+        image: postgres:10.8
+        # Set postgres env variables according to test env.yml config
+        env:
+          POSTGRES_USER: postgres
+          POSTGRES_PASSWORD: postgres
+          POSTGRES_DB: postgres
+        ports:
+          - 5432:5432
+        # Set health checks to wait until postgres has started
+        options: >-
+          --health-cmd pg_isready
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 5
+    steps:
+      - uses: actions/checkout@v2
+      # install python 3.x in order to have mkdocs properly installed
+      - uses: actions/setup-python@v2
+        with:
+          python-version: '3.x'
+      - name: Install mkdocs
+        run: |
+          pip install mkdocs
+          mkdocs --version
+      - name: Use Node.js 12.x
+        uses: actions/setup-node@v1
+        with:
+          node-version: 12.x
+      - name: Install npm/yarn packages using cache
+        uses: bahmutov/npm-install@v1
+      # Inject slug vars, so that we can reference $GITHUB_HEAD_REF_SLUG for branch name
+      - name: Inject slug/short variables
+        uses: rlespinasse/[email protected]
+      - name: Check if End-to-end should run
+        run: ./scripts/check-if-e2e-should-run-on-ci.sh
+      - name: Lint code
+        run: yarn lint
+      - name: Lint messages
+        run: yarn lint-messages
+      - name: Run flow check
+        run: yarn flow
+      - name: Run tests
+        run: yarn test-client
+      - name: Build with minification
+        run: yarn run build -- --minify
+      - name: Build docs
+        run: mkdocs build
+      - name: Start MongoDB
+        if: env.SHOULD_RUN_E2E == 'true'
+        uses: supercharge/[email protected]
+        with:
+          mongodb-version: 4.2
+      - name: Add aws credentials for datatools-server
+        if: env.SHOULD_RUN_E2E == 'true'
+        run: mkdir ~/.aws && printf '%s\n' '[default]' 'aws_access_key_id=${AWS_ACCESS_KEY_ID}' 'aws_secret_access_key=${AWS_SECRET_ACCESS_KEY}' 'region=${AWS_REGION}' > ~/.aws/config
+        env:
+          AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
+          AWS_REGION: ${{ secrets.AWS_REGION }}
+          AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+      - name: Install otp-runner
+        if: env.SHOULD_RUN_E2E == 'true'
+        run: yarn global add https://github.com/ibi-group/otp-runner.git
+      - name: Run e2e tests
+        if: env.SHOULD_RUN_E2E == 'true'
+        run: yarn test-end-to-end
+        env:
+          AUTH0_API_CLIENT: ${{ secrets.AUTH0_API_CLIENT }}
+          AUTH0_API_SECRET: ${{ secrets.AUTH0_API_SECRET }}
+          AUTH0_CLIENT_ID: ${{ secrets.AUTH0_CLIENT_ID }}
+          AUTH0_DOMAIN: ${{ secrets.AUTH0_DOMAIN }}
+          AUTH0_SECRET: ${{ secrets.AUTH0_SECRET }}
+          AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
+          AWS_REGION: ${{ secrets.AWS_REGION }}
+          AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+          E2E_AUTH0_PASSWORD: ${{ secrets.E2E_AUTH0_PASSWORD }}
+          E2E_AUTH0_USERNAME: ${{ secrets.E2E_AUTH0_USERNAME }}
+          GRAPH_HOPPER_KEY: ${{ secrets.GRAPH_HOPPER_KEY }}
+          GTFS_DATABASE_PASSWORD: ${{ secrets.GTFS_DATABASE_PASSWORD }}
+          GTFS_DATABASE_URL: ${{ secrets.GTFS_DATABASE_URL }}
+          GTFS_DATABASE_USER: ${{ secrets.GTFS_DATABASE_USER }}
+          LOGS_S3_BUCKET: ${{ secrets.LOGS_S3_BUCKET }}
+          MAPBOX_ACCESS_TOKEN: ${{ secrets.MAPBOX_ACCESS_TOKEN }}
+          MONGO_DB_NAME: ${{ secrets.MONGO_DB_NAME }}
+          MS_TEAMS_WEBHOOK_URL: ${{ secrets.MS_TEAMS_WEBHOOK_URL }}
+          OSM_VEX: ${{ secrets.OSM_VEX }}
+          RUN_E2E: "true"
+          S3_BUCKET: ${{ secrets.S3_BUCKET }}
+          SPARKPOST_EMAIL: ${{ secrets.SPARKPOST_EMAIL }}
+          SPARKPOST_KEY: ${{ secrets.SPARKPOST_KEY }}
+          TRANSITFEEDS_KEY: ${{ secrets.TRANSITFEEDS_KEY }}
+      # At this point, the build is successful.
+      - name: Semantic Release
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          NPM_TOKEN: ${{ secrets.NPM_TOKEN }}
+        run: yarn semantic-release

.gitignore

@@ -28,3 +28,6 @@ env.yml-original
 .env
 !configurations/test/env.yml
 scripts/*client.json
+
+# Vs code settings
+.vscode/

.husky/.gitignore

@@ -0,0 +1 @@
+_

.husky/pre-commit

@@ -0,0 +1,4 @@
+#!/bin/sh
+. "$(dirname "$0")/_/husky.sh"
+
+npx lint-staged

.travis.yml

@@ -0,0 +1,8 @@
+# CHANGELOG
+----------------------
+ * Fix timetable previous stop time checks from checking text columns.
+ * Add feature allowing routing to avoid highways.
+ * Support Polish language and add initial translations.
+ * Fix bug displaying null continuous_pickup as '0'.
+ * Add route type selector with new extended GTFS route types.
+ * Fix bug to update continuous_pickup/dropoff values correctly.

CHANGELOG.md

@@ -259,6 +259,20 @@ async function expectSelectorToNotContainHtml (selector: string, html: string) {
   expect(innerHTML).not.toContain(html)
 }
 
+/**
+ * Checks that the expected feed version validity dates are displayed.
+ */
+async function expectFeedVersionValidityDates (startDate: string, endDate: string) {
+  await expectSelectorToContainHtml(
+    '[data-test-id="active-feed-version-validity-start"]',
+    startDate
+  )
+  await expectSelectorToContainHtml(
+    '[data-test-id="active-feed-version-validity-end"]',
+    endDate
+  )
+}
+
 /**
  * Create a new project.  Assumes that this is called while the browser is on
  * the home page.
@@ -301,6 +315,7 @@ async function deleteProject (projectId: string) {
   // verify deletion
   await goto(`http://localhost:9966/project/${projectId}`)
   await waitForSelector('.project-not-found')
+  await wait(5000, 'for previously rendered project markup to be removed')
   await expectSelectorToContainHtml('.project-not-found', projectId)
   log.info(`confirmed successful deletion of project with id ${projectId}`)
 }
@@ -549,6 +564,17 @@ async function pickColor (containerSelector: string, color: string) {
   await clearAndType(`${containerSelector} input`, color)
 }
 
+/**
+ * A helper method to choose a route type
+ * in the route editor (but not in the feed editor).
+ */
+async function pickRouteType (containerSelector: string, routeOptionId: string) {
+  await click(`${containerSelector} a`)
+  await waitForSelector(`${containerSelector} .dropdown-content`)
+  await waitForSelector(`[data-test-id="${routeOptionId}"]`)
+  await click(`[data-test-id="${routeOptionId}"] label`)
+}
+
 /**
  * A helper method to type in an autocomplete value and then select an option
  * from an react-select component.
@@ -1085,13 +1111,10 @@ describe('end-to-end', () => {
       await uploadGtfs()
 
       // wait for main tab to show up with version validity info
-      await waitForSelector('[data-test-id="feed-version-validity"]')
+      await waitForSelector('[data-test-id="active-feed-version-validity-start"]')
 
       // verify feed was uploaded
-      await expectSelectorToContainHtml(
-        '[data-test-id="feed-version-validity"]',
-        'Valid from Jan. 01, 2014 to Dec. 31, 2018'
-      )
+      await expectFeedVersionValidityDates('Jan 1, 2014', 'Dec 31, 2018')
     }, defaultTestTimeout)
 
     // this test also sets the feed source as deployable
@@ -1132,10 +1155,7 @@ describe('end-to-end', () => {
       await wait(2000, 'for feed source to update')
 
       // verify that feed was fetched and processed
-      await expectSelectorToContainHtml(
-        '[data-test-id="feed-version-validity"]',
-        'Valid from Apr. 08, 2018 to Jun. 30, 2018'
-      )
+      await expectFeedVersionValidityDates('Apr 8, 2018', 'Jun 30, 2018')
     }, defaultTestTimeout)
 
     if (doNonEssentialSteps) {
@@ -1245,10 +1265,7 @@ describe('end-to-end', () => {
         await wait(2000, 'for data to refresh')
         await waitForSelector('#feed-source-viewer-tabs')
         // verify that the previous feed is now the displayed feed
-        await expectSelectorToContainHtml(
-          '[data-test-id="feed-version-validity"]',
-          'Valid from Apr. 08, 2018 to Jun. 30, 2018'
-        )
+        await expectFeedVersionValidityDates('Apr 8, 2018', 'Jun 30, 2018')
       }, defaultTestTimeout)
     }
   })
@@ -1556,11 +1573,6 @@ describe('end-to-end', () => {
         await waitForSelector('[data-test-id="route-route_id-input-container"]')
 
         // fill out form
-        // set status to approved
-        await page.select(
-          '[data-test-id="route-status-input-container"] select',
-          '2'
-        )
 
         // set public to yes
         await page.select(
@@ -1593,9 +1605,9 @@ describe('end-to-end', () => {
         )
 
         // route type
-        await page.select(
-          '[data-test-id="route-route_type-input-container"] select',
-          '3'
+        await pickRouteType(
+          '[data-test-id="route-route_type-input-container"]',
+          'route-type-option-3'
         )
 
         // route color
@@ -1622,6 +1634,13 @@ describe('end-to-end', () => {
           'example.branding.test'
         )
 
+        // Set status to approved so the route is exported to a snapshot.
+        // Do this last, otherwise the approved status will change back to in-progress.
+        await page.select(
+          '[data-test-id="route-status-input-container"] select',
+          '2'
+        )
+
         // save
         await click('[data-test-id="save-entity-button"]')
         await wait(2000, 'for save to happen')
@@ -1648,6 +1667,13 @@ describe('end-to-end', () => {
           ' updated'
         )
 
+        // Set status to approved so the route is exported to a snapshot.
+        // Do this last, otherwise the approved status will change back to in-progress.
+        await page.select(
+          '[data-test-id="route-status-input-container"] select',
+          '2'
+        )
+
         // save
         await click('[data-test-id="save-entity-button"]')
         await wait(2000, 'for save to happen')
@@ -2338,10 +2364,14 @@ describe('end-to-end', () => {
 
           // add 1st stop
           await reactSelectOption('.pattern-stop-card', 'la', 1, true)
+          await wait(500, 'for 1st stop to be selected')
+          await click('[data-test-id="add-pattern-stop-button"]')
           await wait(2000, 'for 1st stop to save')
 
           // add 2nd stop
           await reactSelectOption('.pattern-stop-card', 'ru', 1, true)
+          await wait(500, 'for 2nd stop to be selected')
+          await click('[data-test-id="add-pattern-stop-button"]')
           await wait(2000, 'for auto-save to happen')
 
           // reload to make sure stuff was saved
@@ -2653,13 +2683,10 @@ describe('end-to-end', () => {
       await click('#feed-source-viewer-tabs-tab-')
 
       // wait for main tab to show up with version validity info
-      await waitForSelector('[data-test-id="feed-version-validity"]')
+      await waitForSelector('[data-test-id="active-feed-version-validity-start"]')
 
       // verify that snapshot was made active version
-      await expectSelectorToContainHtml(
-        '[data-test-id="feed-version-validity"]',
-        'Valid from May. 29, 2018 to May. 29, 2028'
-      )
+      await expectFeedVersionValidityDates('May 29, 2018', 'May 29, 2028')
     }, defaultTestTimeout, 'should create snapshot')
 
     // TODO: download and validate gtfs??
@@ -2688,7 +2715,10 @@ describe('end-to-end', () => {
         '[data-test-id="deployment-router-id"]'
       )
       // get rid of router id text and react tags
-      routerId = innerHTML.replace('Router ID: ', '')
+      // (remove any square brackets too)
+      routerId = innerHTML
+        .replace('Router ID: ', '')
+        .replace(/[[\]]/g, '')
 
       // confirm deployment
       await click('[data-test-id="confirm-deploy-server-button"]')
@@ -2698,9 +2728,11 @@ describe('end-to-end', () => {
     }, defaultTestTimeout + 30000) // Add thirty seconds for deployment job
 
     makeEditorEntityTest('should be able to do a trip plan on otp', async () => {
+      await wait(15000, 'for OTP to pick up the newly-built graph')
       // hit the otp endpoint
+      const url = `${OTP_ROOT}${routerId}/plan?fromPlace=37.04532992924222%2C-122.07542181015015&toPlace=37.04899494106061%2C-122.07432746887208&time=00%3A32&date=2018-07-24&mode=TRANSIT%2CWALK&maxWalkDistance=804.672&arriveBy=false&wheelchair=false&locale=en`
       const response = await fetch(
-        `${OTP_ROOT}${routerId}/plan?fromPlace=37.04532992924222%2C-122.07542181015015&toPlace=37.04899494106061%2C-122.07432746887208&time=12%3A32am&date=07-24-2018&mode=TRANSIT%2CWALK&maxWalkDistance=804.672&arriveBy=false&wheelchair=false&locale=en`,
+        url,
         {
           headers: {
             'Content-Type': 'application/json; charset=utf-8'