Merge pull request #1267 from cmu-delphi/release/delphi-epidata-4.1.8

* Performance testing testing workflow (#1253)

* New CTIS publication (#1255)

* Newer CTIS publication

* bring loggers in sync and add multiproc capabilities (#1254)

* add syntax feature documentation (#1256)

* Newest CTIS publication

* chore: sync to www-covidcast release v3.2.7

* moving quidel signals to non-public access (#1261)

with integration tests!

---------

Co-authored-by: Dmytro Trotsko <[email protected]>

* chore: release delphi-epidata 4.1.8

---------

Co-authored-by: Dmytro Trotsko <[email protected]>
Co-authored-by: Rostyslav Zatserkovnyi <[email protected]>
Co-authored-by: Alex Reinhart <[email protected]>
Co-authored-by: nmdefries <[email protected]>
Co-authored-by: melange396 <[email protected]>
Co-authored-by: minhkhul <[email protected]>
Co-authored-by: minhkhul <[email protected]>
Co-authored-by: melange396 <[email protected]>

Author: 6 people

.bumpversion.cfg

@@ -1,5 +1,5 @@
 [bumpversion]
-current_version = 4.1.7
+current_version = 4.1.8
 commit = False
 tag = False
 

.github/workflows/performance-tests-periodic.yml

@@ -0,0 +1,102 @@
+name: One-time performance testing - 9th August 2023
+
+# Run "At every 30th minute on day-of-month 9 in August"
+on:
+  schedule:
+    - cron: '*/30 * 9 8 *'
+
+# Add some extra perms to comment on a PR
+permissions:
+  pull-requests: write
+  contents: read
+
+jobs:
+  run-perftests:
+    runs-on: ubuntu-latest
+    outputs:
+      request_count: ${{ steps.output.outputs.request_count }}
+      failure_count: ${{ steps.output.outputs.failure_count }}
+      med_time: ${{ steps.output.outputs.med_time }}
+      avg_time: ${{ steps.output.outputs.avg_time }}
+      min_time: ${{ steps.output.outputs.min_time }}
+      max_time: ${{ steps.output.outputs.max_time }}
+      requests_per_sec: ${{ steps.output.outputs.requests_per_sec }}
+    steps:
+      - name: Set up WireGuard
+        uses: egor-tensin/[email protected]
+        with:
+          endpoint: '${{ secrets.WG_PERF_ENDPOINT }}'
+          endpoint_public_key: '${{ secrets.WG_PERF_ENDPOINT_PUBLIC_KEY }}'
+          ips: '${{ secrets.WG_PERF_IPS }}'
+          allowed_ips: '${{ secrets.WG_PERF_ALLOWED_IPS }}'
+          private_key: '${{ secrets.WG_PERF_PRIVATE_KEY }}'
+      - name: Check out repository
+        uses: actions/checkout@v3
+      - name: Set up repository # mimics install.sh in the README except that delphi is cloned from the PR rather than main
+        run: |
+          cd ..
+          mkdir -p driver/repos/delphi
+          cd driver/repos/delphi
+          git clone https://github.com/cmu-delphi/operations
+          git clone https://github.com/cmu-delphi/utils
+          git clone https://github.com/cmu-delphi/flu-contest
+          git clone https://github.com/cmu-delphi/nowcast
+          cd ../../
+
+          cd ..
+          cp -R delphi-epidata driver/repos/delphi/delphi-epidata
+          cd -
+
+          ln -s repos/delphi/delphi-epidata/dev/local/Makefile
+      - name: Build & run epidata
+        run: |
+          cd ../driver
+          sudo make web sql="${{ secrets.DB_CONN_STRING }}"
+          sudo make redis
+      - name: Check out delphi-admin
+        uses: actions/checkout@v3
+        with:
+          repository: cmu-delphi/delphi-admin
+          token: ${{ secrets.CMU_DELPHI_DEPLOY_MACHINE_PAT }}
+          path: delphi-admin
+      - name: Build & run Locust
+        continue-on-error: true # sometimes ~2-5 queries fail, we shouldn't end the run if that's the case
+        run: |
+          cd delphi-admin/load-testing/locust
+          docker build -t locust .
+          export CSV=v4-requests-small.csv
+          touch output_stats.csv && chmod 666 output_stats.csv
+          touch output_stats_history.csv && chmod 666 output_stats_history.csv
+          touch output_failures.csv && chmod 666 output_failures.csv
+          touch output_exceptions.csv && chmod 666 output_exceptions.csv
+          docker run --net=host -v $PWD:/mnt/locust -e CSV="/mnt/locust/${CSV}" locust -f /mnt/locust/v4.py --host http://127.0.0.1:10080/ --users 10 --spawn-rate 1 --headless -i "$(cat ${CSV} | wc -l)" --csv=/mnt/locust/output
+      - name: Produce output for summary
+        id: output
+        uses: jannekem/run-python-script-action@v1
+        with:
+          script: |
+            import os
+
+            def write_string(name, value):
+              with open(os.environ['GITHUB_OUTPUT'], 'a') as fh:
+                print(f'{name}={value}', file=fh)
+
+            def write_float(name, value):
+              write_string(name, "{:.2f}".format(float(value)))
+
+            with open("delphi-admin/load-testing/locust/output_stats.csv", "r", encoding="utf-8", errors="ignore") as scraped:
+              final_line = scraped.readlines()[-1].split(",")
+              write_string('request_count', final_line[2])
+              write_string('failure_count', final_line[3])
+              write_float('med_time', final_line[4])
+              write_float('avg_time', final_line[5])
+              write_float('min_time', final_line[6])
+              write_float('max_time', final_line[7])
+              write_float('requests_per_sec', final_line[9])
+
+      - name: Archive results as artifacts
+        uses: actions/upload-artifact@v3
+        with:
+          name: locust-output
+          path: |
+            delphi-admin/load-testing/locust/output_*.csv

dev/local/setup.cfg

@@ -1,6 +1,6 @@
 [metadata]
 name = Delphi Development
-version = 4.1.7
+version = 4.1.8
 
 [options]
 packages =

docs/api/covidcast-signals/quidel-inactive.md

@@ -15,6 +15,7 @@ grand_parent: COVIDcast Main Endpoint
 1. TOC
 {:toc}
 
+## Accessibility: Delphi-internal only
 
 ## COVID-19 Tests
 These signals are still active. Documentation is available on the [Quidel page](quidel.md).

docs/api/covidcast-signals/quidel.md

@@ -15,6 +15,8 @@ grand_parent: COVIDcast Main Endpoint
 1. TOC
 {:toc}
 
+## Accessibility: Delphi-internal only
+
 ## COVID-19 Tests
 
 * **Earliest issue available:** July 29, 2020 

docs/api/covidcast.md

@@ -113,6 +113,16 @@ and lists.
 The current set of signals available for each data source is returned by the
 [`covidcast_meta`](covidcast_meta.md) endpoint.
 
+#### Alternate Required Parameters
+
+The following parameters help specify multiple source-signal, timetype-timevalue or geotype-geovalue pairs. Use them instead of the usual required parameters.
+
+| Parameter | Replaces | Format | Description | Example |
+| --- | --- | --- | --- | --- |
+| `signal` | `data_source`, `signal` | `signal={source}:{signal1},{signal2}` | Specify multiple source-signal pairs, grouped by source | `signal=src1:sig1`, `signal=src1:sig1,sig2`, `signal=src1:*`, `signal=src1:sig1;src2:sig3` |
+| `time` | `time_type`, `time_values` | `time={timetype}:{timevalue1},{timevalue2}` | Specify multiple timetype-timevalue pairs, grouped by timetype | `time=day:*`, `time=day:20201201`, `time=day:20201201,20201202`, `time=day:20201201-20201204` |
+| `geo` | `geo_type`, `geo_value` | `geo={geotype}:{geovalue1},{geovalue2}` | Specify multiple geotype-geovalue pairs, grouped by geotype | `geo=fips:*`, `geo=fips:04019`, `geo=fips:04019,19143`, `geo=fips:04019;msa:40660`, `geo=fips:*;msa:*` |
+
 #### Optional
 
 Estimates for a specific `time_value` and `geo_value` are sometimes updated
@@ -209,6 +219,12 @@ The `fields` parameter can be used to limit which fields are included in each re
 
 https://api.delphi.cmu.edu/epidata/covidcast/?data_source=fb-survey&signal=smoothed_cli&time_type=day&geo_type=county&time_values=20200406-20200410&geo_value=06001
 
+or
+
+https://api.delphi.cmu.edu/epidata/covidcast/?signal=fb-survey:smoothed_cli&time=day:20200406-20200410&geo=county:06001
+
+Both of these URLs are equivalent and can be used to get the following result:
+
 ```json
 {
   "result": 1,

docs/api/covidcast_signals.md

@@ -36,7 +36,6 @@ dashboard](https://delphi.cmu.edu/covidcast/):
 | Early Indicators | COVID-Like Symptoms | [`fb-survey`](covidcast-signals/fb-survey.md) | `smoothed_wcli` |
 | Early Indicators | COVID-Like Symptoms in Community | [`fb-survey`](covidcast-signals/fb-survey.md) | `smoothed_whh_cmnty_cli` |
 | Early Indicators | COVID-Related Doctor Visits | [`doctor-visits`](covidcast-signals/doctor-visits.md) | `smoothed_adj_cli` |
-| Cases and Testing | COVID Antigen Test Positivity (Quidel) | [`quidel`](covidcast-signals/quidel.md) | `covid_ag_smoothed_pct_positive` |
 | Cases and Testing | COVID Cases | [`jhu-csse`](covidcast-signals/jhu-csse.md) | `confirmed_7dav_incidence_prop` |
 | Late Indicators | COVID Hospital Admissions | [`hhs`](covidcast-signals/hhs.md) | `confirmed_admissions_covid_1d_prop_7dav` |
 | Late Indicators | Deaths | [`jhu-csse`](covidcast-signals/jhu-csse.md) | `deaths_7dav_incidence_prop` |

docs/symptom-survey/publications.md

@@ -26,6 +26,17 @@ Pandemic"](https://www.pnas.org/topic/548) in *PNAS*:
 
 Research publications using the survey data include:
 
+- C.K. Ettman, E. Badillo Goicoechea, and E.A. Stuart (2023). [Evolution of
+  depression and anxiety over the COVID-19 pandemic and across demographic
+  groups in a large sample of U.S. adults](https://doi.org/10.1016/j.focus.2023.100140).
+  *AJPM Focus*.
+- M. Rubinstein, Z. Branson, and E.H. Kennedy (2023). [Heterogeneous
+  interventional effects with multiple mediators: Semiparametric and
+  nonparametric approaches](https://doi.org/10.1515/jci-2022-0070). *Journal of
+  Causal Inference* 11 (1), 20220070. 
+- Uyheng, J., Robertson, D.C. & Carley, K.M. (2023). [Bridging online and offline
+  dynamics of the face mask infodemic](https://doi.org/10.1186/s44247-023-00026-z).
+  *BMC Digital Health* 1, 27. 
 - Kobayashi H, Saenz-Escarcega R, Fulk A, Agusto FB (2023). [Understanding
   mental health trends during COVID-19 pandemic in the United States using
   network analysis](https://doi.org/10.1371/journal.pone.0286857). *PLoS

integrations/server/test_covidcast_endpoints.py

@@ -26,7 +26,19 @@ def localSetUp(self):
         # reset the `covidcast_meta_cache` table (it should always have one row)
         self._db._cursor.execute('update covidcast_meta_cache set timestamp = 0, epidata = "[]"')
 
-    def _fetch(self, endpoint="/", is_compatibility=False, **params):
+        cur = self._db._cursor
+        # NOTE: we must specify the db schema "epidata" here because the cursor/connection are bound to schema "covid"
+        cur.execute("TRUNCATE TABLE epidata.api_user")
+        cur.execute("TRUNCATE TABLE epidata.user_role")
+        cur.execute("TRUNCATE TABLE epidata.user_role_link")
+        cur.execute("INSERT INTO epidata.api_user (api_key, email) VALUES ('quidel_key', 'quidel_email')")
+        cur.execute("INSERT INTO epidata.user_role (name) VALUES ('quidel')")
+        cur.execute(
+            "INSERT INTO epidata.user_role_link (user_id, role_id) SELECT api_user.id, user_role.id FROM epidata.api_user JOIN epidata.user_role WHERE api_key='quidel_key' and user_role.name='quidel'"
+        )
+        cur.execute("INSERT INTO epidata.api_user (api_key, email) VALUES ('key', 'email')")
+
+    def _fetch(self, endpoint="/", is_compatibility=False, auth=AUTH, **params):
         # make the request
         if is_compatibility:
             url = BASE_URL_OLD
@@ -37,7 +49,7 @@ def _fetch(self, endpoint="/", is_compatibility=False, **params):
                 params.setdefault("data_source", params.get("source"))
         else:
             url = f"{BASE_URL}{endpoint}"
-        response = requests.get(url, params=params, auth=AUTH)
+        response = requests.get(url, params=params, auth=auth)
         response.raise_for_status()
         return response.json()
 
@@ -67,6 +79,28 @@ def test_basic(self):
             out = self._fetch("/", signal=first.signal_pair(), geo=first.geo_pair(), time="day:*")
             self.assertEqual(len(out["epidata"]), len(rows))
 
+    def test_basic_restricted_source(self):
+        """Request a signal from the / endpoint."""
+        rows = [CovidcastTestRow.make_default_row(time_value=2020_04_01 + i, value=i, source="quidel") for i in range(10)]
+        first = rows[0]
+        self._insert_rows(rows)
+
+        with self.subTest("validation"):
+            out = self._fetch("/")
+            self.assertEqual(out["result"], -1)
+
+        with self.subTest("no_roles"):
+            out = self._fetch("/", signal=first.signal_pair(), geo=first.geo_pair(), time="day:*")
+            self.assertEqual(len(out["epidata"]), 0)
+
+        with self.subTest("no_api_key"):
+            out = self._fetch("/", auth=None, signal=first.signal_pair(), geo=first.geo_pair(), time="day:*")
+            self.assertEqual(len(out["epidata"]), 0)
+
+        with self.subTest("quidel_role"):
+            out = self._fetch("/", auth=("epidata", "quidel_key"), signal=first.signal_pair(), geo=first.geo_pair(), time="day:*")
+            self.assertEqual(len(out["epidata"]), len(rows))
+
     def test_compatibility(self):
         """Request at the /api.php endpoint."""
         rows = [CovidcastTestRow.make_default_row(source="src", signal="sig", time_value=2020_04_01 + i, value=i) for i in range(10)]
@@ -271,6 +305,35 @@ def test_meta(self):
             out = self._fetch("/meta", signal=f"{first.source}:X")
             self.assertEqual(len(out), 0)
 
+    def test_meta_restricted(self):
+        """Request 'restricted' signals from the /meta endpoint."""
+        # NOTE: this method is nearly identical to ./test_covidcast_meta.py:test_restricted_sources()
+        #       ...except the self._fetch() methods are different, as is the format of those methods' outputs
+        #       (the other covidcast_meta endpoint uses APrinter, this one returns its own unadulterated json).
+        #       additionally, the sample data used here must match entries (that is, named sources and signals)
+        #       from covidcast_utils.model.data_sources (the `data_sources` variable from file
+        #       src/server/endpoints/covidcast_utils/model.py, which is created by the _load_data_sources() method
+        #       and fed by src/server/endpoints/covidcast_utils/db_sources.csv, but also surreptitiously augmened
+        #       by _load_data_signals() which attaches a list of signals to each source,
+        #       in turn fed by src/server/endpoints/covidcast_utils/db_signals.csv)
+
+        # insert data from two different sources, one restricted/protected (quidel), one not
+        self._insert_rows([
+            CovidcastTestRow.make_default_row(source="quidel", signal="raw_pct_negative"),
+            CovidcastTestRow.make_default_row(source="hhs", signal="confirmed_admissions_covid_1d")
+        ])
+
+        # update metadata cache
+        update_cache(args=None)
+
+        # verify unauthenticated (no api key) or unauthorized (user w/o privilege) only see metadata for one source
+        self.assertEqual(len(self._fetch("/meta", auth=None)), 1)
+        self.assertEqual(len(self._fetch("/meta", auth=AUTH)), 1)
+
+        # verify authorized user sees metadata for both sources
+        qauth = ('epidata', 'quidel_key')
+        self.assertEqual(len(self._fetch("/meta", auth=qauth)), 2)
+
     def test_coverage(self):
         """Request a signal from the /coverage endpoint."""
 

integrations/server/test_covidcast_meta.py

@@ -9,6 +9,7 @@
 
 #first party
 from delphi_utils import Nans
+from delphi.epidata.acquisition.covidcast.test_utils import CovidcastBase, CovidcastTestRow
 from delphi.epidata.maintenance.covidcast_meta_cache_updater import main as update_cache
 import delphi.operations.secrets as secrets
 
@@ -17,7 +18,7 @@
 AUTH = ('epidata', 'key')
 
 
-class CovidcastMetaTests(unittest.TestCase):
+class CovidcastMetaTests(CovidcastBase):
   """Tests the `covidcast_meta` endpoint."""
 
   src_sig_lookups = {
@@ -48,7 +49,7 @@ class CovidcastMetaTests(unittest.TestCase):
          %d, %d)
   '''
 
-  def setUp(self):
+  def localSetUp(self):
     """Perform per-test setup."""
 
     # connect to the `epidata` database and clear the `covidcast` table
@@ -68,6 +69,17 @@ def setUp(self):
     # reset the `covidcast_meta_cache` table (it should always have one row)
     cur.execute('update covidcast_meta_cache set timestamp = 0, epidata = "[]"')
 
+    # NOTE: we must specify the db schema "epidata" here because the cursor/connection are bound to schema "covid"
+    cur.execute("TRUNCATE TABLE epidata.api_user")
+    cur.execute("TRUNCATE TABLE epidata.user_role")
+    cur.execute("TRUNCATE TABLE epidata.user_role_link")
+    cur.execute("INSERT INTO epidata.api_user (api_key, email) VALUES ('quidel_key', 'quidel_email')")
+    cur.execute("INSERT INTO epidata.user_role (name) VALUES ('quidel')")
+    cur.execute(
+      "INSERT INTO epidata.user_role_link (user_id, role_id) SELECT api_user.id, user_role.id FROM epidata.api_user JOIN epidata.user_role WHERE api_key='quidel_key' and user_role.name='quidel'"
+    )
+    cur.execute("INSERT INTO epidata.api_user (api_key, email) VALUES ('key', 'email')")
+
     # populate dimension tables
     for (src,sig) in self.src_sig_lookups:
         cur.execute('''
@@ -93,7 +105,7 @@ def setUp(self):
     secrets.db.epi = ('user', 'pass')
 
 
-  def tearDown(self):
+  def localTearDown(self):
     """Perform per-test teardown."""
     self.cur.close()
     self.cnx.close()
@@ -138,10 +150,10 @@ def _get_id(self):
     return self.id_counter
 
   @staticmethod
-  def _fetch(**kwargs):
+  def _fetch(auth=AUTH, **kwargs):
     params = kwargs.copy()
     params['endpoint'] = 'covidcast_meta'
-    response = requests.get(BASE_URL, params=params, auth=AUTH)
+    response = requests.get(BASE_URL, params=params, auth=auth)
     response.raise_for_status()
     return response.json()
 
@@ -161,6 +173,26 @@ def test_round_trip(self):
       'message': 'success',
     })
 
+  def test_restricted_sources(self):
+    # NOTE: this method is nearly identical to ./test_covidcast_endpoints.py:test_meta_restricted()
+
+    # insert data from two different sources, one restricted/protected (quidel), one not
+    self._insert_rows([
+      CovidcastTestRow.make_default_row(source="quidel"),
+      CovidcastTestRow.make_default_row(source="not-quidel")
+    ])
+
+    # generate metadata cache
+    update_cache(args=None)
+
+    # verify unauthenticated (no api key) or unauthorized (user w/o privilege) only see metadata for one source
+    self.assertEqual(len(self._fetch(auth=None)['epidata']), 1)
+    self.assertEqual(len(self._fetch(auth=AUTH)['epidata']), 1)
+
+    # verify authorized user sees metadata for both sources
+    qauth = ('epidata', 'quidel_key')
+    self.assertEqual(len(self._fetch(auth=qauth)['epidata']), 2)
+
   def test_filter(self):
     """Test filtering options some sample data."""
 

src/client/delphi_epidata.R

@@ -15,7 +15,7 @@ Epidata <- (function() {
   # API base url
   BASE_URL <- getOption('epidata.url', default = 'https://api.delphi.cmu.edu/epidata/')
 
-  client_version <- '4.1.7'
+  client_version <- '4.1.8'
 
   auth <- getOption("epidata.auth", default = NA)
 

src/client/delphi_epidata.js

@@ -22,7 +22,7 @@
   }
 })(this, function (exports, fetchImpl, jQuery) {
   const BASE_URL = "https://api.delphi.cmu.edu/epidata/";
-  const client_version = "4.1.7";
+  const client_version = "4.1.8";
 
   // Helper function to cast values and/or ranges to strings
   function _listitem(value) {