Merge pull request #1056 from cmu-delphi/release/delphi-epidata-0.4.4

Release Delphi Epidata 0.4.4

Author: krivard

.bumpversion.cfg

@@ -1,5 +1,5 @@
 [bumpversion]
-current_version = 0.4.3
+current_version = 0.4.4
 commit = False
 tag = False
 
@@ -14,3 +14,5 @@ tag = False
 [bumpversion:file:src/client/packaging/pypi/setup.py]
 
 [bumpversion:file:src/client/packaging/pypi/delphi_epidata/__init__.py]
+
+[bumpversion:file:dev/local/setup.cfg]

.github/workflows/ci.yaml

@@ -51,8 +51,6 @@ jobs:
 
       - name: Build docker images
         run: |
-          docker build -t delphi_database -f repos/delphi/operations/dev/docker/database/Dockerfile .
-          docker build -t delphi_python -f repos/delphi/operations/dev/docker/python/Dockerfile .
           docker build -t delphi_database_epidata -f ./repos/delphi/delphi-epidata/dev/docker/database/epidata/Dockerfile .
           docker build -t delphi_web_python -f repos/delphi/delphi-epidata/dev/docker/python/Dockerfile .
           cd ./repos/delphi/delphi-epidata

dev/docker/database/epidata/Dockerfile

@@ -13,7 +13,7 @@ ENV MYSQL_ROOT_PASSWORD pass
 # create the `epidata` database
 ENV MYSQL_DATABASE epidata
 
-# create the `epi` user account with a development-only password
+# create the `user` account with a development-only password
 ENV MYSQL_USER user
 ENV MYSQL_PASSWORD pass
 

dev/docker/database/epidata/README.md

@@ -1,10 +1,10 @@
 # `delphi_database_epidata`
 
-This image extends Delphi's database by:
+This image extends a Percona database by:
 
-- adding the `epi` user account
-- adding the `epidata` database
-- creating empty tables in `epidata`
+- adding the `user` account
+- adding the `epidata` &  other appropriate databases
+- creating empty tables in those databases
 
 To start a container from this image, run:
 

dev/docker/python/Dockerfile

@@ -1,4 +1,11 @@
-# start with the `delphi_python` image
-FROM delphi_python
+FROM python:3.8-buster
 
-RUN pip install --no-cache-dir -r repos/delphi/delphi-epidata/requirements.txt -r repos/delphi/delphi-epidata/requirements.dev.txt
+WORKDIR /usr/src/app
+
+COPY repos repos
+COPY repos/delphi/delphi-epidata/dev/docker/python/setup.sh .
+
+RUN ln -s -f /usr/share/zoneinfo/America/New_York /etc/localtime && \
+    chmod -R o+r repos/ && \
+    bash setup.sh && \
+    pip install --no-cache-dir -r repos/delphi/delphi-epidata/requirements.api.txt -r repos/delphi/delphi-epidata/requirements.dev.txt

dev/docker/python/setup.sh

@@ -0,0 +1,27 @@
+# This script sets up the correct directory structure within the `delphi_img`
+# docker image.
+
+# Some notes on package structure:
+#   - Python package names can't contain hyphens, so hyphens in repo names are
+#     replaced with underscores in the package hierarchy. (An exception is the
+#     repo `delphi-epidata`, which is renamed to simply `epidata`.)
+#   - Repos are organized such that the main code for the package is inside of
+#     a `src/` directory. When deployed, `src/` is elided. (An exception is the
+#     legacy `undef-analysis` repo, which has sources at the top-level.)
+
+# bail if anything fails
+set -e
+
+# create python package `undefx`
+mkdir undefx
+mv repos/undefx/py3tester/src undefx/py3tester
+mv repos/undefx/undef-analysis undefx/undef_analysis
+
+# create python package `delphi`
+mkdir delphi
+mv repos/delphi/operations/src delphi/operations
+mv repos/delphi/utils/src delphi/utils
+mv repos/delphi/github-deploy-repo/src delphi/github_deploy_repo
+mv repos/delphi/delphi-epidata/src delphi/epidata
+mv repos/delphi/flu-contest/src delphi/flu_contest
+mv repos/delphi/nowcast/src delphi/nowcast

dev/local/Makefile

@@ -7,10 +7,6 @@
 #
 # Checks for the delphi-net bridge and creates if it doesn't exist.
 #
-# Creates all prereq images (delphi_database, delphi_python) only if they don't
-# exist. If you need to rebuild a prereq, you're probably doing something
-# complicated, and can figure out the rebuild command on your own.
-#
 #
 # Commands:
 #
@@ -78,7 +74,7 @@ web:
 	@# Build the web_epidata image
 	@cd repos/delphi/delphi-epidata;\
 		docker build -t delphi_web_epidata -f ./devops/Dockerfile .;\
-		cd ../../../
+		cd -
 
 	@# Run the web server
 	@docker run --rm -p 127.0.0.1:10080:80 \
@@ -97,10 +93,6 @@ db:
 	@# Setup virtual network if it doesn't exist
 	@docker network ls | grep delphi-net || docker network create --driver bridge delphi-net
 
-	@# Only build prereqs if we need them
-	@docker images delphi_database | grep delphi || \
-		docker build -t delphi_database -f repos/delphi/operations/dev/docker/database/Dockerfile .
-
 	@# Build the database_epidata image
 	@docker build -t delphi_database_epidata \
 		-f repos/delphi/delphi-epidata/dev/docker/database/epidata/Dockerfile .
@@ -120,10 +112,6 @@ db:
 
 .PHONY=py
 py:
-	@# Build the python image
-	@docker build -t delphi_python \
-		-f repos/delphi/operations/dev/docker/python/Dockerfile .
-
 	@docker build -t delphi_web_python \
 		-f repos/delphi/delphi-epidata/dev/docker/python/Dockerfile .
 

dev/local/setup.cfg

@@ -1,6 +1,6 @@
 [metadata]
 name = Delphi Development
-version = 0.4.0
+version = 0.4.4
 
 [options]
 packages =

devops/Dockerfile

@@ -5,25 +5,25 @@ RUN npm ci && npm run build
 
 FROM tiangolo/meinheld-gunicorn:python3.8
 LABEL org.opencontainers.image.source=https://github.com/cmu-delphi/delphi-epidata
-# use delphi's timezome
-RUN ln -s -f /usr/share/zoneinfo/America/New_York /etc/localtime
 
-COPY requirements.txt /app/requirements_also.txt
-RUN pip install --no-cache-dir -r /tmp/requirements.txt -r requirements_also.txt
+COPY ./devops/gunicorn_conf.py /app
+COPY ./devops/start_wrapper.sh /
+COPY ./src/server/ /app/app/
+COPY --from=builder ./src/build/lib/ /app/app/lib/
+
+COPY requirements.api.txt /app/requirements_also.txt
+
+RUN ln -s -f /usr/share/zoneinfo/America/New_York /etc/localtime \
+      && rm -rf /app/app/__pycache__ /app/app/*.php \
+      && chmod -R o+r /app/app \
+      && chmod 755 /start_wrapper.sh \
+      && pip install --no-cache-dir -r /tmp/requirements.txt -r requirements_also.txt
 # the file /tmp/requirements.txt is created in the parent docker definition.  (see:
 #   https://github.com/tiangolo/meinheld-gunicorn-docker/blob/master/docker-images/python3.8.dockerfile#L5 )
 # this combined requirements installation ensures all version constrants are accounted for.
 
 # disable python stdout buffering
 ENV PYTHONUNBUFFERED 1
 
-COPY ./devops/gunicorn_conf.py /app
-COPY ./devops/start_wrapper.sh /
-COPY ./src/server/ /app/app/
-COPY --from=builder ./src/build/lib/ /app/app/lib/
-RUN rm -rf /app/app/__pycache__ /app/app/*.php \
-      && chmod -R o+r /app/app \
-      && chmod 755 /start_wrapper.sh
-
 ENTRYPOINT [ "/entrypoint.sh" ]
 CMD [ "/start_wrapper.sh" ]

docs/Gemfile.lock

@@ -211,7 +211,7 @@ GEM
       jekyll-seo-tag (~> 2.1)
     minitest (5.14.4)
     multipart-post (2.1.1)
-    nokogiri (1.13.9)
+    nokogiri (1.13.10)
       mini_portile2 (~> 2.8.0)
       racc (~> 1.4)
     octokit (4.20.0)
@@ -220,7 +220,7 @@ GEM
     pathutil (0.16.2)
       forwardable-extended (~> 2.6)
     public_suffix (4.0.6)
-    racc (1.6.0)
+    racc (1.6.1)
     rb-fsevent (0.10.4)
     rb-inotify (0.10.1)
       ffi (~> 1.0)

docs/epidata_development.md

@@ -119,8 +119,8 @@ above. The base images are built first, followed by the derived
   the Epidata API to the `delphi_web` image.
 - The
   [`delphi_database_epidata` image](https://github.com/cmu-delphi/delphi-epidata/blob/main/dev/docker/database/epidata/README.md)
-  adds the `epi` user account, `epidata` database, and relevant tables
-  (initially empty) to the `delphi_database` image.
+  adds user accounts, `epidata` & other appropriate databases, and relevant tables
+  (initially empty) to a Percona database image.
 
 From the root of your workspace, all of the images can be built as follows:
 

docs/new_endpoint_tutorial.md

@@ -277,8 +277,6 @@ docker build -t delphi_web \
   -f repos/delphi/operations/dev/docker/web/Dockerfile .
 docker build -t delphi_web_epidata \
   -f repos/delphi/delphi-epidata/dev/docker/web/epidata/Dockerfile .
-docker build -t delphi_database \
-  -f repos/delphi/operations/dev/docker/database/Dockerfile .
 docker build -t delphi_database_epidata \
   -f repos/delphi/delphi-epidata/dev/docker/database/epidata/Dockerfile .
 

integrations/server/test_covidcast.py

@@ -63,6 +63,18 @@ def _insert_placeholder_set_three(self):
     self._insert_rows(rows)
     return rows
 
+  def _insert_placeholder_set_four(self):
+    rows = [
+      self._make_placeholder_row(source='src1', signal=str(i)*5, value=i*1., stderr=i*10., sample_size=i*100.)[0]
+      for i in [1, 2, 3]
+    ] + [
+      # signal intended to overlap with the signal above
+      self._make_placeholder_row(source='src2', signal=str(i-3)*5, value=i*1., stderr=i*10., sample_size=i*100.)[0]
+      for i in [4, 5, 6]
+    ]
+    self._insert_rows(rows)
+    return rows 
+
   def test_round_trip(self):
     """Make a simple round-trip with some sample data."""
 
@@ -225,6 +237,26 @@ def test_location_wildcard(self):
       'message': 'success',
     })
 
+  def test_signal_wildcard(self):
+    """Select all signals with a wildcard query."""
+
+    # insert placeholder data
+    rows = self._insert_placeholder_set_four()
+    expected_signals = [
+      self.expected_from_row(r) for r in rows[:3]
+    ]
+
+    # make the request
+    response, _ = self.request_based_on_row(rows[0], signals="*")
+
+    self.maxDiff = None
+    # assert that the right data came back
+    self.assertEqual(response, {
+      'result': 1,
+      'epidata': expected_signals,
+      'message': 'success',
+    })
+
   def test_geo_value(self):
     """test different variants of geo types: single, *, multi."""
 

requirements.txt renamed to requirements.api.txt

@@ -1,15 +1,15 @@
+epiweeks==2.1.2
+Flask==2.2.2
 itsdangerous<2.1
 jinja2==3.0.3
-werkzeug==2.2.2
-Flask==2.2.2
-SQLAlchemy==1.4.40
 mysqlclient==2.1.1
-python-dotenv==0.15.0
+newrelic
 orjson==3.4.7
 pandas==1.2.3
+python-dotenv==0.15.0
 scipy==1.6.2
+SQLAlchemy==1.4.40
+structlog==22.1.0
 tenacity==7.0.0
-newrelic
-epiweeks==2.1.2
 typing-extensions
-structlog==22.1.0
+werkzeug==2.2.2

requirements.dev.txt

@@ -1,8 +1,25 @@
-invoke>=1.4.1
+aiohttp==3.8.3
 black>=20.8b1
-sqlalchemy-stubs>=0.3
+bump2version==1.0.1
+covidcast==0.1.5
+delphi_utils==0.3.6
+docker==6.0.1
+dropbox==11.36.0
+freezegun==1.2.2
+invoke>=1.4.1
+lxml==4.9.1
+matplotlib==3.6.2
 mypy>=0.790
-pytest
+mysql-connector==2.2.9
+numpy==1.22.4
+pycountry==22.3.5
+pymysql==1.0.2
+pytest==7.2.0
+pytest-check==1.3.0
+requests==2.28.1
+sas7bdat==2.2.3
+selenium==4.7.2
+sqlalchemy-stubs>=0.3
+structlog==22.1.0
 tenacity==7.0.0
-bump2version
-requests
+xlrd==2.0.1