diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index 6c6db3fe6..6110ceb5a 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -39,11 +39,11 @@ jobs: steps: - name: Checkout repository - uses: actions/checkout@v2 + uses: actions/checkout@v5 # Initializes the CodeQL tools for scanning. - name: Initialize CodeQL - uses: github/codeql-action/init@v1 + uses: github/codeql-action/init@v4 with: languages: ${{ matrix.language }} # If you wish to specify custom queries, you can do so here or in a config file. @@ -54,7 +54,7 @@ jobs: # Autobuild attempts to build any compiled languages (C/C++, C#, or Java). # If this step fails, then you should remove it and run the build manually (see below) - name: Autobuild - uses: github/codeql-action/autobuild@v1 + uses: github/codeql-action/autobuild@v4 # ℹī¸ Command-line programs to run using the OS shell. # 📚 https://git.io/JvXDl @@ -68,4 +68,4 @@ jobs: # make release - name: Perform CodeQL Analysis - uses: github/codeql-action/analyze@v1 + uses: github/codeql-action/analyze@v4 diff --git a/.github/workflows/testing.yml b/.github/workflows/testing.yml index f71000bae..fe87820e6 100644 --- a/.github/workflows/testing.yml +++ b/.github/workflows/testing.yml @@ -34,9 +34,10 @@ jobs: - name: Install dependencies run: | python${{ matrix.python-version }} -m pip install --upgrade "pip<25.3" build pip-tools pre-commit - python${{ matrix.python-version }} -m piptools compile --extra dev -o requirements.txt mpcontribs-client/setup.py mpcontribs-api/setup.py + python${{ matrix.python-version }} -m piptools compile --extra dev -o requirements.txt mpcontribs-client/pyproject.toml mpcontribs-api/pyproject.toml python${{ matrix.python-version }} -m pip install -r requirements.txt - cd mpcontribs-api && python${{ matrix.python-version }} -m pip install --no-deps . + cd mpcontribs-api + python${{ matrix.python-version }} -m pip install --no-deps . - name: Set SSL_CERT_FILE (Linux) if: matrix.os == 'ubuntu-latest' || matrix.os == 'macos-latest' run: | @@ -50,7 +51,6 @@ jobs: - name: Run pre-commit run: | - python${{ matrix.python-version }} -m pip install pre-commit pre-commit install pre-commit run --all-files @@ -61,9 +61,10 @@ jobs: shell: bash run: | cd mpcontribs-client + python${{ matrix.python-version }} -m pip install --no-deps . python${{ matrix.python-version }} -m flake8 --max-line-length 100 python${{ matrix.python-version }} -m pycodestyle --max-line-length 100 . - python${{ matrix.python-version }} -m pytest -v -s --cov=mpcontribs/client --cov-report=term-missing --cov-report=xml --ignore=bravado + python${{ matrix.python-version }} -m pytest -n auto -v -s --cov=mpcontribs/client --cov-report=term-missing --cov-report=xml --ignore=bravado python${{ matrix.python-version }} -m build --outdir ../dist - name: Install lux and test with pytest diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 3f37230e8..50a0eb904 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -28,12 +28,6 @@ repos: files: cloudformation/.*\.(json|yml|yaml)$ args: ["--ignore-checks=E3030,E3001,E3002,E3012"] - #- repo: https://gitlab.com/pycqa/flake8 - # rev: 3.7.9 - # hooks: - # - id: flake8 - # args: [--max-line-length=99] - - repo: https://github.com/psf/black rev: 25.9.0 hooks: @@ -47,3 +41,8 @@ repos: - --drop-empty-cells - --strip-init-cells - --extra-keys=metadata.kernelspec + + - repo: https://github.com/asottile/pyupgrade + rev: v3.15.1 + hooks: + - id: pyupgrade diff --git a/mpcontribs-api/Dockerfile b/mpcontribs-api/Dockerfile index 5986dfd7d..10513c25c 100644 --- a/mpcontribs-api/Dockerfile +++ b/mpcontribs-api/Dockerfile @@ -7,7 +7,7 @@ RUN apt-get update && apt-get install -y --no-install-recommends gcc git g++ lib ENV PIP_FLAGS "--no-cache-dir --compile" COPY requirements/deployment.txt ./requirements.txt RUN pip install $PIP_FLAGS -r requirements.txt -COPY setup.py . +COPY pyproject.toml . COPY mpcontribs mpcontribs RUN pip install $PIP_FLAGS --no-deps . #ENV SETUPTOOLS_SCM_PRETEND_VERSION 0.0.0 diff --git a/mpcontribs-api/mpcontribs/api/__init__.py b/mpcontribs-api/mpcontribs/api/__init__.py index 515d2b43f..e61583b4b 100644 --- a/mpcontribs-api/mpcontribs/api/__init__.py +++ b/mpcontribs-api/mpcontribs/api/__init__.py @@ -8,6 +8,7 @@ from email.message import EmailMessage from importlib import import_module +from importlib.metadata import version from websocket import create_connection from flask import Flask, current_app, request, jsonify from flask_marshmallow import Marshmallow @@ -26,6 +27,11 @@ from notebook.gateway.managers import GatewayClient from requests.exceptions import ConnectionError, Timeout +try: + __version__ = version("mpcontribs-api") +except Exception: + # package is not installed + pass delimiter, max_depth = ".", 7 # = MAX_NESTING + 2 from client invalidChars = set(punctuation.replace("*", "").replace("|", "") + whitespace) diff --git a/mpcontribs-api/mpcontribs/api/config.py b/mpcontribs-api/mpcontribs/api/config.py index 117d8ffc9..17732a602 100644 --- a/mpcontribs-api/mpcontribs/api/config.py +++ b/mpcontribs-api/mpcontribs/api/config.py @@ -2,11 +2,10 @@ """configuration module for MPContribs Flask API""" import os -import datetime import json import gzip -from semantic_version import Version +from mpcontribs.api import __version__ formulae_path = os.path.join( os.path.dirname(__file__), "contributions", "formulae.json.gz" @@ -15,12 +14,6 @@ with gzip.open(formulae_path) as f: FORMULAE = json.load(f) -now = datetime.datetime.now() -VERSION = str(Version( - major=now.year, minor=now.month, patch=now.day, - prerelease=(str(now.hour), str(now.minute)) -)) - JSON_ADD_STATUS = False SECRET_KEY = "super-secret" # TODO in local prod config @@ -65,7 +58,7 @@ "title": "MPContribs API", "description": "Operations to contribute, update and retrieve materials data on Materials Project", "termsOfService": "https://materialsproject.org/terms", - "version": VERSION, + "version": __version__, "contact": { "name": "MPContribs", "email": "contribs@materialsproject.org", diff --git a/mpcontribs-api/pyproject.toml b/mpcontribs-api/pyproject.toml new file mode 100644 index 000000000..c9f6abbb0 --- /dev/null +++ b/mpcontribs-api/pyproject.toml @@ -0,0 +1,103 @@ +[build-system] +requires = ["setuptools>=80.0.0","setuptools-scm>=8"] +build-backend = "setuptools.build_meta" + +[tool.setuptools_scm] +root = ".." +relative_to = "__file__" + +[tool.setuptools] +include-package-data = true + +[tool.setuptools.packages.find] +where = ["."] +exclude = ["scripts","supervisord"] +include = ["mpcontribs.api"] + +[project] +name = "mpcontribs-api" +dynamic = ["version"] +requires-python = ">=3.11" +description="API for community-contributed Materials Project data" +license = {text = "MIT"} +authors = [ + {name = "Patrick Huck", email = "phuck@lbl.gov"}, + {name = "The Materials Project", email="feedback@materialsproject.org"}, +] +dependencies = [ + "numpy", + "apispec<6", + "asn1crypto", + "blinker", + "boltons", + "css-html-js-minify", + "dateparser", + "ddtrace", + "dnspython", + "filetype", + "flasgger-tschaume>=0.9.7", + "flask-compress", + "flask-marshmallow", + "flask-mongorest-mpcontribs>=3.2.1", + "Flask-RQ2", + "gunicorn[gevent]", + "jinja2", + "json2html", + "marshmallow<4", + "more-itertools", + "nbformat", + "notebook<7", + "pint>=0.24", + "psycopg2-binary", + "pymatgen", + "pyopenssl", + "python-snappy", + "rq<=2.3.2", # see https://github.com/rq/Flask-RQ2/issues/620 + "supervisor", + "setproctitle", + "uncertainties", + "websocket_client", + "zstandard", +] + +[project.urls] +Homepage = "https://github.com/materialsproject/MPContribs" +Documentation = "https://docs.materialsproject.org/services/mpcontribs" + +[project.optional-dependencies] +dev = [ + "flake8", + "pytest", + "pytest-flake8", + "pytest-pycodestyle", + "pytest-xdist", +] +all = [ + "mpcontribs-api[dev]" +] + +[tool.pytest] +markers = [ + "base: basic resource testing", + "extra: all extra views", +] + +[tool.pycodestyle] +count = true +ignore = ["E121","E123","E126","E133","E226","E241","E242","E704","W503","W504","W505","E741","W605"] +max-line-length = 120 +statistics = true +exclude = ["flasgger","flask-mongorest"] + +[tool.flake8] +exclude = [".git","__pycache__","tests","flasgger","flask-mongorest"] +extend-ignore = ["E741",] +max-line-length = 120 + +[tool.pydocstyle] +ignore = ["D105","D2","D4"] + +[tool.mypy] +ignore_missing_imports = true +namespace_packages = true +python_version = 3.11 diff --git a/mpcontribs-api/setup.cfg b/mpcontribs-api/setup.cfg deleted file mode 100644 index 4f3ee2f11..000000000 --- a/mpcontribs-api/setup.cfg +++ /dev/null @@ -1,25 +0,0 @@ -[tool:pytest] -markers = - base: basic resource testing - extra: all extra views - -[pycodestyle] -count = True -ignore = E121,E123,E126,E133,E226,E241,E242,E704,W503,W504,W505,E741,W605 -max-line-length = 120 -statistics = True -exclude = flasgger,flask-mongorest - -[flake8] -exclude = .git,__pycache__,tests,flasgger,flask-mongorest -# max-complexity = 10 -extend-ignore = E741 -max-line-length = 120 - -[pydocstyle] -ignore = D105,D2,D4 - -[mypy] -ignore_missing_imports = True -namespace_packages = True -python_version = 3.7 diff --git a/mpcontribs-api/setup.py b/mpcontribs-api/setup.py deleted file mode 100644 index b343d13be..000000000 --- a/mpcontribs-api/setup.py +++ /dev/null @@ -1,52 +0,0 @@ -# -*- coding: utf-8 -*- -import datetime -from setuptools import setup - -setup( - name="mpcontribs-api", - version=datetime.datetime.today().strftime("%Y.%m.%d"), - description="API for community-contributed Materials Project data", - author="Patrick Huck", - author_email="phuck@lbl.gov", - url="https://mpcontribs.org", - packages=["mpcontribs.api"], - install_requires=[ - "numpy", - "apispec<6", - "asn1crypto", - "blinker", - "boltons", - "css-html-js-minify", - "dateparser", - "ddtrace", - "dnspython", - "filetype", - "flasgger-tschaume>=0.9.7", - "flask-compress", - "flask-marshmallow", - "flask-mongorest-mpcontribs>=3.2.1", - "Flask-RQ2", - "gunicorn[gevent]", - "jinja2", - "json2html", - "marshmallow<4", - "more-itertools", - "nbformat", - "notebook<7", - "pint>=0.24", - "psycopg2-binary", - "pymatgen", - "pyopenssl", - "python-snappy", - "rq<=2.3.2", # see https://github.com/rq/Flask-RQ2/issues/620 - "semantic-version", - "supervisor", - "setproctitle", - "uncertainties", - "websocket_client", - "zstandard", - ], - extras_require={"dev": ["pytest", "flake8"]}, - license="MIT", - zip_safe=False, -) diff --git a/mpcontribs-client/mpcontribs/client/__init__.py b/mpcontribs-client/mpcontribs/client/__init__.py index 16eb8d7b2..3be88b2c5 100644 --- a/mpcontribs-client/mpcontribs/client/__init__.py +++ b/mpcontribs-client/mpcontribs/client/__init__.py @@ -1,5 +1,5 @@ -# -*- coding: utf-8 -*- import io +import importlib.metadata import sys import os import ujson @@ -13,14 +13,12 @@ import functools import requests import logging -import datetime from inspect import getfullargspec from math import isclose -from semantic_version import Version from requests.exceptions import RequestException from bson.objectid import ObjectId -from typing import Union, Type, Optional +from typing import Type from tqdm.auto import tqdm from hashlib import md5 from pathlib import Path @@ -59,7 +57,12 @@ from plotly.express._chart_types import line as line_chart from cachetools import cached, LRUCache from cachetools.keys import hashkey -from pymatgen.core import SETTINGS + +try: + __version__ = importlib.metadata.version("mpcontribs-client") +except Exception: + # package is not installed + pass RETRIES = 3 MAX_WORKERS = 3 @@ -88,6 +91,7 @@ SUPPORTED_FILETYPES = (Gz, Jpeg, Png, Gif, Tiff) SUPPORTED_MIMES = [t().mime for t in SUPPORTED_FILETYPES] DEFAULT_DOWNLOAD_DIR = Path.home() / "mpcontribs-downloads" +VALID_API_KEY_ALIASES = ["MPCONTRIBS_API_KEY", "MP_API_KEY", "PMG_MAPI_KEY"] j2h = Json2Html() pd.options.plotting.backend = "plotly" @@ -126,7 +130,7 @@ class LogFilter(logging.Filter): def __init__(self, level, *args, **kwargs): self.level = level - super(LogFilter, self).__init__(*args, **kwargs) + super().__init__(*args, **kwargs) def filter(self, record): return record.levelno < self.level @@ -144,7 +148,7 @@ class TqdmToLogger(io.StringIO): buf = "" def __init__(self, logger, level=None): - super(TqdmToLogger, self).__init__() + super().__init__() self.logger = logger self.level = level or logging.INFO @@ -177,7 +181,7 @@ class MPContribsClientError(ValueError): """custom error for mpcontribs-client""" -def get_md5(d): +def get_md5(d) -> str: s = ujson.dumps(d, sort_keys=True).encode("utf-8") return md5(s).hexdigest() @@ -478,7 +482,7 @@ def unpack(self) -> str: return unpacked - def write(self, outdir: Optional[Union[str, Path]] = None) -> Path: + def write(self, outdir: str | Path | None = None) -> Path: """Write attachment to file using its name Args: @@ -490,7 +494,7 @@ def write(self, outdir: Optional[Union[str, Path]] = None) -> Path: path.write_bytes(content) return path - def display(self, outdir: Optional[Union[str, Path]] = None): + def display(self, outdir: str | Path | None = None): """Display Image/FileLink for attachment if in IPython/Jupyter Args: @@ -519,7 +523,7 @@ def name(self) -> str: return self["name"] @classmethod - def from_data(cls, data: Union[list, dict], name: str = "attachment"): + def from_data(cls, data: list | dict, name: str = "attachment"): """Construct attachment from data dict or list Args: @@ -539,7 +543,7 @@ def from_data(cls, data: Union[list, dict], name: str = "attachment"): ) @classmethod - def from_file(cls, path: Union[Path, str]): + def from_file(cls, path: str | Path): """Construct attachment from file Args: @@ -616,7 +620,7 @@ def from_list(cls, elements: list): return attachments @classmethod - def from_data(cls, data: Union[list, dict], prefix: str = "attachment"): + def from_data(cls, data: list | dict, prefix: str = "attachment"): """Construct list of attachments from data dict or list Args: @@ -830,32 +834,24 @@ def _expand_params(protocol, host, version, projects_json, apikey=None): def _version(url): retries, max_retries = 0, 3 protocol = urlparse(url).scheme - is_mock_test = "pytest" in sys.modules and protocol == "http" - - if is_mock_test: - now = datetime.datetime.now() - return Version( - major=now.year, - minor=now.month, - patch=now.day, - prerelease=(str(now.hour), str(now.minute)), - ) - else: - while retries < max_retries: - try: - r = requests.get(f"{url}/healthcheck", timeout=5) - if r.status_code in {200, 403}: - return r.json().get("version") - else: - retries += 1 - logger.warning( - f"Healthcheck for {url} failed ({r.status_code})! Wait 30s." - ) - time.sleep(30) - except RequestException as ex: + if "pytest" in sys.modules and protocol == "http": + return __version__ + + while retries < max_retries: + try: + r = requests.get(f"{url}/healthcheck", timeout=5) + if r.status_code in {200, 403}: + return r.json().get("version") + else: retries += 1 - logger.warning(f"Could not connect to {url} ({ex})! Wait 30s.") + logger.warning( + f"Healthcheck for {url} failed ({r.status_code})! Wait 30s." + ) time.sleep(30) + except RequestException as ex: + retries += 1 + logger.warning(f"Could not connect to {url} ({ex})! Wait 30s.") + time.sleep(30) class Client(SwaggerClient): @@ -870,11 +866,11 @@ class Client(SwaggerClient): def __init__( self, - apikey: Optional[str] = None, - headers: Optional[dict] = None, - host: Optional[str] = None, - project: Optional[str] = None, - session: Optional[requests.Session] = None, + apikey: str | None = None, + headers: dict | None = None, + host: str | None = None, + project: str | None = None, + session: requests.Session | None = None, ): """Initialize the client - only reloads API spec from server as needed @@ -892,7 +888,17 @@ def __init__( host = os.environ.get("MPCONTRIBS_API_HOST", DEFAULT_HOST) if not apikey: - apikey = os.environ.get("MPCONTRIBS_API_KEY", SETTINGS.get("PMG_MAPI_KEY")) + try: + apikey = next( + os.environ.get(kalias) + for kalias in VALID_API_KEY_ALIASES + if kalias is not None + ) + except StopIteration: + from pymatgen.core import SETTINGS + + apikey = SETTINGS.get("PMG_MAPI_KEY") + if apikey and len(apikey) != 32: raise MPContribsClientError(f"Invalid API key: {apikey}") @@ -925,7 +931,7 @@ def __init__( def __enter__(self): return self - def __exit__(self, exc_type, exc_val, exc_tb): + def __exit__(self, exc_type, exc_val, exc_tb) -> None: return None @property @@ -934,17 +940,18 @@ def cached_swagger_spec(self): self.protocol, self.host, self.headers_json, self.project, self.version ) - def __dir__(self): + def __dir__(self) -> set[str]: members = set(self.swagger_spec.resources.keys()) - members |= set(k for k in self.__dict__.keys() if not k.startswith("_")) - members |= set(k for k in dir(self.__class__) if not k.startswith("_")) + members |= {k for k in self.__dict__.keys() if not k.startswith("_")} + members |= {k for k in dir(self.__class__) if not k.startswith("_")} return members def _reinit(self): _load.cache_clear() super().__init__(self.cached_swagger_spec) - def _is_valid_payload(self, model: str, data: dict): + def _is_valid_payload(self, model: str, data: dict) -> None: + """Raise an error if a payload is invalid.""" model_spec = deepcopy(self.get_model(f"{model}sSchema")._model_spec) model_spec.pop("required") model_spec["additionalProperties"] = False @@ -952,17 +959,20 @@ def _is_valid_payload(self, model: str, data: dict): try: validate_object(self.swagger_spec, model_spec, data) except ValidationError as ex: - return False, str(ex) - - return True, None + raise MPContribsClientError(str(ex)) - def _is_serializable_dict(self, dct): - for k, v in flatten(dct, reducer="dot").items(): - if v is not None and not isinstance(v, (str, int, float)): - error = f"Value {v} of {type(v)} for key {k} not supported." - return False, error - - return True, None + def _is_serializable_dict(self, dct: dict) -> None: + """Raise an error if an input dict is not JSON serializable.""" + try: + raise MPContribsClientError( + next( + f"Value {v} of {type(v)} for key {k} not supported." + for k, v in flatten(dct, reducer="dot").items() + if v is not None and not isinstance(v, (str, int, float)) + ) + ) + except StopIteration: + pass def _get_per_page_default_max( self, op: str = "query", resource: str = "contributions" @@ -1030,14 +1040,10 @@ def _split_query( for q in queries: # copy over missing parameters - for k, v in query.items(): - if k not in q: - q[k] = v + q.update({k: v for k, v in query.items() if k not in q}) # comma-separated lists - for k, v in q.items(): - if isinstance(v, list): - q[k] = ",".join(v) + q.update({k: ",".join(v) for k, v in q.items() if isinstance(v, list)}) return queries @@ -1047,7 +1053,7 @@ def _get_future( params: dict, rel_url: str = "contributions", op: str = "query", - data: Optional[dict] = None, + data: dict | None = None, ): rname = rel_url.split("/", 1)[0] resource = self.swagger_spec.resources[rname] @@ -1066,7 +1072,7 @@ def _get_future( def available_query_params( self, - startswith: Optional[tuple] = None, + startswith: tuple | None = None, resource: str = "contributions", ) -> list: resources = self.swagger_spec.resources @@ -1083,9 +1089,7 @@ def available_query_params( return [param for param in params if param.startswith(startswith)] - def get_project( - self, name: Optional[str] = None, fields: Optional[list] = None - ) -> Dict: + def get_project(self, name: str | None = None, fields: list | None = None) -> Dict: """Retrieve a project entry Args: @@ -1103,10 +1107,10 @@ def get_project( def query_projects( self, - query: Optional[dict] = None, - term: Optional[str] = None, - fields: Optional[list] = None, - sort: Optional[str] = None, + query: dict | None = None, + term: str | None = None, + fields: list | None = None, + sort: str | None = None, timeout: int = -1, ) -> list[dict]: """Query projects by query and/or term (Atlas Search) @@ -1157,9 +1161,13 @@ def search_future(search_term): if total_pages < 2: return ret["data"] - for field in ["name__in", "_fields"]: - if field in query: - query[field] = ",".join(query[field]) + query.update( + { + field: ",".join(query[field]) + for field in ["name__in", "_fields"] + if field in query + } + ) queries = [] @@ -1172,8 +1180,7 @@ def search_future(search_term): ] responses = _run_futures(futures, total=total_count, timeout=timeout) - for resp in responses.values(): - ret["data"] += resp["result"]["data"] + ret["data"].extend([resp["result"]["data"] for resp in responses.values()]) return ret["data"] @@ -1210,7 +1217,7 @@ def create_project( else: raise MPContribsClientError(resp) - def update_project(self, update: dict, name: Optional[str] = None): + def update_project(self, update: dict, name: str | None = None): """Update project info Args: @@ -1268,15 +1275,12 @@ def update_project(self, update: dict, name: Optional[str] = None): logger.warning("nothing to update") return - valid, error = self._is_valid_payload("Project", payload) - if valid: - resp = self.projects.updateProjectByName(pk=name, project=payload).result() - if not resp.get("count", 0): - raise MPContribsClientError(resp) - else: - raise MPContribsClientError(error) + self._is_valid_payload("Project", payload) + resp = self.projects.updateProjectByName(pk=name, project=payload).result() + if not resp.get("count", 0): + raise MPContribsClientError(resp) - def delete_project(self, name: Optional[str] = None): + def delete_project(self, name: str | None = None): """Delete a project Args: @@ -1295,7 +1299,7 @@ def delete_project(self, name: Optional[str] = None): if resp and "error" in resp: raise MPContribsClientError(resp["error"]) - def get_contribution(self, cid: str, fields: Optional[list] = None) -> Dict: + def get_contribution(self, cid: str, fields: list | None = None) -> Dict: """Retrieve a contribution Args: @@ -1408,7 +1412,7 @@ def get_attachment(self, aid_or_md5: str) -> Attachment: ) def init_columns( - self, columns: Optional[dict] = None, name: Optional[str] = None + self, columns: dict | None = None, name: str | None = None ) -> dict: """initialize columns for a project to set their order and desired units @@ -1560,13 +1564,11 @@ def init_columns( new_columns.append(new_column) payload = {"columns": new_columns} - valid, error = self._is_valid_payload("Project", payload) - if not valid: - raise MPContribsClientError(error) + self._is_valid_payload("Project", payload) return self.projects.updateProjectByName(pk=name, project=payload).result() - def delete_contributions(self, query: Optional[dict] = None, timeout: int = -1): + def delete_contributions(self, query: dict | None = None, timeout: int = -1): """Remove all contributions for a query Args: @@ -1612,7 +1614,7 @@ def delete_contributions(self, query: Optional[dict] = None, timeout: int = -1): def get_totals( self, - query: Optional[dict] = None, + query: dict | None = None, timeout: int = -1, resource: str = "contributions", op: str = "query", @@ -1641,23 +1643,23 @@ def get_totals( query = {k: v for k, v in query.items() if k not in skip_keys} query["_fields"] = [] # only need totals -> explicitly request no fields queries = self._split_query(query, resource=resource, op=op) # don't paginate - result = {"total_count": 0, "total_pages": 0} futures = [ self._get_future(i, q, rel_url=resource) for i, q in enumerate(queries) ] responses = _run_futures(futures, timeout=timeout, desc="Totals") - for resp in responses.values(): - for k in result: - result[k] += resp.get("result", {}).get(k, 0) + result = { + k: sum(resp.get("result", {}).get(k, 0) for resp in responses.values()) + for k in ("total_count", "total_pages") + } return result["total_count"], result["total_pages"] - def count(self, query: Optional[dict] = None) -> int: + def count(self, query: dict | None = None) -> int: """shortcut for get_totals()""" return self.get_totals(query=query)[0] - def get_unique_identifiers_flags(self, query: Optional[dict] = None) -> dict: + def get_unique_identifiers_flags(self, query: dict | None = None) -> dict: """Retrieve values for `unique_identifiers` flags. See `client.available_query_params(resource="projects")` for available query parameters. @@ -1677,10 +1679,10 @@ def get_unique_identifiers_flags(self, query: Optional[dict] = None) -> dict: def get_all_ids( self, - query: Optional[dict] = None, - include: Optional[list[str]] = None, + query: dict | None = None, + include: list[str] | None = None, timeout: int = -1, - data_id_fields: Optional[dict] = None, + data_id_fields: dict | None = None, fmt: str = "sets", op: str = "query", ) -> dict: @@ -1731,7 +1733,7 @@ def get_all_ids( }, ...} """ include = include or [] - components = set(x for x in include if x in COMPONENTS) + components = {x for x in include if x in COMPONENTS} if include and not components: raise MPContribsClientError(f"`include` must be subset of {COMPONENTS}!") @@ -1745,9 +1747,13 @@ def get_all_ids( unique_identifiers = self.get_unique_identifiers_flags() data_id_fields = data_id_fields or {} - for k, v in data_id_fields.items(): - if k in unique_identifiers and isinstance(v, str): - data_id_fields[k] = v + data_id_fields.update( + { + k: v + for k, v in data_id_fields.items() + if k in unique_identifiers and isinstance(v, str) + } + ) ret = {} query = query or {} @@ -1810,34 +1816,40 @@ def get_all_ids( if data_id_field and data_id_field_val: ret[project][identifier][data_id_field] = data_id_field_val - for component in components: - if component in contrib: - ret[project][identifier][component] = { + ret[project][identifier].update( + { + component: { d["name"]: {"id": d["id"], "md5": d["md5"]} for d in contrib[component] } + for component in components + if component in contrib + } + ) elif data_id_field and data_id_field_val: ret[project][identifier] = { data_id_field_val: {"id": contrib["id"]} } - for component in components: - if component in contrib: - ret[project][identifier][data_id_field_val][ - component - ] = { + ret[project][identifier][data_id_field_val].update( + { + component: { d["name"]: {"id": d["id"], "md5": d["md5"]} for d in contrib[component] } + for component in components + if component in contrib + } + ) return ret def query_contributions( self, - query: Optional[dict] = None, - fields: Optional[list] = None, - sort: Optional[str] = None, + query: dict | None = None, + fields: list | None = None, + sort: str | None = None, paginate: bool = False, timeout: int = -1, ) -> dict: @@ -1861,12 +1873,11 @@ def query_contributions( query["project"] = self.project if paginate: - cids = [] - - for v in self.get_all_ids(query).values(): - cids_project = v.get("ids") - if cids_project: - cids.extend(cids_project) + cids = [ + idx + for v in self.get_all_ids(query).values() + for idx in (v.get("ids") or []) + ] if not cids: raise MPContribsClientError("No contributions match the query.") @@ -1891,7 +1902,7 @@ def query_contributions( return ret def update_contributions( - self, data: dict, query: Optional[dict] = None, timeout: int = -1 + self, data: dict, query: dict | None = None, timeout: int = -1 ) -> dict: """Apply the same update to all contributions in a project (matching query) @@ -1906,14 +1917,10 @@ def update_contributions( raise MPContribsClientError("Nothing to update.") tic = time.perf_counter() - valid, error = self._is_valid_payload("Contribution", data) - if not valid: - raise MPContribsClientError(error) + self._is_valid_payload("Contribution", data) if "data" in data: - serializable, error = self._is_serializable_dict(data["data"]) - if not serializable: - raise MPContribsClientError(error) + self._is_serializable_dict(data["data"]) query = query or {} @@ -1939,7 +1946,7 @@ def update_contributions( # get current list of data columns to decide if swagger reload is needed resp = self.projects.getProjectByName(pk=name, _fields=["columns"]).result() - old_paths = set(c["path"] for c in resp["columns"]) + old_paths = {c["path"] for c in resp["columns"]} total = len(cids) cids_query = {"id__in": cids} @@ -1954,7 +1961,7 @@ def update_contributions( if updated: resp = self.projects.getProjectByName(pk=name, _fields=["columns"]).result() - new_paths = set(c["path"] for c in resp["columns"]) + new_paths = {c["path"] for c in resp["columns"]} if new_paths != old_paths: self.init_columns(name=name) @@ -1964,7 +1971,7 @@ def update_contributions( return {"updated": updated, "total": total, "seconds_elapsed": toc - tic} def make_public( - self, query: Optional[dict] = None, recursive: bool = False, timeout: int = -1 + self, query: dict | None = None, recursive: bool = False, timeout: int = -1 ) -> dict: """Publish a project and optionally its contributions @@ -1977,7 +1984,7 @@ def make_public( ) def make_private( - self, query: Optional[dict] = None, recursive: bool = False, timeout: int = -1 + self, query: dict | None = None, recursive: bool = False, timeout: int = -1 ) -> dict: """Make a project and optionally its contributions private @@ -1992,7 +1999,7 @@ def make_private( def _set_is_public( self, is_public: bool, - query: Optional[dict] = None, + query: dict | None = None, recursive: bool = False, timeout: int = -1, ) -> dict: @@ -2124,9 +2131,13 @@ def submit_contributions( resp = self.get_all_ids(dict(id__in=collect_ids), timeout=timeout) project_names |= set(resp.keys()) - for project_name, values in resp.items(): - for cid in values["ids"]: - id2project[cid] = project_name + id2project.update( + { + cid: project_name + for project_name, values in resp.items() + for cid in values["ids"] + } + ) existing = defaultdict(dict) unique_identifiers = defaultdict(dict) @@ -2160,9 +2171,7 @@ def submit_contributions( for contrib in tqdm(contributions, desc="Prepare"): if "data" in contrib: contrib["data"] = unflatten(contrib["data"], splitter="dot") - serializable, error = self._is_serializable_dict(contrib["data"]) - if not serializable: - raise MPContribsClientError(error) + self._is_serializable_dict(contrib["data"]) update = "id" in contrib project_name = id2project[contrib["id"]] if update else contrib["project"] @@ -2282,13 +2291,7 @@ def submit_contributions( digests[project_name][component].add(digest) contribs[project_name][-1][component].append(dct) - valid, error = self._is_valid_payload( - "Contribution", contribs[project_name][-1] - ) - if not valid: - raise MPContribsClientError( - f"{contrib['identifier']} invalid: {error}!" - ) + self._is_valid_payload("Contribution", contribs[project_name][-1]) # submit contributions if contribs: @@ -2425,10 +2428,10 @@ def put_future(pk, payload): def download_contributions( self, - query: Optional[dict] = None, - outdir: Union[str, Path] = DEFAULT_DOWNLOAD_DIR, + query: dict | None = None, + outdir: str | Path = DEFAULT_DOWNLOAD_DIR, overwrite: bool = False, - include: Optional[list[str]] = None, + include: list[str] | None = None, timeout: int = -1, ) -> list: """Download a list of contributions as .json.gz file(s) @@ -2448,7 +2451,7 @@ def download_contributions( include = include or [] outdir = Path(outdir) or Path(".") outdir.mkdir(parents=True, exist_ok=True) - components = set(x for x in include if x in COMPONENTS) + components = {x for x in include if x in COMPONENTS} if include and not components: raise MPContribsClientError(f"`include` must be subset of {COMPONENTS}!") @@ -2527,7 +2530,7 @@ def download_contributions( def download_structures( self, ids: list[str], - outdir: Union[str, Path] = DEFAULT_DOWNLOAD_DIR, + outdir: str | Path = DEFAULT_DOWNLOAD_DIR, overwrite: bool = False, timeout: int = -1, fmt: str = "json", @@ -2556,7 +2559,7 @@ def download_structures( def download_tables( self, ids: list[str], - outdir: Union[str, Path] = DEFAULT_DOWNLOAD_DIR, + outdir: str | Path = DEFAULT_DOWNLOAD_DIR, overwrite: bool = False, timeout: int = -1, fmt: str = "json", @@ -2585,7 +2588,7 @@ def download_tables( def download_attachments( self, ids: list[str], - outdir: Union[str, Path] = DEFAULT_DOWNLOAD_DIR, + outdir: str | Path = DEFAULT_DOWNLOAD_DIR, overwrite: bool = False, timeout: int = -1, fmt: str = "json", @@ -2615,7 +2618,7 @@ def _download_resource( self, resource: str, ids: list[str], - outdir: Union[str, Path] = DEFAULT_DOWNLOAD_DIR, + outdir: str | Path = DEFAULT_DOWNLOAD_DIR, overwrite: bool = False, timeout: int = -1, fmt: str = "json", diff --git a/mpcontribs-client/pyproject.toml b/mpcontribs-client/pyproject.toml new file mode 100644 index 000000000..e7a4fd7e2 --- /dev/null +++ b/mpcontribs-client/pyproject.toml @@ -0,0 +1,64 @@ +[build-system] +requires = ["setuptools>=80.0.0","setuptools-scm>=8"] +build-backend = "setuptools.build_meta" + +[tool.setuptools_scm] +root = ".." +relative_to = "__file__" + +[tool.setuptools] +include-package-data = true + +[tool.setuptools.packages.find] +where = ["."] +exclude = ["tests"] +include = ["mpcontribs.client"] + +[project] +name = "mpcontribs-client" +dynamic = ["version"] +requires-python = ">=3.11" +description="Client library for MPContribs API" +license = {text = "MIT"} +authors = [ + {name = "Patrick Huck", email = "phuck@lbl.gov"}, + {name = "The Materials Project", email="feedback@materialsproject.org"}, +] +dependencies = [ + "numpy", + "boltons", + "bravado", + "filetype", + "flatten-dict", + "ipython", + "json2html", + "pandas", + "pint", + "plotly", + "pyIsEmail", + "pymatgen", + "pymongo", + "requests-futures", + "swagger-spec-validator", + "tqdm", + "ujson", + "cachetools", +] + +[project.urls] +Homepage = "https://github.com/materialsproject/MPContribs" +Documentation = "https://docs.materialsproject.org/services/mpcontribs" + +[project.optional-dependencies] +dev = [ + "flake8", + "pytest", + "pytest-flake8", + "pytest-pycodestyle", + "pytest-cov", + "pytest-xdist", + "py", +] +all = [ + "mpcontribs-client[dev]" +] diff --git a/mpcontribs-client/setup.py b/mpcontribs-client/setup.py deleted file mode 100644 index 9ad2f515e..000000000 --- a/mpcontribs-client/setup.py +++ /dev/null @@ -1,63 +0,0 @@ -# -*- coding: utf-8 -*- -from setuptools import setup - - -def local_version(version): - # https://github.com/pypa/setuptools_scm/issues/342 - return "" - - -with open("README.md", "r") as fh: - long_description = fh.read() - -setup( - name="mpcontribs-client", - python_requires=">=3.8", - author="Patrick Huck", - author_email="phuck@lbl.gov", - description="client library for MPContribs API", - long_description=long_description, - long_description_content_type="text/markdown", - url="https://github.com/materialsproject/MPContribs/tree/master/mpcontribs-client", - packages=["mpcontribs.client"], - install_requires=[ - "numpy", - "boltons", - "bravado", - "filetype", - "flatten-dict", - "ipython", - "json2html", - "pandas", - "pint", - "plotly", - "pyIsEmail", - "pymatgen", - "pymongo", - "requests-futures", - "swagger-spec-validator", - "tqdm", - "ujson", - "semantic-version", - "cachetools", - ], - extras_require={ - "dev": [ - "flake8", - "pytest", - "pytest-flake8", - "pytest-pycodestyle", - "pytest-cov", - "py", - ] - }, - license="MIT", - zip_safe=False, - include_package_data=True, - use_scm_version={ - "root": "..", - "relative_to": __file__, - "local_scheme": local_version, - }, - setup_requires=["setuptools_scm"], -) diff --git a/mpcontribs-client/tests/test_client.py b/mpcontribs-client/tests/test_client.py index 45ded807c..8dd7c344c 100644 --- a/mpcontribs-client/tests/test_client.py +++ b/mpcontribs-client/tests/test_client.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- import pytest import logging diff --git a/mpcontribs-io/pyproject.toml b/mpcontribs-io/pyproject.toml new file mode 100644 index 000000000..5268b0b5b --- /dev/null +++ b/mpcontribs-io/pyproject.toml @@ -0,0 +1,74 @@ +[build-system] +requires = ["setuptools>=80.0.0","setuptools-scm>=8"] +build-backend = "setuptools.build_meta" + +[tool.setuptools_scm] +root = ".." +relative_to = "__file__" + +[tool.setuptools] +include-package-data = true + +[tool.setuptools.packages.find] +where = ["."] +exclude = ["scripts","supervisord"] +include = ["mpcontribs.io.core","mpcontribs.io.archie"] + +[project] +name = "mpcontribs-io" +dynamic = ["version"] +requires-python = ">=3.11" +description="MPContribs I/O Library" +license = {text = "MIT"} +authors = [ + {name = "Patrick Huck", email = "phuck@lbl.gov"}, + {name = "The Materials Project", email="feedback@materialsproject.org"}, +] +dependencies = [ + "archieml", + "ipython", + "pandas", + "plotly", + "pymatgen" +] + +[project.urls] +Homepage = "https://github.com/materialsproject/MPContribs/tree/master/mpcontribs-io" +Documentation = "https://docs.materialsproject.org/services/mpcontribs" + +[tool.tox] +envlist = ["clean","py311","py312"] + +[tool.tox.gh-actions] +python = ["3.11: py312","3.12: py312"] + +[tool.tox.pycodestyle] +max-line-length = 120 + +[tool.tox.pytest] +flake8-max-line-length = 120 +python_files = ["test_*.py",] + +[tool.tox.coverage.paths] +source = "mpcontribs/io" + +[tool.tox.coverage.run] +source = "mpcontribs/io" +omit = ["*test_*.py",] + +[tool.tox.testenv] +deps = [ + "pytest", + "pytest-flake8", + "pytest-pycodestyle", + "pytest-cov", + "-rrequirements.txt", +] +commands = [ + "pytest -v -s --flake8 --pycodestyle --cov={envsitepackagesdir}/mpcontribs/io --cov-report=term-missing --cov-report=xml" +] + +[tool.tox.testenv.clean] +deps = ["coverage"] +skip_install = true +commands = ["coverage", "erase"] diff --git a/mpcontribs-io/setup.py b/mpcontribs-io/setup.py deleted file mode 100644 index 7e1546efc..000000000 --- a/mpcontribs-io/setup.py +++ /dev/null @@ -1,32 +0,0 @@ -# -*- coding: utf-8 -*- -from setuptools import setup - - -def local_version(version): - # https://github.com/pypa/setuptools_scm/issues/342 - return "" - - -with open("README.md", "r") as fh: - long_description = fh.read() - -setup( - name="mpcontribs-io", - author="Patrick Huck", - author_email="phuck@lbl.gov", - description="MPContribs I/O Library", - long_description=long_description, - long_description_content_type="text/markdown", - url="https://github.com/materialsproject/MPContribs/tree/master/mpcontribs-io", - packages=["mpcontribs.io.core", "mpcontribs.io.archie"], - install_requires=["archieml", "ipython", "pandas", "plotly", "pymatgen"], - license="MIT", - zip_safe=False, - include_package_data=True, - use_scm_version={ - "root": "..", - "relative_to": __file__, - "local_scheme": local_version, - }, - setup_requires=["setuptools_scm"], -) diff --git a/mpcontribs-io/tox.ini b/mpcontribs-io/tox.ini deleted file mode 100644 index 57c931167..000000000 --- a/mpcontribs-io/tox.ini +++ /dev/null @@ -1,35 +0,0 @@ -[tox] -envlist = clean,py37,py38 - -[gh-actions] -python = - 3.7: py37 - 3.8: py38 - -[pycodestyle] -max-line-length = 120 - -[pytest] -flake8-max-line-length = 120 -python_files = test_*.py - -[coverage:paths] -source = mpcontribs/io - -[coverage:run] -source = mpcontribs/io -omit = *test_*.py - -[testenv] -deps = - pytest - pytest-flake8 - pytest-pycodestyle - pytest-cov - -rrequirements.txt -commands = pytest -v -s --flake8 --pycodestyle --cov={envsitepackagesdir}/mpcontribs/io --cov-report=term-missing --cov-report=xml - -[testenv:clean] -deps = coverage -skip_install = true -commands = coverage erase diff --git a/mpcontribs-lux/tests/test_autogen.py b/mpcontribs-lux/tests/test_autogen.py index 17b877ddb..71cd04a67 100644 --- a/mpcontribs-lux/tests/test_autogen.py +++ b/mpcontribs-lux/tests/test_autogen.py @@ -60,7 +60,7 @@ def test_schema_generation(test_dir): schema = schemer.schema() # Avoiding `eval` here; `eval` also can't handle import statements - with open(test_py_file := Path("./test_no_kwargs.py").resolve(), "wt") as py_temp: + with open(test_py_file := Path("./test_no_kwargs.py").resolve(), "w") as py_temp: py_temp.write(schema) test_no_kwargs = dynamically_load_module_from_path(test_py_file, "test_no_kwargs") @@ -74,7 +74,7 @@ def test_schema_generation(test_dir): new_desc = {"b0": "Bulk modulus", "cif": "CIF"} with open( - test_py_file := Path("./test_w_name_and_anno.py").resolve(), "wt" + test_py_file := Path("./test_w_name_and_anno.py").resolve(), "w" ) as py_temp: py_temp.write(schemer.schema(model_name="TestClass", descriptions=new_desc)) test_w_kwargs = dynamically_load_module_from_path(test_py_file, "test_w_kwargs") diff --git a/mpcontribs-portal/pyproject.toml b/mpcontribs-portal/pyproject.toml new file mode 100644 index 000000000..4daab63ee --- /dev/null +++ b/mpcontribs-portal/pyproject.toml @@ -0,0 +1,91 @@ +[build-system] +requires = ["setuptools>=80.0.0","setuptools-scm>=8"] +build-backend = "setuptools.build_meta" + +[tool.setuptools_scm] +root = ".." +relative_to = "__file__" + +[tool.setuptools] +include-package-data = true + +[tool.setuptools.packages.find] +where = ["."] +exclude = ["scripts","supervisord"] +include = ["mpcontribs.portal", "mpcontribs.users"] + +[project] +name = "mpcontribs-portal" +dynamic = ["version"] +requires-python = ">=3.11" +description="MPContribs Portal" +license = {text = "MIT"} +authors = [ + {name = "Patrick Huck", email = "phuck@lbl.gov"}, + {name = "The Materials Project", email="feedback@materialsproject.org"}, +] +dependencies = [ + "boltons", + "boto3", + "ddtrace", + "Django>=3.2,<4.0", + "django-extensions", + "django-settings-file", + "django-webpack4-loader", + "fastnumbers", + "gunicorn[gevent]", + "ipykernel", + "ipython-genutils", + "jinja2", + "json2html", + "monty", + "mpcontribs-client", + "nbconvert", + "nbformat", + "redis", + "scipy", + "setproctitle", + "whitenoise", +] + +[project.urls] +Homepage = "https://github.com/materialsproject/MPContribs" +Documentation = "https://docs.materialsproject.org/services/mpcontribs" + +[project.optional-dependencies] +dev = [ + "flake8", + "pytest", + "pytest-flake8", + "pytest-pycodestyle", + "pytest-xdist", +] +all = [ + "mpcontribs-portal[dev]" +] + +[tool.pytest] +markers = [ + "base: basic resource testing", + "extra: all extra views", +] + +[tool.pycodestyle] +count = true +ignore = ["E121","E123","E126","E133","E226","E241","E242","E704","W503","W504","W505","E741","W605"] +max-line-length = 120 +statistics = true +exclude = ["supervisord","notebooks"] + +[tool.flake8] +exclude = [".git","__pycache__","tests","flasgger","flask-mongorest"] +extend-ignore = ["E741",] +max-line-length = 120 + +[tool.pydocstyle] +ignore = ["D105","D2","D4"] + +[tool.mypy] +ignore_missing_imports = true +namespace_packages = true +python_version = 3.11 diff --git a/mpcontribs-portal/setup.py b/mpcontribs-portal/setup.py deleted file mode 100644 index 7ee9d1e22..000000000 --- a/mpcontribs-portal/setup.py +++ /dev/null @@ -1,40 +0,0 @@ -# -*- coding: utf-8 -*- -import datetime -from setuptools import setup - -setup( - name="mpcontribs-portal", - version=datetime.datetime.today().strftime("%Y.%m.%d"), - description="MPContribs Portal", - author="Patrick Huck", - author_email="phuck@lbl.gov", - url="https://docs.mpcontribs.org", - packages=["mpcontribs.portal", "mpcontribs.users"], - install_requires=[ - "boltons", - "boto3", - "ddtrace", - "Django>=3.2,<4.0", - "django-extensions", - "django-settings-file", - "django-webpack4-loader", - "fastnumbers", - "gunicorn[gevent]", - "ipykernel", - "ipython-genutils", - "jinja2", - "json2html", - "monty", - "mpcontribs-client", - "nbconvert", - "nbformat", - "redis", - "scipy", - "setproctitle", - "whitenoise", - ], - extras_require={"dev": ["pytest", "flake8"]}, - license="MIT", - zip_safe=False, - include_package_data=True, -)