From aaae7db7a30c7d9f09ab754c01bc3c619c0819c6 Mon Sep 17 00:00:00 2001
From: Spencer Phillip Young <spencer.young@spyoung.com>
Date: Wed, 3 May 2023 17:57:22 -0700
Subject: [PATCH 1/6] Add 'unasync: remove' feature
---
src/unasync/__init__.py | 41 +++++++++++++++++++++++++++---------
tests/data/async/removals.py | 36 +++++++++++++++++++++++++++++++
tests/data/sync/removals.py | 23 ++++++++++++++++++++
3 files changed, 90 insertions(+), 10 deletions(-)
create mode 100644 tests/data/async/removals.py
create mode 100644 tests/data/sync/removals.py
diff --git a/src/unasync/__init__.py b/src/unasync/__init__.py
index ffc9205..c7ece45 100644
--- a/src/unasync/__init__.py
+++ b/src/unasync/__init__.py
@@ -1,5 +1,6 @@
"""Top-level package for unasync."""
+import ast
import collections
import errno
import os
@@ -68,17 +69,37 @@ def _unasync_file(self, filepath):
encoding, _ = std_tokenize.detect_encoding(f.readline)
with open(filepath, "rt", encoding=encoding) as f:
- tokens = tokenize_rt.src_to_tokens(f.read())
- tokens = self._unasync_tokens(tokens)
- result = tokenize_rt.tokens_to_src(tokens)
- outfilepath = filepath.replace(self.fromdir, self.todir)
- os.makedirs(os.path.dirname(outfilepath), exist_ok=True)
- with open(outfilepath, "wb") as f:
- f.write(result.encode(encoding))
-
- def _unasync_tokens(self, tokens):
+ contents = f.read()
+ tokens = self._unasync_tokenize(contents=contents, filename=filepath)
+ result = tokenize_rt.tokens_to_src(tokens)
+ outfilepath = filepath.replace(self.fromdir, self.todir)
+ os.makedirs(os.path.dirname(outfilepath), exist_ok=True)
+ with open(outfilepath, "wb") as f:
+ f.write(result.encode(encoding))
+
+ def _unasync_tokenize(self, contents, filename):
+ tokens = tokenize_rt.src_to_tokens(contents)
+
+ comment_lines_locations = []
+ for token in tokens:
+ # find line numbers where "unasync: remove" comments are found
+ if token.name == 'COMMENT' and 'unasync: remove' in token.src: # XXX: maybe make this a little more strict
+ comment_lines_locations.append(token.line)
+
+ lines_to_remove = set()
+ if comment_lines_locations: # only parse ast if we actually have "unasync: remove" comments
+ tree = ast.parse(contents, filename=filename)
+ for node in ast.walk(tree):
+ # find nodes whose line number (start line) intersect with unasync: remove comments
+ if hasattr(node, 'lineno') and node.lineno in comment_lines_locations:
+ for lineno in range(node.lineno, node.end_lineno + 1):
+ # find all lines related to each node and mark those lines for removal
+ lines_to_remove.add(lineno)
+
skip_next = False
- for i, token in enumerate(tokens):
+ for token in tokens:
+ if token.line in lines_to_remove:
+ continue
if skip_next:
skip_next = False
continue
diff --git a/tests/data/async/removals.py b/tests/data/async/removals.py
new file mode 100644
index 0000000..6d88118
--- /dev/null
+++ b/tests/data/async/removals.py
@@ -0,0 +1,36 @@
+from common import (
+a, b , c # these should stick around
+)
+
+# these imports should be removed
+from async_only import ( # unasync: remove
+ async_a, async_b,
+ async_c
+)
+
+CONST = 'foo'
+ASYNC_CONST = 'bar' # unasync: remove
+
+async def foo():
+ print('this function should stick around')
+
+async def async_only(): # unasync: remove
+ print('this function will be removed entirely')
+
+
+class AsyncOnly: # unasync: remove
+ async def foo(self):
+ print('the entire class should be removed')
+
+
+class Foo:
+ async def foobar(self):
+ print('This method should stick around')
+
+ async def async_only_method(self): # unasync: remove
+ print('only this method should be removed')
+
+ async def another_method(self):
+ print('This line should stick around')
+ await self.something("the content in this line should be removed") # unasync: remove
+
diff --git a/tests/data/sync/removals.py b/tests/data/sync/removals.py
new file mode 100644
index 0000000..e923281
--- /dev/null
+++ b/tests/data/sync/removals.py
@@ -0,0 +1,23 @@
+from common import (
+a, b , c # these should stick around
+)
+
+# these imports should be removed
+
+CONST = 'foo'
+
+def foo():
+ print('this function should stick around')
+
+
+
+
+
+class Foo:
+ def foobar(self):
+ print('This method should stick around')
+
+
+ def another_method(self):
+ print('This line should stick around')
+
From 4d6f9170e8f7b978f7817a47536d04155c3119f3 Mon Sep 17 00:00:00 2001
From: Spencer Phillip Young <spencer.young@spyoung.com>
Date: Wed, 3 May 2023 17:57:42 -0700
Subject: [PATCH 2/6] Drop 3.7 support
---
.github/workflows/ci.yml | 6 +++---
1 file changed, 3 insertions(+), 3 deletions(-)
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 518b8c3..5493f3d 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -9,7 +9,7 @@ jobs:
strategy:
fail-fast: false
matrix:
- python: ['3.7', '3.8', '3.9', '3.10']
+ python: ['3.8', '3.9', '3.10']
steps:
- name: Checkout
@@ -34,7 +34,7 @@ jobs:
strategy:
fail-fast: false
matrix:
- python: ['3.7', '3.8', '3.9', '3.10', '3.11-dev']
+ python: ['3.8', '3.9', '3.10', '3.11-dev']
check_formatting: ['0']
extra_name: ['']
include:
@@ -70,7 +70,7 @@ jobs:
strategy:
fail-fast: false
matrix:
- python: ['3.7', '3.8', '3.9', '3.10']
+ python: ['3.8', '3.9', '3.10']
steps:
- name: Checkout
uses: actions/checkout@v2
From 01b2c8ad74f8c8f44af3dcb8a2c3cf6811cb8ce2 Mon Sep 17 00:00:00 2001
From: Spencer Phillip Young <spencer.young@spyoung.com>
Date: Wed, 3 May 2023 18:47:27 -0700
Subject: [PATCH 3/6] Drop support for Python 3.7
---
setup.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/setup.py b/setup.py
index 88f3ecc..94e64be 100644
--- a/setup.py
+++ b/setup.py
@@ -19,7 +19,7 @@
package_dir={"": "src"},
install_requires=["tokenize_rt"],
keywords=["async"],
- python_requires=">=3.7",
+ python_requires=">=3.8",
classifiers=[
"License :: OSI Approved :: MIT License",
"License :: OSI Approved :: Apache Software License",
From 14b1e328a1ba53d4527e3382bee00a5475477578 Mon Sep 17 00:00:00 2001
From: Spencer Phillip Young <spencer.young@spyoung.com>
Date: Wed, 3 May 2023 18:12:44 -0700
Subject: [PATCH 4/6] apply black formatting
---
src/unasync/__init__.py | 10 +++++++---
tests/data/async/removals.py | 3 +++
tests/data/sync/removals.py | 3 +++
tests/test_unasync.py | 5 -----
4 files changed, 13 insertions(+), 8 deletions(-)
diff --git a/src/unasync/__init__.py b/src/unasync/__init__.py
index c7ece45..872a1ff 100644
--- a/src/unasync/__init__.py
+++ b/src/unasync/__init__.py
@@ -83,15 +83,19 @@ def _unasync_tokenize(self, contents, filename):
comment_lines_locations = []
for token in tokens:
# find line numbers where "unasync: remove" comments are found
- if token.name == 'COMMENT' and 'unasync: remove' in token.src: # XXX: maybe make this a little more strict
+ if (
+ token.name == "COMMENT" and "unasync: remove" in token.src
+ ): # XXX: maybe make this a little more strict
comment_lines_locations.append(token.line)
lines_to_remove = set()
- if comment_lines_locations: # only parse ast if we actually have "unasync: remove" comments
+ if (
+ comment_lines_locations
+ ): # only parse ast if we actually have "unasync: remove" comments
tree = ast.parse(contents, filename=filename)
for node in ast.walk(tree):
# find nodes whose line number (start line) intersect with unasync: remove comments
- if hasattr(node, 'lineno') and node.lineno in comment_lines_locations:
+ if hasattr(node, "lineno") and node.lineno in comment_lines_locations:
for lineno in range(node.lineno, node.end_lineno + 1):
# find all lines related to each node and mark those lines for removal
lines_to_remove.add(lineno)
diff --git a/tests/data/async/removals.py b/tests/data/async/removals.py
index 6d88118..5640722 100644
--- a/tests/data/async/removals.py
+++ b/tests/data/async/removals.py
@@ -1,3 +1,5 @@
+# isort: skip_file
+# fmt: off
from common import (
a, b , c # these should stick around
)
@@ -34,3 +36,4 @@ async def another_method(self):
print('This line should stick around')
await self.something("the content in this line should be removed") # unasync: remove
+# fmt: on
diff --git a/tests/data/sync/removals.py b/tests/data/sync/removals.py
index e923281..260a79f 100644
--- a/tests/data/sync/removals.py
+++ b/tests/data/sync/removals.py
@@ -1,3 +1,5 @@
+# isort: skip_file
+# fmt: off
from common import (
a, b , c # these should stick around
)
@@ -21,3 +23,4 @@ def foobar(self):
def another_method(self):
print('This line should stick around')
+# fmt: on
diff --git a/tests/test_unasync.py b/tests/test_unasync.py
index 35e0c6c..6b02adc 100644
--- a/tests/test_unasync.py
+++ b/tests/test_unasync.py
@@ -35,7 +35,6 @@ def test_rule_on_short_path():
@pytest.mark.parametrize("source_file", TEST_FILES)
def test_unasync(tmpdir, source_file):
-
rule = unasync.Rule(fromdir=ASYNC_DIR, todir=str(tmpdir))
rule._unasync_file(os.path.join(ASYNC_DIR, source_file))
@@ -64,7 +63,6 @@ def test_unasync_files(tmpdir):
def test_build_py_modules(tmpdir):
-
source_modules_dir = os.path.join(TEST_DIR, "example_mod")
mod_dir = str(tmpdir) + "/" + "example_mod"
shutil.copytree(source_modules_dir, mod_dir)
@@ -84,7 +82,6 @@ def test_build_py_modules(tmpdir):
def test_build_py_packages(tmpdir):
-
source_pkg_dir = os.path.join(TEST_DIR, "example_pkg")
pkg_dir = str(tmpdir) + "/" + "example_pkg"
shutil.copytree(source_pkg_dir, pkg_dir)
@@ -101,7 +98,6 @@ def test_build_py_packages(tmpdir):
def test_project_structure_after_build_py_packages(tmpdir):
-
source_pkg_dir = os.path.join(TEST_DIR, "example_pkg")
pkg_dir = str(tmpdir) + "/" + "example_pkg"
shutil.copytree(source_pkg_dir, pkg_dir)
@@ -121,7 +117,6 @@ def test_project_structure_after_build_py_packages(tmpdir):
def test_project_structure_after_customized_build_py_packages(tmpdir):
-
source_pkg_dir = os.path.join(TEST_DIR, "example_custom_pkg")
pkg_dir = str(tmpdir) + "/" + "example_custom_pkg"
shutil.copytree(source_pkg_dir, pkg_dir)
From 415991bffb445bdaa332861e803d61c5371d2844 Mon Sep 17 00:00:00 2001
From: Spencer Phillip Young <spencer.young@spyoung.com>
Date: Wed, 3 May 2023 19:21:38 -0700
Subject: [PATCH 5/6] fix tests when run locally on Windows/venv
---
tests/test_unasync.py | 11 ++++++-----
1 file changed, 6 insertions(+), 5 deletions(-)
diff --git a/tests/test_unasync.py b/tests/test_unasync.py
index 6b02adc..073e3fd 100644
--- a/tests/test_unasync.py
+++ b/tests/test_unasync.py
@@ -3,6 +3,7 @@
import os
import shutil
import subprocess
+import sys
import pytest
@@ -69,9 +70,9 @@ def test_build_py_modules(tmpdir):
env = copy.copy(os.environ)
env["PYTHONPATH"] = os.path.realpath(os.path.join(TEST_DIR, ".."))
- subprocess.check_call(["python", "setup.py", "build"], cwd=mod_dir, env=env)
+ subprocess.check_call([sys.executable, "setup.py", "build"], cwd=mod_dir, env=env)
# Calling it twice to test the "if not copied" branch
- subprocess.check_call(["python", "setup.py", "build"], cwd=mod_dir, env=env)
+ subprocess.check_call([sys.executable, "setup.py", "build"], cwd=mod_dir, env=env)
unasynced = os.path.join(mod_dir, "build/lib/_sync/some_file.py")
tree_build_dir = list_files(mod_dir)
@@ -88,7 +89,7 @@ def test_build_py_packages(tmpdir):
env = copy.copy(os.environ)
env["PYTHONPATH"] = os.path.realpath(os.path.join(TEST_DIR, ".."))
- subprocess.check_call(["python", "setup.py", "build"], cwd=pkg_dir, env=env)
+ subprocess.check_call([sys.executable, "setup.py", "build"], cwd=pkg_dir, env=env)
unasynced = os.path.join(pkg_dir, "build/lib/example_pkg/_sync/__init__.py")
@@ -104,7 +105,7 @@ def test_project_structure_after_build_py_packages(tmpdir):
env = copy.copy(os.environ)
env["PYTHONPATH"] = os.path.realpath(os.path.join(TEST_DIR, ".."))
- subprocess.check_call(["python", "setup.py", "build"], cwd=pkg_dir, env=env)
+ subprocess.check_call([sys.executable, "setup.py", "build"], cwd=pkg_dir, env=env)
_async_dir_tree = list_files(
os.path.join(source_pkg_dir, "src/example_pkg/_async/.")
@@ -123,7 +124,7 @@ def test_project_structure_after_customized_build_py_packages(tmpdir):
env = copy.copy(os.environ)
env["PYTHONPATH"] = os.path.realpath(os.path.join(TEST_DIR, ".."))
- subprocess.check_call(["python", "setup.py", "build"], cwd=pkg_dir, env=env)
+ subprocess.check_call([sys.executable, "setup.py", "build"], cwd=pkg_dir, env=env)
_async_dir_tree = list_files(os.path.join(source_pkg_dir, "src/ahip/."))
unasynced_dir_path = os.path.join(pkg_dir, "build/lib/hip/.")
From 19617984f7b647d684086c6d9331c299379d6318 Mon Sep 17 00:00:00 2001
From: Spencer Phillip Young <spencer.young@spyoung.com>
Date: Wed, 14 Aug 2024 15:59:16 -0700
Subject: [PATCH 6/6] fix merge conflicts
---
.github/workflows/ci.yml | 26 ++++++++++----------------
.readthedocs.yml | 16 +++++++++-------
ci.sh | 6 ++----
docs/source/history.rst | 8 ++++++++
setup.py | 5 +++--
src/unasync/_version.py | 2 +-
tests/data/async/fstring.py | 5 +++++
tests/data/sync/fstring.py | 5 +++++
8 files changed, 43 insertions(+), 30 deletions(-)
create mode 100644 tests/data/async/fstring.py
create mode 100644 tests/data/sync/fstring.py
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 5493f3d..dead9bd 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -9,13 +9,12 @@ jobs:
strategy:
fail-fast: false
matrix:
- python: ['3.8', '3.9', '3.10']
-
+ python: ['3.8', '3.9', '3.10', '3.11', '3.12']
steps:
- name: Checkout
- uses: actions/checkout@v2
+ uses: actions/checkout@v4
- name: Setup python
- uses: actions/setup-python@v2
+ uses: actions/setup-python@v5
with:
python-version: ${{ matrix.python }}
cache: pip
@@ -34,7 +33,7 @@ jobs:
strategy:
fail-fast: false
matrix:
- python: ['3.8', '3.9', '3.10', '3.11-dev']
+ python: ['3.8', '3.9', '3.10', '3.11', '3.12', '3.13']
check_formatting: ['0']
extra_name: ['']
include:
@@ -43,19 +42,14 @@ jobs:
extra_name: ', check formatting'
steps:
- name: Checkout
- uses: actions/checkout@v2
+ uses: actions/checkout@v4
- name: Setup python
- uses: actions/setup-python@v2
- if: "!endsWith(matrix.python, '-dev')"
+ uses: actions/setup-python@v5
with:
python-version: ${{ matrix.python }}
+ allow-prereleases: true
cache: pip
cache-dependency-path: test-requirements.txt
- - name: Setup python (dev)
- uses: deadsnakes/action@v2.0.2
- if: endsWith(matrix.python, '-dev')
- with:
- python-version: '${{ matrix.python }}'
- name: Run tests
run: ./ci.sh
env:
@@ -70,12 +64,12 @@ jobs:
strategy:
fail-fast: false
matrix:
- python: ['3.8', '3.9', '3.10']
+ python: ['3.8', '3.9', '3.10', '3.11', '3.12']
steps:
- name: Checkout
- uses: actions/checkout@v2
+ uses: actions/checkout@v4
- name: Setup python
- uses: actions/setup-python@v2
+ uses: actions/setup-python@v5
with:
python-version: ${{ matrix.python }}
cache: pip
diff --git a/.readthedocs.yml b/.readthedocs.yml
index da6abdf..917704b 100644
--- a/.readthedocs.yml
+++ b/.readthedocs.yml
@@ -1,10 +1,12 @@
-# https://docs.readthedocs.io/en/latest/yaml-config.html
-formats:
- - htmlzip
- - epub
+version: 2
+
+build:
+ os: ubuntu-22.04
+ tools:
+ python: "3.12"
-requirements_file: ci/rtd-requirements.txt
python:
- version: 3
- pip_install: True
+ install:
+ - requirements: ci/rtd-requirements.txt
+ - path: .
diff --git a/ci.sh b/ci.sh
index 9934548..abf5b67 100755
--- a/ci.sh
+++ b/ci.sh
@@ -2,7 +2,7 @@
set -ex
-BLACK_VERSION=22.6.0
+BLACK_VERSION=24.4.2
python -m pip install -U pip setuptools wheel
@@ -55,6 +55,4 @@ fi
# Actual tests
pip install -Ur test-requirements.txt
-pytest -W error -ra -v tests --cov --cov-config=.coveragerc
-
-bash <(curl -s https://codecov.io/bash)
+pytest -W error -ra -v tests --cov --cov-config=.coveragerc --cov-fail-under=93
diff --git a/docs/source/history.rst b/docs/source/history.rst
index 6b4518c..30e8d37 100644
--- a/docs/source/history.rst
+++ b/docs/source/history.rst
@@ -4,3 +4,11 @@ Release history
.. currentmodule:: unasync
.. towncrier release notes start
+
+unasync 0.6.0 (2024-05-03)
+--------------------------
+
+* Drop support for Python 2.7, 3.5, 3.6 and 3.7
+* Add support for Python 3.9, 3.10, 3.11 and 3.12
+* Replace ``tokenize`` with ``tokenize-rt`` which roundtrips correctly and
+ handles Python 3.12 f-strings correctly.
diff --git a/setup.py b/setup.py
index 94e64be..16e3ec4 100644
--- a/setup.py
+++ b/setup.py
@@ -17,7 +17,7 @@
include_package_data=True,
packages=find_packages("src"),
package_dir={"": "src"},
- install_requires=["tokenize_rt"],
+ install_requires=["tokenize_rt", "setuptools"],
keywords=["async"],
python_requires=">=3.8",
classifiers=[
@@ -27,10 +27,11 @@
"Operating System :: POSIX :: Linux",
"Operating System :: MacOS :: MacOS X",
"Operating System :: Microsoft :: Windows",
- "Programming Language :: Python :: 3.7",
"Programming Language :: Python :: 3.8",
"Programming Language :: Python :: 3.9",
"Programming Language :: Python :: 3.10",
+ "Programming Language :: Python :: 3.11",
+ "Programming Language :: Python :: 3.12",
"Programming Language :: Python :: Implementation :: CPython",
"Programming Language :: Python :: Implementation :: PyPy",
],
diff --git a/src/unasync/_version.py b/src/unasync/_version.py
index 0a95219..dff3253 100644
--- a/src/unasync/_version.py
+++ b/src/unasync/_version.py
@@ -1,3 +1,3 @@
# This file is imported from __init__.py and exec'd from setup.py
-__version__ = "0.5.0+dev"
+__version__ = "0.6.0+dev"
diff --git a/tests/data/async/fstring.py b/tests/data/async/fstring.py
new file mode 100644
index 0000000..e58f2c8
--- /dev/null
+++ b/tests/data/async/fstring.py
@@ -0,0 +1,5 @@
+similarity_algo = f"""
+if (dotProduct < 0) {{
+ return 1;
+}}
+"""
diff --git a/tests/data/sync/fstring.py b/tests/data/sync/fstring.py
new file mode 100644
index 0000000..e58f2c8
--- /dev/null
+++ b/tests/data/sync/fstring.py
@@ -0,0 +1,5 @@
+similarity_algo = f"""
+if (dotProduct < 0) {{
+ return 1;
+}}
+"""