generate protobuf from public proto

Author: rekby

.gitignore

@@ -1,4 +1,6 @@
 __pycache__
 ydb.egg-info/
 /.idea
+/tox
 /venv
+/ydb_certs

.gitmodules

@@ -0,0 +1,3 @@
+[submodule "ydb-api-protos"]
+	path = ydb-api-protos
+	url = https://github.com/ydb-platform/ydb-api-protos.git

generate_protoc.py

@@ -1,22 +1,53 @@
 import os
+import pathlib
+import shutil
+from typing import List
+
 from grpc_tools import command
 
-command.build_package_protos('.')
 
-init_files_to_create = []
+def files_filter(dir, items: List[str]) -> List[str]:
+    ignored_dirs = ['protos', '.git']
+
+    ignore = []
+    for item in items:
+        fullpath = os.path.join(dir, item)
+        if os.path.isdir(fullpath) and item not in ignored_dirs:
+            continue
+        if item.endswith(".proto"):
+            continue
+        ignore.append(item)
+
+    print("items: ", items)
+    print("ignore: ", ignore)
+    return ignore
+
+
+def create_init_files(dir):
+    for root, _dirs, _files in os.walk(dir):
+        init_path = pathlib.Path(os.path.join(root, '__init__.py'))
+        if not init_path.exists():
+            init_path.touch()
+
+
+def replace_protos_dir():
+    def replace_dir(src, dst):
+        shutil.rmtree(dst)
+        shutil.copytree(src, dst, ignore=files_filter)
+        create_init_files(dst)
+
+    replace_dir("ydb-api-protos", "ydb/public/api/grpc")
+    replace_dir("ydb-api-protos/protos", "ydb/public/api/protos")
 
-for root, dirs, files in os.walk('kikimr'):
-    if '__init__.py' in files:
-        continue
 
-    init_files_to_create.append(
-        os.path.join(
-            root,
-            '__init__.py'
-        )
-    )
+def remove_protos():
+    for root, _dirs, files in os.walk("ydb/public/api"):
+        for file in files:
+            if file.endswith(".proto"):
+                os.remove(os.path.join(root, file))
 
 
-for filename in init_files_to_create:
-    with open(filename, 'w') as f:
-        pass
+if __name__ == '__main__':
+    replace_protos_dir()
+    command.build_package_protos('./ydb/public/api/')
+    remove_protos()

requirements.txt

@@ -1,4 +1,4 @@
-grpcio==1.38.0
-protobuf>3.17.3
+grpcio==1.39.0
+protobuf>3.17.3,<4.0.0
 pytest==6.2.4
 aiohttp==3.7.4

test-requirements.txt

@@ -10,15 +10,16 @@ docker==5.0.0
 docker-compose==1.29.2
 dockerpty==0.4.1
 docopt==0.6.2
-grpcio>=1.38.0
+grpcio==1.39.0
+grpcio-tools==1.39.0
 idna==3.2
 importlib-metadata==4.6.1
 iniconfig==1.1.1
 jsonschema==3.2.0
 packaging==21.0
 paramiko==2.10.1
 pluggy==0.13.1
-protobuf>3.17.3
+protobuf>3.17.3,<4.0.0
 py==1.10.0
 pycparser==2.20
 PyNaCl==1.4.0