From ba8f78890842c4768d42b2c3b5906e7c8c7cc6b6 Mon Sep 17 00:00:00 2001 From: coyotte508 Date: Tue, 21 Nov 2023 20:01:03 +0100 Subject: [PATCH 1/2] =?UTF-8?q?=E2=99=BB=EF=B8=8F=20Use=20hf=5Fhub.create?= =?UTF-8?q?=5Fcommit=20instead=20of=20upload=5Ffolder?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/doc_builder/commands/push.py | 24 ++++++++---------------- 1 file changed, 8 insertions(+), 16 deletions(-) diff --git a/src/doc_builder/commands/push.py b/src/doc_builder/commands/push.py index 63759e4ab..9a02f8c80 100644 --- a/src/doc_builder/commands/push.py +++ b/src/doc_builder/commands/push.py @@ -20,19 +20,10 @@ from pathlib import Path from time import sleep, time -from huggingface_hub import HfApi +from huggingface_hub import HfApi, CommitOperationAdd REPO_TYPE = "dataset" -SEPARATOR = "/" - - -def create_zip_name(library_name, version, with_ext=True): - file_name = f"{library_name}{SEPARATOR}{version}" - if with_ext: - file_name += ".zip" - return file_name - def push_command(args): """ @@ -63,7 +54,7 @@ def push_command_add(args): ) doc_version_folder = str(doc_version_folder) - zip_file_path = create_zip_name(library_name, doc_version_folder) + zip_file_path = f"{library_name}/{doc_version_folder}" # eg create ./transformers/v4.0.zip with '/transformers/v4.0/*' file architecture inside # Use subprocess.run instead of shutil.make_archive to avoid corrupted files, see https://github.com/huggingface/doc-builder/issues/348 print(f"Running zip command: zip -r {zip_file_path} {path_docs_built}") @@ -76,13 +67,14 @@ def push_command_add(args): while number_of_retries: try: if args.upload_version_yml: - # removing doc artifact folder to upload 2 files using `upload_folder`: _version.yml and zipped doc artifact file - shutil.rmtree(f"{library_name}/{doc_version_folder}") - api.upload_folder( + operations = [ + CommitOperationAdd(path_in_repo=zip_file_path, path_or_fileobj=zip_file_path), + CommitOperationAdd(path_in_repo=f"{library_name}/_versions.yml",path_or_fileobj=f"{library_name}/_versions.yml"), + ] + api.create_commit( repo_id=args.doc_build_repo_id, repo_type=REPO_TYPE, - folder_path=library_name, - path_in_repo=library_name, + operations=operations, commit_message=args.commit_msg, token=args.token, ) From a4b93de1eacdf9053cc0a875294617afcd674dd1 Mon Sep 17 00:00:00 2001 From: coyotte508 Date: Tue, 21 Nov 2023 20:03:14 +0100 Subject: [PATCH 2/2] =?UTF-8?q?=F0=9F=9A=A8?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/doc_builder/commands/push.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/doc_builder/commands/push.py b/src/doc_builder/commands/push.py index 9a02f8c80..91ee5910e 100644 --- a/src/doc_builder/commands/push.py +++ b/src/doc_builder/commands/push.py @@ -25,6 +25,7 @@ REPO_TYPE = "dataset" + def push_command(args): """ Commit file doc builds changes using: 1. zip doc build artifacts 2. hf_hub client to upload/delete zip file @@ -69,7 +70,9 @@ def push_command_add(args): if args.upload_version_yml: operations = [ CommitOperationAdd(path_in_repo=zip_file_path, path_or_fileobj=zip_file_path), - CommitOperationAdd(path_in_repo=f"{library_name}/_versions.yml",path_or_fileobj=f"{library_name}/_versions.yml"), + CommitOperationAdd( + path_in_repo=f"{library_name}/_versions.yml", path_or_fileobj=f"{library_name}/_versions.yml" + ), ] api.create_commit( repo_id=args.doc_build_repo_id,