Skip to content

♻️ Use hf_hub.create_commit instead of upload_folder #453

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 2 commits into
base: main
Choose a base branch
from
Open

♻️ Use hf_hub.create_commit instead of upload_folder #453

Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
ba8f788
♻️ Use hf_hub.create_commit instead of upload_folder
coyotte508 Nov 21, 2023
a4b93de
🚨
coyotte508 Nov 21, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
25 changes: 10 additions & 15 deletions src/doc_builder/commands/push.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,18 +20,10 @@
from pathlib import Path
from time import sleep, time

from huggingface_hub import HfApi
from huggingface_hub import HfApi, CommitOperationAdd


REPO_TYPE = "dataset"
SEPARATOR = "/"


def create_zip_name(library_name, version, with_ext=True):
file_name = f"{library_name}{SEPARATOR}{version}"
if with_ext:
file_name += ".zip"
return file_name


def push_command(args):
Expand Down Expand Up @@ -63,7 +55,7 @@ def push_command_add(args):
)
doc_version_folder = str(doc_version_folder)

zip_file_path = create_zip_name(library_name, doc_version_folder)
zip_file_path = f"{library_name}/{doc_version_folder}"
# eg create ./transformers/v4.0.zip with '/transformers/v4.0/*' file architecture inside
# Use subprocess.run instead of shutil.make_archive to avoid corrupted files, see https://github.com/huggingface/doc-builder/issues/348
print(f"Running zip command: zip -r {zip_file_path} {path_docs_built}")
Expand All @@ -76,13 +68,16 @@ def push_command_add(args):
while number_of_retries:
try:
if args.upload_version_yml:
# removing doc artifact folder to upload 2 files using `upload_folder`: _version.yml and zipped doc artifact file
shutil.rmtree(f"{library_name}/{doc_version_folder}")
api.upload_folder(
operations = [
CommitOperationAdd(path_in_repo=zip_file_path, path_or_fileobj=zip_file_path),
CommitOperationAdd(
path_in_repo=f"{library_name}/_versions.yml", path_or_fileobj=f"{library_name}/_versions.yml"
),
]
api.create_commit(
repo_id=args.doc_build_repo_id,
repo_type=REPO_TYPE,
folder_path=library_name,
path_in_repo=library_name,
operations=operations,
commit_message=args.commit_msg,
token=args.token,
)
Expand Down