Buckets:

guarin
/

fsx-backup

Files

xet

guarin/fsx-backup / tipsv2 /copy_to_google.py

guarin

2 months ago

download

raw

7.19 kB

"""Copy each guarin/tipsv2-* repo into the matching google/tipsv2-* repo via a pull request.
For every source repo, all files are downloaded locally and re-uploaded into the target repo as
a pull request using CommitOperationAdd. This works with only `discussion.write` access on the
target (the community-PR flow): the uploaded files, including LFS weights, land in the pull
request's own storage, so write access to the google org is not required. The trade-off is that
the weights are actually transferred rather than duplicated server-side by hash. The changes are
opened as a pull request on each google/* target so they can be reviewed before being merged.
"""

from pathlib import Path

from huggingface_hub import (
    CommitOperationAdd,
    CommitOperationDelete,
    HfApi,
    list_repo_files,
    snapshot_download,
)

# Stable PR title per source repo. Reusing the same title lets repeated runs detect an
# already-open PR and skip it, so re-running the script never opens a duplicate or changes
# an existing PR.
def pr_title(source_repo: str) -> str:
    return f"Update files for transformers integration"

# Source repos to copy. Each is mirrored to a target repo with the same name under the google org.
# Both the DPT repos and their backbones are copied.
SOURCE_REPOS = [
    # "guarin/tipsv2-b14-dpt",
    # "guarin/tipsv2-b14",
    "guarin/tipsv2-l14-dpt",
    # "guarin/tipsv2-l14",
    "guarin/tipsv2-so400m14-dpt",
    # "guarin/tipsv2-so400m14",
    "guarin/tipsv2-g14-dpt",
    # "guarin/tipsv2-g14",
]

api = HfApi()


def target_repo(source_repo: str) -> str:
    """Map a guarin/<name> source repo to its google/<name> target repo."""
    name = source_repo.split("/", 1)[1]
    return f"google/{name}"


def find_open_pr(target: str, title: str) -> int | None:
    """Return the number of an already-open PR on the target with the given title, if any."""
    for discussion in api.get_repo_discussions(
        repo_id=target, repo_type="model", discussion_type="pull_request", discussion_status="open"
    ):
        if discussion.title == title:
            return discussion.num
    return None


def copy_repo(source_repo: str) -> None:
    target = target_repo(source_repo)
    title = pr_title(source_repo)
    print(f"\n=== {source_repo} -> {target} ===")

    # Skip if a matching PR is already open, so repeated runs neither open duplicates nor
    # modify the existing PR.
    existing_pr = find_open_pr(target, title)
    if existing_pr is not None:
        print(f"  PR #{existing_pr} already open with this title; skipping.")
        return

    source_files = list_repo_files(source_repo, repo_type="model")
    print(f"  Files to copy: {len(source_files)}")

    # Download the full source repo locally, then re-upload its files into the target PR. The
    # community-PR flow only needs `discussion.write` on the target, but it cannot duplicate LFS
    # files server-side, so the weights are downloaded and uploaded here.
    print("  Downloading source files ...")
    local_dir = snapshot_download(source_repo, repo_type="model")

    operations = [
        CommitOperationAdd(
            path_in_repo=file_name,
            path_or_fileobj=f"{local_dir}/{file_name}",
        )
        for file_name in source_files
    ]

    print(f"  Opening PR on {target} (uploading {len(operations)} files) ...")
    commit_info = api.create_commit(
        repo_id=target,
        repo_type="model",
        operations=operations,
        commit_message=title,
        commit_description=(
            "Updates the repo to smoothly integrate with transformers.\n"
            "\n"
            "PR: https://github.com/huggingface/transformers/pull/46347\n"
            "\n"
            "Changes:\n"
            "- Update `config.json` to match transformers structure\n"
            "- Update config classes to handle new `config.json`\n"
            "- Add processor and tokenizer configs\n"
            "- Update DPT `model.safetensors` files to include DPT head + backbone weights\n"
            "- Update DPT model code to handle extra weights in `model.safetensors` file\n"
        ),
        create_pr=True,
    )

    print(f"  PR opened: {commit_info.pr_url}")
    print("  Done.")


# Local configuration files uploaded to the top level of each PR. The DPT config goes only to
# the DPT repos, the TIPS config only to the non-DPT (backbone) repos.
CONFIGURATION_DPT = Path(__file__).parent / "configuration_dpt.py"
MODELING_DPT = Path(__file__).parent / "modeling_dpt.py"
CONFIGURATION_TIPS = Path(__file__).parent / "configuration_tips.py"


def update_pr(source_repo: str) -> None:
    """Add a commit to the already-open PR for a source repo.
    The integration PRs were already opened by `copy_repo`. This adds a new commit to each
    PR (on its `refs/pr/<number>` revision) rather than opening a fresh PR. For DPT repos it
    removes `image_encoder.py` and uploads `configuration_dpt.py` and `modeling_dpt.py`; for
    non-DPT repos it uploads `configuration_tips.py`.
    """
    target = target_repo(source_repo)
    title = pr_title(source_repo)
    is_dpt = source_repo.endswith("-dpt")
    print(f"\n=== {source_repo} -> {target} ===")

    pr_number = find_open_pr(target, title)
    if pr_number is None:
        print("  No open PR with this title; skipping.")
        return

    revision = f"refs/pr/{pr_number}"
    pr_files = list_repo_files(target, repo_type="model", revision=revision)

    operations = []
    if is_dpt:
        if "image_encoder.py" in pr_files:
            operations.append(CommitOperationDelete(path_in_repo="image_encoder.py"))
        operations.append(
            CommitOperationAdd(
                path_in_repo=CONFIGURATION_DPT.name,
                path_or_fileobj=str(CONFIGURATION_DPT),
            )
        )
        operations.append(
            CommitOperationAdd(
                path_in_repo=MODELING_DPT.name,
                path_or_fileobj=str(MODELING_DPT),
            )
        )
        commit_message = "Update files"
        commit_description = (
            "Update files"
        )
    else:
        operations.append(
            CommitOperationAdd(
                path_in_repo=CONFIGURATION_TIPS.name,
                path_or_fileobj=str(CONFIGURATION_TIPS),
            )
        )
        commit_message = "Update configuration_tips.py"
        commit_description = "Uploads configuration_tips.py."

    print(f"  Updating PR #{pr_number}: {len(operations)} operation(s) ...")
    commit_info = api.create_commit(
        repo_id=target,
        repo_type="model",
        operations=operations,
        commit_message=commit_message,
        commit_description=commit_description,
        revision=revision,
        create_pr=False,
    )

    print(f"  Committed: {commit_info.commit_url}")
    print("  Done.")


def main() -> None:
    # The integration PRs are already open. Add a commit to each PR: DPT repos get
    # image_encoder.py removed and configuration_dpt.py uploaded, non-DPT repos get
    # configuration_tips.py uploaded.
    for source_repo in SOURCE_REPOS:
        update_pr(source_repo)


if __name__ == "__main__":
    main()

Xet Storage Details

Size:: 7.19 kB
Xet hash:: 6ff917d5325e3cfb4b1910f6c95fa0acb608ce9fe508ce0761a2078b5a8ff591

Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.

	"""Copy each guarin/tipsv2-* repo into the matching google/tipsv2-* repo via a pull request.

	For every source repo, all files are downloaded locally and re-uploaded into the target repo as
	a pull request using CommitOperationAdd. This works with only `discussion.write` access on the
	target (the community-PR flow): the uploaded files, including LFS weights, land in the pull
	request's own storage, so write access to the google org is not required. The trade-off is that
	the weights are actually transferred rather than duplicated server-side by hash. The changes are
	opened as a pull request on each google/* target so they can be reviewed before being merged.
	"""

	from pathlib import Path

	from huggingface_hub import (
	CommitOperationAdd,
	CommitOperationDelete,
	HfApi,
	list_repo_files,
	snapshot_download,
	)

	# Stable PR title per source repo. Reusing the same title lets repeated runs detect an
	# already-open PR and skip it, so re-running the script never opens a duplicate or changes
	# an existing PR.
	def pr_title(source_repo: str) -> str:
	return f"Update files for transformers integration"

	# Source repos to copy. Each is mirrored to a target repo with the same name under the google org.
	# Both the DPT repos and their backbones are copied.
	SOURCE_REPOS = [
	# "guarin/tipsv2-b14-dpt",
	# "guarin/tipsv2-b14",
	"guarin/tipsv2-l14-dpt",
	# "guarin/tipsv2-l14",
	"guarin/tipsv2-so400m14-dpt",
	# "guarin/tipsv2-so400m14",
	"guarin/tipsv2-g14-dpt",
	# "guarin/tipsv2-g14",
	]

	api = HfApi()


	def target_repo(source_repo: str) -> str:
	"""Map a guarin/<name> source repo to its google/<name> target repo."""
	name = source_repo.split("/", 1)[1]
	return f"google/{name}"


	def find_open_pr(target: str, title: str) -> int \| None:
	"""Return the number of an already-open PR on the target with the given title, if any."""
	for discussion in api.get_repo_discussions(
	repo_id=target, repo_type="model", discussion_type="pull_request", discussion_status="open"
	):
	if discussion.title == title:
	return discussion.num
	return None


	def copy_repo(source_repo: str) -> None:
	target = target_repo(source_repo)
	title = pr_title(source_repo)
	print(f"\n=== {source_repo} -> {target} ===")

	# Skip if a matching PR is already open, so repeated runs neither open duplicates nor
	# modify the existing PR.
	existing_pr = find_open_pr(target, title)
	if existing_pr is not None:
	print(f" PR #{existing_pr} already open with this title; skipping.")
	return

	source_files = list_repo_files(source_repo, repo_type="model")
	print(f" Files to copy: {len(source_files)}")

	# Download the full source repo locally, then re-upload its files into the target PR. The
	# community-PR flow only needs `discussion.write` on the target, but it cannot duplicate LFS
	# files server-side, so the weights are downloaded and uploaded here.
	print(" Downloading source files ...")
	local_dir = snapshot_download(source_repo, repo_type="model")

	operations = [
	CommitOperationAdd(
	path_in_repo=file_name,
	path_or_fileobj=f"{local_dir}/{file_name}",
	)
	for file_name in source_files
	]

	print(f" Opening PR on {target} (uploading {len(operations)} files) ...")
	commit_info = api.create_commit(
	repo_id=target,
	repo_type="model",
	operations=operations,
	commit_message=title,
	commit_description=(
	"Updates the repo to smoothly integrate with transformers.\n"
	"\n"
	"PR: https://github.com/huggingface/transformers/pull/46347\n"
	"\n"
	"Changes:\n"
	"- Update `config.json` to match transformers structure\n"
	"- Update config classes to handle new `config.json`\n"
	"- Add processor and tokenizer configs\n"
	"- Update DPT `model.safetensors` files to include DPT head + backbone weights\n"
	"- Update DPT model code to handle extra weights in `model.safetensors` file\n"
	),
	create_pr=True,
	)

	print(f" PR opened: {commit_info.pr_url}")
	print(" Done.")


	# Local configuration files uploaded to the top level of each PR. The DPT config goes only to
	# the DPT repos, the TIPS config only to the non-DPT (backbone) repos.
	CONFIGURATION_DPT = Path(__file__).parent / "configuration_dpt.py"
	MODELING_DPT = Path(__file__).parent / "modeling_dpt.py"
	CONFIGURATION_TIPS = Path(__file__).parent / "configuration_tips.py"


	def update_pr(source_repo: str) -> None:
	"""Add a commit to the already-open PR for a source repo.

	The integration PRs were already opened by `copy_repo`. This adds a new commit to each
	PR (on its `refs/pr/<number>` revision) rather than opening a fresh PR. For DPT repos it
	removes `image_encoder.py` and uploads `configuration_dpt.py` and `modeling_dpt.py`; for
	non-DPT repos it uploads `configuration_tips.py`.
	"""
	target = target_repo(source_repo)
	title = pr_title(source_repo)
	is_dpt = source_repo.endswith("-dpt")
	print(f"\n=== {source_repo} -> {target} ===")

	pr_number = find_open_pr(target, title)
	if pr_number is None:
	print(" No open PR with this title; skipping.")
	return

	revision = f"refs/pr/{pr_number}"
	pr_files = list_repo_files(target, repo_type="model", revision=revision)

	operations = []
	if is_dpt:
	if "image_encoder.py" in pr_files:
	operations.append(CommitOperationDelete(path_in_repo="image_encoder.py"))
	operations.append(
	CommitOperationAdd(
	path_in_repo=CONFIGURATION_DPT.name,
	path_or_fileobj=str(CONFIGURATION_DPT),
	)
	)
	operations.append(
	CommitOperationAdd(
	path_in_repo=MODELING_DPT.name,
	path_or_fileobj=str(MODELING_DPT),
	)
	)
	commit_message = "Update files"
	commit_description = (
	"Update files"
	)
	else:
	operations.append(
	CommitOperationAdd(
	path_in_repo=CONFIGURATION_TIPS.name,
	path_or_fileobj=str(CONFIGURATION_TIPS),
	)
	)
	commit_message = "Update configuration_tips.py"
	commit_description = "Uploads configuration_tips.py."

	print(f" Updating PR #{pr_number}: {len(operations)} operation(s) ...")
	commit_info = api.create_commit(
	repo_id=target,
	repo_type="model",
	operations=operations,
	commit_message=commit_message,
	commit_description=commit_description,
	revision=revision,
	create_pr=False,
	)

	print(f" Committed: {commit_info.commit_url}")
	print(" Done.")


	def main() -> None:
	# The integration PRs are already open. Add a commit to each PR: DPT repos get
	# image_encoder.py removed and configuration_dpt.py uploaded, non-DPT repos get
	# configuration_tips.py uploaded.
	for source_repo in SOURCE_REPOS:
	update_pr(source_repo)


	if __name__ == "__main__":
	main()