Buckets:
| """A local server for interacting with project repos.""" | |
| from __future__ import annotations | |
| import logging | |
| import os | |
| import platform | |
| import re | |
| import subprocess | |
| from typing import Literal | |
| import dvc | |
| import dvc.config | |
| import dvc.repo | |
| import dvc.repo.data | |
| import dvc.repo.status | |
| from dvc.exceptions import NotDvcRepoError | |
| from fastapi import FastAPI, HTTPException | |
| from pydantic import BaseModel | |
| from starlette.middleware.cors import CORSMiddleware | |
| import calkit | |
| import calkit.jupyter | |
| from calkit.dvc import get_dvc_repo, run_dvc_command | |
| logger = logging.getLogger(__package__) | |
| logger.setLevel(logging.INFO) | |
| app = FastAPI(title="calkit-server") | |
| app.add_middleware( | |
| CORSMiddleware, | |
| allow_origins=[ | |
| "http://localhost:5173", | |
| "http://localhost", | |
| "https://calkit.io", | |
| "https://staging.calkit.io", | |
| ], | |
| allow_credentials=True, | |
| allow_methods=["*"], | |
| allow_headers=["*"], | |
| ) | |
| class Message(BaseModel): | |
| message: str | |
| def get_health() -> str: | |
| return "All good!" | |
| class LocalProject(BaseModel): | |
| owner_name: str | |
| project_name: str | |
| wdir: str | |
| jupyter_url: str | None | |
| jupyter_token: str | None | |
| def get_root(get_jupyter_servers: bool = True) -> list[LocalProject]: | |
| """Return information about the current running server. | |
| - The project owner | |
| - The project name | |
| - The current working directory | |
| - A Jupyter server running here, if applicable | |
| """ | |
| resp = [] | |
| logger.info("Finding project directories") | |
| project_dirs = calkit.find_project_dirs() | |
| if get_jupyter_servers: | |
| logger.info("Getting Jupyter servers") | |
| servers = calkit.jupyter.get_servers() | |
| else: | |
| servers = [] | |
| for pdir in project_dirs: | |
| logger.info(f"Inspecting {pdir}") | |
| try: | |
| project = calkit.detect_project_name(wdir=pdir) | |
| except ValueError: | |
| logger.warning(f"Can't detect project name in {pdir}") | |
| continue | |
| owner, name = project.split("/") | |
| url = token = None | |
| for server in servers: | |
| if server.wdir == pdir: | |
| url = server.url | |
| token = server.token | |
| break | |
| resp.append( | |
| LocalProject( | |
| owner_name=owner, | |
| project_name=name, | |
| wdir=os.path.abspath(pdir), | |
| jupyter_token=token, | |
| jupyter_url=url, | |
| ) | |
| ) | |
| return resp | |
| def get_local_project( | |
| owner_name: str, project_name: str, get_jupyter_server: bool = True | |
| ) -> LocalProject: | |
| all_projects = get_root(get_jupyter_servers=get_jupyter_server) | |
| for project in all_projects: | |
| if ( | |
| project.owner_name == owner_name | |
| and project.project_name == project_name | |
| ): | |
| return project | |
| raise HTTPException(404) | |
| def get_project_jupyter_server( | |
| owner_name: str, | |
| project_name: str, | |
| autostart=False, | |
| no_browser=False, | |
| ) -> calkit.jupyter.Server | None: | |
| project = get_local_project( | |
| owner_name=owner_name, project_name=project_name | |
| ) | |
| if project is None: | |
| return | |
| if project.jupyter_url is not None: | |
| return calkit.jupyter.Server( | |
| url=project.jupyter_url, | |
| token=project.jupyter_token, | |
| wdir=project.wdir, | |
| ) | |
| if autostart: | |
| calkit.jupyter.start_server(project.wdir, no_browser=no_browser) | |
| servers = calkit.jupyter.get_servers() | |
| for server in servers: | |
| if server.wdir == project.wdir: | |
| return server | |
| def stop_project_jupyter_server(owner_name: str, project_name: str) -> None: | |
| project = get_local_project( | |
| owner_name=owner_name, project_name=project_name | |
| ) | |
| if project is None: | |
| return | |
| if project.jupyter_url is not None: | |
| calkit.jupyter.stop_server(url=project.jupyter_url) | |
| def get_cwd() -> str: | |
| return os.getcwd() | |
| def get_ls(owner_name: str, project_name: str) -> list[dict]: | |
| project = get_local_project(owner_name, project_name) | |
| repo = calkit.git.get_repo(project.wdir) | |
| contents = os.listdir(project.wdir) | |
| resp = [] | |
| for item in contents: | |
| if item == ".git" or repo.ignored(item): | |
| continue | |
| if os.path.isfile(os.path.join(project.wdir, item)): | |
| kind = "file" | |
| else: | |
| kind = "dir" | |
| resp.append(dict(name=item, type=kind)) | |
| return sorted(resp, key=lambda item: (item["type"], item["name"])) | |
| def run_git_command( | |
| owner_name: str, | |
| project_name: str, | |
| command: str, | |
| params: dict | None = None, | |
| ): | |
| if params is None: | |
| params = {} | |
| project = get_local_project(owner_name, project_name) | |
| func = getattr(calkit.git.get_repo(project.wdir).git, command) | |
| return func(**params) | |
| class GitPushPost(BaseModel): | |
| remote_name: str = "origin" | |
| branch_name: str | None = None | |
| def git_push(owner_name: str, project_name: str, req: GitPushPost) -> Message: | |
| logger.info(f"Looking for project {owner_name}/{project_name}") | |
| project = get_local_project(owner_name, project_name) | |
| logger.info(f"Found project at {project.wdir}") | |
| git_repo = calkit.git.get_repo(project.wdir) | |
| branch_name = req.branch_name or git_repo.active_branch.name | |
| logger.info(f"Git pushing to {req.remote_name} {branch_name}") | |
| git_repo.git.push(req.remote_name, branch_name) | |
| return Message(message="Success!") | |
| def git_pull(owner_name: str, project_name: str) -> Message: | |
| logger.info(f"Looking for project {owner_name}/{project_name}") | |
| project = get_local_project(owner_name, project_name) | |
| logger.info(f"Found project at {project.wdir}") | |
| git_repo = calkit.git.get_repo(project.wdir) | |
| git_repo.git.pull("--ff-only") | |
| return Message(message="Success!") | |
| def dvc_pull(owner_name: str, project_name: str) -> Message: | |
| logger.info(f"Looking for project {owner_name}/{project_name}") | |
| project = get_local_project(owner_name, project_name) | |
| logger.info(f"Found project at {project.wdir}") | |
| result = run_dvc_command(["pull"], cwd=project.wdir) | |
| if result != 0: | |
| raise HTTPException(500, "DVC pull failed") | |
| return Message(message="Success!") | |
| def dvc_push(owner_name: str, project_name: str) -> Message: | |
| logger.info(f"Looking for project {owner_name}/{project_name}") | |
| project = get_local_project(owner_name, project_name) | |
| logger.info(f"Found project at {project.wdir}") | |
| result = run_dvc_command(["push"], cwd=project.wdir) | |
| if result != 0: | |
| raise HTTPException(500, "DVC push failed") | |
| return Message(message="Success!") | |
| def get_diff(): | |
| """Get differences in working directory, from both Git and DVC.""" | |
| raise HTTPException(501) | |
| class Status(BaseModel): | |
| dvc: dict | None | |
| git: dict | |
| errors: list[dict] | None = None | |
| def get_status( | |
| owner_name: str, | |
| project_name: str, | |
| fetch_git: bool = True, | |
| fetch_dvc: bool = True, | |
| ): | |
| """Get status in working directory, from both Git and DVC.""" | |
| errors = [] | |
| logger.info(f"Looking for project {owner_name}/{project_name}") | |
| project = get_local_project( | |
| owner_name, project_name, get_jupyter_server=False | |
| ) | |
| logger.info(f"Found project at {project.wdir}") | |
| git_repo = calkit.git.get_repo(project.wdir) | |
| if fetch_git: | |
| git_repo.git.fetch() | |
| untracked_git_files = git_repo.untracked_files | |
| # Get a list of diffs of the working tree to the index | |
| git_diff = git_repo.index.diff(None) | |
| git_diff_files = [d.a_path for d in git_diff] | |
| git_staged = git_repo.index.diff("HEAD") | |
| git_staged_files = [d.a_path for d in git_staged] | |
| # See if we're ahead and/or behind origin remote | |
| # From https://stackoverflow.com/a/52757014/2284865 | |
| ahead = 0 | |
| behind = 0 | |
| # Porcelain v2 is easier to parse, branch shows ahead/behind | |
| repo_status = git_repo.git.status(porcelain="v2", branch=True) | |
| ahead_behind_match = re.search( | |
| r"#\sbranch\.ab\s\+(\d+)\s-(\d+)", repo_status | |
| ) | |
| # If no remotes exist or the HEAD is detached, there is no ahead/behind | |
| if ahead_behind_match: | |
| ahead = int(ahead_behind_match.group(1)) | |
| behind = int(ahead_behind_match.group(2)) | |
| # If the DVC remote is not configured properly, we might raise a | |
| # dvc.config.ConfigError here | |
| try: | |
| dvc_repo = get_dvc_repo(project.wdir) | |
| # Get a dictionary of DVC artifacts that have changed, keyed by the DVC | |
| # file, where values are a list, which may include a dict with a | |
| # 'changed outs' key, e.g., | |
| # { | |
| # "data/jhtdb-transitional-bl/all-stats.h5.dvc": [ | |
| # { | |
| # "changed outs": { | |
| # "data/jhtdb-transitional-bl/all-stats.h5": "modified" | |
| # } | |
| # } | |
| # ] | |
| # } | |
| dvc_pipeline_status = dvc.repo.status.status(dvc_repo) | |
| logger.info(f"Raw DVC pipeline status: {dvc_pipeline_status}") | |
| # Frozen stages are intentionally pinned and hidden from status. | |
| # Always-run entries are kept so clients can surface them. | |
| frozen_stages = calkit.pipeline.frozen_stage_base_names( | |
| wdir=project.wdir | |
| ) | |
| dvc_pipeline_status = { | |
| k.split("dvc.yaml:")[-1]: v | |
| for k, v in dvc_pipeline_status.items() | |
| if not k.endswith(".dvc") | |
| and k.split("dvc.yaml:")[-1].split("@")[0] not in frozen_stages | |
| } | |
| logger.info( | |
| f"DVC pipeline status after filtering: {dvc_pipeline_status}" | |
| ) | |
| dvc_data_status = dvc.repo.data.status( | |
| dvc_repo, not_in_remote=fetch_dvc, remote_refresh=fetch_dvc | |
| ) | |
| # Reformat this a bit, since it can be a little hard to understand | |
| # DVC calls a path committed when its DVC file is staged | |
| dvc_data_status["changed"] = dvc_data_status.get( | |
| "uncommitted", {} | |
| ).get("modified", []) | |
| dvc_data_status["staged"] = dvc_data_status.get("committed", {}).get( | |
| "modified", [] | |
| ) | |
| dvc_status = dict(pipeline=dvc_pipeline_status, data=dvc_data_status) | |
| except (dvc.config.ConfigError, NotDvcRepoError) as e: | |
| errors.append(dict(type=str(type(e)), info=str(e))) | |
| dvc_status = None | |
| # TODO: Structure this in an intelligent way | |
| return { | |
| "dvc": dvc_status, | |
| "git": { | |
| "untracked": untracked_git_files, | |
| "changed": git_diff_files, | |
| "staged": git_staged_files, | |
| "commits_ahead": ahead, | |
| "commits_behind": behind, | |
| }, | |
| "errors": errors, | |
| } | |
| class GitIgnorePut(BaseModel): | |
| path: str | |
| commit: bool = True | |
| commit_message: str | None = None | |
| push: bool = False | |
| def put_git_ignored( | |
| owner_name: str, project_name: str, req: GitIgnorePut | |
| ) -> Message: | |
| project = get_local_project(owner_name, project_name) | |
| git_repo = calkit.git.get_repo(project.wdir) | |
| path = req.path | |
| if git_repo.ignored(path): | |
| logger.info(f"{path} is already ignored") | |
| return Message(message=f"{path} is already ignored") | |
| ignore_fpath = os.path.join(git_repo.working_dir, ".gitignore") | |
| if os.path.isfile(ignore_fpath): | |
| with open(ignore_fpath) as f: | |
| txt = f.read() | |
| else: | |
| txt = "" | |
| txt += "\n" + path + "\n" | |
| with open(ignore_fpath, "w") as f: | |
| f.write(txt) | |
| if req.commit and git_repo.git.diff(): | |
| git_repo.git.add(".gitignore") | |
| if req.commit_message is None: | |
| msg = f"Add {path} to gitignore" | |
| else: | |
| msg = req.commit_message | |
| git_repo.git.commit(["-m", msg]) | |
| if req.push: | |
| git_repo.git.push(["origin", git_repo.active_branch.name]) | |
| return Message(message="Success!") | |
| class AddPost(BaseModel): | |
| paths: list[str] | |
| to: Literal["git", "dvc"] | None = None | |
| commit: bool = True | |
| commit_message: str | None = None | |
| push: bool = False | |
| def calkit_add(owner_name: str, project_name: str, req: AddPost) -> Message: | |
| project = get_local_project(owner_name, project_name) | |
| cmd = ["calkit", "add"] + req.paths | |
| paths_txt = ", ".join(req.paths) | |
| if req.commit: | |
| msg = req.commit_message | |
| if msg is None: | |
| msg = f"Add {paths_txt}" | |
| cmd += ["--commit-message", msg] | |
| if req.to is not None: | |
| cmd += ["--to", req.to] | |
| if req.push: | |
| cmd += ["--push"] | |
| try: | |
| subprocess.call(cmd, cwd=project.wdir) | |
| except subprocess.CalledProcessError as e: | |
| raise HTTPException(500, f"Failed to run {cmd}: {e}") | |
| return Message(message=f"Added paths: {paths_txt}") | |
| class CommitPost(BaseModel): | |
| paths: list[str] | |
| to: Literal["git", "dvc"] | None = None | |
| commit_message: str | None = None | |
| push: bool = False | |
| def calkit_add_and_commit( | |
| owner_name: str, project_name: str, req: AddPost | |
| ) -> Message: | |
| project = get_local_project(owner_name, project_name) | |
| git_repo = calkit.git.get_repo(project.wdir) | |
| # Unstage any staged files since we're going to add and commit here | |
| git_repo.git.reset() | |
| cmd = ["calkit", "add"] + req.paths | |
| paths_txt = ", ".join(req.paths) | |
| msg = req.commit_message | |
| if msg is None: | |
| msg = f"Update {paths_txt}" | |
| cmd += ["--commit-message", msg] | |
| if req.to is not None: | |
| cmd += ["--to", req.to] | |
| if req.push: | |
| cmd += ["--push"] | |
| try: | |
| subprocess.call(cmd, cwd=project.wdir) | |
| except subprocess.CalledProcessError as e: | |
| raise HTTPException(500, f"Failed to run {cmd}: {e}") | |
| return Message(message=f"Added and committed paths: {paths_txt}") | |
| def discard_changes(owner_name: str, project_name: str) -> Message: | |
| project = get_local_project(owner_name, project_name) | |
| git_repo = calkit.git.get_repo(project.wdir) | |
| # Stash any git changes | |
| git_repo.git.stash() | |
| # Checkout any DVC changes | |
| # If the DVC remote is not configured properly, we might raise a | |
| # dvc.config.ConfigError here | |
| try: | |
| dvc_repo = get_dvc_repo(project.wdir) | |
| dvc_data_status = dvc.repo.data.status(dvc_repo) | |
| # Get only the changed files since anything will be uncommitted after | |
| # the git stash | |
| changed = dvc_data_status.get("uncommitted", {}).get("modified", []) | |
| for path in changed: | |
| logger.info(f"Checking out {path} with DVC") | |
| result = run_dvc_command( | |
| ["checkout", path, "--force"], | |
| cwd=project.wdir, | |
| ) | |
| if result != 0: | |
| raise HTTPException(500, f"Failed to checkout {path} with DVC") | |
| except dvc.config.ConfigError: | |
| pass | |
| return Message(message="Changes successfully discarded") | |
| def open_vscode(owner_name: str, project_name: str) -> int: | |
| project = get_local_project(owner_name, project_name) | |
| return os.system(f"code {project.wdir}") | |
| def get_jupyter_servers() -> list[calkit.jupyter.Server]: | |
| return calkit.jupyter.get_servers() | |
| def run_pipeline(owner_name: str, project_name: str) -> Message: | |
| project = get_local_project(owner_name, project_name) | |
| try: | |
| subprocess.check_call(["calkit", "run"], cwd=project.wdir) | |
| except subprocess.CalledProcessError as e: | |
| raise HTTPException(500, f"Pipeline failed to run: {e}") | |
| return Message(message="Success!") | |
| class Pipeline(BaseModel): | |
| raw_yaml: str | None = None | |
| stages: dict | |
| mermaid: str | None = None | |
| def get_pipeline(owner_name: str, project_name: str) -> Pipeline: | |
| project = get_local_project(owner_name, project_name) | |
| fpath = os.path.join(project.wdir, "dvc.yaml") | |
| if os.path.isfile(fpath): | |
| with open(fpath) as f: | |
| raw_yaml = f.read() | |
| pipeline = calkit.ryaml.load(raw_yaml) | |
| import io | |
| from contextlib import redirect_stdout | |
| mermaid_output = io.StringIO() | |
| with redirect_stdout(mermaid_output): | |
| result = run_dvc_command(["dag", "--mermaid"], cwd=project.wdir) | |
| if result != 0: | |
| raise HTTPException(500, "Failed to get DVC DAG") | |
| mermaid = mermaid_output.getvalue() | |
| return Pipeline( | |
| raw_yaml=raw_yaml, | |
| stages=pipeline.get("stages", {}), | |
| mermaid=mermaid, | |
| ) | |
| return Pipeline(raw_yaml=None, stages={}, mermaid=None) | |
| class StageObject(BaseModel): | |
| title: str | |
| description: str | |
| class StagePost(BaseModel): | |
| name: str | |
| cmd: str | |
| deps: list[str] | None | |
| outs: list[str] | None | |
| calkit_type: Literal["figure", "dataset", "publication"] | None = None | |
| calkit_object: StageObject | None = None | |
| commit: bool = True | |
| push: bool = False | |
| def post_pipeline_stage( | |
| owner_name: str, project_name: str, req: StagePost | |
| ) -> Message: | |
| project = get_local_project(owner_name, project_name) | |
| dvc_fpath = os.path.join(project.wdir, "dvc.yaml") | |
| if req.calkit_type is not None and req.calkit_object is None: | |
| raise HTTPException(422, "Calkit object info must be provided") | |
| if req.calkit_type is not None: | |
| if req.outs is None or len(req.outs) != 1: | |
| raise HTTPException(400, "One output must be provided") | |
| if os.path.isfile(dvc_fpath): | |
| with open(dvc_fpath) as f: | |
| pipeline = calkit.ryaml.load(f) | |
| else: | |
| pipeline = {} | |
| stages = pipeline.get("stages", {}) | |
| stage_names = list(stages.keys()) | |
| if req.name in stage_names: | |
| raise HTTPException( | |
| 400, "Stage with same name already exists in pipeline" | |
| ) | |
| # Make sure we don't have any conflicting outputs | |
| all_outs = [] | |
| for _, stage in stages.items(): | |
| all_outs += stage.get("outs", []) | |
| if req.outs is not None: | |
| for out in req.outs: | |
| if out in all_outs: | |
| raise HTTPException( | |
| 400, "Output is already part of another stage" | |
| ) | |
| new_stage = dict(cmd=req.cmd) | |
| if req.deps: | |
| new_stage["deps"] = req.deps | |
| if req.outs: | |
| new_stage["outs"] = req.outs | |
| stages[req.name] = new_stage | |
| pipeline["stages"] = stages | |
| with open(dvc_fpath, "w") as f: | |
| calkit.ryaml.dump(pipeline, f) | |
| repo = calkit.git.get_repo(project.wdir) | |
| repo.git.add("dvc.yaml") | |
| # Add Calkit object if applicable | |
| if req.calkit_type is not None: | |
| ck_info = calkit.load_calkit_info(wdir=project.wdir) | |
| ck_objs = ck_info.get(req.calkit_type + "s", []) | |
| existing_paths = [obj.get("path") for obj in ck_objs] | |
| out = req.outs[0] | |
| if out in existing_paths: | |
| raise HTTPException( | |
| 400, f"{req.calkit_type} already exists at {out}" | |
| ) | |
| new_obj = ( | |
| dict(path=out, stage=req.name) | req.calkit_object.model_dump() | |
| ) | |
| ck_objs.append(new_obj) | |
| ck_info[req.calkit_type + "s"] = ck_objs | |
| with open(os.path.join(project.wdir, "calkit.yaml"), "w") as f: | |
| calkit.ryaml.dump(ck_info, f) | |
| repo.git.add("calkit.yaml") | |
| if req.commit: | |
| repo.git.commit(["-m", f"Add new pipeline stage {req.name}"]) | |
| if req.push: | |
| repo.git.push(["origin", repo.active_branch.name]) | |
| return Message(message="Successfully added stage") | |
| def open_folder(owner_name: str, project_name: str) -> Message: | |
| project = get_local_project(owner_name, project_name) | |
| if platform.system() == "Windows": | |
| cmd = ["explorer"] | |
| else: | |
| cmd = ["open"] | |
| cmd.append(project.wdir) | |
| subprocess.Popen(cmd, cwd=project.wdir) | |
| return Message(message=f"Opened {project.wdir} with `{cmd[0]}`") | |
| class ClonePost(BaseModel): | |
| git_repo_url: str | |
| protocol: Literal["https", "ssh"] = "https" | |
| def clone_repo(req: ClonePost) -> Message: | |
| parent_dir = os.path.join(os.path.expanduser("~"), "calkit") | |
| os.makedirs(parent_dir, exist_ok=True) | |
| dest_dir = req.git_repo_url.split("/")[-1].removesuffix(".git") | |
| abs_dest_dir = os.path.join(parent_dir, dest_dir) | |
| if os.path.exists(abs_dest_dir): | |
| raise HTTPException(400, "Destination directory already exists") | |
| if req.protocol == "ssh": | |
| url = req.git_repo_url.replace( | |
| "https://github.com/", "git@github.com:" | |
| ) | |
| else: | |
| url = req.git_repo_url | |
| if not url.endswith(".git"): | |
| url += ".git" | |
| cmd = ["calkit", "clone", url] | |
| try: | |
| subprocess.call(cmd, cwd=parent_dir) | |
| except subprocess.CalledProcessError as e: | |
| logger.error(f"Failed to clone: {e}") | |
| raise HTTPException(500, f"Failed to clone: {e}") | |
| return Message(message=f"Successfully cloned into {abs_dest_dir}") | |
Xet Storage Details
- Size:
- 22 kB
- Xet hash:
- ece44e7c59188be56ec9bb59ae1a89365b10e9df676851fab520c9252baa90b6
·
Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.