feat: Inititial commit
Browse files- .github/copilot-instructions.md +1 -0
- .github/workflows/upload-to-pypi.yml +34 -0
- MANIFEST.in +3 -0
- pyproject.toml +3 -0
- scribe_python_client/__init__.py +1 -0
- scribe_python_client/cli.py +47 -0
- scribe_python_client/client.py +146 -0
- scribe_python_client/constants.py +5 -0
- scribe_python_client/utils.py +67 -0
- scribe_python_client/version.py +1 -0
- setup.py +27 -0
.github/copilot-instructions.md
ADDED
|
@@ -0,0 +1 @@
|
|
|
|
|
|
|
| 1 |
+
<!-- Use this file to provide workspace-specific custom instructions to Copilot. For more details, visit https://code.visualstudio.com/docs/copilot/copilot-customization#_use-a-githubcopilotinstructionsmd-file -->
|
.github/workflows/upload-to-pypi.yml
ADDED
|
@@ -0,0 +1,34 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
name: Upload To PyPI
|
| 2 |
+
|
| 3 |
+
on:
|
| 4 |
+
workflow_dispatch:
|
| 5 |
+
|
| 6 |
+
jobs:
|
| 7 |
+
build_pkg:
|
| 8 |
+
runs-on: ubuntu-latest
|
| 9 |
+
|
| 10 |
+
steps:
|
| 11 |
+
- name: Checkout
|
| 12 |
+
uses: actions/checkout@v4
|
| 13 |
+
|
| 14 |
+
- name: Set up Python
|
| 15 |
+
uses: actions/setup-python@v4
|
| 16 |
+
with:
|
| 17 |
+
python-version: '3.x'
|
| 18 |
+
|
| 19 |
+
- name: Install dependencies
|
| 20 |
+
run: |
|
| 21 |
+
python -m pip install --upgrade pip
|
| 22 |
+
pip install setuptools wheel twine bump2version
|
| 23 |
+
|
| 24 |
+
- name: Bump Version
|
| 25 |
+
run: |
|
| 26 |
+
bump2version patch --allow-dirty
|
| 27 |
+
|
| 28 |
+
- name: Build and Upload
|
| 29 |
+
env:
|
| 30 |
+
TWINE_USERNAME: __token__
|
| 31 |
+
TWINE_PASSWORD: ${{ secrets.PYPI_TOKEN }}
|
| 32 |
+
run: |
|
| 33 |
+
python setup.py sdist bdist_wheel
|
| 34 |
+
twine upload dist/*
|
MANIFEST.in
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
include README.md
|
| 2 |
+
include LICENSE
|
| 3 |
+
recursive-include scribe_python_client *
|
pyproject.toml
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
[build-system]
|
| 2 |
+
requires = ["setuptools", "wheel"]
|
| 3 |
+
build-backend = "setuptools.build_meta"
|
scribe_python_client/__init__.py
ADDED
|
@@ -0,0 +1 @@
|
|
|
|
|
|
|
| 1 |
+
# Initialize the scribe_python_client package
|
scribe_python_client/cli.py
ADDED
|
@@ -0,0 +1,47 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import argparse
|
| 2 |
+
import logging
|
| 3 |
+
from .client import ScribeClient
|
| 4 |
+
|
| 5 |
+
def main():
|
| 6 |
+
parser = argparse.ArgumentParser(description="Scribe Client CLI")
|
| 7 |
+
parser.add_argument("--api_call", choices=[
|
| 8 |
+
"get-products",
|
| 9 |
+
"get-datasets",
|
| 10 |
+
"get-policy-results",
|
| 11 |
+
"list-attestations",
|
| 12 |
+
"get-attestation",
|
| 13 |
+
"get-product-vulnerabilities",
|
| 14 |
+
"get-latest-attestation"
|
| 15 |
+
], help="Which API call to execute")
|
| 16 |
+
parser.add_argument("--api_token", required=False, default=None, help="Your API token from ScribeHub integrations page")
|
| 17 |
+
parser.add_argument("--env", default="prod", choices=["prod", "dev", "test", "ci"], help="Which environment to use")
|
| 18 |
+
|
| 19 |
+
args = parser.parse_args()
|
| 20 |
+
|
| 21 |
+
base_url = "https://api.scribesecurity.com"
|
| 22 |
+
if args.env != "prod":
|
| 23 |
+
base_url = f"https://api.{args.env}.scribesecurity.com"
|
| 24 |
+
|
| 25 |
+
if not args.api_token:
|
| 26 |
+
logging.error("No API token provided")
|
| 27 |
+
exit()
|
| 28 |
+
|
| 29 |
+
client = ScribeClient(api_token=args.api_token, base_url=base_url)
|
| 30 |
+
|
| 31 |
+
if args.api_call == "get-products":
|
| 32 |
+
print(client.get_products_str())
|
| 33 |
+
elif args.api_call == "get-datasets":
|
| 34 |
+
print(client.get_datasets())
|
| 35 |
+
elif args.api_call == "get-policy-results":
|
| 36 |
+
print(client.get_policy_results_str())
|
| 37 |
+
elif args.api_call == "list-attestations":
|
| 38 |
+
print(client.list_attestations())
|
| 39 |
+
elif args.api_call == "get-attestation":
|
| 40 |
+
print(client.get_attestation("example-attestation-id"))
|
| 41 |
+
elif args.api_call == "get-product-vulnerabilities":
|
| 42 |
+
print(client.get_product_vulnerabilities("example-logical-app"))
|
| 43 |
+
elif args.api_call == "get-latest-attestation":
|
| 44 |
+
print(client.get_latest_attestation())
|
| 45 |
+
|
| 46 |
+
if __name__ == "__main__":
|
| 47 |
+
main()
|
scribe_python_client/client.py
ADDED
|
@@ -0,0 +1,146 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import logging
|
| 2 |
+
import datetime
|
| 3 |
+
import jwt
|
| 4 |
+
import json
|
| 5 |
+
import os
|
| 6 |
+
from .utils import convert_timestamps, send_post_request, send_get_request
|
| 7 |
+
from .constants import (
|
| 8 |
+
SCRIBE_VULNERABILITIES_DATASET,
|
| 9 |
+
SCRIBE_PRODUCTS_DATASET,
|
| 10 |
+
SCRIBE_POLICY_DATASET,
|
| 11 |
+
SCRIBE_TEAM_STAT_DATASET,
|
| 12 |
+
SCRIBE_LINEAGE_DATASET
|
| 13 |
+
)
|
| 14 |
+
|
| 15 |
+
class ScribeClient:
|
| 16 |
+
def __init__(self, api_token=None, ignore_env=True, base_url="https://api.scribesecurity.com", refresh_interval=600, cached_data={}):
|
| 17 |
+
if ignore_env:
|
| 18 |
+
self.api_token = api_token
|
| 19 |
+
else:
|
| 20 |
+
self.api_token = api_token or os.getenv('SCRIBE_TOKEN')
|
| 21 |
+
self.base_url = base_url
|
| 22 |
+
self.refresh_interval = refresh_interval
|
| 23 |
+
|
| 24 |
+
self.jwt_token = None
|
| 25 |
+
self.superset_token = None
|
| 26 |
+
self.dataset_ids = None
|
| 27 |
+
self.datasets = None
|
| 28 |
+
self.product_list = None
|
| 29 |
+
self.team_id = None
|
| 30 |
+
self.team_name = None
|
| 31 |
+
self.team_info = None
|
| 32 |
+
|
| 33 |
+
if self.api_token:
|
| 34 |
+
self.last_refresh = datetime.datetime.now()
|
| 35 |
+
self.refresh_data(force=True)
|
| 36 |
+
|
| 37 |
+
def refresh_data(self, force=False):
|
| 38 |
+
if not self.api_token:
|
| 39 |
+
logging.error("No API token provided")
|
| 40 |
+
return
|
| 41 |
+
|
| 42 |
+
need_refresh = (
|
| 43 |
+
force or
|
| 44 |
+
not self.dataset_ids or
|
| 45 |
+
not self.product_list or
|
| 46 |
+
(datetime.datetime.now() - self.last_refresh).seconds > self.refresh_interval
|
| 47 |
+
)
|
| 48 |
+
|
| 49 |
+
if not need_refresh:
|
| 50 |
+
return
|
| 51 |
+
|
| 52 |
+
self.last_refresh = datetime.datetime.now()
|
| 53 |
+
|
| 54 |
+
self.jwt_token = self.login(self.api_token, self.base_url)
|
| 55 |
+
if not self.jwt_token:
|
| 56 |
+
logging.error("Refresh: Failed to get JWT token")
|
| 57 |
+
|
| 58 |
+
try:
|
| 59 |
+
decoded_token = jwt.decode(self.jwt_token, options={"verify_signature": False})
|
| 60 |
+
self.decoded_token = decoded_token
|
| 61 |
+
except jwt.InvalidTokenError as e:
|
| 62 |
+
logging.error(f"Invalid JWT token: {e}")
|
| 63 |
+
self.decoded_token = None
|
| 64 |
+
return
|
| 65 |
+
|
| 66 |
+
self.superset_token = self.get_superset_token(self.jwt_token, self.base_url)
|
| 67 |
+
if not self.superset_token:
|
| 68 |
+
logging.error("Refresh: Failed to get Superset token")
|
| 69 |
+
|
| 70 |
+
def login(self, api_token, base_url):
|
| 71 |
+
url = f"{base_url}/v1/login"
|
| 72 |
+
body = {"api_token": api_token}
|
| 73 |
+
response = send_post_request(url=url, body=body)
|
| 74 |
+
|
| 75 |
+
if response and 'token' in response.json():
|
| 76 |
+
return response.json()['token']
|
| 77 |
+
else:
|
| 78 |
+
logging.error("Token not found in the response.")
|
| 79 |
+
return None
|
| 80 |
+
|
| 81 |
+
def get_superset_token(self, jwt_token, base_url):
|
| 82 |
+
url = f"{base_url}/dataset/token"
|
| 83 |
+
response = send_get_request(url=url, token=jwt_token)
|
| 84 |
+
if not response:
|
| 85 |
+
return None
|
| 86 |
+
|
| 87 |
+
response_json = json.loads(response.text)
|
| 88 |
+
if 'access_token' in response_json:
|
| 89 |
+
return response_json['access_token']
|
| 90 |
+
else:
|
| 91 |
+
logging.error("Access token not found in the response.")
|
| 92 |
+
return None
|
| 93 |
+
|
| 94 |
+
def get_products(self):
|
| 95 |
+
if not self.validate_dataset(SCRIBE_PRODUCTS_DATASET):
|
| 96 |
+
return None
|
| 97 |
+
|
| 98 |
+
url = f"{self.base_url}/dataset/data"
|
| 99 |
+
body = {
|
| 100 |
+
"superset_token": self.superset_token,
|
| 101 |
+
"validate": False,
|
| 102 |
+
"query": {
|
| 103 |
+
"datasource": {
|
| 104 |
+
"id": self.dataset_ids[SCRIBE_PRODUCTS_DATASET]["id"],
|
| 105 |
+
"type": "table"
|
| 106 |
+
},
|
| 107 |
+
"force": "false",
|
| 108 |
+
"queries": [
|
| 109 |
+
{
|
| 110 |
+
"columns": [
|
| 111 |
+
"logical_app",
|
| 112 |
+
"logical_app_version",
|
| 113 |
+
"version_timestamp"
|
| 114 |
+
],
|
| 115 |
+
"filters": [],
|
| 116 |
+
"metrics": [],
|
| 117 |
+
"orderby": [["logical_app", True], ["version_timestamp", False]],
|
| 118 |
+
"row_limit": 0
|
| 119 |
+
}
|
| 120 |
+
],
|
| 121 |
+
"result_format": "json",
|
| 122 |
+
"result_type": "results"
|
| 123 |
+
}
|
| 124 |
+
}
|
| 125 |
+
|
| 126 |
+
response = send_post_request(url=url, token=self.jwt_token, body=body)
|
| 127 |
+
if not response:
|
| 128 |
+
return None
|
| 129 |
+
|
| 130 |
+
try:
|
| 131 |
+
data = response.json()
|
| 132 |
+
return convert_timestamps(data, "version_timestamp")
|
| 133 |
+
except Exception as e:
|
| 134 |
+
logging.error(f"Error decoding JSON: {e}")
|
| 135 |
+
return None
|
| 136 |
+
|
| 137 |
+
def validate_dataset(self, dataset_name):
|
| 138 |
+
if not self.dataset_ids:
|
| 139 |
+
self.refresh_data()
|
| 140 |
+
if not self.dataset_ids:
|
| 141 |
+
logging.error("Failed to get dataset IDs")
|
| 142 |
+
return False
|
| 143 |
+
if dataset_name not in self.dataset_ids:
|
| 144 |
+
logging.error(f"Dataset {dataset_name} not found")
|
| 145 |
+
return False
|
| 146 |
+
return True
|
scribe_python_client/constants.py
ADDED
|
@@ -0,0 +1,5 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
SCRIBE_VULNERABILITIES_DATASET = "API Vulnerabilities"
|
| 2 |
+
SCRIBE_PRODUCTS_DATASET = "asbom_filtered"
|
| 3 |
+
SCRIBE_POLICY_DATASET = "Policy results v5"
|
| 4 |
+
SCRIBE_TEAM_STAT_DATASET = "MARKETING Team activity"
|
| 5 |
+
SCRIBE_LINEAGE_DATASET = "extended_lineage_new"
|
scribe_python_client/utils.py
ADDED
|
@@ -0,0 +1,67 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import datetime
|
| 2 |
+
import logging
|
| 3 |
+
import json
|
| 4 |
+
import requests
|
| 5 |
+
|
| 6 |
+
def convert_timestamps(data, col_substr="timestamp"):
|
| 7 |
+
"""
|
| 8 |
+
Recursively convert all fields containing 'timestamp' in their key
|
| 9 |
+
from epoch-milliseconds to UTC ISO8601 formatted strings.
|
| 10 |
+
"""
|
| 11 |
+
if isinstance(data, dict):
|
| 12 |
+
new_data = {}
|
| 13 |
+
for k, v in data.items():
|
| 14 |
+
if col_substr in k and isinstance(v, (int, float)):
|
| 15 |
+
ts_seconds = v / 1000.0
|
| 16 |
+
dt = datetime.datetime.fromtimestamp(ts_seconds)
|
| 17 |
+
new_data[k] = dt.strftime("%Y-%m-%dT%H:%M:%SZ")
|
| 18 |
+
else:
|
| 19 |
+
new_data[k] = convert_timestamps(v, col_substr=col_substr)
|
| 20 |
+
return new_data
|
| 21 |
+
|
| 22 |
+
elif isinstance(data, list):
|
| 23 |
+
return [convert_timestamps(item, col_substr=col_substr) for item in data]
|
| 24 |
+
|
| 25 |
+
return data
|
| 26 |
+
|
| 27 |
+
def send_post_request(url, token=None, body=None):
|
| 28 |
+
try:
|
| 29 |
+
headers = {}
|
| 30 |
+
if token is not None:
|
| 31 |
+
headers['Authorization'] = f'Bearer {token}'
|
| 32 |
+
headers['Content-Type'] = 'application/json'
|
| 33 |
+
|
| 34 |
+
logging.debug(f"Sending POST request to {url} with body: {json.dumps(body, indent=2)}")
|
| 35 |
+
|
| 36 |
+
response = requests.post(url=url, headers=headers, json=body)
|
| 37 |
+
|
| 38 |
+
if response.status_code == 200:
|
| 39 |
+
logging.info(f"POST request to {url} successful with status code: {response.status_code}")
|
| 40 |
+
return response
|
| 41 |
+
else:
|
| 42 |
+
logging.warning(f"POST request to {url} failed with status code: {response.status_code}, {response.text}")
|
| 43 |
+
return None
|
| 44 |
+
|
| 45 |
+
except requests.exceptions.RequestException as e:
|
| 46 |
+
logging.error(f"An error occurred: {e}")
|
| 47 |
+
return None
|
| 48 |
+
|
| 49 |
+
def send_get_request(url, token=None):
|
| 50 |
+
try:
|
| 51 |
+
headers = {}
|
| 52 |
+
if token is not None:
|
| 53 |
+
headers['Authorization'] = f'Bearer {token}'
|
| 54 |
+
headers['Content-Type'] = 'application/json'
|
| 55 |
+
|
| 56 |
+
response = requests.get(url=url, headers=headers)
|
| 57 |
+
|
| 58 |
+
if response.status_code == 200:
|
| 59 |
+
logging.info(f"GET request from {url} successful with status code: {response.status_code}")
|
| 60 |
+
return response
|
| 61 |
+
else:
|
| 62 |
+
logging.warning(f"GET request from {url} failed with status code: {response.status_code}\n{response.text}")
|
| 63 |
+
return None
|
| 64 |
+
|
| 65 |
+
except requests.exceptions.RequestException as e:
|
| 66 |
+
logging.error(f"An error occurred: {e}")
|
| 67 |
+
return None
|
scribe_python_client/version.py
ADDED
|
@@ -0,0 +1 @@
|
|
|
|
|
|
|
| 1 |
+
__version__ = "0.1.0"
|
setup.py
ADDED
|
@@ -0,0 +1,27 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from setuptools import setup, find_packages
|
| 2 |
+
from scribe_python_client.version import __version__
|
| 3 |
+
|
| 4 |
+
setup(
|
| 5 |
+
name='scribe-python-client',
|
| 6 |
+
version=__version__,
|
| 7 |
+
packages=find_packages(),
|
| 8 |
+
install_requires=[
|
| 9 |
+
'requests',
|
| 10 |
+
'PyJWT'
|
| 11 |
+
],
|
| 12 |
+
entry_points={
|
| 13 |
+
'console_scripts': [
|
| 14 |
+
'scribe-client=scribe_python_client.cli:main',
|
| 15 |
+
],
|
| 16 |
+
},
|
| 17 |
+
description='A Python client for interacting with ScribeHub.',
|
| 18 |
+
author='Daniel Nebenzahl',
|
| 19 |
+
author_email='dn@scribesecurity.com',
|
| 20 |
+
url='https://github.com/scribe-security/scribe-python-client',
|
| 21 |
+
classifiers=[
|
| 22 |
+
'Programming Language :: Python :: 3',
|
| 23 |
+
'License :: OSI Approved :: MIT License',
|
| 24 |
+
'Operating System :: OS Independent',
|
| 25 |
+
],
|
| 26 |
+
python_requires='>=3.6',
|
| 27 |
+
)
|