Download copilot/adapters/storage.py from Wardline/cloak-api: direct link, hf CLI and curl.
- Browser
- Download file 5.75 kB
-
https://huggingface.co/spaces/Wardline/cloak-api/resolve/main/copilot/adapters/storage.py
- Command line
-
hf download hf://spaces/Wardline/cloak-api/copilot/adapters/storage.py
-
curl -L -o storage.py https://huggingface.co/spaces/Wardline/cloak-api/resolve/main/copilot/adapters/storage.py
5.75 kB
| """S3, GCS, Azure Blob and GitHub policies: at-rest scanning of buckets and repositories. | |
| These scan and record. Nothing is blocked, deleted or quarantined. "Specific data" uses | |
| detection-rule mode; "anything sensitive" uses broad discovery. | |
| """ | |
| from __future__ import annotations | |
| from ..interpret import Request | |
| from ..model import Draft, Question | |
| from ..platform.catalog import TYPE_LABEL | |
| from ..platform.context import PlatformContext | |
| from ..rules import RuleDecision | |
| from .base import (domains_of, size_bytes, qualifiers_unsupported, Build, answer_for, base_body, channel_group_check, exceptions_unsupported, pick_connectors, | |
| rule_questions, schedule_unsupported, thresholds_unsupported) | |
| RECORD_VERBS = {None, 'LOG', 'ALERT', 'ENABLE', 'CREATE'} | |
| def build(req: Request, rd: RuleDecision, ctx: PlatformContext, index: int, answers: dict, kind: str) -> Build: | |
| d = Draft(kind, None, rd) | |
| b = Build(d, consumed_exceptions=list(req.consumed)) | |
| body = base_body(kind) | |
| cfg = body['channels'][kind] | |
| c = req.content | |
| label = TYPE_LABEL[kind] | |
| if req.verb not in RECORD_VERBS and answer_for(answers, 'action', index) != 'log': | |
| d.issue('ACTION_NOT_AVAILABLE', 'blocking', | |
| f'{label} policies scan and record what they find; they cannot block, delete or quarantine.', | |
| requested=(req.verb or '').lower(), type=kind, available=['log']) | |
| d.ask(Question('action', f'{label} can only scan and record findings. Continue with that?', | |
| [{'value': 'log', 'label': 'Scan and record'}, {'value': 'cancel', 'label': 'Cancel'}], | |
| True, 'log')) | |
| cfg['connectorIds'] = pick_connectors(d, req, ctx, kind, index, answers) | |
| specific = rd.mode != 'none' and not rd.assumed and 'discovery' not in c.mods | |
| if specific: | |
| cfg['scanMode'] = 'rules' | |
| cfg['detectionRuleIds'] = list(rd.rule_ids) | |
| else: | |
| cfg['scanMode'] = 'discovery' | |
| if rd.mode != 'none': | |
| d.issue('DISCOVERY_FOR_GENERIC_DATA', 'info', 'For "any sensitive data" the scan uses broad discovery, ' | |
| 'which flags every kind of PII, PHI and card data it finds.') | |
| rd.mode, rd.rule_ids, rd.assumed = 'none', [], False | |
| if 'auto_sync' in c.mods or c.interval_minutes: | |
| cfg['autoSync']['enabled'] = True | |
| if c.interval_minutes: | |
| lo = 15 if kind == 'github' else 1 | |
| if c.interval_minutes < lo: | |
| d.issue('INTERVAL_RAISED', 'info', f'{label} re-scans at most every {lo} minutes.') | |
| cfg['autoSync']['intervalMinutes'] = max(lo, int(c.interval_minutes)) | |
| handled: set[str] = {'sizes'} | |
| sizes = list(c.sizes) | |
| for k, e in enumerate(req.exceptions): | |
| if e.sizes and k not in b.consumed_exceptions: | |
| sizes += e.sizes | |
| b.consumed_exceptions.append(k) | |
| for s in sizes: | |
| n = size_bytes(s) | |
| if n: | |
| cfg['maxFileBytes'] = n | |
| d.issue('SCAN_SIZE_LIMIT', 'info', f'Files larger than {s.upper()} are listed but not read.') | |
| break | |
| count_key = 'maxFiles' if kind == 'github' else 'maxObjects' | |
| for n, noun in c.thresholds: | |
| if noun in ('files', 'objects', 'items') and not (kind == 'github' and noun == 'items' and 'clone' in c.mods): | |
| cfg[count_key] = max(1, int(n)) | |
| handled |= {'files', 'objects'} | |
| d.issue('SCAN_COUNT_LIMIT', 'info', f'Each scan reads at most {n} files.') | |
| break | |
| if domains_of(c): | |
| d.issue('DOMAINS_NOT_USED', 'warning', f'{", ".join(domains_of(c))} has no meaning in a scan policy and was ' | |
| 'not used.') | |
| if kind == 'github': | |
| repos = [n for n, noun in c.thresholds if noun in ('repos', 'times', 'items')] | |
| if repos or 'clone' in c.mods: | |
| if repos: | |
| cfg['bulkRepoThreshold'] = max(1, int(repos[0])) | |
| handled |= {'repos', 'times', 'items'} | |
| d.issue('GITHUB_CLONE_ALERT', 'info', | |
| f'Raises an incident when one person clones or downloads {cfg["bulkRepoThreshold"]} or more ' | |
| 'repositories in an hour. This needs the GitHub Enterprise Cloud audit log.') | |
| extra = list(c.paths) | |
| for k, e in enumerate(req.exceptions): | |
| # "exclude node_modules and test folders": paths the scan should never read | |
| if k not in b.consumed_exceptions and (e.paths or e.words) and e.data.empty() and not e.groups: | |
| extra += e.paths + [w if ('.' in w or '/' in w) else f'{w}/' for w in e.words] | |
| b.consumed_exceptions.append(k) | |
| for p in extra: | |
| if p not in cfg['excludePaths']: | |
| cfg['excludePaths'].append(p) | |
| if extra: | |
| d.issue('GITHUB_EXCLUDES', 'info', f'The scan will skip {", ".join(extra)}.') | |
| d.issue('SCAN_IS_MANUAL', 'info', f'{label} scans are started from the {label} dashboard; this policy decides ' | |
| 'what they look for.') | |
| channel_group_check(d, req, ctx, index, answers) | |
| thresholds_unsupported(d, req, index, answers, handled=handled) | |
| schedule_unsupported(d, req, index, answers) | |
| qualifiers_unsupported(d, req, index, answers, handled=({'bulk'} if kind == 'github' else set()) | {'sizes'}) | |
| exceptions_unsupported(d, req, b.consumed_exceptions, index, answers, ctx) | |
| rule_questions(d, rd, ctx) | |
| d.spans = [{'start': s, 'end': e, 'label': l} for s, e, l in req.spans] | |
| d.policyBody = body | |
| return b | |