cloak-api / copilot /adapters /storage.py
n1th1sh's picture
Upload 35 files
375880c verified
Raw History Blame Contribute Delete
5.75 kB
"""S3, GCS, Azure Blob and GitHub policies: at-rest scanning of buckets and repositories.
These scan and record. Nothing is blocked, deleted or quarantined. "Specific data" uses
detection-rule mode; "anything sensitive" uses broad discovery.
"""
from __future__ import annotations
from ..interpret import Request
from ..model import Draft, Question
from ..platform.catalog import TYPE_LABEL
from ..platform.context import PlatformContext
from ..rules import RuleDecision
from .base import (domains_of, size_bytes, qualifiers_unsupported, Build, answer_for, base_body, channel_group_check, exceptions_unsupported, pick_connectors,
rule_questions, schedule_unsupported, thresholds_unsupported)
RECORD_VERBS = {None, 'LOG', 'ALERT', 'ENABLE', 'CREATE'}
def build(req: Request, rd: RuleDecision, ctx: PlatformContext, index: int, answers: dict, kind: str) -> Build:
d = Draft(kind, None, rd)
b = Build(d, consumed_exceptions=list(req.consumed))
body = base_body(kind)
cfg = body['channels'][kind]
c = req.content
label = TYPE_LABEL[kind]
if req.verb not in RECORD_VERBS and answer_for(answers, 'action', index) != 'log':
d.issue('ACTION_NOT_AVAILABLE', 'blocking',
f'{label} policies scan and record what they find; they cannot block, delete or quarantine.',
requested=(req.verb or '').lower(), type=kind, available=['log'])
d.ask(Question('action', f'{label} can only scan and record findings. Continue with that?',
[{'value': 'log', 'label': 'Scan and record'}, {'value': 'cancel', 'label': 'Cancel'}],
True, 'log'))
cfg['connectorIds'] = pick_connectors(d, req, ctx, kind, index, answers)
specific = rd.mode != 'none' and not rd.assumed and 'discovery' not in c.mods
if specific:
cfg['scanMode'] = 'rules'
cfg['detectionRuleIds'] = list(rd.rule_ids)
else:
cfg['scanMode'] = 'discovery'
if rd.mode != 'none':
d.issue('DISCOVERY_FOR_GENERIC_DATA', 'info', 'For "any sensitive data" the scan uses broad discovery, '
'which flags every kind of PII, PHI and card data it finds.')
rd.mode, rd.rule_ids, rd.assumed = 'none', [], False
if 'auto_sync' in c.mods or c.interval_minutes:
cfg['autoSync']['enabled'] = True
if c.interval_minutes:
lo = 15 if kind == 'github' else 1
if c.interval_minutes < lo:
d.issue('INTERVAL_RAISED', 'info', f'{label} re-scans at most every {lo} minutes.')
cfg['autoSync']['intervalMinutes'] = max(lo, int(c.interval_minutes))
handled: set[str] = {'sizes'}
sizes = list(c.sizes)
for k, e in enumerate(req.exceptions):
if e.sizes and k not in b.consumed_exceptions:
sizes += e.sizes
b.consumed_exceptions.append(k)
for s in sizes:
n = size_bytes(s)
if n:
cfg['maxFileBytes'] = n
d.issue('SCAN_SIZE_LIMIT', 'info', f'Files larger than {s.upper()} are listed but not read.')
break
count_key = 'maxFiles' if kind == 'github' else 'maxObjects'
for n, noun in c.thresholds:
if noun in ('files', 'objects', 'items') and not (kind == 'github' and noun == 'items' and 'clone' in c.mods):
cfg[count_key] = max(1, int(n))
handled |= {'files', 'objects'}
d.issue('SCAN_COUNT_LIMIT', 'info', f'Each scan reads at most {n} files.')
break
if domains_of(c):
d.issue('DOMAINS_NOT_USED', 'warning', f'{", ".join(domains_of(c))} has no meaning in a scan policy and was '
'not used.')
if kind == 'github':
repos = [n for n, noun in c.thresholds if noun in ('repos', 'times', 'items')]
if repos or 'clone' in c.mods:
if repos:
cfg['bulkRepoThreshold'] = max(1, int(repos[0]))
handled |= {'repos', 'times', 'items'}
d.issue('GITHUB_CLONE_ALERT', 'info',
f'Raises an incident when one person clones or downloads {cfg["bulkRepoThreshold"]} or more '
'repositories in an hour. This needs the GitHub Enterprise Cloud audit log.')
extra = list(c.paths)
for k, e in enumerate(req.exceptions):
# "exclude node_modules and test folders": paths the scan should never read
if k not in b.consumed_exceptions and (e.paths or e.words) and e.data.empty() and not e.groups:
extra += e.paths + [w if ('.' in w or '/' in w) else f'{w}/' for w in e.words]
b.consumed_exceptions.append(k)
for p in extra:
if p not in cfg['excludePaths']:
cfg['excludePaths'].append(p)
if extra:
d.issue('GITHUB_EXCLUDES', 'info', f'The scan will skip {", ".join(extra)}.')
d.issue('SCAN_IS_MANUAL', 'info', f'{label} scans are started from the {label} dashboard; this policy decides '
'what they look for.')
channel_group_check(d, req, ctx, index, answers)
thresholds_unsupported(d, req, index, answers, handled=handled)
schedule_unsupported(d, req, index, answers)
qualifiers_unsupported(d, req, index, answers, handled=({'bulk'} if kind == 'github' else set()) | {'sizes'})
exceptions_unsupported(d, req, b.consumed_exceptions, index, answers, ctx)
rule_questions(d, rd, ctx)
d.spans = [{'start': s, 'end': e, 'label': l} for s, e, l in req.spans]
d.policyBody = body
return b