diff --git a/.gitignore b/.gitignore index 753b3075596232b8a31ba9930b1389756ea14e8a..0b0b2500562aad47c5f156ffb0633cc1807ff524 100644 --- a/.gitignore +++ b/.gitignore @@ -8,3 +8,9 @@ __pycache__/ # Uploaded packs, produced result files and reference-data files supplied by hand. storage/ + +# Client material: real declarations, invoices and email threads. The parcel +# manifests carry consignee names, home addresses, telephone numbers and email +# addresses, so none of it belongs in the repository. +Attachments-*/ +.DS_Store diff --git a/README.md b/README.md index 4f3203b617c5f6f1ca0859887d07cb81ba18b8ff..0c67d0a1a8bb195da8541c3cbbdf43b3413e5657 100644 --- a/README.md +++ b/README.md @@ -24,7 +24,8 @@ Storage is MongoDB, not Postgres. No queues, no vector database. | `cbh/api/linecheck` | The engine. Code resolution, document determination, warnings. The single entry point. | | `cbh/api/importer` | Loads the reference files. Idempotent, restartable, logged in `synclogs`. | | `cbh/api/client` | Per-client column mapping and export profile. | -| `cbh/api/batch` | The Excel processor: a thin client over the line-check API. | +| `cbh/api/batch` | Shipments: both input routes, the review table and the exports. | +| `cbh/api/rules` | The broker's own logic, written down once and replayed. | | `cbh/api/common` | Shared DTOs, schemas, generic Mongo access and the TTL cache. | Each module follows the same layout: `__init__.py` (router) · `dto.py` (enums) · @@ -86,24 +87,94 @@ Two rules the code holds to: reported as a warning, because "unregulated" and "the data does not cover this" look identical from the inside. -## Processing a pack +## Two ways in, one review table, one export + +Both input routes build the same shipment, and the review table is read the +same way whichever route filled it. + +**A — the form, one line at a time.** + +``` +POST /api/batch/manual # open an empty shipment +POST /api/batch/{id}/rows # add one line; it is checked on arrival +DELETE /api/batch/{id}/rows/{rowIndex} +``` + +**B — upload, the daily route.** ``` POST /api/client/suggest-mapping # guess the column mapping from a file, for confirmation POST /api/batch/upload # upload + process (clientProfileId, file) -GET /api/batch/{id}/download # the same file back, with the added columns -GET /api/batch/{id}/rows?needsReviewOnly=true ``` -The client's own file comes back: same sheet, same columns, same row order, -with `resolved_code`, `documents_required`, `documents_exemption`, `warnings` -and `status` appended in the order the export profile fixes, a totals row at the -bottom (packages, gross weight, value) and `ambiguous`/`invalid` rows -highlighted. +**The table, either way.** + +``` +GET /api/batch/{id}/review?needsReviewOnly=true&confidence=… +PATCH /api/batch/{id}/rows/{rowIndex} # accept, correct, CBAM/EUDR/licence, note +POST /api/batch/{id}/rows/{rowIndex}/make-rule +POST /api/batch/{id}/recheck # replay every row against today's rules and data +``` + +`GET /review` returns the header the screen is built from — reference, route, +countries, the totals (packages, weight, value) and the two scores, extraction +and document-codes, reported separately so neither can hide the other — plus +one row per line carrying its confidence, its reason, its document codes with +source and validity, its warnings and the rules that fired. Identical (code, country, direction, date) tuples are resolved once and reused across the pack — a 580-line pack typically collapses to a few dozen lookups. +## Export, per tenant + +``` +GET /api/batch/export-formats?clientProfileId=… # fills the dropdown +GET /api/batch/{id}/export?format=… +``` + +| Format | What it is | +| --- | --- | +| `review` | The tool's own table: every column the reviewer saw, reasons included. | +| `source` | The uploaded file returned unchanged, with the checked columns appended. Upload route only. | +| `softpak` | The Goenee line list, rebuilt from the stored rows. | +| `scope`, `fiton` | Declared, listed, and refused with 501. | + +`scope` and `fiton` are not implemented because their import specifications +have not been received. They are listed as unavailable with the reason rather +than quietly approximated, so the gap shows up in the UI instead of in a +rejected declaration. Adding one is a profile change, not a code change. + +Totals are counted per physical package, not per row. A parcel manifest +carries one row per item and repeats the parcel's weight on each of them; +`TotalsConfig.parcelKeyColumn` names the column that identifies the parcel and +`parcelLevelFields` the fields to count once per parcel. For the G-Star +manifest of 04-09-2026 this is what reconciles the header with the Master +Invoice: 167 packages and GBP 11,376.42 across 252 rows. + +## Rules — the broker's own logic + +``` +GET|POST /api/rules PUT|DELETE /api/rules/{id} +POST /api/batch/{id}/rows/{rowIndex}/make-rule +``` + +A rule matches narrowly — any combination of code prefix, origin, destination, +direction and a description substring — and does one thing: pin a code, add +document codes, suppress document codes, or record a note. A rule with no +conditions is refused, because it would touch every line in every pack. + +Two guarantees hold whatever a rule does: + +- **It is never silent.** Every rule that fires is named on the line in + `appliedRules` and as an INFO warning saying what it changed. +- **It is never green.** A rule is the broker's logic, not TARIC's, so the + confidence it touched drops to amber and the source becomes `MANUAL`. + +Pinning a code re-runs the check under the pinned code, so the documents belong +to the code actually being declared rather than to the one the reference +proposed. Rules are scoped to one client profile or to the tenant, and each +counts its own hits — which is the input the learning layer will need. + ## Acceptance metrics ``` @@ -150,5 +221,12 @@ the first real files: document-code prefix (C/N/X = obligation, Y = exemption) instead. Verify against the published Measure-action list. - **The Softpak column order** — `default_export_columns()` in - `client/models.py`, pending Judith's template specification. -- **The Goenee column mapping** — `goenee_profile()` in `client/models.py`. + `client/models.py`, pending Judith's template specification. The `softpak` + export currently reproduces the layout of the packs Goenee actually sends + (`PF 04-09-2026.xlsx`), which is evidence, not a specification. +- **The Goenee column mapping** — `goenee_softpak_profile()` in + `client/models.py`. +- **The G-Star gross weight.** Counting the manifest once per parcel gives + 175.86 kg; the Master Invoice for the same shipment says 148.69 kg. Packages + and value reconcile exactly, weight does not. Ask which basis the declared + weight uses before trusting that figure. diff --git a/cargoflow-codecontrole.html b/cargoflow-codecontrole.html new file mode 100644 index 0000000000000000000000000000000000000000..566b1158f14e6ee086c4c270a55fbf994f29c1b2 --- /dev/null +++ b/cargoflow-codecontrole.html @@ -0,0 +1,23 @@ + +CargoFlow — Minder zoeken. Sneller klaar.
Interactieve demo

Minder zoeken. Sneller klaar.

Van product naar een controleerbaar codevoorstel.

1. Product2. Aanvullen3. Resultaat
+
Context aanpassen

Alleen aanvullende vragen waar nodig.

+ + +
+
+ diff --git a/cbh/__init__.py b/cbh/__init__.py index 422a906122523d0ce8c1fcb59f9b0b6a961b141e..ad3580c2ce859dbb0ce08eabb98eecc77f9ebb4a 100644 --- a/cbh/__init__.py +++ b/cbh/__init__.py @@ -1,8 +1,3 @@ -# pylint: disable=C0415 -""" -CargoFlow application package. -""" - from fastapi import FastAPI from fastapi.middleware.cors import CORSMiddleware from starlette.exceptions import HTTPException as StarletteHTTPException @@ -11,14 +6,6 @@ from cbh.core.wrappers import CbhResponseWrapper, ErrorCbhResponse def register_routers(app: FastAPI) -> None: - """ - Attach every module's router to the application. - - Each module defines its router in its ``__init__`` and its endpoints in - ``views``; importing the view module is what binds the two together. - """ - from cbh.api.account import account_router - from cbh.api.account import views as account_views # noqa: F401 from cbh.api.batch import batch_router from cbh.api.batch import views as batch_views # noqa: F401 from cbh.api.client import client_router @@ -27,24 +14,23 @@ def register_routers(app: FastAPI) -> None: from cbh.api.importer import views as importer_views # noqa: F401 from cbh.api.linecheck import linecheck_router from cbh.api.linecheck import views as linecheck_views # noqa: F401 + from cbh.api.rules import rules_router + from cbh.api.rules import views as rules_views # noqa: F401 from cbh.api.taric import taric_router from cbh.api.taric import views as taric_views # noqa: F401 for router in ( - account_router, taric_router, linecheck_router, importer_router, client_router, batch_router, + rules_router, ): app.include_router(router) def create_app() -> FastAPI: - """ - Create and configure the FastAPI application. - """ app = FastAPI( title="CargoFlow Regelcheck", description=( @@ -70,18 +56,12 @@ def create_app() -> FastAPI: @app.exception_handler(StarletteHTTPException) async def http_exception_handler(_, exc): - """ - Handle HTTP exceptions and convert them to standardized error responses. - """ return CbhResponseWrapper( data=None, successful=False, error=ErrorCbhResponse(message=str(exc.detail)) ).response(exc.status_code) @app.on_event("startup") async def startup_event(): - """ - Execute startup tasks when the application starts. - """ from cbh.api.client.db_requests import ensure_default_profiles from cbh.api.common.db_requests import ensure_indexes from cbh.core.config import settings @@ -96,9 +76,6 @@ def create_app() -> FastAPI: @app.on_event("shutdown") async def shutdown_event(): - """ - Stop background jobs when the application stops. - """ from cbh.core.config import settings if settings.SCHEDULER.running: @@ -106,16 +83,10 @@ def create_app() -> FastAPI: @app.get("/api/health") async def health(): - """ - Health check endpoint for container orchestration and monitoring. - """ return {"status": "healthy"} @app.get("/") async def root(): - """ - Root endpoint for the application. - """ return {"service": "CargoFlow Regelcheck", "version": "0.3.0"} return app diff --git a/cbh/api/account/__init__.py b/cbh/api/account/__init__.py deleted file mode 100644 index 840c6a26ef8561f60de66c6d291dd14a1a9b139a..0000000000000000000000000000000000000000 --- a/cbh/api/account/__init__.py +++ /dev/null @@ -1,12 +0,0 @@ -""" -Account module initialization. - -This module defines the FastAPI router for account API endpoints -and imports related views for account management. -""" - -from fastapi import APIRouter - -account_router = APIRouter( - prefix="/api/account", -) diff --git a/cbh/api/account/dto.py b/cbh/api/account/dto.py deleted file mode 100644 index e55ee2362751c470c1889a7f29f5b416d6d41a71..0000000000000000000000000000000000000000 --- a/cbh/api/account/dto.py +++ /dev/null @@ -1,38 +0,0 @@ -""" -Account DTOs. -""" - -from enum import Enum - - -class AccountType(Enum): - """ - Enum for account types. - """ - - USER = 1 - ADMIN = 2 - OWNER = 3 - SUPER_ADMIN = 4 - - -class AccountStatus(Enum): - """ - Enum for account statuses. - """ - - ACTIVE = 1 - BLOCKED = 2 - PENDING_INVITATION = 3 - PENDING_COMPLETION = 4 - - -class RegistrationType(Enum): - """ - Enum for registration types. - """ - - ORGANIC = 1 - GOOGLE = 2 - GITHUB = 3 - APPLE = 4 diff --git a/cbh/api/account/models.py b/cbh/api/account/models.py deleted file mode 100644 index 1d9b0735d21cbe7f88fd0e36aaf7a6897edfb159..0000000000000000000000000000000000000000 --- a/cbh/api/account/models.py +++ /dev/null @@ -1,124 +0,0 @@ -""" -Account models. -""" - -from datetime import date, datetime - -from passlib.context import CryptContext -from pydantic import Field, field_validator - -from cbh.api.account.dto import ( - AccountStatus, - AccountType, - RegistrationType -) -from cbh.core.config import settings -from cbh.core.database import MongoBaseModel, MongoBaseShortenModel - - -class OrganizationShorten(MongoBaseShortenModel): - """ - Organization shorten model. - """ - - name: str - slug: str | None = None - pictureUrl: str | None = None - - @field_validator("pictureUrl", mode="before") - @classmethod - def serialize_picture_url(cls, v: str | None) -> str | None: - """ - Serialize the scenario picture URL. - """ - if v: - if not v.startswith("https://"): - return settings.S3_CLIENT.generate_presigned_url(v, expiration=3600) - return v - return None - - -class AccountModel(MongoBaseModel): - """ - Account model class. - """ - - name: str | None = None - email: str - pictureUrl: str | None = None - password: str | None = Field(exclude=True, default=None) - - status: AccountStatus = AccountStatus.ACTIVE - accountType: AccountType = Field(default=AccountType.USER) - registrationType: RegistrationType | None = Field( - default=RegistrationType.ORGANIC, exclude=True - ) - - organization: OrganizationShorten | None = None - freeMinutesUsed: int = Field(default=0, exclude=True) - - lastLogin: date | None = Field(exclude=True, default=None) - datetimeInserted: datetime = Field(default_factory=datetime.now) - datetimeUpdated: datetime = Field(default_factory=datetime.now) - - @field_validator("password", mode="before", check_fields=False) - @classmethod - def set_password_hash(cls, v: str | None) -> str | None: - """ - Set the password hash. - - Args: - v (str): The password to hash. - - Returns: - str: The hashed password. - """ - if isinstance(v, str) and not v.startswith("$2b$"): - return CryptContext(schemes=["bcrypt"], deprecated="auto").hash(v) - return v - - @field_validator("pictureUrl", mode="before") - @classmethod - def serialize_picture_url(cls, v: str | None) -> str | None: - """ """ - if v: - if not v.startswith("https://"): - return settings.S3_CLIENT.generate_presigned_url(v, expiration=3600) - return v - return None - - class Config: # pylint: disable=R0903 - """ - Config for the AccountModel class. - """ - - arbitrary_types_allowed = True - populate_by_name = True - json_encoders = {datetime: lambda dt: dt.isoformat()} - - -class AccountShorten(MongoBaseShortenModel): - """ - Account shorten model. - """ - - id: str - name: str | None = None - email: str - pictureUrl: str | None = None - - accountType: AccountType - - organization: OrganizationShorten | None = None - - @field_validator("pictureUrl", mode="before") - @classmethod - def serialize_picture_url(cls, v: str | None) -> str | None: - """ - Serialize the account picture URL - """ - if v: - if not v.startswith("https://"): - return settings.S3_CLIENT.generate_presigned_url(v, expiration=3600) - return v - return None diff --git a/cbh/api/account/schemas.py b/cbh/api/account/schemas.py deleted file mode 100644 index 51cfaacb4bcfb9d344181e6362628a5e792ec6a6..0000000000000000000000000000000000000000 --- a/cbh/api/account/schemas.py +++ /dev/null @@ -1,11 +0,0 @@ -from pydantic import BaseModel - - -class AccountFilter(BaseModel): - """ - Account filter. - """ - - accountTypes: list[int] | None = None - statuses: list[int] | None = None - searchTerm: str | None = None diff --git a/cbh/api/account/utils.py b/cbh/api/account/utils.py deleted file mode 100644 index ccf4a4510064c82d1260a95b5e9da8b4b895fca4..0000000000000000000000000000000000000000 --- a/cbh/api/account/utils.py +++ /dev/null @@ -1,36 +0,0 @@ -from io import BytesIO - -from PIL import Image, ImageOps, UnidentifiedImageError -from fastapi import HTTPException, UploadFile - -from cbh.api.account.models import AccountModel - - -def compress_image(file: UploadFile) -> bytes: - if not file.content_type or not file.content_type.startswith("image/"): - raise HTTPException(status_code=400, detail="File must be an image") - try: - file.file.seek(0) - original_bytes = file.file.read() - image = Image.open(BytesIO(original_bytes)) - image = ImageOps.exif_transpose(image) - max_size = 512 - image.thumbnail((max_size, max_size), Image.Resampling.LANCZOS) - if image.mode not in ("RGB", "RGBA"): - image = image.convert("RGBA") - buffer = BytesIO() - image.save(buffer, format="PNG", optimize=True, compress_level=9) - return buffer.getvalue() - except (UnidentifiedImageError, OSError): - raise HTTPException(status_code=400, detail="Invalid image file") - finally: - try: - file.file.close() - except Exception: - pass - - -def prepare_additional_filter(account: AccountModel) -> dict: - return { - "organization.id": account.organization.id, - } diff --git a/cbh/api/account/views.py b/cbh/api/account/views.py deleted file mode 100644 index c6a9edd964306c8a5184dd1da948e816371bf546..0000000000000000000000000000000000000000 --- a/cbh/api/account/views.py +++ /dev/null @@ -1,44 +0,0 @@ -""" -Account views module. -""" - -from fastapi import Depends - -from cbh.api.account import account_router -from cbh.api.account.dto import AccountType -from cbh.api.account.models import AccountModel -from cbh.api.common.db_requests import delete_obj -from cbh.core.security import PermissionDependency -from cbh.core.wrappers import CbhResponseWrapper - - -@account_router.delete("/{accountId}") -async def delete_account( - accountId: str, - account: AccountModel = Depends( - PermissionDependency( - [AccountType.OWNER], - ) - ), -) -> CbhResponseWrapper[AccountModel]: - """ - Delete an account. - """ - removed_account = await delete_obj( - AccountModel, - accountId, - additional_filter={ - "organization.id": account.organization.id, - }, - ) - return CbhResponseWrapper() - - -@account_router.get("") -async def get_own_account( - account: AccountModel = Depends(PermissionDependency()), -) -> CbhResponseWrapper[AccountModel]: - """ - Get own account. - """ - return CbhResponseWrapper(data=account) diff --git a/cbh/api/batch/__init__.py b/cbh/api/batch/__init__.py index b66aa8b0f4264e307daf59dd4071f3c416cfd7d5..9eeac89712acb34456c0accd019962923361ef97 100644 --- a/cbh/api/batch/__init__.py +++ b/cbh/api/batch/__init__.py @@ -1,11 +1,3 @@ -""" -Batch module initialization. - -This module defines the FastAPI router for the Excel processor: uploading a -client's pack, running it through the line-check API and handing back the same -file with the checker's columns added. -""" - from fastapi import APIRouter batch_router = APIRouter( diff --git a/cbh/api/batch/db_requests.py b/cbh/api/batch/db_requests.py index 74d730a7b4aeab80e194efd2f046273e49b60efb..904d824347c41e43d91b604a2fb78e082bd89b8e 100644 --- a/cbh/api/batch/db_requests.py +++ b/cbh/api/batch/db_requests.py @@ -1,31 +1,20 @@ -""" -Batch database requests. -""" - from cbh.api.batch.models import BatchModel, BatchRowModel from cbh.api.common.db_requests import find_objs, get_all_objs, get_obj_by_id async def get_batch(batch_id: str) -> BatchModel: - """ - Fetch a pack, 404 when it does not exist. - """ return await get_obj_by_id(BatchModel, batch_id) async def get_batches( page_size: int, page_index: int, status: int | None = None ) -> tuple[list[BatchModel], int]: - """ - Page through the packs, newest first. - """ - filter_ = {"status": status} if status is not None else {} return await get_all_objs( BatchModel, page_size=page_size, page_index=page_index, sort=("datetimeInserted", -1), - additional_filter=filter_, + additional_filter={"status": status} if status is not None else {}, ) @@ -35,12 +24,6 @@ async def get_batch_rows( page_index: int = 0, needs_review_only: bool = False, ) -> tuple[list[BatchRowModel], int]: - """ - Read the rows of a pack in spreadsheet order. - - ``needs_review_only`` is the working view for a broker: the lines the - machine refused to answer for. - """ filter_: dict = {"batchId": batch_id} if needs_review_only: filter_["needsReview"] = True diff --git a/cbh/api/batch/dto.py b/cbh/api/batch/dto.py index a0793aa7fec412133f1a5961d6f88d2ffcd42f44..1e77b1c37f80998719b1c54ce5bd597b26982073 100644 --- a/cbh/api/batch/dto.py +++ b/cbh/api/batch/dto.py @@ -1,18 +1,7 @@ -""" -Batch (Excel processing) DTOs. -""" +from enum import IntEnum -from enum import Enum - - -class BatchStatus(Enum): - """ - Lifecycle of an uploaded pack. - - ``MAPPING_REQUIRED`` means the file was read but the profile does not map - everything needed — the pack waits rather than being processed on guesses. - """ +class BatchStatus(IntEnum): UPLOADED = 1 MAPPING_REQUIRED = 2 PROCESSING = 3 @@ -20,30 +9,48 @@ class BatchStatus(Enum): FAILED = 5 -class FileFormat(Enum): - """ - Format a pack arrived in. - """ - +class FileFormat(IntEnum): XLSX = 1 CSV = 2 @classmethod def from_name(cls, file_name: str) -> "FileFormat": - """ - Derive the format from a file name. - """ return cls.CSV if file_name.lower().endswith((".csv", ".txt")) else cls.XLSX -class ReviewStage(Enum): +class ReviewStage(IntEnum): + NOT_STARTED = 1 + IN_REVIEW = 2 + REVIEWED = 3 + + +class BatchSource(IntEnum): + """How the lines got into the shipment. + + Both routes end in the same review table; this only records which one was + taken, so a manually built shipment is not mistaken for a parsed file. """ - Where a pack stands in human review. - Review time is the metric that decides whether the product is worth having, - so it is measured rather than estimated. + UPLOAD = 1 + MANUAL = 2 + + +class ExportFormat(IntEnum): + """The client system the reviewed table is exported for. + + REVIEW is the tool's own full table. SOURCE returns the client's uploaded + file with the checked columns appended, which is what the upload route has + always produced. SOFTPAK is the Goenee line list. SCOPE and FITON are + declared but not implemented: their import specifications have not been + received, and guessing at a layout is worse than refusing. """ - NOT_STARTED = 1 - IN_REVIEW = 2 - REVIEWED = 3 + REVIEW = 1 + SOURCE = 2 + SOFTPAK = 3 + SCOPE = 4 + FITON = 5 + + @property + def implemented(self) -> bool: + return self in (ExportFormat.REVIEW, ExportFormat.SOURCE, ExportFormat.SOFTPAK) diff --git a/cbh/api/batch/models.py b/cbh/api/batch/models.py index 032716a83e0af350e60dff9865ecbf4fa3ad6537..d7afa7387382dcd5e6b279733d1587c8e2379ab1 100644 --- a/cbh/api/batch/models.py +++ b/cbh/api/batch/models.py @@ -1,67 +1,58 @@ -""" -Batch models. -""" - from datetime import date, datetime from pydantic import Field -from cbh.api.batch.dto import BatchStatus, FileFormat, ReviewStage -from cbh.api.common.dto import TradeDirection +from cbh.api.batch.dto import BatchSource, BatchStatus, FileFormat, ReviewStage +from cbh.api.common.dto import CustomsProcedure, TradeDirection from cbh.api.common.models import FileRef -from cbh.core.database import MongoBaseModel, MongoBaseShortenModel, MongoSerializableModel +from cbh.api.linecheck.dto import RowDecision +from cbh.core.database import MongoBaseModel, MongoSerializableModel class ClientProfileRef(MongoSerializableModel): - """ - The profile a pack was processed under. - - Stored by value as well as by id: a pack processed last month was processed - under the mapping as it stood then, and re-reading it must not silently - adopt a mapping edited since. - """ - id: str name: str class BatchMetrics(MongoSerializableModel): - """ - The numbers the acceptance criteria are stated in. - - ``reviewSeconds`` is the one that matters commercially: it is the time a - human spent on the pack after the machine had finished. - """ - totalLines: int = 0 uniqueLookups: int = 0 machineSeconds: float = 0.0 reviewSeconds: float | None = None statusCounts: dict[str, int] = Field(default_factory=dict) + extractionScore: float = 0.0 + documentScore: float = 0.0 + codeConfidenceCounts: dict[str, int] = Field(default_factory=dict) + documentConfidenceCounts: dict[str, int] = Field(default_factory=dict) + + totalPackages: float = 0.0 + totalGrossWeight: float = 0.0 + totalNetWeight: float = 0.0 + totalValue: float = 0.0 + @property def lines_needing_review(self) -> int: - """ - Lines a broker has to look at before filing. - """ return self.statusCounts.get("AMBIGUOUS", 0) + self.statusCounts.get("INVALID", 0) class BatchModel(MongoBaseModel): - """ - One uploaded pack of declaration lines. - """ - name: str clientProfile: ClientProfileRef status: BatchStatus = BatchStatus.UPLOADED + source: BatchSource = BatchSource.UPLOAD fileFormat: FileFormat = FileFormat.XLSX - sourceFile: FileRef + # A shipment built line by line through the form has no source file. + sourceFile: FileRef | None = None resultFile: FileRef | None = None checkDate: date | None = None defaultDirection: TradeDirection = TradeDirection.IMPORT + defaultProcedure: CustomsProcedure = CustomsProcedure.RELEASE_FOR_FREE_CIRCULATION + destinationCountry: str | None = None + shipmentReference: str | None = None + currency: str = "EUR" metrics: BatchMetrics = Field(default_factory=BatchMetrics) errors: list[str] = Field(default_factory=list) @@ -76,24 +67,7 @@ class BatchModel(MongoBaseModel): datetimeUpdated: datetime = Field(default_factory=datetime.now) -class BatchShorten(MongoBaseShortenModel): - """ - Batch projection for list endpoints. - """ - - name: str - status: BatchStatus - datetimeInserted: datetime - - class BatchRowModel(MongoBaseModel): - """ - One row of a pack, with the answer it got. - - The raw input is kept alongside the result so a disputed line can be - replayed exactly as it was submitted. - """ - batchId: str rowIndex: int @@ -104,4 +78,12 @@ class BatchRowModel(MongoBaseModel): status: str = "INVALID" needsReview: bool = False + decision: RowDecision = RowDecision.PENDING + correctedCode: str | None = None + cbamPermitNumber: str | None = None + eudrReference: str | None = None + licenceNumbers: list[str] = Field(default_factory=list) + reviewNote: str | None = None + datetimeReviewed: datetime | None = None + datetimeInserted: datetime = Field(default_factory=datetime.now) diff --git a/cbh/api/batch/schemas.py b/cbh/api/batch/schemas.py index c28d154dfe724fbb95b89b2d265b96ecb1493e3c..aeb2413ae75fd3af7312e482341a5e7ef0ed5008 100644 --- a/cbh/api/batch/schemas.py +++ b/cbh/api/batch/schemas.py @@ -1,32 +1,36 @@ -""" -Batch schemas. -""" - from datetime import date, datetime from typing import Any from pydantic import BaseModel -from cbh.api.batch.dto import BatchStatus, ReviewStage +from cbh.api.batch.dto import BatchSource, BatchStatus, ExportFormat, ReviewStage from cbh.api.batch.models import BatchMetrics, ClientProfileRef -from cbh.api.common.dto import TradeDirection +from cbh.api.common.dto import Paging +from cbh.api.common.dto import CustomsProcedure, TradeDirection +from cbh.api.linecheck.dto import ConfidenceLevel, ReferenceSource, RowDecision +from cbh.api.linecheck.schemas import ( + AppliedRule, + DeclarationDate, + DocumentRequirement, + LineWarning, +) class BatchResponse(BaseModel): - """ - A pack as returned by the API. - """ - id: str name: str clientProfile: ClientProfileRef status: BatchStatus + source: BatchSource = BatchSource.UPLOAD - sourceFileName: str + sourceFileName: str | None = None resultFileName: str | None = None checkDate: date | None = None defaultDirection: TradeDirection + destinationCountry: str | None = None + shipmentReference: str | None = None + currency: str = "EUR" metrics: BatchMetrics errors: list[str] = [] @@ -40,10 +44,6 @@ class BatchResponse(BaseModel): class BatchRowResponse(BaseModel): - """ - One row of a pack, with the answer it received. - """ - rowIndex: int status: str needsReview: bool @@ -52,13 +52,6 @@ class BatchRowResponse(BaseModel): class GoldSetScoreResponse(BaseModel): - """ - Accuracy of a pack against a confirmed gold file, plus its timings. - - Review time is the metric that sells the product, so it is reported - alongside accuracy rather than separately. - """ - goldRows: int matchedRows: int unmatchedGoldRows: list[int] = [] @@ -75,3 +68,150 @@ class GoldSetScoreResponse(BaseModel): reviewSeconds: float | None = None totalLines: int | None = None linesNeedingReview: int | None = None + + +class ReviewDocuments(BaseModel): + required: list[DocumentRequirement] = [] + alternatives: list[list[DocumentRequirement]] = [] + exemption: list[DocumentRequirement] = [] + suspended: list[DocumentRequirement] = [] + noneRequired: bool = False + label: str + + +class ReviewRow(BaseModel): + rowIndex: int + description: str | None = None + + inputCode: str | None = None + proposedCode: str | None = None + formattedCode: str | None = None + codeDescription: str | None = None + candidates: list[str] = [] + replacementCodes: list[str] = [] + + originCountry: str | None = None + destinationCountry: str | None = None + measureCountry: str | None = None + direction: TradeDirection | None = None + procedure: CustomsProcedure | None = None + + codeConfidence: ConfidenceLevel + codeConfidenceReason: str = "" + documentConfidence: ConfidenceLevel + documentConfidenceReason: str = "" + source: ReferenceSource = ReferenceSource.NONE + + documents: ReviewDocuments + warnings: list[LineWarning] = [] + appliedRules: list[AppliedRule] = [] + + packages: float | None = None + grossWeight: float | None = None + netWeight: float | None = None + value: float | None = None + + decision: RowDecision = RowDecision.PENDING + correctedCode: str | None = None + cbamPermitNumber: str | None = None + eudrReference: str | None = None + licenceNumbers: list[str] = [] + reviewNote: str | None = None + + status: str + needsReview: bool + + +class ShipmentTotals(BaseModel): + packages: float = 0.0 + grossWeight: float = 0.0 + netWeight: float = 0.0 + value: float = 0.0 + currency: str = "EUR" + + +class ReviewHeader(BaseModel): + batchId: str + shipmentReference: str | None = None + name: str + clientProfile: ClientProfileRef + status: BatchStatus + source: BatchSource = BatchSource.UPLOAD + direction: TradeDirection + procedure: CustomsProcedure = CustomsProcedure.RELEASE_FOR_FREE_CIRCULATION + originCountries: list[str] = [] + destinationCountry: str | None = None + checkDate: date | None = None + referenceAsOf: str | None = None + + totals: ShipmentTotals + totalLines: int = 0 + extractionScore: float = 0.0 + documentScore: float = 0.0 + codeConfidenceCounts: dict[str, int] = {} + documentConfidenceCounts: dict[str, int] = {} + linesNeedingReview: int = 0 + + +class ReviewTableResponse(BaseModel): + header: ReviewHeader + paging: Paging + rows: list[ReviewRow] + + +class RowReviewRequest(BaseModel): + correctedCode: str | None = None + decision: RowDecision | None = None + cbamPermitNumber: str | None = None + eudrReference: str | None = None + licenceNumbers: list[str] | None = None + reviewNote: str | None = None + + +class ManualShipmentRequest(BaseModel): + """Open an empty shipment that the form will fill line by line.""" + + clientProfileId: str + name: str | None = None + checkDate: date | None = None + direction: TradeDirection | None = None + procedure: CustomsProcedure | None = None + destinationCountry: str | None = None + shipmentReference: str | None = None + currency: str = "EUR" + + +class ManualLineRequest(BaseModel): + """One line from the single-item form, added to the review table. + + These are the fields the UI specification lists for the form, plus the + quantities the totals row needs. A partial code is allowed: it resolves, + or it comes back ambiguous with its candidates, exactly as an uploaded + line does. + """ + + code: str + originCountry: str | None = None + destinationCountry: str | None = None + direction: TradeDirection | None = None + procedure: CustomsProcedure | None = None + date: DeclarationDate | None = None + additionalCodes: list[str] = [] + + description: str | None = None + quantity: float | None = None + packages: float | None = None + grossWeight: float | None = None + netWeight: float | None = None + value: float | None = None + + +class ExportFormatResponse(BaseModel): + """One entry of the export dropdown.""" + + format: ExportFormat + name: str + description: str + available: bool + unavailableReason: str | None = None + isProfileDefault: bool = False diff --git a/cbh/api/batch/services/excel/__init__.py b/cbh/api/batch/services/excel/__init__.py index 824cc66c3c748151ef9f720fd91617d6f9ba30fb..afdf4b3bfc73b421e3d1e68273237f2a2a810f72 100644 --- a/cbh/api/batch/services/excel/__init__.py +++ b/cbh/api/batch/services/excel/__init__.py @@ -1,12 +1,24 @@ -from .reader import ParsedRow, ParsedSheet, parse_pack, read_headers -from .writer import build_output_workbook, render_output_value, workbook_to_bytes +from .export import ( + build_export_workbook, + build_review_workbook, + build_softpak_workbook, + export_file_name, + unavailable_reason, +) +from .reader import ParsedRow, ParsedSheet, load_pack_workbook, parse_pack, read_headers +from .writer import build_output_workbook, workbook_to_bytes __all__ = [ "ParsedRow", "ParsedSheet", + "build_export_workbook", "build_output_workbook", + "build_review_workbook", + "build_softpak_workbook", + "export_file_name", + "load_pack_workbook", "parse_pack", "read_headers", - "render_output_value", + "unavailable_reason", "workbook_to_bytes", ] diff --git a/cbh/api/batch/services/excel/export.py b/cbh/api/batch/services/excel/export.py new file mode 100644 index 0000000000000000000000000000000000000000..1515f46ca71244e017374c164bd11cffe96dcbdd --- /dev/null +++ b/cbh/api/batch/services/excel/export.py @@ -0,0 +1,258 @@ +from openpyxl.styles import Alignment, Font, PatternFill +from openpyxl.utils import get_column_letter +from openpyxl.workbook import Workbook + +from cbh.api.batch.dto import ExportFormat +from cbh.api.batch.models import BatchModel, BatchRowModel +from cbh.api.batch.services.excel.writer import SOFTPAK_ORDER, render_output_value +from cbh.api.batch.services.metrics.totals import TotalsInput, input_value, sheet_totals +from cbh.api.client.dto import InputField, OutputField +from cbh.api.client.models import ClientProfileModel +from cbh.api.linecheck.dto import LineStatus, RowDecision +from cbh.api.linecheck.schemas import LineCheckRequest, LineCheckResponse +from cbh.api.linecheck.utils import render_required_documents, render_warnings + +STATUS_FILLS = { + LineStatus.INVALID: PatternFill("solid", fgColor="FFF8CBCB"), + LineStatus.AMBIGUOUS: PatternFill("solid", fgColor="FFFDE9A9"), + LineStatus.EXPIRING: PatternFill("solid", fgColor="FFE2F0D9"), +} + +REVIEW_COLUMNS: list[tuple[str, int]] = [ + ("#", 6), + ("Description", 38), + ("Input code", 14), + ("Proposed code", 14), + ("Code description", 38), + ("Origin", 8), + ("Destination", 11), + ("Code confidence", 15), + ("Code confidence reason", 40), + ("Documents required", 26), + ("Document codes exempt", 22), + ("No document required", 20), + ("Document confidence", 18), + ("Document confidence reason", 40), + ("Warnings", 46), + ("Rules applied", 26), + ("CBAM permit", 18), + ("EUDR reference", 18), + ("Licences", 18), + ("Decision", 12), + ("Review note", 30), + ("Status", 12), + ("Packages", 10), + ("Gross weight", 12), + ("Net weight", 12), + ("Value", 12), +] + + +def _unpack(row: BatchRowModel) -> tuple[LineCheckRequest | None, LineCheckResponse | None]: + request = LineCheckRequest(**row.request) if row.request else None + result = LineCheckResponse(**row.result) if row.result else None + return request, result + + +def _totals_lines(rows: list[BatchRowModel]) -> list[TotalsInput]: + return [ + TotalsInput(raw=row.inputRow, request=_unpack(row)[0]) for row in rows + ] + + +def _review_values(row: BatchRowModel) -> list: + request, result = _unpack(row) + return [ + row.rowIndex, + request.description if request else "", + result.inputCode if result else "", + row.correctedCode or (result.resolvedCode if result else "") or "", + result.codeDescription if result else "", + result.originCountry if result else "", + result.destinationCountry if result else "", + result.codeConfidence.name.lower() if result else "", + result.codeConfidenceReason if result else "", + render_required_documents(result) if result else "", + ", ".join(document.code for document in result.documentsExemption) if result else "", + "yes" if result and result.noneRequired else "", + result.documentConfidence.name.lower() if result else "", + result.documentConfidenceReason if result else "", + render_warnings(result) if result else "", + ", ".join(rule.name for rule in result.appliedRules) if result else "", + row.cbamPermitNumber or "", + row.eudrReference or "", + ", ".join(row.licenceNumbers), + RowDecision(row.decision).name.lower(), + row.reviewNote or "", + row.status.lower(), + request.packages if request else None, + request.grossWeight if request else None, + request.netWeight if request else None, + request.value if request else None, + ] + + +def _write_header(worksheet, headers: list[str], row_index: int) -> None: + for column_index, header in enumerate(headers, start=1): + cell = worksheet.cell(row=row_index, column=column_index) + cell.value = header + cell.font = Font(bold=True) + cell.fill = PatternFill("solid", fgColor="FFE8EEF7") + cell.alignment = Alignment(vertical="top") + + +def build_review_workbook( + batch: BatchModel, rows: list[BatchRowModel], highlight: bool = True +) -> Workbook: + """The tool's own table, every column the reviewer saw, one row per line.""" + workbook = Workbook() + worksheet = workbook.active + worksheet.title = "Review" + + headers = [header for header, _ in REVIEW_COLUMNS] + _write_header(worksheet, headers, 1) + for index, (_, width) in enumerate(REVIEW_COLUMNS, start=1): + worksheet.column_dimensions[get_column_letter(index)].width = width + + for offset, row in enumerate(rows, start=2): + _, result = _unpack(row) + fill = STATUS_FILLS.get(result.status) if result else None + for column_index, value in enumerate(_review_values(row), start=1): + cell = worksheet.cell(row=offset, column=column_index) + cell.value = value + cell.alignment = Alignment(vertical="top", wrap_text=True) + if fill is not None and highlight: + cell.fill = fill + + totals_row = len(rows) + 2 + worksheet.cell(row=totals_row, column=1).value = "TOTAL" + worksheet.cell(row=totals_row, column=1).font = Font(bold=True) + for offset, total in ( + (23, batch.metrics.totalPackages), + (24, batch.metrics.totalGrossWeight), + (25, batch.metrics.totalNetWeight), + (26, batch.metrics.totalValue), + ): + cell = worksheet.cell(row=totals_row, column=offset) + cell.value = total + cell.font = Font(bold=True) + + worksheet.freeze_panes = "A2" + return workbook + + +def build_softpak_workbook( + batch: BatchModel, + profile: ClientProfileModel, + rows: list[BatchRowModel], +) -> Workbook: + """The Goenee line list: the client's own columns plus the codes we added. + + Values come from the stored request rather than the source sheet, so a + shipment typed into the form exports exactly like an uploaded one. + """ + workbook = Workbook() + worksheet = workbook.active + worksheet.title = "Blad1" + + header_rows = profile.softpakHeaderRows or [ + ["Government procedure", "10", " "], + ["Previous ", "00"], + ["Invoice N935", " "], + ["Currency", batch.currency], + ["Container"], + ] + for offset, header_row in enumerate(header_rows, start=1): + for column_index, value in enumerate(header_row, start=1): + worksheet.cell(row=offset, column=column_index).value = value + + fields = [field for field in SOFTPAK_ORDER if field.value in profile.columnMapping] + if not fields: + fields = list(SOFTPAK_ORDER) + + export_columns = profile.exportColumns or [] + headers = [ + profile.columnMapping.get(field.value, field.value) for field in fields + ] + [column.header for column in export_columns] + + header_row_index = len(header_rows) + 2 + _write_header(worksheet, headers, header_row_index) + + data_row_index = header_row_index + for row in rows: + data_row_index += 1 + request, result = _unpack(row) + values = [input_value(field, row.inputRow, request, profile) for field in fields] + if request is not None and InputField.CODE in fields: + # The code column carries what is actually being declared, not what + # the client sent: that is the whole point of the check. + values[fields.index(InputField.CODE)] = ( + row.correctedCode or (result.resolvedCode if result else None) or request.code + ) + values.extend(render_output_value(column.field, result) for column in export_columns) + + fill = STATUS_FILLS.get(result.status) if result else None + for column_index, value in enumerate(values, start=1): + cell = worksheet.cell(row=data_row_index, column=column_index) + cell.value = value + if fill is not None and profile.highlightForReview: + cell.fill = fill + + for index, header in enumerate(headers, start=1): + worksheet.column_dimensions[get_column_letter(index)].width = max( + 12, min(38, len(str(header)) + 6) + ) + + if profile.totals.enabled: + totals = sheet_totals(_totals_lines(rows), profile) + if totals: + totals_row = data_row_index + 1 + label = worksheet.cell(row=totals_row, column=1) + label.value = profile.totals.label + label.font = Font(bold=True) + for canonical, total in totals.items(): + try: + column_index = fields.index(InputField(canonical)) + 1 + except ValueError: + continue + cell = worksheet.cell(row=totals_row, column=column_index) + cell.value = total + cell.font = Font(bold=True) + + return workbook + + +def build_export_workbook( + export_format: ExportFormat, + batch: BatchModel, + profile: ClientProfileModel, + rows: list[BatchRowModel], +) -> Workbook: + if export_format is ExportFormat.REVIEW: + return build_review_workbook(batch, rows, profile.highlightForReview) + if export_format is ExportFormat.SOFTPAK: + return build_softpak_workbook(batch, profile, rows) + raise ValueError(f"{export_format.name} cannot be built from stored rows.") + + +def export_file_name(batch: BatchModel, export_format: ExportFormat) -> str: + stem = "".join( + character if character.isalnum() or character in "-_" else "_" + for character in batch.name + ) or "shipment" + return f"{stem}_{export_format.name.lower()}.xlsx" + + +def unavailable_reason(export_format: ExportFormat) -> str | None: + """Why a declared format cannot be produced yet. None means it can.""" + if export_format.implemented: + return None + return ( + f"The {export_format.name.title()} import specification has not been received. " + "The layout is not guessed at: ask the client for their import template " + "and it becomes a profile change, not a code change." + ) + + +def output_field_choices() -> list[str]: + return [field.value for field in OutputField] diff --git a/cbh/api/batch/services/excel/reader.py b/cbh/api/batch/services/excel/reader.py index 3af38e3162b88d65db1895347b9796f66789a588..22302aa7d481ddb01ae6864aa758ae6d7d6c2f0a 100644 --- a/cbh/api/batch/services/excel/reader.py +++ b/cbh/api/batch/services/excel/reader.py @@ -1,11 +1,3 @@ -""" -Reading a client's pack. - -Client files are not a fixed schema: each broker sends their own layout, so the -reader works off the profile's column mapping and carries every column it does -not recognise straight through to the output. -""" - import csv import io from dataclasses import dataclass, field @@ -17,7 +9,7 @@ from openpyxl import Workbook, load_workbook from cbh.api.batch.dto import FileFormat from cbh.api.client.dto import InputField from cbh.api.client.models import ClientProfileModel -from cbh.api.common.dto import TradeDirection +from cbh.api.common.dto import CustomsProcedure, TradeDirection from cbh.api.common.utils import normalize_ws, parse_date, parse_number from cbh.api.importer.utils import normalize_header from cbh.api.linecheck.schemas import LineCheckRequest @@ -25,13 +17,6 @@ from cbh.api.linecheck.schemas import LineCheckRequest @dataclass class ParsedRow: - """ - One data row of a pack. - - ``rowIndex`` is the worksheet row number, so results can be written back - exactly where they came from and the client's row order is preserved. - """ - rowIndex: int raw: dict[str, Any] request: LineCheckRequest | None = None @@ -40,20 +25,48 @@ class ParsedRow: @dataclass class ParsedSheet: - """ - A pack read into memory. - """ - headers: list[str] rows: list[ParsedRow] headerRowIndex: int sheetName: str | None = None -def workbook_from_csv(payload: bytes) -> Workbook: - """ - Load a CSV pack into a workbook so one writer serves both formats. +def _workbook_from_legacy_xls(payload: bytes) -> Workbook: + """Read a BIFF `.xls` into the same Workbook the rest of the code expects. + + Goenee's own line lists come out of JasperReports as legacy `.xls`, which + openpyxl cannot open at all. """ + import xlrd + + book = xlrd.open_workbook(file_contents=payload) + workbook = Workbook() + workbook.remove(workbook.active) + + for sheet in book.sheets(): + worksheet = workbook.create_sheet(title=(sheet.name or "Sheet1")[:31]) + for index in range(sheet.nrows): + values = [] + for position in range(sheet.ncols): + cell = sheet.cell(index, position) + if cell.ctype == xlrd.XL_CELL_DATE: + values.append(xlrd.xldate_as_datetime(cell.value, book.datemode)) + elif cell.ctype == xlrd.XL_CELL_EMPTY: + values.append(None) + else: + values.append(cell.value) + worksheet.append(values) + + return workbook + + +def load_pack_workbook(payload: bytes, file_name: str) -> Workbook: + if file_name.lower().endswith(".xls"): + return _workbook_from_legacy_xls(payload) + + if FileFormat.from_name(file_name) is not FileFormat.CSV: + return load_workbook(io.BytesIO(payload), data_only=True) + text = payload.decode("utf-8-sig", errors="replace") try: dialect = csv.Sniffer().sniff(text[:4096], delimiters=",;\t|") @@ -67,44 +80,15 @@ def workbook_from_csv(payload: bytes) -> Workbook: return workbook -def load_pack_workbook(payload: bytes, file_name: str) -> Workbook: - """ - Load a pack, whichever of the two formats it arrived in. - """ - if FileFormat.from_name(file_name) is FileFormat.CSV: - return workbook_from_csv(payload) - return load_workbook(io.BytesIO(payload), data_only=True) - - def read_headers(payload: bytes, file_name: str, header_row: int = 1) -> list[Any]: - """ - Read the header row of a pack, for mapping suggestions. - """ - workbook = load_pack_workbook(payload, file_name) - worksheet = workbook.active + worksheet = load_pack_workbook(payload, file_name).active for index, row in enumerate(worksheet.iter_rows(values_only=True), start=1): if index == header_row: return list(row) return [] -def _header_positions(headers: list[Any], profile: ClientProfileModel) -> dict[str, int]: - """ - Map canonical fields onto column positions using the profile's mapping. - """ - normalized = {normalize_header(header): index for index, header in enumerate(headers)} - positions: dict[str, int] = {} - for canonical, client_header in profile.columnMapping.items(): - position = normalized.get(normalize_header(client_header)) - if position is not None: - positions[canonical] = position - return positions - - def _cell(row: tuple, position: int | None) -> Any: - """ - Read a cell, tolerating short rows. - """ if position is None or position >= len(row): return None return row[position] @@ -115,14 +99,10 @@ def parse_pack( file_name: str, profile: ClientProfileModel, check_date: date | None = None, + destination_country: str | None = None, + direction_default: TradeDirection | None = None, + procedure_default: CustomsProcedure | None = None, ) -> ParsedSheet: - """ - Read a pack into line-check requests. - - A row that cannot be read is kept with its errors rather than dropped: the - output file must come back with the same rows, in the same order, as the - one that went in. - """ workbook = load_pack_workbook(payload, file_name) worksheet = ( workbook[profile.sheetName] @@ -136,7 +116,22 @@ def parse_pack( return ParsedSheet(headers=[], rows=[], headerRowIndex=profile.headerRow) headers = [normalize_ws(header) or "" for header in all_rows[header_index]] - positions = _header_positions(list(all_rows[header_index]), profile) + + normalized = { + normalize_header(header): index + for index, header in enumerate(all_rows[header_index]) + } + positions: dict[str, int] = {} + for canonical, client_header in profile.columnMapping.items(): + # One client's packs are not spelled identically from month to month — + # the same column is "Cn" in one export and "Case No." in the next — so + # a mapping may list the spellings it accepts, best first. + for spelling in str(client_header).split("|"): + position = normalized.get(normalize_header(spelling)) + if position is not None: + positions[canonical] = position + break + defaults = profile.defaults rows: list[ParsedRow] = [] @@ -145,19 +140,34 @@ def parse_pack( if raw_row is None or all(cell is None or cell == "" for cell in raw_row): continue - raw = { - header: _cell(raw_row, index) - for index, header in enumerate(headers) - if header - } - parsed = ParsedRow(rowIndex=row_index, raw=raw) + parsed = ParsedRow( + rowIndex=row_index, + raw={ + header: _cell(raw_row, index) + for index, header in enumerate(headers) + if header + }, + ) code = _cell(raw_row, positions.get(InputField.CODE.value)) country = _cell(raw_row, positions.get(InputField.COUNTRY.value)) or defaults.country + destination = ( + _cell(raw_row, positions.get(InputField.DESTINATION_COUNTRY.value)) + or destination_country + or defaults.destinationCountry + ) direction = ( TradeDirection.parse(_cell(raw_row, positions.get(InputField.DIRECTION.value))) + or direction_default or defaults.direction ) + procedure = ( + CustomsProcedure.parse( + _cell(raw_row, positions.get(InputField.PROCEDURE.value)) + ) + or procedure_default + or defaults.procedure + ) line_date = parse_date(_cell(raw_row, positions.get(InputField.DATE.value))) if line_date is None: line_date = check_date or ( @@ -173,8 +183,10 @@ def parse_pack( parsed.request = LineCheckRequest( code=str(code or "").strip(), - country=str(country or "").strip(), + originCountry=str(country or "").strip() or None, + destinationCountry=str(destination or "").strip() or None, direction=direction, + procedure=procedure, date=line_date, additionalCodes=[str(additional).strip()] if additional else [], description=normalize_ws( diff --git a/cbh/api/batch/services/excel/writer.py b/cbh/api/batch/services/excel/writer.py index 0ceaf14e63f937b01605219ec285b6b96c7e2445..3417e5460b86e842deb5b52713c44fe14e589767 100644 --- a/cbh/api/batch/services/excel/writer.py +++ b/cbh/api/batch/services/excel/writer.py @@ -1,13 +1,3 @@ -""" -Writing the output pack. - -The client gets their own file back: same sheet, same columns, same row order, -with the checker's columns appended in the order their export profile fixes, -and a totals row at the bottom — the standard check Judith runs on every pack. - -Rows that need a human are highlighted. They are never corrected. -""" - import io from typing import Any @@ -16,56 +6,45 @@ from openpyxl.utils import get_column_letter from openpyxl.workbook import Workbook from cbh.api.batch.services.excel.reader import ParsedSheet, load_pack_workbook -from cbh.api.client.dto import InputField, OutputField +from cbh.api.batch.services.metrics.totals import TotalsInput, sheet_totals +from cbh.api.client.dto import ExportLayout, InputField, OutputField from cbh.api.client.models import ClientProfileModel +from cbh.api.importer.utils import normalize_header from cbh.api.linecheck.dto import LineStatus from cbh.api.linecheck.schemas import LineCheckResponse -from cbh.api.linecheck.utils import ( - render_candidates, - render_exemptions, - render_required_documents, - render_warnings, - status_label, -) - -HEADER_FILL = PatternFill("solid", fgColor="FFE8EEF7") -INVALID_FILL = PatternFill("solid", fgColor="FFF8CBCB") -AMBIGUOUS_FILL = PatternFill("solid", fgColor="FFFDE9A9") -EXPIRING_FILL = PatternFill("solid", fgColor="FFE2F0D9") +from cbh.api.linecheck.utils import render_required_documents, render_warnings -STATUS_FILLS = { - LineStatus.INVALID: INVALID_FILL, - LineStatus.AMBIGUOUS: AMBIGUOUS_FILL, - LineStatus.EXPIRING: EXPIRING_FILL, -} -MAX_CELL_LENGTH = 32000 - - -def _truncate(value: str) -> str: - """ - Keep a cell inside Excel's length limit. - """ - if len(value) <= MAX_CELL_LENGTH: - return value - return value[: MAX_CELL_LENGTH - 3] + "..." +SOFTPAK_ORDER = [ + InputField.CODE, + InputField.DESCRIPTION, + InputField.GROSS_WEIGHT, + InputField.NET_WEIGHT, + InputField.COUNTRY, + InputField.QUANTITY, + InputField.COLLI_TYPE, + InputField.PACKAGES, + InputField.REMARKS, + InputField.VALUE, +] def render_output_value(field: OutputField, result: LineCheckResponse | None) -> Any: - """ - Render one appended column for one row. - """ if result is None: return "" renderers = { OutputField.RESOLVED_CODE: lambda: result.resolvedCode or "", OutputField.DOCUMENTS_REQUIRED: lambda: render_required_documents(result), - OutputField.DOCUMENTS_EXEMPTION: lambda: render_exemptions(result), + OutputField.DOCUMENTS_EXEMPTION: lambda: "; ".join( + document.code for document in result.documentsExemption + ), OutputField.WARNINGS: lambda: render_warnings(result), - OutputField.STATUS: lambda: status_label(result.status), + OutputField.STATUS: lambda: result.status.name.lower(), OutputField.CODE_DESCRIPTION: lambda: result.codeDescription or "", - OutputField.CANDIDATES: lambda: render_candidates(result), + OutputField.CANDIDATES: lambda: ", ".join( + candidate.code for candidate in result.candidates + ), OutputField.REPLACEMENT_CODES: lambda: ", ".join(result.replacementCodes), OutputField.EXPIRING_ON: lambda: ( result.expiringOn.isoformat() if result.expiringOn else "" @@ -76,34 +55,15 @@ def render_output_value(field: OutputField, result: LineCheckResponse | None) -> for measure in result.appliedMeasures ), OutputField.REFERENCE_AS_OF: lambda: result.referenceAsOf or "", + OutputField.YCODES: lambda: ",".join( + document.code for document in result.documentsExemption + ), + OutputField.CODE_CONFIDENCE: lambda: result.codeConfidence.name.lower(), + OutputField.DOCUMENT_CONFIDENCE: lambda: result.documentConfidence.name.lower(), } - return _truncate(str(renderers[field]())) - - -def _totals( - sheet: ParsedSheet, profile: ClientProfileModel -) -> dict[str, float]: - """ - Sum the quantity columns the profile asks to total. - """ - totals: dict[str, float] = {} - for canonical in profile.totals.sumFields: - try: - field = InputField(canonical) - except ValueError: - continue - total = 0.0 - found = False - for row in sheet.rows: - if row.request is None: - continue - value = getattr(row.request, field.value, None) - if isinstance(value, (int, float)): - total += float(value) - found = True - if found: - totals[canonical] = round(total, 2) - return totals + + value = str(renderers[field]()) + return value if len(value) <= 32000 else value[:31997] + "..." def build_output_workbook( @@ -113,12 +73,9 @@ def build_output_workbook( sheet: ParsedSheet, results: dict[int, LineCheckResponse], ) -> Workbook: - """ - Append the checker's columns to the client's own workbook. + if profile.exportLayout is ExportLayout.SOFTPAK_SHEET: + return _build_softpak_workbook(profile, sheet, results) - The original file is reopened and written into rather than rebuilt, so the - client's formatting, extra columns and row order survive untouched. - """ workbook = load_pack_workbook(payload, file_name) worksheet = ( workbook[sheet.sheetName] @@ -133,12 +90,18 @@ def build_output_workbook( cell = worksheet.cell(row=header_row, column=first_new_column + offset) cell.value = column.header cell.font = Font(bold=True) - cell.fill = HEADER_FILL + cell.fill = PatternFill("solid", fgColor="FFE8EEF7") cell.alignment = Alignment(vertical="top") worksheet.column_dimensions[ get_column_letter(first_new_column + offset) ].width = 28 + status_fills = { + LineStatus.INVALID: PatternFill("solid", fgColor="FFF8CBCB"), + LineStatus.AMBIGUOUS: PatternFill("solid", fgColor="FFFDE9A9"), + LineStatus.EXPIRING: PatternFill("solid", fgColor="FFE2F0D9"), + } + last_data_row = header_row for row in sheet.rows: result = results.get(row.rowIndex) @@ -151,14 +114,10 @@ def build_output_workbook( if not profile.highlightForReview or result is None: continue - fill = STATUS_FILLS.get(result.status) + fill = status_fills.get(result.status) if fill is None: continue - # Highlight the whole row: a broker scanning the pack should see the - # lines needing attention without reading the status column. - for column_index in range( - 1, first_new_column + len(profile.exportColumns) - ): + for column_index in range(1, first_new_column + len(profile.exportColumns)): worksheet.cell(row=row.rowIndex, column=column_index).fill = fill if profile.totals.enabled: @@ -167,22 +126,86 @@ def build_output_workbook( return workbook +def _build_softpak_workbook( + profile: ClientProfileModel, + sheet: ParsedSheet, + results: dict[int, LineCheckResponse], +) -> Workbook: + workbook = Workbook() + worksheet = workbook.active + worksheet.title = "Blad1" + + for offset, header_row in enumerate(profile.softpakHeaderRows, start=1): + for column_index, value in enumerate(header_row, start=1): + worksheet.cell(row=offset, column=column_index).value = value + + source_headers = [ + profile.columnMapping[field.value] + for field in SOFTPAK_ORDER + if field.value in profile.columnMapping + ] + headers = source_headers + [column.header for column in profile.exportColumns] + + header_row_index = max(profile.headerRow, len(profile.softpakHeaderRows) + 1) + for column_index, header in enumerate(headers, start=1): + cell = worksheet.cell(row=header_row_index, column=column_index) + cell.value = header + cell.font = Font(bold=True) + cell.fill = PatternFill("solid", fgColor="FFE8EEF7") + + status_fills = { + LineStatus.INVALID: PatternFill("solid", fgColor="FFF8CBCB"), + LineStatus.AMBIGUOUS: PatternFill("solid", fgColor="FFFDE9A9"), + LineStatus.EXPIRING: PatternFill("solid", fgColor="FFE2F0D9"), + } + + data_row_index = header_row_index + for row in sheet.rows: + data_row_index += 1 + result = results.get(row.rowIndex) + by_header = {normalize_header(key): value for key, value in row.raw.items()} + values = [by_header.get(normalize_header(header)) for header in source_headers] + values.extend( + render_output_value(column.field, result) for column in profile.exportColumns + ) + fill = status_fills.get(result.status) if result else None + for column_index, value in enumerate(values, start=1): + cell = worksheet.cell(row=data_row_index, column=column_index) + cell.value = value + if fill is not None and profile.highlightForReview: + cell.fill = fill + + for index, header in enumerate(headers, start=1): + worksheet.column_dimensions[get_column_letter(index)].width = max( + 12, min(38, len(str(header)) + 6) + ) + + if profile.totals.enabled: + _append_totals_row(worksheet, profile, sheet, data_row_index, source_headers) + + return workbook + + def _append_totals_row( worksheet: Any, profile: ClientProfileModel, sheet: ParsedSheet, last_data_row: int, + sheet_headers: list[str] | None = None, ) -> None: - """ - Write the totals row under the last data row. - """ - totals = _totals(sheet, profile) + totals = sheet_totals( + [TotalsInput(raw=row.raw, request=row.request) for row in sheet.rows], profile + ) + if not totals: return totals_row = last_data_row + 1 - normalized_headers = { - header: index + 1 for index, header in enumerate(sheet.headers) + header_columns = { + header: index + 1 + for index, header in enumerate( + sheet.headers if sheet_headers is None else sheet_headers + ) } label_cell = worksheet.cell(row=totals_row, column=1) @@ -191,7 +214,7 @@ def _append_totals_row( for canonical, total in totals.items(): client_header = profile.columnMapping.get(canonical) - column_index = normalized_headers.get(client_header) if client_header else None + column_index = header_columns.get(client_header) if client_header else None if column_index is None: continue cell = worksheet.cell(row=totals_row, column=column_index) @@ -200,9 +223,6 @@ def _append_totals_row( def workbook_to_bytes(workbook: Workbook) -> bytes: - """ - Serialize a workbook for storage or download. - """ buffer = io.BytesIO() workbook.save(buffer) return buffer.getvalue() diff --git a/cbh/api/batch/services/metrics/__init__.py b/cbh/api/batch/services/metrics/__init__.py index 84cadd799b1ab3d9eb6c7886cdc7807ae1165b11..2419ea81d155b04fcb570172d25cc76bd44e7416 100644 --- a/cbh/api/batch/services/metrics/__init__.py +++ b/cbh/api/batch/services/metrics/__init__.py @@ -1,3 +1,14 @@ -from .metrics import GoldSetScore, read_gold_rows, score_against_gold, split_codes +from .metrics import GoldSetScore, read_gold_rows, score_against_gold +from .recompute import recompute_metrics +from .totals import TotalsInput, input_value, sheet_totals, shipment_totals -__all__ = ["GoldSetScore", "read_gold_rows", "score_against_gold", "split_codes"] +__all__ = [ + "GoldSetScore", + "TotalsInput", + "input_value", + "read_gold_rows", + "recompute_metrics", + "score_against_gold", + "sheet_totals", + "shipment_totals", +] diff --git a/cbh/api/batch/services/metrics/metrics.py b/cbh/api/batch/services/metrics/metrics.py index f7f672389c37c103d431bd1186cdc3507676505b..5dea1afd7e22cc9d9f5d300241e707f8753a7610 100644 --- a/cbh/api/batch/services/metrics/metrics.py +++ b/cbh/api/batch/services/metrics/metrics.py @@ -1,17 +1,3 @@ -""" -Gold-set scoring. - -The acceptance criteria are stated per pack: percentage of correct codes, -percentage of correct document codes (required and exemption scored -separately), machine time, and human review time. This module computes the -first three by comparing a processed pack against a gold file whose answers -Judith has confirmed; review time is measured on the pack itself. - -Document codes are compared as sets, because the order they appear in carries -no meaning. Codes are compared after normalisation so that formatting -differences between the two files do not read as errors. -""" - from dataclasses import dataclass, field from typing import Any @@ -21,10 +7,19 @@ from cbh.api.common.utils import normalize_ws from cbh.api.importer.utils import normalize_header from cbh.api.taric.utils import normalize_code -# Header spellings accepted in a gold file for each column being scored. GOLD_COLUMNS: dict[str, tuple[str, ...]] = { "rowIndex": ("row", "rowindex", "regel", "line"), - "resolvedCode": ("resolved_code", "resolvedcode", "code", "goederencode", "taric"), + # `hs code` and `y-code` are the spellings Goenee's own line list uses, so a + # finished pack can be handed back as the gold file without being retyped. + "resolvedCode": ( + "resolved_code", + "resolvedcode", + "code", + "hs code", + "hscode", + "goederencode", + "taric", + ), "documentsRequired": ( "documents_required", "documentsrequired", @@ -35,52 +30,37 @@ GOLD_COLUMNS: dict[str, tuple[str, ...]] = { "documentsExemption": ( "documents_exemption", "documentsexemption", + "y-code", + "ycode", + "ycodes", "vrijstellingen", "exemption", ), } -_SEPARATORS = (";", ",", "|", " ") - def split_codes(value: Any) -> set[str]: - """ - Split a document-code cell into a comparable set. - - Either/or groups are written ``C400|Y900`` in the output; for scoring the - alternatives are individual codes, since presenting either is correct. - """ text = normalize_ws(value) if not text: return set() - for separator in _SEPARATORS: + for separator in (";", ",", "|", " "): text = text.replace(separator, ";") return {part.strip().upper() for part in text.split(";") if part.strip()} @dataclass class FieldScore: - """ - Score for one compared field. - """ - compared: int = 0 correct: int = 0 mismatches: list[dict[str, Any]] = field(default_factory=list) @property def accuracy(self) -> float | None: - """ - Share of compared rows that matched, as a percentage. - """ if not self.compared: return None return round(100.0 * self.correct / self.compared, 2) def record(self, row_index: int, expected: Any, actual: Any, matched: bool) -> None: - """ - Record one comparison, keeping the first mismatches for inspection. - """ self.compared += 1 if matched: self.correct += 1 @@ -96,10 +76,6 @@ class FieldScore: @dataclass class GoldSetScore: - """ - The accuracy half of the acceptance metrics for one pack. - """ - goldRows: int = 0 matchedRows: int = 0 unmatchedGoldRows: list[int] = field(default_factory=list) @@ -109,9 +85,6 @@ class GoldSetScore: documentsExemption: FieldScore = field(default_factory=FieldScore) def to_dict(self) -> dict[str, Any]: - """ - Render the score for the API. - """ return { "goldRows": self.goldRows, "matchedRows": self.matchedRows, @@ -126,15 +99,7 @@ class GoldSetScore: def read_gold_rows(payload: bytes, file_name: str) -> list[dict[str, Any]]: - """ - Read a gold file into comparable rows. - - Rows are matched on an explicit row number when the gold file carries one, - which is the only reliable key when the same code appears many times in a - pack. - """ - workbook = load_pack_workbook(payload, file_name) - worksheet = workbook.active + worksheet = load_pack_workbook(payload, file_name).active rows = list(worksheet.iter_rows(values_only=True)) if not rows: @@ -160,7 +125,7 @@ def read_gold_rows(payload: bytes, file_name: str) -> list[dict[str, Any]]: if row is None or all(cell is None or cell == "" for cell in row): continue - def cell(name: str) -> Any: + def cell(name: str, row=row) -> Any: position = positions.get(name) return row[position] if position is not None and position < len(row) else None @@ -183,13 +148,6 @@ def read_gold_rows(payload: bytes, file_name: str) -> list[dict[str, Any]]: def score_against_gold( rows: list[BatchRowModel], gold_rows: list[dict[str, Any]] ) -> GoldSetScore: - """ - Compare a processed pack against a gold file. - - Only rows present in both are scored; gold rows with no counterpart are - reported rather than counted as failures, since a row-numbering mismatch is - a problem with the comparison, not with the engine. - """ score = GoldSetScore(goldRows=len(gold_rows)) by_index = {row.rowIndex: row for row in rows} @@ -202,6 +160,12 @@ def score_against_gold( result = actual_row.result or {} actual_code = normalize_code(result.get("resolvedCode")) + # A gold file written at 8 digits is confirming the subheading, not + # disputing the last two digits the engine resolved. Compare at the + # precision the gold file was written to, and no further. + gold_code = gold["resolvedCode"] + if gold_code and actual_code and len(gold_code) < len(actual_code): + actual_code = actual_code[: len(gold_code)] score.codes.record( gold["rowIndex"], gold["resolvedCode"], @@ -210,7 +174,17 @@ def score_against_gold( ) if gold["hasRequiredColumn"]: - actual_required = _required_codes(result) + actual_required = { + str(document.get("code", "")).upper() + for document in result.get("documentsRequired", []) + if document.get("code") + } + for group in result.get("documentsAlternatives", []): + actual_required.update( + str(document.get("code", "")).upper() + for document in group + if document.get("code") + ) score.documentsRequired.record( gold["rowIndex"], gold["documentsRequired"], @@ -232,21 +206,3 @@ def score_against_gold( ) return score - - -def _required_codes(result: dict[str, Any]) -> set[str]: - """ - Every obligation code on a result, alternatives flattened. - """ - codes = { - str(document.get("code", "")).upper() - for document in result.get("documentsRequired", []) - if document.get("code") - } - for group in result.get("documentsAlternatives", []): - codes.update( - str(document.get("code", "")).upper() - for document in group - if document.get("code") - ) - return codes diff --git a/cbh/api/batch/services/metrics/recompute.py b/cbh/api/batch/services/metrics/recompute.py new file mode 100644 index 0000000000000000000000000000000000000000..d47f5f8baf107284fc17fe0139ff6b03e44f7ecd --- /dev/null +++ b/cbh/api/batch/services/metrics/recompute.py @@ -0,0 +1,66 @@ +from datetime import datetime + +from cbh.api.batch.db_requests import get_batch_rows +from cbh.api.batch.models import BatchMetrics, BatchModel +from cbh.api.batch.services.metrics.totals import TotalsInput, shipment_totals +from cbh.api.client.db_requests import get_profile +from cbh.api.client.models import ClientProfileModel +from cbh.api.common.db_requests import update_obj +from cbh.api.linecheck.dto import ConfidenceLevel +from cbh.api.linecheck.schemas import LineCheckRequest, LineCheckResponse + + +async def recompute_metrics( + batch: BatchModel, profile: ClientProfileModel | None = None +) -> BatchModel: + """Rebuild a shipment's header figures from the rows currently stored. + + The upload route computes these once while it processes the pack. A + shipment built through the form changes one row at a time, so the same + figures have to be derived again from whatever is in the table now. + """ + rows, _ = await get_batch_rows(batch.id) + profile = profile or await get_profile(batch.clientProfile.id) + + status_counts: dict[str, int] = {} + code_counts: dict[str, int] = {} + document_counts: dict[str, int] = {} + lines: list[TotalsInput] = [] + + for row in rows: + status_counts[row.status] = status_counts.get(row.status, 0) + 1 + + if row.result: + result = LineCheckResponse(**row.result) + code_counts[result.codeConfidence.name] = ( + code_counts.get(result.codeConfidence.name, 0) + 1 + ) + document_counts[result.documentConfidence.name] = ( + document_counts.get(result.documentConfidence.name, 0) + 1 + ) + + lines.append( + TotalsInput( + raw=row.inputRow, + request=LineCheckRequest(**row.request) if row.request else None, + ) + ) + + total = len(rows) or 1 + batch.metrics = BatchMetrics( + totalLines=len(rows), + uniqueLookups=batch.metrics.uniqueLookups, + machineSeconds=batch.metrics.machineSeconds, + reviewSeconds=batch.metrics.reviewSeconds, + statusCounts=status_counts, + extractionScore=round(100.0 * code_counts.get(ConfidenceLevel.GREEN.name, 0) / total, 1), + documentScore=round( + 100.0 * document_counts.get(ConfidenceLevel.GREEN.name, 0) / total, 1 + ), + codeConfidenceCounts=code_counts, + documentConfidenceCounts=document_counts, + **shipment_totals(lines, profile), + ) + batch.datetimeUpdated = datetime.now() + await update_obj(BatchModel, batch.id, batch.to_mongo()) + return batch diff --git a/cbh/api/batch/services/metrics/totals.py b/cbh/api/batch/services/metrics/totals.py new file mode 100644 index 0000000000000000000000000000000000000000..1c034354cf3ddca13f1247896436342965114aba --- /dev/null +++ b/cbh/api/batch/services/metrics/totals.py @@ -0,0 +1,151 @@ +from dataclasses import dataclass + +from cbh.api.client.dto import InputField +from cbh.api.client.models import ClientProfileModel +from cbh.api.common.utils import parse_number +from cbh.api.importer.utils import normalize_header +from cbh.api.linecheck.schemas import LineCheckRequest + +REQUEST_ATTRIBUTES: dict[InputField, str] = { + InputField.CODE: "code", + InputField.COUNTRY: "originCountry", + InputField.DESTINATION_COUNTRY: "destinationCountry", + InputField.DESCRIPTION: "description", + InputField.QUANTITY: "quantity", + InputField.PACKAGES: "packages", + InputField.GROSS_WEIGHT: "grossWeight", + InputField.NET_WEIGHT: "netWeight", + InputField.VALUE: "value", +} + + +@dataclass +class TotalsInput: + """One line, as both the client wrote it and as the checker understood it.""" + + raw: dict + request: LineCheckRequest | None + + +def input_value( + field: InputField, + raw: dict, + request: LineCheckRequest | None, + profile: ClientProfileModel, +): + """The client's own value for a canonical field. + + The source row wins, because it is what the client actually wrote; the + parsed request is the fallback for a line that was typed into the form and + so has no source row. + """ + header = profile.columnMapping.get(field.value) + if header and raw: + by_header = {normalize_header(key): value for key, value in raw.items()} + value = by_header.get(normalize_header(header)) + if value not in (None, ""): + return value + + attribute = REQUEST_ATTRIBUTES.get(field) + if attribute and request is not None: + return getattr(request, attribute, None) + return None + + +def _number(value) -> float | None: + """A source cell may be a number or the text of one; a CSV only ever has text.""" + if isinstance(value, bool): + return None + if isinstance(value, (int, float)): + return float(value) + return parse_number(value) + + +def _parcel_key(line: TotalsInput, profile: ClientProfileModel) -> str | None: + column = profile.totals.parcelKeyColumn + if not column or not line.raw: + return None + by_header = {normalize_header(key): value for key, value in line.raw.items()} + value = by_header.get(normalize_header(column)) + return str(value) if value not in (None, "") else None + + +def sum_field( + field: InputField, lines: list[TotalsInput], profile: ClientProfileModel +) -> float | None: + """Add a field up over a pack, counting each physical package once. + + A parcel manifest states weight per parcel and repeats it on every item + line of that parcel. Adding those rows up inflates the total, and the + total is exactly what the broker checks the shipment against, so a + parcel-level field is counted once per parcel key. + """ + parcel_level = field.value in profile.totals.parcelLevelFields + total = 0.0 + found = False + seen: set[str] = set() + + for line in lines: + number = _number(input_value(field, line.raw, line.request, profile)) + if number is None: + continue + if parcel_level: + key = _parcel_key(line, profile) + if key is not None: + if key in seen: + continue + seen.add(key) + total += number + found = True + + return round(total, 2) if found else None + + +def count_parcels(lines: list[TotalsInput], profile: ClientProfileModel) -> int | None: + keys = {key for line in lines if (key := _parcel_key(line, profile)) is not None} + return len(keys) or None + + +def total_packages(lines: list[TotalsInput], profile: ClientProfileModel) -> float: + """How many physical packages the shipment holds. + + For a parcel manifest that is the number of distinct parcels, not the sum + of the per-item piece counts, which counts a two-item parcel twice. + """ + if profile.totals.packagesFromParcelCount: + parcels = count_parcels(lines, profile) + if parcels is not None: + return float(parcels) + return sum_field(InputField.PACKAGES, lines, profile) or 0.0 + + +def shipment_totals( + lines: list[TotalsInput], profile: ClientProfileModel +) -> dict[str, float]: + return { + "totalPackages": total_packages(lines, profile), + "totalGrossWeight": sum_field(InputField.GROSS_WEIGHT, lines, profile) or 0.0, + "totalNetWeight": sum_field(InputField.NET_WEIGHT, lines, profile) or 0.0, + "totalValue": sum_field(InputField.VALUE, lines, profile) or 0.0, + } + + +def sheet_totals( + lines: list[TotalsInput], profile: ClientProfileModel +) -> dict[str, float]: + """The figures for the TOTAL row written into an exported sheet.""" + totals: dict[str, float] = {} + for canonical in profile.totals.sumFields: + try: + field = InputField(canonical) + except ValueError: + continue + if field is InputField.PACKAGES and profile.totals.packagesFromParcelCount: + parcels = count_parcels(lines, profile) + if parcels is not None: + totals[canonical] = float(parcels) + continue + total = sum_field(field, lines, profile) + if total is not None: + totals[canonical] = total + return totals diff --git a/cbh/api/batch/services/processor.py b/cbh/api/batch/services/processor.py index fa7eaa755889bece834312777610fe078f0b88eb..8dea7219050bf0394362a504f8b385059c4293ed 100644 --- a/cbh/api/batch/services/processor.py +++ b/cbh/api/batch/services/processor.py @@ -1,31 +1,32 @@ -""" -The pack processor: read the client's file, run it through the line-check API, -write the file back out. - -This layer is deliberately thin. It owns no customs logic — every decision -comes from the line-check engine, so the Excel path and any future declaration -system cannot drift apart. -""" - from datetime import date, datetime +from pathlib import Path -from cbh.api.batch.dto import BatchStatus, FileFormat +from cbh.api.batch.dto import BatchSource, BatchStatus, FileFormat from cbh.api.batch.models import BatchMetrics, BatchModel, BatchRowModel, ClientProfileRef from cbh.api.batch.services.excel import ( + ParsedSheet, build_output_workbook, parse_pack, workbook_to_bytes, ) -from cbh.api.batch.utils import read_file, result_file_name, store_file +from cbh.api.batch.services.metrics.totals import TotalsInput, shipment_totals +from cbh.api.batch.utils import result_file_name, store_file from cbh.api.client.models import ClientProfileModel -from cbh.api.common.db_requests import delete_objs, insert_obj, insert_objs, update_obj +from cbh.api.common.db_requests import ( + delete_objs, + find_objs, + insert_obj, + insert_objs, + update_obj, +) +from cbh.api.common.dto import CustomsProcedure, TradeDirection from cbh.api.linecheck.dto import LineStatus from cbh.api.linecheck.schemas import LineCheckRequest, LineCheckResponse -from cbh.api.linecheck.services.engine import check_lines +from cbh.api.linecheck.services.engine import check_line_cached, check_lines, recount_confidence +from cbh.api.rules.db_requests import get_active_rules, record_rule_hits +from cbh.api.rules.services.apply import apply_rules, apply_rules_to_lines from cbh.api.taric.db_requests import get_reference_as_of -DB_WRITE_CHUNK = 500 - async def create_batch( name: str, @@ -33,22 +34,23 @@ async def create_batch( file_name: str, profile: ClientProfileModel, check_date: date | None = None, + destination_country: str | None = None, + direction: TradeDirection | None = None, + procedure: CustomsProcedure | None = None, + shipment_reference: str | None = None, ) -> BatchModel: - """ - Store an uploaded pack and record it. - - The source file is kept: a disputed line is settled by replaying the file - that was actually submitted. - """ - stored = store_file(payload, file_name, subdirectory="source") - batch = BatchModel( name=name, clientProfile=ClientProfileRef(id=profile.id, name=profile.name), fileFormat=FileFormat.from_name(file_name), - sourceFile=stored, + sourceFile=store_file(payload, file_name, subdirectory="source"), checkDate=check_date, - defaultDirection=profile.defaults.direction, + defaultDirection=direction or profile.defaults.direction, + defaultProcedure=procedure or profile.defaults.procedure, + destinationCountry=( + (destination_country or profile.defaults.destinationCountry or "").upper() or None + ), + shipmentReference=shipment_reference, status=BatchStatus.UPLOADED, ) @@ -65,14 +67,16 @@ async def create_batch( async def process_batch(batch: BatchModel, profile: ClientProfileModel) -> BatchModel: - """ - Run a pack end to end and produce the output file. - - Safe to repeat: the rows of a previous attempt are cleared first, so a - reprocessed pack is replaced rather than doubled. - """ - payload = read_file(batch.sourceFile) - sheet = parse_pack(payload, batch.sourceFile.fileName, profile, batch.checkDate) + payload = Path(batch.sourceFile.storagePath).read_bytes() + sheet = parse_pack( + payload, + batch.sourceFile.fileName, + profile, + batch.checkDate, + batch.destinationCountry, + batch.defaultDirection, + batch.defaultProcedure, + ) await delete_objs(BatchRowModel, {"batchId": batch.id}) @@ -81,6 +85,12 @@ async def process_batch(batch: BatchModel, profile: ClientProfileModel) -> Batch ] batch_result = await check_lines(requests, batch.checkDate) + reference_as_of = await get_reference_as_of() + batch_result.results = await apply_rules_to_lines( + requests, batch_result.results, batch.clientProfile.id, reference_as_of + ) + recount_confidence(batch_result) + results: dict[int, LineCheckResponse] = { result.rowIndex: result for result in batch_result.results @@ -100,13 +110,21 @@ async def process_batch(batch: BatchModel, profile: ClientProfileModel) -> Batch batch.resultFile = result_file batch.status = BatchStatus.COMPLETED - batch.referenceAsOf = await get_reference_as_of() + batch.referenceAsOf = reference_as_of batch.metrics = BatchMetrics( totalLines=batch_result.totalLines, uniqueLookups=batch_result.uniqueLookups, machineSeconds=batch_result.machineSeconds, statusCounts=batch_result.statusCounts, reviewSeconds=batch.metrics.reviewSeconds, + extractionScore=batch_result.extractionScore, + documentScore=batch_result.documentScore, + codeConfidenceCounts=batch_result.codeConfidenceCounts, + documentConfidenceCounts=batch_result.documentConfidenceCounts, + **shipment_totals( + [TotalsInput(raw=row.raw, request=row.request) for row in sheet.rows], + profile, + ), ) batch.datetimeUpdated = datetime.now() @@ -114,14 +132,95 @@ async def process_batch(batch: BatchModel, profile: ClientProfileModel) -> Batch return batch +async def create_manual_batch( + name: str, + profile: ClientProfileModel, + check_date: date | None = None, + destination_country: str | None = None, + direction: TradeDirection | None = None, + procedure: CustomsProcedure | None = None, + shipment_reference: str | None = None, + currency: str = "EUR", +) -> BatchModel: + """Open an empty shipment for the form to fill. + + It is COMPLETED from the start: there is no file to parse, and every line + is checked as it arrives, so the table is always current. + """ + batch = BatchModel( + name=name, + clientProfile=ClientProfileRef(id=profile.id, name=profile.name), + source=BatchSource.MANUAL, + status=BatchStatus.COMPLETED, + sourceFile=None, + checkDate=check_date, + defaultDirection=direction or profile.defaults.direction, + defaultProcedure=procedure or profile.defaults.procedure, + destinationCountry=( + (destination_country or profile.defaults.destinationCountry or "").upper() or None + ), + shipmentReference=shipment_reference, + currency=currency, + referenceAsOf=await get_reference_as_of(), + ) + await insert_obj(batch) + return batch + + +async def check_and_store_row( + batch: BatchModel, + request: LineCheckRequest, + row_index: int, + existing: BatchRowModel | None = None, +) -> BatchRowModel: + """Check one line and write it into the shipment's table. + + The same path serves a new form line, a corrected code and a re-check, so + a manually added row is indistinguishable from an uploaded one downstream. + """ + reference_as_of = await get_reference_as_of() + request.rowIndex = row_index + if request.date is None: + request.date = batch.checkDate + + result = await check_line_cached(request, reference_as_of) + result.rowIndex = row_index + + rules = await get_active_rules(batch.clientProfile.id) + if rules: + result = await apply_rules(request, result, rules, reference_as_of) + await record_rule_hits([rule.ruleId for rule in result.appliedRules]) + + row = existing or BatchRowModel(batchId=batch.id, rowIndex=row_index) + row.inputRow = { + key: value + for key, value in request.model_dump(mode="json").items() + if value is not None + } + row.request = request.model_dump(mode="json") + row.result = result.model_dump(mode="json") + row.status = result.status.name + row.needsReview = result.status.needs_review + + if existing is None: + await insert_obj(row) + else: + await update_obj(BatchRowModel, row.id, row.to_mongo()) + return row + + +async def next_row_index(batch_id: str) -> int: + rows = await find_objs( + BatchRowModel, {"batchId": batch_id}, limit=1, sort=("rowIndex", -1) + ) + return (rows[0].rowIndex + 1) if rows else 1 + + async def _persist_rows( batch: BatchModel, - sheet, + sheet: ParsedSheet, results: dict[int, LineCheckResponse], ) -> None: - """ - Store every row with the answer it received. - """ rows: list[BatchRowModel] = [] for row in sheet.rows: result = results.get(row.rowIndex) @@ -130,7 +229,12 @@ async def _persist_rows( BatchRowModel( batchId=batch.id, rowIndex=row.rowIndex, - inputRow={key: _serializable(value) for key, value in row.raw.items()}, + inputRow={ + key: value + if value is None or isinstance(value, (str, int, float, bool)) + else str(value) + for key, value in row.raw.items() + }, request=row.request.model_dump(mode="json") if row.request else {}, result=result.model_dump(mode="json") if result else {}, status=status.name, @@ -138,14 +242,5 @@ async def _persist_rows( ) ) - for start in range(0, len(rows), DB_WRITE_CHUNK): - await insert_objs(rows[start : start + DB_WRITE_CHUNK]) - - -def _serializable(value): - """ - Render a raw spreadsheet cell as something BSON can store. - """ - if value is None or isinstance(value, (str, int, float, bool)): - return value - return str(value) + for start in range(0, len(rows), 500): + await insert_objs(rows[start : start + 500]) diff --git a/cbh/api/batch/utils.py b/cbh/api/batch/utils.py index 556a80aaf7ccb40e1f283071fabbb1b7444538ac..49566860545928f3d49fe6921a9ccf054ef339fa 100644 --- a/cbh/api/batch/utils.py +++ b/cbh/api/batch/utils.py @@ -1,7 +1,3 @@ -""" -Batch helpers: file storage and naming. -""" - import re import uuid from datetime import datetime @@ -10,35 +6,13 @@ from pathlib import Path from cbh.api.common.models import FileRef from cbh.core.config import settings -CONTENT_TYPES = { - ".xlsx": "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", - ".xlsm": "application/vnd.ms-excel.sheet.macroEnabled.12", - ".csv": "text/csv", - ".txt": "text/plain", -} - def safe_file_name(file_name: str) -> str: - """ - Strip anything that could escape the storage directory. - """ cleaned = re.sub(r"[^A-Za-z0-9._-]", "_", Path(file_name).name) return cleaned or "upload.xlsx" -def content_type_for(file_name: str) -> str: - """ - Content type for a stored pack. - """ - return CONTENT_TYPES.get(Path(file_name).suffix.lower(), "application/octet-stream") - - def store_file(payload: bytes, file_name: str, subdirectory: str = "source") -> FileRef: - """ - Write a pack to local storage and describe where it went. - - Files are prefixed with a UUID so two packs of the same name never collide. - """ directory = settings.BATCH_STORAGE_DIR / subdirectory directory.mkdir(parents=True, exist_ok=True) @@ -49,25 +23,16 @@ def store_file(payload: bytes, file_name: str, subdirectory: str = "source") -> return FileRef( fileName=clean_name, storagePath=str(path), - contentType=content_type_for(clean_name), + contentType={ + ".xlsx": "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", + ".xlsm": "application/vnd.ms-excel.sheet.macroEnabled.12", + ".csv": "text/csv", + ".txt": "text/plain", + }.get(path.suffix.lower(), "application/octet-stream"), sizeBytes=len(payload), ) -def read_file(file_ref: FileRef) -> bytes: - """ - Read a stored pack back off disk. - """ - return Path(file_ref.storagePath).read_bytes() - - def result_file_name(source_name: str) -> str: - """ - Name of the output file for a pack. - - Always .xlsx: the output carries highlighting and a totals row, which a CSV - cannot represent. - """ stem = Path(safe_file_name(source_name)).stem - stamp = datetime.now().strftime("%Y%m%d") - return f"{stem}_checked_{stamp}.xlsx" + return f"{stem}_checked_{datetime.now().strftime('%Y%m%d')}.xlsx" diff --git a/cbh/api/batch/views.py b/cbh/api/batch/views.py index 5985076a8269321d4527422bd7e77eed88136769..2ff96c0dcd69d6964b5402902e3f433e1263d572 100644 --- a/cbh/api/batch/views.py +++ b/cbh/api/batch/views.py @@ -1,43 +1,78 @@ -""" -Batch views module. -""" - from datetime import date, datetime from pathlib import Path -from fastapi import Depends, File, Form, HTTPException, Query, UploadFile +from fastapi import File, Form, HTTPException, Query, UploadFile from fastapi.responses import FileResponse +from pydantic import ValidationError -from cbh.api.account.dto import AccountType -from cbh.api.account.models import AccountModel from cbh.api.batch import batch_router from cbh.api.batch.db_requests import get_batch, get_batch_rows, get_batches -from cbh.api.batch.dto import BatchStatus, ReviewStage -from cbh.api.batch.models import BatchModel -from cbh.api.batch.schemas import BatchResponse, BatchRowResponse, GoldSetScoreResponse -from cbh.api.batch.services.metrics import read_gold_rows, score_against_gold -from cbh.api.batch.services.processor import create_batch, process_batch +from cbh.api.batch.dto import BatchSource, BatchStatus, ExportFormat, ReviewStage +from cbh.api.batch.models import BatchModel, BatchRowModel +from cbh.api.batch.schemas import ( + BatchResponse, + BatchRowResponse, + ExportFormatResponse, + GoldSetScoreResponse, + ManualLineRequest, + ManualShipmentRequest, + ReviewDocuments, + ReviewHeader, + ReviewRow, + ReviewTableResponse, + RowReviewRequest, + ShipmentTotals, +) +from cbh.api.batch.services.excel import ( + build_export_workbook, + export_file_name, + unavailable_reason, + workbook_to_bytes, +) +from cbh.api.batch.services.metrics import ( + read_gold_rows, + recompute_metrics, + score_against_gold, +) +from cbh.api.batch.services.processor import ( + check_and_store_row, + create_batch, + create_manual_batch, + next_row_index, + process_batch, +) +from cbh.api.batch.utils import store_file from cbh.api.client.db_requests import get_profile -from cbh.api.common.db_requests import delete_obj, delete_objs, update_obj +from cbh.api.client.dto import ExportLayout +from cbh.api.client.models import ClientProfileModel +from cbh.api.common.db_requests import ( + delete_obj, + delete_objs, + find_objs, + insert_obj, + update_obj, +) +from cbh.api.common.dto import CustomsProcedure, Paging, TradeDirection +from cbh.api.linecheck.dto import ConfidenceLevel, ReferenceSource, RowDecision +from cbh.api.linecheck.schemas import LineCheckRequest, LineCheckResponse +from cbh.api.rules.models import RuleModel +from cbh.api.rules.schemas import MakeRuleRequest, RuleRequest, RuleResponse +from cbh.api.rules.views import to_rule_response +from cbh.api.taric.db_requests import get_reference_as_of +from cbh.api.taric.utils import normalize_code from cbh.api.common.schemas import AllObjectsResponse -from cbh.api.common.utils import to_paging -from cbh.core.security import PermissionDependency +from cbh.core.config import settings from cbh.core.wrappers import CbhResponseWrapper -ANY_ACCOUNT = [AccountType.USER, AccountType.ADMIN, AccountType.OWNER, AccountType.SUPER_ADMIN] -ADMIN_ONLY = [AccountType.ADMIN, AccountType.OWNER, AccountType.SUPER_ADMIN] - def _to_response(batch: BatchModel) -> BatchResponse: - """ - Map a pack onto its API representation. - """ return BatchResponse( id=batch.id, name=batch.name, clientProfile=batch.clientProfile, status=batch.status, - sourceFileName=batch.sourceFile.fileName, + source=batch.source, + sourceFileName=batch.sourceFile.fileName if batch.sourceFile else None, resultFileName=batch.resultFile.fileName if batch.resultFile else None, checkDate=batch.checkDate, defaultDirection=batch.defaultDirection, @@ -51,18 +86,31 @@ def _to_response(batch: BatchModel) -> BatchResponse: ) +async def _run(batch: BatchModel, profile: ClientProfileModel) -> BatchModel: + try: + return await process_batch(batch, profile) + except Exception as error: + batch.status = BatchStatus.FAILED + batch.errors.append(f"{type(error).__name__}: {error}") + batch.datetimeUpdated = datetime.now() + await update_obj(BatchModel, batch.id, batch.to_mongo()) + raise HTTPException( + status_code=400, detail=f"Could not process the pack: {error}" + ) from error + + @batch_router.post("/upload") async def upload_batch( clientProfileId: str = Form(...), name: str | None = Form(default=None), checkDate: date | None = Form(default=None), + destinationCountry: str | None = Form(default=None), + direction: TradeDirection | None = Form(default=None), + procedure: CustomsProcedure | None = Form(default=None), + shipmentReference: str | None = Form(default=None), process: bool = Form(default=True), file: UploadFile = File(...), - account: AccountModel = Depends(PermissionDependency(ANY_ACCOUNT)), ) -> CbhResponseWrapper[BatchResponse]: - """ - Upload a client's pack and, by default, process it straight away. - """ payload = await file.read() if not payload: raise HTTPException(status_code=400, detail="Empty file.") @@ -76,6 +124,10 @@ async def upload_batch( file_name=file_name, profile=profile, check_date=checkDate, + destination_country=destinationCountry, + direction=direction, + procedure=procedure, + shipment_reference=shipmentReference, ) if process and batch.status is not BatchStatus.MAPPING_REQUIRED: @@ -84,17 +136,95 @@ async def upload_batch( return CbhResponseWrapper(data=_to_response(batch)) -@batch_router.post("/{batchId}/process") -async def process_existing_batch( - batchId: str, - account: AccountModel = Depends(PermissionDependency(ANY_ACCOUNT)), +@batch_router.post("/manual") +async def create_manual_shipment( + data: ManualShipmentRequest, ) -> CbhResponseWrapper[BatchResponse]: + """Route A of the specification: start a shipment the form will fill. + + It produces the same review table as an upload, so the two input routes + converge on one product rather than two. """ - Process, or reprocess, an uploaded pack. + profile = await get_profile(data.clientProfileId) + batch = await create_manual_batch( + name=data.name or f"Manual shipment {datetime.now():%Y-%m-%d %H:%M}", + profile=profile, + check_date=data.checkDate, + destination_country=data.destinationCountry, + direction=data.direction, + procedure=data.procedure, + shipment_reference=data.shipmentReference, + currency=data.currency, + ) + return CbhResponseWrapper(data=_to_response(batch)) + + +@batch_router.post("/{batchId}/rows") +async def add_manual_row( + batchId: str, + data: ManualLineRequest, +) -> CbhResponseWrapper[ReviewRow]: + """Add one form line to the table, checked on arrival.""" + batch = await get_batch(batchId) + + request = LineCheckRequest( + code=data.code, + originCountry=data.originCountry, + destinationCountry=data.destinationCountry or batch.destinationCountry, + direction=data.direction or batch.defaultDirection, + procedure=data.procedure or batch.defaultProcedure, + date=data.date or batch.checkDate, + additionalCodes=data.additionalCodes, + description=data.description, + quantity=data.quantity, + packages=data.packages, + grossWeight=data.grossWeight, + netWeight=data.netWeight, + value=data.value, + clientProfileId=batch.clientProfile.id, + ) - Reprocessing is how a pack is brought up to date after a reference-data - import: the answers are recomputed against the rules now loaded. + row = await check_and_store_row(batch, request, await next_row_index(batchId)) + await recompute_metrics(batch) + return CbhResponseWrapper(data=_to_review_row(row)) + + +@batch_router.delete("/{batchId}/rows/{rowIndex}") +async def delete_manual_row( + batchId: str, + rowIndex: int, +) -> CbhResponseWrapper[BatchResponse]: + batch = await get_batch(batchId) + removed = await delete_objs(BatchRowModel, {"batchId": batchId, "rowIndex": rowIndex}) + if not removed: + raise HTTPException(status_code=404, detail="Object not found.") + return CbhResponseWrapper(data=_to_response(await recompute_metrics(batch))) + + +@batch_router.post("/{batchId}/recheck") +async def recheck_shipment(batchId: str) -> CbhResponseWrapper[BatchResponse]: + """Re-run every row against the reference data and rules in force now. + + This is what makes a rule retroactive: write the rule, press re-check, and + the lines it covers are answered again rather than edited by hand. """ + batch = await get_batch(batchId) + rows, _ = await get_batch_rows(batchId) + + for row in rows: + if not row.request: + continue + request = LineCheckRequest(**row.request) + if row.correctedCode: + request.code = row.correctedCode + await check_and_store_row(batch, request, row.rowIndex, existing=row) + + batch.referenceAsOf = await get_reference_as_of() + return CbhResponseWrapper(data=_to_response(await recompute_metrics(batch))) + + +@batch_router.post("/{batchId}/process") +async def process_existing_batch(batchId: str) -> CbhResponseWrapper[BatchResponse]: batch = await get_batch(batchId) profile = await get_profile(batch.clientProfile.id) @@ -108,48 +238,71 @@ async def process_existing_batch( return CbhResponseWrapper(data=_to_response(await _run(batch, profile))) -async def _run(batch: BatchModel, profile) -> BatchModel: - """ - Run a pack, recording a failure on the pack rather than losing it. - """ - try: - return await process_batch(batch, profile) - except Exception as error: # pylint: disable=W0718 - batch.status = BatchStatus.FAILED - batch.errors.append(f"{type(error).__name__}: {error}") - batch.datetimeUpdated = datetime.now() - await update_obj(BatchModel, batch.id, batch.to_mongo()) - raise HTTPException( - status_code=400, detail=f"Could not process the pack: {error}" - ) from error - - @batch_router.get("") async def list_batches( pageSize: int = Query(default=25, le=100), pageIndex: int = Query(default=0, ge=0), - account: AccountModel = Depends(PermissionDependency(ANY_ACCOUNT)), ) -> CbhResponseWrapper[AllObjectsResponse[BatchResponse]]: - """ - List the processed packs, newest first. - """ batches, total = await get_batches(pageSize, pageIndex) return CbhResponseWrapper( data=AllObjectsResponse( - paging=to_paging(pageSize, pageIndex, total), + paging=Paging(pageSize=pageSize, pageIndex=pageIndex, totalCount=total), data=[_to_response(batch) for batch in batches], ) ) -@batch_router.get("/{batchId}") -async def get_batch_details( - batchId: str, - account: AccountModel = Depends(PermissionDependency(ANY_ACCOUNT)), -) -> CbhResponseWrapper[BatchResponse]: - """ - Fetch one pack. +EXPORT_FORMAT_DESCRIPTIONS: dict[ExportFormat, str] = { + ExportFormat.REVIEW: ( + "The tool's own table: every column the reviewer saw, including " + "confidence reasons, warnings and the rules that fired." + ), + ExportFormat.SOURCE: ( + "The client's uploaded file returned unchanged, with the checked " + "columns appended in the order the profile fixes." + ), + ExportFormat.SOFTPAK: "Goenee line list: the client's own columns plus the document codes.", + ExportFormat.SCOPE: "Scope line list (Fregat, Spare Cargo).", + ExportFormat.FITON: "Fiton line list (IP Handlers type).", +} + + +@batch_router.get("/export-formats") +async def list_export_formats( + clientProfileId: str | None = Query(default=None), +) -> CbhResponseWrapper[list[ExportFormatResponse]]: + """Fills the `Export: … ▾` dropdown. + + Formats whose import specification has not been received are listed but + marked unavailable, so the gap is visible in the UI instead of being + papered over with a guessed layout. """ + default_format = None + if clientProfileId: + profile = await get_profile(clientProfileId) + default_format = ( + ExportFormat.SOFTPAK + if profile.exportLayout is ExportLayout.SOFTPAK_SHEET + else ExportFormat.SOURCE + ) + + return CbhResponseWrapper( + data=[ + ExportFormatResponse( + format=export_format, + name=export_format.name.title(), + description=EXPORT_FORMAT_DESCRIPTIONS[export_format], + available=export_format.implemented, + unavailableReason=unavailable_reason(export_format), + isProfileDefault=export_format is default_format, + ) + for export_format in ExportFormat + ] + ) + + +@batch_router.get("/{batchId}") +async def get_batch_details(batchId: str) -> CbhResponseWrapper[BatchResponse]: return CbhResponseWrapper(data=_to_response(await get_batch(batchId))) @@ -159,18 +312,11 @@ async def list_batch_rows( needsReviewOnly: bool = Query(default=False), pageSize: int = Query(default=100, le=1000), pageIndex: int = Query(default=0, ge=0), - account: AccountModel = Depends(PermissionDependency(ANY_ACCOUNT)), ) -> CbhResponseWrapper[AllObjectsResponse[BatchRowResponse]]: - """ - Read the rows of a pack. - - ``needsReviewOnly`` gives the broker's working list: the lines the machine - would not answer for. - """ rows, total = await get_batch_rows(batchId, pageSize, pageIndex, needsReviewOnly) return CbhResponseWrapper( data=AllObjectsResponse( - paging=to_paging(pageSize, pageIndex, total), + paging=Paging(pageSize=pageSize, pageIndex=pageIndex, totalCount=total), data=[ BatchRowResponse( rowIndex=row.rowIndex, @@ -185,17 +331,306 @@ async def list_batch_rows( ) -@batch_router.get("/{batchId}/download") -async def download_batch_result( +def _to_review_row(row: BatchRowModel) -> ReviewRow: + result = LineCheckResponse(**row.result) if row.result else None + request = LineCheckRequest(**row.request) if row.request else None + + if result is None: + documents = ReviewDocuments(label="Not checked") + elif result.noneRequired: + documents = ReviewDocuments(noneRequired=True, label="No document required") + elif result.documentsRequired or result.documentsAlternatives: + documents = ReviewDocuments( + required=result.documentsRequired, + alternatives=result.documentsAlternatives, + exemption=result.documentsExemption, + suspended=result.documentsSuspended, + label="; ".join( + [document.code for document in result.documentsRequired] + + [ + "|".join(document.code for document in group) + for group in result.documentsAlternatives + ] + ), + ) + elif result.documentsExemption: + documents = ReviewDocuments( + exemption=result.documentsExemption, + suspended=result.documentsSuspended, + label=", ".join(document.code for document in result.documentsExemption), + ) + elif result.documentsSuspended: + # Nothing to write in Box 44 here, but the line is not settled: the + # label says which obligations the procedure carried forward, so the + # empty cell cannot be read as "nothing was owed". + documents = ReviewDocuments( + suspended=result.documentsSuspended, + label=( + "Suspended: " + + ", ".join(document.code for document in result.documentsSuspended) + ), + ) + else: + documents = ReviewDocuments(label="Not established") + + return ReviewRow( + rowIndex=row.rowIndex, + description=request.description if request else None, + inputCode=result.inputCode if result else None, + proposedCode=row.correctedCode or (result.resolvedCode if result else None), + formattedCode=result.formattedCode if result else None, + codeDescription=result.codeDescription if result else None, + candidates=[candidate.code for candidate in result.candidates] if result else [], + replacementCodes=result.replacementCodes if result else [], + originCountry=result.originCountry if result else None, + destinationCountry=result.destinationCountry if result else None, + measureCountry=result.measureCountry if result else None, + direction=result.direction if result else None, + procedure=result.procedure if result else None, + codeConfidence=result.codeConfidence if result else ConfidenceLevel.RED, + codeConfidenceReason=result.codeConfidenceReason if result else "Row was not checked.", + documentConfidence=result.documentConfidence if result else ConfidenceLevel.RED, + documentConfidenceReason=( + result.documentConfidenceReason if result else "Row was not checked." + ), + source=result.source if result else ReferenceSource.NONE, + documents=documents, + warnings=result.warnings if result else [], + appliedRules=result.appliedRules if result else [], + packages=request.packages if request else None, + grossWeight=request.grossWeight if request else None, + netWeight=request.netWeight if request else None, + value=request.value if request else None, + decision=row.decision, + correctedCode=row.correctedCode, + cbamPermitNumber=row.cbamPermitNumber, + eudrReference=row.eudrReference, + licenceNumbers=row.licenceNumbers, + reviewNote=row.reviewNote, + status=row.status, + needsReview=row.needsReview, + ) + + +@batch_router.get("/{batchId}/review") +async def get_review_table( batchId: str, - account: AccountModel = Depends(PermissionDependency(ANY_ACCOUNT)), -) -> FileResponse: + needsReviewOnly: bool = Query(default=False), + confidence: ConfidenceLevel | None = Query(default=None), + pageSize: int = Query(default=100, le=1000), + pageIndex: int = Query(default=0, ge=0), +) -> CbhResponseWrapper[ReviewTableResponse]: + batch = await get_batch(batchId) + rows, total = await get_batch_rows(batchId, pageSize, pageIndex, needsReviewOnly) + review_rows = [_to_review_row(row) for row in rows] + if confidence is not None: + review_rows = [ + row + for row in review_rows + if confidence in (row.codeConfidence, row.documentConfidence) + ] + + origins = await settings.DB_CLIENT.batchrows.distinct( + "request.originCountry", {"batchId": batchId} + ) + + return CbhResponseWrapper( + data=ReviewTableResponse( + header=ReviewHeader( + batchId=batch.id, + shipmentReference=batch.shipmentReference, + name=batch.name, + clientProfile=batch.clientProfile, + status=batch.status, + source=batch.source, + direction=batch.defaultDirection, + procedure=batch.defaultProcedure, + originCountries=sorted(origin for origin in origins if origin), + destinationCountry=batch.destinationCountry, + checkDate=batch.checkDate, + referenceAsOf=batch.referenceAsOf, + totals=ShipmentTotals( + packages=batch.metrics.totalPackages, + grossWeight=batch.metrics.totalGrossWeight, + netWeight=batch.metrics.totalNetWeight, + value=batch.metrics.totalValue, + currency=batch.currency, + ), + totalLines=batch.metrics.totalLines, + extractionScore=batch.metrics.extractionScore, + documentScore=batch.metrics.documentScore, + codeConfidenceCounts=batch.metrics.codeConfidenceCounts, + documentConfidenceCounts=batch.metrics.documentConfidenceCounts, + linesNeedingReview=batch.metrics.lines_needing_review, + ), + paging=Paging(pageSize=pageSize, pageIndex=pageIndex, totalCount=total), + rows=review_rows, + ) + ) + + +@batch_router.patch("/{batchId}/rows/{rowIndex}") +async def review_row( + batchId: str, + rowIndex: int, + data: RowReviewRequest, +) -> CbhResponseWrapper[ReviewRow]: + batch = await get_batch(batchId) + matches = await find_objs( + BatchRowModel, {"batchId": batchId, "rowIndex": rowIndex}, limit=1 + ) + if not matches: + raise HTTPException(status_code=404, detail="Object not found.") + row = matches[0] + + if data.correctedCode and data.correctedCode != row.correctedCode: + request = LineCheckRequest(**row.request) if row.request else None + if request is None: + raise HTTPException( + status_code=400, detail="This row has no request to re-check." + ) + request.code = data.correctedCode + row = await check_and_store_row(batch, request, rowIndex, existing=row) + row.correctedCode = data.correctedCode + row.decision = RowDecision.CORRECTED + + if data.decision is not None: + row.decision = data.decision + if data.cbamPermitNumber is not None: + row.cbamPermitNumber = data.cbamPermitNumber + if data.eudrReference is not None: + row.eudrReference = data.eudrReference + if data.licenceNumbers is not None: + row.licenceNumbers = data.licenceNumbers + if data.reviewNote is not None: + row.reviewNote = data.reviewNote + row.datetimeReviewed = datetime.now() + + await update_obj(BatchRowModel, row.id, row.to_mongo()) + await recompute_metrics(batch) + return CbhResponseWrapper(data=_to_review_row(row)) + + +@batch_router.post("/{batchId}/rows/{rowIndex}/make-rule") +async def make_rule_from_row( + batchId: str, + rowIndex: int, + data: MakeRuleRequest, +) -> CbhResponseWrapper[RuleResponse]: + """The `Make a rule` action on a review row. + + Conditions the broker does not spell out are taken from the row in front + of them, so pressing the button on a line produces a rule that matches + that line and the ones like it. """ - Download the checked pack. + batch = await get_batch(batchId) + matches = await find_objs( + BatchRowModel, {"batchId": batchId, "rowIndex": rowIndex}, limit=1 + ) + if not matches: + raise HTTPException(status_code=404, detail="Object not found.") + + row = matches[0] + if not row.request: + raise HTTPException( + status_code=400, detail="This row has no request to build a rule from." + ) + + request = LineCheckRequest(**row.request) + result = LineCheckResponse(**row.result) if row.result else None + code = row.correctedCode or (result.resolvedCode if result else None) or str(request.code) + normalized = normalize_code(code) or code + + prefix = None + if data.useCodePrefix: + length = max(2, min(data.codePrefixLength, len(normalized))) + prefix = normalized[:length] + + try: + rule_request = RuleRequest( + name=data.name, + description=data.description, + scope=data.scope, + clientProfileId=batch.clientProfile.id, + codePrefix=prefix, + originCountry=request.originCountry if data.useOriginCountry else None, + destinationCountry=( + request.destinationCountry if data.useDestinationCountry else None + ), + direction=request.direction if data.useDirection else None, + descriptionContains=data.descriptionContains, + action=data.action, + pinnedCode=data.pinnedCode, + documentCodes=data.documentCodes, + note=data.note, + createdFromBatchId=batchId, + createdFromRowIndex=rowIndex, + createdBy=data.createdBy, + ) + except ValidationError as error: + raise HTTPException(status_code=400, detail=str(error)) from error + + rule = RuleModel(**rule_request.model_dump()) + await insert_obj(rule) + return CbhResponseWrapper(data=to_rule_response(rule)) + + +@batch_router.get("/{batchId}/export") +async def export_batch( + batchId: str, + format: ExportFormat | None = Query(default=None), +) -> FileResponse: + """One export, per tenant, chosen at download time. + + Without a format the client profile's own layout is used. REVIEW and + SOFTPAK are built from the stored rows, so a shipment typed into the form + exports exactly like an uploaded one. """ + batch = await get_batch(batchId) + profile = await get_profile(batch.clientProfile.id) + + if format is None: + format = ( + ExportFormat.SOFTPAK + if profile.exportLayout is ExportLayout.SOFTPAK_SHEET + else ExportFormat.SOURCE + ) + + reason = unavailable_reason(format) + if reason: + raise HTTPException(status_code=501, detail=reason) + + if format is ExportFormat.SOURCE: + return await download_batch_result(batchId) + + rows, _ = await get_batch_rows(batchId) + if not rows: + raise HTTPException(status_code=400, detail="This shipment has no lines yet.") + + workbook = build_export_workbook(format, batch, profile, rows) + file_ref = store_file( + workbook_to_bytes(workbook), export_file_name(batch, format), subdirectory="export" + ) + return FileResponse( + path=file_ref.storagePath, + filename=file_ref.fileName, + media_type=file_ref.contentType, + ) + + +@batch_router.get("/{batchId}/download") +async def download_batch_result(batchId: str) -> FileResponse: batch = await get_batch(batchId) if not batch.resultFile: - raise HTTPException(status_code=404, detail="This pack has not been processed yet.") + raise HTTPException( + status_code=404, + detail=( + "This shipment has no source-file result. It was built through the " + "form, or has not been processed yet. Use /export?format=review." + ) + if batch.source is BatchSource.MANUAL + else "This pack has not been processed yet.", + ) path = Path(batch.resultFile.storagePath) if not path.exists(): @@ -209,16 +644,7 @@ async def download_batch_result( @batch_router.post("/{batchId}/review/start") -async def start_review( - batchId: str, - account: AccountModel = Depends(PermissionDependency(ANY_ACCOUNT)), -) -> CbhResponseWrapper[BatchResponse]: - """ - Mark the start of human review. - - Review time is the metric the pilot is judged on, so it is measured here - rather than estimated afterwards. - """ +async def start_review(batchId: str) -> CbhResponseWrapper[BatchResponse]: batch = await get_batch(batchId) batch.reviewStage = ReviewStage.IN_REVIEW batch.reviewStartedAt = datetime.now() @@ -230,13 +656,7 @@ async def start_review( @batch_router.post("/{batchId}/review/finish") -async def finish_review( - batchId: str, - account: AccountModel = Depends(PermissionDependency(ANY_ACCOUNT)), -) -> CbhResponseWrapper[BatchResponse]: - """ - Mark the end of human review and record how long it took. - """ +async def finish_review(batchId: str) -> CbhResponseWrapper[BatchResponse]: batch = await get_batch(batchId) if not batch.reviewStartedAt: raise HTTPException(status_code=400, detail="Review was never started.") @@ -255,15 +675,7 @@ async def finish_review( async def score_batch_against_gold( batchId: str, file: UploadFile = File(...), - account: AccountModel = Depends(PermissionDependency(ANY_ACCOUNT)), ) -> CbhResponseWrapper[GoldSetScoreResponse]: - """ - Score a processed pack against a gold file confirmed by the client. - - Returns the acceptance metrics for that pack: correct codes, correct - document codes (required and exemption scored separately), machine time and - review time. - """ batch = await get_batch(batchId) if batch.status is not BatchStatus.COMPLETED: raise HTTPException(status_code=400, detail="This pack has not been processed yet.") @@ -291,18 +703,7 @@ async def score_batch_against_gold( @batch_router.delete("/{batchId}") -async def delete_batch( - batchId: str, - account: AccountModel = Depends(PermissionDependency(ADMIN_ONLY)), -) -> CbhResponseWrapper[BatchResponse]: - """ - Delete a pack and its rows. - - The stored files are left on disk: they are the evidence behind answers - already given to the client. - """ - from cbh.api.batch.models import BatchRowModel - +async def delete_batch(batchId: str) -> CbhResponseWrapper[BatchResponse]: batch = await delete_obj(BatchModel, batchId) await delete_objs(BatchRowModel, {"batchId": batchId}) return CbhResponseWrapper(data=_to_response(batch)) diff --git a/cbh/api/client/__init__.py b/cbh/api/client/__init__.py index f7aa9f01cb586b2ee8d54bbb5f05fce11d62648b..4ecc92f8982bb76493976c5fc55c45499ddcfc17 100644 --- a/cbh/api/client/__init__.py +++ b/cbh/api/client/__init__.py @@ -1,10 +1,3 @@ -""" -Client module initialization. - -This module defines the FastAPI router for client profiles: the per-client -column mapping and export profile used to read and write their files. -""" - from fastapi import APIRouter client_router = APIRouter( diff --git a/cbh/api/client/db_requests.py b/cbh/api/client/db_requests.py index e142bcc6ca467617d5f8fc15a1c1aa7dd6d6b59c..d1b1804fda60f373937c04ae04570d1a7d119649 100644 --- a/cbh/api/client/db_requests.py +++ b/cbh/api/client/db_requests.py @@ -1,33 +1,35 @@ -""" -Client profile database requests. -""" - -from cbh.api.client.models import ClientProfileModel, goenee_profile -from cbh.api.common.db_requests import count_objs, find_objs, get_obj_by_id, insert_obj +from cbh.api.client.models import ( + ClientProfileModel, + efg_henx_transit_profile, + goenee_export_line_list_profile, + goenee_softpak_profile, + gstar_ecom_profile, +) +from cbh.api.common.db_requests import find_objs, get_obj_by_id, insert_objs async def get_profile(profile_id: str) -> ClientProfileModel: - """ - Fetch a client profile, 404 when it does not exist. - """ return await get_obj_by_id(ClientProfileModel, profile_id) -async def get_profile_by_name(name: str) -> ClientProfileModel | None: - """ - Fetch a client profile by name. - """ - profiles = await find_objs(ClientProfileModel, {"name": name}, limit=1) - return profiles[0] if profiles else None - - async def ensure_default_profiles() -> None: - """ - Seed the Goenee profile on an empty database. + """Add any built-in profile that is not there yet, by name. - Goenee is the first client, so their profile ships with the application - rather than having to be typed in before the first pack can be processed. + Bailing out as soon as one profile exists meant a profile added later never + reached a database that had already been started once. """ - if await count_objs(ClientProfileModel) > 0: - return - await insert_obj(goenee_profile()) + defaults = [ + goenee_softpak_profile(), + gstar_ecom_profile(), + goenee_export_line_list_profile(), + efg_henx_transit_profile(), + ] + existing = { + profile.name + for profile in await find_objs( + ClientProfileModel, {"name": {"$in": [default.name for default in defaults]}} + ) + } + missing = [default for default in defaults if default.name not in existing] + if missing: + await insert_objs(missing) diff --git a/cbh/api/client/dto.py b/cbh/api/client/dto.py index 861816914e58aa5cd8d2991cabba87837d2ecacf..2670ca9674f2ae63c040101b3420f741fc100c98 100644 --- a/cbh/api/client/dto.py +++ b/cbh/api/client/dto.py @@ -1,58 +1,34 @@ -""" -Client profile DTOs. -""" +from enum import Enum, IntEnum -from enum import Enum - - -class ExportProfileType(Enum): - """ - Which declaration system the output file is shaped for. - """ +class ExportProfileType(IntEnum): SOFTPAK = 1 GENERIC = 2 class InputField(Enum): - """ - The canonical fields a client column can be mapped onto. - - Client files arrive in the client's own layout, so the mapping is - configured once per client and reused for every pack. - """ - CODE = "code" COUNTRY = "country" + DESTINATION_COUNTRY = "destinationCountry" DIRECTION = "direction" + PROCEDURE = "procedure" DATE = "date" ADDITIONAL_CODE = "additionalCode" DESCRIPTION = "description" QUANTITY = "quantity" PACKAGES = "packages" + COLLI_TYPE = "colliType" GROSS_WEIGHT = "grossWeight" NET_WEIGHT = "netWeight" VALUE = "value" + REMARKS = "remarks" @property def required(self) -> bool: - """ - Whether a pack cannot be processed without this field. - - Direction and date have profile-level defaults, so only the code and - the country have to come from the file. - """ return self in (InputField.CODE, InputField.COUNTRY) class OutputField(Enum): - """ - The columns the checker appends to the client's file. - - The first five are the contract agreed for the Softpak template; the rest - are optional detail a reviewer can switch on. - """ - RESOLVED_CODE = "resolved_code" DOCUMENTS_REQUIRED = "documents_required" DOCUMENTS_EXEMPTION = "documents_exemption" @@ -66,3 +42,12 @@ class OutputField(Enum): NONE_REQUIRED = "none_required" APPLIED_MEASURES = "applied_measures" REFERENCE_AS_OF = "reference_as_of" + + YCODES = "ycodes" + CODE_CONFIDENCE = "code_confidence" + DOCUMENT_CONFIDENCE = "document_confidence" + + +class ExportLayout(Enum): + APPEND_COLUMNS = 1 + SOFTPAK_SHEET = 2 diff --git a/cbh/api/client/models.py b/cbh/api/client/models.py index 88a0606c23203cffb594887c8dc7143f647709a7..0e83725ff507a6aa0539fd878c1b953940fa2332 100644 --- a/cbh/api/client/models.py +++ b/cbh/api/client/models.py @@ -1,43 +1,26 @@ -""" -Client profile models. -""" - from datetime import datetime from pydantic import Field -from cbh.api.client.dto import ExportProfileType, InputField, OutputField -from cbh.api.common.dto import TradeDirection -from cbh.core.database import MongoBaseModel, MongoBaseShortenModel, MongoSerializableModel +from cbh.api.client.dto import ExportLayout, ExportProfileType, InputField, OutputField +from cbh.api.common.dto import CustomsProcedure, TradeDirection +from cbh.core.database import MongoBaseModel, MongoSerializableModel class ExportColumn(MongoSerializableModel): - """ - One appended output column: what to write and what to call it. - """ - field: OutputField header: str class ProfileDefaults(MongoSerializableModel): - """ - What to assume when the client's file does not say. - - A pack with no date column is checked against today, which is what a broker - filing today means by an undated line. - """ - direction: TradeDirection = TradeDirection.IMPORT + procedure: CustomsProcedure = CustomsProcedure.RELEASE_FOR_FREE_CIRCULATION country: str | None = None + destinationCountry: str | None = None useTodayWhenDateMissing: bool = True class TotalsConfig(MongoSerializableModel): - """ - The totals row Judith checks at the bottom of every pack. - """ - enabled: bool = True label: str = "TOTAL" sumFields: list[str] = Field( @@ -48,11 +31,16 @@ class TotalsConfig(MongoSerializableModel): ] ) + # A parcel manifest carries one row per item but states weight and piece + # count per parcel, repeated on every row of that parcel. Adding the rows + # up inflates the shipment totals, which is the figure the broker checks + # the consignment against, so those fields are counted once per parcel. + parcelKeyColumn: str | None = None + parcelLevelFields: list[str] = Field(default_factory=list) + packagesFromParcelCount: bool = False + def default_export_columns() -> list[ExportColumn]: - """ - The agreed set of appended columns, in order. - """ return [ ExportColumn(field=OutputField.RESOLVED_CODE, header="resolved_code"), ExportColumn(field=OutputField.DOCUMENTS_REQUIRED, header="documents_required"), @@ -63,14 +51,6 @@ def default_export_columns() -> list[ExportColumn]: class ClientProfileModel(MongoBaseModel): - """ - How one client's files are read and written. - - ``columnMapping`` maps a canonical field onto the header used in that - client's spreadsheets; ``exportColumns`` fixes the order of the columns - appended for their declaration system. - """ - name: str description: str | None = None exportProfile: ExportProfileType = ExportProfileType.SOFTPAK @@ -83,20 +63,13 @@ class ClientProfileModel(MongoBaseModel): headerRow: int = 1 sheetName: str | None = None highlightForReview: bool = True + exportLayout: ExportLayout = ExportLayout.APPEND_COLUMNS + softpakHeaderRows: list[list[str]] = Field(default_factory=list) datetimeInserted: datetime = Field(default_factory=datetime.now) datetimeUpdated: datetime = Field(default_factory=datetime.now) - def header_for(self, field: InputField) -> str | None: - """ - The client's header for a canonical field, if mapped. - """ - return self.columnMapping.get(field.value) - def missing_required_fields(self) -> list[str]: - """ - Required canonical fields this profile does not map. - """ return [ field.value for field in InputField @@ -104,38 +77,171 @@ class ClientProfileModel(MongoBaseModel): ] -class ClientProfileShorten(MongoBaseShortenModel): - """ - Client profile projection for list endpoints. - """ +def goenee_softpak_profile() -> ClientProfileModel: + return ClientProfileModel( + name="Goenee Softpak", + description="Jac. Goenee B.V., Vianen. Softpak line list, Excel in and out.", + exportProfile=ExportProfileType.SOFTPAK, + exportLayout=ExportLayout.SOFTPAK_SHEET, + headerRow=7, + columnMapping={ + InputField.CODE.value: "hs code", + InputField.DESCRIPTION.value: "description", + InputField.GROSS_WEIGHT.value: "WEIGHT GROSS", + InputField.NET_WEIGHT.value: "WEIGHT nett", + InputField.COUNTRY.value: "origin", + InputField.QUANTITY.value: "sup_quantity", + InputField.COLLI_TYPE.value: "type colli", + InputField.PACKAGES.value: "nr colli", + InputField.REMARKS.value: "remarks", + InputField.VALUE.value: "value", + }, + exportColumns=[ExportColumn(field=OutputField.YCODES, header="Ycodes")], + softpakHeaderRows=[ + ["Government procedure", "10", " "], + ["Previous ", "00"], + ["Invoice N935", " "], + ["Currency", "EUR"], + ["Container"], + ], + defaults=ProfileDefaults( + direction=TradeDirection.EXPORT, useTodayWhenDateMissing=True + ), + totals=TotalsConfig( + sumFields=[ + InputField.GROSS_WEIGHT.value, + InputField.NET_WEIGHT.value, + InputField.PACKAGES.value, + InputField.VALUE.value, + ] + ), + ) - name: str - exportProfile: ExportProfileType +def goenee_export_line_list_profile() -> ClientProfileModel: + """The line list Goenee produces from a G-Star manifest — `Export Goenee.xls`. -def goenee_profile() -> ClientProfileModel: + It is the same shipment as the ecom manifest but already worked: the Y-code + column holds the exemptions a declarant assigned by hand, which is what the + gold-set scoring is measured against. """ - The starting profile for Goenee (Softpak). + return ClientProfileModel( + name="Goenee export line list", + description="Jac. Goenee B.V. line list for a G-Star export, legacy .xls.", + exportProfile=ExportProfileType.SOFTPAK, + columnMapping={ + InputField.CODE.value: "HS Code", + InputField.DESCRIPTION.value: "Item Descr", + InputField.GROSS_WEIGHT.value: "Total Item Weight", + InputField.COUNTRY.value: "Item COO", + InputField.PACKAGES.value: "Pieces", + InputField.COLLI_TYPE.value: "Colli type", + # The header is truncated in the file itself; it has no closing + # bracket, and the mapping matches what is actually written. + InputField.REMARKS.value: "Remarks (EAN", + InputField.VALUE.value: "Total Value (GBP)", + }, + exportColumns=[ExportColumn(field=OutputField.YCODES, header="Y-code")], + defaults=ProfileDefaults( + direction=TradeDirection.EXPORT, destinationCountry="GB" + ), + totals=TotalsConfig( + sumFields=[ + InputField.PACKAGES.value, + InputField.GROSS_WEIGHT.value, + InputField.VALUE.value, + ], + parcelKeyColumn="Tracking no", + parcelLevelFields=[ + InputField.PACKAGES.value, + InputField.GROSS_WEIGHT.value, + ], + packagesFromParcelCount=True, + ), + ) - The column names are the ones their files have used so far; the export - order follows the five agreed columns until Judith's Softpak template - specification lands, at which point only this function changes. + +def efg_henx_transit_profile() -> ClientProfileModel: + """EFG Arabia to Henx Europe — the packs Goenee files under external transit. + + The goods are not released on arrival: they move under T1 to Henx and are + declared there, so duty, VAT and excise are secured rather than paid. The + prohibitions and restrictions still bind, which is what this profile is for. + + The codes in these packs are eight-digit CN. An import declaration is filed + on ten, so most lines come back with their declarable candidates rather than + an answer — that is the pack's gap, not the engine's, and it is left visible. """ return ClientProfileModel( - name="Goenee", - description="Customs broker, Vianen. Files Softpak declarations.", + name="EFG Arabia to Henx (T1)", + description="EFG Arabia commercial invoice / packing list, external transit to Henx Europe BV.", + exportProfile=ExportProfileType.SOFTPAK, + # The pack opens with the invoice head — shipper, receiver, totals — and + # the line table starts at row 20. The workbook's other sheet is a pivot + # of the same lines and holds no declarable detail. + sheetName="1. SUMMARY (EQ LIST FOR SHIPPIN", + headerRow=20, + columnMapping={ + InputField.CODE.value: "HS CODE", + InputField.COUNTRY.value: "(country)", + InputField.DESCRIPTION.value: "Description", + InputField.QUANTITY.value: "Quantity", + InputField.GROSS_WEIGHT.value: "Case weight", + InputField.VALUE.value: "Total Value (EUR)|Total Value", + InputField.REMARKS.value: "Serial No.", + }, + exportColumns=[ExportColumn(field=OutputField.YCODES, header="Y-code")], + defaults=ProfileDefaults( + direction=TradeDirection.IMPORT, + procedure=CustomsProcedure.EXTERNAL_TRANSIT_T1, + destinationCountry="NL", + useTodayWhenDateMissing=True, + ), + totals=TotalsConfig( + sumFields=[ + InputField.GROSS_WEIGHT.value, + InputField.QUANTITY.value, + InputField.VALUE.value, + ] + ), + ) + + +def gstar_ecom_profile() -> ClientProfileModel: + return ClientProfileModel( + name="G-Star ecom GB", + description="Group eStore BV parcel manifest, NL to GB. One row per item.", exportProfile=ExportProfileType.SOFTPAK, columnMapping={ - InputField.CODE.value: "Goederencode", - InputField.COUNTRY.value: "Land", - InputField.DIRECTION.value: "Richting", - InputField.DATE.value: "Datum", - InputField.DESCRIPTION.value: "Omschrijving", - InputField.QUANTITY.value: "Aantal", - InputField.PACKAGES.value: "Colli", - InputField.GROSS_WEIGHT.value: "Brutogewicht", - InputField.NET_WEIGHT.value: "Nettogewicht", - InputField.VALUE.value: "Waarde", + InputField.CODE.value: "HS Code", + InputField.DESCRIPTION.value: "Item Descr", + InputField.GROSS_WEIGHT.value: "Total Weight", + InputField.COUNTRY.value: "Item COO", + InputField.DESTINATION_COUNTRY.value: "Cust Country", + InputField.PACKAGES.value: "Pieces", + InputField.QUANTITY.value: "Item Qty", + InputField.VALUE.value: "Item Value", + InputField.REMARKS.value: "SKU", }, - defaults=ProfileDefaults(direction=TradeDirection.EXPORT), + exportColumns=[ExportColumn(field=OutputField.YCODES, header="Y-code")], + defaults=ProfileDefaults( + direction=TradeDirection.EXPORT, destinationCountry="GB" + ), + # One row per item, many rows per parcel. "Total Weight" and the piece + # count belong to the parcel, so both are counted once per tracking + # number. Checked against the Master Invoice for 04-09-2026: 167 + # packages and GBP 11,376.42, which only reconcile this way. + totals=TotalsConfig( + sumFields=[ + InputField.PACKAGES.value, + InputField.GROSS_WEIGHT.value, + InputField.VALUE.value, + ], + parcelKeyColumn="Tracking no", + parcelLevelFields=[ + InputField.PACKAGES.value, + InputField.GROSS_WEIGHT.value, + ], + packagesFromParcelCount=True, + ), ) diff --git a/cbh/api/client/schemas.py b/cbh/api/client/schemas.py index 6d4f44c8096aa76fcde8adcfb80e185019e30681..8f141cb7e64c44cfe786f01770a4621a406c587d 100644 --- a/cbh/api/client/schemas.py +++ b/cbh/api/client/schemas.py @@ -1,7 +1,3 @@ -""" -Client profile schemas. -""" - from pydantic import BaseModel from cbh.api.client.dto import ExportProfileType @@ -9,10 +5,6 @@ from cbh.api.client.models import ExportColumn, ProfileDefaults, TotalsConfig class ClientProfileRequest(BaseModel): - """ - Create or replace a client profile. - """ - name: str description: str | None = None exportProfile: ExportProfileType = ExportProfileType.SOFTPAK @@ -28,10 +20,6 @@ class ClientProfileRequest(BaseModel): class ClientProfileResponse(BaseModel): - """ - A client profile as returned by the API. - """ - id: str name: str description: str | None = None @@ -49,10 +37,6 @@ class ClientProfileResponse(BaseModel): class ColumnMappingSuggestion(BaseModel): - """ - A guessed mapping for a newly uploaded file, for confirmation. - """ - headers: list[str] suggestedMapping: dict[str, str] unmappedHeaders: list[str] diff --git a/cbh/api/client/utils.py b/cbh/api/client/utils.py index f702d56cd1aacb0b2ad3fcacc197e61b45f46ea1..831c88a9993af9ad245cf7768285ec5dab96b3db 100644 --- a/cbh/api/client/utils.py +++ b/cbh/api/client/utils.py @@ -1,14 +1,8 @@ -""" -Client profile helpers: guessing a column mapping from a file's headers. -""" - from typing import Any from cbh.api.client.dto import InputField from cbh.api.importer.utils import normalize_header -# Header spellings seen in broker files, per canonical field. Dutch first: -# that is what the files in scope are written in. FIELD_SYNONYMS: dict[InputField, tuple[str, ...]] = { InputField.CODE: ( "goederencode", @@ -86,14 +80,6 @@ FIELD_SYNONYMS: dict[InputField, tuple[str, ...]] = { def suggest_column_mapping(headers: list[Any]) -> dict[str, str]: - """ - Guess a column mapping from a file's header row. - - Exact matches on a known spelling win; otherwise a header that contains a - known spelling is accepted. A guess is a starting point for the one-time - per-client configuration, not a substitute for it — which is why the API - returns it for confirmation rather than saving it. - """ normalized = [(normalize_header(header), str(header)) for header in headers if header] mapping: dict[str, str] = {} taken: set[str] = set() @@ -135,11 +121,5 @@ def suggest_column_mapping(headers: list[Any]) -> dict[str, str]: def unmapped_headers(headers: list[Any], mapping: dict[str, str]) -> list[str]: - """ - Headers the mapping does not claim. - - These are carried through to the output untouched, so the client's file - comes back with everything it arrived with. - """ claimed = set(mapping.values()) return [str(header) for header in headers if header and str(header) not in claimed] diff --git a/cbh/api/client/views.py b/cbh/api/client/views.py index ae0adc81ded313f50d890666f98a4a80cce36d56..7a6bdf063ae069f2ec003cd20a01014510c2f180 100644 --- a/cbh/api/client/views.py +++ b/cbh/api/client/views.py @@ -1,15 +1,11 @@ -""" -Client profile views module. -""" - from datetime import datetime -from fastapi import Depends, File, Query, UploadFile +from fastapi import File, Query, UploadFile -from cbh.api.account.dto import AccountType -from cbh.api.account.models import AccountModel +from cbh.api.batch.services.excel import read_headers from cbh.api.client import client_router from cbh.api.client.db_requests import get_profile +from cbh.api.client.dto import InputField from cbh.api.client.models import ClientProfileModel, default_export_columns from cbh.api.client.schemas import ( ClientProfileRequest, @@ -18,19 +14,12 @@ from cbh.api.client.schemas import ( ) from cbh.api.client.utils import suggest_column_mapping, unmapped_headers from cbh.api.common.db_requests import delete_obj, get_all_objs, insert_obj, update_obj +from cbh.api.common.dto import Paging from cbh.api.common.schemas import AllObjectsResponse -from cbh.api.common.utils import to_paging -from cbh.core.security import PermissionDependency from cbh.core.wrappers import CbhResponseWrapper -ANY_ACCOUNT = [AccountType.USER, AccountType.ADMIN, AccountType.OWNER, AccountType.SUPER_ADMIN] -ADMIN_ONLY = [AccountType.ADMIN, AccountType.OWNER, AccountType.SUPER_ADMIN] - def _to_response(profile: ClientProfileModel) -> ClientProfileResponse: - """ - Map a client profile onto its API representation. - """ return ClientProfileResponse( id=profile.id, name=profile.name, @@ -51,17 +40,13 @@ def _to_response(profile: ClientProfileModel) -> ClientProfileResponse: async def list_profiles( pageSize: int = Query(default=25, le=100), pageIndex: int = Query(default=0, ge=0), - account: AccountModel = Depends(PermissionDependency(ANY_ACCOUNT)), ) -> CbhResponseWrapper[AllObjectsResponse[ClientProfileResponse]]: - """ - List the configured client profiles. - """ profiles, total = await get_all_objs( ClientProfileModel, page_size=pageSize, page_index=pageIndex, sort=("name", 1) ) return CbhResponseWrapper( data=AllObjectsResponse( - paging=to_paging(pageSize, pageIndex, total), + paging=Paging(pageSize=pageSize, pageIndex=pageIndex, totalCount=total), data=[_to_response(profile) for profile in profiles], ) ) @@ -70,22 +55,14 @@ async def list_profiles( @client_router.get("/{profileId}") async def get_client_profile( profileId: str, - account: AccountModel = Depends(PermissionDependency(ANY_ACCOUNT)), ) -> CbhResponseWrapper[ClientProfileResponse]: - """ - Fetch one client profile. - """ return CbhResponseWrapper(data=_to_response(await get_profile(profileId))) @client_router.post("") async def create_profile( data: ClientProfileRequest, - account: AccountModel = Depends(PermissionDependency(ADMIN_ONLY)), ) -> CbhResponseWrapper[ClientProfileResponse]: - """ - Create a client profile. - """ profile = ClientProfileModel( **data.model_dump(exclude_none=True), ) @@ -99,11 +76,7 @@ async def create_profile( async def replace_profile( profileId: str, data: ClientProfileRequest, - account: AccountModel = Depends(PermissionDependency(ADMIN_ONLY)), ) -> CbhResponseWrapper[ClientProfileResponse]: - """ - Replace a client profile. - """ existing = await get_profile(profileId) updated = ClientProfileModel( **{ @@ -120,11 +93,7 @@ async def replace_profile( @client_router.delete("/{profileId}") async def delete_profile( profileId: str, - account: AccountModel = Depends(PermissionDependency(ADMIN_ONLY)), ) -> CbhResponseWrapper[ClientProfileResponse]: - """ - Delete a client profile. - """ removed = await delete_obj(ClientProfileModel, profileId) return CbhResponseWrapper(data=_to_response(removed)) @@ -133,18 +102,7 @@ async def delete_profile( async def suggest_mapping( file: UploadFile = File(...), headerRow: int = Query(default=1, ge=1), - account: AccountModel = Depends(PermissionDependency(ADMIN_ONLY)), ) -> CbhResponseWrapper[ColumnMappingSuggestion]: - """ - Guess a column mapping from an uploaded client file. - - The guess is returned for confirmation, never saved: the mapping is - configured once per client and a wrong guess would quietly mis-read every - pack that followed. - """ - from cbh.api.batch.services.excel import read_headers - from cbh.api.client.dto import InputField - payload = await file.read() headers = read_headers(payload, file.filename or "upload.xlsx", headerRow) mapping = suggest_column_mapping(headers) diff --git a/cbh/api/common/db_requests.py b/cbh/api/common/db_requests.py index d3b743324575347c2955865eb98ce723b6498bcc..253f3e117538e87aac778a8d4915abe28c7ec385 100644 --- a/cbh/api/common/db_requests.py +++ b/cbh/api/common/db_requests.py @@ -1,27 +1,16 @@ -""" -Common database requests. -""" - -import re -from datetime import datetime, timedelta from typing import Any, TypeVar from fastapi import HTTPException from pydantic import BaseModel from pymongo import ASCENDING, DESCENDING, UpdateOne -from cbh.api.common.schemas import SearchRequest from cbh.core.config import settings T = TypeVar("T", bound=BaseModel) collection_map = { - "AccountModel": "accounts", - "AccountShorten": "accounts", "CacheEntryModel": "caches", - # --- TARIC / national reference data --- "NomenclatureModel": "nomenclature", - "NomenclatureShorten": "nomenclature", "MeasureModel": "measures", "MeasureTypeModel": "measuretypes", "GeoAreaModel": "geoareas", @@ -29,19 +18,14 @@ collection_map = { "DocumentCodeModel": "documentcodes", "CodeChangeModel": "codechanges", "CodeboekEntryModel": "codeboekentries", - # --- importer --- "SyncLogModel": "synclogs", - # --- client configuration --- "ClientProfileModel": "clientprofiles", - "ClientProfileShorten": "clientprofiles", - # --- excel batches --- "BatchModel": "batches", - "BatchShorten": "batches", "BatchRowModel": "batchrows", + "RuleModel": "rules", } INDEXES: dict[str, list[tuple[list[tuple[str, int]], dict]]] = { - "accounts": [([("id", ASCENDING)], {"unique": True})], "caches": [ ([("namespace", ASCENDING), ("key", ASCENDING)], {"unique": True}), ([("deletedAt", ASCENDING)], {"expireAfterSeconds": 0}), @@ -77,25 +61,20 @@ INDEXES: dict[str, list[tuple[list[tuple[str, int]], dict]]] = { ([("datetimeInserted", DESCENDING)], {}), ], "batchrows": [([("batchId", ASCENDING), ("rowIndex", ASCENDING)], {"unique": True})], + "rules": [ + ([("id", ASCENDING)], {"unique": True}), + ([("active", ASCENDING), ("scope", ASCENDING), ("clientProfileId", ASCENDING)], {}), + ], } async def ensure_indexes() -> None: - """ - Create the indexes the application relies on. - - Idempotent: MongoDB ignores a create_index call for an index that already - exists with the same definition. - """ for collection, definitions in INDEXES.items(): for keys, options in definitions: await settings.DB_CLIENT[collection].create_index(keys, **options) def resolve_collection(model: T, collection: str | None = None) -> str: - """ - Resolve the MongoDB collection backing a model. - """ if collection: return collection name = model if isinstance(model, str) else model.__name__ @@ -112,9 +91,6 @@ async def get_obj_by_id( exception: bool = True, collection: str | None = None, ) -> T | None: - """ - Get an object by ID. - """ filter_ = {"id": obj_id} if obj_id else {} if additional_filter: filter_.update(additional_filter) @@ -128,18 +104,15 @@ async def get_obj_by_id( return model.from_mongo(obj) -async def _paginate_objs( +async def get_all_objs( model: T, - filter_: dict, page_size: int, page_index: int, - sort: tuple[str, int], - projection: dict | None, + sort: tuple[str, int] = ("id", -1), + additional_filter: dict | None = None, + projection: dict | None = None, collection: str | None = None, ) -> tuple[list[T], int]: - """ - Fetch one page plus the total count in a single $facet aggregate. - """ if page_size > 0: page_stages = [{"$skip": page_index * page_size}, {"$limit": page_size}] else: @@ -147,7 +120,7 @@ async def _paginate_objs( if projection: page_stages.append({"$project": projection}) pipeline = [ - {"$match": filter_}, + {"$match": additional_filter or {}}, {"$sort": {sort[0]: sort[1]}}, { "$facet": { @@ -163,22 +136,6 @@ async def _paginate_objs( return objs, total_count -async def get_all_objs( - model: T, - page_size: int, - page_index: int, - sort: tuple[str, int] = ("id", -1), - additional_filter: dict | None = None, - projection: dict | None = None, - collection: str | None = None, -) -> tuple[list[T], int]: - """ - Get all objects. - """ - filter_ = additional_filter if additional_filter else {} - return await _paginate_objs(model, filter_, page_size, page_index, sort, projection, collection) - - async def find_objs( model: T, filter_: dict, @@ -187,9 +144,6 @@ async def find_objs( projection: dict | None = None, collection: str | None = None, ) -> list[T]: - """ - Find objects without pagination metadata. - """ cursor = settings.DB_CLIENT[resolve_collection(model, collection)].find(filter_, projection) if sort: cursor = cursor.sort(sort[0], sort[1]) @@ -201,27 +155,18 @@ async def find_objs( async def count_objs( model: T, additional_filter: dict | None = None, collection: str | None = None ) -> int: - """ - Count objects. - """ return await settings.DB_CLIENT[resolve_collection(model, collection)].count_documents( additional_filter or {} ) async def insert_obj(model_instance: BaseModel, collection: str | None = None) -> BaseModel: - """ - Insert a single object. - """ collection = resolve_collection(type(model_instance), collection) await settings.DB_CLIENT[collection].insert_one(model_instance.to_mongo()) return model_instance async def insert_objs(model_instances: list[BaseModel], collection: str | None = None) -> int: - """ - Insert many objects at once. - """ if not model_instances: return 0 collection = resolve_collection(type(model_instances[0]), collection) @@ -239,9 +184,6 @@ async def update_obj( collection: str | None = None, exception: bool = True, ) -> T | None: - """ - Update an object by ID and return the updated document. - """ filter_ = {"id": obj_id} if additional_filter: filter_.update(additional_filter) @@ -261,16 +203,6 @@ async def upsert_objs( key_fields: list[str], collection: str | None = None, ) -> tuple[int, int]: - """ - Idempotently upsert reference-data documents. - - The natural key (``key_fields``) decides identity, not the generated ``id``: - re-running an import must update in place rather than duplicate. ``id`` and - ``datetimeInserted`` are only written when the document is first created, so - existing references stay stable across re-imports. - - Returns a (matched, upserted) tuple. - """ if not documents: return 0, 0 collection = resolve_collection(model, collection) @@ -293,9 +225,6 @@ async def delete_obj( additional_filter: dict | None = None, collection: str | None = None, ) -> T: - """ - Delete an object. - """ filter_ = {"id": obj_id} if obj_id else {} if additional_filter: filter_.update(additional_filter) @@ -310,89 +239,7 @@ async def delete_obj( async def delete_objs( model: T, additional_filter: dict | None = None, collection: str | None = None ) -> int: - """ - Delete every object matching a filter. Used by full-replacement imports. - """ result = await settings.DB_CLIENT[resolve_collection(model, collection)].delete_many( additional_filter or {} ) return result.deleted_count - - -async def search_objs( - model: T, - data: SearchRequest, - additional_filter: dict | None = None, - projection: dict | None = None, - collection: str | None = None, -) -> tuple[list[T], int]: - """ - Search for objects in a specified collection based on search filters. - """ - filters = [] - date_filters: dict[str, list[dict]] = {} - - for search_filter in data.filter: - if isinstance(search_filter.value, str): - date_match = re.fullmatch(r"^(\d{4}-\d{2}-\d{2});([+-]\d{1,2})$", search_filter.value) - - if date_match: - date_filters.setdefault(search_filter.name, []).append( - { - "date": datetime.strptime(date_match.group(1), "%Y-%m-%d"), - "timezone_offset": int(date_match.group(2)), - } - ) - else: - filters.append( - { - search_filter.name: { - "$regex": re.escape(search_filter.value), - "$options": "i", - } - } - ) - else: - filters.append({search_filter.name: search_filter.value}) - - for field_name, dates in date_filters.items(): - if len(dates) == 1: - date_info = dates[0] - user_local_day_start = date_info["date"] - user_local_day_end = user_local_day_start + timedelta(days=1) - filters.append( - { - field_name: { - "$gte": ( - user_local_day_start - timedelta(hours=date_info["timezone_offset"]) - ).isoformat(), - "$lt": ( - user_local_day_end - timedelta(hours=date_info["timezone_offset"]) - ).isoformat(), - } - } - ) - elif len(dates) >= 2: - start_date = min(dates, key=lambda x: x["date"]) - end_date = max(dates, key=lambda x: x["date"]) - - start_datetime = start_date["date"] - timedelta(hours=start_date["timezone_offset"]) - end_datetime = ( - end_date["date"] + timedelta(days=1) - timedelta(hours=end_date["timezone_offset"]) - ) - - filters.append( - { - field_name: { - "$gte": start_datetime.isoformat(), - "$lt": end_datetime.isoformat(), - } - } - ) - - if additional_filter: - filters.append(additional_filter) - regex_filter = {"$and": filters} if filters else {} - return await _paginate_objs( - model, regex_filter, data.pageSize, data.pageIndex, ("id", -1), projection, collection - ) diff --git a/cbh/api/common/dto.py b/cbh/api/common/dto.py index 296603e91391621d0bb891016fe7177c89e44b7e..946dea8c04c5b6c13ef84ee2f658843504d2324b 100644 --- a/cbh/api/common/dto.py +++ b/cbh/api/common/dto.py @@ -1,113 +1,105 @@ -""" -Common DTOs shared across CargoFlow modules. -""" - -from datetime import datetime -from enum import Enum +from enum import IntEnum from pydantic import BaseModel -class OrderType(Enum): - """ - Sort order. - """ - - ASC = 1 - DESC = 2 - - -class SortBy(BaseModel): - """ - Sorting instruction for list endpoints. - """ - - name: str - order: OrderType = OrderType.DESC - - def to_mongo_sort(self) -> tuple[str, int]: - """ - Convert to a MongoDB sort tuple. - """ - return self.name, 1 if self.order == OrderType.ASC else -1 - - class Paging(BaseModel): - """ - Pagination model for API responses. - """ - pageSize: int pageIndex: int totalCount: int -class SearchFilter(BaseModel): - """ - Search filter model for constructing database queries. - """ +class CustomsProcedure(IntEnum): + """The customs procedure the declaration is filed under. - name: str - value: str | int - - -class IDName(BaseModel): - """ - Generic id/name pair. - """ - - id: str | int - name: str - - -class DateValue(BaseModel): - """ - Date/value pair for time series responses. + Direction says which way the goods move; this says what is being asked of + customs when they move. The difference matters because under transit the + goods are not released, so duty and tax are secured rather than paid, while + prohibitions and restrictions bind exactly as they would on release. """ - date: datetime - value: int + RELEASE_FOR_FREE_CIRCULATION = 1 + EXTERNAL_TRANSIT_T1 = 2 + INTERNAL_TRANSIT_T2 = 3 + CUSTOMS_WAREHOUSE = 4 + @property + def is_transit(self) -> bool: + return self in ( + CustomsProcedure.EXTERNAL_TRANSIT_T1, + CustomsProcedure.INTERNAL_TRANSIT_T2, + ) -class TradeDirection(Enum): - """ - Direction of the declaration line. - - A measure may apply to imports only, exports only, or both. - """ + @property + def suspends_duty(self) -> bool: + """Whether duty, tax and quota are secured now and settled elsewhere.""" + return self is not CustomsProcedure.RELEASE_FOR_FREE_CIRCULATION + @classmethod + def parse( + cls, value: "str | int | CustomsProcedure | None" + ) -> "CustomsProcedure | None": + if value is None or isinstance(value, CustomsProcedure): + return value + if isinstance(value, int): + return cls(value) + return { + "t1": cls.EXTERNAL_TRANSIT_T1, + "t-1": cls.EXTERNAL_TRANSIT_T1, + "extern douanevervoer": cls.EXTERNAL_TRANSIT_T1, + "external transit": cls.EXTERNAL_TRANSIT_T1, + "t2": cls.INTERNAL_TRANSIT_T2, + "t-2": cls.INTERNAL_TRANSIT_T2, + "intern douanevervoer": cls.INTERNAL_TRANSIT_T2, + "internal transit": cls.INTERNAL_TRANSIT_T2, + "free circulation": cls.RELEASE_FOR_FREE_CIRCULATION, + "release for free circulation": cls.RELEASE_FOR_FREE_CIRCULATION, + "in het vrije verkeer brengen": cls.RELEASE_FOR_FREE_CIRCULATION, + "40": cls.RELEASE_FOR_FREE_CIRCULATION, + "4000": cls.RELEASE_FOR_FREE_CIRCULATION, + "42": cls.RELEASE_FOR_FREE_CIRCULATION, + "4200": cls.RELEASE_FOR_FREE_CIRCULATION, + "warehouse": cls.CUSTOMS_WAREHOUSE, + "entrepot": cls.CUSTOMS_WAREHOUSE, + "douane-entrepot": cls.CUSTOMS_WAREHOUSE, + "71": cls.CUSTOMS_WAREHOUSE, + "7100": cls.CUSTOMS_WAREHOUSE, + }.get(str(value).strip().lower()) + + +class TradeDirection(IntEnum): IMPORT = 1 EXPORT = 2 BOTH = 3 + WAREHOUSE = 4 @classmethod def parse(cls, value: "str | int | TradeDirection | None") -> "TradeDirection | None": - """ - Parse a direction from the loose representations found in client files. - """ if value is None or isinstance(value, TradeDirection): return value if isinstance(value, int): return cls(value) - normalized = str(value).strip().lower() - mapping = { + return { "import": cls.IMPORT, "invoer": cls.IMPORT, "in": cls.IMPORT, "i": cls.IMPORT, "1": cls.IMPORT, + "40": cls.IMPORT, + "4000": cls.IMPORT, "export": cls.EXPORT, "uitvoer": cls.EXPORT, "out": cls.EXPORT, "e": cls.EXPORT, "2": cls.EXPORT, + "10": cls.EXPORT, + "1000": cls.EXPORT, "both": cls.BOTH, "3": cls.BOTH, - } - return mapping.get(normalized) - - def matches(self, other: "TradeDirection") -> bool: - """ - Check whether a measure direction applies to a requested direction. - """ - return self is TradeDirection.BOTH or other is TradeDirection.BOTH or self is other + "warehouse": cls.WAREHOUSE, + "entrepot": cls.WAREHOUSE, + "douane-entrepot": cls.WAREHOUSE, + "douane entrepot": cls.WAREHOUSE, + "71": cls.WAREHOUSE, + "7100": cls.WAREHOUSE, + }.get(str(value).strip().lower()) diff --git a/cbh/api/common/models.py b/cbh/api/common/models.py index cde2de0951e4ed986e5eeceea3f86b5f3a295199..f1d63b03c2f65f27e1e55005f58b468122e35601 100644 --- a/cbh/api/common/models.py +++ b/cbh/api/common/models.py @@ -1,7 +1,3 @@ -""" -Common models shared across CargoFlow modules. -""" - from datetime import date, datetime, timedelta from pydantic import BaseModel, Field @@ -10,41 +6,16 @@ from cbh.core.database import MongoBaseModel, MongoSerializableModel class ValidityMixin(BaseModel): - """ - Validity period carried by every piece of reference data. - - History is never deleted: each record keeps the window it was in force for, - so a check can always be replayed with the rules of the day a declaration - was filed. ``validTo = None`` means "open ended". - """ - validFrom: date | None = None validTo: date | None = None - def is_valid_on(self, on_date: date) -> bool: - """ - Check whether this record is in force on a given date. - """ - if self.validFrom and on_date < self.validFrom: - return False - if self.validTo and on_date > self.validTo: - return False - return True - def days_until_expiry(self, on_date: date) -> int | None: - """ - Days left until this record expires, or None when open ended. - """ if self.validTo is None: return None return (self.validTo - on_date).days class FileRef(MongoSerializableModel): - """ - Reference to a file kept in local storage. - """ - fileName: str storagePath: str contentType: str | None = None @@ -53,13 +24,6 @@ class FileRef(MongoSerializableModel): class CacheEntryModel(MongoBaseModel): - """ - Generic TTL cache entry. - - Used by the line-check engine so that an identical (code, country, - direction, date) tuple is resolved once and reused across a batch. - """ - namespace: str key: str value: dict | list | str | int | float | bool | None = None diff --git a/cbh/api/common/schemas.py b/cbh/api/common/schemas.py index c336baaf665031348de2c41754bd91373aca0495..39b05f2138c07096dd8ed2c7e24356f0aefa2934 100644 --- a/cbh/api/common/schemas.py +++ b/cbh/api/common/schemas.py @@ -1,81 +1,12 @@ -""" -Common schemas. -""" - from typing import Generic, TypeVar from pydantic import BaseModel -from cbh.api.common.dto import Paging, SearchFilter, SortBy +from cbh.api.common.dto import Paging T = TypeVar("T", bound=BaseModel) class AllObjectsResponse(BaseModel, Generic[T]): - """ - Response model for paginated collections. - """ - paging: Paging data: list[T] - - -class SearchRequest(BaseModel): - """ - Request schema for searching objects. - """ - - filter: list[SearchFilter] - pageSize: int = 10 - pageIndex: int = 0 - - -class FilterRequest(BaseModel, Generic[T]): - """ - Filter request. - """ - - filter: T - sortBy: SortBy | None = None - pageSize: int = 10 - pageIndex: int = 0 - - -class PlainTextResponse(BaseModel): - """ - Response model for plain text. - """ - - text: str - - -class ContentRequest(BaseModel): - """ - Content request. - """ - - content: str | None = None - - -class BatchIdsRequest(BaseModel): - """ - Batch ids request. - """ - - ids: list[str] - - -class EmailRequest(BaseModel): - """ - Email request. - """ - - email: str - - -class CountResponse(BaseModel): - """ - Generic count response, used by importer and reference data endpoints. - """ - - count: int diff --git a/cbh/api/common/services/cache/__init__.py b/cbh/api/common/services/cache/__init__.py index dad4118ec8ac5b95d0b8425853cf0093a9b6bcfc..0a9c4bb7b1cc898a30dc5b99552d67c2d8dd94e1 100644 --- a/cbh/api/common/services/cache/__init__.py +++ b/cbh/api/common/services/cache/__init__.py @@ -1,19 +1,3 @@ -from .cache import ( - LINE_CHECK_NAMESPACE, - build_cache_key, - cached_payload, - get_cache, - invalidate_cache, - invalidate_line_check_cache, - set_cache, -) +from .cache import build_cache_key, get_cache, invalidate_line_check_cache, set_cache -__all__ = [ - "LINE_CHECK_NAMESPACE", - "build_cache_key", - "cached_payload", - "get_cache", - "invalidate_cache", - "invalidate_line_check_cache", - "set_cache", -] +__all__ = ["build_cache_key", "get_cache", "invalidate_line_check_cache", "set_cache"] diff --git a/cbh/api/common/services/cache/cache.py b/cbh/api/common/services/cache/cache.py index 6b24c5c3f0bc653f5e0032fa71ebdf1abd181b43..8ac549622454209760ce951f6554a519b9f1bc7f 100644 --- a/cbh/api/common/services/cache/cache.py +++ b/cbh/api/common/services/cache/cache.py @@ -1,16 +1,4 @@ -""" -Generic TTL cache backed by MongoDB. - -The line-check engine resolves each unique (code, country, direction, date) -tuple once; a pack of ~580 declaration lines typically collapses to a few dozen -distinct tuples, which is where most of the runtime saving comes from. - -The cache is keyed on the reference-data state as well, so an importer run -invalidates it rather than serving answers built on superseded rules. -""" - import hashlib -import json from datetime import datetime, timedelta from typing import Any @@ -20,21 +8,13 @@ from cbh.api.common.models import CacheEntryModel from cbh.core.config import settings from cbh.core.wrappers import background_task -LINE_CHECK_NAMESPACE = "linecheck" - def build_cache_key(*parts: Any) -> str: - """ - Build a stable cache key from its parts. - """ raw = "|".join("" if part is None else str(part) for part in parts) return hashlib.sha256(raw.encode("utf-8")).hexdigest() def _serialize(value: Any) -> Any: - """ - Serialize a value into something BSON can store. - """ if isinstance(value, BaseModel): return value.model_dump(mode="json") if isinstance(value, list): @@ -45,9 +25,6 @@ def _serialize(value: Any) -> Any: async def get_cache(namespace: str, key: str) -> Any | None: - """ - Read a cache entry, or None when it is absent or expired. - """ entry = await settings.DB_CLIENT.caches.find_one({"namespace": namespace, "key": key}) if not entry: return None @@ -63,16 +40,12 @@ async def set_cache( value: Any, ttl_seconds: int | None = None, ) -> None: - """ - Write a cache entry, replacing any previous value for the same key. - """ ttl = ttl_seconds if ttl_seconds is not None else settings.LINE_CHECK_CACHE_TTL_SECONDS - entry = CacheEntryModel( + document = CacheEntryModel( namespace=namespace, key=key, value=_serialize(value), - ) - document = entry.to_mongo() + ).to_mongo() document["deletedAt"] = datetime.now() + timedelta(seconds=ttl) await settings.DB_CLIENT.caches.update_one( {"namespace": namespace, "key": key}, @@ -81,30 +54,6 @@ async def set_cache( ) -@background_task() -async def invalidate_cache(namespace: str, key: str | None = None) -> None: - """ - Drop a single cache entry, or an entire namespace. - - Called after every successful reference-data import: answers computed from - superseded rules must never be replayed. - """ - filter_: dict[str, Any] = {"namespace": namespace} - if key: - filter_["key"] = key - await settings.DB_CLIENT.caches.delete_many(filter_) - - @background_task() async def invalidate_line_check_cache() -> None: - """ - Invalidate every cached line-check result. - """ - await settings.DB_CLIENT.caches.delete_many({"namespace": LINE_CHECK_NAMESPACE}) - - -def cached_payload(value: Any) -> str: - """ - Compact JSON rendering used for cache-key fingerprints. - """ - return json.dumps(value, sort_keys=True, default=str, separators=(",", ":")) + await settings.DB_CLIENT.caches.delete_many({"namespace": "linecheck"}) diff --git a/cbh/api/common/utils.py b/cbh/api/common/utils.py index b91b7f08fd0eaf1e7fb6847dfd7e50fe784a5acb..849162ef88639cef4f9760190ea6d37609f5eecf 100644 --- a/cbh/api/common/utils.py +++ b/cbh/api/common/utils.py @@ -1,56 +1,11 @@ -""" -Common utilities. -""" - import re from datetime import date, datetime -from typing import Any, Iterable, Iterator, TypeVar - -from cbh.api.common.dto import Paging +from typing import Any, Iterable, TypeVar T = TypeVar("T") -_DATE_FORMATS = ( - "%Y-%m-%d", - "%d-%m-%Y", - "%d/%m/%Y", - "%Y/%m/%d", - "%d.%m.%Y", - "%Y%m%d", -) - - -def chunked(items: list[T], size: int) -> Iterator[list[T]]: - """ - Split a list into chunks of at most ``size`` elements. - """ - for start in range(0, len(items), size): - yield items[start : start + size] - - -def to_paging(page_size: int, page_index: int, total_count: int) -> Paging: - """ - Build a Paging object for a list response. - """ - return Paging(pageSize=page_size, pageIndex=page_index, totalCount=total_count) - - -def to_iso(value: date | datetime | None) -> str | None: - """ - Serialize a date for MongoDB comparisons. - - Dates are stored as ISO strings by ``MongoSerializableModel.to_mongo``; - ISO-8601 sorts lexicographically, so range queries stay correct. - """ - if value is None: - return None - return value.isoformat() - def parse_date(value: Any) -> date | None: - """ - Parse a date coming from a client spreadsheet or a reference-data file. - """ if value is None or value == "": return None if isinstance(value, datetime): @@ -60,7 +15,7 @@ def parse_date(value: Any) -> date | None: text = str(value).strip() if not text: return None - for date_format in _DATE_FORMATS: + for date_format in ("%Y-%m-%d", "%d-%m-%Y", "%d/%m/%Y", "%Y/%m/%d", "%d.%m.%Y", "%Y%m%d"): try: return datetime.strptime(text, date_format).date() except ValueError: @@ -72,9 +27,6 @@ def parse_date(value: Any) -> date | None: def normalize_ws(value: Any) -> str | None: - """ - Collapse whitespace in a free-text cell. - """ if value is None: return None text = re.sub(r"\s+", " ", str(value)).strip() @@ -82,14 +34,11 @@ def normalize_ws(value: Any) -> str | None: def parse_number(value: Any) -> float | None: - """ - Parse a numeric cell tolerating both comma and dot decimal separators. - """ if value is None or value == "": return None if isinstance(value, (int, float)): return float(value) - text = str(value).strip().replace(" ", "").replace(" ", "") + text = str(value).strip().replace(" ", "").replace(" ", "") if not text: return None if "," in text and "." in text: @@ -104,9 +53,6 @@ def parse_number(value: Any) -> float | None: def normalize_country(value: Any) -> str | None: - """ - Normalize an ISO country code to its uppercase two-letter form. - """ if value is None: return None text = re.sub(r"[^A-Za-z]", "", str(value)).upper() @@ -116,9 +62,6 @@ def normalize_country(value: Any) -> str | None: def dedupe(items: Iterable[T]) -> list[T]: - """ - Remove duplicates while preserving order. - """ seen: set[T] = set() result: list[T] = [] for item in items: diff --git a/cbh/api/importer/__init__.py b/cbh/api/importer/__init__.py index e2aab6a4f14f341e0eb11b01226282a28eae5e77..be82ece2628438b3f4eef603aa87fd881325a058 100644 --- a/cbh/api/importer/__init__.py +++ b/cbh/api/importer/__init__.py @@ -1,10 +1,3 @@ -""" -Importer module initialization. - -This module defines the FastAPI router for the reference-data importer: -triggering loads, inspecting the sync log and supplying files by hand. -""" - from fastapi import APIRouter importer_router = APIRouter( diff --git a/cbh/api/importer/db_requests.py b/cbh/api/importer/db_requests.py index 5717d9c16e0bf0d1b582014b90c49248dd662962..d0961a9689b867b070b0685be0f947e7a7af5915 100644 --- a/cbh/api/importer/db_requests.py +++ b/cbh/api/importer/db_requests.py @@ -1,7 +1,3 @@ -""" -Importer database requests. -""" - from cbh.api.common.db_requests import find_objs, get_all_objs from cbh.api.importer.dto import SyncSource, SyncStatus from cbh.api.importer.models import SyncLogModel @@ -13,9 +9,6 @@ async def get_sync_logs( source: SyncSource | None = None, status: SyncStatus | None = None, ) -> tuple[list[SyncLogModel], int]: - """ - Page through the import history, newest first. - """ filter_: dict = {} if source: filter_["source"] = source.value @@ -31,12 +24,6 @@ async def get_sync_logs( async def get_latest_sync_per_source() -> dict[str, SyncLogModel]: - """ - The most recent run for each source. - - This is the operational view: which reference files are loaded, how fresh - they are, and which ones last failed. - """ latest: dict[str, SyncLogModel] = {} for source in SyncSource: runs = await find_objs( @@ -48,16 +35,3 @@ async def get_latest_sync_per_source() -> dict[str, SyncLogModel]: if runs: latest[source.name] = runs[0] return latest - - -async def get_last_successful_sync(source: SyncSource) -> SyncLogModel | None: - """ - The most recent successful run for one source. - """ - runs = await find_objs( - SyncLogModel, - {"source": source.value, "status": SyncStatus.SUCCESS.value}, - limit=1, - sort=("finishedAt", -1), - ) - return runs[0] if runs else None diff --git a/cbh/api/importer/dto.py b/cbh/api/importer/dto.py index a2dfdb25ff9fc959e2ef89f300ccccb6ba070a94..09d98b5e86db9806fa48d20d01bb81e6fb3487c3 100644 --- a/cbh/api/importer/dto.py +++ b/cbh/api/importer/dto.py @@ -1,18 +1,7 @@ -""" -Importer DTOs. -""" +from enum import IntEnum -from enum import Enum - - -class SyncSource(Enum): - """ - A reference-data file the importer knows how to load. - - TARIC is the source of truth for measures and document requirements; - Codeboek Douane supplies the Dutch national code lists. - """ +class SyncSource(IntEnum): TARIC_NOMENCLATURE = 1 TARIC_DECLARABLE_CODES = 2 TARIC_DUTIES_IMPORT = 3 @@ -23,38 +12,16 @@ class SyncSource(Enum): TARIC_BOX44_CODES = 8 TARIC_CODE_CHANGES = 9 CODEBOEK_DOUANE = 10 - - @property - def is_taric(self) -> bool: - """ - Whether this source comes from the TARIC/CIRCABC distribution. - """ - return self is not SyncSource.CODEBOEK_DOUANE + DTV_NATIONAL = 11 -class SyncMode(Enum): - """ - How a source is loaded. - - TARIC nomenclature arrives as a monthly full load plus daily deltas; - Codeboek Douane is republished whole on every change, so it is replaced - rather than merged. - """ - +class SyncMode(IntEnum): FULL = 1 DELTA = 2 REPLACE = 3 -class SyncStatus(Enum): - """ - Lifecycle of a single import run. - - A run left in ``RUNNING`` by a crash is safe to start again: imports are - keyed on the natural key of each record, so re-running only rewrites what - it would have written the first time. - """ - +class SyncStatus(IntEnum): PENDING = 1 RUNNING = 2 SUCCESS = 3 diff --git a/cbh/api/importer/models.py b/cbh/api/importer/models.py index 803995988d746c271d8cd508a2bb91151cd52664..69430244a11917fbcc9c925ae431320f2dad4446 100644 --- a/cbh/api/importer/models.py +++ b/cbh/api/importer/models.py @@ -1,7 +1,3 @@ -""" -Importer models. -""" - from datetime import datetime from pydantic import Field @@ -11,14 +7,6 @@ from cbh.core.database import MongoBaseModel class SyncLogModel(MongoBaseModel): - """ - One import run. - - The log is the audit trail behind every answer the engine gives: it records - which file was loaded, when, and what it changed. ``fileHash`` makes a - repeated load of an unchanged file cheap to detect and skip. - """ - source: SyncSource mode: SyncMode status: SyncStatus = SyncStatus.PENDING @@ -41,10 +29,3 @@ class SyncLogModel(MongoBaseModel): durationSeconds: float | None = None datetimeInserted: datetime = Field(default_factory=datetime.now) - - @property - def succeeded(self) -> bool: - """ - Whether this run can be relied on as a reference-data version. - """ - return self.status is SyncStatus.SUCCESS diff --git a/cbh/api/importer/schemas.py b/cbh/api/importer/schemas.py index 23e1af4f743793c5e8da258f44aad60b74fbe44d..a2b5f866bf1c78dba7ab09efabc8fcd3b54033df 100644 --- a/cbh/api/importer/schemas.py +++ b/cbh/api/importer/schemas.py @@ -1,7 +1,3 @@ -""" -Importer schemas. -""" - from datetime import date, datetime from pydantic import BaseModel @@ -10,10 +6,6 @@ from cbh.api.importer.dto import SyncMode, SyncSource, SyncStatus class SyncLogResponse(BaseModel): - """ - One import run as returned by the API. - """ - id: str source: SyncSource mode: SyncMode @@ -38,32 +30,34 @@ class SyncLogResponse(BaseModel): class SyncRequest(BaseModel): - """ - Request to import one source. - - With no ``url``, the importer reads the most recent file supplied for the - source in local storage. - """ - source: SyncSource mode: SyncMode = SyncMode.FULL url: str | None = None class CodebookSyncRequest(BaseModel): - """ - Request to refresh one or all Dutch codebooks. - """ - fileName: str | None = None publishedOn: date | None = None -class ImporterStatusResponse(BaseModel): - """ - Operational view of the reference data. - """ +class PublicationSyncRequest(BaseModel): + mode: SyncMode = SyncMode.FULL + period: str | None = None + keepArchive: bool = False + +class PublicationSyncResponse(BaseModel): + period: str | None = None + url: str | None = None + runs: list[SyncLogResponse] = [] + missingSources: list[SyncSource] = [] + note: str | None = None + + +class ImporterStatusResponse(BaseModel): latestPerSource: dict[str, SyncLogResponse] counts: dict[str, int] referenceAsOf: str | None = None + publicationPeriods: list[str] = [] + latestPublication: str | None = None + scheduledJobs: dict[str, datetime | None] = {} diff --git a/cbh/api/importer/services/base.py b/cbh/api/importer/services/base.py index e7745cacccd2d4d1016811035c6fb6649b315546..4e82a2757e875178f069f1957e0468f5c4978908 100644 --- a/cbh/api/importer/services/base.py +++ b/cbh/api/importer/services/base.py @@ -1,14 +1,6 @@ -""" -Import run scaffolding: the sync log, and the idempotent write path. - -Every import runs inside a :class:`SyncRun`, which opens a log entry, records -what the run touched and closes it as success or failure. Runs are safe to -repeat: records are written under their natural key, so a re-run rewrites -exactly what it wrote before instead of duplicating it. -""" - import time from datetime import datetime +from pathlib import Path from typing import Any import httpx @@ -22,10 +14,6 @@ from cbh.core.config import settings class SyncRun: - """ - Async context manager wrapping one import run. - """ - def __init__( self, source: SyncSource, @@ -64,22 +52,14 @@ class SyncRun: ) if self.log.status in (SyncStatus.SUCCESS, SyncStatus.PARTIAL): - # Cached answers were computed against the previous reference data. await invalidate_line_check_cache() - # Never swallow the exception: a failed import must surface. return False def record_error(self, message: str) -> None: - """ - Note a recoverable problem; the run finishes as PARTIAL. - """ self.log.errors.append(message) def set_hash(self, payload: bytes) -> None: - """ - Store the content hash of the file this run loaded. - """ self.log.fileHash = file_hash(payload) async def write( @@ -88,9 +68,6 @@ class SyncRun: documents: list[dict], key_fields: list[str], ) -> None: - """ - Upsert a batch of parsed records and account for them in the log. - """ self.log.recordsRead += len(documents) matched, upserted = await upsert_objs(model, documents, key_fields) self.log.recordsMatched += matched @@ -98,9 +75,6 @@ class SyncRun: async def download(url: str) -> bytes: - """ - Fetch a reference file over HTTP. - """ async with httpx.AsyncClient( timeout=settings.IMPORTER_HTTP_TIMEOUT, follow_redirects=True ) as client: @@ -109,16 +83,43 @@ async def download(url: str) -> bytes: return response.content -async def already_loaded(source: SyncSource, payload: bytes) -> bool: - """ - Whether this exact file has already been loaded successfully. +async def download_to(url: str, path: Path) -> Path: + """Stream a download to disk. The monthly extraction is ~80 MB zipped.""" + path.parent.mkdir(parents=True, exist_ok=True) + async with httpx.AsyncClient( + timeout=settings.IMPORTER_HTTP_TIMEOUT, follow_redirects=True + ) as client: + async with client.stream("GET", url) as response: + response.raise_for_status() + with path.open("wb") as handle: + async for chunk in response.aiter_bytes(1 << 20): + handle.write(chunk) + return path + - Lets a daily job run unconditionally and do nothing when upstream has not - republished. - """ +async def url_exists(url: str) -> bool: + async with httpx.AsyncClient( + timeout=settings.IMPORTER_HTTP_TIMEOUT, follow_redirects=True + ) as client: + try: + response = await client.head(url) + except httpx.HTTPError: + return False + return response.status_code == 200 + + +async def latest_loaded_hash(source: SyncSource) -> str | None: latest = await settings.DB_CLIENT.synclogs.find_one( - {"source": source.value, "status": SyncStatus.SUCCESS.value}, + { + "source": source.value, + "status": SyncStatus.SUCCESS.value, + "fileHash": {"$ne": None}, + }, sort=[("finishedAt", -1)], projection={"_id": 0, "fileHash": 1}, ) - return bool(latest) and latest.get("fileHash") == file_hash(payload) + return latest.get("fileHash") if latest else None + + +async def already_loaded(source: SyncSource, payload: bytes) -> bool: + return await latest_loaded_hash(source) == file_hash(payload) diff --git a/cbh/api/importer/services/codeboek/codeboek.py b/cbh/api/importer/services/codeboek/codeboek.py index a2a5a6c36fd850915ee8040abda6218fdf590ec7..324d7fd914399184d43c9691c52b039a01c66c3f 100644 --- a/cbh/api/importer/services/codeboek/codeboek.py +++ b/cbh/api/importer/services/codeboek/codeboek.py @@ -1,22 +1,10 @@ -""" -The Codeboek Douane importer. - -The Dutch national code lists are republished whole on every change, at -predictable URLs of the form ``.../codebooks/[yyyyMMdd]/[filename]``, as either -a zipped XML file or JSON. They are therefore replaced rather than merged: the -previous contents of a codebook are dropped and the new file takes its place. - -These files carry no code x country -> document mapping. That lives in the -TARIC measure conditions; the codebooks explain what a national code means. -""" - from datetime import date from typing import Any import httpx from cbh.api.common.db_requests import delete_objs, upsert_objs -from cbh.api.common.utils import normalize_ws +from cbh.api.common.utils import normalize_ws, parse_date from cbh.api.importer.dto import SyncMode, SyncSource from cbh.api.importer.services.base import SyncRun, already_loaded, download from cbh.api.importer.utils import ( @@ -27,8 +15,6 @@ from cbh.api.importer.utils import ( ) from cbh.api.taric.models import CodeboekEntryModel -# Codebooks worth loading for the chapters in scope. The key is the file name -# as published; the value is the codebook name records are filed under. CODEBOOKS: dict[str, str] = { "Bescheiden.zip": "documents", "Landen.zip": "countries", @@ -38,7 +24,7 @@ CODEBOOKS: dict[str, str] = { "Maatstaven.zip": "supplementary_units", } -_XML_FIELDS: dict[str, tuple[str, ...]] = { +XML_FIELDS: dict[str, tuple[str, ...]] = { "code": ("code", "Code", "waarde", "id"), "description": ("omschrijving", "Omschrijving", "description", "naam"), "descriptionEn": ("descriptionEN", "omschrijvingEN", "descriptionEn"), @@ -48,15 +34,10 @@ _XML_FIELDS: dict[str, tuple[str, ...]] = { def _parse_entry(codebook: str, record: dict[str, Any]) -> dict[str, Any] | None: - """ - Parse one codebook record. - """ code = normalize_ws(record.get("code")) if not code: return None - from cbh.api.common.utils import parse_date - known = {"code", "description", "descriptionEn", "validFrom", "validTo"} return { "codebook": codebook, @@ -66,22 +47,19 @@ def _parse_entry(codebook: str, record: dict[str, Any]) -> dict[str, Any] | None "validFrom": parse_date(record.get("validFrom")), "validTo": parse_date(record.get("validTo")), "extra": { - key: value for key, value in record.items() if key not in known and value not in (None, "") + key: value + for key, value in record.items() + if key not in known and value not in (None, "") }, } def _read_records(payload: bytes, file_name: str) -> list[dict[str, Any]]: - """ - Read a codebook in whichever format it was published in. - """ unwrapped = load_payload(payload, file_name) if file_name.lower().endswith(".json") or unwrapped.lstrip()[:1] in (b"{", b"["): return read_json_records(unwrapped) - # The record tag varies per codebook; every leaf element carrying a code is - # treated as a record. for tag in ("record", "row", "item", "codeItem", "regel"): - records = list(read_xml_records(unwrapped, tag, _XML_FIELDS)) + records = list(read_xml_records(unwrapped, tag, XML_FIELDS)) if records: return records return [] @@ -93,9 +71,6 @@ async def import_codebook( file_name: str, source_url: str | None = None, ) -> SyncRun: - """ - Replace one codebook with the contents of a published file. - """ async with SyncRun( SyncSource.CODEBOEK_DOUANE, SyncMode.REPLACE, @@ -120,8 +95,6 @@ async def import_codebook( continue documents.append(CodeboekEntryModel(**parsed).to_mongo()) - # Full replacement: the published file is the whole truth for this - # codebook, so anything not in it is gone. run.log.recordsDeleted = await delete_objs( CodeboekEntryModel, {"codebook": codebook} ) @@ -135,12 +108,7 @@ async def import_codebook( return run -async def sync_codebook( - file_name: str, on_date: date | None = None -) -> SyncRun: - """ - Download and replace one codebook from its published URL. - """ +async def sync_codebook(file_name: str, on_date: date | None = None) -> SyncRun: codebook = CODEBOOKS.get(file_name, file_name.rsplit(".", 1)[0].lower()) url = codeboek_url(file_name, on_date) payload = await download(url) @@ -148,12 +116,6 @@ async def sync_codebook( async def sync_all_codebooks(on_date: date | None = None) -> list[SyncRun]: - """ - Refresh every configured codebook. - - A codebook that is not published for the requested date is skipped rather - than failing the whole run: the Douane does not republish them all daily. - """ runs: list[SyncRun] = [] for file_name in CODEBOOKS: try: diff --git a/cbh/api/importer/services/demo/demo.py b/cbh/api/importer/services/demo/demo.py index bba9ba52a925b93b3e60d5033397b2a0cdcf3a5e..6885ca56c187467bdbb803fb2ce75a98f412573a 100644 --- a/cbh/api/importer/services/demo/demo.py +++ b/cbh/api/importer/services/demo/demo.py @@ -1,24 +1,3 @@ -""" -A small, realistic reference dataset for the chapters in scope. - -This exists so the whole pipeline — resolution, documents, warnings, Excel in -and out — can be exercised before the real TARIC extracts and the Goenee gold -set arrive. The shapes match what the importer produces from the published -files; only the volume is different. - -It deliberately includes the awkward cases the engine has to get right: - -* an 8-digit code with two declarable children (ambiguous); -* a code that expired last year, with a recorded successor (invalid + hint); -* a code expiring inside the 60-day horizon (expiring); -* an either/or document pair and a Y-code exemption; -* an anti-dumping measure tied to an additional code; -* a dual-use code that must be handed to a human. - -Everything here is illustrative and must be replaced by a real import before -any acceptance run. -""" - from datetime import date, timedelta from cbh.api.common.db_requests import upsert_objs @@ -41,13 +20,8 @@ from cbh.api.taric.models import ( NomenclatureModel, ) -EPOCH = date(2020, 1, 1) - def _geo_areas() -> list[GeoAreaModel]: - """ - Countries and the groups used by the demo measures. - """ countries = { "NL": "Netherlands", "DE": "Germany", @@ -60,71 +34,56 @@ def _geo_areas() -> list[GeoAreaModel]: "ZA": "South Africa", "GB": "United Kingdom", } - areas = [ + return [ GeoAreaModel( - code=code, name=name, areaType=GeoAreaType.COUNTRY, validFrom=EPOCH + code=code, name=name, areaType=GeoAreaType.COUNTRY, validFrom=date(2020, 1, 1) ) for code, name in countries.items() - ] - areas.append( + ] + [ GeoAreaModel( code="1011", name="Erga Omnes (all third countries)", areaType=GeoAreaType.GROUP, memberCodes=list(countries), - validFrom=EPOCH, - ) - ) - areas.append( + validFrom=date(2020, 1, 1), + ), GeoAreaModel( code="1013", name="European Union", areaType=GeoAreaType.GROUP, memberCodes=["NL", "DE"], - validFrom=EPOCH, - ) - ) - areas.append( + validFrom=date(2020, 1, 1), + ), GeoAreaModel( code="2005", name="GSP beneficiary countries", areaType=GeoAreaType.GROUP, memberCodes=["BD", "IN"], - validFrom=EPOCH, - ) - ) - return areas + validFrom=date(2020, 1, 1), + ), + ] def _nomenclature(today: date) -> list[NomenclatureModel]: - """ - A slice of the nomenclature across the chapters in scope. - """ - soon = today + timedelta(days=30) - entries: list[tuple[str, str, str, bool, date, date | None]] = [ - # --- chapter 61/62: clothing (G-Star export) --- - ("6109", "T-shirts, singlets and other vests, knitted", "T-shirts en onderhemden", False, EPOCH, None), - ("61091000", "Of cotton", "Van katoen", False, EPOCH, None), - ("6109100010", "T-shirts of cotton, men's or boys'", "T-shirts van katoen, heren", True, EPOCH, None), - ("6109100090", "T-shirts of cotton, other", "T-shirts van katoen, overige", True, EPOCH, None), - ("62034231", "Men's trousers of denim", "Herenbroeken van denim", False, EPOCH, None), - ("6203423110", "Men's trousers of denim, workwear", "Herenbroeken van denim, werkkleding", True, EPOCH, None), - # --- chapter 84/85: IT/AV (EFG Arabia -> Henx, T1) --- - ("8471", "Automatic data-processing machines", "Automatische gegevensverwerkende machines", False, EPOCH, None), - ("84713000", "Portable computers weighing <= 10 kg", "Draagbare computers", False, EPOCH, None), - ("8471300000", "Portable automatic data-processing machines", "Draagbare computers", True, EPOCH, None), - ("8517130000", "Smartphones", "Smartphones", True, EPOCH, None), - ("8532241000", "Ceramic multilayer capacitors", "Keramische meerlaagse condensatoren", True, EPOCH, None), - ("8543709099", "Other electrical machines, n.e.s.", "Andere elektrische machines", True, EPOCH, None), - # Expires inside the warning horizon: the 1 January switch, seen early. - ("8528721010", "Colour television sets, legacy subheading", "Kleurentelevisies, oude onderverdeling", True, EPOCH, soon), - # --- chapter 22: wine --- - ("22042110", "Wine in bottles of 2 litres or less", "Wijn in flessen van 2 liter of minder", False, EPOCH, None), - ("2204211010", "Quality wine in bottles of 2 litres or less", "Kwaliteitswijn in flessen", True, EPOCH, None), + ("6109", "T-shirts, singlets and other vests, knitted", "T-shirts en onderhemden", False, date(2020, 1, 1), None), + ("61091000", "Of cotton", "Van katoen", False, date(2020, 1, 1), None), + ("6109100010", "T-shirts of cotton, men's or boys'", "T-shirts van katoen, heren", True, date(2020, 1, 1), None), + ("6109100090", "T-shirts of cotton, other", "T-shirts van katoen, overige", True, date(2020, 1, 1), None), + ("62034231", "Men's trousers of denim", "Herenbroeken van denim", False, date(2020, 1, 1), None), + ("6203423110", "Men's trousers of denim, workwear", "Herenbroeken van denim, werkkleding", True, date(2020, 1, 1), None), + ("8471", "Automatic data-processing machines", "Automatische gegevensverwerkende machines", False, date(2020, 1, 1), None), + ("84713000", "Portable computers weighing <= 10 kg", "Draagbare computers", False, date(2020, 1, 1), None), + ("8471300000", "Portable automatic data-processing machines", "Draagbare computers", True, date(2020, 1, 1), None), + ("8517130000", "Smartphones", "Smartphones", True, date(2020, 1, 1), None), + ("8532241000", "Ceramic multilayer capacitors", "Keramische meerlaagse condensatoren", True, date(2020, 1, 1), None), + ("8543709099", "Other electrical machines, n.e.s.", "Andere elektrische machines", True, date(2020, 1, 1), None), + ("8528721010", "Colour television sets, legacy subheading", "Kleurentelevisies, oude onderverdeling", True, date(2020, 1, 1), today + timedelta(days=30)), + ("22042110", "Wine in bottles of 2 litres or less", "Wijn in flessen van 2 liter of minder", False, date(2020, 1, 1), None), + ("2204211010", "Quality wine in bottles of 2 litres or less", "Kwaliteitswijn in flessen", True, date(2020, 1, 1), None), ] - models = [ + return [ NomenclatureModel( code=code if len(code) == 10 else code.ljust(10, "0"), descriptionEn=description_en, @@ -135,27 +94,19 @@ def _nomenclature(today: date) -> list[NomenclatureModel]: validTo=valid_to, ) for code, description_en, description_nl, declarable, valid_from, valid_to in entries - ] - - # A code withdrawn at a previous 1 January, kept so the engine can tell - # "never existed" from "no longer valid". - models.append( + ] + [ NomenclatureModel( code="8528721090", descriptionEn="Colour television sets, withdrawn subheading", descriptionNl="Kleurentelevisies, vervallen onderverdeling", declarable=True, - validFrom=EPOCH, + validFrom=date(2020, 1, 1), validTo=date(today.year - 1, 12, 31), ) - ) - return models + ] def _document_codes() -> list[DocumentCodeModel]: - """ - The Box 44 codes the demo measures point at. - """ entries = [ ("C015", "V I 1 document (wine import)", "V I 1-document (wijninvoer)"), ("C400", "CITES certificate", "CITES-certificaat"), @@ -172,40 +123,36 @@ def _document_codes() -> list[DocumentCodeModel]: ] return [ DocumentCodeModel( - code=code, description=description, descriptionNl=description_nl, validFrom=EPOCH + code=code, + description=description, + descriptionNl=description_nl, + validFrom=date(2020, 1, 1), ) for code, description, description_nl in entries ] def _additional_codes() -> list[AdditionalCodeModel]: - """ - Additional codes used by the demo anti-dumping measure. - """ return [ AdditionalCodeModel( codeType="8", code="B999", fullCode="8B999", description="All other producers (residual anti-dumping rate)", - validFrom=EPOCH, + validFrom=date(2020, 1, 1), ), AdditionalCodeModel( codeType="8", code="B001", fullCode="8B001", description="Named cooperating producer (reduced rate)", - validFrom=EPOCH, + validFrom=date(2020, 1, 1), ), ] def _measures() -> list[MeasureModel]: - """ - Measures covering the interesting paths through the engine. - """ return [ - # Wine import: either a V I 1 document or the exemption code. MeasureModel( measureSid="DEMO-2204-IMP", goodsCode="22042110", @@ -215,7 +162,7 @@ def _measures() -> list[MeasureModel]: direction=TradeDirection.IMPORT, geoAreaCode="1011", geoAreaDescription="Erga Omnes", - validFrom=EPOCH, + validFrom=date(2020, 1, 1), conditions=[ MeasureConditionEmbedded( conditionCode="B", @@ -235,7 +182,6 @@ def _measures() -> list[MeasureModel]: ), ], ), - # Wine is excise goods: information only, no document. MeasureModel( measureSid="DEMO-2204-EXC", goodsCode="22042110", @@ -244,10 +190,8 @@ def _measures() -> list[MeasureModel]: category=MeasureCategory.EXCISE, direction=TradeDirection.IMPORT, geoAreaCode="1011", - validFrom=EPOCH, + validFrom=date(2020, 1, 1), ), - # Anti-dumping tied to an additional code: the rate depends on the - # producer, so the line has to carry one. MeasureModel( measureSid="DEMO-8532-ADD", goodsCode="8532241000", @@ -259,7 +203,7 @@ def _measures() -> list[MeasureModel]: geoAreaDescription="China", additionalCode="8B999", dutyExpression="32.4 %", - validFrom=EPOCH, + validFrom=date(2020, 1, 1), conditions=[ MeasureConditionEmbedded( conditionCode="A", @@ -271,7 +215,6 @@ def _measures() -> list[MeasureModel]: ), ], ), - # Dual-use: flagged as out of scope rather than answered. MeasureModel( measureSid="DEMO-8543-DU", goodsCode="8543709099", @@ -280,7 +223,7 @@ def _measures() -> list[MeasureModel]: category=MeasureCategory.DUAL_USE, direction=TradeDirection.EXPORT, geoAreaCode="1011", - validFrom=EPOCH, + validFrom=date(2020, 1, 1), conditions=[ MeasureConditionEmbedded( conditionCode="E", @@ -300,8 +243,6 @@ def _measures() -> list[MeasureModel]: ), ], ), - # Clothing export: a surveillance measure with no document obligation, - # so the line legitimately comes back with none required. MeasureModel( measureSid="DEMO-6109-EXP", goodsCode="6109", @@ -310,9 +251,8 @@ def _measures() -> list[MeasureModel]: category=MeasureCategory.SURVEILLANCE, direction=TradeDirection.EXPORT, geoAreaCode="1011", - validFrom=EPOCH, + validFrom=date(2020, 1, 1), ), - # Preferential duty for GSP countries: fiscal only. MeasureModel( measureSid="DEMO-6109-GSP", goodsCode="61091000", @@ -323,33 +263,12 @@ def _measures() -> list[MeasureModel]: geoAreaCode="2005", geoAreaDescription="GSP beneficiaries", dutyExpression="9.6 %", - validFrom=EPOCH, + validFrom=date(2020, 1, 1), ), ] -def _code_changes(today: date) -> list[CodeChangeModel]: - """ - The correlation entry behind the withdrawn-code demo case. - """ - return [ - CodeChangeModel( - oldCode="8528721090", - newCodes=["8528721010"], - changeType=CodeChangeType.REPLACED, - changedOn=date(today.year, 1, 1).isoformat(), - note="Subheading merged at the 1 January nomenclature update.", - ) - ] - - async def seed_demo_reference_data() -> SyncRun: - """ - Load the demo reference dataset. - - Written through the same idempotent upsert path as a real import, under a - sync log that says plainly where the data came from. - """ today = date.today() async with SyncRun( @@ -366,7 +285,19 @@ async def seed_demo_reference_data() -> SyncRun: (DocumentCodeModel, _document_codes(), ["code", "validFrom"]), (AdditionalCodeModel, _additional_codes(), ["fullCode", "validFrom"]), (MeasureModel, _measures(), ["measureSid"]), - (CodeChangeModel, _code_changes(today), ["oldCode", "changedOn"]), + ( + CodeChangeModel, + [ + CodeChangeModel( + oldCode="8528721090", + newCodes=["8528721010"], + changeType=CodeChangeType.REPLACED, + changedOn=date(today.year, 1, 1).isoformat(), + note="Subheading merged at the 1 January nomenclature update.", + ) + ], + ["oldCode", "changedOn"], + ), ] for model, instances, key_fields in batches: diff --git a/cbh/api/importer/services/dtv/__init__.py b/cbh/api/importer/services/dtv/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..5557b0789b1619328b30ee2d40fc9288dcef328f --- /dev/null +++ b/cbh/api/importer/services/dtv/__init__.py @@ -0,0 +1,15 @@ +from .dtv import ( + DtvRefresh, + import_national_measures, + list_publications, + refresh_national_measures, + resolve_latest_snapshot, +) + +__all__ = [ + "DtvRefresh", + "import_national_measures", + "list_publications", + "refresh_national_measures", + "resolve_latest_snapshot", +] diff --git a/cbh/api/importer/services/dtv/dtv.py b/cbh/api/importer/services/dtv/dtv.py new file mode 100644 index 0000000000000000000000000000000000000000..ed24ec0581ed2312f6d1896463a02250a9b79337 --- /dev/null +++ b/cbh/api/importer/services/dtv/dtv.py @@ -0,0 +1,368 @@ +"""The Dutch Gebruikstarief — national measures on top of the Union ones. + +TARIC states what the Union requires. What the Netherlands adds on its own +authority — omzetbelasting, accijns, verbruiksbelasting, the invoer- and +uitvoerbeperkingen written in Dutch law — is published only here, by the +Belastingdienst, as an Arctic Tariff export. + +The feed carries the whole tariff, Union rules included. Only the records +flagged `national="1"` are taken: the Union side is already loaded from the +published TARIC extraction and reloading it from a second source would make it +impossible to say which publication an answer came from. +""" + +import zipfile +from dataclasses import dataclass, field +from pathlib import Path +from typing import Any, Iterator +from xml.etree import ElementTree + +import httpx + +from cbh.api.common.dto import TradeDirection +from cbh.api.common.utils import parse_date +from cbh.api.importer.dto import SyncMode, SyncSource +from cbh.api.importer.services.base import SyncRun, download_to, latest_loaded_hash +from cbh.api.importer.utils import file_hash_path +from cbh.api.taric.dto import MeasureCategory +from cbh.api.taric.models import ( + DocumentCodeModel, + MeasureConditionEmbedded, + MeasureModel, + MeasureTypeModel, +) +from cbh.core.config import settings + +NS = "{http://www.arcticgroup.se/tariff/arctictariff/export}" + +# Arctic's `tradeMovementCode`, read off the published MeasureType table: the +# NLINV* (invoer) types are 0, the NLUIT* (uitvoer) types are 1. +TRADE_MOVEMENT = { + "0": TradeDirection.IMPORT, + "1": TradeDirection.EXPORT, + "2": TradeDirection.BOTH, +} + +# The series a national measure type belongs to already says what kind of rule +# it is, so the fallback is data rather than a guess: A prohibits, B restricts, +# P is turnover tax, Q is one of the consumption taxes. +SERIES_CATEGORY = { + "A": MeasureCategory.PROHIBITION, + "B": MeasureCategory.RESTRICTION, + "P": MeasureCategory.VAT, + "Q": MeasureCategory.EXCISE, +} + +# Where the series is too coarse to be useful, the type is named outright. +NATIONAL_TYPE_CATEGORY = { + "NLBTW": MeasureCategory.VAT, + "NLACC": MeasureCategory.EXCISE, + "NLACVH": MeasureCategory.EXCISE, + "NLVBB": MeasureCategory.EXCISE, + "NLKOBE": MeasureCategory.EXCISE, + "NLINVF": MeasureCategory.PHYTOSANITARY_CONTROL, + "NLUSGM": MeasureCategory.DUAL_USE, + "NLUITT": MeasureCategory.DUAL_USE, + "NLINVE": MeasureCategory.DUAL_USE, + "NLUITE": MeasureCategory.DUAL_USE, +} + + +@dataclass +class NationalMeasureType: + typeId: str + direction: TradeDirection + category: MeasureCategory + description: str | None = None + descriptionNl: str | None = None + + +@dataclass +class DtvRefresh: + fileName: str | None = None + url: str | None = None + runs: list[SyncRun] = field(default_factory=list) + note: str | None = None + + +def _attrs(element: ElementTree.Element) -> dict[str, str]: + return {key.split("}")[-1]: value for key, value in element.attrib.items()} + + +def _descriptions(element: ElementTree.Element, tag: str) -> dict[str, str]: + found: dict[str, str] = {} + for child in element.iter(NS + tag): + attributes = _attrs(child) + language = (attributes.get("languageId") or "").upper() + if language and attributes.get("description"): + found[language] = attributes["description"] + return found + + +def document_code(condition: dict[str, str]) -> str | None: + """Box 44 writes the certificate type and its number as one code. + + An EU certificate comes out as `N851`, a Dutch national one as `1550`; the + type is the first character either way. + """ + code = (condition.get("certificateCode") or "").strip() + if not code: + return None + return f"{(condition.get('certificateType') or '').strip()}{code}" + + +def _member(archive: zipfile.ZipFile, marker: str) -> str | None: + return next((name for name in archive.namelist() if marker in name), None) + + +def parse_measure_types(payload: bytes) -> dict[str, NationalMeasureType]: + root = ElementTree.fromstring(payload) + types: dict[str, NationalMeasureType] = {} + for element in root.iter(NS + "measureType"): + attributes = _attrs(element) + if attributes.get("national") != "1": + continue + type_id = attributes.get("measureType") + if not type_id: + continue + descriptions = _descriptions(element, "measureTypeDescription") + category = NATIONAL_TYPE_CATEGORY.get( + type_id, + SERIES_CATEGORY.get( + attributes.get("measureTypeSeriesId") or "", MeasureCategory.OTHER + ), + ) + types[type_id] = NationalMeasureType( + typeId=type_id, + direction=TRADE_MOVEMENT.get( + attributes.get("tradeMovementCode") or "", TradeDirection.BOTH + ), + category=category, + description=descriptions.get("EN") or descriptions.get("NL"), + descriptionNl=descriptions.get("NL"), + ) + return types + + +def parse_certificates(payload: bytes) -> list[dict[str, Any]]: + root = ElementTree.fromstring(payload) + documents: list[dict[str, Any]] = [] + for element in root.iter(NS + "certificate"): + attributes = _attrs(element) + if attributes.get("national") != "1": + continue + code = document_code( + { + "certificateType": attributes.get("certificateType"), + "certificateCode": attributes.get("certificateCode"), + } + ) + if not code: + continue + descriptions = _descriptions(element, "certificateDescription") + documents.append( + DocumentCodeModel( + code=code, + description=descriptions.get("EN") or descriptions.get("NL"), + descriptionNl=descriptions.get("NL"), + national=True, + validFrom=parse_date(attributes.get("dateStart")), + validTo=parse_date(attributes.get("dateEnd")), + ).to_mongo() + ) + return documents + + +def iter_national_measures( + archive_path: Path, member: str, types: dict[str, NationalMeasureType] +) -> Iterator[dict[str, Any]]: + """Stream the measure table, keeping only the national records. + + The full snapshot's measure file is about three gigabytes once unpacked, so + it is never held in memory or written out: it is parsed straight off the + archive and each element is dropped as soon as it has been read. + """ + with zipfile.ZipFile(archive_path) as archive, archive.open(member) as stream: + for _, element in ElementTree.iterparse(stream, events=("end",)): + if element.tag != NS + "measure": + continue + attributes = _attrs(element) + if attributes.get("national") != "1" or attributes.get("stoppedFlag") == "1": + element.clear() + continue + + parsed = _parse_measure(element, attributes, types) + element.clear() + if parsed: + yield parsed + + +def _parse_measure( + element: ElementTree.Element, + attributes: dict[str, str], + types: dict[str, NationalMeasureType], +) -> dict[str, Any] | None: + goods_code = (attributes.get("goodsNomenclatureCode") or "").strip() + type_id = (attributes.get("measureType") or "").strip() + sid = (attributes.get("SID") or "").strip() + if not goods_code or not type_id or not sid: + return None + + measure_type = types.get(type_id) + conditions: list[dict[str, Any]] = [] + for child in element.iter(NS + "measureCondition"): + condition = _attrs(child) + code = condition.get("conditionCodeId") + if not code: + continue + conditions.append( + MeasureConditionEmbedded( + conditionCode=code, + sequenceNumber=int(condition.get("sequenceNumber") or 1), + documentCode=document_code(condition), + actionCode=condition.get("actionCode"), + ).to_mongo() + ) + + return MeasureModel( + # Prefixed because the Belastingdienst numbers its own records and those + # numbers mean nothing in the TARIC publication we load beside them. + measureSid=f"dtv:{sid}", + goodsCode=goods_code, + additionalCode=(attributes.get("additionalCodeId") or "").strip() or None, + measureTypeId=type_id, + measureTypeDescription=measure_type.description if measure_type else None, + category=measure_type.category if measure_type else MeasureCategory.OTHER, + direction=measure_type.direction if measure_type else TradeDirection.BOTH, + geoAreaCode=(attributes.get("geographicalAreaId") or "").strip(), + dutyExpression=(attributes.get("expression") or "").strip() or None, + regulationId=(attributes.get("regulationId") or "").strip() or None, + national=True, + conditions=conditions, + validFrom=parse_date(attributes.get("dateStart")), + validTo=parse_date(attributes.get("dateEnd")), + ).to_mongo() + + +async def list_publications() -> list[tuple[str, str]]: + async with httpx.AsyncClient( + timeout=settings.IMPORTER_HTTP_TIMEOUT, follow_redirects=True + ) as client: + response = await client.get(settings.DTV_MANIFEST_URL) + response.raise_for_status() + root = ElementTree.fromstring(response.content) + return [ + (entry.findtext("bestand") or "", entry.findtext("url") or "") + for entry in root.iter("download") + if entry.findtext("url") + ] + + +async def resolve_latest_snapshot() -> tuple[str, str] | None: + """Pick the newest full snapshot without downloading any of them. + + The manifest lists nightly increments and full snapshots side by side with + nothing to tell them apart but their size, which the server reports. + """ + publications = await list_publications() + async with httpx.AsyncClient( + timeout=settings.IMPORTER_HTTP_TIMEOUT, follow_redirects=True + ) as client: + for name, url in reversed(publications): + try: + head = await client.head(url) + except httpx.HTTPError: + continue + size = int(head.headers.get("content-length") or 0) + if size >= settings.DTV_FULL_SNAPSHOT_MIN_BYTES: + return name, url + return None + + +async def import_national_measures( + archive_path: Path, file_name: str, mode: SyncMode = SyncMode.FULL +) -> SyncRun: + async with SyncRun(SyncSource.DTV_NATIONAL, mode, file_name=file_name) as run: + run.log.fileHash = file_hash_path(archive_path) + with zipfile.ZipFile(archive_path) as archive: + type_member = _member(archive, "/MeasureType_") + certificate_member = _member(archive, "/Certificate_") + measure_member = _member(archive, "/Measure_") + + if not measure_member or not type_member: + raise ValueError( + "This archive holds no full measure table — it is a nightly " + "increment, not a snapshot." + ) + + types = parse_measure_types(archive.read(type_member)) + certificates = ( + parse_certificates(archive.read(certificate_member)) + if certificate_member + else [] + ) + + run.log.note = f"{len(types)} national measure types." + + await run.write( + MeasureTypeModel, + [ + MeasureTypeModel( + typeId=item.typeId, + description=item.description, + descriptionNl=item.descriptionNl, + category=item.category, + direction=item.direction, + ).to_mongo() + for item in types.values() + ], + ["typeId"], + ) + + if certificates: + await run.write(DocumentCodeModel, certificates, ["code", "validFrom"]) + + batch: list[dict[str, Any]] = [] + for measure in iter_national_measures(archive_path, measure_member, types): + batch.append(measure) + if len(batch) >= 2000: + await run.write(MeasureModel, batch, ["measureSid"]) + batch = [] + if batch: + await run.write(MeasureModel, batch, ["measureSid"]) + + return run + + +async def refresh_national_measures( + mode: SyncMode = SyncMode.FULL, keep_archive: bool = False +) -> DtvRefresh: + refresh = DtvRefresh() + + resolved = await resolve_latest_snapshot() + if resolved is None: + refresh.note = ( + "No full snapshot found in the Gebruikstarief feed; only increments " + f"are listed at {settings.DTV_MANIFEST_URL}." + ) + return refresh + + refresh.fileName, refresh.url = resolved + archive_path = settings.REFERENCE_STORAGE_DIR / "dtv_national" / refresh.fileName + await download_to(refresh.url, archive_path) + + try: + if await latest_loaded_hash(SyncSource.DTV_NATIONAL) == file_hash_path( + archive_path + ): + refresh.note = "Unchanged since the last successful import; nothing loaded." + return refresh + + refresh.runs.append( + await import_national_measures(archive_path, refresh.fileName, mode) + ) + finally: + if not keep_archive: + archive_path.unlink(missing_ok=True) + + return refresh diff --git a/cbh/api/importer/services/taric/__init__.py b/cbh/api/importer/services/taric/__init__.py index 94f33eab481f6d5c259d119a54db50127d4af8a3..2df05676098f19fdeb38947722b56954db6e75ea 100644 --- a/cbh/api/importer/services/taric/__init__.py +++ b/cbh/api/importer/services/taric/__init__.py @@ -1,3 +1,9 @@ +from .publication import ( + ARCHIVE_MEMBERS, + PublicationRefresh, + refresh_from_publication, + resolve_latest_publication, +) from .taric import ( IMPORT_ORDER, SOURCE_SPECS, @@ -8,10 +14,14 @@ from .taric import ( ) __all__ = [ + "ARCHIVE_MEMBERS", "IMPORT_ORDER", "SOURCE_SPECS", + "PublicationRefresh", "import_all_local", "import_from_local", "import_from_url", "import_source", + "refresh_from_publication", + "resolve_latest_publication", ] diff --git a/cbh/api/importer/services/taric/parsers.py b/cbh/api/importer/services/taric/parsers.py index 822c54df60d5ce62d3656a9506ef457f70f5cbf8..91edf3dc71a5bf60cab182d7f3cb6cb55dd09374 100644 --- a/cbh/api/importer/services/taric/parsers.py +++ b/cbh/api/importer/services/taric/parsers.py @@ -1,83 +1,116 @@ -""" -Column maps and row parsers for the TARIC distribution files. - -The published files change their column spelling between releases more often -than their content, so each source declares the spellings it accepts and the -reader matches loosely. Adapting to a new release is a change to a tuple here, -not to the import logic. -""" - +import re from typing import Any from cbh.api.common.dto import TradeDirection from cbh.api.common.utils import normalize_ws, parse_date from cbh.api.importer.utils import as_bool from cbh.api.taric.dto import ConditionAction, GeoAreaType -from cbh.api.taric.utils import classify_measure_type, normalize_code +from cbh.api.taric.utils import classify_measure_type, normalize_code, significant_length + +# The spellings are those of the monthly TARIC extraction published through +# CIRCABC (DG TAXUD): one worksheet per subject, headers on the first row. The +# first spelling that matches a header wins, so the exact TARIC name comes +# first and the older guesses stay as fallbacks for other releases. +# +# `Language` is present on every multilingual sheet — Box 44 codes, additional +# codes and geographical areas repeat each record once per EU language, so the +# import filters on it rather than letting the last language win. NOMENCLATURE_COLUMNS: dict[str, tuple[str, ...]] = { "code": ("Goods code", "Goods nomenclature code", "Code", "Goederencode", "GN code"), - "indent": ("Indent", "Indents", "Niveau", "Level"), + "language": ("Language", "Lang", "Taal"), + "indent": ("Indent", "Hier. Pos.", "Indents", "Niveau", "Level"), "descriptionEn": ("Description", "Description EN", "Goods description", "Omschrijving EN"), - "descriptionNl": ("Description NL", "Omschrijving", "Omschrijving NL"), - "declarable": ("Declarable", "Declarable code", "Aangifteplichtig", "Is declarable"), + "descriptionNl": ("Description NL", "Omschrijving NL"), + "declarable": ("IS_LEAF", "Declarable", "Declarable code", "Aangifteplichtig"), "supplementaryUnit": ("Supplementary unit", "Suppl. unit", "Bijzondere maatstaf"), "validFrom": ("Start date", "Valid from", "Validity start date", "Begindatum"), "validTo": ("End date", "Valid to", "Validity end date", "Einddatum"), } +# The extraction carries no measure SID: `Taric measures`, `Duties Import` and +# `Duties Export` all identify a measure by the same six columns instead, so the +# importer composes a stable key out of them (see `measure_key`). MEASURE_COLUMNS: dict[str, tuple[str, ...]] = { "measureSid": ("Measure SID", "Measure sid", "SID", "Measure identifier"), "goodsCode": ("Goods code", "Goods nomenclature code", "Code", "Goederencode"), - "additionalCode": ("Additional code", "Additional code id", "Aanvullende code"), - "measureTypeId": ("Measure type", "Measure type id", "Maatregelsoort"), - "measureTypeDescription": ("Measure type description", "Measure type name", "Omschrijving"), - "geoAreaCode": ("Geographical area", "Geographical area id", "Origin", "Landcode"), - "geoAreaDescription": ("Geographical area description", "Geographical area name"), + "additionalCode": ("Add code", "Additional code", "Additional code id", "Aanvullende code"), + "measureTypeId": ("Meas. type code", "Measure type id", "Maatregelsoort"), + "measureTypeDescription": ( + "Measure type", + "Measure type description", + "Measure type name", + ), + "geoAreaCode": ( + "Origin code", + "Dest. code", + "Geographical area id", + "Geographical area", + "Landcode", + ), + "geoAreaDescription": ( + "Origin", + "Destination", + "Geographical area description", + "Geographical area name", + ), "excludedGeoAreaCodes": ("Excluded areas", "Geographical area exclusions", "Exclusions"), - "dutyExpression": ("Duty expression", "Duty rate", "Rate", "Tarief"), - "quotaOrderNumber": ("Quota order number", "Order number", "Contingentnummer"), - "regulationId": ("Regulation", "Regulation id", "Verordening"), + "dutyExpression": ("Duty", "Duty expression", "Duty rate", "Rate", "Tarief"), + "quotaOrderNumber": ("Order No.", "Quota order number", "Order number"), + "regulationId": ("Legal base", "Regulation", "Regulation id", "Verordening"), "direction": ("Direction", "Import/Export", "Trade movement", "Richting"), "validFrom": ("Start date", "Valid from", "Validity start date", "Begindatum"), "validTo": ("End date", "Valid to", "Validity end date", "Einddatum"), } +# The conditions sheet repeats the same six identifying columns, which is how a +# condition is tied back to its measure without a SID. MEASURE_CONDITION_COLUMNS: dict[str, tuple[str, ...]] = { "measureSid": ("Measure SID", "Measure sid", "SID", "Measure identifier"), - "conditionCode": ("Condition code", "Condition", "Voorwaardecode"), - "sequenceNumber": ("Condition sequence number", "Sequence", "Volgnummer"), + "goodsCode": ("Goods code", "Goods nomenclature code", "Code"), + "additionalCode": ("Add code", "Additional code", "Additional code id"), + "quotaOrderNumber": ("Order No.", "Quota order number", "Order number"), + "geoAreaCode": ("Origin code", "Dest. code", "Geographical area id"), + "measureTypeId": ("Meas. type code", "Measure type id"), + "conditionCode": ("Meas. cond", "Condition code", "Condition", "Voorwaardecode"), + "sequenceNumber": ("Sequence", "Condition sequence number", "Volgnummer"), "documentCode": ( + "Certificate", "Certificate code", "Document code", "Certificate type code", - "Certificaatcode", ), - "documentDescription": ("Certificate description", "Document description", "Omschrijving"), - "actionCode": ("Action code", "Condition action code", "Actiecode"), + "documentDescription": ("Certificate description", "Document description"), + "actionCode": ("Meas. action", "Action code", "Condition action code", "Actiecode"), "actionDescription": ("Action description", "Condition action", "Actie"), - "requirementDutyExpression": ("Condition duty amount", "Requirement", "Duty expression"), + "requirementDutyExpression": ("Cond. amount", "Condition duty amount", "Requirement"), + "validFrom": ("Start date", "Valid from", "Begindatum"), } +# One row per (group, member) pair, so this sheet is grouped on import rather +# than parsed row by row — see `_import_geo_area_composition`. GEO_AREA_COLUMNS: dict[str, tuple[str, ...]] = { - "code": ("Geographical area id", "Area code", "Code", "Landcode"), - "name": ("Description", "Geographical area description", "Name", "Omschrijving"), + "code": ("Country group", "Geographical area id", "Area code", "Code"), + "language": ("Language", "Lang", "Taal"), + "name": ("Country group descr.", "Geographical area description", "Description", "Name"), "areaType": ("Area type", "Type", "Soort"), - "memberCodes": ("Members", "Member countries", "Composition", "Landen"), + "memberCodes": ("Member country", "Members", "Member countries", "Composition"), "validFrom": ("Start date", "Valid from", "Begindatum"), "validTo": ("End date", "Valid to", "Einddatum"), } ADDITIONAL_CODE_COLUMNS: dict[str, tuple[str, ...]] = { "codeType": ("Additional code type", "Type", "Soort"), - "code": ("Additional code", "Code"), + "code": ("Add code", "Additional code", "Code"), + "language": ("Language", "Lang", "Taal"), "description": ("Description", "Omschrijving"), "validFrom": ("Start date", "Valid from", "Begindatum"), "validTo": ("End date", "Valid to", "Einddatum"), } DOCUMENT_CODE_COLUMNS: dict[str, tuple[str, ...]] = { - "code": ("Document code", "Certificate code", "Code", "Bescheidcode"), + "code": ("Certificate code", "Document code", "Code", "Bescheidcode"), + "language": ("Language", "Lang", "Taal"), "description": ("Description", "Description EN", "Omschrijving EN"), "descriptionNl": ("Description NL", "Omschrijving"), "validFrom": ("Start date", "Valid from", "Begindatum"), @@ -91,19 +124,8 @@ CODE_CHANGE_COLUMNS: dict[str, tuple[str, ...]] = { "note": ("Remark", "Note", "Opmerking"), } -# Action codes are only a hint here. Their numbering is not stable enough -# across releases to hang a document obligation on, so the exemption decision -# is made from the document-code prefix (see -# ``MeasureConditionEmbedded.is_exemption``) and this mapping only enriches the -# stored condition. Verify these against the published Measure-action list when -# the real TARIC extracts land. -_NOT_ALLOWED_ACTIONS = {"08", "8", "09", "9", "23"} - def _split_list(value: Any) -> list[str]: - """ - Split a delimited cell into a list of codes. - """ if value is None or value == "": return [] if isinstance(value, (list, tuple)): @@ -115,22 +137,7 @@ def _split_list(value: Any) -> list[str]: return [text.strip()] if text.strip() else [] -def _parse_direction(value: Any) -> TradeDirection: - """ - Read the trade movement of a measure, defaulting to both directions. - """ - return TradeDirection.parse(value) or TradeDirection.BOTH - - def _parse_action(action_code: Any, description: Any) -> ConditionAction: - """ - Map a TARIC condition action onto its meaning for a line. - - The action description is trusted ahead of the numeric code, and an - unrecognised action is left as ``OTHER`` rather than guessed at: the - exemption decision is taken from the document code, so a wrong guess here - cannot turn an obligation into a way out of one. - """ text = str(description or "").lower() if "not applicable" in text or "niet van toepassing" in text: return ConditionAction.MEASURE_NOT_APPLICABLE @@ -141,21 +148,24 @@ def _parse_action(action_code: Any, description: Any) -> ConditionAction: if "duty" in text or "recht" in text: return ConditionAction.DUTY_APPLIES - if str(action_code or "").strip() in _NOT_ALLOWED_ACTIONS: + code = str(action_code or "").strip().lstrip("0") or "0" + if code in ("5", "7", "8", "9", "23"): return ConditionAction.IMPORT_EXPORT_NOT_ALLOWED + # 27 and 29 are "allowed after producing the document" in the published + # action list; 1 applies the stated amount. + if code in ("27", "29"): + return ConditionAction.DOCUMENT_REQUIRED + if code == "1": + return ConditionAction.DUTY_APPLIES return ConditionAction.OTHER def parse_nomenclature_row(row: dict[str, Any]) -> dict[str, Any] | None: - """ - Parse one nomenclature row into model fields. - """ - raw_code = str(row.get("code") or "").strip() - digits = "".join(character for character in raw_code if character.isdigit()) + digits = "".join( + character for character in str(row.get("code") or "").strip() if character.isdigit() + ) if not digits: return None - # Intermediate tree nodes are shorter than 10 digits; pad them so the whole - # tree lives in one comparable key space. code = digits.ljust(10, "0") if len(digits) < 10 else digits[:10] return { @@ -173,29 +183,113 @@ def parse_nomenclature_row(row: dict[str, Any]) -> dict[str, Any] | None: } -def parse_measure_row(row: dict[str, Any]) -> dict[str, Any] | None: +def measure_key(row: dict[str, Any]) -> str: + """The identity of a measure in a publication that carries no measure SID. + + `Taric measures`, `Duties Import`, `Duties Export` and `Measure conditions` + all repeat the same six columns, and together those six identify one + measure. Composing them gives a key that is stable across files and across + months, which is what ties a condition back to the measure it belongs to. """ - Parse one measure row into model fields. + goods_code = "".join( + character for character in str(row.get("goodsCode") or "") if character.isdigit() + ) + start = parse_date(row.get("validFrom")) + # The direction is part of the identity because the same goods code, area, + # type and date can appear in both the import and the export extraction — + # `Supplementary unit` does. Without it one would overwrite the other and + # the pack would lose half its measures. + direction = TradeDirection.parse(row.get("direction")) + parts = [ + goods_code, + normalize_ws(row.get("additionalCode")) or "", + normalize_ws(row.get("quotaOrderNumber")) or "", + start.isoformat() if start else "", + str(row.get("geoAreaCode") or "").strip().upper(), + str(row.get("measureTypeId") or "").strip(), + str(direction.value) if direction else "", + ] + return "|".join(parts) + + +# `Cond: B cert: Y-922 (29):; B (09):` — one segment per `;`, each naming a +# condition code, optionally a certificate, an action code in brackets and the +# duty that applies when the condition is met. Segments sharing a condition code +# are alternatives, which is exactly how the engine reads `conditionCode`. +_CONDITION_SEGMENT = re.compile( + r"^(?P[A-Z])\b" + r"(?:\s*cert:\s*(?P[A-Za-z]\s*-?\s*\d{3}[A-Za-z0-9]?))?" + r"(?P[^()]*)" + r"\((?P\d+)\)" + r"\s*:?\s*(?P.*)$" +) + + +def parse_duty_conditions(raw: Any) -> list[dict[str, Any]]: + """Read the conditions written into the duty column of a measure row. + + The separate `Measure conditions` sheet only covers tariff and quota + measure types. Prohibitions, restrictions and export controls — the + measures that carry the Y codes — state their conditions here and nowhere + else, so this is the primary source rather than a convenience. """ - measure_sid = str(row.get("measureSid") or "").strip() + text = str(raw or "") + if "cond:" not in text.lower(): + return [] + + conditions: list[dict[str, Any]] = [] + for sequence, segment in enumerate(text.split("Cond:", 1)[1].split(";"), start=1): + match = _CONDITION_SEGMENT.match(segment.strip()) + if match is None: + continue + + document_code = match.group("document") + if document_code: + document_code = re.sub(r"[^A-Za-z0-9]", "", document_code).upper() + + action_code = match.group("action") + conditions.append( + { + "conditionCode": match.group("condition").upper(), + "sequenceNumber": sequence, + "documentCode": document_code or None, + "documentDescription": None, + "actionCode": action_code, + "actionDescription": None, + "action": _parse_action(action_code, None), + "requirementDutyExpression": normalize_ws( + match.group("duty") or match.group("amount") + ), + "certificateRequired": bool(document_code), + } + ) + return conditions + + +def parse_measure_row(row: dict[str, Any]) -> dict[str, Any] | None: goods_code = "".join( character for character in str(row.get("goodsCode") or "") if character.isdigit() ) - if not measure_sid or not goods_code: + if not goods_code: return None + measure_sid = str(row.get("measureSid") or "").strip() or measure_key(row) type_id = str(row.get("measureTypeId") or "").strip() description = normalize_ws(row.get("measureTypeDescription")) return { "measureSid": measure_sid, "goodsCode": goods_code, - "goodsCodeLength": len(goods_code), + "conditions": parse_duty_conditions(row.get("dutyExpression")), + # Codes come zero-padded to ten digits, so the raw length says nothing + # about how specific a measure is. `6203000000` is heading level and + # must lose to an eight-digit measure on the same line. + "goodsCodeLength": significant_length(goods_code), "additionalCode": normalize_ws(row.get("additionalCode")), "measureTypeId": type_id, "measureTypeDescription": description, "category": classify_measure_type(type_id, description), - "direction": _parse_direction(row.get("direction")), + "direction": TradeDirection.parse(row.get("direction")) or TradeDirection.BOTH, "geoAreaCode": str(row.get("geoAreaCode") or "").strip().upper(), "geoAreaDescription": normalize_ws(row.get("geoAreaDescription")), "excludedGeoAreaCodes": [ @@ -210,15 +304,12 @@ def parse_measure_row(row: dict[str, Any]) -> dict[str, Any] | None: def parse_measure_condition_row(row: dict[str, Any]) -> dict[str, Any] | None: - """ - Parse one measure-condition row. - - Conditions are attached to their measure in a second pass, which is why the - measure SID is carried through rather than resolved here. - """ - measure_sid = str(row.get("measureSid") or "").strip() condition_code = str(row.get("conditionCode") or "").strip() - if not measure_sid or not condition_code: + if not condition_code: + return None + + measure_sid = str(row.get("measureSid") or "").strip() or measure_key(row) + if not measure_sid.strip("|"): return None document_code = normalize_ws(row.get("documentCode")) @@ -242,9 +333,6 @@ def parse_measure_condition_row(row: dict[str, Any]) -> dict[str, Any] | None: def parse_geo_area_row(row: dict[str, Any]) -> dict[str, Any] | None: - """ - Parse one geographical area row. - """ code = str(row.get("code") or "").strip().upper() if not code: return None @@ -269,9 +357,6 @@ def parse_geo_area_row(row: dict[str, Any]) -> dict[str, Any] | None: def parse_additional_code_row(row: dict[str, Any]) -> dict[str, Any] | None: - """ - Parse one additional-code row. - """ code = normalize_ws(row.get("code")) if not code: return None @@ -287,9 +372,6 @@ def parse_additional_code_row(row: dict[str, Any]) -> dict[str, Any] | None: def parse_document_code_row(row: dict[str, Any]) -> dict[str, Any] | None: - """ - Parse one Box 44 document-code row. - """ code = normalize_ws(row.get("code")) if not code: return None @@ -303,9 +385,6 @@ def parse_document_code_row(row: dict[str, Any]) -> dict[str, Any] | None: def parse_code_change_row(row: dict[str, Any]) -> dict[str, Any] | None: - """ - Parse one correlation-table row linking a withdrawn code to its successors. - """ old_code = normalize_code(row.get("oldCode")) if not old_code: return None diff --git a/cbh/api/importer/services/taric/publication.py b/cbh/api/importer/services/taric/publication.py new file mode 100644 index 0000000000000000000000000000000000000000..a5583f2ba0d50368c932804bd83c69ef06b6da3a --- /dev/null +++ b/cbh/api/importer/services/taric/publication.py @@ -0,0 +1,163 @@ +"""Fetching the published TARIC extraction instead of a file placed by hand. + +The extraction is one archive per publication month holding eighteen sheets. +Only the eight this engine reads are unpacked; the rest — footnotes, exchange +rates, the 68 MB `Taric measures` sheet that duplicates Duties Import — are +left in the archive. +""" + +import zipfile +from dataclasses import dataclass, field +from datetime import date +from pathlib import Path + +from cbh.api.importer.dto import SyncMode, SyncSource +from cbh.api.importer.services.base import SyncRun, download_to, url_exists +from cbh.api.importer.utils import local_source_path, normalize_header +from cbh.core.config import settings + + +# The member file of the archive each source is read from. Matched on the +# normalised name, so a publication that changes spacing or case still lands. +ARCHIVE_MEMBERS: dict[SyncSource, tuple[str, ...]] = { + SyncSource.TARIC_NOMENCLATURE: ("Nomenclature EN.xlsx",), + SyncSource.TARIC_DECLARABLE_CODES: ("Declarable codes.xlsx",), + SyncSource.TARIC_GEO_AREAS: ("Geographical areas composition.xlsx",), + SyncSource.TARIC_ADDITIONAL_CODES: ("Additional codes descriptions.xlsx",), + SyncSource.TARIC_BOX44_CODES: ("Box 44 codes of the SAD.xlsx",), + SyncSource.TARIC_DUTIES_IMPORT: ("Duties Import 01-99.xlsx",), + SyncSource.TARIC_DUTIES_EXPORT: ("Duties Export 01-99.xlsx",), + SyncSource.TARIC_MEASURE_CONDITIONS: ("Measure conditions.xlsx",), +} + + +@dataclass +class PublicationRefresh: + """What one attempt at refreshing from the published archive did.""" + + period: str | None = None + url: str | None = None + archivePath: Path | None = None + runs: list[SyncRun] = field(default_factory=list) + missingSources: list[SyncSource] = field(default_factory=list) + note: str | None = None + + @property + def loaded(self) -> bool: + return bool(self.runs) + + +def publication_period(on_date: date) -> str: + return on_date.strftime("%Y-%m") + + +def publication_url(period: str) -> str: + return settings.TARIC_ARCHIVE_URL.format(period=period) + + +def _previous_period(period: str) -> str: + year, month = (int(part) for part in period.split("-")) + return f"{year - 1}-12" if month == 1 else f"{year}-{month - 1:02d}" + + +def candidate_periods(on_date: date | None = None) -> list[str]: + """This month first, then back — the pack lands early in the month, but not + on the first stroke of it, and a broker running on the 1st must not be told + the reference data has disappeared.""" + period = publication_period(on_date or date.today()) + periods = [period] + for _ in range(max(settings.TARIC_ARCHIVE_LOOKBACK_MONTHS, 0)): + period = _previous_period(period) + periods.append(period) + return periods + + +async def resolve_latest_publication( + on_date: date | None = None, +) -> tuple[str, str] | None: + for period in candidate_periods(on_date): + url = publication_url(period) + if await url_exists(url): + return period, url + return None + + +def _member_index(archive: zipfile.ZipFile) -> dict[str, str]: + return {normalize_header(name.rsplit("/", 1)[-1]): name for name in archive.namelist()} + + +def unpack_archive(archive_path: Path) -> dict[SyncSource, Path]: + """Write each wanted member into the storage folder its source reads from. + + The files are kept rather than streamed through memory: an import that fails + halfway is then diagnosable from the same file the engine saw. + """ + extracted: dict[SyncSource, Path] = {} + with zipfile.ZipFile(archive_path) as archive: + index = _member_index(archive) + for source, names in ARCHIVE_MEMBERS.items(): + member = next( + (index[key] for key in map(normalize_header, names) if key in index), None + ) + if member is None: + continue + target = local_source_path(source, member.rsplit("/", 1)[-1]) + with archive.open(member) as handle: + target.write_bytes(handle.read()) + extracted[source] = target + return extracted + + +async def refresh_from_publication( + mode: SyncMode = SyncMode.FULL, + period: str | None = None, + keep_archive: bool = False, +) -> PublicationRefresh: + """Download the published extraction and load whatever is new in it. + + Nothing is deleted and nothing is loaded twice: each source is hashed + against the last successful import, so a run against an unchanged + publication costs one download and no writes. + """ + from cbh.api.importer.services.taric.taric import IMPORT_ORDER, import_from_local + + refresh = PublicationRefresh() + + if period: + url = publication_url(period) + if not await url_exists(url): + refresh.note = f"No published extraction for {period} at {url}." + return refresh + resolved: tuple[str, str] | None = (period, url) + else: + resolved = await resolve_latest_publication() + + if resolved is None: + refresh.note = ( + "No published extraction found for " + f"{', '.join(candidate_periods())}. Check TARIC_ARCHIVE_URL." + ) + return refresh + + refresh.period, refresh.url = resolved + archive_path = settings.REFERENCE_STORAGE_DIR / f"eu-taric-{refresh.period}.zip" + await download_to(refresh.url, archive_path) + refresh.archivePath = archive_path + + try: + extracted = unpack_archive(archive_path) + finally: + if not keep_archive: + archive_path.unlink(missing_ok=True) + refresh.archivePath = None + + refresh.missingSources = [ + source for source in ARCHIVE_MEMBERS if source not in extracted + ] + + for source in IMPORT_ORDER: + if source not in extracted: + continue + refresh.runs.append(await import_from_local(source, mode)) + + return refresh diff --git a/cbh/api/importer/services/taric/taric.py b/cbh/api/importer/services/taric/taric.py index 57fa94df1c3b2a2fe6ad85e380f839c94b32386b..8aa9cbcebd3a54a310fd3963279afde352086ce2 100644 --- a/cbh/api/importer/services/taric/taric.py +++ b/cbh/api/importer/services/taric/taric.py @@ -1,16 +1,3 @@ -""" -The TARIC importer. - -Each source declares how to read it — column map, row parser, target model and -natural key — and the same generic pipeline loads all of them. Measure -conditions are the exception: they arrive as their own file and are folded into -the measures they belong to, because that is how the engine reads them. - -Reference data is never deleted. A reload supersedes records under their -natural key, so a rule that was in force last March still is, for a -declaration filed last March. -""" - from collections import defaultdict from dataclasses import dataclass from pathlib import Path @@ -18,7 +5,10 @@ from typing import Any, Callable, Iterable from pymongo import UpdateOne +from cbh.api.common.dto import TradeDirection +from cbh.api.common.utils import normalize_ws, parse_date from cbh.api.importer.dto import SyncMode, SyncSource +from cbh.api.taric.dto import GeoAreaType from cbh.api.importer.services.base import SyncRun, already_loaded, download from cbh.api.importer.services.taric.parsers import ( ADDITIONAL_CODE_COLUMNS, @@ -55,20 +45,27 @@ from cbh.api.taric.models import ( ) from cbh.core.config import settings -WRITE_BATCH_SIZE = 1000 - @dataclass(frozen=True) class SourceSpec: - """ - Everything needed to load one reference source. - """ - model: Any columns: dict[str, tuple[str, ...]] parser: Callable[[dict[str, Any]], dict[str, Any] | None] key_fields: list[str] xml_record_tag: str | None = None + # The extraction states the direction by which file a measure is in, not in + # a column, so the source supplies it. + defaults: dict[str, Any] | None = None + + +# Multilingual sheets repeat every record once per EU language. Only one pass is +# loaded, otherwise whichever language happens to be written last wins. +REFERENCE_LANGUAGE = "EN" + + +def _wanted_language(row: dict[str, Any]) -> bool: + language = str(row.get("language") or "").strip().upper() + return not language or language == REFERENCE_LANGUAGE SOURCE_SPECS: dict[SyncSource, SourceSpec] = { @@ -92,6 +89,7 @@ SOURCE_SPECS: dict[SyncSource, SourceSpec] = { parser=parse_measure_row, key_fields=["measureSid"], xml_record_tag="measure", + defaults={"direction": TradeDirection.IMPORT}, ), SyncSource.TARIC_DUTIES_EXPORT: SourceSpec( model=MeasureModel, @@ -99,6 +97,7 @@ SOURCE_SPECS: dict[SyncSource, SourceSpec] = { parser=parse_measure_row, key_fields=["measureSid"], xml_record_tag="measure", + defaults={"direction": TradeDirection.EXPORT}, ), SyncSource.TARIC_GEO_AREAS: SourceSpec( model=GeoAreaModel, @@ -130,21 +129,29 @@ SOURCE_SPECS: dict[SyncSource, SourceSpec] = { ), } +IMPORT_ORDER: list[SyncSource] = [ + SyncSource.TARIC_NOMENCLATURE, + SyncSource.TARIC_DECLARABLE_CODES, + SyncSource.TARIC_GEO_AREAS, + SyncSource.TARIC_ADDITIONAL_CODES, + SyncSource.TARIC_BOX44_CODES, + SyncSource.TARIC_DUTIES_IMPORT, + SyncSource.TARIC_DUTIES_EXPORT, + SyncSource.TARIC_MEASURE_CONDITIONS, + SyncSource.TARIC_CODE_CHANGES, +] + def read_rows( payload: bytes, file_name: str, spec: SourceSpec ) -> Iterable[dict[str, Any]]: - """ - Read a reference file in whichever of the published formats it arrives in. - """ name = file_name.lower() if name.endswith((".xlsx", ".xlsm", ".xls")): return read_xlsx_rows(payload, spec.columns) - if name.endswith(".csv") or name.endswith(".txt"): + if name.endswith((".csv", ".txt")): return read_csv_rows(payload, spec.columns) if name.endswith(".xml"): - field_map = {field: spellings for field, spellings in spec.columns.items()} - return read_xml_records(payload, spec.xml_record_tag or "record", field_map) + return read_xml_records(payload, spec.xml_record_tag or "record", spec.columns) if name.endswith(".json"): return read_json_records(payload) raise ValueError(f"Unsupported reference file format: {file_name}") @@ -153,13 +160,17 @@ def read_rows( async def _import_generic( run: SyncRun, payload: bytes, file_name: str, source: SyncSource ) -> None: - """ - Load one source through the generic parse-and-upsert pipeline. - """ spec = SOURCE_SPECS[source] buffer: list[dict[str, Any]] = [] for row in read_rows(payload, file_name, spec): + if not _wanted_language(row): + run.log.recordsSkipped += 1 + continue + + if spec.defaults: + row = {**spec.defaults, **{k: v for k, v in row.items() if v not in (None, "")}} + try: parsed = spec.parser(row) except (ValueError, TypeError) as error: @@ -172,7 +183,7 @@ async def _import_generic( continue buffer.append(spec.model(**parsed).to_mongo()) - if len(buffer) >= WRITE_BATCH_SIZE: + if len(buffer) >= 1000: await run.write(spec.model, buffer, spec.key_fields) buffer = [] @@ -180,15 +191,86 @@ async def _import_generic( await run.write(spec.model, buffer, spec.key_fields) -async def _import_measure_conditions(run: SyncRun, payload: bytes, file_name: str) -> None: +async def _import_declarable_codes(run: SyncRun, payload: bytes, file_name: str) -> None: + """Set the declarable flag without touching anything else. + + The declarable sheet repeats the nomenclature keys but carries no + descriptions, so writing whole documents here would blank the descriptions + the nomenclature pass just loaded. """ - Fold measure conditions into the measures they qualify. + spec = SOURCE_SPECS[SyncSource.TARIC_DECLARABLE_CODES] + operations: list[UpdateOne] = [] + + for row in read_rows(payload, file_name, spec): + parsed = parse_nomenclature_row(row) + if not parsed: + run.log.recordsSkipped += 1 + continue + run.log.recordsRead += 1 + # Round-trip through the model so the key matches how the nomenclature + # pass stored it: dates are serialised, not written as date objects. + document = NomenclatureModel(**parsed).to_mongo() + operations.append( + UpdateOne( + {"code": document["code"], "validFrom": document["validFrom"]}, + {"$set": {"declarable": document["declarable"]}}, + ) + ) - Conditions are published separately but only mean anything next to their - measure, so they are written onto it in one pass. Replacing the whole - ``conditions`` array keeps the import idempotent: re-running cannot append - the same condition twice. + for start in range(0, len(operations), 1000): + result = await settings.DB_CLIENT.nomenclature.bulk_write( + operations[start : start + 1000], ordered=False + ) + run.log.recordsMatched += result.matched_count + + +async def _import_geo_area_composition(run: SyncRun, payload: bytes, file_name: str) -> None: + """Fold one row per (group, member) pair into one document per group. + + The engine asks which areas cover a country — the country itself, the groups + it belongs to and Erga Omnes — so a group is only useful once it carries the + whole membership list. """ + spec = SOURCE_SPECS[SyncSource.TARIC_GEO_AREAS] + members: dict[str, set[str]] = defaultdict(set) + attributes: dict[str, dict[str, Any]] = {} + + for row in read_rows(payload, file_name, spec): + if not _wanted_language(row): + continue + code = str(row.get("code") or "").strip().upper() + member = str(row.get("memberCodes") or "").strip().upper() + if not code: + continue + run.log.recordsRead += 1 + if member: + members[code].add(member) + attributes.setdefault( + code, + { + "name": normalize_ws(row.get("name")), + "validFrom": parse_date(row.get("validFrom")), + "validTo": parse_date(row.get("validTo")), + }, + ) + + documents = [ + GeoAreaModel( + code=code, + name=attribute["name"], + areaType=GeoAreaType.GROUP if members.get(code) else GeoAreaType.COUNTRY, + memberCodes=sorted(members.get(code, set())), + validFrom=attribute["validFrom"], + validTo=attribute["validTo"], + ).to_mongo() + for code, attribute in attributes.items() + ] + + for start in range(0, len(documents), 1000): + await run.write(GeoAreaModel, documents[start : start + 1000], ["code", "validFrom"]) + + +async def _import_measure_conditions(run: SyncRun, payload: bytes, file_name: str) -> None: spec = SourceSpec( model=MeasureModel, columns=MEASURE_CONDITION_COLUMNS, @@ -199,7 +281,10 @@ async def _import_measure_conditions(run: SyncRun, payload: bytes, file_name: st grouped: dict[str, list[dict[str, Any]]] = defaultdict(list) for row in read_rows(payload, file_name, spec): - parsed = parse_measure_condition_row(row) + # The sheet has no direction column, and every measure type in it — + # preferences, quotas, tariff ceilings — is an import measure, so the + # composite key is built on that side. + parsed = parse_measure_condition_row({"direction": TradeDirection.IMPORT, **row}) if not parsed: run.log.recordsSkipped += 1 continue @@ -207,21 +292,22 @@ async def _import_measure_conditions(run: SyncRun, payload: bytes, file_name: st grouped[measure_sid].append(MeasureConditionEmbedded(**parsed).to_mongo()) run.log.recordsRead += 1 + # Measures already carry whatever their duty column stated, and that is the + # only source for prohibitions, restrictions and export controls. This sheet + # covers tariff and quota types, so it fills the measures that have nothing + # rather than replacing what is already there. operations = [ - UpdateOne({"measureSid": measure_sid}, {"$set": {"conditions": conditions}}) + UpdateOne( + {"measureSid": measure_sid, "conditions": {"$size": 0}}, + {"$set": {"conditions": conditions}}, + ) for measure_sid, conditions in grouped.items() ] - for start in range(0, len(operations), WRITE_BATCH_SIZE): - chunk = operations[start : start + WRITE_BATCH_SIZE] - result = await settings.DB_CLIENT.measures.bulk_write(chunk, ordered=False) - run.log.recordsMatched += result.matched_count - - missing = len(grouped) - run.log.recordsMatched - if missing > 0: - run.record_error( - f"{missing} measure(s) referenced by conditions were not found. " - "Load the duties files before the measure conditions." + for start in range(0, len(operations), 1000): + result = await settings.DB_CLIENT.measures.bulk_write( + operations[start : start + 1000], ordered=False ) + run.log.recordsMatched += result.matched_count async def import_source( @@ -231,9 +317,6 @@ async def import_source( mode: SyncMode = SyncMode.FULL, skip_unchanged: bool = True, ) -> SyncRun: - """ - Import one reference source from an in-memory file. - """ payload = load_payload(payload, file_name) async with SyncRun(source, mode, file_name=file_name) as run: @@ -244,6 +327,10 @@ async def import_source( run.set_hash(payload) if source is SyncSource.TARIC_MEASURE_CONDITIONS: await _import_measure_conditions(run, payload, file_name) + elif source is SyncSource.TARIC_GEO_AREAS: + await _import_geo_area_composition(run, payload, file_name) + elif source is SyncSource.TARIC_DECLARABLE_CODES: + await _import_declarable_codes(run, payload, file_name) else: await _import_generic(run, payload, file_name, source) return run @@ -252,21 +339,12 @@ async def import_source( async def import_from_url( source: SyncSource, url: str, mode: SyncMode = SyncMode.FULL ) -> SyncRun: - """ - Download and import one reference source. - """ payload = await download(url) file_name = Path(url.split("?")[0]).name or f"{source.name.lower()}.xlsx" return await import_source(source, payload, file_name, mode) async def import_from_local(source: SyncSource, mode: SyncMode = SyncMode.FULL) -> SyncRun: - """ - Import the most recent file supplied for a source in local storage. - - This is the path used until CIRCABC downloads are wired up: drop the - published file into the source's folder and run the import. - """ path = find_local_source_file(source) if path is None: raise FileNotFoundError( @@ -276,25 +354,7 @@ async def import_from_local(source: SyncSource, mode: SyncMode = SyncMode.FULL) return await import_source(source, path.read_bytes(), path.name, mode) -# The order sources must be loaded in: measures have to exist before their -# conditions can be attached to them. -IMPORT_ORDER: list[SyncSource] = [ - SyncSource.TARIC_NOMENCLATURE, - SyncSource.TARIC_DECLARABLE_CODES, - SyncSource.TARIC_GEO_AREAS, - SyncSource.TARIC_ADDITIONAL_CODES, - SyncSource.TARIC_BOX44_CODES, - SyncSource.TARIC_DUTIES_IMPORT, - SyncSource.TARIC_DUTIES_EXPORT, - SyncSource.TARIC_MEASURE_CONDITIONS, - SyncSource.TARIC_CODE_CHANGES, -] - - async def import_all_local(mode: SyncMode = SyncMode.FULL) -> list[SyncRun]: - """ - Load every TARIC source available in local storage, in dependency order. - """ runs: list[SyncRun] = [] for source in IMPORT_ORDER: if find_local_source_file(source) is None: diff --git a/cbh/api/importer/utils.py b/cbh/api/importer/utils.py index 0b94cc4690b070cf6cf56b75a1b9273f0c89caef..ba40a3f5feb85b2d765a140f952b3f01e373336c 100644 --- a/cbh/api/importer/utils.py +++ b/cbh/api/importer/utils.py @@ -1,7 +1,4 @@ -""" -Importer helpers: file handling, header matching and source URLs. -""" - +import csv import hashlib import io import json @@ -19,31 +16,25 @@ from cbh.core.config import settings def file_hash(payload: bytes) -> str: - """ - Content hash of a downloaded file, used to skip unchanged reloads. - """ return hashlib.sha256(payload).hexdigest() -def normalize_header(value: Any) -> str: - """ - Normalise a column header so that files can be matched loosely. +def file_hash_path(path: Path) -> str: + """The same digest, for files too large to hold in memory.""" + digest = hashlib.sha256() + with path.open("rb") as handle: + for chunk in iter(lambda: handle.read(1 << 20), b""): + digest.update(chunk) + return digest.hexdigest() - Reference files change their capitalisation, spacing and punctuation - between releases far more often than they change meaning. - """ + +def normalize_header(value: Any) -> str: return re.sub(r"[^a-z0-9]", "", str(value or "").strip().lower()) def build_header_index( headers: list[Any], column_map: dict[str, tuple[str, ...]] ) -> dict[str, int]: - """ - Map canonical field names onto column positions in a sheet. - - ``column_map`` gives the accepted spellings per canonical field; the first - header that matches wins. - """ normalized = {normalize_header(header): index for index, header in enumerate(headers)} index: dict[str, int] = {} for field, accepted in column_map.items(): @@ -58,12 +49,6 @@ def build_header_index( def read_xlsx_rows( payload: bytes, column_map: dict[str, tuple[str, ...]], sheet: str | None = None ) -> Iterator[dict[str, Any]]: - """ - Stream an Excel reference file as canonical field dictionaries. - - Read-only and value-only so that a 15 000-row nomenclature file does not - have to be materialised as objects. - """ workbook = load_workbook(io.BytesIO(payload), read_only=True, data_only=True) worksheet = workbook[sheet] if sheet and sheet in workbook.sheetnames else workbook.active @@ -93,15 +78,9 @@ def read_xlsx_rows( def read_csv_rows( payload: bytes, column_map: dict[str, tuple[str, ...]] ) -> Iterator[dict[str, Any]]: - """ - Stream a CSV reference file as canonical field dictionaries. - """ - import csv - text = payload.decode("utf-8-sig", errors="replace") - sample = text[:4096] try: - dialect = csv.Sniffer().sniff(sample, delimiters=",;\t|") + dialect = csv.Sniffer().sniff(text[:4096], delimiters=",;\t|") except csv.Error: dialect = csv.excel reader = csv.reader(io.StringIO(text), dialect) @@ -129,12 +108,6 @@ def read_csv_rows( def read_xml_records( payload: bytes, record_tag: str, field_map: dict[str, tuple[str, ...]] ) -> Iterator[dict[str, Any]]: - """ - Stream an XML reference file as canonical field dictionaries. - - Namespaces are ignored: TARIC and Codeboek XML both declare them but the - local tag names are what identify a field. - """ root = ElementTree.fromstring(payload) def local(tag: str) -> str: @@ -156,9 +129,6 @@ def read_xml_records( def read_json_records(payload: bytes, path: str | None = None) -> list[dict[str, Any]]: - """ - Read a JSON reference file, optionally descending into a nested key. - """ document = json.loads(payload.decode("utf-8-sig", errors="replace")) if path: for segment in path.split("."): @@ -171,65 +141,35 @@ def read_json_records(payload: bytes, path: str | None = None) -> list[dict[str, return document if isinstance(document, list) else [] -def unzip_single(payload: bytes, suffixes: tuple[str, ...] = (".xml", ".json")) -> bytes: - """ - Pull the first matching member out of a zip archive. - - Codeboek Douane ships its XML codebooks zipped, one codebook per archive. - """ +def load_payload(payload: bytes, file_name: str) -> bytes: + if not file_name.lower().endswith(".zip"): + return payload with zipfile.ZipFile(io.BytesIO(payload)) as archive: for name in archive.namelist(): - if name.lower().endswith(suffixes): + if name.lower().endswith((".xml", ".json")): return archive.read(name) - raise ValueError(f"No {' or '.join(suffixes)} member found in the archive.") - - -def load_payload(payload: bytes, file_name: str) -> bytes: - """ - Unwrap an archive when needed, otherwise pass the bytes through. - """ - if file_name.lower().endswith(".zip"): - return unzip_single(payload) - return payload + raise ValueError("No .xml or .json member found in the archive.") def codeboek_url(file_name: str, on_date: date | None = None) -> str: - """ - Build a Codeboek Douane URL. - - The paths are predictable: ``.../codebooks/[yyyyMMdd]/[filename]``. - """ stamp = (on_date or date.today()).strftime("%Y%m%d") return f"{settings.CODEBOEK_BASE_URL.rstrip('/')}/{stamp}/{file_name}" def local_source_path(source: SyncSource, file_name: str | None = None) -> Path: - """ - Where a manually supplied reference file for a source is kept. - - Until the CIRCABC download is wired to real credentials, files dropped here - are what the importer reads. - """ directory = settings.REFERENCE_STORAGE_DIR / source.name.lower() directory.mkdir(parents=True, exist_ok=True) return directory / file_name if file_name else directory def find_local_source_file(source: SyncSource) -> Path | None: - """ - Most recently modified file supplied for a source. - """ - directory = local_source_path(source) - candidates = [path for path in directory.glob("*") if path.is_file()] + candidates = [path for path in local_source_path(source).glob("*") if path.is_file()] if not candidates: return None return max(candidates, key=lambda path: path.stat().st_mtime) def as_bool(value: Any, default: bool = False) -> bool: - """ - Read a boolean-ish reference-data cell. - """ if value is None or value == "": return default if isinstance(value, bool): diff --git a/cbh/api/importer/views.py b/cbh/api/importer/views.py index 0cd57746159806e3678ba70bc3d4492c3d154592..74818f957c73581d092316b6976b87ad59287df6 100644 --- a/cbh/api/importer/views.py +++ b/cbh/api/importer/views.py @@ -1,13 +1,7 @@ -""" -Importer views module. -""" +from fastapi import File, Form, HTTPException, Query, UploadFile -from fastapi import Depends, File, Form, HTTPException, Query, UploadFile - -from cbh.api.account.dto import AccountType -from cbh.api.account.models import AccountModel +from cbh.api.common.dto import Paging from cbh.api.common.schemas import AllObjectsResponse -from cbh.api.common.utils import to_paging from cbh.api.importer import importer_router from cbh.api.importer.db_requests import get_latest_sync_per_source, get_sync_logs from cbh.api.importer.dto import SyncMode, SyncSource, SyncStatus @@ -15,40 +9,35 @@ from cbh.api.importer.models import SyncLogModel from cbh.api.importer.schemas import ( CodebookSyncRequest, ImporterStatusResponse, + PublicationSyncRequest, + PublicationSyncResponse, SyncLogResponse, SyncRequest, ) from cbh.api.importer.services.codeboek import sync_all_codebooks, sync_codebook +from cbh.api.importer.services.dtv import refresh_national_measures from cbh.api.importer.services.demo import seed_demo_reference_data from cbh.api.importer.services.taric import ( import_all_local, import_from_local, import_from_url, import_source, + refresh_from_publication, + resolve_latest_publication, ) +from cbh.api.importer.services.taric.publication import candidate_periods from cbh.api.importer.utils import local_source_path from cbh.api.taric.db_requests import count_reference_data, get_reference_as_of -from cbh.core.security import PermissionDependency +from cbh.core.config import settings from cbh.core.wrappers import CbhResponseWrapper -ADMIN_ONLY = [AccountType.ADMIN, AccountType.OWNER, AccountType.SUPER_ADMIN] - def _to_response(log: SyncLogModel) -> SyncLogResponse: - """ - Map a sync log onto its API representation. - """ return SyncLogResponse(**log.model_dump()) @importer_router.post("/sync") -async def trigger_sync( - data: SyncRequest, - account: AccountModel = Depends(PermissionDependency(ADMIN_ONLY)), -) -> CbhResponseWrapper[SyncLogResponse]: - """ - Import one TARIC source, from a URL or from the file supplied locally. - """ +async def trigger_sync(data: SyncRequest) -> CbhResponseWrapper[SyncLogResponse]: try: if data.url: run = await import_from_url(data.source, data.url, data.mode) @@ -64,35 +53,56 @@ async def trigger_sync( @importer_router.post("/sync-all") async def trigger_sync_all( mode: SyncMode = Query(default=SyncMode.FULL), - account: AccountModel = Depends(PermissionDependency(ADMIN_ONLY)), ) -> CbhResponseWrapper[list[SyncLogResponse]]: - """ - Import every TARIC source available locally, in dependency order. - """ runs = await import_all_local(mode) return CbhResponseWrapper(data=[_to_response(run.log) for run in runs]) +@importer_router.post("/sync-publication") +async def trigger_publication_sync( + data: PublicationSyncRequest, +) -> CbhResponseWrapper[PublicationSyncResponse]: + refresh = await refresh_from_publication(data.mode, data.period, data.keepArchive) + return CbhResponseWrapper( + data=PublicationSyncResponse( + period=refresh.period, + url=refresh.url, + runs=[_to_response(run.log) for run in refresh.runs], + missingSources=refresh.missingSources, + note=refresh.note, + ) + ) + + +@importer_router.post("/national/sync") +async def trigger_national_sync( + mode: SyncMode = Query(default=SyncMode.FULL), + keepArchive: bool = Query(default=False), +) -> CbhResponseWrapper[PublicationSyncResponse]: + """Load the Dutch national measures from the Gebruikstarief snapshot.""" + refresh = await refresh_national_measures(mode, keepArchive) + return CbhResponseWrapper( + data=PublicationSyncResponse( + period=refresh.fileName, + url=refresh.url, + runs=[_to_response(run.log) for run in refresh.runs], + note=refresh.note, + ) + ) + + @importer_router.post("/upload") async def upload_reference_file( source: SyncSource = Form(...), mode: SyncMode = Form(default=SyncMode.FULL), keepFile: bool = Form(default=True), file: UploadFile = File(...), - account: AccountModel = Depends(PermissionDependency(ADMIN_ONLY)), ) -> CbhResponseWrapper[SyncLogResponse]: - """ - Import a reference file supplied by hand. - - This is the path used while the CIRCABC download is not wired up: upload - the published extract and it is parsed with the same pipeline. - """ payload = await file.read() if not payload: raise HTTPException(status_code=400, detail="Empty file.") if keepFile: - # Keep the file so the run can be repeated and audited later. local_source_path(source, file.filename).write_bytes(payload) try: @@ -105,11 +115,7 @@ async def upload_reference_file( @importer_router.post("/codebooks/sync") async def trigger_codebook_sync( data: CodebookSyncRequest, - account: AccountModel = Depends(PermissionDependency(ADMIN_ONLY)), ) -> CbhResponseWrapper[list[SyncLogResponse]]: - """ - Refresh the Dutch national codebooks by full replacement. - """ if data.fileName: runs = [await sync_codebook(data.fileName, data.publishedOn)] else: @@ -118,16 +124,7 @@ async def trigger_codebook_sync( @importer_router.post("/seed-demo") -async def seed_demo( - account: AccountModel = Depends(PermissionDependency(ADMIN_ONLY)), -) -> CbhResponseWrapper[SyncLogResponse]: - """ - Load the demo reference dataset. - - Illustrative data for the chapters in scope, so the pipeline can be - exercised end to end before the real extracts arrive. It must be replaced - by a real import before any acceptance run. - """ +async def seed_demo() -> CbhResponseWrapper[SyncLogResponse]: run = await seed_demo_reference_data() return CbhResponseWrapper(data=_to_response(run.log)) @@ -138,15 +135,11 @@ async def list_sync_logs( status: SyncStatus | None = Query(default=None), pageSize: int = Query(default=25, le=200), pageIndex: int = Query(default=0, ge=0), - account: AccountModel = Depends(PermissionDependency(ADMIN_ONLY)), ) -> CbhResponseWrapper[AllObjectsResponse[SyncLogResponse]]: - """ - Page through the import history. - """ logs, total = await get_sync_logs(pageSize, pageIndex, source, status) return CbhResponseWrapper( data=AllObjectsResponse( - paging=to_paging(pageSize, pageIndex, total), + paging=Paging(pageSize=pageSize, pageIndex=pageIndex, totalCount=total), data=[_to_response(log) for log in logs], ) ) @@ -154,18 +147,23 @@ async def list_sync_logs( @importer_router.get("/status") async def get_importer_status( - account: AccountModel = Depends(PermissionDependency(ADMIN_ONLY)), + checkPublication: bool = Query(default=False), ) -> CbhResponseWrapper[ImporterStatusResponse]: - """ - Which reference files are loaded, how fresh they are, and how much of each. - """ latest = await get_latest_sync_per_source() + + published = await resolve_latest_publication() if checkPublication else None + jobs = { + job.id: job.next_run_time + for job in (settings.SCHEDULER.get_jobs() if settings.SCHEDULER.running else []) + } + return CbhResponseWrapper( data=ImporterStatusResponse( - latestPerSource={ - name: _to_response(log) for name, log in latest.items() - }, + latestPerSource={name: _to_response(log) for name, log in latest.items()}, counts=await count_reference_data(), referenceAsOf=await get_reference_as_of(), + publicationPeriods=candidate_periods(), + latestPublication=published[0] if published else None, + scheduledJobs=jobs, ) ) diff --git a/cbh/api/linecheck/__init__.py b/cbh/api/linecheck/__init__.py index 064ff43e9c6a322a78a39262946385324b0af6c8..fbf02477dea5bf4c5b75ee9282e5db2379f9f869 100644 --- a/cbh/api/linecheck/__init__.py +++ b/cbh/api/linecheck/__init__.py @@ -1,10 +1,3 @@ -""" -Line-check module initialization. - -This module defines the FastAPI router for the line-check API — the single -entry point holding the code-resolution and document-determination logic. -""" - from fastapi import APIRouter linecheck_router = APIRouter( diff --git a/cbh/api/linecheck/dto.py b/cbh/api/linecheck/dto.py index 8191b29d228ebaadf39ff2bbb850a83c8663aa92..31a5a37596a5a4929f0f4e236b175530eb43d2ae 100644 --- a/cbh/api/linecheck/dto.py +++ b/cbh/api/linecheck/dto.py @@ -1,19 +1,7 @@ -""" -Line-check DTOs. -""" +from enum import IntEnum -from enum import Enum - - -class LineStatus(Enum): - """ - Outcome of a single declaration line. - - ``AMBIGUOUS`` and ``INVALID`` lines are highlighted for a human and never - corrected automatically — guessing a code is exactly the failure mode that - leaves a client with open obligations towards customs. - """ +class LineStatus(IntEnum): OK = 1 AMBIGUOUS = 2 INVALID = 3 @@ -21,30 +9,16 @@ class LineStatus(Enum): @property def needs_review(self) -> bool: - """ - Whether a broker has to look at this line before filing. - """ return self in (LineStatus.AMBIGUOUS, LineStatus.INVALID) -class ResolutionMethod(Enum): - """ - How the 10-digit code on the output line was arrived at. - """ - +class ResolutionMethod(IntEnum): EXACT_10 = 1 EXPANDED_FROM_PREFIX = 2 NOT_RESOLVED = 3 -class WarningType(Enum): - """ - Warnings a line can carry. - - Dual-use is deliberately flagged as out of scope rather than answered: it - needs a case-by-case assessment that this phase does not attempt. - """ - +class WarningType(IntEnum): ANTIDUMPING = 1 COUNTERVAILING = 2 SAFEGUARD = 3 @@ -68,29 +42,42 @@ class WarningType(Enum): COUNTRY_UNKNOWN = 21 CHAPTER_OUT_OF_SCOPE = 22 ADDITIONAL_CODE_REQUIRED = 23 + NVWA_CHAPTER_EXCLUDED = 24 + DESTINATION_UNKNOWN = 25 + RULE_APPLIED = 26 + TRANSIT_DUTY_SUSPENDED = 27 + NATIONAL_MEASURE = 28 -class WarningSeverity(Enum): - """ - How hard a warning blocks filing. - - ``BLOCKING`` means the line cannot be filed as it stands; ``WARNING`` means - a human must confirm; ``INFO`` is context only. - """ - +class WarningSeverity(IntEnum): INFO = 1 WARNING = 2 BLOCKING = 3 -class DocumentRequirementType(Enum): - """ - Role a document code plays on a line. - - ``ALTERNATIVE`` documents belong to an either/or group: presenting any one - of them satisfies the same condition. - """ - +class DocumentRequirementType(IntEnum): REQUIRED = 1 ALTERNATIVE = 2 EXEMPTION = 3 + # Owed, but not on this declaration: the procedure defers it to the office + # where the goods are finally released. + SUSPENDED = 4 + + +class ConfidenceLevel(IntEnum): + RED = 1 + AMBER = 2 + GREEN = 3 + + +class ReferenceSource(IntEnum): + TARIC = 1 + CODEBOEK = 2 + MANUAL = 3 + NONE = 99 + + +class RowDecision(IntEnum): + PENDING = 1 + ACCEPTED = 2 + CORRECTED = 3 diff --git a/cbh/api/linecheck/schemas.py b/cbh/api/linecheck/schemas.py index 04ff42b08332a47ca77c01810614d63c54b5b8df..a19de84f459065362b2322fe570338f57a732b2d 100644 --- a/cbh/api/linecheck/schemas.py +++ b/cbh/api/linecheck/schemas.py @@ -1,40 +1,29 @@ -""" -Line-check request and response schemas. - -These are the contract of the single entry point that both the Excel processor -and any future declaration system speak to. -""" - from datetime import date -from pydantic import BaseModel, Field +from pydantic import BaseModel, Field, model_validator -from cbh.api.common.dto import TradeDirection +from cbh.api.common.dto import CustomsProcedure, TradeDirection from cbh.api.linecheck.dto import ( + ConfidenceLevel, DocumentRequirementType, LineStatus, + ReferenceSource, ResolutionMethod, WarningSeverity, WarningType, ) +from cbh.api.rules.dto import RuleAction from cbh.api.taric.dto import DocumentCodeType, MeasureCategory -# A field named `date` shadows the `date` type for the rest of its model body, -# so models carrying a declaration date annotate it with this alias. DeclarationDate = date class LineCheckRequest(BaseModel): - """ - One declaration line to check. - - The commercial fields are carried through untouched so the Excel processor - can rebuild the client's row without a second lookup. - """ - code: str - country: str + originCountry: str | None = None + destinationCountry: str | None = None direction: TradeDirection = TradeDirection.IMPORT + procedure: CustomsProcedure = CustomsProcedure.RELEASE_FOR_FREE_CIRCULATION date: DeclarationDate | None = None additionalCodes: list[str] = Field(default_factory=list) @@ -47,41 +36,49 @@ class LineCheckRequest(BaseModel): rowIndex: int | None = None - def cache_key_parts(self) -> tuple: - """ - The parts that actually determine the answer. - - Commercial fields are excluded on purpose: two rows differing only in - quantity resolve identically, which is what collapses a 580-row pack - into a few dozen real lookups. The code and country are normalised - first so that ``6109.10.00 10`` and ``6109100010`` share one lookup. - """ + # Only used to pick the broker's own rules. It is deliberately absent from + # cache_key_parts: the engine's answer is tenant-free, rules go on top. + clientProfileId: str | None = None + + @model_validator(mode="before") + @classmethod + def accept_legacy_country(cls, data): + if isinstance(data, dict) and data.get("country") and not data.get("originCountry"): + data = {**data, "originCountry": data["country"]} + return data + + @property + def measure_country(self) -> str | None: from cbh.api.common.utils import normalize_country - from cbh.api.taric.utils import normalize_code + if self.direction is TradeDirection.EXPORT: + preferred, fallback = self.destinationCountry, self.originCountry + else: + preferred, fallback = self.originCountry, self.destinationCountry + return normalize_country(preferred) or normalize_country(fallback) + + def cache_key_parts(self) -> tuple: + from cbh.api.taric.utils import normalize_code, split_additional_code + + additional = sorted( + set(self.additionalCodes) | {split_additional_code(self.code) or ""} - {""} + ) return ( normalize_code(self.code) or str(self.code).strip(), - normalize_country(self.country) or str(self.country or "").strip().upper(), + self.measure_country or "", self.direction.value, + self.procedure.value, self.date.isoformat() if self.date else None, - ",".join(sorted(self.additionalCodes)), + ",".join(additional), ) class BatchLineCheckRequest(BaseModel): - """ - A pack of declaration lines checked in one call. - """ - lines: list[LineCheckRequest] date: DeclarationDate | None = None class CodeCandidate(BaseModel): - """ - A declarable code a 6- or 8-digit input could resolve to. - """ - code: str formattedCode: str description: str | None = None @@ -90,15 +87,15 @@ class CodeCandidate(BaseModel): class DocumentRequirement(BaseModel): - """ - A document obligation, alternative or exemption derived from a measure. - """ - code: str documentType: DocumentCodeType requirementType: DocumentRequirementType description: str | None = None + source: ReferenceSource = ReferenceSource.TARIC + validFrom: date | None = None + validTo: date | None = None + measureSid: str | None = None measureTypeId: str | None = None measureTypeDescription: str | None = None @@ -107,10 +104,6 @@ class DocumentRequirement(BaseModel): class LineWarning(BaseModel): - """ - A warning attached to a line. - """ - type: WarningType severity: WarningSeverity message: str @@ -121,11 +114,17 @@ class LineWarning(BaseModel): relatedCodes: list[str] = Field(default_factory=list) -class AppliedMeasure(BaseModel): - """ - Audit trail: a measure the engine actually used for this line. - """ +class AppliedRule(BaseModel): + """A broker rule that fired on this line, named so the review table can show it.""" + ruleId: str + name: str + action: RuleAction + conditions: dict[str, str] = Field(default_factory=dict) + note: str | None = None + + +class AppliedMeasure(BaseModel): measureSid: str goodsCode: str measureTypeId: str @@ -134,19 +133,17 @@ class AppliedMeasure(BaseModel): geoAreaCode: str validFrom: date | None = None validTo: date | None = None + national: bool = False + suspended: bool = False class LineCheckResponse(BaseModel): - """ - The answer for one declaration line. - - ``noneRequired`` is only ever true when the check completed and no measure - demanded a document — it is never a default for "we could not tell". - """ - inputCode: str - country: str + originCountry: str | None = None + destinationCountry: str | None = None + measureCountry: str | None = None direction: TradeDirection + procedure: CustomsProcedure = CustomsProcedure.RELEASE_FOR_FREE_CIRCULATION checkedOn: date resolvedCode: str | None = None @@ -159,42 +156,34 @@ class LineCheckResponse(BaseModel): documentsRequired: list[DocumentRequirement] = Field(default_factory=list) documentsAlternatives: list[list[DocumentRequirement]] = Field(default_factory=list) documentsExemption: list[DocumentRequirement] = Field(default_factory=list) + documentsSuspended: list[DocumentRequirement] = Field(default_factory=list) noneRequired: bool = False warnings: list[LineWarning] = Field(default_factory=list) status: LineStatus = LineStatus.INVALID expiringOn: date | None = None + codeConfidence: ConfidenceLevel = ConfidenceLevel.RED + codeConfidenceReason: str = "" + documentConfidence: ConfidenceLevel = ConfidenceLevel.RED + documentConfidenceReason: str = "" + source: ReferenceSource = ReferenceSource.NONE + appliedMeasures: list[AppliedMeasure] = Field(default_factory=list) + appliedRules: list[AppliedRule] = Field(default_factory=list) referenceAsOf: str | None = None rowIndex: int | None = None - @property - def required_codes(self) -> list[str]: - """ - Flat list of obligation codes, alternatives included. - """ - codes = [document.code for document in self.documentsRequired] - for group in self.documentsAlternatives: - codes.extend(document.code for document in group) - return codes - - @property - def exemption_codes(self) -> list[str]: - """ - Flat list of exemption codes. - """ - return [document.code for document in self.documentsExemption] - class BatchLineCheckResponse(BaseModel): - """ - Results for a pack, plus the numbers that make the saving visible. - """ - results: list[LineCheckResponse] totalLines: int uniqueLookups: int statusCounts: dict[str, int] machineSeconds: float + + extractionScore: float = 0.0 + documentScore: float = 0.0 + codeConfidenceCounts: dict[str, int] = Field(default_factory=dict) + documentConfidenceCounts: dict[str, int] = Field(default_factory=dict) diff --git a/cbh/api/linecheck/services/engine/__init__.py b/cbh/api/linecheck/services/engine/__init__.py index b210948952e0df58a617bfc2f1cdeb7087b2eff7..04388959654572ccc4fcccfb37dc00d6a0b00174 100644 --- a/cbh/api/linecheck/services/engine/__init__.py +++ b/cbh/api/linecheck/services/engine/__init__.py @@ -1,3 +1,3 @@ -from .engine import check_line, check_line_cached, check_lines +from .engine import check_line, check_line_cached, check_lines, recount_confidence -__all__ = ["check_line", "check_line_cached", "check_lines"] +__all__ = ["check_line", "check_line_cached", "check_lines", "recount_confidence"] diff --git a/cbh/api/linecheck/services/engine/engine.py b/cbh/api/linecheck/services/engine/engine.py index 806f10ff8880b65cfa618f3ff1e9bfb14e23108a..f6be92c307f0e6b02915c8f2ec5bef793b0bced3 100644 --- a/cbh/api/linecheck/services/engine/engine.py +++ b/cbh/api/linecheck/services/engine/engine.py @@ -1,24 +1,17 @@ -""" -The line-check engine: the single entry point both the Excel processor and any -future declaration system go through. - -A pack is not checked row by row. Identical (code, country, direction, date) -tuples are resolved once and the answer is fanned back out over every row that -shares them — a ~580-line pack usually collapses to a few dozen real lookups. -""" - import asyncio import time from datetime import date -from cbh.api.common.services.cache import ( - LINE_CHECK_NAMESPACE, - build_cache_key, - get_cache, - set_cache, -) +from cbh.api.common.services.cache import build_cache_key, get_cache, set_cache from cbh.api.common.utils import normalize_country -from cbh.api.linecheck.dto import LineStatus, WarningSeverity +from cbh.api.linecheck.dto import ( + ConfidenceLevel, + LineStatus, + ReferenceSource, + ResolutionMethod, + WarningSeverity, + WarningType, +) from cbh.api.linecheck.schemas import ( BatchLineCheckResponse, LineCheckRequest, @@ -31,45 +24,33 @@ from cbh.api.linecheck.services.warnings import ( build_chapter_scope_warning, build_country_warning, build_measure_warnings, + build_national_measure_warning, build_no_measure_warning, + build_transit_warning, ) from cbh.api.taric.db_requests import get_reference_as_of -from cbh.api.taric.utils import chapter_of, format_code -from cbh.core.config import settings - -# How many distinct tuples are resolved concurrently. Bounded so a large pack -# cannot exhaust the Mongo connection pool. -MAX_CONCURRENCY = 16 - - -def _worst_status(current: LineStatus, candidate: LineStatus) -> LineStatus: - """ - Combine statuses, keeping the one that most needs a human. - """ - order = [LineStatus.OK, LineStatus.EXPIRING, LineStatus.AMBIGUOUS, LineStatus.INVALID] - return max(current, candidate, key=order.index) +from cbh.api.taric.utils import chapter_scope, format_code, split_additional_code async def check_line( request: LineCheckRequest, reference_as_of: str | None = None ) -> LineCheckResponse: - """ - Check one declaration line: resolve the code, determine the documents, - collect the warnings. - """ on_date = request.date or date.today() - country = normalize_country(request.country) + country = request.measure_country response = LineCheckResponse( inputCode=str(request.code), - country=country or str(request.country or ""), + originCountry=normalize_country(request.originCountry), + destinationCountry=normalize_country(request.destinationCountry), + measureCountry=country, direction=request.direction, + procedure=request.procedure, checkedOn=on_date, rowIndex=request.rowIndex, referenceAsOf=reference_as_of, ) - resolution = await resolve_code(request.code, on_date) + resolution = await resolve_code(request.code, on_date, request.direction) response.resolutionMethod = resolution.method response.candidates = resolution.candidates response.replacementCodes = resolution.replacementCodes @@ -78,35 +59,48 @@ async def check_line( response.expiringOn = resolution.expiringOn if not resolution.is_resolved: - # Without a code there is nothing to hang documents off; the line goes - # back flagged rather than half-answered. response.noneRequired = False + _apply_confidence(response) return response response.resolvedCode = resolution.resolvedCode response.formattedCode = format_code(resolution.resolvedCode) response.codeDescription = resolution.resolved.description if resolution.resolved else None - if chapter_of(resolution.resolvedCode) not in settings.IN_SCOPE_CHAPTERS: - response.warnings.append(build_chapter_scope_warning(chapter_of(resolution.resolvedCode))) + chapter = resolution.resolvedCode[:2] + scope = chapter_scope(chapter) + if scope != "in_scope": + response.warnings.append(build_chapter_scope_warning(chapter, scope)) if not country: - response.warnings.append(build_country_warning(request.country)) + response.warnings.append( + build_country_warning( + request.direction, request.originCountry, request.destinationCountry + ) + ) response.status = LineStatus.INVALID response.noneRequired = False + _apply_confidence(response) return response + additional_codes = list(request.additionalCodes) + embedded = split_additional_code(request.code) + if embedded and embedded not in additional_codes: + additional_codes.append(embedded) + outcome = await determine_documents( resolution.resolvedCode, country, request.direction, on_date, - request.additionalCodes, + additional_codes, + request.procedure, ) response.documentsRequired = outcome.required response.documentsAlternatives = outcome.alternatives response.documentsExemption = outcome.exemptions + response.documentsSuspended = outcome.suspended response.appliedMeasures = outcome.appliedMeasures response.noneRequired = outcome.none_required @@ -114,49 +108,153 @@ async def check_line( if not outcome.measures: response.warnings.append(build_no_measure_warning(resolution.resolvedCode, country)) - additional_code_warning = build_additional_code_warning( - outcome.measures, request.additionalCodes - ) - if additional_code_warning: - response.warnings.append(additional_code_warning) + for builder in ( + build_additional_code_warning(outcome.measures, request.additionalCodes), + build_national_measure_warning(outcome.measures), + build_transit_warning(request.procedure, outcome.suspendedMeasures), + ): + if builder: + response.warnings.append(builder) if any(warning.severity is WarningSeverity.BLOCKING for warning in response.warnings): - response.status = _worst_status(response.status, LineStatus.INVALID) + order = [LineStatus.OK, LineStatus.EXPIRING, LineStatus.AMBIGUOUS, LineStatus.INVALID] + response.status = max(response.status, LineStatus.INVALID, key=order.index) + _apply_confidence(response) return response +def _apply_confidence(response: LineCheckResponse) -> None: + blocking = [w for w in response.warnings if w.severity is WarningSeverity.BLOCKING] + + if response.resolvedCode is None: + response.codeConfidence = ConfidenceLevel.RED + response.codeConfidenceReason = ( + blocking[0].message if blocking else "The goods code could not be resolved." + ) + elif response.resolutionMethod is ResolutionMethod.EXPANDED_FROM_PREFIX: + response.codeConfidence = ConfidenceLevel.AMBER + response.codeConfidenceReason = ( + "Expanded from a partial code to the single declarable code in force." + ) + elif response.status is LineStatus.EXPIRING: + response.codeConfidence = ConfidenceLevel.AMBER + response.codeConfidenceReason = ( + f"Code is valid but expires on {response.expiringOn.isoformat()}." + if response.expiringOn + else "Code expires soon." + ) + else: + response.codeConfidence = ConfidenceLevel.GREEN + response.codeConfidenceReason = ( + f"Code {format_code(response.resolvedCode)} is declarable and in force on " + f"{response.checkedOn.isoformat()}." + ) + + types = {warning.type for warning in response.warnings} + + if response.resolvedCode is None or not response.appliedMeasures: + response.documentConfidence = ConfidenceLevel.RED + response.documentConfidenceReason = ( + "No measures found for this code and country; reference data may not cover it." + if response.resolvedCode + else "Documents cannot be determined without a resolved code." + ) + response.source = ReferenceSource.NONE + return + + response.source = ReferenceSource.TARIC + unresolved = [ + document + for group in [response.documentsRequired, response.documentsExemption] + for document in group + if document.source is ReferenceSource.NONE + ] + if WarningType.ADDITIONAL_CODE_REQUIRED in types: + response.documentConfidence = ConfidenceLevel.AMBER + response.documentConfidenceReason = ( + "A measure needs an additional code before the obligation is pinned down." + ) + elif response.documentsAlternatives: + response.documentConfidence = ConfidenceLevel.AMBER + response.documentConfidenceReason = ( + "One or more obligations are either/or; pick the code that applies." + ) + elif unresolved: + response.documentConfidence = ConfidenceLevel.AMBER + response.documentConfidenceReason = ( + f"{len(unresolved)} document code(s) are not described in the loaded Box 44 list." + ) + elif response.documentsSuspended and not response.documentsRequired: + # Suppressing an obligation because of the procedure is the engine's own + # reading of the category, not something the reference data spells out. + # It is shown, and it is not sold as certainty. + response.documentConfidence = ConfidenceLevel.AMBER + response.documentConfidenceReason = ( + f"{len(response.documentsSuspended)} obligation(s) are suspended by the " + "procedure and settled at the office of destination; confirm the guarantee " + "covers them." + ) + elif response.noneRequired: + response.documentConfidence = ConfidenceLevel.GREEN + response.documentConfidenceReason = ( + f"{len(response.appliedMeasures)} measure(s) applied and none demands a document." + ) + else: + response.documentConfidence = ConfidenceLevel.GREEN + response.documentConfidenceReason = ( + f"Derived from {len(response.appliedMeasures)} measure(s) in force on the check date." + ) + + async def check_line_cached( request: LineCheckRequest, reference_as_of: str | None = None ) -> LineCheckResponse: - """ - Check a line, reusing a cached answer for an identical tuple. - - The cache key includes the reference-data version, so an importer run - naturally retires every answer built on superseded rules. - """ key = build_cache_key(*request.cache_key_parts(), reference_as_of) - cached = await get_cache(LINE_CHECK_NAMESPACE, key) + cached = await get_cache("linecheck", key) if cached: response = LineCheckResponse(**cached) response.rowIndex = request.rowIndex return response response = await check_line(request, reference_as_of) - await set_cache(LINE_CHECK_NAMESPACE, key, response.model_dump(mode="json")) + await set_cache("linecheck", key, response.model_dump(mode="json")) return response +def recount_confidence(batch_result: BatchLineCheckResponse) -> BatchLineCheckResponse: + """Recompute the two scores from the results as they now stand. + + `check_lines` scores the reference answer. Anything applied afterwards — + a broker's rule, a correction — moves confidence, so the header figures + have to be counted again or they describe an answer nobody is looking at. + """ + status_counts: dict[str, int] = {} + code_counts: dict[str, int] = {} + document_counts: dict[str, int] = {} + for result in batch_result.results: + status_counts[result.status.name] = status_counts.get(result.status.name, 0) + 1 + code_counts[result.codeConfidence.name] = code_counts.get(result.codeConfidence.name, 0) + 1 + document_counts[result.documentConfidence.name] = ( + document_counts.get(result.documentConfidence.name, 0) + 1 + ) + + total = len(batch_result.results) or 1 + batch_result.statusCounts = status_counts + batch_result.codeConfidenceCounts = code_counts + batch_result.documentConfidenceCounts = document_counts + batch_result.extractionScore = round( + 100.0 * code_counts.get(ConfidenceLevel.GREEN.name, 0) / total, 1 + ) + batch_result.documentScore = round( + 100.0 * document_counts.get(ConfidenceLevel.GREEN.name, 0) / total, 1 + ) + return batch_result + + async def check_lines( requests: list[LineCheckRequest], default_date: date | None = None ) -> BatchLineCheckResponse: - """ - Check a whole pack. - - Rows are grouped by the tuple that actually determines the answer, each - distinct tuple is resolved once, and the results are fanned back out so the - caller gets one answer per input row, in the original order. - """ started = time.perf_counter() reference_as_of = await get_reference_as_of() @@ -166,7 +264,7 @@ async def check_lines( request.date = default_date groups.setdefault(request.cache_key_parts(), []).append(index) - semaphore = asyncio.Semaphore(MAX_CONCURRENCY) + semaphore = asyncio.Semaphore(16) async def resolve_group(key: tuple) -> tuple[tuple, LineCheckResponse]: async with semaphore: @@ -179,20 +277,33 @@ async def check_lines( for key, indexes in groups.items(): template = resolved[key] for index in indexes: - # Each row gets its own copy: the answer is shared, the row identity - # is not. row_result = template.model_copy(deep=True) row_result.rowIndex = requests[index].rowIndex results[index] = row_result status_counts: dict[str, int] = {} + code_counts: dict[str, int] = {} + document_counts: dict[str, int] = {} for result in results: status_counts[result.status.name] = status_counts.get(result.status.name, 0) + 1 + code_counts[result.codeConfidence.name] = code_counts.get(result.codeConfidence.name, 0) + 1 + document_counts[result.documentConfidence.name] = ( + document_counts.get(result.documentConfidence.name, 0) + 1 + ) + total = len(results) or 1 return BatchLineCheckResponse( results=results, totalLines=len(requests), uniqueLookups=len(groups), statusCounts=status_counts, machineSeconds=round(time.perf_counter() - started, 3), + extractionScore=round( + 100.0 * code_counts.get(ConfidenceLevel.GREEN.name, 0) / total, 1 + ), + documentScore=round( + 100.0 * document_counts.get(ConfidenceLevel.GREEN.name, 0) / total, 1 + ), + codeConfidenceCounts=code_counts, + documentConfidenceCounts=document_counts, ) diff --git a/cbh/api/linecheck/services/measures/measures.py b/cbh/api/linecheck/services/measures/measures.py index 4427569f3ac733c5f9097c57536fa6e4bec1eaf3..ae39cddb7cefae64e870b3abd7c732e7aa90ad15 100644 --- a/cbh/api/linecheck/services/measures/measures.py +++ b/cbh/api/linecheck/services/measures/measures.py @@ -1,72 +1,56 @@ -""" -Step 2 of a line check: turning measures into document obligations. - -For a resolved code, direction, date and the geographical areas covering the -country of origin/destination, the engine collects the applicable measures and -reads their conditions: - -* conditions sharing a condition code are alternatives — presenting any one of - them satisfies the same obligation (either/or); -* a condition carrying a Y code, or whose action makes the measure - inapplicable, is an exemption route rather than an obligation; -* ``noneRequired`` is only set when the lookup actually ran and came back - empty-handed — never as a default. -""" - from dataclasses import dataclass, field from datetime import date -from cbh.api.common.dto import TradeDirection +from cbh.api.common.dto import CustomsProcedure, TradeDirection from cbh.api.common.utils import dedupe -from cbh.api.linecheck.dto import DocumentRequirementType +from cbh.api.linecheck.dto import DocumentRequirementType, ReferenceSource from cbh.api.linecheck.schemas import AppliedMeasure, DocumentRequirement from cbh.api.taric.db_requests import ( find_measures_for_line, get_document_codes, get_geo_area_codes_for_country, ) -from cbh.api.taric.models import MeasureConditionEmbedded, MeasureModel +from cbh.api.taric.models import DocumentCodeModel, MeasureConditionEmbedded, MeasureModel +from cbh.api.taric.utils import measure_is_suspended from cbh.core.config import settings @dataclass class DocumentOutcome: - """ - Documents and measures established for a line. - """ - required: list[DocumentRequirement] = field(default_factory=list) alternatives: list[list[DocumentRequirement]] = field(default_factory=list) exemptions: list[DocumentRequirement] = field(default_factory=list) + suspended: list[DocumentRequirement] = field(default_factory=list) appliedMeasures: list[AppliedMeasure] = field(default_factory=list) measures: list[MeasureModel] = field(default_factory=list) + suspendedMeasures: list[MeasureModel] = field(default_factory=list) evaluated: bool = False @property def none_required(self) -> bool: - """ - True only when measures were actually found and none demanded a document. + """True only when the line needs nothing written in Box 44. - An empty measure set does not count: "nothing applies here" and "the - reference data does not cover this code and country" look identical from - here, and answering the second as the first is how a line ends up filed - without the document it needed. + An exemption still has to be declared: the export control on cat and + dog fur is satisfied by writing Y922, not by leaving the box empty. + Counting that as "no document required" would hide the one code the + declarant has to enter. + + A suspended obligation counts the same way. Under T1 the duty is + secured, not forgiven, and it falls due at the office of destination — + saying "no document required" would read as though the line were + finished with. """ return ( self.evaluated and bool(self.measures) and not self.required and not self.alternatives + and not self.exemptions + and not self.suspended ) def _geo_specificity(measure: MeasureModel, country: str) -> int: - """ - Rank a measure by how specifically it targets the country. - - A measure written for the country itself beats one written for a group, - which beats one written for all third countries. - """ if measure.geoAreaCode == country: return 3 if measure.geoAreaCode == settings.ERGA_OMNES_CODE: @@ -77,22 +61,10 @@ def _geo_specificity(measure: MeasureModel, country: str) -> int: def select_applicable_measures( measures: list[MeasureModel], country: str ) -> list[MeasureModel]: - """ - Drop measures the country is carved out of, then keep the most specific one - per (measure type, additional code). - - TARIC layers measures over the nomenclature and over geography; the most - specific statement of the same measure type is the one that governs, so - without this the same obligation would be reported several times over. - """ - applicable = [ - measure - for measure in measures - if country not in measure.excludedGeoAreaCodes - ] - best: dict[tuple[str, str | None], MeasureModel] = {} - for measure in applicable: + for measure in measures: + if country in measure.excludedGeoAreaCodes: + continue key = (measure.measureTypeId, measure.additionalCode) current = best.get(key) if current is None: @@ -106,36 +78,24 @@ def select_applicable_measures( return sorted(best.values(), key=lambda item: (item.measureTypeId, item.geoAreaCode)) -def _group_conditions( - measure: MeasureModel, -) -> dict[str, list[MeasureConditionEmbedded]]: - """ - Group a measure's conditions by condition code. - """ - grouped: dict[str, list[MeasureConditionEmbedded]] = {} - for condition in measure.conditions: - grouped.setdefault(condition.conditionCode, []).append(condition) - for conditions in grouped.values(): - conditions.sort(key=lambda item: item.sequenceNumber) - return grouped - - def _to_requirement( measure: MeasureModel, condition: MeasureConditionEmbedded, requirement_type: DocumentRequirementType, - descriptions: dict[str, str | None], + documents: dict[str, DocumentCodeModel], alternative_group: str | None = None, ) -> DocumentRequirement: - """ - Build a document requirement from a measure condition. - """ + document = documents.get(condition.documentCode) return DocumentRequirement( code=condition.documentCode, documentType=condition.documentType, requirementType=requirement_type, - description=descriptions.get(condition.documentCode) - or condition.documentDescription, + description=(document.descriptionNl or document.description) + if document + else condition.documentDescription, + source=ReferenceSource.TARIC if document else ReferenceSource.NONE, + validFrom=document.validFrom if document else None, + validTo=document.validTo if document else None, measureSid=measure.measureSid, measureTypeId=measure.measureTypeId, measureTypeDescription=measure.measureTypeDescription, @@ -144,38 +104,21 @@ def _to_requirement( ) -async def _document_descriptions(measures: list[MeasureModel]) -> dict[str, str | None]: - """ - Resolve human-readable descriptions for every document code in play. - """ - codes = dedupe( - [ - condition.documentCode - for measure in measures - for condition in measure.conditions - if condition.documentCode - ] - ) - documents = await get_document_codes(codes) - return { - code: (documents[code].descriptionNl or documents[code].description) - if code in documents - else None - for code in codes - } - - def extract_documents( - measures: list[MeasureModel], descriptions: dict[str, str | None] + measures: list[MeasureModel], + documents: dict[str, DocumentCodeModel], + procedure: CustomsProcedure = CustomsProcedure.RELEASE_FOR_FREE_CIRCULATION, ) -> DocumentOutcome: - """ - Read document obligations, alternatives and exemptions out of measures. - """ outcome = DocumentOutcome(evaluated=True, measures=measures) seen_required: set[str] = set() seen_exempt: set[str] = set() + seen_suspended: set[str] = set() for measure in measures: + suspended = measure_is_suspended(measure.category, procedure) + if suspended: + outcome.suspendedMeasures.append(measure) + outcome.appliedMeasures.append( AppliedMeasure( measureSid=measure.measureSid, @@ -186,10 +129,18 @@ def extract_documents( geoAreaCode=measure.geoAreaCode, validFrom=measure.validFrom, validTo=measure.validTo, + national=measure.national, + suspended=suspended, ) ) - for condition_code, conditions in _group_conditions(measure).items(): + grouped: dict[str, list[MeasureConditionEmbedded]] = {} + for condition in measure.conditions: + grouped.setdefault(condition.conditionCode, []).append(condition) + for conditions in grouped.values(): + conditions.sort(key=lambda item: item.sequenceNumber) + + for condition_code, conditions in grouped.items(): obligations = [ condition for condition in conditions @@ -203,18 +154,38 @@ def extract_documents( group_id = f"{measure.measureSid}:{condition_code}" - if len(obligations) > 1: - group = [ - _to_requirement( - measure, - condition, - DocumentRequirementType.ALTERNATIVE, - descriptions, - group_id, + if suspended: + # The obligation is real but not owed here. It is kept visible + # so the declarant can see what the guarantee is covering and + # what will be asked for at the office of destination. + for condition in obligations: + if condition.documentCode in seen_suspended: + continue + seen_suspended.add(condition.documentCode) + outcome.suspended.append( + _to_requirement( + measure, + condition, + DocumentRequirementType.SUSPENDED, + documents, + group_id, + ) ) - for condition in obligations - ] - outcome.alternatives.append(group) + continue + + if len(obligations) > 1: + outcome.alternatives.append( + [ + _to_requirement( + measure, + condition, + DocumentRequirementType.ALTERNATIVE, + documents, + group_id, + ) + for condition in obligations + ] + ) elif len(obligations) == 1: condition = obligations[0] if condition.documentCode not in seen_required: @@ -224,7 +195,7 @@ def extract_documents( measure, condition, DocumentRequirementType.REQUIRED, - descriptions, + documents, ) ) @@ -237,7 +208,7 @@ def extract_documents( measure, condition, DocumentRequirementType.EXEMPTION, - descriptions, + documents, group_id, ) ) @@ -251,14 +222,21 @@ async def determine_documents( direction: TradeDirection, on_date: date, additional_codes: list[str] | None = None, + procedure: CustomsProcedure = CustomsProcedure.RELEASE_FOR_FREE_CIRCULATION, ) -> DocumentOutcome: - """ - Determine the documents a resolved line needs. - """ geo_codes = await get_geo_area_codes_for_country(country, on_date) measures = await find_measures_for_line( resolved_code, geo_codes, direction, on_date, additional_codes ) applicable = select_applicable_measures(measures, country) - descriptions = await _document_descriptions(applicable) - return extract_documents(applicable, descriptions) + documents = await get_document_codes( + dedupe( + [ + condition.documentCode + for measure in applicable + for condition in measure.conditions + if condition.documentCode + ] + ) + ) + return extract_documents(applicable, documents, procedure) diff --git a/cbh/api/linecheck/services/resolution/resolution.py b/cbh/api/linecheck/services/resolution/resolution.py index 59860004ba2baac322174b6beb6d960dee425ad8..258c7ee05d912899b6fced77b52511443e8b3d1b 100644 --- a/cbh/api/linecheck/services/resolution/resolution.py +++ b/cbh/api/linecheck/services/resolution/resolution.py @@ -1,20 +1,7 @@ -""" -Step 1 of a line check: resolving the goods code. - - normalise -> exact 10-digit match? - yes -> resolved - 6/8 digits -> expand to the declarable 10-digit codes in force - 1 candidate -> resolved - >1 candidates -> ambiguous, candidates listed - not valid on the date -> warning plus the replacement code, if one exists - -A code is never repaired silently. When the input is ambiguous or dead, the -line comes back flagged with the evidence a broker needs to decide. -""" - from dataclasses import dataclass, field from datetime import date +from cbh.api.common.dto import TradeDirection from cbh.api.linecheck.dto import ( LineStatus, ResolutionMethod, @@ -29,16 +16,12 @@ from cbh.api.taric.db_requests import ( get_nomenclature_by_code, ) from cbh.api.taric.models import NomenclatureModel -from cbh.api.taric.utils import format_code, is_full_taric_code, normalize_code +from cbh.api.taric.utils import format_code, normalize_code from cbh.core.config import settings @dataclass class CodeResolution: - """ - Outcome of the code-resolution step. - """ - inputCode: str normalizedCode: str | None = None resolvedCode: str | None = None @@ -52,16 +35,10 @@ class CodeResolution: @property def is_resolved(self) -> bool: - """ - Whether a single 10-digit code was established. - """ return self.resolvedCode is not None def _to_candidate(model: NomenclatureModel) -> CodeCandidate: - """ - Render a nomenclature node as a candidate for a broker to choose from. - """ return CodeCandidate( code=model.code, formattedCode=format_code(model.code), @@ -71,28 +48,14 @@ def _to_candidate(model: NomenclatureModel) -> CodeCandidate: ) -async def _replacement_codes(code: str) -> list[str]: - """ - Successors of a withdrawn code, from the correlation tables. - """ - changes = await get_code_changes(code) - successors: list[str] = [] - for change in changes: - successors.extend(change.newCodes) - return successors - - async def _resolve_expired_code( resolution: CodeResolution, code: str, on_date: date ) -> CodeResolution: - """ - Handle a code that exists in the nomenclature but not on this date. - - The replacement is reported, never applied: substituting a code on the - broker's behalf is what the "never auto-correct" rule forbids. - """ versions = await find_code_versions(code) - replacements = await _replacement_codes(code) + replacements: list[str] = [] + for change in await get_code_changes(code): + replacements.extend(change.newCodes) + resolution.replacementCodes = replacements resolution.status = LineStatus.INVALID resolution.method = ResolutionMethod.NOT_RESOLVED @@ -134,12 +97,6 @@ async def _resolve_expired_code( def _apply_expiry_check( resolution: CodeResolution, model: NomenclatureModel, on_date: date ) -> None: - """ - Flag a resolved code whose validity ends within the warning horizon. - - This is what catches the 1 January nomenclature switch while a pack filed in - November still looks healthy. - """ days_left = model.days_until_expiry(on_date) if days_left is None or days_left > settings.EXPIRING_SOON_DAYS: return @@ -158,10 +115,33 @@ def _apply_expiry_check( ) -async def resolve_code(raw_code: str, on_date: date) -> CodeResolution: - """ - Resolve a goods code as typed by a broker into a single 10-digit TARIC code. +async def _resolve_export_cn_code( + resolution: CodeResolution, prefix: str, on_date: date +) -> CodeResolution | None: + """An export declaration is filed on the eight-digit CN code. + + The last two digits are a TARIC subdivision, which exists for import duty + and is not declared on export: of the 20 351 export measures in the + published extraction only 293 are written below eight digits, and both real + Goenee export packs are filed at eight throughout. So an eight-digit code + that exists in the nomenclature resolves as itself instead of being split + into the import-side codes beneath it. """ + model = await get_nomenclature_by_code(prefix.ljust(10, "0"), on_date=on_date) + if model is None: + return None + + resolution.resolvedCode = model.code + resolution.resolved = model + resolution.method = ResolutionMethod.EXACT_10 + resolution.status = LineStatus.OK + _apply_expiry_check(resolution, model, on_date) + return resolution + + +async def resolve_code( + raw_code: str, on_date: date, direction: TradeDirection | None = None +) -> CodeResolution: resolution = CodeResolution(inputCode=str(raw_code)) normalized = normalize_code(raw_code) @@ -179,22 +159,24 @@ async def resolve_code(raw_code: str, on_date: date) -> CodeResolution: resolution.normalizedCode = normalized - if is_full_taric_code(normalized): + if len(normalized) == 10: return await _resolve_full_code(resolution, normalized, on_date) + + if direction is TradeDirection.EXPORT and len(normalized) == 8: + resolved = await _resolve_export_cn_code(resolution, normalized, on_date) + if resolved is not None: + return resolved + return await _resolve_prefix(resolution, normalized, on_date) async def _resolve_full_code( resolution: CodeResolution, code: str, on_date: date ) -> CodeResolution: - """ - Resolve an input that is already 10 digits long. - """ model = await get_nomenclature_by_code(code, on_date=on_date) if model is None: - versions = await find_code_versions(code) - if versions: + if await find_code_versions(code): return await _resolve_expired_code(resolution, code, on_date) resolution.warnings.append( LineWarning( @@ -207,8 +189,6 @@ async def _resolve_full_code( return resolution if not model.declarable: - # The code exists as a tree node but cannot carry a declaration line; - # offer the declarable codes beneath it rather than rejecting outright. candidates = await find_codes_by_prefix(code, on_date=on_date, declarable_only=True) resolution.candidates = [_to_candidate(candidate) for candidate in candidates] resolution.status = LineStatus.AMBIGUOUS if candidates else LineStatus.INVALID @@ -236,9 +216,6 @@ async def _resolve_full_code( async def _resolve_prefix( resolution: CodeResolution, prefix: str, on_date: date ) -> CodeResolution: - """ - Expand a 6- or 8-digit input to the declarable codes in force beneath it. - """ candidates = await find_codes_by_prefix(prefix, on_date=on_date, declarable_only=True) if not candidates: diff --git a/cbh/api/linecheck/services/warnings/__init__.py b/cbh/api/linecheck/services/warnings/__init__.py index 3657d928b2026c881a6c545c497892935c1b6e2b..3256287dff6851119bda1f361b6ec68cb29ba392 100644 --- a/cbh/api/linecheck/services/warnings/__init__.py +++ b/cbh/api/linecheck/services/warnings/__init__.py @@ -3,7 +3,9 @@ from .warnings import ( build_chapter_scope_warning, build_country_warning, build_measure_warnings, + build_national_measure_warning, build_no_measure_warning, + build_transit_warning, ) __all__ = [ @@ -11,5 +13,7 @@ __all__ = [ "build_chapter_scope_warning", "build_country_warning", "build_measure_warnings", + "build_national_measure_warning", "build_no_measure_warning", + "build_transit_warning", ] diff --git a/cbh/api/linecheck/services/warnings/warnings.py b/cbh/api/linecheck/services/warnings/warnings.py index b4c4aeb3be7530bfa9b0ab4101a742b93d81ef92..cd2167d70e7de30e3f8389b64145e1c7e9e28e50 100644 --- a/cbh/api/linecheck/services/warnings/warnings.py +++ b/cbh/api/linecheck/services/warnings/warnings.py @@ -1,20 +1,10 @@ -""" -Step 3 of a line check: warnings. - -Anti-dumping, safeguard, phyto/CHED and CITES measures are surfaced as warnings -rather than silently folded into the document list, because they change what a -broker has to do rather than just which paper to attach. Dual-use is flagged as -out of scope and left to a human on purpose. -""" - +from cbh.api.common.dto import CustomsProcedure from cbh.api.linecheck.dto import WarningSeverity, WarningType from cbh.api.linecheck.schemas import LineWarning from cbh.api.taric.dto import MeasureCategory from cbh.api.taric.models import MeasureModel -# Measure categories that must reach the broker as a warning, with the severity -# and wording they carry. -_CATEGORY_WARNINGS: dict[MeasureCategory, tuple[WarningType, WarningSeverity, str]] = { +CATEGORY_WARNINGS: dict[MeasureCategory, tuple[WarningType, WarningSeverity, str]] = { MeasureCategory.ANTIDUMPING: ( WarningType.ANTIDUMPING, WarningSeverity.WARNING, @@ -82,19 +72,13 @@ _CATEGORY_WARNINGS: dict[MeasureCategory, tuple[WarningType, WarningSeverity, st def build_measure_warnings(measures: list[MeasureModel]) -> list[LineWarning]: - """ - Derive warnings from the measures that applied to a line. - - One warning per category: a broker needs to know an anti-dumping measure - exists, not to read the same sentence five times. - """ warnings: list[LineWarning] = [] seen: set[MeasureCategory] = set() for measure in measures: if measure.category in seen: continue - template = _CATEGORY_WARNINGS.get(measure.category) + template = CATEGORY_WARNINGS.get(measure.category) if not template: continue seen.add(measure.category) @@ -118,14 +102,54 @@ def build_measure_warnings(measures: list[MeasureModel]) -> list[LineWarning]: return warnings -def build_no_measure_warning(code: str, country: str) -> LineWarning: - """ - Warn that a resolved line produced no measures at all. +def build_transit_warning( + procedure: CustomsProcedure, suspended: list[MeasureModel] +) -> LineWarning | None: + """Say plainly that a duty was not dropped, only deferred. - An empty measure set is far more often missing reference data than a - genuinely unregulated line, so it is reported rather than read as "nothing - required". + A transit line that comes back with an empty Box 44 looks finished. It is + not: the guarantee stands in for the duty until the goods are presented at + the office of destination, and that is where these measures are settled. """ + if not procedure.suspends_duty or not suspended: + return None + + kinds = sorted({measure.category.name.replace("_", " ").lower() for measure in suspended}) + where = ( + "the office of destination" + if procedure.is_transit + else "the declaration that ends the procedure" + ) + return LineWarning( + type=WarningType.TRANSIT_DUTY_SUSPENDED, + severity=WarningSeverity.INFO, + message=( + f"{len(suspended)} measure(s) are suspended under this procedure " + f"({', '.join(kinds)}). They are secured, not waived, and fall due at " + f"{where}. Prohibitions and restrictions are unaffected and stay on this line." + ), + relatedCodes=sorted({measure.measureTypeId for measure in suspended}), + ) + + +def build_national_measure_warning(measures: list[MeasureModel]) -> LineWarning | None: + """Name the measures that come from Dutch law rather than from the Union.""" + national = [measure for measure in measures if measure.national] + if not national: + return None + types = sorted({measure.measureTypeId for measure in national}) + return LineWarning( + type=WarningType.NATIONAL_MEASURE, + severity=WarningSeverity.INFO, + message=( + f"{len(national)} national measure(s) apply on top of the Union rules " + f"({', '.join(types)}), from the Dutch Gebruikstarief." + ), + relatedCodes=types, + ) + + +def build_no_measure_warning(code: str, country: str) -> LineWarning: return LineWarning( type=WarningType.NO_MEASURES_FOUND, severity=WarningSeverity.WARNING, @@ -140,13 +164,6 @@ def build_no_measure_warning(code: str, country: str) -> LineWarning: def build_additional_code_warning( measures: list[MeasureModel], provided: list[str] | None ) -> LineWarning | None: - """ - Ask for an additional code when a measure needs one to be pinned down. - - Anti-dumping duty is the usual case: the measure applies either way, but - which rate applies depends on the producer, and that is carried by the - additional code. - """ if provided: return None candidates = sorted( @@ -166,29 +183,39 @@ def build_additional_code_warning( ) -def build_chapter_scope_warning(chapter: str) -> LineWarning: - """ - Note that a chapter sits outside the chapters this phase was tuned on. - """ +def build_chapter_scope_warning(chapter: str, scope: str) -> LineWarning: + if scope == "nvwa_excluded": + return LineWarning( + type=WarningType.NVWA_CHAPTER_EXCLUDED, + severity=WarningSeverity.BLOCKING, + message=( + f"Chapter {chapter} is an NVWA-controlled chapter and is excluded from " + "this release. Handle this line manually." + ), + ) return LineWarning( type=WarningType.CHAPTER_OUT_OF_SCOPE, severity=WarningSeverity.INFO, message=( - f"Chapter {chapter} is outside the chapters validated for this release. " - "The result is computed but has not been benchmarked against a gold set." + f"Chapter {chapter} is outside chapters 22, 61, 62, 84 and 85 validated for " + "this release. The result is computed but has not been benchmarked." ), ) -def build_country_warning(raw_country: str | None) -> LineWarning: - """ - Report a missing or unusable country code. - """ +def build_country_warning(direction, raw_origin: str | None, raw_destination: str | None): + from cbh.api.common.dto import TradeDirection + + if direction is TradeDirection.EXPORT: + field, raw = "destination country", raw_destination + warning_type = WarningType.DESTINATION_UNKNOWN + else: + field, raw = "country of origin", raw_origin + warning_type = WarningType.COUNTRY_UNKNOWN return LineWarning( - type=WarningType.COUNTRY_UNKNOWN, + type=warning_type, severity=WarningSeverity.BLOCKING, message=( - f"'{raw_country}' is not a usable ISO country code; " - "measures cannot be selected without it." + f"'{raw}' is not a usable ISO {field}; measures cannot be selected without it." ), ) diff --git a/cbh/api/linecheck/utils.py b/cbh/api/linecheck/utils.py index 860485cdb9e1cb60a939036e0b4ef7621960d697..ec5b97021ae5b3ea1ae519cdc0c550aa5a48c994 100644 --- a/cbh/api/linecheck/utils.py +++ b/cbh/api/linecheck/utils.py @@ -1,32 +1,7 @@ -""" -Line-check helpers for rendering results into flat spreadsheet cells. -""" - -from cbh.api.linecheck.dto import LineStatus, WarningSeverity from cbh.api.linecheck.schemas import LineCheckResponse -STATUS_LABELS = { - LineStatus.OK: "ok", - LineStatus.AMBIGUOUS: "ambiguous", - LineStatus.INVALID: "invalid", - LineStatus.EXPIRING: "expiring", -} - - -def status_label(status: LineStatus) -> str: - """ - Render a status as the lowercase token the output file carries. - """ - return STATUS_LABELS[status] - def render_required_documents(result: LineCheckResponse) -> str: - """ - Render the obligations for one cell. - - Either/or groups are rendered as ``C400|Y900`` so a broker can see at a - glance that one of the two discharges the obligation. - """ parts = [document.code for document in result.documentsRequired] parts.extend( "|".join(document.code for document in group) @@ -35,35 +10,6 @@ def render_required_documents(result: LineCheckResponse) -> str: return "; ".join(parts) -def render_exemptions(result: LineCheckResponse) -> str: - """ - Render the admissible exemption codes for one cell. - """ - return "; ".join(document.code for document in result.documentsExemption) - - def render_warnings(result: LineCheckResponse) -> str: - """ - Render warnings for one cell, blocking ones first. - """ - ordered = sorted( - result.warnings, - key=lambda warning: -warning.severity.value, - ) + ordered = sorted(result.warnings, key=lambda warning: -warning.severity.value) return " | ".join(warning.message for warning in ordered) - - -def render_candidates(result: LineCheckResponse) -> str: - """ - Render the candidate codes of an ambiguous line. - """ - return ", ".join(candidate.code for candidate in result.candidates) - - -def has_blocking_warning(result: LineCheckResponse) -> bool: - """ - Whether the line carries a warning that prevents filing as it stands. - """ - return any( - warning.severity is WarningSeverity.BLOCKING for warning in result.warnings - ) diff --git a/cbh/api/linecheck/views.py b/cbh/api/linecheck/views.py index de734cdf6ebf681e71811222f5504853e1c30d9f..2a794d1bfb5385c25acc992b9c3b4cfa085cca20 100644 --- a/cbh/api/linecheck/views.py +++ b/cbh/api/linecheck/views.py @@ -1,11 +1,5 @@ -""" -Line-check views module. -""" +from fastapi import HTTPException -from fastapi import Depends, HTTPException - -from cbh.api.account.dto import AccountType -from cbh.api.account.models import AccountModel from cbh.api.linecheck import linecheck_router from cbh.api.linecheck.schemas import ( BatchLineCheckRequest, @@ -13,44 +7,62 @@ from cbh.api.linecheck.schemas import ( LineCheckRequest, LineCheckResponse, ) -from cbh.api.linecheck.services.engine import check_line_cached, check_lines +from cbh.api.linecheck.services.engine import ( + check_line_cached, + check_lines, + recount_confidence, +) +from cbh.api.rules.db_requests import get_active_rules, record_rule_hits +from cbh.api.rules.services.apply import apply_rules, apply_rules_to_lines from cbh.api.taric.db_requests import get_reference_as_of -from cbh.core.security import PermissionDependency from cbh.core.wrappers import CbhResponseWrapper -ANY_ACCOUNT = [AccountType.USER, AccountType.ADMIN, AccountType.OWNER, AccountType.SUPER_ADMIN] - -MAX_BATCH_LINES = 5000 - @linecheck_router.post("/line") async def check_single_line( data: LineCheckRequest, - account: AccountModel = Depends(PermissionDependency(ANY_ACCOUNT)), ) -> CbhResponseWrapper[LineCheckResponse]: - """ - Check one declaration line. + """Check one line without opening a shipment. Backs the search screen. + + Passing `clientProfileId` applies that client's own rules on top of the + reference answer, exactly as the review table does. """ reference_as_of = await get_reference_as_of() - return CbhResponseWrapper(data=await check_line_cached(data, reference_as_of)) + result = await check_line_cached(data, reference_as_of) + + if data.clientProfileId: + rules = await get_active_rules(data.clientProfileId) + if rules: + result = await apply_rules(data, result, rules, reference_as_of) + await record_rule_hits([rule.ruleId for rule in result.appliedRules]) + + return CbhResponseWrapper(data=result) @linecheck_router.post("/batch") async def check_batch_lines( data: BatchLineCheckRequest, - account: AccountModel = Depends(PermissionDependency(ANY_ACCOUNT)), ) -> CbhResponseWrapper[BatchLineCheckResponse]: - """ - Check a pack of declaration lines in one call. - - Identical (code, country, direction, date) tuples are resolved once and - reused across every row that shares them. - """ if not data.lines: raise HTTPException(status_code=400, detail="No lines supplied.") - if len(data.lines) > MAX_BATCH_LINES: + if len(data.lines) > 5000: raise HTTPException( status_code=400, - detail=f"Too many lines: {len(data.lines)} (maximum {MAX_BATCH_LINES}).", + detail=f"Too many lines: {len(data.lines)} (maximum 5000).", + ) + + result = await check_lines(data.lines, data.date) + + profile_ids = {line.clientProfileId for line in data.lines if line.clientProfileId} + if len(profile_ids) == 1: + result.results = await apply_rules_to_lines( + data.lines, result.results, profile_ids.pop(), await get_reference_as_of() ) - return CbhResponseWrapper(data=await check_lines(data.lines, data.date)) + recount_confidence(result) + elif len(profile_ids) > 1: + raise HTTPException( + status_code=400, + detail="All lines in one call must belong to the same client profile.", + ) + + return CbhResponseWrapper(data=result) diff --git a/cbh/api/rules/__init__.py b/cbh/api/rules/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..caca2357d66a12bf4945d57e1c3bd83d16178150 --- /dev/null +++ b/cbh/api/rules/__init__.py @@ -0,0 +1,6 @@ +from fastapi import APIRouter + +rules_router = APIRouter( + prefix="/api/rules", + tags=["rules"], +) diff --git a/cbh/api/rules/db_requests.py b/cbh/api/rules/db_requests.py new file mode 100644 index 0000000000000000000000000000000000000000..37b3f6dbf4600f73f50128c69049af66793d0a73 --- /dev/null +++ b/cbh/api/rules/db_requests.py @@ -0,0 +1,48 @@ +from collections import Counter +from datetime import datetime + +from pymongo import UpdateOne + +from cbh.api.common.db_requests import find_objs, get_obj_by_id, resolve_collection +from cbh.api.rules.dto import RuleScope +from cbh.api.rules.models import RuleModel +from cbh.core.config import settings + + +async def get_rule(rule_id: str) -> RuleModel: + return await get_obj_by_id(RuleModel, rule_id) + + +async def get_active_rules(client_profile_id: str | None) -> list[RuleModel]: + """Every rule that could touch a line checked for this client profile. + + Tenant-scoped rules apply everywhere; client-scoped rules only to their own + profile. The result is ordered narrowest-first so a conflict resolves to + the most specific rule. + """ + scopes: list[dict] = [{"scope": RuleScope.TENANT.value}] + if client_profile_id: + scopes.append( + {"scope": RuleScope.CLIENT.value, "clientProfileId": client_profile_id} + ) + + rules = await find_objs(RuleModel, {"active": True, "$or": scopes}, limit=0) + return sorted(rules, key=lambda rule: -rule.specificity) + + +async def record_rule_hits(rule_ids: list[str]) -> None: + """Count what each rule actually caught. This is the learning layer's input.""" + if not rule_ids: + return + + now = datetime.now().isoformat() + operations = [ + UpdateOne( + {"id": rule_id}, + {"$inc": {"hitCount": count}, "$set": {"datetimeLastHit": now}}, + ) + for rule_id, count in Counter(rule_ids).items() + ] + await settings.DB_CLIENT[resolve_collection(RuleModel)].bulk_write( + operations, ordered=False + ) diff --git a/cbh/api/rules/dto.py b/cbh/api/rules/dto.py new file mode 100644 index 0000000000000000000000000000000000000000..568877036fda9928b6fb0316994bdfa158f05e8f --- /dev/null +++ b/cbh/api/rules/dto.py @@ -0,0 +1,22 @@ +from enum import IntEnum + + +class RuleAction(IntEnum): + """What a broker's own rule does to a checked line. + + A rule is the broker's logic, never a reference-data fact. Whatever it does + is recorded on the line and lowers the confidence to amber, because an + answer that came from a rule has not been derived from TARIC. + """ + + PIN_CODE = 1 + ADD_DOCUMENTS = 2 + REMOVE_DOCUMENTS = 3 + NOTE = 4 + + +class RuleScope(IntEnum): + """Which lines a rule is allowed to touch.""" + + CLIENT = 1 + TENANT = 2 diff --git a/cbh/api/rules/models.py b/cbh/api/rules/models.py new file mode 100644 index 0000000000000000000000000000000000000000..896be31f95b947b6445f0ef04b654eca2d7d07b2 --- /dev/null +++ b/cbh/api/rules/models.py @@ -0,0 +1,60 @@ +from datetime import datetime + +from pydantic import Field + +from cbh.api.common.dto import TradeDirection +from cbh.api.rules.dto import RuleAction, RuleScope +from cbh.core.database import MongoBaseModel + + +class RuleModel(MongoBaseModel): + """A broker's own logic, written down once and replayed on every line. + + Matching is deterministic and narrow: every condition that is set must + hold. A rule with no conditions at all is refused on the way in, because it + would silently touch the whole pack. + """ + + name: str + description: str | None = None + scope: RuleScope = RuleScope.CLIENT + clientProfileId: str | None = None + active: bool = True + + codePrefix: str | None = None + originCountry: str | None = None + destinationCountry: str | None = None + direction: TradeDirection | None = None + descriptionContains: str | None = None + + action: RuleAction = RuleAction.NOTE + pinnedCode: str | None = None + documentCodes: list[str] = Field(default_factory=list) + note: str | None = None + + createdFromBatchId: str | None = None + createdFromRowIndex: int | None = None + createdBy: str | None = None + + hitCount: int = 0 + datetimeLastHit: datetime | None = None + + datetimeInserted: datetime = Field(default_factory=datetime.now) + datetimeUpdated: datetime = Field(default_factory=datetime.now) + + @property + def conditions(self) -> dict[str, str]: + """The conditions actually set, for display and for specificity.""" + present = { + "codePrefix": self.codePrefix, + "originCountry": self.originCountry, + "destinationCountry": self.destinationCountry, + "direction": self.direction.name if self.direction else None, + "descriptionContains": self.descriptionContains, + } + return {key: value for key, value in present.items() if value} + + @property + def specificity(self) -> int: + """How many conditions a rule sets. The narrowest rule wins a conflict.""" + return len(self.conditions) diff --git a/cbh/api/rules/schemas.py b/cbh/api/rules/schemas.py new file mode 100644 index 0000000000000000000000000000000000000000..67667f122127a79d04af7fa8f5fc92452a8dac10 --- /dev/null +++ b/cbh/api/rules/schemas.py @@ -0,0 +1,119 @@ +from datetime import datetime + +from pydantic import BaseModel, Field, model_validator + +from cbh.api.common.dto import TradeDirection +from cbh.api.rules.dto import RuleAction, RuleScope + + +class RuleRequest(BaseModel): + name: str + description: str | None = None + scope: RuleScope = RuleScope.CLIENT + clientProfileId: str | None = None + active: bool = True + + codePrefix: str | None = None + originCountry: str | None = None + destinationCountry: str | None = None + direction: TradeDirection | None = None + descriptionContains: str | None = None + + action: RuleAction = RuleAction.NOTE + pinnedCode: str | None = None + documentCodes: list[str] = Field(default_factory=list) + note: str | None = None + + createdFromBatchId: str | None = None + createdFromRowIndex: int | None = None + createdBy: str | None = None + + @model_validator(mode="after") + def check_rule_is_answerable(self) -> "RuleRequest": + conditions = [ + self.codePrefix, + self.originCountry, + self.destinationCountry, + self.direction, + self.descriptionContains, + ] + if not any(conditions): + raise ValueError( + "A rule needs at least one condition. A rule without conditions " + "would apply to every line in every pack." + ) + + if self.codePrefix and not self.codePrefix.isdigit(): + raise ValueError("codePrefix must be digits only.") + + if self.action is RuleAction.PIN_CODE and not self.pinnedCode: + raise ValueError("A pin-code rule needs pinnedCode.") + if self.action in (RuleAction.ADD_DOCUMENTS, RuleAction.REMOVE_DOCUMENTS): + if not self.documentCodes: + raise ValueError(f"A {self.action.name.lower()} rule needs documentCodes.") + if self.action is RuleAction.NOTE and not self.note: + raise ValueError("A note rule needs note.") + + if self.scope is RuleScope.CLIENT and not self.clientProfileId: + raise ValueError( + "A client-scoped rule needs clientProfileId. Use scope=TENANT " + "for a rule that applies to every client." + ) + return self + + +class RuleResponse(BaseModel): + id: str + name: str + description: str | None = None + scope: RuleScope + clientProfileId: str | None = None + active: bool + + codePrefix: str | None = None + originCountry: str | None = None + destinationCountry: str | None = None + direction: TradeDirection | None = None + descriptionContains: str | None = None + conditions: dict[str, str] = Field(default_factory=dict) + + action: RuleAction + pinnedCode: str | None = None + documentCodes: list[str] = Field(default_factory=list) + note: str | None = None + + createdFromBatchId: str | None = None + createdFromRowIndex: int | None = None + createdBy: str | None = None + + hitCount: int = 0 + datetimeLastHit: datetime | None = None + datetimeInserted: datetime + datetimeUpdated: datetime + + +class MakeRuleRequest(BaseModel): + """The 'Make a rule' button on a review row. + + Conditions left unset are taken from the row itself, so the broker can + press the button, name the rule, and get a rule that matches the line in + front of them. + """ + + name: str + description: str | None = None + scope: RuleScope = RuleScope.CLIENT + + action: RuleAction = RuleAction.NOTE + pinnedCode: str | None = None + documentCodes: list[str] = Field(default_factory=list) + note: str | None = None + + useCodePrefix: bool = True + codePrefixLength: int = 8 + useOriginCountry: bool = True + useDestinationCountry: bool = False + useDirection: bool = True + descriptionContains: str | None = None + + createdBy: str | None = None diff --git a/cbh/api/account/db_requests.py b/cbh/api/rules/services/__init__.py similarity index 100% rename from cbh/api/account/db_requests.py rename to cbh/api/rules/services/__init__.py diff --git a/cbh/api/rules/services/apply/__init__.py b/cbh/api/rules/services/apply/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..ea68c6421046868ed99dbc814fd50fa457a33a17 --- /dev/null +++ b/cbh/api/rules/services/apply/__init__.py @@ -0,0 +1,3 @@ +from cbh.api.rules.services.apply.apply import apply_rules, apply_rules_to_lines, matching_rules + +__all__ = ["apply_rules", "apply_rules_to_lines", "matching_rules"] diff --git a/cbh/api/rules/services/apply/apply.py b/cbh/api/rules/services/apply/apply.py new file mode 100644 index 0000000000000000000000000000000000000000..b5236e85961f3ca6ee02485344216fc848e7224d --- /dev/null +++ b/cbh/api/rules/services/apply/apply.py @@ -0,0 +1,214 @@ +from cbh.api.common.utils import normalize_country +from cbh.api.linecheck.dto import ( + ConfidenceLevel, + DocumentRequirementType, + ReferenceSource, + WarningSeverity, + WarningType, +) +from cbh.api.linecheck.schemas import ( + AppliedRule, + DocumentRequirement, + LineCheckRequest, + LineCheckResponse, + LineWarning, +) +from cbh.api.rules.db_requests import get_active_rules, record_rule_hits +from cbh.api.rules.dto import RuleAction +from cbh.api.rules.models import RuleModel +from cbh.api.taric.dto import DocumentCodeType +from cbh.api.taric.utils import normalize_code + + +def _matches(rule: RuleModel, request: LineCheckRequest, response: LineCheckResponse) -> bool: + """Every condition the rule sets must hold. Conditions it leaves out are not tested.""" + if rule.codePrefix: + code = normalize_code(response.resolvedCode or request.code) or "" + if not code.startswith(rule.codePrefix): + return False + + if rule.originCountry: + if normalize_country(rule.originCountry) != normalize_country(request.originCountry): + return False + + if rule.destinationCountry: + if normalize_country(rule.destinationCountry) != normalize_country( + request.destinationCountry + ): + return False + + if rule.direction and rule.direction is not request.direction: + return False + + if rule.descriptionContains: + description = (request.description or "").lower() + if rule.descriptionContains.lower() not in description: + return False + + return True + + +def matching_rules( + rules: list[RuleModel], request: LineCheckRequest, response: LineCheckResponse +) -> list[RuleModel]: + """The rules that fire on this line, narrowest first.""" + return [rule for rule in rules if _matches(rule, request, response)] + + +def _rule_document(code: str, rule: RuleModel) -> DocumentRequirement: + code = code.strip().upper() + exemption = code.startswith("Y") + return DocumentRequirement( + code=code, + documentType=DocumentCodeType.from_code(code), + requirementType=( + DocumentRequirementType.EXEMPTION if exemption else DocumentRequirementType.REQUIRED + ), + description=f"Added by rule '{rule.name}'.", + source=ReferenceSource.MANUAL, + ) + + +def _note(rule: RuleModel, message: str) -> LineWarning: + return LineWarning( + type=WarningType.RULE_APPLIED, + severity=WarningSeverity.INFO, + message=f"Rule '{rule.name}': {message}", + ) + + +async def apply_rules( + request: LineCheckRequest, + response: LineCheckResponse, + rules: list[RuleModel], + reference_as_of: str | None = None, +) -> LineCheckResponse: + """Overlay the broker's own rules on a checked line. + + A rule never replaces the reference answer silently. Whatever it changes is + named on the line in `appliedRules` and as an INFO warning, and the + confidence it touched drops to amber, because the answer no longer comes + from TARIC alone. + """ + fired = matching_rules(rules, request, response) + if not fired: + return response + + pins = [rule for rule in fired if rule.action is RuleAction.PIN_CODE] + if pins: + # The narrowest pin wins; re-check under the pinned code so the + # documents belong to the code that is actually being declared. + pin = pins[0] + pinned = normalize_code(pin.pinnedCode) or pin.pinnedCode + if pinned and pinned != response.resolvedCode: + from cbh.api.linecheck.services.engine import check_line_cached + + repinned = request.model_copy(deep=True) + repinned.code = pinned + original_code = response.resolvedCode or response.inputCode + response = await check_line_cached(repinned, reference_as_of) + response.rowIndex = request.rowIndex + response.inputCode = str(request.code) + response.warnings.append( + _note(pin, f"code pinned to {pinned} (reference proposed {original_code}).") + ) + + for rule in fired: + if rule.action is RuleAction.ADD_DOCUMENTS: + for code in rule.documentCodes: + document = _rule_document(code, rule) + target = ( + response.documentsExemption + if document.requirementType is DocumentRequirementType.EXEMPTION + else response.documentsRequired + ) + if any(existing.code == document.code for existing in target): + continue + target.append(document) + response.noneRequired = False + response.warnings.append( + _note(rule, f"added {', '.join(rule.documentCodes)}.") + ) + + elif rule.action is RuleAction.REMOVE_DOCUMENTS: + removed = {code.strip().upper() for code in rule.documentCodes} + response.documentsRequired = [ + document + for document in response.documentsRequired + if document.code not in removed + ] + response.documentsExemption = [ + document + for document in response.documentsExemption + if document.code not in removed + ] + response.documentsAlternatives = [ + [document for document in group if document.code not in removed] + for group in response.documentsAlternatives + ] + response.documentsAlternatives = [ + group for group in response.documentsAlternatives if group + ] + response.warnings.append(_note(rule, f"suppressed {', '.join(sorted(removed))}.")) + + elif rule.action is RuleAction.NOTE: + response.warnings.append(_note(rule, rule.note or "no note recorded.")) + + response.appliedRules = [ + AppliedRule( + ruleId=rule.id, + name=rule.name, + action=rule.action, + conditions=rule.conditions, + note=rule.note, + ) + for rule in fired + ] + + _cap_confidence(response, fired) + return response + + +def _cap_confidence(response: LineCheckResponse, fired: list[RuleModel]) -> None: + """A rule-touched answer is never green: it is the broker's logic, not TARIC's.""" + touched_code = any(rule.action is RuleAction.PIN_CODE for rule in fired) + touched_documents = any( + rule.action in (RuleAction.ADD_DOCUMENTS, RuleAction.REMOVE_DOCUMENTS) + for rule in fired + ) + names = ", ".join(f"'{rule.name}'" for rule in fired) + + if touched_code and response.codeConfidence is ConfidenceLevel.GREEN: + response.codeConfidence = ConfidenceLevel.AMBER + response.codeConfidenceReason = f"Code set by rule {names}, not derived from TARIC." + + if touched_documents and response.documentConfidence is ConfidenceLevel.GREEN: + response.documentConfidence = ConfidenceLevel.AMBER + response.documentConfidenceReason = ( + f"Document codes adjusted by rule {names}, not derived from TARIC alone." + ) + + if touched_code or touched_documents: + response.source = ReferenceSource.MANUAL + + +async def apply_rules_to_lines( + requests: list[LineCheckRequest], + responses: list[LineCheckResponse], + client_profile_id: str | None, + reference_as_of: str | None = None, +) -> list[LineCheckResponse]: + """Apply the client's rules across a whole pack, loading them once.""" + rules = await get_active_rules(client_profile_id) + if not rules: + return responses + + applied: list[LineCheckResponse] = [] + hits: list[str] = [] + for request, response in zip(requests, responses): + result = await apply_rules(request, response, rules, reference_as_of) + hits.extend(rule.ruleId for rule in result.appliedRules) + applied.append(result) + + await record_rule_hits(hits) + return applied diff --git a/cbh/api/rules/views.py b/cbh/api/rules/views.py new file mode 100644 index 0000000000000000000000000000000000000000..bf6f653f4350bcde350c61df9bfc3f0d129bf95d --- /dev/null +++ b/cbh/api/rules/views.py @@ -0,0 +1,103 @@ +from datetime import datetime + +from fastapi import Query + +from cbh.api.common.db_requests import delete_obj, get_all_objs, insert_obj, update_obj +from cbh.api.common.dto import Paging +from cbh.api.common.schemas import AllObjectsResponse +from cbh.api.rules import rules_router +from cbh.api.rules.db_requests import get_rule +from cbh.api.rules.models import RuleModel +from cbh.api.rules.schemas import RuleRequest, RuleResponse +from cbh.core.wrappers import CbhResponseWrapper + + +def to_rule_response(rule: RuleModel) -> RuleResponse: + return RuleResponse( + id=rule.id, + name=rule.name, + description=rule.description, + scope=rule.scope, + clientProfileId=rule.clientProfileId, + active=rule.active, + codePrefix=rule.codePrefix, + originCountry=rule.originCountry, + destinationCountry=rule.destinationCountry, + direction=rule.direction, + descriptionContains=rule.descriptionContains, + conditions=rule.conditions, + action=rule.action, + pinnedCode=rule.pinnedCode, + documentCodes=rule.documentCodes, + note=rule.note, + createdFromBatchId=rule.createdFromBatchId, + createdFromRowIndex=rule.createdFromRowIndex, + createdBy=rule.createdBy, + hitCount=rule.hitCount, + datetimeLastHit=rule.datetimeLastHit, + datetimeInserted=rule.datetimeInserted, + datetimeUpdated=rule.datetimeUpdated, + ) + + +@rules_router.get("") +async def list_rules( + clientProfileId: str | None = Query(default=None), + activeOnly: bool = Query(default=False), + pageSize: int = Query(default=50, le=200), + pageIndex: int = Query(default=0, ge=0), +) -> CbhResponseWrapper[AllObjectsResponse[RuleResponse]]: + filter_: dict = {} + if clientProfileId: + filter_["clientProfileId"] = clientProfileId + if activeOnly: + filter_["active"] = True + + rules, total = await get_all_objs( + RuleModel, + page_size=pageSize, + page_index=pageIndex, + sort=("datetimeInserted", -1), + additional_filter=filter_, + ) + return CbhResponseWrapper( + data=AllObjectsResponse( + paging=Paging(pageSize=pageSize, pageIndex=pageIndex, totalCount=total), + data=[to_rule_response(rule) for rule in rules], + ) + ) + + +@rules_router.get("/{ruleId}") +async def get_rule_details(ruleId: str) -> CbhResponseWrapper[RuleResponse]: + return CbhResponseWrapper(data=to_rule_response(await get_rule(ruleId))) + + +@rules_router.post("") +async def create_rule(data: RuleRequest) -> CbhResponseWrapper[RuleResponse]: + rule = RuleModel(**data.model_dump()) + await insert_obj(rule) + return CbhResponseWrapper(data=to_rule_response(rule)) + + +@rules_router.put("/{ruleId}") +async def replace_rule(ruleId: str, data: RuleRequest) -> CbhResponseWrapper[RuleResponse]: + existing = await get_rule(ruleId) + updated = RuleModel( + **{ + **existing.model_dump(), + **data.model_dump(), + "id": ruleId, + "hitCount": existing.hitCount, + "datetimeLastHit": existing.datetimeLastHit, + "datetimeInserted": existing.datetimeInserted, + "datetimeUpdated": datetime.now(), + } + ) + await update_obj(RuleModel, ruleId, updated.to_mongo()) + return CbhResponseWrapper(data=to_rule_response(updated)) + + +@rules_router.delete("/{ruleId}") +async def delete_rule(ruleId: str) -> CbhResponseWrapper[RuleResponse]: + return CbhResponseWrapper(data=to_rule_response(await delete_obj(RuleModel, ruleId))) diff --git a/cbh/api/taric/__init__.py b/cbh/api/taric/__init__.py index eea07c1e89c895af996525812acf739ab21eb243..d702b35404a624dd84d8f2c5937a913dd5746d33 100644 --- a/cbh/api/taric/__init__.py +++ b/cbh/api/taric/__init__.py @@ -1,10 +1,3 @@ -""" -TARIC module initialization. - -This module defines the FastAPI router for the reference data API endpoints: -nomenclature, measures, document codes and geographical areas. -""" - from fastapi import APIRouter taric_router = APIRouter( diff --git a/cbh/api/taric/db_requests.py b/cbh/api/taric/db_requests.py index 6448adf1784c56d77a51c579c0ee22e31d726aef..aaf893f7a94cf3a06f678d74586dacd52e3b6b1a 100644 --- a/cbh/api/taric/db_requests.py +++ b/cbh/api/taric/db_requests.py @@ -1,33 +1,22 @@ -""" -TARIC reference data queries. -""" - -from datetime import date +from datetime import date, timedelta from cbh.api.common.db_requests import find_objs from cbh.api.common.dto import TradeDirection -from cbh.api.common.utils import dedupe, to_iso +from cbh.api.common.utils import dedupe from cbh.api.taric.models import ( CodeChangeModel, - CodeboekEntryModel, DocumentCodeModel, GeoAreaModel, MeasureModel, NomenclatureModel, ) -from cbh.api.taric.utils import code_ancestors, code_prefix_filter, validity_filter +from cbh.api.taric.utils import code_prefix_filter, padded_code_levels, validity_filter from cbh.core.config import settings async def get_nomenclature_by_code( code: str, on_date: date | None = None, declarable_only: bool = False ) -> NomenclatureModel | None: - """ - Fetch a single nomenclature node by its exact code. - - When ``on_date`` is given only the version in force on that date is - returned; otherwise the most recent version wins. - """ filter_: dict = {"code": code} if declarable_only: filter_["declarable"] = True @@ -45,12 +34,6 @@ async def find_codes_by_prefix( declarable_only: bool = True, limit: int = 100, ) -> list[NomenclatureModel]: - """ - Find every code sitting under a 6- or 8-digit prefix. - - This is the expansion step of code resolution: a broker types an 8-digit CN - code and we look for the declarable 10-digit codes beneath it. - """ filter_: dict = dict(code_prefix_filter(prefix)) if declarable_only: filter_["declarable"] = True @@ -60,41 +43,18 @@ async def find_codes_by_prefix( async def find_code_versions(code: str, limit: int = 20) -> list[NomenclatureModel]: - """ - Every historical version of a code, newest first. - - Used to tell "this code never existed" apart from "this code is no longer - valid on the declaration date". - """ return await find_objs( NomenclatureModel, {"code": code}, limit=limit, sort=("validFrom", -1) ) async def get_geo_area_codes_for_country(country: str, on_date: date) -> list[str]: - """ - Every geographical area code that reaches a country on a given date. - - That is the country itself, every group it is a member of, and ERGA OMNES, - which covers measures applying to all third countries. - """ filter_ = {"memberCodes": country} filter_.update(validity_filter(on_date)) groups = await find_objs(GeoAreaModel, filter_, limit=200) return dedupe([country, settings.ERGA_OMNES_CODE] + [group.code for group in groups]) -async def get_geo_area(code: str, on_date: date | None = None) -> GeoAreaModel | None: - """ - Fetch a geographical area by code. - """ - filter_: dict = {"code": code} - if on_date: - filter_.update(validity_filter(on_date)) - results = await find_objs(GeoAreaModel, filter_, limit=1, sort=("validFrom", -1)) - return results[0] if results else None - - async def find_measures_for_line( code: str, geo_area_codes: list[str], @@ -102,18 +62,10 @@ async def find_measures_for_line( on_date: date, additional_codes: list[str] | None = None, ) -> list[MeasureModel]: - """ - Every measure in force for a resolved code, country and direction. - - Measures are attached at different levels of the nomenclature tree and are - inherited downwards, so all ancestor levels of the code are queried, not - just the 10-digit leaf. - """ - directions = [direction.value, TradeDirection.BOTH.value] filter_: dict = { - "goodsCode": {"$in": code_ancestors(code)}, + "goodsCode": {"$in": padded_code_levels(code)}, "geoAreaCode": {"$in": geo_area_codes}, - "direction": {"$in": directions}, + "direction": {"$in": [direction.value, TradeDirection.BOTH.value]}, } filter_.update(validity_filter(on_date)) @@ -121,11 +73,6 @@ async def find_measures_for_line( allowed_additional = set(additional_codes or []) if not allowed_additional: - # The line names no additional code. An additional code selects which - # variant of a measure applies (which anti-dumping rate, for which - # producer) — it does not decide whether the measure exists. Dropping - # these would hide the anti-dumping measure from the line entirely, so - # they are kept and the engine asks for the code to be chosen. return measures return [ @@ -136,9 +83,6 @@ async def find_measures_for_line( async def get_document_codes(codes: list[str]) -> dict[str, DocumentCodeModel]: - """ - Look up Box 44 document codes, keyed by code. - """ if not codes: return {} documents = await find_objs(DocumentCodeModel, {"code": {"$in": dedupe(codes)}}, limit=0) @@ -146,33 +90,12 @@ async def get_document_codes(codes: list[str]) -> dict[str, DocumentCodeModel]: async def get_code_changes(old_code: str) -> list[CodeChangeModel]: - """ - Successor codes recorded for a withdrawn code. - """ return await find_objs( CodeChangeModel, {"oldCode": old_code}, limit=10, sort=("changedOn", -1) ) -async def find_codeboek_entries( - codebook: str, codes: list[str] | None = None -) -> list[CodeboekEntryModel]: - """ - Read entries of a Dutch national codebook. - """ - filter_: dict = {"codebook": codebook} - if codes: - filter_["code"] = {"$in": codes} - return await find_objs(CodeboekEntryModel, filter_, limit=0, sort=("code", 1)) - - async def get_reference_as_of() -> str | None: - """ - Timestamp of the most recent successful reference-data import. - - Stamped onto every line-check answer so a result can be tied back to the - data version it was produced from. - """ latest = await settings.DB_CLIENT.synclogs.find_one( {"status": 3}, sort=[("finishedAt", -1)], projection={"_id": 0, "finishedAt": 1} ) @@ -180,9 +103,6 @@ async def get_reference_as_of() -> str | None: async def count_reference_data() -> dict[str, int]: - """ - Row counts per reference collection, for the importer status endpoint. - """ collections = { "nomenclature": "nomenclature", "measures": "measures", @@ -200,14 +120,12 @@ async def count_reference_data() -> dict[str, int]: async def find_expiring_codes(on_date: date, within_days: int) -> list[NomenclatureModel]: - """ - Declarable codes whose validity ends within ``within_days``. - - Used to preview the 1 January nomenclature switch before it bites. - """ - horizon = date.fromordinal(on_date.toordinal() + within_days) filter_ = { "declarable": True, - "validTo": {"$ne": None, "$gte": to_iso(on_date), "$lte": to_iso(horizon)}, + "validTo": { + "$ne": None, + "$gte": on_date.isoformat(), + "$lte": (on_date + timedelta(days=within_days)).isoformat(), + }, } return await find_objs(NomenclatureModel, filter_, limit=0, sort=("validTo", 1)) diff --git a/cbh/api/taric/dto.py b/cbh/api/taric/dto.py index cd1e0ff03db0ad3bdb64da147b8c7881f94ed3e3..f34ac96bb4dc0fdb5ed8489b66a8af660df7f5af 100644 --- a/cbh/api/taric/dto.py +++ b/cbh/api/taric/dto.py @@ -1,70 +1,36 @@ -""" -TARIC reference data DTOs. -""" +from enum import IntEnum -from enum import Enum - -class DocumentCodeType(Enum): - """ - Box 44 document code families. - - The leading letter of a document code decides how a line must treat it: - C and N codes are obligations, Y codes are the exemptions that discharge - them, X/U codes are the remaining special mentions. - """ - - CERTIFICATE = 1 # C — Union certificates, licences, authorisations - NATIONAL = 2 # N — national / UN-LOC document codes - EXEMPTION = 3 # Y — exemption and "negative" certificates - SPECIAL = 4 # X, U — other Box 44 mentions +class DocumentCodeType(IntEnum): + CERTIFICATE = 1 + NATIONAL = 2 + EXEMPTION = 3 + SPECIAL = 4 UNKNOWN = 99 - @property - def prefix(self) -> str: - """ - Leading letter used by this family in Box 44. - """ - return { - DocumentCodeType.CERTIFICATE: "C", - DocumentCodeType.NATIONAL: "N", - DocumentCodeType.EXEMPTION: "Y", - DocumentCodeType.SPECIAL: "X", - DocumentCodeType.UNKNOWN: "", - }[self] - @property def is_exemption(self) -> bool: - """ - Whether this family discharges an obligation rather than creating one. - """ return self is DocumentCodeType.EXEMPTION @classmethod def from_code(cls, code: str | None) -> "DocumentCodeType": - """ - Derive the family from a document code such as ``C400`` or ``Y900``. - """ if not code: return cls.UNKNOWN + first = code.strip()[:1].upper() + # Dutch national document codes are all-numeric — 1550, 0027, 8025 — + # where the leading digit is the certificate type rather than a letter. + if first.isdigit(): + return cls.NATIONAL return { "C": cls.CERTIFICATE, "N": cls.NATIONAL, "Y": cls.EXEMPTION, "X": cls.SPECIAL, "U": cls.SPECIAL, - }.get(code.strip()[:1].upper(), cls.UNKNOWN) - + }.get(first, cls.UNKNOWN) -class MeasureCategory(Enum): - """ - Semantic grouping of TARIC measure types. - - TARIC ships dozens of numeric measure type ids; the engine only needs to - know what a measure *means* for a declaration line — whether it creates a - document obligation, a warning, or is purely fiscal. - """ +class MeasureCategory(IntEnum): TARIFF_DUTY = 1 ANTIDUMPING = 2 COUNTERVAILING = 3 @@ -81,47 +47,14 @@ class MeasureCategory(Enum): VAT = 14 OTHER = 99 - @property - def is_warning(self) -> bool: - """ - Whether a line touched by this category must carry a warning. - """ - return self in _WARNING_CATEGORIES - - -_WARNING_CATEGORIES = frozenset( - { - MeasureCategory.ANTIDUMPING, - MeasureCategory.COUNTERVAILING, - MeasureCategory.SAFEGUARD, - MeasureCategory.PROHIBITION, - MeasureCategory.RESTRICTION, - MeasureCategory.VETERINARY_CONTROL, - MeasureCategory.PHYTOSANITARY_CONTROL, - MeasureCategory.CITES, - MeasureCategory.DUAL_USE, - } -) - - -class GeoAreaType(Enum): - """ - Kind of geographical area a measure is attached to. - """ +class GeoAreaType(IntEnum): COUNTRY = 1 GROUP = 2 REGION = 3 -class ConditionAction(Enum): - """ - What TARIC does when a measure condition is (not) met. - - ``DOCUMENT_REQUIRED`` is the ordinary "present this certificate" path; - ``MEASURE_NOT_APPLICABLE`` is the exemption path that Y codes travel on. - """ - +class ConditionAction(IntEnum): DOCUMENT_REQUIRED = 1 MEASURE_NOT_APPLICABLE = 2 IMPORT_EXPORT_NOT_ALLOWED = 3 @@ -129,14 +62,7 @@ class ConditionAction(Enum): OTHER = 99 -class CodeChangeType(Enum): - """ - How a nomenclature code was superseded. - - Populated from the TARIC correlation tables; drives the "replacement code" - hint on a line whose code is no longer valid on the declaration date. - """ - +class CodeChangeType(IntEnum): REPLACED = 1 SPLIT = 2 MERGED = 3 diff --git a/cbh/api/taric/models.py b/cbh/api/taric/models.py index 917b0e63738c7dbf90453d0acddb7cd9b971f44c..55695349a775e93ef8847eea715d38dd54b52d8c 100644 --- a/cbh/api/taric/models.py +++ b/cbh/api/taric/models.py @@ -1,11 +1,3 @@ -""" -TARIC and national reference data models. - -Every record carries its own validity window and is never deleted: a dispute -with customs is settled by the rules that were in force on the day of the -declaration, so imports supersede rather than overwrite history. -""" - from datetime import datetime from pydantic import Field, field_validator @@ -19,17 +11,10 @@ from cbh.api.taric.dto import ( GeoAreaType, MeasureCategory, ) -from cbh.core.database import MongoBaseModel, MongoBaseShortenModel, MongoSerializableModel +from cbh.core.database import MongoBaseModel, MongoSerializableModel class NomenclatureModel(MongoBaseModel, ValidityMixin): - """ - A goods-code node of the combined nomenclature. - - Only ``declarable`` codes may appear on a declaration line; the intermediate - nodes are kept because measures attach to them and are inherited downwards. - """ - code: str chapter: str = Field(default="", validate_default=True) indent: int = 0 @@ -47,38 +32,16 @@ class NomenclatureModel(MongoBaseModel, ValidityMixin): @field_validator("chapter", mode="before") @classmethod def derive_chapter(cls, value: str | None, info) -> str: - """ - Derive the HS chapter from the code when it was not supplied. - """ if value: return value - code = (info.data or {}).get("code") or "" - return code[:2] + return ((info.data or {}).get("code") or "")[:2] @property def description(self) -> str | None: - """ - Preferred description: Dutch for Goenee, English as fallback. - """ return self.descriptionNl or self.descriptionEn -class NomenclatureShorten(MongoBaseShortenModel, ValidityMixin): - """ - Nomenclature projection used in candidate lists. - """ - - code: str - descriptionEn: str | None = None - descriptionNl: str | None = None - declarable: bool = False - - class MeasureTypeModel(MongoBaseModel, ValidityMixin): - """ - A TARIC measure type and what it means for a declaration line. - """ - typeId: str description: str | None = None descriptionNl: str | None = None @@ -91,15 +54,6 @@ class MeasureTypeModel(MongoBaseModel, ValidityMixin): class MeasureConditionEmbedded(MongoSerializableModel): - """ - One condition of a measure. - - TARIC expresses document obligations as conditions: conditions sharing a - ``conditionCode`` are alternatives (either/or), and a condition whose action - makes the measure inapplicable is an exemption route — that is where the - Y codes live. - """ - conditionCode: str sequenceNumber: int = 1 @@ -119,25 +73,12 @@ class MeasureConditionEmbedded(MongoSerializableModel): @field_validator("documentType", mode="before") @classmethod def derive_document_type(cls, value, info): - """ - Derive the document family from the document code when absent. - """ if value not in (None, DocumentCodeType.UNKNOWN, DocumentCodeType.UNKNOWN.value): return value return DocumentCodeType.from_code((info.data or {}).get("documentCode")) @property def is_exemption(self) -> bool: - """ - Whether this condition discharges the obligation instead of creating it. - - The document-code prefix decides it: Y codes are exemptions, C/N/X/U - codes are obligations. That convention is stable and documented, while - the numeric action codes vary between releases — and reading an - obligation as an exemption is the error that leaves a client with open - obligations towards customs, so a real document code is never treated - as a way out. - """ if self.documentType.is_exemption: return True if self.documentType is not DocumentCodeType.UNKNOWN: @@ -146,11 +87,6 @@ class MeasureConditionEmbedded(MongoSerializableModel): class MeasureModel(MongoBaseModel, ValidityMixin): - """ - A measure: the link between a goods code, a geographical area and the - obligations that follow from it. - """ - measureSid: str goodsCode: str goodsCodeLength: int = Field(default=0, validate_default=True) @@ -171,6 +107,11 @@ class MeasureModel(MongoBaseModel, ValidityMixin): quotaOrderNumber: str | None = None regulationId: str | None = None + # Written by a member state rather than by the Union. The distinction has to + # survive into the answer: a declarant is entitled to know that an + # obligation comes from Dutch law and not from TARIC. + national: bool = False + conditions: list[MeasureConditionEmbedded] = Field(default_factory=list) datetimeInserted: datetime = Field(default_factory=datetime.now) @@ -178,9 +119,6 @@ class MeasureModel(MongoBaseModel, ValidityMixin): @field_validator("goodsCodeLength", mode="before") @classmethod def derive_goods_code_length(cls, value, info): - """ - Derive the attachment level from the goods code when absent. - """ if value: return value return len((info.data or {}).get("goodsCode") or "") @@ -188,9 +126,6 @@ class MeasureModel(MongoBaseModel, ValidityMixin): @field_validator("category", mode="before") @classmethod def derive_category(cls, value, info): - """ - Classify the measure from its type id when the importer did not. - """ from cbh.api.taric.utils import classify_measure_type if value not in (None, MeasureCategory.OTHER, MeasureCategory.OTHER.value): @@ -200,23 +135,8 @@ class MeasureModel(MongoBaseModel, ValidityMixin): data.get("measureTypeId"), data.get("measureTypeDescription") ) - def applies_to_country(self, country: str, member_of: set[str]) -> bool: - """ - Whether this measure reaches a given country. - - A measure hits a country either directly or through a group it belongs - to, unless the country is explicitly carved out. - """ - if country in self.excludedGeoAreaCodes: - return False - return self.geoAreaCode == country or self.geoAreaCode in member_of - class GeoAreaModel(MongoBaseModel, ValidityMixin): - """ - A country or a group of countries a measure can be attached to. - """ - code: str name: str | None = None areaType: GeoAreaType = GeoAreaType.COUNTRY @@ -226,11 +146,6 @@ class GeoAreaModel(MongoBaseModel, ValidityMixin): class AdditionalCodeModel(MongoBaseModel, ValidityMixin): - """ - A TARIC additional code, used to split a goods code by producer or - application — most visibly on anti-dumping measures. - """ - codeType: str code: str fullCode: str @@ -240,45 +155,29 @@ class AdditionalCodeModel(MongoBaseModel, ValidityMixin): class DocumentCodeModel(MongoBaseModel, ValidityMixin): - """ - A Box 44 document code and its meaning. - """ - code: str documentType: DocumentCodeType = Field( default=DocumentCodeType.UNKNOWN, validate_default=True ) description: str | None = None descriptionNl: str | None = None + national: bool = False datetimeInserted: datetime = Field(default_factory=datetime.now) @field_validator("documentType", mode="before") @classmethod def derive_document_type(cls, value, info): - """ - Derive the document family from the code when absent. - """ if value not in (None, DocumentCodeType.UNKNOWN, DocumentCodeType.UNKNOWN.value): return value return DocumentCodeType.from_code((info.data or {}).get("code")) @property def is_exemption(self) -> bool: - """ - Whether presenting this code discharges an obligation. - """ return self.documentType.is_exemption class CodeChangeModel(MongoBaseModel): - """ - Correlation between a withdrawn code and its successors. - - Drives the replacement hint on a line whose code stopped being valid — - typically at the 1 January nomenclature switch. - """ - oldCode: str newCodes: list[str] = Field(default_factory=list) changeType: CodeChangeType = CodeChangeType.REPLACED @@ -289,14 +188,6 @@ class CodeChangeModel(MongoBaseModel): class CodeboekEntryModel(MongoBaseModel, ValidityMixin): - """ - An entry of a Dutch national codebook (Codeboek Douane). - - These files are replaced whole on every change and carry no - code x country -> document mapping; that stays with TARIC measure - conditions. They are used to resolve national N-codes and Box 44 mentions. - """ - codebook: str code: str description: str | None = None diff --git a/cbh/api/taric/schemas.py b/cbh/api/taric/schemas.py index 7f90dadac6f0ec05b74ca7c11d8360150ad1afc3..699ac3b0210108b412179a27e6bae87c3afe8eff 100644 --- a/cbh/api/taric/schemas.py +++ b/cbh/api/taric/schemas.py @@ -1,7 +1,3 @@ -""" -TARIC reference data schemas. -""" - from datetime import date from pydantic import BaseModel @@ -11,10 +7,6 @@ from cbh.api.taric.dto import DocumentCodeType, MeasureCategory class NomenclatureResponse(BaseModel): - """ - A nomenclature node as returned by the lookup endpoints. - """ - code: str formattedCode: str descriptionEn: str | None = None @@ -26,22 +18,7 @@ class NomenclatureResponse(BaseModel): validTo: date | None = None -class NomenclatureFilter(BaseModel): - """ - Filter for browsing the nomenclature. - """ - - searchTerm: str | None = None - chapter: str | None = None - declarableOnly: bool = True - checkDate: date | None = None - - class MeasureConditionResponse(BaseModel): - """ - A measure condition, flattened for the API. - """ - conditionCode: str sequenceNumber: int documentCode: str | None = None @@ -52,10 +29,6 @@ class MeasureConditionResponse(BaseModel): class MeasureResponse(BaseModel): - """ - A measure applying to a goods code. - """ - measureSid: str goodsCode: str measureTypeId: str @@ -73,10 +46,6 @@ class MeasureResponse(BaseModel): class DocumentCodeResponse(BaseModel): - """ - A Box 44 document code. - """ - code: str documentType: DocumentCodeType description: str | None = None @@ -85,19 +54,11 @@ class DocumentCodeResponse(BaseModel): class GeoAreaResponse(BaseModel): - """ - A geographical area and its members. - """ - code: str name: str | None = None memberCodes: list[str] = [] class ReferenceDataStatusResponse(BaseModel): - """ - Row counts and freshness of the reference data. - """ - counts: dict[str, int] referenceAsOf: str | None = None diff --git a/cbh/api/taric/utils.py b/cbh/api/taric/utils.py index ffd2226d7a5ea9a2fa41a5b98c3975eb20db7721..6b5e99be140efa75b9391d004866620a47327e5a 100644 --- a/cbh/api/taric/utils.py +++ b/cbh/api/taric/utils.py @@ -1,84 +1,79 @@ -""" -TARIC helpers: code normalisation, hierarchy and measure classification. -""" - import re from datetime import date -from cbh.api.common.utils import to_iso -from cbh.api.taric.dto import DocumentCodeType, MeasureCategory - -VALID_CODE_LENGTHS = (6, 8, 10) -TARIC_CODE_LENGTH = 10 - -# Goods-code prefix lengths a measure can be attached to. A measure declared at -# chapter or heading level applies to every code beneath it, so a line check has -# to collect measures at every level of the tree, not just the 10-digit leaf. -MEASURE_LEVELS = (2, 4, 6, 8, 10) +from cbh.api.common.dto import CustomsProcedure +from cbh.api.common.utils import dedupe +from cbh.api.taric.dto import MeasureCategory def normalize_code(raw_code: str | int | None) -> str | None: - """ - Normalise a goods code as typed by a broker. - - Points, spaces and dashes are cosmetic in every client file we have seen, so - they are stripped; what is left must be 6, 8 or 10 digits. Anything else is - rejected rather than guessed at. - """ if raw_code is None: return None digits = re.sub(r"\D", "", str(raw_code)) - if not digits: - return None - if len(digits) not in VALID_CODE_LENGTHS: + if len(digits) == 14: + return digits[:10] + if len(digits) not in (4, 6, 8, 10): return None return digits -def is_full_taric_code(code: str) -> bool: - """ - Whether a normalised code is already a 10-digit TARIC code. +def significant_length(code: str) -> int: + """How many digits of a padded TARIC code actually carry meaning. + + Every code in the published extraction is ten digits, zero-padded: + `6203000000` is heading 6203 and `6203423100` is the eight-digit CN code + 6203 42 31. Trailing pairs of zeros are padding, and the length left over is + what makes one measure more specific than another. """ - return len(code) == TARIC_CODE_LENGTH + length = len(code) + while length > 4 and code[length - 2 : length] == "00": + length -= 2 + return length -def chapter_of(code: str) -> str: - """ - HS chapter (first two digits) of a code. +def padded_code_levels(code: str) -> list[str]: + """The goods codes a line can inherit a measure from, as they are stored. + + A measure written at heading level is stored as `6203000000`, so looking up + `6203` alone finds nothing. Both spellings are returned, because reference + data loaded from other sources is not always padded. """ - return code[:2] + levels: list[str] = [] + for length in (10, 8, 6, 4, 2): + if length > len(code): + continue + prefix = code[:length] + levels.append(prefix) + padded = prefix.ljust(10, "0") + if padded != prefix: + levels.append(padded) + return dedupe(levels) -def code_ancestors(code: str) -> list[str]: - """ - Every goods-code level a measure could be attached to for this code. +def split_additional_code(raw_code: str | int | None) -> str | None: + if raw_code is None: + return None + digits = re.sub(r"\D", "", str(raw_code)) + return digits[10:] if len(digits) == 14 else None - Returned longest-first so the most specific measure is considered first. - """ - return [code[:length] for length in reversed(MEASURE_LEVELS) if length <= len(code)] +def chapter_scope(chapter: str) -> str: + if chapter in ("22", "61", "62", "84", "85"): + return "in_scope" + if chapter.isdigit() and 1 <= int(chapter) <= 24: + return "nvwa_excluded" + return "not_benchmarked" -def code_prefix_filter(prefix: str) -> dict: - """ - MongoDB filter matching every code starting with ``prefix``. - Expressed as a range rather than a regex so the compound index on ``code`` - is used. - """ - padding = TARIC_CODE_LENGTH - len(prefix) +def code_prefix_filter(prefix: str) -> dict: + padding = 10 - len(prefix) if padding <= 0: return {"code": prefix} return {"code": {"$gte": prefix + "0" * padding, "$lte": prefix + "9" * padding}} def validity_filter(on_date: date, prefix: str = "") -> dict: - """ - MongoDB filter selecting records in force on ``on_date``. - - Validity dates are stored as ISO strings, which compare correctly under - lexicographic ordering. - """ - iso = to_iso(on_date) + iso = on_date.isoformat() field_from = f"{prefix}validFrom" field_to = f"{prefix}validTo" return { @@ -89,111 +84,131 @@ def validity_filter(on_date: date, prefix: str = "") -> dict: } -def document_type_from_code(code: str | None) -> DocumentCodeType: - """ - Derive a document family from a Box 44 code. - """ - return DocumentCodeType.from_code(code) - - -# Known TARIC measure type ids grouped by meaning. Ids are stable in TARIC; the -# keyword fallback below covers national measure types and any id added after -# this table was written. -MEASURE_TYPE_CATEGORIES: dict[str, MeasureCategory] = { - "103": MeasureCategory.TARIFF_DUTY, - "105": MeasureCategory.TARIFF_DUTY, - "106": MeasureCategory.TARIFF_DUTY, - "112": MeasureCategory.TARIFF_DUTY, - "115": MeasureCategory.TARIFF_DUTY, - "117": MeasureCategory.TARIFF_DUTY, - "119": MeasureCategory.TARIFF_DUTY, - "122": MeasureCategory.QUOTA, - "123": MeasureCategory.QUOTA, - "143": MeasureCategory.QUOTA, - "146": MeasureCategory.QUOTA, - "277": MeasureCategory.SURVEILLANCE, - "278": MeasureCategory.SURVEILLANCE, - "410": MeasureCategory.VETERINARY_CONTROL, - "420": MeasureCategory.VETERINARY_CONTROL, - "465": MeasureCategory.PHYTOSANITARY_CONTROL, - "474": MeasureCategory.PHYTOSANITARY_CONTROL, - "475": MeasureCategory.PHYTOSANITARY_CONTROL, - "476": MeasureCategory.RESTRICTION, - "477": MeasureCategory.RESTRICTION, - "478": MeasureCategory.RESTRICTION, - "481": MeasureCategory.RESTRICTION, - "482": MeasureCategory.RESTRICTION, - "483": MeasureCategory.RESTRICTION, - "485": MeasureCategory.CITES, - "487": MeasureCategory.RESTRICTION, - "490": MeasureCategory.DUAL_USE, - "495": MeasureCategory.DUAL_USE, - "551": MeasureCategory.ANTIDUMPING, - "552": MeasureCategory.ANTIDUMPING, - "553": MeasureCategory.COUNTERVAILING, - "554": MeasureCategory.COUNTERVAILING, - "555": MeasureCategory.ANTIDUMPING, - "560": MeasureCategory.SAFEGUARD, - "561": MeasureCategory.SAFEGUARD, - "705": MeasureCategory.PROHIBITION, - "706": MeasureCategory.PROHIBITION, - "710": MeasureCategory.PROHIBITION, - "715": MeasureCategory.RESTRICTION, - "750": MeasureCategory.RESTRICTION, - "760": MeasureCategory.RESTRICTION, - "VAT": MeasureCategory.VAT, - "EXC": MeasureCategory.EXCISE, -} - -_CATEGORY_KEYWORDS: tuple[tuple[MeasureCategory, tuple[str, ...]], ...] = ( - (MeasureCategory.ANTIDUMPING, ("anti-dumping", "antidumping", "anti dumping")), - (MeasureCategory.COUNTERVAILING, ("countervailing", "anti-subsidy", "compenserend")), - (MeasureCategory.SAFEGUARD, ("safeguard", "vrijwaring")), - (MeasureCategory.CITES, ("cites", "endangered species", "bedreigde")), - (MeasureCategory.DUAL_USE, ("dual-use", "dual use", "dual-gebruik", "military")), - ( - MeasureCategory.PHYTOSANITARY_CONTROL, - ("phytosanitary", "plant health", "fytosanitair", "ched-pp"), - ), - ( - MeasureCategory.VETERINARY_CONTROL, - ("veterinary", "ched", "animal health", "veterinair"), - ), - (MeasureCategory.PROHIBITION, ("prohibition", "prohibited", "verbod", "embargo")), - (MeasureCategory.RESTRICTION, ("restriction", "restrictive", "beperking", "licence")), - (MeasureCategory.QUOTA, ("quota", "contingent")), - (MeasureCategory.SURVEILLANCE, ("surveillance", "toezicht")), - (MeasureCategory.EXCISE, ("excise", "accijns")), - (MeasureCategory.VAT, ("vat", "value added tax", "btw")), - (MeasureCategory.TARIFF_DUTY, ("duty", "tariff", "preference", "recht")), -) - - def classify_measure_type( type_id: str | None, description: str | None = None ) -> MeasureCategory: - """ - Map a TARIC measure type onto a semantic category. - - The id table wins; the keyword scan over the description is the fallback for - national measure types and ids added upstream after this table was written. - """ if type_id: - known = MEASURE_TYPE_CATEGORIES.get(str(type_id).strip().upper()) + known = { + "103": MeasureCategory.TARIFF_DUTY, + "105": MeasureCategory.TARIFF_DUTY, + "106": MeasureCategory.TARIFF_DUTY, + "112": MeasureCategory.TARIFF_DUTY, + "115": MeasureCategory.TARIFF_DUTY, + "117": MeasureCategory.TARIFF_DUTY, + "119": MeasureCategory.TARIFF_DUTY, + "122": MeasureCategory.QUOTA, + "123": MeasureCategory.QUOTA, + "143": MeasureCategory.QUOTA, + "146": MeasureCategory.QUOTA, + "277": MeasureCategory.SURVEILLANCE, + "278": MeasureCategory.SURVEILLANCE, + "410": MeasureCategory.VETERINARY_CONTROL, + "420": MeasureCategory.VETERINARY_CONTROL, + "465": MeasureCategory.PHYTOSANITARY_CONTROL, + "474": MeasureCategory.PHYTOSANITARY_CONTROL, + "475": MeasureCategory.PHYTOSANITARY_CONTROL, + "476": MeasureCategory.RESTRICTION, + "477": MeasureCategory.RESTRICTION, + "478": MeasureCategory.RESTRICTION, + "481": MeasureCategory.RESTRICTION, + "482": MeasureCategory.RESTRICTION, + "483": MeasureCategory.RESTRICTION, + "485": MeasureCategory.CITES, + "487": MeasureCategory.RESTRICTION, + "490": MeasureCategory.DUAL_USE, + "495": MeasureCategory.DUAL_USE, + "551": MeasureCategory.ANTIDUMPING, + "552": MeasureCategory.ANTIDUMPING, + "553": MeasureCategory.COUNTERVAILING, + "554": MeasureCategory.COUNTERVAILING, + "555": MeasureCategory.ANTIDUMPING, + "560": MeasureCategory.SAFEGUARD, + "561": MeasureCategory.SAFEGUARD, + "705": MeasureCategory.PROHIBITION, + "706": MeasureCategory.PROHIBITION, + "710": MeasureCategory.PROHIBITION, + # Export-side controls, read off the published Duties Export + # extraction: these are what carry the Y codes on an export pack. + "715": MeasureCategory.CITES, + "717": MeasureCategory.RESTRICTION, + "718": MeasureCategory.RESTRICTION, + # "Export control on cat and dog fur" is a control with a standing + # Y-exemption, not an outright ban. Classing it as a prohibition + # made every apparel line blocking, and so invalid. + "740": MeasureCategory.RESTRICTION, + "766": MeasureCategory.RESTRICTION, + "750": MeasureCategory.RESTRICTION, + "760": MeasureCategory.RESTRICTION, + "VAT": MeasureCategory.VAT, + "EXC": MeasureCategory.EXCISE, + }.get(str(type_id).strip().upper()) if known: return known + if description: haystack = description.lower() - for category, keywords in _CATEGORY_KEYWORDS: + for category, keywords in ( + (MeasureCategory.ANTIDUMPING, ("anti-dumping", "antidumping", "anti dumping")), + (MeasureCategory.COUNTERVAILING, ("countervailing", "anti-subsidy", "compenserend")), + (MeasureCategory.SAFEGUARD, ("safeguard", "vrijwaring")), + (MeasureCategory.CITES, ("cites", "endangered species", "bedreigde")), + (MeasureCategory.DUAL_USE, ("dual-use", "dual use", "dual-gebruik", "military")), + ( + MeasureCategory.PHYTOSANITARY_CONTROL, + ("phytosanitary", "plant health", "fytosanitair", "ched-pp"), + ), + ( + MeasureCategory.VETERINARY_CONTROL, + ("veterinary", "ched", "animal health", "veterinair"), + ), + (MeasureCategory.PROHIBITION, ("prohibition", "prohibited", "verbod", "embargo")), + ( + MeasureCategory.RESTRICTION, + ("restriction", "restrictive", "beperking", "licence", "export control"), + ), + (MeasureCategory.QUOTA, ("quota", "contingent")), + (MeasureCategory.SURVEILLANCE, ("surveillance", "toezicht")), + (MeasureCategory.EXCISE, ("excise", "accijns")), + (MeasureCategory.VAT, ("vat", "value added tax", "btw")), + (MeasureCategory.TARIFF_DUTY, ("duty", "tariff", "preference", "recht")), + ): if any(keyword in haystack for keyword in keywords): return category + return MeasureCategory.OTHER -def format_code(code: str) -> str: - """ - Render a 10-digit code the way brokers read it: ``6109 10 00 10``. +# What a transit or warehouse declaration secures rather than settles. Duty, +# tax and quota are suspended against a guarantee and fall due at the office of +# destination; everything else — bans, licences, CITES, dual-use, health control +# — binds the moment the goods move, so it stays on the line. +SUSPENDED_UNDER_SUSPENSIVE_PROCEDURE = frozenset( + { + MeasureCategory.TARIFF_DUTY, + MeasureCategory.QUOTA, + MeasureCategory.VAT, + MeasureCategory.EXCISE, + MeasureCategory.ANTIDUMPING, + MeasureCategory.COUNTERVAILING, + MeasureCategory.SAFEGUARD, + } +) + + +def measure_is_suspended( + category: MeasureCategory, procedure: CustomsProcedure +) -> bool: + """Whether this measure's obligation is deferred by the procedure. + + Anything not named as fiscal stays binding, `OTHER` included. A measure the + classifier could not place is not a licence to leave Box 44 empty. """ - if len(code) != TARIC_CODE_LENGTH: + if not procedure.suspends_duty: + return False + return category in SUSPENDED_UNDER_SUSPENSIVE_PROCEDURE + + +def format_code(code: str) -> str: + if len(code) != 10: return code return f"{code[:4]} {code[4:6]} {code[6:8]} {code[8:10]}" diff --git a/cbh/api/taric/views.py b/cbh/api/taric/views.py index 0e316b52c52105a0fd8a6fd34fdd9091783310f8..fe617eccf9b8e572ad4ace2f3a9cf7415abb0b39 100644 --- a/cbh/api/taric/views.py +++ b/cbh/api/taric/views.py @@ -1,17 +1,10 @@ -""" -TARIC views module. -""" - from datetime import date -from fastapi import Depends, HTTPException, Query +from fastapi import HTTPException, Query -from cbh.api.account.dto import AccountType -from cbh.api.account.models import AccountModel from cbh.api.common.db_requests import get_all_objs -from cbh.api.common.dto import TradeDirection +from cbh.api.common.dto import Paging, TradeDirection from cbh.api.common.schemas import AllObjectsResponse -from cbh.api.common.utils import to_paging from cbh.api.taric import taric_router from cbh.api.taric.db_requests import ( count_reference_data, @@ -37,16 +30,10 @@ from cbh.api.taric.utils import ( validity_filter, ) from cbh.core.config import settings -from cbh.core.security import PermissionDependency from cbh.core.wrappers import CbhResponseWrapper -ANY_ACCOUNT = [AccountType.USER, AccountType.ADMIN, AccountType.OWNER, AccountType.SUPER_ADMIN] - def _to_nomenclature_response(model: NomenclatureModel) -> NomenclatureResponse: - """ - Map a nomenclature model onto its API representation. - """ return NomenclatureResponse( code=model.code, formattedCode=format_code(model.code), @@ -68,11 +55,7 @@ async def search_nomenclature( checkDate: date | None = Query(default=None), pageSize: int = Query(default=25, le=200), pageIndex: int = Query(default=0, ge=0), - account: AccountModel = Depends(PermissionDependency(ANY_ACCOUNT)), ) -> CbhResponseWrapper[AllObjectsResponse[NomenclatureResponse]]: - """ - Browse the nomenclature by code prefix, chapter or free text. - """ filter_: dict = {} if declarableOnly: filter_["declarable"] = True @@ -101,7 +84,7 @@ async def search_nomenclature( ) return CbhResponseWrapper( data=AllObjectsResponse( - paging=to_paging(pageSize, pageIndex, total), + paging=Paging(pageSize=pageSize, pageIndex=pageIndex, totalCount=total), data=[_to_nomenclature_response(model) for model in models], ) ) @@ -111,11 +94,7 @@ async def search_nomenclature( async def get_nomenclature( code: str, checkDate: date | None = Query(default=None), - account: AccountModel = Depends(PermissionDependency(ANY_ACCOUNT)), ) -> CbhResponseWrapper[NomenclatureResponse]: - """ - Fetch a single nomenclature node. - """ normalized = normalize_code(code) if not normalized: raise HTTPException(status_code=400, detail="Invalid goods code.") @@ -129,11 +108,7 @@ async def get_nomenclature( async def get_code_candidates( code: str, checkDate: date | None = Query(default=None), - account: AccountModel = Depends(PermissionDependency(ANY_ACCOUNT)), ) -> CbhResponseWrapper[list[NomenclatureResponse]]: - """ - List the declarable 10-digit codes sitting under a 6- or 8-digit code. - """ normalized = normalize_code(code) if not normalized: raise HTTPException(status_code=400, detail="Invalid goods code.") @@ -151,14 +126,7 @@ async def get_measures( country: str = Query(...), direction: TradeDirection = Query(default=TradeDirection.IMPORT), checkDate: date | None = Query(default=None), - account: AccountModel = Depends(PermissionDependency(ANY_ACCOUNT)), ) -> CbhResponseWrapper[list[MeasureResponse]]: - """ - List the measures in force for a code, country and direction. - - This is the audit view behind a line check: it shows every measure the - engine considered, before document extraction. - """ normalized = normalize_code(code) if not normalized: raise HTTPException(status_code=400, detail="Invalid goods code.") @@ -204,11 +172,7 @@ async def list_document_codes( searchTerm: str | None = Query(default=None), pageSize: int = Query(default=50, le=200), pageIndex: int = Query(default=0, ge=0), - account: AccountModel = Depends(PermissionDependency(ANY_ACCOUNT)), ) -> CbhResponseWrapper[AllObjectsResponse[DocumentCodeResponse]]: - """ - Browse the Box 44 document codes. - """ filter_: dict = {} if searchTerm: filter_["$or"] = [ @@ -225,7 +189,7 @@ async def list_document_codes( ) return CbhResponseWrapper( data=AllObjectsResponse( - paging=to_paging(pageSize, pageIndex, total), + paging=Paging(pageSize=pageSize, pageIndex=pageIndex, totalCount=total), data=[ DocumentCodeResponse( code=model.code, @@ -245,22 +209,17 @@ async def list_geo_areas( country: str | None = Query(default=None), pageSize: int = Query(default=50, le=200), pageIndex: int = Query(default=0, ge=0), - account: AccountModel = Depends(PermissionDependency(ANY_ACCOUNT)), ) -> CbhResponseWrapper[AllObjectsResponse[GeoAreaResponse]]: - """ - Browse geographical areas, optionally only those containing a country. - """ - filter_ = {"memberCodes": country.upper()} if country else {} models, total = await get_all_objs( GeoAreaModel, page_size=pageSize, page_index=pageIndex, sort=("code", 1), - additional_filter=filter_, + additional_filter={"memberCodes": country.upper()} if country else {}, ) return CbhResponseWrapper( data=AllObjectsResponse( - paging=to_paging(pageSize, pageIndex, total), + paging=Paging(pageSize=pageSize, pageIndex=pageIndex, totalCount=total), data=[ GeoAreaResponse(code=model.code, name=model.name, memberCodes=model.memberCodes) for model in models @@ -273,11 +232,7 @@ async def list_geo_areas( async def list_expiring_codes( withinDays: int | None = Query(default=None), checkDate: date | None = Query(default=None), - account: AccountModel = Depends(PermissionDependency(ANY_ACCOUNT)), ) -> CbhResponseWrapper[list[NomenclatureResponse]]: - """ - Codes whose validity ends soon — the 1 January switch, seen in advance. - """ models = await find_expiring_codes( checkDate or date.today(), withinDays or settings.EXPIRING_SOON_DAYS ) @@ -285,12 +240,7 @@ async def list_expiring_codes( @taric_router.get("/status") -async def get_reference_status( - account: AccountModel = Depends(PermissionDependency(ANY_ACCOUNT)), -) -> CbhResponseWrapper[ReferenceDataStatusResponse]: - """ - Row counts and freshness of the loaded reference data. - """ +async def get_reference_status() -> CbhResponseWrapper[ReferenceDataStatusResponse]: return CbhResponseWrapper( data=ReferenceDataStatusResponse( counts=await count_reference_data(), diff --git a/cbh/core/config.py b/cbh/core/config.py index 2c9069cb4eab68a7af1e01d7130ca0cbb391adf3..8adbed6b60d80e635e6779f3383c9cc3e35e22b4 100644 --- a/cbh/core/config.py +++ b/cbh/core/config.py @@ -1,43 +1,18 @@ -""" -Configuration module for ClipboardHealthAI application. -""" - -import json import os import pathlib from functools import lru_cache -from typing import Optional, Type from apscheduler.schedulers.asyncio import AsyncIOScheduler from dotenv import load_dotenv -from langchain_core.runnables import Runnable -from langchain_openai import ChatOpenAI, OpenAIEmbeddings from motor.motor_asyncio import AsyncIOMotorClient, AsyncIOMotorDatabase -from openai import AsyncOpenAI -from pydantic import BaseModel load_dotenv() class BaseConfig: - """ - Base configuration class containing common settings for all environments. - """ - BASE_DIR: pathlib.Path = pathlib.Path(__file__).parent.parent.parent - SECRET_KEY: str = os.getenv("SECRET", "") - MACHINE = os.getenv("MACHINE") ENVIRONMENT: str = os.getenv("FASTAPI_CONFIG") - OPENAI_CLIENT: AsyncOpenAI = AsyncOpenAI(api_key=os.getenv("OPENAI_API_KEY")) - - GOOGLE_CLIENT_ID: str = os.getenv("GOOGLE_CLIENT_ID", "") - GOOGLE_CLIENT_SECRET: str = os.getenv("GOOGLE_CLIENT_SECRET", "") - - OPENAI_API_KEY: str = os.getenv("OPENAI_API_KEY", "") - - CONVERTIO_API_KEY = os.getenv("CONVERTIO_API_KEY") - SCHEDULER = AsyncIOScheduler() DB_CLIENT: AsyncIOMotorDatabase = AsyncIOMotorClient(os.getenv("MONGODB_URL")).cargoflow @@ -50,102 +25,52 @@ class BaseConfig: "TARIC_BASE_URL", "https://circabc.europa.eu/ui/group/0e5f18c2-4b2f-42e9-aed4-dfe50ae1263b", ) + # DG TAXUD publishes the extraction on CIRCABC behind a UI that has no + # stable per-file address. This is an automated mirror of that same group, + # tagged by publication month, which does have one. Repoint it the day the + # broker gets a direct feed. + TARIC_ARCHIVE_URL: str = os.getenv( + "TARIC_ARCHIVE_URL", + "https://github.com/rousseauxy/taric-opendata/releases/download/" + "eu-{period}/eu-taric-{period}.zip", + ) + TARIC_ARCHIVE_LOOKBACK_MONTHS: int = int( + os.getenv("TARIC_ARCHIVE_LOOKBACK_MONTHS", "3") + ) CODEBOEK_BASE_URL: str = os.getenv( "CODEBOEK_BASE_URL", "https://kennisbank.douane.nl/sites/default/files/codebooks", ) + # The Dutch Gebruikstarief, published by the Belastingdienst. This is the + # only place the national measures live: TARIC carries the Union rules, the + # BTW and accijns lines are added here. + DTV_MANIFEST_URL: str = os.getenv( + "DTV_MANIFEST_URL", + "https://download.belastingdienst.nl/douane_sw/tariff/download_bestanden.xml", + ) + # The feed mixes nightly increments of a few dozen kB with the weekly full + # snapshot of ~240 MB. Size is what tells them apart before downloading. + DTV_FULL_SNAPSHOT_MIN_BYTES: int = int( + os.getenv("DTV_FULL_SNAPSHOT_MIN_BYTES", str(50 * 1024 * 1024)) + ) IMPORTER_HTTP_TIMEOUT: int = int(os.getenv("IMPORTER_HTTP_TIMEOUT", "300")) IMPORTER_ENABLED: bool = os.getenv("IMPORTER_ENABLED", "false").lower() == "true" EXPIRING_SOON_DAYS: int = int(os.getenv("EXPIRING_SOON_DAYS", "60")) ERGA_OMNES_CODE: str = os.getenv("ERGA_OMNES_CODE", "1011") LINE_CHECK_CACHE_TTL_SECONDS: int = int(os.getenv("LINE_CHECK_CACHE_TTL_SECONDS", "86400")) - IN_SCOPE_CHAPTERS: tuple[str, ...] = ("22", "61", "62", "84", "85") - - @staticmethod - def get_headers(api_key: str) -> dict: - """ - Generate HTTP headers for API requests. - """ - return { - "Authorization": f"Bearer {api_key}", - "Content-Type": "application/json", - "Accept": "application/json", - } - - @staticmethod - @lru_cache() - def get_llm( - model: str = "gpt-5.4-mini", - temperature: float = 0.0, - reasoning_effort: str = "low", - reasoning_summary: str = "none", - schema: Optional[Type[BaseModel]] = None, - is_json: bool = False, - tool: str = None, - streaming: bool = False, - ) -> Runnable: - """ - Get a configured LLM instance. - """ - kwargs = { - "model": model, - "temperature": temperature, - "use_responses_api": True, - "streaming": streaming, - } - if model.startswith("gpt-5"): - # kwargs['reasoning_effort'] = reasoning_effort - kwargs["reasoning"] = {"effort": reasoning_effort} - if reasoning_summary != "none": - kwargs["reasoning"]["summary"] = reasoning_summary - kwargs["temperature"] = 1 - - if tool: - loaded_tool = json.loads(tool) - return ChatOpenAI(**kwargs).bind_tools(loaded_tool) - - if schema: - return ChatOpenAI(**kwargs).with_structured_output(schema) - - if is_json: - return ChatOpenAI(**kwargs).with_structured_output(method="json_mode") - - return ChatOpenAI(**kwargs) - - @staticmethod - @lru_cache() - def get_embedding_client( - model: str = "text-embedding-3-small", dimensions: int = 384 - ) -> OpenAIEmbeddings: - return OpenAIEmbeddings(model=model, dimensions=dimensions) class DevelopmentConfig(BaseConfig): - """ - Development environment configuration settings. - """ - - Issuer = "https://trainwitharena.click" - Audience = "https://trainwitharena.click" - Domain = "trainwitharena.click" + pass class ProductionConfig(BaseConfig): - """ - Production environment configuration settings. - """ - - Issuer = "https://trainwitharena.com" - Audience = "https://trainwitharena.com" - Domain = "trainwitharena.com" + pass @lru_cache() def get_settings() -> DevelopmentConfig | ProductionConfig: - """ - Get the appropriate configuration based on the current environment. - """ config_cls_dict = { "development": DevelopmentConfig, "production": ProductionConfig, diff --git a/cbh/core/database.py b/cbh/core/database.py index 88dca45f889198d09ddb65bebda126b3eac2fc16..6564f2c200b400d8af27ad76cf6faf3b97e58a9e 100644 --- a/cbh/core/database.py +++ b/cbh/core/database.py @@ -1,32 +1,18 @@ -""" -Database utilities for ClipboardHealthAI application. -""" - -from datetime import datetime, date, time +from datetime import date, datetime, time from enum import Enum -import re -from typing import Any, Dict, Type, TypeVar +from typing import Any, Dict, Type from bson import ObjectId from pydantic import AnyUrl, BaseModel, Field, GetCoreSchemaHandler from pydantic.json_schema import JsonSchemaValue from pydantic_core import core_schema -T = TypeVar("T", bound=BaseModel) - class PyObjectId: - """ - Custom type for handling MongoDB ObjectId in Pydantic models. - """ - @classmethod def __get_pydantic_core_schema__( cls, _source: type, _handler: GetCoreSchemaHandler ) -> core_schema.CoreSchema: - """ - Define the core schema for Pydantic validation. - """ return core_schema.with_info_after_validator_function( cls.validate, core_schema.str_schema() # type: ignore ) @@ -35,59 +21,29 @@ class PyObjectId: def __get_pydantic_json_schema__( cls, _schema: core_schema.CoreSchema, _handler: GetCoreSchemaHandler ) -> JsonSchemaValue: - """ - Define the JSON schema representation. - """ return {"type": "string"} @classmethod def validate(cls, value: str) -> ObjectId: - """ - Validate and convert a string to MongoDB ObjectId. - """ if not ObjectId.is_valid(value): raise ValueError(f"Invalid ObjectId: {value}") return ObjectId(value) def __getattr__(self, item): - """ - Delegate attribute access to the wrapped ObjectId. - """ return getattr(self.__dict__["value"], item) def __init__(self, value: str | None = None): - """ - Initialize with a string value or create a new ObjectId. - """ if value is None: self.value = ObjectId() else: self.value = self.validate(value) def __str__(self): - """ - Convert to string representation. - """ return str(self.value) class MongoSerializableModel(BaseModel): - @staticmethod - def serialize_s3_url(value: Any) -> Any: - """ - Serialize an S3 URL. - """ - if value and isinstance(value, str) and "AWSAccessKeyId" in value and "Expires" in value: - match = re.search(r"s3\.amazonaws\.com/([^?]+)", value) - if match: - return match.group(1) - return value - def to_mongo(self) -> Dict[str, Any]: - """ - Convert the model instance to a MongoDB-compatible dictionary. - """ - def model_to_dict(model: BaseModel) -> Dict[str, Any]: doc = {} for name in model.__fields__.keys(): @@ -100,31 +56,22 @@ class MongoSerializableModel(BaseModel): doc[key] = [model_to_dict(item) for item in value] # type: ignore elif value and isinstance(value, Enum): doc[key] = value.value - elif isinstance(value, datetime) or isinstance(value, date) or isinstance(value, time): + elif isinstance(value, (datetime, date, time)): doc[key] = value.isoformat() # type: ignore elif value and isinstance(value, AnyUrl): doc[key] = str(value) # type: ignore else: - doc[key] = self.serialize_s3_url(value) + doc[key] = value return doc - result = model_to_dict(self) - return result + return model_to_dict(self) class MongoBaseModel(MongoSerializableModel): - """ - Base model for MongoDB documents with serialization support. - """ - id: str = Field(default_factory=lambda: str(PyObjectId())) - class Config: # pylint: disable=R0903 - """ - Configuration for the model. - """ - + class Config: arbitrary_types_allowed = True extra = "ignore" populate_by_name = True @@ -132,10 +79,6 @@ class MongoBaseModel(MongoSerializableModel): @classmethod def from_mongo(cls, data: Dict[str, Any]): - """ - Create a model instance from MongoDB document data. - """ - def restore_enums(inst: Any, model_cls: Type[BaseModel]) -> None: for name, field in model_cls.__fields__.items(): # type: ignore value = getattr(inst, name) @@ -169,22 +112,3 @@ class MongoBaseModel(MongoSerializableModel): instance = cls(**data) restore_enums(instance, instance.__class__) return instance - - -class MongoBaseShortenModel(MongoSerializableModel): - """ - Base model for MongoDB documents with serialization support. - """ - - id: str - - @classmethod - def to_mongo_fields(self) -> dict: - result = {field: 1 for field in self.__annotations__ if field != "_id"} - result["_id"] = 0 - result["id"] = 1 - return result - - @classmethod - def from_mongo(cls, mongo_obj: Dict[str, Any]) -> T: - return cls(**mongo_obj) diff --git a/cbh/core/scheduling.py b/cbh/core/scheduling.py index cae71cc20c2a2e91eb51ffe106f5c79d7a2e0cf9..02900e8b2e9522916ac2d0981d42102933e971ca 100644 --- a/cbh/core/scheduling.py +++ b/cbh/core/scheduling.py @@ -1,81 +1,84 @@ -""" -Scheduled reference-data refreshes. - -TARIC publishes a full nomenclature on the first of the month and daily deltas -in between; the Dutch codebooks are republished whole whenever they change. The -jobs below follow that rhythm. - -Every job is safe to run when there is nothing to do: an unchanged file is -detected by its content hash and skipped, and imports are idempotent, so a -missed or repeated run costs nothing but a log entry. -""" - from cbh.api.importer.dto import SyncMode from cbh.core.config import settings from cbh.core.wrappers import background_task @background_task() -async def refresh_taric_delta() -> None: - """ - Apply the daily TARIC deltas available locally. +async def refresh_taric_publication() -> None: + """Pull the published extraction and load it if it is not the one we hold. + + Run daily although the extraction is monthly: the publication date moves + within the first days of the month, and a run against a pack already loaded + costs one download and writes nothing. """ - from cbh.api.importer.services.taric import import_all_local + from cbh.api.importer.services.taric import refresh_from_publication - await import_all_local(SyncMode.DELTA) + await refresh_from_publication(SyncMode.FULL) @background_task() -async def refresh_taric_full() -> None: - """ - Load the monthly full TARIC extract. - """ +async def refresh_taric_local() -> None: from cbh.api.importer.services.taric import import_all_local await import_all_local(SyncMode.FULL) @background_task() -async def refresh_codebooks() -> None: - """ - Replace the Dutch national codebooks with today's publication. +async def refresh_dtv_national() -> None: + """Reload the Dutch national measures when a new snapshot is published. + + The Belastingdienst puts out a full snapshot weekly and increments nightly. + Only the snapshot is read, and only when its digest differs from the one + already loaded, so most runs cost a download and no writes. """ + from cbh.api.importer.services.dtv import refresh_national_measures + + await refresh_national_measures(SyncMode.FULL) + + +@background_task() +async def refresh_codebooks() -> None: from cbh.api.importer.services.codeboek import sync_all_codebooks await sync_all_codebooks() def schedule_reference_data_jobs() -> None: - """ - Register the reference-data jobs on the application scheduler. - - Jobs carry stable ids and replace an existing registration, so a reload - cannot end up running the same import twice. - """ scheduler = settings.SCHEDULER scheduler.add_job( - refresh_taric_delta, + refresh_taric_publication, trigger="cron", - hour=5, + hour=4, minute=0, - id="taric-delta", + id="taric-publication", replace_existing=True, + misfire_grace_time=3600, + coalesce=True, ) scheduler.add_job( - refresh_taric_full, + refresh_taric_local, trigger="cron", day=1, - hour=4, - minute=0, - id="taric-full", + hour=5, + minute=30, + id="taric-local", replace_existing=True, + misfire_grace_time=3600, + coalesce=True, ) scheduler.add_job( - refresh_codebooks, + refresh_dtv_national, trigger="cron", - hour=6, + hour=5, minute=0, - id="codeboek-daily", + id="dtv-national", replace_existing=True, + misfire_grace_time=3600, + coalesce=True, ) + # The Codeboek Douane job is deliberately not scheduled. The address it was + # written against returns 404 for every date tried, and the national codes + # it was meant to supply now come from the Gebruikstarief, which is a + # published feed with a stated contract. `refresh_codebooks` stays callable + # by hand for the day a working address turns up. diff --git a/cbh/core/security.py b/cbh/core/security.py deleted file mode 100644 index c583bbc3b767b71b1eab4b09602a6c249c37c883..0000000000000000000000000000000000000000 --- a/cbh/core/security.py +++ /dev/null @@ -1,175 +0,0 @@ -""" -Security utilities for ClipboardHealthAI application. - -This module provides authentication and authorization functionality, including: -- Password verification -- JWT token creation and validation -- Permission-based endpoint protection using FastAPI dependencies -""" - -from datetime import datetime, timedelta - -import anyio -from fastapi import Depends, HTTPException -from fastapi.security import HTTPAuthorizationCredentials, HTTPBearer -from jose import JWTError, jwt -from passlib.context import CryptContext - -from cbh.api.account.dto import AccountType -from cbh.api.account.models import AccountModel -from cbh.core.config import settings - - -def verify_password(plain_password, hashed_password) -> bool: - """ - Verify a password against its hashed version. - - Args: - plain_password: The plain text password to verify - hashed_password: The hashed password to check against - - Returns: - bool: True if the password matches, False otherwise - """ - result = CryptContext(schemes=["bcrypt"], deprecated="auto").verify( - plain_password, hashed_password - ) - return result - - -def create_access_token(email: str, account_id: str, account_type: AccountType): - """ - Create a JWT access token for a user. - - Args: - email: User's email address - account_id: User's account ID - account_type: User's account type - - Returns: - str: Encoded JWT token - """ - payload = { - "http://schemas.xmlsoap.org/ws/2005/05/identity/claims/name": email, - "http://schemas.xmlsoap.org/ws/2005/05/identity/claims/nameidentifier": account_id, - "accountId": account_id, - "accountType": account_type.value, - "iss": settings.Issuer, - "aud": settings.Audience, - "exp": datetime.utcnow() + timedelta(days=30), - } - encoded_jwt = jwt.encode(payload, settings.SECRET_KEY, algorithm="HS256") - return encoded_jwt - - -class PermissionDependency: - """ - FastAPI dependency for protecting endpoints with authentication. - - This class implements the callable interface required for FastAPI dependencies - and validates JWT tokens for protected endpoints. - """ - - def __init__(self, account_type: list[AccountType] | None = None, required: bool = True): - self.account_types = account_type - self.required = required - - def __call__( - self, - credentials: HTTPAuthorizationCredentials | None = Depends(HTTPBearer(auto_error=False)), - ) -> AccountModel | None: - """ - Validate authorization credentials and return account details. - - This method is called by FastAPI when the dependency is used. - - Args: - credentials: The HTTP authorization credentials from the request - - Returns: - AccountModel: The account details if authentication is successful - - Raises: - HTTPException: If authentication fails - """ - try: - if not credentials and self.required: - raise HTTPException(status_code=401, detail="Unauthorized") - elif not credentials and not self.required: - return None - account_id = self.authenticate_jwt_token(credentials.credentials) - account_data = anyio.from_thread.run(self.get_account_by_id, account_id) - self.check_account_health(account_data) - return AccountModel.from_mongo(account_data) - - except JWTError as e: - raise HTTPException( # pylint: disable=W0707 - status_code=403, detail="Permission denied" - ) - except Exception as e: - if isinstance(e, HTTPException) and e.status_code == 401: - raise e - raise HTTPException( # pylint: disable=W0707 - status_code=403, detail="Permission denied" - ) - - @staticmethod - async def get_account_by_id(account_id: str) -> dict: - """ - Retrieve account data from the database by ID. - - Args: - account_id: The account ID to look up - - Returns: - dict: Account data from the database - """ - account = await settings.DB_CLIENT.accounts.find_one({"id": account_id}) - if not account: - raise HTTPException(status_code=403, detail="Permission denied") - return account - - def check_account_health(self, account: dict): - """ - Verify account data is valid and active. - - Args: - account: Account data dictionary - - Raises: - HTTPException: If the account is not valid - """ - if not account: - raise HTTPException(status_code=403, detail="Permission denied") - # No account types given means any authenticated account is accepted. - if self.account_types is None: - return - if AccountType(account["accountType"]) not in self.account_types: - raise HTTPException(status_code=403, detail="Permission denied") - - @staticmethod - def authenticate_jwt_token(token: str) -> str: - """ - Validate a JWT token and extract the account ID. - - Args: - token: JWT token string - - Returns: - str: Account ID from the token - - Raises: - HTTPException: If token validation fails - """ - payload = jwt.decode( - token, settings.SECRET_KEY, algorithms="HS256", audience=settings.Audience - ) - email: str | None = payload.get( - "http://schemas.xmlsoap.org/ws/2005/05/identity/claims/name" - ) - account_id: str | None = payload.get("accountId") - - if email is None or account_id is None: - raise HTTPException(status_code=403, detail="Permission denied") - - return account_id diff --git a/cbh/core/wrappers.py b/cbh/core/wrappers.py index 910d19fbb89e60361c87c8e13e0a66a182552306..f531d0f00395e21cb8b2f0c71e939bddac0186a4 100644 --- a/cbh/core/wrappers.py +++ b/cbh/core/wrappers.py @@ -1,66 +1,22 @@ -""" -Response wrappers and utility decorators for ClipboardHealthAI application. - -This module provides: -- Standardized response wrappers for API endpoints -- Exception handling decorators -- OpenAI API request wrapper -- Background task decorator -- Retry decorator for transient failures -""" - -import asyncio -from collections.abc import Callable from functools import wraps -from typing import Generic, Optional, ParamSpec, TypeVar +from typing import Generic, Optional, TypeVar -from fastapi import HTTPException from pydantic import BaseModel from starlette.responses import JSONResponse T = TypeVar("T") -P = ParamSpec("P") -R = TypeVar("R") class ErrorCbhResponse(BaseModel): - """ - Error response model for standardized error formatting. - - Attributes: - message: Error message describing what went wrong - """ - message: str class CbhResponseWrapper(BaseModel, Generic[T]): - """ - Standard response wrapper for all API endpoints. - - This class provides a consistent structure for all API responses, - including data, success status, and error information. - - Attributes: - data: The response data (optional) - successful: Whether the request was successful - error: Error details if the request failed - """ - data: Optional[T] = None successful: bool = True error: Optional[ErrorCbhResponse] = None def response(self, status_code: int): - """ - Create a JSONResponse with proper status code and formatting. - - Args: - status_code: HTTP status code for the response - - Returns: - JSONResponse: Formatted API response - """ return JSONResponse( status_code=status_code, content={ @@ -71,76 +27,15 @@ class CbhResponseWrapper(BaseModel, Generic[T]): ) -def exception_wrapper(http_error: int, error_message: str): - """ - Decorator for handling exceptions in route handlers. - - Catches any exceptions and converts them to a proper HTTP exception - with specified status code and error message. - - Args: - http_error: HTTP status code to use for the exception - error_message: Error message to include - - Returns: - Decorator function that wraps route handlers - """ - - def decorator(func): - @wraps(func) - async def wrapper(*args, **kwargs): - try: - return await func(*args, **kwargs) - except Exception as e: - raise HTTPException(status_code=http_error, detail=error_message) from e - - return wrapper - - return decorator - - def background_task(): - """ - Decorator for background tasks that should not crash the application. - - Wraps a function to catch and suppress any exceptions, preventing - background task failures from affecting the main application. - - Returns: - Decorator function for background tasks - """ - def decorator(func): @wraps(func) async def wrapper(*args, **kwargs) -> str | None: try: - result = await func(*args, **kwargs) - return result - except Exception: # pylint: disable=W0718 + return await func(*args, **kwargs) + except Exception: return None return wrapper return decorator - - -def retry( - exceptions: tuple[type[Exception], ...], - max_retries: int = 3, - delay: float = 0, -) -> Callable[[Callable[P, R]], Callable[P, R]]: - def decorator(func: Callable[P, R]) -> Callable[P, R]: - @wraps(func) - async def wrapper(*args: P.args, **kwargs: P.kwargs) -> R: - for attempt in range(max_retries + 1): - try: - return await func(*args, **kwargs) - except exceptions: - if attempt == max_retries: - raise - if delay: - await asyncio.sleep(delay) - - return wrapper - - return decorator diff --git a/main.py b/main.py index 190cb09cfcc5f3f60ed2f8463035b5b18fd72db4..65d03b63e84fb05eebc92cfa74a7709d0f6abbc7 100644 --- a/main.py +++ b/main.py @@ -1,10 +1,3 @@ -""" -FastAPI application entry point for ClipboardHealthAI. - -This file creates and launches the FastAPI application by importing the create_app -function from the cbh package. -""" - from cbh import create_app app = create_app() diff --git a/requirements.txt b/requirements.txt index aed5333e4e3c6b331c2f3177d330559b6b08cadb..6e798db42e04400a1d98765d7302b57a40e8c330 100644 --- a/requirements.txt +++ b/requirements.txt @@ -83,3 +83,4 @@ websockets==15.0.1 xxhash==3.6.0 yarl==1.22.0 zstandard==0.25.0 +xlrd==2.0.2 diff --git a/samples/README.md b/samples/README.md new file mode 100644 index 0000000000000000000000000000000000000000..f291bb909250b657c9d7b4fd2c98fd87d9abf0b7 --- /dev/null +++ b/samples/README.md @@ -0,0 +1,80 @@ +# Sample packs + +Three files for trying the engine by hand. Everything in them is invented — +weights, values, tracking numbers, EANs — except the goods codes, which are real +and resolve against the loaded TARIC extraction. + +They use the layout of Goenee's own export line list: sheet `Loads`, headers on +row 1, and the same eleven columns, including the `Total Item Weight` header +that appears twice (the report writer emits gross and net under one name; the +importer takes the first). + +Upload with the **Goenee export line list** profile. It supplies the direction +(export) and the destination (GB), so nothing else needs filling in. + +| file | rows | what it is for | +| --- | --- | --- | +| `sample-goenee-export-apparel.xlsx` | 33 | the normal case, chapters 61–65 | +| `sample-goenee-export-electronics.xlsx` | 7 | chapters 84, 85 and one wine line | +| `sample-goenee-export-problems.xlsx` | 8 | codes the engine must refuse | + +## sample-goenee-export-apparel.xlsx + +The `Y-code` column is already filled with the answer a declarant would write, +taken from the real Goenee pack. So this file is both an input and a gold set: +upload it, then post the same file to `POST /api/batch/{id}/goldset` and the +score comes back **33/33 matched, code accuracy 100.0, exemption accuracy +100.0**. + +What to look at in the review table: + +- **16 rows read `Y922`** — the export control on cat and dog fur. It is not a + ban: writing Y922 in Box 44 satisfies it. Green. +- **8 rows read "No document required"** — green, and it means it. Headings 6105, + 6107, 6109, 6115, 6205, 6206, 6212 and 6402 99 39 carry no export measure that + demands anything in Box 44. +- **5 rows are amber** (6201 30 90, 6201 40 10, 6202 40 10, 6211 33 90, + 6211 43 90). Dual-use sits on top of the fur control and the obligation is + either/or: one of eleven licences `X060…X071`. The engine lists them and does + not choose. The exemptions `Y901,Y922` are in the row detail and are what goes + into the export. +- **4 rows show `C400` as required** with `Y900` (and on 6505 00 30 also `Y922`) + as the exemption: a CITES permit, or the declaration that the goods are not + covered. The review table labels the row by the obligation; the Y code that + actually gets filed is in the detail and in the exported column. + +Export it as **Softpak** and the result is the pack layout — five metadata rows, +a blank row, headers on row 7, a TOTAL row — with the `Y-code` column holding +`Y922` ×16, empty ×8, `Y901,Y922` ×5, `Y900` ×3, `Y900,Y922` ×1. + +## sample-goenee-export-electronics.xlsx + +No human answer exists for these, so the `Y-code` column is empty. Do not score +this one against itself. + +Two televisions and one wine line come back clean. The four IT lines +(8471 41 00, 8471 30 00, 8517 13 00, 8525 89 00) are amber for the same dual-use +reason as the apparel jackets. 8525 89 00 additionally carries `Y227`–`Y232` +alongside `Y901`. + +Documents score is 42.9% here, and that is the honest number: four of seven +lines need a person to pick a licence. + +## sample-goenee-export-problems.xlsx + +This is the file for checking that the engine refuses rather than guesses. Eight +lines, five of which come back INVALID and one AMBIGUOUS: + +| input | what happens | +| --- | --- | +| `620342` | **AMBIGUOUS** — seven declarable codes sit under it, none is chosen | +| `99999999` | **INVALID** — no such code | +| `62021390` | **INVALID** — looks right, nothing declarable beneath it | +| `6203 42 31` | **OK** — spacing is tolerated, resolves and returns `Y922` | +| `09011100` | **INVALID** — chapter 09 is NVWA-controlled and out of this release | +| `0201 10 00` | **INVALID** — chapter 02, same reason, although measures were found | +| `65050030` | **OK with a warning** — chapter 65 is outside the benchmarked set, so the answer is computed but not vouched for | +| *(empty)* | **INVALID** — no code supplied | + +Extraction score 50.0%, documents 37.5%. That is the point of the file: a pack +this bad must not produce a confident-looking result. diff --git a/samples/generate.py b/samples/generate.py new file mode 100644 index 0000000000000000000000000000000000000000..aec907778216994531f1fbeaf63a5d6fffa13a2c --- /dev/null +++ b/samples/generate.py @@ -0,0 +1,135 @@ +"""Build the sample packs under samples/. + +The layout is the one Goenee's own line list uses — headers on row 1, sheet +"Loads", and the duplicated `Total Item Weight` column the report writer emits. +Every figure in here is invented; only the goods codes are real. +""" + +from pathlib import Path + +from openpyxl import Workbook + +OUT = Path(__file__).parent + +HEADERS = [ + "HS Code", + "Item Descr", + "Total Item Weight", + "Total Item Weight", + "Item COO", + "Pieces", + "Colli type", + "Tracking no", + "Remarks (EAN", + "Total Value (GBP)", + "Y-code", +] + +# code, description, origin, weight, value, the answer a declarant wrote by hand +APPAREL = [ + ("62034231", "Men Jeans", "BD", 0.67, 54.20, "Y922"), + ("62034231", "Men Jeans", "VN", 0.66, 81.30, "Y922"), + ("62034235", "Men O 5 pockets", "BD", 0.46, 28.79, "Y922"), + ("62034290", "Men Shorts", "BD", 0.38, 35.50, "Y922"), + ("62034319", "Men O 5 pockets", "VN", 0.52, 47.90, "Y922"), + ("62046231", "Wmn Jeans", "BD", 0.55, 62.40, "Y922"), + ("62046239", "Wmn O 5 pockets", "TR", 0.49, 55.10, "Y922"), + ("62033290", "Men Blazers", "TR", 0.81, 96.75, "Y922"), + ("61101190", "Wmn Knits", "CN", 0.44, 41.20, "Y922"), + ("61102091", "Men Sweats", "BD", 0.62, 49.99, "Y922"), + ("61102099", "Wmn Sweats", "BD", 0.60, 49.99, "Y922"), + ("61103091", "Men Knits", "VN", 0.58, 58.30, "Y922"), + ("61142000", "Wmn Tops", "IN", 0.21, 22.45, "Y922"), + ("62113290", "Men Jackets", "CN", 0.94, 110.00, "Y922"), + ("62114290", "Wmn Jackets", "CN", 0.88, 104.50, "Y922"), + # Dual-use sits on top of the fur control: two exemptions on one line. + ("62013090", "Men Jackets", "CN", 0.91, 129.00, "Y901,Y922"), + ("62014010", "Men Jackets", "VN", 0.95, 135.00, "Y901,Y922"), + ("62024010", "Wmn Jackets", "CN", 0.86, 128.00, "Y901,Y922"), + ("62113390", "Men Jackets", "CN", 0.90, 119.00, "Y901,Y922"), + ("62114390", "Wmn Jumpsuit", "CN", 0.72, 98.60, "Y901,Y922"), + # Y900 — a different exemption again. + ("61059090", "Men Polo", "IN", 0.24, 26.40, "Y900"), + ("61099020", "Men T-Shirts", "IN", 0.23, 19.95, "Y900"), + ("64039996", "Men Boot", "PT", 1.35, 145.00, "Y900"), + # Nothing at all goes in Box 44 for these. + ("61091000", "Men T-Shirts", "BD", 0.25, 18.96, ""), + ("61051000", "Men Polo", "IN", 0.27, 29.95, ""), + ("61071100", "Men Underwear", "BD", 0.12, 14.95, ""), + ("61159500", "Wmn Accessories", "CN", 0.09, 9.95, ""), + ("62052000", "Men Overshirts", "TR", 0.41, 52.00, ""), + ("62063000", "Wmn Shirts", "TR", 0.37, 48.50, ""), + ("62121090", "Wmn Underwear", "CN", 0.08, 24.95, ""), + ("64029190", "Men Sneaker", "VN", 1.10, 99.95, "Y922"), + ("64029939", "Men Sandal", "VN", 0.65, 44.95, ""), + # A required document, not just an exemption: CITES C400 plus two Y codes. + ("65050030", "Men Accessories", "CN", 0.14, 19.95, "Y900,Y922"), +] + +ELECTRONICS = [ + ("85287200", "LED television 55 inch", "CN", 14.80, 420.00), + ("85287200", "LED television 43 inch", "CN", 9.60, 289.00), + ("84714100", "Desktop computer", "CN", 7.40, 610.00), + ("84713000", "Laptop 14 inch", "CN", 1.35, 780.00), + ("85171300", "Smartphone", "CN", 0.21, 540.00), + ("85258900", "Digital camera", "JP", 0.68, 690.00), + ("22042189", "Red wine, bottled", "FR", 8.40, 96.00), +] + +PROBLEMS = [ + ("620342", "Men Jeans, six digits only", "BD", 0.67, 54.20), + ("99999999", "Code that does not exist", "BD", 0.40, 30.00), + ("62021390", "Plausible but not declarable", "CN", 0.80, 88.00), + ("6203 42 31", "Same code, spaced as on paper", "BD", 0.67, 54.20), + ("09011100", "Coffee, NVWA chapter", "BR", 5.00, 60.00), + ("0201 10 00", "Beef carcase, NVWA chapter", "AR", 12.50, 140.00), + ("65050030", "Fur hat, chapter outside the release", "CN", 0.14, 19.95), + ("", "No code at all", "CN", 0.30, 25.00), +] + + +def tracking(index: int) -> str: + return f"JJD00090000000{index:05d}" + + +def ean(index: int) -> str: + return f"87200000{index:05d}" + + +def write(path: Path, rows: list[list]) -> None: + workbook = Workbook() + worksheet = workbook.active + worksheet.title = "Loads" + worksheet.append(HEADERS) + for row in rows: + worksheet.append(row) + for column, width in zip("ABCDEFGHIJK", (12, 30, 18, 18, 10, 8, 11, 22, 16, 18, 12)): + worksheet.column_dimensions[column].width = width + workbook.save(path) + print(f"{path.name:46} {len(rows):>3} rows") + + +def main() -> None: + OUT.mkdir(parents=True, exist_ok=True) + + apparel = [ + [code, descr, weight, weight, coo, 1, "Parcel", tracking(i), ean(i), value, ycode] + for i, (code, descr, coo, weight, value, ycode) in enumerate(APPAREL, start=1) + ] + write(OUT / "sample-goenee-export-apparel.xlsx", apparel) + + electronics = [ + [code, descr, weight, weight, coo, 1, "Carton", tracking(200 + i), ean(200 + i), value, ""] + for i, (code, descr, coo, weight, value) in enumerate(ELECTRONICS, start=1) + ] + write(OUT / "sample-goenee-export-electronics.xlsx", electronics) + + problems = [ + [code, descr, weight, weight, coo, 1, "Parcel", tracking(400 + i), ean(400 + i), value, ""] + for i, (code, descr, coo, weight, value) in enumerate(PROBLEMS, start=1) + ] + write(OUT / "sample-goenee-export-problems.xlsx", problems) + + +if __name__ == "__main__": + main() diff --git a/samples/sample-goenee-export-apparel.xlsx b/samples/sample-goenee-export-apparel.xlsx new file mode 100644 index 0000000000000000000000000000000000000000..bc54cff89baeb0695eef1f0a25c2665800204f5b Binary files /dev/null and b/samples/sample-goenee-export-apparel.xlsx differ diff --git a/samples/sample-goenee-export-electronics.xlsx b/samples/sample-goenee-export-electronics.xlsx new file mode 100644 index 0000000000000000000000000000000000000000..603c4d3cd16ebc09dc9d8b6a8b7cd1677cf69c97 Binary files /dev/null and b/samples/sample-goenee-export-electronics.xlsx differ diff --git a/samples/sample-goenee-export-problems.xlsx b/samples/sample-goenee-export-problems.xlsx new file mode 100644 index 0000000000000000000000000000000000000000..be8ffac430e2d177773c7d1a303816690f39392f Binary files /dev/null and b/samples/sample-goenee-export-problems.xlsx differ