Buckets:
AnonymousSpottedToad/candidate_bucket / codebase /python /RS-PYTHON_rs-server /resources /sync_config_files.py
| # Copyright 2023-2026 Airbus, CS Group | |
| # | |
| # Licensed under the Apache License, Version 2.0 (the "License"); | |
| # you may not use this file except in compliance with the License. | |
| # You may obtain a copy of the License at | |
| # | |
| # http://www.apache.org/licenses/LICENSE-2.0 | |
| # | |
| # Unless required by applicable law or agreed to in writing, software | |
| # distributed under the License is distributed on an "AS IS" BASIS, | |
| # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | |
| # See the License for the specific language governing permissions and | |
| # limitations under the License. | |
| """ | |
| Create rs-server configuration files from templates. | |
| Copy them to rs-demo, rs-helm and rs-server-deployment repositories. | |
| """ | |
| import collections.abc | |
| import copy | |
| import json | |
| import logging | |
| import numbers | |
| import os | |
| import re | |
| import shutil | |
| from collections.abc import Callable, Iterable | |
| from dataclasses import dataclass | |
| from pathlib import Path | |
| from typing import Any | |
| import yaml | |
| from yaml.representer import SafeRepresenter | |
| # Avoid yaml references, see: https://stackoverflow.com/a/30682604 | |
| yaml.Dumper.ignore_aliases = lambda *_: True # type: ignore | |
| logger = logging.getLogger(__name__) | |
| # Supported stations: list of CADIP, AUXIP and PRIP stations | |
| # for which we want to include the config file in the helm chart | |
| SUPPORTED_CADIP_STATIONS = ["cadip", "sgs", "mti"] | |
| SUPPORTED_ADGS_STATIONS = ["adgs", "adgs2"] | |
| SUPPORTED_PRIP_STATIONS = ["s1a", "s2b"] | |
| # | |
| # Class definition | |
| class Stations: # pylint: disable=too-few-public-methods | |
| """Check a station name""" | |
| def __init__(self): | |
| self.value: str = "" | |
| self.adgs: bool = False | |
| self.cadip: bool = False | |
| self.lta: bool = False | |
| self.prip: bool = False | |
| def update(self, station: str): | |
| """Update the station name from a string.""" | |
| if any(station.startswith(s) for s in ("adgs", "auxip")): | |
| self.adgs = True | |
| self.value = station | |
| elif any(station.startswith(s) for s in ("cadip", "ins", "mps", "mti", "nsg", "sgs")): | |
| self.cadip = True | |
| self.value = station | |
| elif any(station == s for s in ("lta",)): | |
| self.lta = True | |
| self.value = station | |
| elif any(station == s for s in ("s1a", "s2b")): | |
| self.prip = True | |
| self.value = station | |
| class HelmOrInfraParams: | |
| """ | |
| Parameters to copy a single configuration file to rs-helm or rs-server-deployment. | |
| Attributes: | |
| input_path_relative: input configuration file path, relative to the rs-server root dir | |
| input_root_tags: root yaml tags from where the input values must be read | |
| output_root_tags: root yaml tags where the output values must be written | |
| output_doc_index: output document index (start at 0) to write (only for yaml files with multiple documents) | |
| post_processing: function to execute on the output configuration under output_root_tags | |
| """ | |
| input_path_relative: str | |
| input_root_tags: list[str] | |
| output_root_tags: list[str] | |
| output_doc_index: int | |
| post_processing: Callable[[dict], None] | None = None | |
| prune_missing: bool = False | |
| # | |
| # Utility functions | |
| def recursive_update(old, new): | |
| """Recursive dict update, taken from: https://stackoverflow.com/a/3233356""" | |
| for key, value in new.items(): | |
| if isinstance(value, collections.abc.Mapping): | |
| old[key] = recursive_update(old.get(key) or {}, value) | |
| else: | |
| old[key] = value | |
| return old | |
| class LiteralStr(str): | |
| """ | |
| To print literal yaml strings with | | |
| See: https://stackoverflow.com/a/20863889 | |
| """ | |
| class QuotedStr(str): | |
| """To print quoted yaml strings with single quotes.""" | |
| def change_yaml_style(style, representer): | |
| """Goes with LiteralStr above""" | |
| def new_representer(dumper, data): | |
| scalar = representer(dumper, data) | |
| scalar.style = style | |
| return scalar | |
| return new_representer | |
| # | |
| # Init global variables | |
| represent_literal_str = change_yaml_style("|", SafeRepresenter.represent_str) | |
| yaml.add_representer(LiteralStr, represent_literal_str) | |
| represent_quoted_str = change_yaml_style("'", SafeRepresenter.represent_str) | |
| yaml.add_representer(QuotedStr, represent_quoted_str) | |
| # Replace local urls like http(s)://(127.0.0.1|localhost):5xxx | |
| REGEX_URL = re.compile(r"(https?://)(127.0.0.1|localhost):5\d+") | |
| REGEX_DOMAIN = re.compile(r"mockup-.+.processing.svc.cluster.local") | |
| # Replace k8s values | |
| REGEX_RANGE_START = r"({{-?\s*range\s.*-?}})" | |
| REGEX_RANGE_END = r"({{-?\s*end\s.*-?}})" | |
| DCB_OPEN = "DOUBLE_CURLY_BRACES_OPEN" | |
| DCB_CLOSE = "DOUBLE_CURLY_BRACES_CLOSE" | |
| # The config files will be copied in rs-demo, rs-helm and rs-server-deployment if these projects | |
| # are checkout under the same directory than rs-server. | |
| rs_server_dir = Path(__file__).parent.parent | |
| rs_demo_dir = rs_server_dir.parent / "rs-demo" | |
| rs_helm_dir = rs_server_dir.parent / "rs-helm" | |
| rs_deploy_dir = rs_server_dir.parent / "rs-server-deployment" | |
| if not rs_demo_dir.is_dir(): | |
| logger.warning(f"No 'rs-demo' repository found under: '{rs_demo_dir!s}'") | |
| if not rs_helm_dir.is_dir(): | |
| logger.warning(f"No 'rs-helm' repository found under: '{rs_helm_dir!s}'") | |
| if not rs_deploy_dir.is_dir(): | |
| logger.warning(f"No 'rs-server-deployment' repository found under: '{rs_deploy_dir!s}'") | |
| # Extract the copyright header from this current file. It will be added to yaml files modified from a template. | |
| COPYRIGHT_HEADER = "" | |
| with open(__file__, encoding="utf-8") as this_script: | |
| for line in this_script: | |
| line = line.strip() | |
| if not line: | |
| continue | |
| if line.startswith("#"): | |
| COPYRIGHT_HEADER += line + "\n" | |
| else: | |
| break | |
| # Save the template file paths that were used to create each final configuration file. | |
| # Key=final file, values=template files. | |
| TEMPLATE_PATHS: dict[Path, list[str]] = {} | |
| # | |
| # Implement main features | |
| def get_header(template_paths: list[str] | None = None, final_paths: Iterable[Path] | None = None): | |
| """ | |
| Return header for configuration file created from template files. | |
| Args: | |
| template_paths: template file paths, relative to the rs-server root dir. | |
| final_paths: final configuration file absolute paths | |
| """ | |
| if template_paths is None: | |
| template_paths = [] | |
| if final_paths is None: | |
| final_paths = [] | |
| # Get the template files used to create the final files | |
| for path in final_paths: | |
| if path in TEMPLATE_PATHS: | |
| template_paths += TEMPLATE_PATHS[path] | |
| else: | |
| logger.warning(f"Missing template mapping for: '{path!s}'") | |
| sep = "\n# - rs-server/" | |
| header_paths = sep + sep.join(sorted(template_paths) + [os.path.relpath(__file__, rs_server_dir)]) | |
| return ( | |
| COPYRIGHT_HEADER | |
| + f"\n# THIS FILE WAS AUTOMATICALLY CREATED FROM:{header_paths}\n# DON'T MODIFY IT DIRECTLY !\n\n" | |
| ) | |
| def create_from_template( | |
| template_paths: list[str], | |
| ): # pylint: disable=too-many-locals,too-many-branches,too-many-statements | |
| """ | |
| Create a configuration file from one or several template paths. | |
| Args: | |
| template_paths: template file paths, relative to the rs-server root dir. | |
| """ | |
| all_files: dict = {} | |
| output_paths: set[Path] = set() | |
| for relative_path in template_paths: | |
| template_path: Path = rs_server_dir / relative_path | |
| is_json = template_path.suffix.lower() == ".json" | |
| is_yaml = template_path.suffix.lower() in (".yml", ".yaml") | |
| if not (is_json or is_yaml): | |
| raise RuntimeError(f"Unexpected file extension: '{template_path.absolute()!s}'") | |
| # The output path is the same than the template file witout the ".template*" extension. | |
| template_ext = ".template" | |
| suffixes = template_path.suffixes | |
| for i, suffix in enumerate(suffixes): | |
| if template_ext in suffix: | |
| suffixes.pop(i) # remove .template* from the suffixes | |
| break | |
| if suffixes == template_path.suffixes: | |
| raise RuntimeError(f"'{template_ext}*' is missing from filename: '{template_path.absolute()!s}'") | |
| output_paths.add( | |
| template_path.parent | |
| / template_path.name.replace( | |
| "".join(template_path.suffixes), | |
| "".join(suffixes), | |
| ), | |
| ) | |
| if len(output_paths) > 1: | |
| raise RuntimeError(f"Incoherent output files: {output_paths}") | |
| # Read the yaml or json file | |
| with open(template_path, encoding="utf-8") as opened: | |
| if is_json: | |
| file = json.loads(opened.read()) | |
| else: | |
| file = yaml.safe_load(opened) | |
| # A template should defined at the top of the file and be applied to each file node | |
| template = file.pop("template") | |
| # If the file contains a single node, we guess that it is a root title. | |
| # We apply the template to its children. | |
| nodes = file | |
| if len(file) == 1: | |
| nodes = next(iter(file.values())) | |
| if isinstance(nodes, list): | |
| for i, node in enumerate(nodes): | |
| nodes[i] = recursive_update(copy.deepcopy(template), node) | |
| elif isinstance(nodes, dict): | |
| for key, node in nodes.items(): | |
| nodes[key] = recursive_update(copy.deepcopy(template), node) | |
| else: | |
| raise RuntimeError(f"Unrecognized data structure for: '{template_path.absolute()!s}'") | |
| all_files.update(file) | |
| # We should have a single output file | |
| if len(output_paths) != 1: | |
| raise RuntimeError(f"Not a single output file: {output_paths}") | |
| output_path: Path = output_paths.pop() | |
| logger.info(f"Update: '{output_path!s}'") | |
| # Save the template file paths that were used to create each final configuration file. | |
| TEMPLATE_PATHS[output_path] = template_paths | |
| # Specific post-processing for files that must keep literal block style | |
| def post_process(file_path: Path, content: dict): | |
| """Apply file-specific tweaks after template rendering.""" | |
| if file_path.name == "rs-server.yaml" and isinstance(content, dict): | |
| # Avoid inheriting OAuth2 fields for non-OAuth auth types in generated configs. | |
| sources = content.get("external_data_sources") | |
| if isinstance(sources, dict): | |
| for source in sources.values(): | |
| if not isinstance(source, dict): | |
| continue | |
| auth = source.get("authentication") | |
| if not isinstance(auth, dict): | |
| continue | |
| auth_type = auth.get("auth_type") | |
| if auth_type in ("s3", "ftp", "ftpes"): | |
| # Keep only relevant fields for these auth types. | |
| for key in ( | |
| "token_url", | |
| "grant_type", | |
| "scope", | |
| "client_id", | |
| "client_secret", | |
| "authorization", | |
| ): | |
| auth.pop(key, None) | |
| if auth_type == "s3": | |
| # S3 should not have username/password. | |
| auth.pop("username", None) | |
| auth.pop("password", None) | |
| post_process(output_path, all_files) | |
| # Write back the templated file | |
| with open(output_path, "w", encoding="utf-8") as opened: | |
| opened.write(get_header(template_paths)) | |
| if is_json: | |
| json.dump(all_files, opened, indent=2, sort_keys=False) | |
| else: | |
| yaml.dump(all_files, opened, default_flow_style=False, sort_keys=False) | |
| # | |
| # rs-demo | |
| def extract_structure(value: Any) -> Any: | |
| """Return a structure-only representation of the input value.""" | |
| if isinstance(value, dict): | |
| return {key: extract_structure(subvalue) for key, subvalue in value.items()} | |
| if isinstance(value, list): | |
| return [extract_structure(subvalue) for subvalue in value] | |
| return None | |
| def get_structure(file_contents: dict | None) -> Any: | |
| """Return structure-only representation, parsing embedded stations yaml if needed.""" | |
| if not isinstance(file_contents, dict): | |
| return None | |
| stations_value = file_contents.get("stations") | |
| if isinstance(stations_value, str): | |
| # Normalize placeholders so template vs. local values compare equal. | |
| stations_value = re.sub(r"\{[A-Z0-9_]+\}", "PLACEHOLDER", stations_value) | |
| try: | |
| stations_value = yaml.safe_load(stations_value) | |
| except yaml.YAMLError: | |
| # Keep as string if stations content is not valid YAML. | |
| stations_value = str(stations_value) | |
| file_contents = dict(file_contents) | |
| file_contents["stations"] = stations_value | |
| return extract_structure(file_contents) | |
| def update_demo_values(parent_key: str, config: dict, station: Stations | None = None): | |
| """ | |
| Recursive function to update values from the config file. | |
| Args: | |
| parent_key: parent yaml tag name | |
| config: current yaml block | |
| station: is the current yaml block implementing an adgs station, or cadip station, or ... | |
| """ | |
| if not isinstance(config, dict): | |
| raise RuntimeError(f"Invalid argument: {config}") | |
| if station is None: | |
| station = Stations() | |
| # Check station name from parent key | |
| station = copy.deepcopy(station) # save the instance so the previous recursive calls are not impacted | |
| station.update(parent_key) | |
| def update_single_value(value: str) -> str: | |
| """Return a single updated url value.""" | |
| if not isinstance(value, str): | |
| raise RuntimeError(f"Invalid argument: {value}") | |
| if station.adgs: | |
| return re.sub(REGEX_DOMAIN, "adgs-station", re.sub(REGEX_URL, r"\g<1>adgs-station:5000", value)) | |
| if station.cadip: | |
| return re.sub(REGEX_DOMAIN, "cadip-station", re.sub(REGEX_URL, r"\g<1>cadip-station:5000", value)) | |
| if station.lta: | |
| return re.sub(REGEX_DOMAIN, "lta-station", re.sub(REGEX_URL, r"\g<1>lta-station:5000", value)) | |
| if station.prip: | |
| return re.sub(REGEX_DOMAIN, "prip-station", re.sub(REGEX_URL, r"\g<1>prip-station:5000", value)) | |
| # No modification | |
| return value | |
| # Apply regex recursively | |
| for key, value in config.items(): | |
| # Recursive calls on dicts | |
| if isinstance(value, dict): | |
| update_demo_values(key, value, station) | |
| # Update string value | |
| elif isinstance(value, str): | |
| config[key] = update_single_value(value) | |
| # Recursive calls on lists... | |
| elif isinstance(value, list): | |
| for i, subvalue in enumerate(value): | |
| # ... on list dicts | |
| if isinstance(subvalue, dict): | |
| update_demo_values(key, subvalue, station) | |
| # ... or on list string values | |
| elif isinstance(subvalue, str): | |
| value[i] = update_single_value(subvalue) | |
| def copy_to_demo(input_path_relative: str): | |
| """ | |
| Copy and update a configuration file from rs-server to rs-demo. | |
| Args: | |
| input_path_relative: input configuration file path, relative to the rs-server root dir | |
| """ | |
| if not rs_demo_dir.is_dir(): | |
| return | |
| # Copy the file, keep the same name | |
| input_path = rs_server_dir / input_path_relative | |
| config_path = rs_demo_dir / "local-mode/config" / input_path.name | |
| logger.info(f"Update: '{config_path!s}'") | |
| shutil.copyfile(input_path, config_path) | |
| # Open the output file. | |
| # There are only yaml files for now | |
| if config_path.suffix.lower() not in (".yml", ".yaml"): | |
| raise RuntimeError(f"File type not supported: {config_path}") | |
| with open(config_path, encoding="utf-8") as opened: | |
| file = yaml.safe_load(opened) | |
| update_demo_values("", file) | |
| # Write the modified output file | |
| with open(config_path, "w", encoding="utf-8") as opened: | |
| opened.write(get_header(final_paths=[input_path])) | |
| yaml.dump(file, opened, default_flow_style=False, sort_keys=False) | |
| # | |
| # rs-helm and rs-server-deployment | |
| def copy_to_helm_or_infra(all_params: list[HelmOrInfraParams], output_path: Path, ignore_existing_file: bool = False): | |
| """ | |
| Copy and update a configuration file from rs-server to rs-helm or rs-server-deployment. | |
| Args: | |
| all_params: parameters to copy each configuration file | |
| output_path: output configuration absolute path | |
| ignore_existing_file: if there is already a file with the same name as the output path, ignore it | |
| and completely rewrite it. Use this option ONLY for files that contain only configuration (not for | |
| configmap for example) and that don't have helm templating in them. | |
| """ | |
| logger.info(f"Update: '{output_path!s}'") | |
| # The k8s configmap files contain strings that contain yaml contents. | |
| yaml_as_string = output_path.name == "configmap.yaml" | |
| # Open the output file if it exists. | |
| output_configs = None | |
| if output_path.is_file() and not ignore_existing_file: | |
| with open(output_path, encoding="utf-8") as opened: | |
| output_configs = read_helm_or_infra(opened.read(), yaml_as_string) | |
| # Else just start from empty dicts | |
| if not output_configs: | |
| output_configs = [{}] * len(all_params) | |
| # For each input configuration file | |
| for params in all_params: | |
| # Check that the document exists in the output file | |
| if params.output_doc_index > len(output_configs): | |
| raise RuntimeError(f"Document index #{params.output_doc_index} not found in: {output_path!r}") | |
| # Call the sub-function on a single doc | |
| try: | |
| copy_to_helm_or_infra_single_doc(params, output_configs[params.output_doc_index]) | |
| except Exception as e: # pylint: disable=broad-exception-caught | |
| logger.error("Failed to update single infra doc: %s", params, exc_info=e) | |
| # Get the file header from the list of input configuration files | |
| input_paths = {rs_server_dir / param.input_path_relative for param in all_params} | |
| header = get_header(final_paths=input_paths) | |
| # Write the modified output file into a string | |
| yaml_contents = write_helm_or_infra(output_configs, yaml_as_string) | |
| if output_path == rs_helm_dir / "charts/rs-server-station-secrets/values.yaml": | |
| yaml_contents = re.sub( | |
| r"(^\s+-----END (?:CERTIFICATE|RSA PRIVATE KEY)-----\n)(?!\n)", | |
| r"\1\n", | |
| yaml_contents, | |
| flags=re.MULTILINE, | |
| ) | |
| yaml_contents = re.sub(r"\n(?:[ \t]*\n)+\Z", "\n", yaml_contents) | |
| with open(output_path, "w", encoding="utf-8") as opened: | |
| opened.write(header) | |
| opened.write(yaml_contents) | |
| def read_helm_or_infra(yaml_contents: str, yaml_as_string: bool) -> list[dict]: | |
| """ | |
| Read Kubernetes configuration files that contain not yaml-compliant values and need special care. | |
| Args: | |
| yaml_contents: yaml contents as a string | |
| yaml_as_string: should configmap.yaml string values be converted to yaml dict ? | |
| Returns: | |
| list of yaml dicts = one per document | |
| """ | |
| if not isinstance(yaml_contents, str): | |
| raise RuntimeError(f"Invalid argument: {yaml_contents}") | |
| # Python yaml cannot open the {{- range ... }}' and '{{- end }}' tags | |
| # that do not contain a : at the end, so we add the : here | |
| yaml_contents = re.sub(re.compile(REGEX_RANGE_START), r"\g<1>:", yaml_contents) | |
| yaml_contents = re.sub(re.compile(REGEX_RANGE_END), r"\g<1>:", yaml_contents) | |
| # Python yaml cannot open files with k8s values like {{ some.thing }} | |
| # so we replace them with any other strings. | |
| yaml_contents = yaml_contents.replace("{{", DCB_OPEN) | |
| yaml_contents = yaml_contents.replace("}}", DCB_CLOSE) | |
| # Read the configuration file as a multidoc file (with docs separated by '---') | |
| output_configs = list(yaml.safe_load_all(yaml_contents)) or [] | |
| # Parse strings that contain yaml values | |
| if yaml_as_string: | |
| for output_config in output_configs: | |
| data = output_config.get("data", {}) | |
| for key, value in data.items(): | |
| if isinstance(value, str): | |
| data[key] = yaml.safe_load(value) | |
| return output_configs | |
| def write_helm_or_infra(output_configs: list[dict], yaml_as_string: bool) -> str: | |
| """ | |
| Write Kubernetes configuration files that contain not yaml-compliant values and need special care. | |
| Args: | |
| output_configs: list of yaml dicts = one per document | |
| yaml_as_string: should configmap.yaml string values be converted from yaml dict ? | |
| Returns: | |
| String contents. | |
| """ | |
| if not isinstance(output_configs, list): | |
| raise RuntimeError(f"Invalid argument: {output_configs}") | |
| # Use a large width so we don't split long lines | |
| witdh = 1000 | |
| # Unparse yaml contents into literal strings (indented with |) | |
| if yaml_as_string: | |
| def format_config_value(config_value: Any) -> Any: | |
| """Format configmap values that embed yaml.""" | |
| if isinstance(config_value, dict): | |
| stations_value = config_value.get("stations") | |
| if isinstance(stations_value, str) and "\n" in stations_value: | |
| if not stations_value.endswith("\n"): | |
| stations_value += "\n" | |
| config_value["stations"] = LiteralStr(stations_value) | |
| return LiteralStr(yaml.dump(config_value, default_flow_style=False, sort_keys=False, width=witdh)) | |
| if isinstance(config_value, str) and "\n" in config_value: | |
| return LiteralStr(config_value if config_value.endswith("\n") else config_value + "\n") | |
| return config_value | |
| for output_config in output_configs: | |
| data = output_config.get("data", {}) | |
| for key, value in data.items(): | |
| data[key] = format_config_value(value) | |
| # Write the configuration file as a multidoc file (with docs separated by '---') | |
| yaml_contents = yaml.dump_all(output_configs, default_flow_style=False, sort_keys=False, width=witdh) | |
| # Restore the double curly braces | |
| yaml_contents = yaml_contents.replace(DCB_OPEN, "{{") | |
| yaml_contents = yaml_contents.replace(DCB_CLOSE, "}}") | |
| # Remove the added : after the k8s values | |
| suffix = r":(\s*null)?" # yaml parsing added ': null' after the tag | |
| yaml_contents = re.sub(re.compile(REGEX_RANGE_START + suffix), r"\g<1>", yaml_contents) | |
| yaml_contents = re.sub(re.compile(REGEX_RANGE_END + suffix), r"\g<1>", yaml_contents) | |
| return yaml_contents | |
| def copy_to_helm_or_infra_single_doc( # pylint: disable=too-many-statements | |
| params: HelmOrInfraParams, | |
| output_config: dict, | |
| ): | |
| """ | |
| Copy and update a single yaml document from rs-server to rs-helm or rs-server-deployment. | |
| Args: | |
| params (HelmOrInfraParams): parameters to copy a single configuration file | |
| output_config: current output yaml block | |
| """ | |
| # There are only yaml files for now | |
| input_path = rs_server_dir / params.input_path_relative | |
| if input_path.suffix.lower() not in (".yml", ".yaml"): | |
| raise RuntimeError(f"File type not supported: {input_path}") | |
| # Open the input file | |
| with open(input_path, encoding="utf-8") as opened: | |
| input_config = yaml.safe_load(opened) | |
| # Start from the input and output root tag(s) | |
| for tag in params.input_root_tags: | |
| input_config = input_config[tag] # input tags must exist | |
| for tag in params.output_root_tags: | |
| if tag not in output_config: # output tags are optional | |
| output_config[tag] = {} | |
| logger.warning(f"Tag does not exist in output file: {tag!r}") | |
| output_config = output_config[tag] | |
| def update_all_values( # pylint: disable=too-many-branches | |
| parent_keys: list[str], | |
| input_config: dict, | |
| output_config: dict, | |
| station: Stations = Stations(), | |
| ): | |
| """ | |
| Recursive function to update values from the config file. | |
| Args: | |
| parent_keys: parent yaml tag names | |
| input_config: current input yaml block | |
| output_config: current output yaml block | |
| station: is the current yaml block implementing an adgs station, or cadip station, or ... | |
| """ | |
| if not isinstance(input_config, dict): | |
| raise RuntimeError(f"Invalid input_config: {input_config}") | |
| if not isinstance(output_config, dict): | |
| raise RuntimeError(f"Invalid output_config: {output_config}") | |
| # Check station name from parent key | |
| station = copy.deepcopy(station) # save the instance so the previous recursive calls are not impacted | |
| station.update(parent_keys[-1] if parent_keys else "") | |
| def update_single_value(input_value: Any, output_value: str) -> str: | |
| """Return a single updated value.""" | |
| if not isinstance(output_value, str): | |
| raise RuntimeError(f"Invalid argument: {output_value}") | |
| # If the output value is like {{ some.thing }}, it's a k8s value, we don't change it. | |
| if (DCB_OPEN in output_value) and (DCB_CLOSE in output_value): | |
| return output_value | |
| # Else try to update url in the input value | |
| updated_value = input_value | |
| if station.adgs: | |
| updated_value = re.sub( | |
| REGEX_URL, | |
| rf"\g<1>mockup-station-{station.value}.processing.svc.cluster.local:8080", | |
| input_value, | |
| ) | |
| elif station.cadip: | |
| updated_value = re.sub( | |
| REGEX_URL, | |
| rf"\g<1>mockup-station-cadip-{station.value}.processing.svc.cluster.local:8080", | |
| input_value, | |
| ) | |
| elif station.lta: | |
| updated_value = re.sub( | |
| REGEX_URL, | |
| rf"\g<1>mockup-lta-{station.value}.processing.svc.cluster.local:8080", | |
| input_value, | |
| ) | |
| elif station.prip: | |
| updated_value = re.sub( | |
| REGEX_URL, | |
| rf"\g<1>mockup-prip-{station.value}.processing.svc.cluster.local:8080", | |
| input_value, | |
| ) | |
| return updated_value | |
| for key in output_config.keys(): | |
| if key not in input_config.keys(): | |
| logger.warning(f"Missing from rs-server: {'.'.join(parent_keys + [key])!r}") | |
| # Copy values recursively from input to output | |
| for key, input_value in input_config.items(): | |
| if key not in output_config: | |
| output_config[key] = input_value | |
| # Recursive calls on dicts | |
| output_value = output_config[key] | |
| if isinstance(output_value, dict): | |
| if not isinstance(input_value, dict): | |
| output_config[key] = input_value | |
| continue | |
| update_all_values(parent_keys + [key], input_value, output_value, station) | |
| # Update string value | |
| elif isinstance(output_value, str): | |
| if isinstance(input_value, str): | |
| output_config[key] = update_single_value(input_value, output_value) | |
| else: | |
| output_config[key] = input_value | |
| # Update number values | |
| elif isinstance(output_value, numbers.Number): | |
| if not isinstance(input_value, numbers.Number): | |
| raise RuntimeError(f"Invalid argument: {input_value}") | |
| output_config[key] = input_value | |
| # Recursive calls on lists... | |
| elif isinstance(output_value, list): | |
| # If the input value is not a list or does not have the same length, | |
| # I don't really know what to do, so just copy the input value. | |
| if (not isinstance(input_value, list)) or (len(input_value) != len(output_value)): | |
| output_config[key] = input_value | |
| # Else, do a recursive call on each lists element ... | |
| else: | |
| for i, output_subvalue in enumerate(output_value): | |
| input_subvalue = input_value[i] | |
| # ... on list dicts | |
| if isinstance(output_subvalue, dict): | |
| update_all_values(parent_keys + [key], input_subvalue, output_subvalue, station) | |
| # ... or on list string values | |
| elif isinstance(output_subvalue, str): | |
| output_value[i] = update_single_value(input_subvalue, output_subvalue) | |
| # ... or on list number values | |
| elif isinstance(output_subvalue, numbers.Number): | |
| output_value[i] = input_subvalue | |
| def prune_missing_keys(input_value: Any, output_value: Any): | |
| """Remove output keys that are not present in the input structure.""" | |
| if isinstance(input_value, dict) and isinstance(output_value, dict): | |
| for key in list(output_value.keys()): | |
| if key not in input_value: | |
| output_value.pop(key) | |
| continue | |
| prune_missing_keys(input_value[key], output_value[key]) | |
| elif isinstance(input_value, list) and isinstance(output_value, list) and len(input_value) == len(output_value): | |
| for input_subvalue, output_subvalue in zip(input_value, output_value): | |
| prune_missing_keys(input_subvalue, output_subvalue) | |
| if isinstance(input_config, dict) and isinstance(output_config, dict): | |
| update_all_values([], input_config, output_config) | |
| if params.prune_missing: | |
| prune_missing_keys(input_config, output_config) | |
| else: | |
| logger.error(f"Cannot update values for {input_config} => {output_config}") | |
| # Call post-processing on output configuration | |
| if params.post_processing: | |
| params.post_processing(output_config) | |
| if __name__ == "__main__": | |
| # Create configuration files from templates. Paths are given from the rs-server root dir. | |
| for templates in ( | |
| ["services/common/config/rs-server.template.yaml"], | |
| ["services/adgs/config/adgs_search_config.template.yaml"], | |
| ["services/adgs/config/adgs_ws_config.template.yaml"], | |
| ["services/cadip/config/cadip_search_config.template.yaml"], | |
| [ | |
| "services/cadip/config/cadip_ws_config.template.yaml", | |
| "services/cadip/config/cadip_ws_config.template_session.yaml", | |
| ], | |
| ["services/prip/config/prip_search_config.template.yaml"], | |
| ["services/prip/config/prip_ws_config.template.yaml"], | |
| ): | |
| create_from_template(templates) | |
| # Copy resulting files to rs-demo | |
| for config_path_relative in ( | |
| "services/common/config/rs-server.yaml", | |
| "services/adgs/config/adgs_ws_config.yaml", | |
| "services/cadip/config/cadip_ws_config.yaml", | |
| "services/prip/config/prip_ws_config.yaml", | |
| ): | |
| copy_to_demo(config_path_relative) | |
| # | |
| # Copy resulting files to rs-helm and rs-server-deployment | |
| def remove_session_stations(output_config: dict): | |
| """For this file, don't copy the cadip "_session" stations.""" | |
| if not isinstance(output_config, dict): | |
| return | |
| for station in list(output_config.keys()): | |
| if station.endswith("_session"): | |
| output_config.pop(station) | |
| for station_name in ("pedc", "bedc"): | |
| station = output_config.get(station_name) | |
| if not isinstance(station, dict): | |
| continue | |
| authentication = station.get("authentication") | |
| if not isinstance(authentication, dict): | |
| continue | |
| for key in ("username", "password"): | |
| if key in authentication: | |
| authentication[key] = QuotedStr(str(authentication[key])) | |
| for key in ("ca_crt", "client_crt", "client_key"): | |
| if key in authentication and authentication[key] is not None: | |
| cert_value = str(authentication[key]).rstrip("\n") | |
| authentication[key] = LiteralStr(cert_value) | |
| station_params = HelmOrInfraParams( | |
| "services/common/config/rs-server.yaml", | |
| ["external_data_sources"], | |
| ["app", "stations"], | |
| 0, | |
| remove_session_stations, | |
| prune_missing=True, | |
| ) | |
| copy_to_helm_or_infra([station_params], rs_helm_dir / "charts/rs-server-station-secrets/values.yaml") | |
| for adgs_station in SUPPORTED_ADGS_STATIONS: | |
| copy_to_helm_or_infra( | |
| [ | |
| HelmOrInfraParams( | |
| "services/adgs/config/adgs_ws_config.yaml", | |
| [adgs_station], # use the station name as input value | |
| [adgs_station], | |
| 0, # output doc index | |
| prune_missing=True, | |
| ), | |
| ], | |
| rs_helm_dir / f"charts/rs-server-adgs/config/{adgs_station}_mockup_service_config.yaml", | |
| ) | |
| copy_to_helm_or_infra( | |
| [ | |
| HelmOrInfraParams( | |
| "services/adgs/config/adgs_search_config.yaml", | |
| [], | |
| [], | |
| 0, | |
| prune_missing=True, | |
| ), | |
| ], | |
| rs_helm_dir / "charts/rs-server-adgs/config/adgs_search_config.yaml", | |
| ignore_existing_file=True, | |
| ) | |
| for cadip_station in SUPPORTED_CADIP_STATIONS: | |
| copy_to_helm_or_infra( | |
| [ | |
| HelmOrInfraParams( | |
| "services/cadip/config/cadip_ws_config.yaml", | |
| [cadip_station], # use the station name as input value | |
| [cadip_station], | |
| 0, # output doc index | |
| prune_missing=True, | |
| ), | |
| ], | |
| rs_helm_dir / f"charts/rs-server-cadip/config/{cadip_station}_mockup_service_config.yaml", | |
| ) | |
| copy_to_helm_or_infra( | |
| [ | |
| HelmOrInfraParams( # same for _session stations | |
| "services/cadip/config/cadip_ws_config.yaml", | |
| [f"{cadip_station}_session"], | |
| [f"{cadip_station}_session"], | |
| 0, # output doc index | |
| prune_missing=True, | |
| ), | |
| ], | |
| rs_helm_dir / f"charts/rs-server-cadip/config/{cadip_station}_mockup_service_config.yaml", | |
| ) | |
| copy_to_helm_or_infra( | |
| [ | |
| HelmOrInfraParams( | |
| "services/cadip/config/cadip_search_config.yaml", | |
| [], | |
| [], | |
| 0, | |
| prune_missing=True, | |
| ), | |
| ], | |
| rs_helm_dir / "charts/rs-server-cadip/config/cadip_search_config.yaml", | |
| ignore_existing_file=True, | |
| ) | |
| for prip_station in SUPPORTED_PRIP_STATIONS: | |
| copy_to_helm_or_infra( | |
| [ | |
| HelmOrInfraParams( | |
| "services/prip/config/prip_ws_config.yaml", | |
| [prip_station], # use the station name as input value | |
| [prip_station], | |
| 0, # output doc index | |
| prune_missing=True, | |
| ), | |
| ], | |
| rs_helm_dir / f"charts/rs-server-prip/config/{prip_station}_mockup_service_config.yaml", | |
| ) | |
| copy_to_helm_or_infra( | |
| [ | |
| HelmOrInfraParams( | |
| "services/prip/config/prip_search_config.yaml", | |
| [], | |
| [], | |
| 0, | |
| prune_missing=True, | |
| ), | |
| ], | |
| rs_helm_dir / "charts/rs-server-prip/config/prip_search_config.yaml", | |
| ignore_existing_file=True, | |
| ) | |
Xet Storage Details
- Size:
- 36.2 kB
- Xet hash:
- a7804dde6ee6ee3f0c7fd4739bb452f67e2cd2e2ccc61e712fbf4577f21da1fc
·
Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.