"""  Module Name: Frontend App with Taipy GUI  Author: Kenneth Leung  Last Modified: 25 Mar 2023  """  # Launch app with the command in CLI: python main.py  from src.config import *  from src.functions import *  from pages.analysis_md import *  from pages.data_viewer_md import *  import taipy as tp  from taipy.gui import Gui, Icon, navigate    with open('config.yml') as f:      cfg = yaml.safe_load(f)    # =======================  #       Setup menu  # =======================  menu = [("Analysis", Icon('assets/histogram_menu_2.png', 'Analysis')),          ('Data', Icon('assets/data_menu.png', 'Data'))]    page_markdown = """  <|toggle|theme|>  <|menu|label=Menu|lov={menu}|on_action=menu_function|>  """    pages = {"/":page_markdown,           "Analysis":analysis_page,           "Data":data_page}    def menu_function(state, var_name: str, fct: str, var_value: list):      # Change the value of the state.page variable in order to render the correct page      navigate(state, var_value["args"][0])    # Run application  if __name__ == "__main__":      tp.Core().run()        # Create and execute scenario      scenario_selector = [(s.id, s.name) for s in tp.get_scenarios()]      scenario = tp.create_scenario(scenario_cfg, name="Default Scenario")      selected_scenario = scenario.id      tp.submit(scenario)        df = scenario.pipeline_keyword_analysis.data_keywords_df.read()      df_keywords_count = scenario.pipeline_keyword_analysis.data_keywords_count.read()        Gui(pages=pages).run(title="Keyword Extraction and Analysis with KeyBERT and Taipy",                            dark_mode=False,                            port=8020,                            use_reloader=True)


"""  Module Name: Page Markdown Template (Analysis Page)  Author: Kenneth Leung  Last Modified: 8 Apr 2023  """  import taipy as tp  from taipy.gui import notify  from src.config import scenario_cfg    import yaml  with open('config.yml') as f:      cfg = yaml.safe_load(f)    query = cfg['QUERY']  ngram_min = cfg['NGRAM_MIN']  ngram_max = cfg['NGRAM_MAX']  diversity_algo = cfg['DIVERSITY_ALGO']  diversity_algo_options = ['mmr', 'maxsum']  diversity = cfg['DIVERSITY']  top_n = cfg['TOP_N']  nr_candidates = cfg['NR_CANDIDATES']      # Input section of dashboard  input_page = """  # Keyword Extraction and Analysis with KeyBERT and Taipy
    <|layout|columns=1 1 1 1 1 1 1|  <|{query}|input|label=Query Topic|>    <|{ngram_min}|number|label=Min N-gram|>    <|{ngram_max}|number|label=Max N-gram|>    <|{top_n}|number|label=Top n results|>    <|{diversity}|number|label=Diversity (for MMR)|>    <|{nr_candidates}|number|label=No. of Candidates (for MaxSum)|>    <|{diversity_algo}|selector|lov={diversity_algo_options}|dropdown|label=Diversity Algorithm|>    <|Update Analysis|button|on_action=submit_scenario|>  |>

     <|{selected_scenario}|selector|lov={scenario_selector}|dropdown|label=Scenario|on_change=synchronize_gui_core|value_by_id|>     <|Save Scenario|button|on_action=create_scenario|>
    """    # Chart types: https://docs.taipy.io/en/latest/manuals/gui/viselements/chart/  chart_properties = {"type":"bar",                      "y":"keyword",                      "x":"count",                      "orientation": "h",                      "layout": {                          "xaxis": {"title": "Frequency Count"},                          "yaxis": {"title": None},                          "showlegend": False, # Hide the legend                          "title": 'Keyword Frequency Bar Plot',                          "margin": {'pad': 0}                          }                  }    # Output section of dashboard  output_page = """  <|layout|columns=1 1|gap=10px|  <|{df_keywords_count}|table|width=30|page_size=10|height=20|>    <|{df_keywords_count}|chart|properties={chart_properties}|height=20|>  |>    """    # Combine layout segments  analysis_page = input_page + output_page      # ======= Scenario Setup =========  def create_scenario(state):      print("Creating scenario...")      scenario = tp.create_scenario(scenario_cfg, name=f"Scenario N {len(state.scenario_selector)} {state.query}")      state.scenario_selector += [(scenario.id, scenario.name)]      state.selected_scenario = scenario.id      notify(state, 'success', 'Scenario created!')      submit_scenario(state)      def update_chart(state):      # Select the right scenario and pipeline      scenario = tp.get(state.selected_scenario)      # Update the chart based on this pipeline      state.df = scenario.pipeline_keyword_analysis.data_keywords_df.read()      state.df_keywords_count = scenario.pipeline_keyword_analysis.data_keywords_count.read()      def submit_scenario(state):       print("Submitting scenario...")       notify(state, 'info', 'Submitting scenario...')       # Get the selected scenario: in this current step a single scenario is created then modified here.       scenario = tp.get(state.selected_scenario)         # Change the default parameters by writing in the datanodes       scenario.query.write(str(state.query))       scenario.ngram_max.write(int(state.ngram_max))       scenario.diversity_algo.write(str(state.diversity_algo))       scenario.diversity.write(float(state.diversity))       scenario.top_n.write(int(state.top_n))       scenario.nr_candidates.write(int(state.nr_candidates))         # Execute the pipelines/code       tp.submit(scenario)       notify(state, 'success', 'Execution finished!')       # Update the chart when we change the scenario       update_chart(state)      def synchronize_gui_core(state):      scenario = tp.get(state.selected_scenario)      # get the information of the selected scenario and display it on the GUI      state.query = scenario.query.read()      state.ngram_max = scenario.ngram_max.read()      state.diversity_algo = scenario.diversity_algo.read()      state.diversity = scenario.diversity.read()      state.top_n = scenario.top_n.read()      state.nr_candidates = scenario.nr_candidates.read()      update_chart(state)


"""  Module Name: Page Markdown Template  Author: Kenneth Leung  Last Modified: 19 Mar 2023  """    data_page = """  # Extracted arXiv Abstracts    <|{df}|table|>    """


"""  Module Name: Core Functions (Backend)  Author: Kenneth Leung  Last Modified: 19 Mar 2023  """  import arxiv  import pandas as pd  import yaml  from keybert import KeyBERT      with open('config.yml') as f:      cfg = yaml.safe_load(f)    def extract_arxiv(query: str):      search = arxiv.Search(                  query=query,                  max_results=cfg['MAX_ABSTRACTS'], # Limit number of abstracts                  sort_by=arxiv.SortCriterion.SubmittedDate,                  sort_order=arxiv.SortOrder.Descending              )             # Returns arxiv object      return search      def save_in_dataframe(search):      df = pd.DataFrame([{'uid': result.entry_id.split('.')[-1],                          'title': result.title,                          'date_published': result.published,                          'abstract': result.summary} for result in search.results()])      return df      def preprocess_data(df: pd.DataFrame):      df['date_published'] = pd.to_datetime(df['date_published'])        # Create empty column to store keyword extraction output      df['keywords_and_scores'] = ''        # Create empty column to store top keywords      df['keywords'] = ''        return df      def run_keybert(df: pd.DataFrame, ngram_min: int, ngram_max: int, diversity_algo: str, top_n: int, diversity: float, nr_candidates: int):      kw_model = KeyBERT(model='all-MiniLM-L6-v2')      use_mmr = diversity_algo.lower() == 'mmr'      use_maxsum = diversity_algo.lower() == 'maxsum'            for i, row in df.iterrows():          abstract_text = row['abstract']          kw_output = kw_model.extract_keywords(abstract_text,                                                keyphrase_ngram_range=(ngram_min, ngram_max),                                                stop_words='english',                                                use_mmr=use_mmr,                                                use_maxsum=use_maxsum,                                                top_n=top_n,                                                diversity=diversity,                                                nr_candidates=nr_candidates)          df.at[i, 'keywords_and_scores'] = kw_output          df.at[i, 'keywords'] = [pair[0] for pair in kw_output]      return df      def get_keyword_value_counts(df):      keywords_count = pd.Series(df['keywords'].explode()).value_counts().reset_index()      keywords_count.columns = ['keyword', 'count']        return keywords_count


"""  Module Name: Taipy Configurations  Author: Kenneth Leung  Last Modified: 19 Mar 2023  """    import yaml  from taipy import Config, Scope  from src.functions import *    with open('config.yml') as f:      cfg = yaml.safe_load(f)    # ====================  #   Input Data Nodes  # ====================  # To place input values as DataNodeConfig objects  data_query_cfg = Config.configure_data_node(id='query', default_data=cfg['QUERY'])  data_ngram_min_cfg = Config.configure_data_node(id='ngram_min', default_data=cfg['NGRAM_MIN'])  data_ngram_max_cfg = Config.configure_data_node(id='ngram_max', default_data=cfg['NGRAM_MAX'])  data_diversity_algo_cfg = Config.configure_data_node(id='diversity_algo', default_data=cfg['DIVERSITY_ALGO'])  data_top_n_cfg = Config.configure_data_node(id='top_n', default_data=cfg['TOP_N'])  data_diversity_cfg = Config.configure_data_node(id='diversity', default_data=cfg['DIVERSITY'])  data_nr_candidates_cfg = Config.configure_data_node(id='nr_candidates', default_data=cfg['NR_CANDIDATES'])    # ===================  #   Key Data Nodes  # ===================  data_arxiv_search_cfg = Config.configure_data_node(id='data_arxiv_search')  data_raw_df_cfg = Config.configure_data_node(id='data_raw_df')  data_processed_df_cfg = Config.configure_data_node(id='data_processed_df')  data_keywords_df_cfg = Config.configure_data_node(id='data_keywords_df')  data_keywords_count_cfg = Config.configure_data_node(id='data_keywords_count')    # =================  #      Tasks  # =================  task_arxiv_extraction_cfg = Config.configure_task(id="task_arxiv_extraction",                                                    function=extract_arxiv,                                                    input=data_query_cfg,                                                    output=data_arxiv_search_cfg,                                                    skippable=True)    task_save_in_df_cfg = Config.configure_task(id="task_save_in_df",                                              function=save_in_dataframe,                                              input=data_arxiv_search_cfg,                                              output=data_raw_df_cfg,                                              skippable=True)    task_process_data_cfg = Config.configure_task(id='task_process_data',                                                function=preprocess_data,                                                input=data_raw_df_cfg,                                                output=data_processed_df_cfg,                                                skippable=True)    task_extract_keywords_cfg = Config.configure_task(id='task_extract_keywords',                                                    function=run_keybert,                                                    input=[data_processed_df_cfg,                                                            data_ngram_min_cfg,                                                           data_ngram_max_cfg,                                                            data_diversity_algo_cfg,                                                            data_top_n_cfg,                                                           data_diversity_cfg,                                                            data_nr_candidates_cfg],                                                    output=data_keywords_df_cfg)    task_get_kw_count_cfg = Config.configure_task(id='task_count_keywords',                                                    function=get_keyword_value_counts,                                                    input=data_keywords_df_cfg,                                                    output=data_keywords_count_cfg)      # =================  #     Pipelines  # =================  pipeline_data_prep_cfg = Config.configure_pipeline(id='pipeline_data_prep',                                                     task_configs=[task_arxiv_extraction_cfg,                                                                   task_save_in_df_cfg,                                                                   task_process_data_cfg,                                                                   ])    pipeline_keyword_analysis_cfg = Config.configure_pipeline(id='pipeline_keyword_analysis',                                                            task_configs=[task_extract_keywords_cfg,                                                                          task_get_kw_count_cfg                                                                          ])    # =============  #   Scenario   # =============  scenario_cfg = Config.configure_scenario(id="scenario",                                            pipeline_configs=[pipeline_data_prep_cfg,                                                              pipeline_keyword_analysis_cfg                                                             ])


from taipy.gui import Gui  from pages import *      pages = {      "/": root_page,      "pages": pages  }      if __name__ == "__main__":        gui = Gui(pages=pages)      gui.run(title="ui")


from .root import root_page  from .pages.pages import pages


"""  """    from taipy.gui import Markdown    root_page = Markdown("pages/root.md")


"""  <|{value}|text|format=%.2f|>    """    from taipy.gui import Markdown    pages = Markdown("pages/pages/pages.md")


import random    # Define a dictionary to store the year-wise and month-wise electricity usage  electricity_usage = {}    months = ['January', 'February', 'March', 'April', 'May', 'June', 'July', 'August', 'September', 'October', 'November', 'December']    # For each year and each month, generate a random number between 1000 and 5000 to simulate the electricity usage for that month  for year in range(1, 6):      electricity_usage[year] = {}      for month in months:          electricity_usage[year][month] = random.randint(1000, 5000)    import csv  # Open a new CSV file in write mode  with open('electricity_usage.csv', 'w', newline='') as file:      writer = csv.writer(file)        # Write the header row      writer.writerow(['YearMonth', 'Usage'])        # Iterate over the years from 2001 to 2010, and for each month, write a row to the CSV file      for year in range(2001, 2011):          for month in range(1, 13):              usage = random.randint(1000, 5000)              writer.writerow([f'{year}/{month}', usage])


from taipy.gui import Gui  import taipy as tp  import pandas as pd  import os      section_1 = """  ## Power Consumption Dashboard  <|{dataset}|chart|x=YearMonth|y[1]=Usage|color=blue|>  """    dataset = pd.read_csv("data.csv")    rest = tp.Rest()  gui = Gui(page=section_1)    tp.run(      rest=rest,      gui=gui,      title="Taipy Dashboard",      host='0.0.0.0',      port=os.environ.get('PORT', '5000'),      debug=False,  )


"""  A multi-page Taipy application, which includes 3 pages:  - A rootpage which is shared by other pages.  - Two pages named page_1 and page_2.    Please refer to https://docs.taipy.io/en/latest/manuals/gui/pages for more details.  """    from pages import data_viz, scenario_page, performance  from pages.root import *    from config.config import *    from taipy.gui import Gui  import taipy as tp      def on_change(state, var_name: str, var_value):      state['scenario'].on_change(state, var_name, var_value)      pages = {      "/": root_page,      "data_viz": data_viz,      "scenario": scenario_page,      "performance": performance  }      if __name__ == "__main__":      tp.Core().run()      gui = Gui(pages=pages)      gui.run(title="Taipy Application")


import datetime as dt  import pandas as pd    from taipy import Config, Scope, Frequency    from algos.algos import *    path_to_csv = "data/dataset.csv"    # Datanodes (3.1)  ## Input Data Nodes  initial_dataset_cfg = Config.configure_data_node(id="initial_dataset",                                                   storage_type="csv",                                                   path=path_to_csv,                                                   scope=Scope.GLOBAL)    # We assume the current day is the 26th of July 2021.  # This day can be changed to simulate multiple executions of scenarios on different days  day_cfg = Config.configure_data_node(id="day", default_data=dt.datetime(2021, 7, 26))    n_predictions_cfg = Config.configure_data_node(id="n_predictions", default_data=40)    max_capacity_cfg = Config.configure_data_node(id="max_capacity", default_data=200)    ## Remaining Data Nodes  cleaned_dataset_cfg = Config.configure_data_node(id="cleaned_dataset",                                                   scope=Scope.GLOBAL)    predictions_baseline_cfg = Config.configure_data_node(id="predictions_baseline")  predictions_ml_cfg = Config.configure_data_node(id="predictions_ml")    full_predictions_cfg = Config.configure_data_node(id="full_predictions")    metrics_baseline_cfg = Config.configure_data_node(id="metrics_baseline")  metrics_ml_cfg = Config.configure_data_node(id="metrics_ml")    # Functions (3.2)    # Tasks (3.3)  clean_data_task_cfg = Config.configure_task(id="task_clean_data",                                              function=clean_data,                                              input=initial_dataset_cfg,                                              output=cleaned_dataset_cfg,                                              skippable=True)    predict_baseline_task_cfg = Config.configure_task(id="predict_baseline",                                                    function=predict_baseline,                                                    input=[cleaned_dataset_cfg, n_predictions_cfg, day_cfg,                                                           max_capacity_cfg],                                                    output=predictions_baseline_cfg)      # Create the task configuration of the predict_ml function.  ## We use the same input and ouput as the previous predict_baseline task but we change the funtion  predict_ml_task_cfg = Config.configure_task(id="task_predict_ml",                                              function=predict_ml,                                              input=[cleaned_dataset_cfg,                                                     n_predictions_cfg, day_cfg,                                                     max_capacity_cfg],                                              output=predictions_ml_cfg)      metrics_baseline_task_cfg = Config.configure_task(id="task_metrics_baseline",                                              function=compute_metrics,                                              input=[cleaned_dataset_cfg,                                                     predictions_baseline_cfg],                                              output=metrics_baseline_cfg)    metrics_ml_task_cfg = Config.configure_task(id="task_metrics_ml",                                              function=compute_metrics,                                              input=[cleaned_dataset_cfg,                                                     predictions_ml_cfg],                                              output=metrics_ml_cfg)    full_predictions_task_cfg = Config.configure_task(id="task_full_predictions",                                              function=create_predictions_dataset,                                              input=[predictions_baseline_cfg,                                                     predictions_ml_cfg,                                                    day_cfg,                                                    n_predictions_cfg,                                                    cleaned_dataset_cfg],                                              output=full_predictions_cfg)          # Configure our scenario which is our business problem.  scenario_cfg = Config.configure_scenario_from_tasks(id="scenario",                                                      task_configs=[clean_data_task_cfg,                                                                    predict_baseline_task_cfg,                                                                    predict_ml_task_cfg,                                                                    metrics_baseline_task_cfg,                                                                    metrics_ml_task_cfg,                                                                    full_predictions_task_cfg],                                                      frequency=Frequency.WEEKLY)    Config.export('config/config.toml')


# For the sake of clarity, we have used an AutoRegressive model rather than a pure ML model such as:  # Random Forest, Linear Regression, LSTM, etc     from statsmodels.tsa.ar_model import AutoReg  import pandas as pd  from sklearn.metrics import mean_squared_error, mean_absolute_error  import numpy as np  import datetime as dt    def clean_data(initial_dataset: pd.DataFrame):      print("     Cleaning data")      # Convert the date column to datetime      initial_dataset['Date'] = pd.to_datetime(initial_dataset['Date'])      cleaned_dataset = initial_dataset.copy()      return cleaned_dataset      def predict_baseline(cleaned_dataset: pd.DataFrame, n_predictions: int, day: dt.datetime, max_capacity: int):      print("     Predicting baseline")      # Select the train data      train_dataset = cleaned_dataset[cleaned_dataset['Date'] < day]        predictions = train_dataset['Value'][-n_predictions:].reset_index(drop=True)      predictions = predictions.apply(lambda x: min(x, max_capacity))      return predictions      # This is the function that will be used by the task  def predict_ml(cleaned_dataset: pd.DataFrame, n_predictions: int, day: dt.datetime, max_capacity: int):      print("     Predicting with ML")      # Select the train data      train_dataset = cleaned_dataset[cleaned_dataset["Date"] < day]        # Fit the AutoRegressive model      model = AutoReg(train_dataset["Value"], lags=7).fit()        # Get the n_predictions forecasts      predictions = model.forecast(n_predictions).reset_index(drop=True)      predictions = predictions.apply(lambda x: min(x, max_capacity))      return predictions      def compute_metrics(historical_data, predicted_data):      historical_to_compare = historical_data[-len(predicted_data):]['Value']      rmse = mean_squared_error(historical_to_compare, predicted_data)      mae = mean_absolute_error(historical_to_compare, predicted_data)      return rmse, mae    def create_predictions_dataset(predictions_baseline, predictions_ml, day, n_predictions, cleaned_data):      print("Creating predictions dataset...")        # Set arbitrarily the time window for the chart as 5 times the number of predictions      window = 5 * n_predictions        # Create the historical dataset that will be displayed      new_length = len(cleaned_data[cleaned_data["Date"] < day]) + n_predictions      temp_df = cleaned_data[:new_length]      temp_df = temp_df[-window:].reset_index(drop=True)        # Create the series that will be used in the concat      historical_values = pd.Series(temp_df["Value"], name="Historical values")      predicted_values_ml = pd.Series([np.NaN] * len(temp_df), name="Predicted values ML")      predicted_values_ml[-len(predictions_ml):] = predictions_ml       predicted_values_baseline = pd.Series([np.NaN] * len(temp_df), name="Predicted values Baseline")      predicted_values_baseline[-len(predictions_baseline):] = predictions_baseline        # Create the predictions dataset      # Columns : [Date, Historical values, Predicted values]      predictions_dataset = pd.concat([temp_df["Date"],                                      historical_values,                                      predicted_values_ml,                                      predicted_values_baseline], axis=1)            return predictions_dataset


from .data_viz import data_viz  from .scenario import scenario_page  from .performance import performance  from .root import root_page


"""  The rootpage of the application.  Page content is imported from the root.md file.    Please refer to https://docs.taipy.io/en/latest/manuals/gui/pages for more details.  """    from taipy.gui import Markdown    root_page = Markdown("pages/root.md")


from .scenario import scenario_page


"""  The second page of the application.  Page content is imported from the page_2.md file.    Please refer to https://docs.taipy.io/en/latest/manuals/gui/pages for more details.  """    from taipy.gui import Markdown, notify  import datetime as dt  import pandas as pd      scenario = None  day = dt.datetime(2021, 7, 26)  n_predictions = 40  max_capacity = 200  predictions_dataset = {"Date":[0],                          "Predicted values ML":[0],                         "Predicted values Baseline":[0],                         "Historical values":[0]}        def save(state):      print("Saving scenario...")      # Get the currently selected scenario        # Conversion to the right format      state_day = dt.datetime(state.day.year, state.day.month, state.day.day)        # Change the default parameters by writing in the Data Nodes      state.scenario.day.write(state_day)      state.scenario.n_predictions.write(int(state.n_predictions))      state.scenario.max_capacity.write(int(state.max_capacity))      notify(state, "success", "Saved!")          def on_change(state, var_name, var_value):      if var_name == "scenario" and var_value:          state.day = state.scenario.day.read()          state.n_predictions = state.scenario.n_predictions.read()          state.max_capacity = state.scenario.max_capacity.read()                    if state.scenario.full_predictions.is_ready_for_reading:              state.predictions_dataset = state.scenario.full_predictions.read()          else:              state.predictions_dataset = predictions_dataset        scenario_page = Markdown("pages/scenario/scenario.md")


from .data_viz import data_viz


"""  The first page of the application.  Page content is imported from the page_1.md file.    Please refer to https://docs.taipy.io/en/latest/manuals/gui/pages for more details.  """    from taipy.gui import Markdown  import pandas as pd    def get_data(path_to_csv: str):      # pandas.read_csv() returns a pd.DataFrame      dataset = pd.read_csv(path_to_csv)      dataset["Date"] = pd.to_datetime(dataset["Date"])      return dataset    # Read the dataframe  path_to_csv = "data/dataset.csv"  dataset = get_data(path_to_csv)    # Initial value  n_week = 10    # Select the week based on the slider value  dataset_week = dataset[dataset["Date"].dt.isocalendar().week == n_week]    def on_slider(state):      state.dataset_week = dataset[dataset["Date"].dt.isocalendar().week == state.n_week]      data_viz = Markdown("pages/data_viz/data_viz.md")


from .performance import performance


from taipy.gui import Markdown    import pandas as pd  import taipy as tp    # Initial dataset for comparison  comparison_scenario = pd.DataFrame(columns=["Scenario Name",                                              "RMSE baseline",                                              "MAE baseline",                                              "RMSE ML",                                              "MAE ML"])      # Selector for metrics  metric_selector = ["RMSE", "MAE"]  selected_metric = metric_selector[0]      def compare(state):      print("Comparing...")        # Initialize lists for comparison      scenario_data = []        # Go through all the primary scenarios      all_scenarios = sorted(tp.get_primary_scenarios(), key=lambda x: x.creation_date.timestamp())        for scenario in all_scenarios:          rmse_baseline, mae_baseline = scenario.metrics_baseline.read()          rmse_ml, mae_ml = scenario.metrics_ml.read()              # Store scenario data in a dictionary          scenario_data.append({              "Scenario Name": scenario.name,              "RMSE baseline": rmse_baseline,              "MAE baseline": mae_baseline,              "RMSE ML": rmse_ml,              "MAE ML": mae_ml          })        # Create a DataFrame from the scenario_data list      state.comparison_scenario = pd.DataFrame(scenario_data)    performance = Markdown("pages/performance/performance.md")


import os  import re  import shutil  import subprocess    from _fetch_source_file import CLI, GitContext, read_doc_version_from_mkdocs_yml_template_file    # Assuming this script is in taipy-doc/tools  TOOLS_PATH = os.path.dirname(os.path.abspath(__file__))  ROOT_DIR = os.path.dirname(TOOLS_PATH)  TOP_DIR = os.path.dirname(ROOT_DIR)    # Where all the code from all directories/repositories is copied  DEST_DIR_NAME = "taipy"    REPOS = ["config", "core", "gui", "rest", "taipy", "templates"]  PRIVATE_REPOS = ["auth", "enterprise"]    OPTIONAL_PACKAGES = {      "gui": ["pyarrow", "pyngrok", "python-magic", "python-magic-bin"]  }    args = CLI(os.path.basename(__file__), REPOS).get_args()    # Read version from mkdocs.yml template  mkdocs_yml_version = read_doc_version_from_mkdocs_yml_template_file(ROOT_DIR)    # Gather version information for each repository  repo_defs = {repo if repo == "taipy" else f"taipy-{repo}": {"version": "local", "tag": None} for repo in REPOS + PRIVATE_REPOS}  CATCH_VERSION_RE = re.compile(r"(^\d+\.\d+?)(?:(\.\d+)(\..*)?)?|develop|local$")  for version in args.version:      repo = None      if version == "MKDOCS":          version = mkdocs_yml_version          tag = None      else:          try:              colon = version.index(":")              repo = version[:colon]              if not repo.startswith("taipy"):                  repo = f"taipy-{repo}"              version = version[colon + 1:]          except ValueError as e:              pass          version_match = CATCH_VERSION_RE.fullmatch(version)          if not version_match:              raise ValueError(f"'{version}' is not a valid version.")          tag = None          if version_match.group(1):              version = version_match.group(1)              if version_match.group(2):                  tag = version + version_match.group(2)                  if version_match.group(3):                      tag += version_match.group(3)      if repo:          if not repo in repo_defs:              raise ValueError(f"'{repo}' is not a valid repository name.")          repo_defs[repo]["version"] = version          repo_defs[repo]["tag"] = tag      else:          for repo in repo_defs.keys():              repo_defs[repo]["version"] = version              repo_defs[repo]["tag"] = tag    # Test git, if needed  git_command = "git"  if args.no_pull and all(v["version"] == "local" for v in repo_defs.values()):      git_command = None  else:      git_path = shutil.which(git_command)      if git_path is None or subprocess.run(f"\"{git_path}\" --version", shell=True, capture_output=True) is None:          raise IOError(f"Couldn't find command \"{git_command}\"")      git_command = git_path    # Check that directory, branches and tags exist for each repository  github_token = os.environ.get("GITHUB_TOKEN", "")  if github_token:      github_token += "@"  github_root = f"https://{github_token}github.com/Avaiga/"  for repo in repo_defs.keys():      version = repo_defs[repo]["version"]      if version == "local":          repo_path = os.path.join(TOP_DIR, repo)          repo_defs[repo]["path"] = repo_path          if not os.path.isdir(repo_path):              if repo in PRIVATE_REPOS:                  repo_defs[repo]["skip"] = True              else:                  raise IOError(f"Repository '{repo}' must be cloned in \"{TOP_DIR}\".")      elif version == "develop":          with GitContext(repo, PRIVATE_REPOS):              cmd = subprocess.run(f"\"{git_path}\" ls-remote -q -h {github_root}{repo}.git", shell=True, capture_output=True,                                   text=True)              if cmd.returncode:                  if repo in PRIVATE_REPOS:                      repo_defs[repo]["skip"] = True                      continue                  else:                      raise SystemError(f"Problem with {repo}:\nOutput: {cmd.stdout}\nError: {cmd.stderr}")      else:          with GitContext(repo, PRIVATE_REPOS):              cmd = subprocess.run(f"\"{git_path}\" ls-remote --exit-code --heads {github_root}{repo}.git", shell=True,                                   capture_output=True, text=True)              if cmd.returncode:                  if repo in PRIVATE_REPOS:                      repo_defs[repo]["skip"] = True                      continue                  else:                      raise SystemError(f"Couldn't query branches from {github_root}{repo}.")              if not f"release/{version}\n" in cmd.stdout:                  raise ValueError(f"No branch 'release/{version}' in repository '{repo}'.")              tag = repo_defs[repo]["tag"]              if tag:                  cmd = subprocess.run(f"\"{git_path}\" ls-remote -t --refs {github_root}{repo}.git", shell=True,                                       capture_output=True, text=True)                  if not f"refs/tags/{tag}\n" in cmd.stdout:                      raise ValueError(f"No tag '{tag}' in repository '{repo}'.")    if args.check:      print(f"Fetch should perform properly with the following settings:")      for repo in repo_defs.keys():          if not repo_defs[repo].get("skip", False):              version = repo_defs[repo]['version']              version = "(local)" if version == "local" else f"branch:{version if version == 'develop' else f'release/{version}'}"              tag = repo_defs[repo]["tag"]              if tag:                  version += f" tag:{tag}"              print(f"- {repo} {version}")      exit(0)    DEST_DIR = os.path.join(ROOT_DIR, DEST_DIR_NAME)      def safe_rmtree(dir: str):      if os.path.isdir(dir):          shutil.rmtree(dir)      # Remove target 'taipy' directory  safe_rmtree(DEST_DIR)  os.makedirs(DEST_DIR)  # If a leftover of a previous failed run  safe_rmtree(os.path.join(TOOLS_PATH, DEST_DIR_NAME))    pipfile_packages = {}  PIPFILE_PACKAGE_RE = re.compile(r"(.*?)\s?=\s?(.*)")      # Fetch files  def move_files(repo: str, src_path: str):      # Read Pipfile dependency packages      pipfile_path = os.path.join(src_path, "Pipfile")      if os.path.isfile(pipfile_path):          reading_packages = False          repo_optional_packages = OPTIONAL_PACKAGES.get(repo, None)          with open(pipfile_path, "r") as pipfile:              while True:                  line = pipfile.readline()                  if str(line) == "" or (reading_packages and (not line.strip() or line[0] == "[")):                      break                  line = line.strip()                  if line == "[packages]":                      reading_packages = True                  elif reading_packages:                      match = PIPFILE_PACKAGE_RE.fullmatch(line)                      if match and not match.group(1).startswith("taipy"):                          package = match.group(1).lower()                          version = match.group(2)                          if repo_optional_packages is None or not package in repo_optional_packages:                              if package in pipfile_packages:                                  versions = pipfile_packages[package]                                  if version in versions:                                      versions[version].append(repo)                                  else:                                      versions[version] = [repo]                              else:                                  pipfile_packages[package] = {version: [repo]}      # Copy relevant files for doc generation      if repo.startswith("taipy-getting-started"):          gs_dir = os.path.join(ROOT_DIR, "docs", "getting_started", repo[6:])          # safe_rmtree(os.path.join(gs_dir, "src"))          for step_dir in [step_dir for step_dir in os.listdir(gs_dir) if                           step_dir.startswith("step_") and os.path.isdir(os.path.join(gs_dir, step_dir))]:              safe_rmtree(os.path.join(gs_dir, step_dir))          for step_dir in [step_dir for step_dir in os.listdir(src_path) if                           step_dir.startswith("step_") and os.path.isdir(os.path.join(src_path, step_dir))]:              shutil.copytree(os.path.join(src_path, step_dir), os.path.join(gs_dir, step_dir))          safe_rmtree(os.path.join(gs_dir, "src"))          shutil.copytree(os.path.join(src_path, "src"), os.path.join(gs_dir, "src"))          shutil.copy(os.path.join(src_path, "index.md"), os.path.join(gs_dir, "index.md"))          saved_dir = os.getcwd()          os.chdir(os.path.join(ROOT_DIR, "docs", "getting_started", repo[6:]))          subprocess.run(f"python {os.path.join(src_path, 'generate_notebook.py')}",                         shell=True,                         capture_output=True,                         text=True)          os.chdir(saved_dir)      else:          tmp_dir = os.path.join(ROOT_DIR, f"{repo}.tmp")          safe_rmtree(tmp_dir)          gui_dir = os.path.join(ROOT_DIR, f"gui") if repo == "taipy-gui" else None          if repo == "taipy-gui" and os.path.isdir(gui_dir):              if os.path.isdir(os.path.join(gui_dir, "node_modules")):                  shutil.move(os.path.join(gui_dir, "node_modules"), os.path.join(ROOT_DIR, f"gui_node_modules"))              shutil.rmtree(gui_dir)          try:              def copy_source(src_path: str, repo: str):                  def copy(item: str, src: str, dst: str, rel_path: str):                      full_src = os.path.join(src, item)                      full_dst = os.path.join(dst, item)                      if os.path.isdir(full_src):                          if item in ["__pycache__", "node_modules"]:                              return                          if not os.path.isdir(full_dst):                              os.makedirs(full_dst)                          rel_path = f"{rel_path}/{item}"                          for sub_item in os.listdir(full_src):                              copy(sub_item, full_src, full_dst, rel_path)                      elif any(item.endswith(ext) for ext in [".py", ".pyi", ".json", ".ipynb"]):                          if os.path.isfile(full_dst): # File exists - compare                              with open(full_src, "r") as f:                                  src = f.read()                              with open(full_dst, "r") as f:                                  dst = f.read()                              if src != dst:                                  raise FileExistsError(f"File {rel_path}/{item} already exists and is different (copying repository {repo})")                          else:                              shutil.copy(full_src, full_dst)                  dest_path = os.path.join(ROOT_DIR, "taipy")                  if not os.path.exists(dest_path):                      os.makedirs(dest_path)                  src_path = os.path.join(src_path, "src", "taipy")                  for item in os.listdir(src_path):                      copy(item, src_path, dest_path, "")              copy_source(src_path, repo)                if gui_dir:                  if not os.path.isdir(gui_dir):                      os.mkdir(gui_dir)                  src_gui_dir = os.path.join(src_path, "gui")                  shutil.copytree(os.path.join(src_gui_dir, "src"), os.path.join(gui_dir, "src"))                  for f in [f for f in os.listdir(src_gui_dir) if f.endswith(".md") or f.endswith(".json")]:                      shutil.copy(os.path.join(src_gui_dir, f), os.path.join(gui_dir, f))                  if os.path.isdir(os.path.join(ROOT_DIR, "gui_node_modules")):                      shutil.move(os.path.join(ROOT_DIR, "gui_node_modules"), os.path.join(gui_dir, "node_modules"))          finally:              pass              """              shutil.rmtree(tmp_dir)              """      for repo in repo_defs.keys():      if repo_defs[repo].get("skip", False):          continue      version = repo_defs[repo]['version']      print(f"Fetching file for repository {repo} ({version})", flush=True)      if version == "local":          src_path = repo_defs[repo]['path']          if not args.no_pull:              subprocess.run(f"\"{git_path}\" pull {src_path}", shell=True, capture_output=True, text=True)          print(f"    Copying from {src_path}...", flush=True)          move_files(repo, src_path)      else:          clone_dir = os.path.join(ROOT_DIR, f"{repo}.clone")          if version != "develop":              version = f"release/{version}"          print("    Cloning...", flush=True)          subprocess.run(f"\"{git_path}\" clone -b {version} {github_root}{repo}.git {clone_dir}", shell=True,                         capture_output=True, text=True)          tag = repo_defs[repo]['tag']          if tag:              # Checkout tag version              saved_dir = os.getcwd()              os.chdir(clone_dir)              subprocess.run(f"\"{git_path}\" checkout {tag}", shell=True, capture_output=True, text=True)              os.chdir(saved_dir)          move_files(repo, clone_dir)              # For some reason, we need to protect the removal of the clone dirs...          # See https://stackoverflow.com/questions/1213706/what-user-do-python-scripts-run-as-in-windows          def handleRemoveReadonly(func, path, exc):              import errno, stat              if func == os.unlink and exc[1].errno == errno.EACCES:                  os.chmod(path, stat.S_IRWXU | stat.S_IRWXG | stat.S_IRWXO)  # 0777                  func(path)              else:                  raise              shutil.rmtree(clone_dir, onerror=handleRemoveReadonly)    # Manually add the taipy.run() function.  # TODO: Automate this, grabbing the function from the 'taipy' repository,  # so we benefit from potential updates.  init_path = os.path.join(ROOT_DIR, "taipy", "__init__.py")  with open(init_path, "a") as init:      run_method = """  import typing as t    def run(*services: t.Union[Gui, Rest, Core], **kwargs) -> t.Optional[t.Union[Gui, Rest, Core]]:      \"\"\"Run one or multiple Taipy services.        A Taipy service is an instance of a class that runs code as a web application.        Parameters:          services (Union[`Gui^`, `Rest^`, `Core^`]): Services to run.<br/>              If several services are provided, all the services run simultaneously. If this is empty or set to None,              this method does nothing.          **kwargs (dict[str, any]): Other parameters to provide to the services.      \"\"\"      pass\n"""      init.write(run_method)    # Generate Pipfile from package dependencies from all repositories  pipfile_path = os.path.join(ROOT_DIR, "Pipfile")  pipfile_message = "WARNING: Package versions mismatch in Pipfiles - Pipfile not updated."  for package, versions in pipfile_packages.items():      if len(versions) != 1:          if pipfile_message:              print(pipfile_message)              pipfile_message = None          print(f"- {package}:")          for version, repos in versions.items():              print(f"  {version} in {', '.join(repos)}.")          pipfile_path = None  # Update Pipfile from all other Pipfiles  if pipfile_path:      pipfile_lines = None      with open(pipfile_path, "r") as pipfile:          pipfile_lines = pipfile.readlines()      new_pipfile_path = os.path.join(ROOT_DIR, "Pipfile.new")      in_packages_section = False      legacy_pipfile_packages = {}      pipfile_changes = []      with open(new_pipfile_path, "w") as new_pipfile:          for line in pipfile_lines:              if in_packages_section:                  if not line or line[0] == "[":                      in_packages_section = False                      # List packages                      for package in sorted(pipfile_packages.keys(), key=str.casefold):                          versions = pipfile_packages[package]                          version = list(versions.keys())[0]                          if package == "modin":                              # Remove 'extras' from modin package requirements                              version = re.sub(r"\{\s*extras.*?,\s*version\s*=\s*(.*?)\s*}", r"\1", version)                          new_pipfile.write(f"{package} = {version}\n")                          if not package in legacy_pipfile_packages:                              pipfile_changes.append(f"Package '{package}' added ({version})")                          elif legacy_pipfile_packages[package] != version:                              pipfile_changes.append(                                  f"Package '{package}' version changed from {legacy_pipfile_packages[package]} to {version}")                              del legacy_pipfile_packages[package]                          else:                              del legacy_pipfile_packages[package]                      new_pipfile.write("\n")                      skip_line = True                      new_pipfile.write(line)                  match = PIPFILE_PACKAGE_RE.fullmatch(line.strip())                  if match and not match.group(1).startswith("taipy"):                      legacy_pipfile_packages[match.group(1).lower()] = match.group(2)              else:                  new_pipfile.write(line)                  if line.strip() == "[packages]":                      in_packages_section = True      for package, version in legacy_pipfile_packages.items():          pipfile_changes.append(f"Package '{package}' removed ({version})")      if pipfile_changes:          print(f"Pipfile was updated (Pipfile.bak saved):")          for change in pipfile_changes:              print(f"- {change}")          shutil.move(pipfile_path, os.path.join(ROOT_DIR, "Pipfile.bak"))          shutil.move(new_pipfile_path, pipfile_path)          print(f"You may want to rebuild you virtual environment:")          print(f"  - pipenv --rm")          print(f"  - pipenv install --dev")      else:          print(f"No changes in Pipfile")          os.remove(new_pipfile_path)


# ################################################################################  # setup_generation.py  #   Prepares all files before running MkDocs to generate the complete  #   Taipy documentation set.  #  # This setup is performed by executing successive steps, depending on the  # topic (Reference Manual generation, JavaScript documentation integration  # within MkDocs...)  # ################################################################################    import os    from _setup_generation import run_setup    steps = None  # --------------------------------------------------------------------------------  # You may want to replace that 'step' initial value with a list of steps you  # want to run.  # i.e. say you want to run only the Reference Manual generation step.  # You could add the following code:  #   from _setup_generation.step_refman import RefManStep  #   steps = [RefManStep()]  # --------------------------------------------------------------------------------    # Assuming that this script is located in <taipy-doc>/tools  run_setup(os.path.dirname(os.path.dirname(os.path.abspath(__file__))), steps)


# ######################################################################  #  # Syntax for cross-references to the Reference Manual:  #   `ClassName^`  #       generates a link from `ClassName` to the class doc page.  #   `functionName(*)^`  #       generates a link from `functionName(*)` to the function doc section.  #   `ClassName.methodName(*)^`  #       generates a link from `ClassName.methodName(*)` to the method doc section.  #   `(ClassName.)methodName(*)^`  #       generates a link from `methodName(*)` to the method doc section, hiding the class name.  #   `package.functionName(*)^`  #       generates a link from `package.functionName(*)` to the function doc section.  #   `(package.)functionName(*)^`  #       generates a link from `functionName(*)` to the function doc section, hiding the package.  #   `(package.)ClassName^`  #       generates a link from `ClassName` to the class doc page, hiding the package.  #   `(package.)ClassName.methodName(*)^`  #       generates a link from `ClassName.methodName(*)` to the method doc section, hiding the  #       package.  #   `(package.ClassName.)methodName(*)^`  #       generates a link from `methodName(*)` to the method doc section, hiding the package and the  #       class name.  # ######################################################################  import os  import re  from typing import Dict  import logging  import json    def define_env(env):      """      Mandatory to make this a proper MdDocs macro      """      match = re.search(r"/en/(develop|(?:release-(\d+\.\d+)))/$", env.conf["site_url"])      env.conf["branch"] = (f"release/{match[2]}" if match[2] else match[1]) if match else "unknown"      TOC_ENTRY_PART1 = r"
\s*([^<]*)\s*
\s*"  # XREF_RE details:  #  group(1) - The package, class or function name with a trailing '.'.  #    Can be empty.  #    This text will not appear as the anchor text.  #  group(2) - The name of a class, function, method... This text will appear in the anchor text  #  group(3) - The function parameters, with their (). Can be empty.  #  group(4) - The anchor text. Can be empty, then group(2)+group(3) is used.  ## XREF_RE = re.compile(r"(\()?((?:[^\d\W]\w*\.)*)"  ##                     + r"(?:\))?([^\d\W][\w\.]*)(\(.*?\))?\^<\/code>")  XREF_RE = re.compile(r"(?:\(([\w*\.]*)\))?([\.\w]*)(\(.*?\))?\^<\/code>(?:\(([^\)]*)\))?")      def find_dummy_h3_entries(content: str) -> Dict[str, str]:      """      Find 'dummy
' entries.        These are
 tags that are just redirections to another page.      These need to be removed, and redirection must be used in TOC.      """        ids = {}      TOC_ENTRY = re.compile(TOC_ENTRY_PART1 + r"(#[^\"]+)" + TOC_ENTRY_PART2, re.M | re.S)      while True:          toc_entry = TOC_ENTRY.search(content)          if toc_entry is None:              break          content = content[toc_entry.end() :]          id = toc_entry.group(1)[1:]          dummy_h3 = re.search(r"
\s* str:      """      Removes dummy
 entries and fix TOC.      """        for id, redirect in ids.items():          # Replace redirection in TOC          content = re.sub(              TOC_ENTRY_PART1 + "#" + id + TOC_ENTRY_PART2,              '
\\1
\n',              content,              re.M | re.S,          )          # Remove dummy
          content = re.sub(r"
\s*<a\s+href=\".*?\".*?
", "", content, re.M | re.S)      return content      def on_post_build(env):      """      Post-build actions for Taipy documentation      """        log = logging.getLogger("mkdocs")      site_dir = env.conf["site_dir"]      xrefs = {}      multi_xrefs = {}      if os.path.exists(site_dir + "/manuals/xrefs"):          with open(site_dir + "/manuals/xrefs") as xrefs_file:              xrefs = json.load(xrefs_file)      if xrefs is None:          log.error(f"Could not read xrefs in 'manuals/xrefs'")      x_packages = set()      for xref, xref_desc in xrefs.items():          if isinstance(xref_desc, list):              x_packages.add(xref_desc[0])              if xref_desc[2]:                  x_packages.update(xref_desc[2])          else:              descs = [xrefs[f"{xref}/{i}"] for i in range(0, xref_desc)]              # If unspecified, the first xref will be used (the one with the shortests package)              multi_xrefs[xref] = sorted(descs, key=lambda p: len(p[0]))      manuals_files_path = os.path.join(site_dir, "manuals")      ref_files_path = os.path.join(manuals_files_path, "reference")      fixed_cross_refs = {}      for root, _, file_list in os.walk(site_dir):          for f in file_list:              # Remove the *_template files              if f.endswith("_template"):                  os.remove(os.path.join(root, f))              # Post-process generated '.html' files              elif f.endswith(".html"):                  filename = os.path.join(root, f)                  file_was_changed = False                  with open(filename) as html_file:                      try:                          html_content = html_file.read()                      except Exception as e:                          print(f"Couldn't read HTML file {filename}")                          raise e                      # Rebuild coherent links from TOC to sub-pages                      ids = find_dummy_h3_entries(html_content)                      if ids:                          html_content = remove_dummy_h3(html_content, ids)                          file_was_changed = True                      # Remove
Index
 part of relevant pages                      INDEX_H1_RE = re.compile(                          r"
Index
\s*
(.*?)
", re.M | re.S                      )                      match = INDEX_H1_RE.search(html_content)                      if match:                          before = html_content[:match.start()]                          new_title = match.group(2)                          if new_title.startswith("Package"):                              USELESS_TITLE_RE = re.compile(r"(?<=<title>)Index(.*?)(?=</title>)", re.M | re.S)                              t_match = USELESS_TITLE_RE.search(before)                              if t_match:                                  new_title = re.sub(r"<a\s+.*?", "", new_title)                                  new_title, n = re.subn(r"(.*?)", r"\g<1>", new_title)                                  new_title = "Taipy p" + new_title[1:]                          before = before[:t_match.start()] + new_title + before[t_match.end():]                          html_content = (before                              + f"
{match.group(2)}
"                              + html_content[match.end():])                          file_was_changed = True                      #"""                      # Collapse doubled
/
 page titles                      REPEATED_H1_H2 = re.compile(                          r"
(.*?)
\s*
\1(<a\s+class=\"headerlink\".*?)?
", re.M | re.S                      )                      html_content, n_changes = REPEATED_H1_H2.subn('
\\1\\3
', html_content)                      if n_changes != 0:                          file_was_changed = True                      #"""                      # Specific processing for Getting Started documentation files                      if "getting_started" in filename:                          GS_H1_H2 = re.compile(r"
(.*?)
(.*?
\1)<", re.M | re.S)                          html_content, n_changes = GS_H1_H2.subn('\\2<', html_content)                          if n_changes != 0:                              file_was_changed = True                          gs_rel_path = os.path.relpath(site_dir, filename).replace("\\", "/").replace("../", "", 1)                          GS_DOCLINK = re.compile(r"(href=\")https://docs\.taipy\.io/en/latest(.*?\")", re.M | re.S)                          html_content, n_changes = GS_DOCLINK.subn(f"\\1{gs_rel_path}\\2", html_content)                          if n_changes != 0:                              file_was_changed = True                          GS_IPYNB = re.compile(r"(", re.M | re.S)                          html_content, n_changes = GS_IPYNB.subn(f"\\1 download>", html_content)                          if n_changes != 0:                              file_was_changed = True                      # Add external link icons (and open in new tab)                      # Note we want this only for the simple [text](http*://ext_url) cases                      EXTLINK = re.compile(                          r".*?<\/a>)", re.M | re.S                      )                      html_content, n_changes = EXTLINK.subn('<a class="ext-link" target="_blank" \\1', html_content)                      if n_changes != 0:                          file_was_changed = True                      # Find and resolve automatic cross-references to the Reference Manual                      # The syntax in Markdown is `(class.)method()^` and similar.                      rel_path = os.path.relpath(ref_files_path, filename).replace("\\", "/").replace("../", "", 1)                      new_content = ""                      last_location = 0                      for xref in XREF_RE.finditer(html_content):                          all_parts = (xref[1] + xref[2]) if xref[1] else xref[2]                          parts = all_parts.split(".")                          c_name: str = parts.pop() # Class of function name                          m_name: str = None # Method name                          # Get potential destination link descriptor from last part                          desc = xrefs.get(c_name)                          # Check for class.method (looking at part before last)                          if len(parts) > 0:                              potential_class_name = parts[-1]                              if class_xref := xrefs.get(potential_class_name):                                  desc = class_xref                                  m_name = c_name                                  c_name = potential_class_name                                  parts.pop()                          if isinstance(desc, int):                              if parts:                                  package = "." . join(parts)                                  desc = next(d for d in multi_xrefs.get(c_name) if d[0].endswith(package))                              else:                                  desc = multi_xrefs.get(c_name)[0]                          link = None                          if not desc:                              if os.path.exists(f"{ref_files_path}/pkg_{all_parts}/index.html"):                                  link = f"pkg_{all_parts}"                              else:                                  (dir, file) = os.path.split(filename)                                  (dir, dir1) = os.path.split(dir)                                  (dir, dir2) = os.path.split(dir)                                  message = f"Unresolve crossref '{re.sub(r'</?code>', '`', xref[0])}' found in "                                  if file == "index.html":                                      (dir, dir3) = os.path.split(dir)                                      log.error(f"{message}{dir3}/{dir2}/{dir1}.md")                                  else:                                      log.error(f"{message}{dir2}/{dir1}/{file}")                          else:                              link = f"{desc[0]}.{c_name}"                              if m_name:                                  link += f"/index.html#{desc[0]}.{c_name}.{m_name}"                          if link:                              new_content += html_content[last_location:xref.start()]                              new_content += f""                              if xref[4]: # Anchor text                                  new_content += xref[4]                              elif xref[2] or xref[3]:                                  new_content += ""                                  if xref[2]: new_content += xref[2]                                  if xref[3]: new_content += xref[3]                                  new_content += ""                              else:                                  new_content += f"NO CONTENT"                              new_content += f""                          last_location = xref.end()                      if last_location:                          file_was_changed = True                          html_content = new_content + html_content[last_location:]                      # Find 'free' crossrefs to the Reference Manual                      # Syntax in Markdown is [free text]((class.)method()^) and similar                      new_content = ""                      last_location = 0                      FREE_XREF_RE = re.compile(r"(.*?)")                      for xref in FREE_XREF_RE.finditer(html_content):                          groups = xref.groups()                          entry = groups[1]                          method = groups[2]                          # Function in package? Or method in class?                          packages  = [entry, None] if entry in x_packages else xrefs.get(entry)                          if packages is None:                              (dir, file) = os.path.split(filename)                              (dir, dir1) = os.path.split(dir)                              (dir, dir2) = os.path.split(dir)                              bad_xref = xref.group(0)                              message = f"Unresolve crossref '{bad_xref}' found in "                              if file == "index.html":                                  (dir, dir3) = os.path.split(dir)                                  log.error(f"{message}{dir3}/{dir2}/{dir1}.md")                              else:                                  log.error(f"{message}{dir2}/{dir1}/{file}")                              continue                          else: # Free XRef was found: forget about the previous warning after all                              md_file=filename[len(site_dir):]                              sep=md_file[0]                              (dir, file) = os.path.split(md_file[1:]) # Drop the separator                              (dir, dir1) = os.path.split(dir)                              if file == "index.html": # Other cases to be treated as they come                                  source = f"{dir}{sep}{dir1}.md"                                  dest = f"{dir}{sep}{entry}{method}{groups[3]}^"                                  sources = fixed_cross_refs.get(dest, None)                                  if sources:                                      sources.add(source)                                  else:                                      fixed_cross_refs[dest] = {source}                          package = packages[0]                          orig_package = packages[1]                          new_content += html_content[last_location:xref.start()]                          new_content += f"{groups[0]}{rel_path}/{package}."                          if orig_package:                              new_content += f"{entry}"                              if method:                                  new_content += f"/index.html#{orig_package}.{entry}{method}\""                          else:                              new_content += f"{method}/"                          new_content += f"\"{groups[4]}"                          last_location = xref.end()                      if last_location:                          file_was_changed = True                          html_content = new_content + html_content[last_location:]                        # Finding xrefs in 'Parameters', 'Returns' and similar constructs.                      # These would be <typeName>^ fragments located in potentially                      # complex constructs such as "Optional[Union[str, <typeName>^]]"                      #                      # These fragments appear in single-line  blocks.                      #                      # At this point, this code is pretty sub-optimal and heavily                      # depends on the MkDocs generation. Time will tell if we can                      # keep it as is...                      typing_code = re.compile(r"(\s*)(.*\^.*)(\s*	)")                      # This will match a single <typeName>^ fragment.                      typing_type = re.compile(r"\w+\^")                      for xref in typing_code.finditer(html_content):                          groups = xref.groups()                          table_line = groups[1]                          table_line_to_replace = "".join(groups)                          new_table_line = table_line_to_replace                          typing_xref_found = False                            for type_ in typing_type.finditer(table_line):                              class_ = type_[0][:-1] # Remove ^                              packages = xrefs.get(class_)                              if packages:                                  # TODO - Retrieve the actual XREF                                  if isinstance(packages, int):                                      packages = xrefs.get(f"{class_}/0") # WRONG                                  typing_xref_found = True                                  new_content = f"{class_}"                                  new_table_line = new_table_line.replace(f"{class_}^", new_content)                          if typing_xref_found:                              file_was_changed = True                              html_content = html_content.replace(table_line_to_replace, new_table_line)                        # Replace data-source attributes in h<N> tags to links to                      # files in the appropriate repositores.                      process = process_data_source_attr(html_content, env)                      if process[0]:                          html_content = process[1]                          file_was_changed = True                      # Replace hrefs to GitHub containg [BRANCH] with proper branch name.                      process = process_links_to_github(html_content, env)                      if process[0]:                          html_content = process[1]                          file_was_changed = True                      # Shorten Table of contents in REST API files                      if "rest/apis_" in filename or "rest\\apis_" in filename:                          REST_TOC_ENTRY_RE = re.compile(r"(\s+)"                                                       + r"/api/v\d+(.*?)(?=\s+)")                          new_content = ""                          last_location = 0                          for toc_entry in REST_TOC_ENTRY_RE.finditer(html_content):                              new_content += html_content[last_location:toc_entry.start()]                              new_content += f"{toc_entry.group(1)}{toc_entry.group(2)}"                              last_location = toc_entry.end()                          if last_location:                              file_was_changed = True                              html_content = new_content + html_content[last_location:]                        # Rename the GUI Extension API type aliases                      elif "reference_guiext" in filename:                          for in_out in [("TaipyAction", "Action", "../interfaces/Action"),                                         ("TaipyContext", "Context", "#context")]:                              LINK_RE = re.compile(f"{in_out[0]}")                              new_content = ""                              last_location = 0                              for link in LINK_RE.finditer(html_content):                                  new_content += html_content[last_location:link.start()]                                  new_content += f"{in_out[1]}"                                  last_location = link.end()                              if last_location:                                  file_was_changed = True                                  html_content = new_content + html_content[last_location:]                        # Processing for visual element pages:                      # - Remove <tt> from title                      # - Add breadcrumbs to Taipy GUI's control, part and core element pages                      fn_match = re.search(r"(/|\\)gui\1(vis|cor)elements\1(.*?)\1index.html", filename)                      element_category = None                      if fn_match is not None:                          if title_match := re.search(r"<title><tt>(.*?)</tt> - Taipy</title>", html_content):                              html_content = (html_content[:title_match.start()]                                              + f"<title>{title_match.group(1)} - Taipy</title>"                                              + html_content[title_match.end():])                              file_was_changed = True                          if category_match := re.search(r"", html_content):                              element_category = category_match[1]                          elif re.match(r"^charts(/|\\).*$", fn_match[3]):                              element_category = "chart"                      if element_category:                          # Insert breadcrumbs                          ARTICLE_RE = re.compile(r"(
)(\s*<article)")                          if article_match := ARTICLE_RE.search(html_content):                              repl = "\n
"                              if fn_match[2] == "cor":                                  repl += f"
Visual Elements
"                                  repl += "
Scenario management controls
"                              else:                                  chart_part = "../" if element_category == "chart" else ""                                  repl += f"
Visual Elements
"                                  repl += (f"
Blocks
" if element_category == "blocks"                                          else f"
Standard controls
")                                  if chart_part:                                      repl += f"
Charts
"                              repl += "
"                              html_content = (html_content[:article_match.start()]                                              + article_match.group(1)                                              + repl                                              + article_match.group(2)                                              + html_content[article_match.end():])                              file_was_changed = True                      # Processing for the page builder API:                      fn_match = re.search(r"(/|\\)reference\1taipy\.gui\.builder.(.*?)\1index.html", filename)                      if fn_match is not None:                          element_name = fn_match[2]                          # Default value of properties appear as "dynamic(type" as "indexed(type"                          prop_re = re.compile(r"
\s*.*?	"                                             + r"\s*\s*(.*?)\s*	\s*",                                               re.S)                          new_content = ""                          last_location = 0                          for prop in prop_re.finditer(html_content):                              if default_value_re := re.match(r"(dynamic|indexed)\((.*)", prop[1]):                                  new_content += html_content[last_location:prop.start(1)]                                  new_content += f"{default_value_re[2]}
{default_value_re[1]}"                                  last_location = prop.end(1)                          if last_location:                              file_was_changed = True                              html_content = new_content + html_content[last_location:]                          # 'default value' -> default value                          dv_re = re.compile(r"&\#39;<i>(.*?)</i>&\#39;", re.S)                          new_content = ""                          last_location = 0                          for dv in dv_re.finditer(html_content):                              new_content += html_content[last_location:dv.start()] + f"{dv[1]}"                              last_location = dv.end()                          if last_location:                              file_was_changed = True                              html_content = new_content + html_content[last_location:]                      # Handle title and header in packages documentation file                      FONT_SIZE_CHANGE=''                      def code(s: str) -> str:                          return f"{s}"                        fn_match = re.search(r"manuals(/|\\)reference\1pkg_taipy\1index.html", filename)                      if fn_match is not None: # The root 'taipy' package                          html_content = re.sub(f"(
)taipy(
)", f"\\1{code('taipy')}\\2", html_content)                          file_was_changed = True                      fn_match = re.search(r"manuals(/|\\)reference\1pkg_taipy(\..*)\1index.html", filename)                      if fn_match is not None:                          pkg = fn_match[2]                          sub_match = re.search(r"(\.\w+)(\..*)", pkg)                          if sub_match is None:                              html_content = re.sub(f"(<title>)Index\\s", f"\\1taipy{pkg} ", html_content)                              html_content = re.sub(f"(
)Index(
)", f"\\1{code('taipy'+pkg)}\\2", html_content)                              html_content = re.sub(f"(
){pkg}(
)", f"\\1{code('taipy'+pkg)}\\2", html_content)                          else:                              html_content = re.sub(f"(<title>)({sub_match[2]})\\s", f"\\1taipy{sub_match[1]}\\2 ", html_content)                              html_content = re.sub(f"(
)(?:){sub_match[2]}(?:)(
)", f"\\1{code('taipy'+pkg)}\\2", html_content)                          file_was_changed = True                    if file_was_changed:                      with open(filename, "w") as html_file:                          html_file.write(html_content)              # Replace path to doc in '.ipynb' files              elif f.endswith(".ipynb"):                  filename = os.path.join(root, f)                  with open(filename) as ipynb_file:                      try:                          content = ipynb_file.read()                      except Exception as e:                          print(f"Couldn't read Notebook file {filename}")                          raise e                      (new_content, n) = re.subn("(?<=https://docs.taipy.io/en/)latest", f"{env.conf['branch']}", content)                      if n > 0:                          with open(filename, "w") as ipynb_file:                              ipynb_file.write(new_content)      if fixed_cross_refs:          for dest in sorted(fixed_cross_refs.keys()):              sources = ", ".join(sorted(fixed_cross_refs[dest]))              log.info(f"FIXED cross-ref to '{dest}' from {sources}")      # ################################################################################  # Functions that are used in the postprocessing phase.  # I wish we could move them elsewhere, but couldn't figure out how  # ################################################################################  def process_data_source_attr(html: str, env):      _DATASOURCE_RE = re.compile(r"(<(h\d+))\s+data-source=\"(.*?)\"(.*</\2>)")      new_content = ""      last_location = 0      for m in _DATASOURCE_RE.finditer(html):          ref = m.group(3)          repo_m = re.search(r"^([\w\d]+):", ref)          if repo_m:              ref = ("https://github.com/Avaiga/taipy-" +                     f"{repo_m.group(0)[:-1]}/blob/{env.conf['branch']}/{ref[repo_m.end():]}")          new_content += (html[last_location:m.start()]                          + f"{m.group(1)}{m.group(4)}"                          + f"\nYou can download the entire source code used in this "                          + f"section from the GitHub repository."                          )          last_location = m.end()      if last_location:          return (True, new_content + html[last_location:])      else:          return (False, None)      def process_links_to_github(html: str, env):      _LINK_RE = re.compile(r"(?<=href=\"https://github.com/Avaiga/)(taipy-gui/tree/)\[BRANCH\](.*?\")")      new_content = ""      last_location = 0      for m in _LINK_RE.finditer(html):          new_content += html[last_location:m.start()] + m.group(1) + env.conf['branch'] + m.group(2)          last_location = m.end()      if last_location:          return (True, new_content + html[last_location:])      else:          return (False, None)


import os  import re  import logging    SITE_DIR = "site"    def to_unix(p):      return p.replace("\\","/")  # Assuming that this script is located in <taipy-doc>/tools  tools_dir = os.path.dirname(__file__)  site_path = os.path.join(os.path.dirname(tools_dir), SITE_DIR)  ref_site_path = to_unix(site_path)    logger = logging.getLogger("taipy-doc-checker")  logging.basicConfig(level=logging.INFO)  logger.info(f"Site: {site_path}")  if not os.path.isdir(site_path):      logger.critical("Site does not exist. Please run 'mkdocs build' first.")      exit(1)    A_RE = re.compile(r"<a href=\"(.*?)\"")  for root, _, file_list in os.walk(site_path):      this_dir = to_unix(root)      for f in file_list:          if f.endswith(".html"):              filename = to_unix(os.path.join(root, f))              ref_filename = filename[len(ref_site_path):]              with open(filename) as html_file:                  ref_root = to_unix(root)                  try:                      html_content = html_file.read()                      for anchor in A_RE.finditer(html_content):                          href = anchor.group(1)                          if href.startswith("/en/develop/"):                              href = href[12:]                          if href != "." and href != ".." and not href.startswith("http") and not href.startswith("mailto"):                              if href.startswith("#"):                                  id=href[1:]                                  if not id in html_content:                                      logger.error(f"No id {id} in {ref_filename}")                              else:                                  if href.endswith("/"):                                      target = to_unix(os.path.join(ref_root, href))[:-1]                                      if not os.path.isdir(target) or not os.path.isfile(target + "/index.html"):                                          logger.error(f"Bad xref '{href}' in {ref_filename}")                  except Exception as e:                      logger.error(f"Couldn't read HTML file {filename}")                      raise e


"""  Check that parameters' documentation have a uniform format.    - `<param_name>`: [The |A |An ]*.  """  import sys    COMMENTS_CAN_START_WITH = ' The ', ' If ', ' An ', ' A ', ' TODO '    filename = sys.argv[1]    for line in open(filename).readlines():      if line.replace(' ', '').startswith('-`') and ':' in line:          try:              variable, comment = line.split(':', maxsplit=1)                assert variable.endswith(' ') is False              # Too early for this              # ------------------              # assert any(comment.startswith(x) for x in COMMENTS_CAN_START_WITH)              # if not comment.startswith(" TODO "):              #     assert comment[-1] == '.' or comment[-2] == '.'          except Exception as _:              print(f"Error in file '{filename}': '{line}'")              exit(1)


# ################################################################################  # Taipy REST API Reference Manual generation setup step.  #  # A Taipy REST server is created and the API is queries dynamically.  # Then documentation pages are generated from the retrieved structure.  # ################################################################################  from .setup import Setup, SetupStep      class RestRefManStep(SetupStep):      def __init__(self):          self.navigation = None        def get_id(self) -> str:          return "rest"        def get_description(self) -> str:          return "Generation of the REST API Reference Manual pages"        def enter(self, setup: Setup):          self.navigation = None        def setup(self, setup: Setup) -> None:          setup.move_package_to_tools(Setup.ROOT_PACKAGE)          try:              self.generate_rest_refman_pages(setup)          except Exception as e:              raise e          finally:              setup.move_package_to_root(Setup.ROOT_PACKAGE)        def generate_rest_refman_pages(self, setup: Setup):          REST_REF_DIR_PATH = setup.manuals_dir + "/reference_rest"          from taipy.rest.app import create_app as rest_create_app            app = rest_create_app()            from taipy.rest.extensions import apispec as rest_apispec            with app.app_context():              rest_specs = rest_apispec.swagger_json().json            self.navigation = ""            path_descs = {              "cycle": "Entry points that interact with `Cycle^` entities.",              "scenario": "Entry points that interact with `Scenario^` entities.",              "sequence": "Entry points that interact with `Sequence^` entities.",              "datanode": "Entry points that interact with `DataNode^` entities.",              "task": "Entry points that interact with `Task^` entities.",              "job": "Entry points that interact with `Job^` entities.",              "auth": "Entry points that deal with authentication.",          }          path_files = {}          enterprise_paths = ["auth"]            response_statuses = {              "200": ("OK", "https://httpwg.org/specs/rfc7231.html#status.200"),              "201": ("Created", "https://httpwg.org/specs/rfc7231.html#status.201"),              "202": ("Accepted", "https://httpwg.org/specs/rfc7231.html#status.202"),              "204": ("No Content", "https://httpwg.org/specs/rfc7231.html#status.204"),              "401": ("Unauthorized", "https://httpwg.org/specs/rfc7235.html#status.401"),              "404": ("Not Found", "https://httpwg.org/specs/rfc7231.html#status.404"),          }            def get_property_type(property_desc, from_schemas):              type_name = property_desc.get("type", None)              a_pre = ""              a_post = ""              if type_name == "array":                  property_desc = property_desc["items"]                  type_name = property_desc.get("type", None)                  a_pre = "\\["                  a_post = "\\]"              if type_name is None:                  type_name = property_desc.get("$ref", None)                  if type_name is None:                      # print(f"WARNING - No type...")                      return ""                  type_name = type_name[type_name.rfind("/") + 1 :]                  prefix = "" if from_schemas else "schemas.md"                  type_name = f"[`{type_name}`]({prefix}#{type_name.lower()})"              return f"{a_pre}{type_name}{a_post}"            for path, desc in rest_specs["paths"].items():              dest_path = None              dest_path_desc = None              for p, d in path_descs.items():                  if f"/{p}" in path:                      dest_path = p                      dest_path_desc = d                      break              if dest_path is None:                  print(f"Path {path} has no paths bucket")                  continue                file = path_files.get(dest_path)              if file is None:                  file = open(f"{REST_REF_DIR_PATH}/apis_{dest_path}.md", "w")                  if dest_path in enterprise_paths:                      file.write(Setup.ENTERPRISE_BANNER)                  file.write(f"{dest_path_desc}\n\n")                  self.navigation += f"- \"Paths for {dest_path}\": manuals/reference_rest/apis_{dest_path}.md\n"                  path_files[dest_path] = file                file.write(f"# `{path}`\n\n")              for method, method_desc in desc.items():                  file.write(f"## {method.upper()}\n\n")                  file.write(f"{method_desc['description']}\n\n")                  parameters = method_desc.get("parameters")                  if parameters is not None:                      file.write(f"
Parameters
\n\n")                      file.write("|Name|Type|Required|Description|\n")                      file.write("|---|---|---|---|\n")                      for param in parameters:                          param_type = "TODO"                          if "schema" in param and len(param["schema"]) == 1:                              param_type = param["schema"]["type"]                          file.write(                              f"|{param['name']}|{param_type}|{'Yes' if param['name'] else 'No'}|{param.get('description', '-')}|\n"                          )                      file.write("\n")                  request_body = method_desc.get("requestBody")                  if request_body is not None:                      file.write(f'

Request body:
\n')                      schema = request_body["content"]["application/json"]["schema"]["$ref"]                      schema = schema[schema.rfind("/") + 1 :]                      file.write(f'{schema}
\n\n')                  responses = method_desc.get("responses")                  if responses is not None:                      file.write(f"
Responses
\n\n")                      for response, response_desc in responses.items():                          file.write(f"- {response}
\n")                          status = response_statuses[response]                          file.write(f"    Status: [`{status[0]}`]({status[1]})\n\n")                          description = response_desc.get("description")                          if description is not None:                              file.write(f"    {description}\n")                          content = response_desc.get("content", None)                          if content is not None:                              schema = content["application/json"]["schema"]                              allOf = schema.get("allOf")                              if allOf is not None:                                  schema = allOf[0]                              properties = schema.get("properties")                              if properties is not None:                                  file.write("    |Name|Type|\n")                                  file.write("    |---|---|\n")                                  for property, property_desc in properties.items():                                      file.write(f"    |{property}|{get_property_type(property_desc, False)}|\n")                              else:                                  print(f"No properties in content in response {response} in method {method} of {path}")                          elif response != "404":                              print(f"No content in response {response} in method {method} of {path}")                          file.write("\n")            for f in path_files.values():              f.close()            with open(f"{REST_REF_DIR_PATH}/schemas.md", "w") as file:              for schema, schema_desc in rest_specs["components"]["schemas"].items():                  properties = schema_desc.get("properties")                  if properties is not None:                      file.write(f"## {schema}\n\n")                      file.write("|Name|Type|Description|\n")                      file.write("|---|---|---|\n")                      for property, property_desc in properties.items():                          file.write(f"|{property}|{get_property_type(property_desc, True)}|-|\n")                      file.write("\n")          self.navigation += "- \"Schemas\": manuals/reference_rest/schemas.md\n"          def exit(self, setup: Setup):          setup.update_mkdocs_yaml_template(              r"^\s*\[REST_REFERENCE_CONTENT\]\s*\n", self.navigation if self.navigation else ""          )


from .setup import run_setup


# ################################################################################  # Taipy GUI Extension Reference Manual generation setup step.  #  # The Reference Manual pages for the Taipy GUI JavaScript extension module  # is generated using the typedoc tools, directly from the JavaScript  # source files.  # ################################################################################  from .setup import Setup, SetupStep  import os  import re  import shutil  import subprocess      class GuiExtRefManStep(SetupStep):      def get_id(self) -> str:          return "guiext"        def get_description(self) -> str:          return "Generating the GUI Extension API documentation"        def enter(self, setup: Setup):          self.GUI_EXT_REF_DIR_PATH = setup.manuals_dir + "/reference_guiext"          if os.path.exists(self.GUI_EXT_REF_DIR_PATH):              shutil.rmtree(self.GUI_EXT_REF_DIR_PATH)          npm_path = shutil.which("npm")          if npm_path:              if " " in npm_path:                  npm_path = f"\"{npm_path}\""              try:                  subprocess.run(f"{npm_path} --version", shell=True, capture_output=True)              except OSError:                  print(f"WARNING: Couldn't run npm, ignoring this step.", flush=True)                  npm_path = None          self.npm_path = npm_path        def setup(self, setup: Setup) -> None:          if self.npm_path:              saved_cwd = os.getcwd()              gui_path = os.path.join(setup.root_dir, "gui")              os.chdir(gui_path)              print(f"... Installing node modules...", flush=True)              subprocess.run(f"{self.npm_path} run inst", shell=True)              print(f"... Generating documentation...", flush=True)              subprocess.run(f"{self.npm_path} run mkdocs", shell=True)              # Process and copy files to docs/manuals              os.mkdir(self.GUI_EXT_REF_DIR_PATH)              dst_dir = os.path.abspath(self.GUI_EXT_REF_DIR_PATH)              src_dir = os.path.abspath(os.path.join(gui_path, "reference_guiext"))              JS_EXT_RE = re.compile(r"^(.*?)(\n#.*?\n)", re.MULTILINE | re.DOTALL)              for root, dirs, files in os.walk(src_dir):                  for file in files:                      file_content = None                      path_name = os.path.join(root, file)                      with open(path_name) as input:                          file_content = input.read()                      match = JS_EXT_RE.search(file_content)                      if match:                          file_content = match.group(2) + match.group(1) + file_content[match.end():]                      path_name = path_name.replace(src_dir, dst_dir)                      with open(path_name, "w") as output:                          output.write(file_content)                  for dir in dirs:                      os.mkdir(os.path.join(dst_dir, dir))              os.chdir(saved_cwd)