def run_arxiv_cleaner(parameters):
  """Core of the code, runs the actual arXiv cleaner."""

  files_to_delete = [
      r'\.aux$',
      r'\.sh$',
      r'\.blg$',
      r'\.brf$',
      r'\.log$',
      r'\.out$',
      r'\.ps$',
      r'\.dvi$',
      r'\.synctex.gz$',
      '~$',
      r'\.backup$',
      r'\.gitignore$',
      r'\.DS_Store$',
      r'\.svg$',
      r'^\.idea',
      r'\.dpth$',
      r'\.md5$',
      r'\.dep$',
      r'\.auxlock$',
      r'\.fls$',
      r'\.fdb_latexmk$',
  ]

  if not parameters['keep_bib']:
    files_to_delete.append(r'\.bib$')

  parameters.update({
      'to_delete': files_to_delete,
      'figures_to_copy_if_referenced': [
          r'\.png$',
          r'\.jpg$',
          r'\.jpeg$',
          r'\.pdf$',
      ],
  })

  logging.info('Collecting file structure.')
  parameters['output_folder'] = _create_out_folder(parameters['input_folder'])

  from_zip = parameters['input_folder'].endswith('.zip')
  tempdir_context = (
      tempfile.TemporaryDirectory() if from_zip else contextlib.suppress()
  )

  with tempdir_context as tempdir:

    if from_zip:
      logging.info('Unzipping input folder.')
      shutil.unpack_archive(parameters['input_folder'], tempdir)
      parameters['input_folder'] = tempdir

    splits = _split_all_files(parameters)

    logging.info('Reading all tex files')
    tex_contents = _read_all_tex_contents(
        splits['tex_in_root'] + splits['tex_not_in_root'], parameters
    )

    for tex_file in tex_contents:
      logging.info('Removing comments in file %s.', tex_file)
      tex_contents[tex_file] = _remove_comments_and_commands_to_delete(
          tex_contents[tex_file], parameters
      )

    for tex_file in tex_contents:
      logging.info('Replacing \\includesvg calls in file %s.', tex_file)
      tex_contents[tex_file] = _replace_includesvg(
          tex_contents[tex_file], splits['svg_inkscape']
      )

    for tex_file in tex_contents:
      logging.info('Replacing Tikz Pictures in file %s.', tex_file)
      content = _replace_tikzpictures(
          tex_contents[tex_file], splits['external_tikz_figures']
      )
      # If file ends with '\n' already, the split in last line would add an extra
      # '\n', so we remove it.
      tex_contents[tex_file] = content.split('\n')

    _keep_only_referenced_tex(tex_contents, splits)
    _add_root_tex_files(splits)

    for tex_file in splits['tex_to_copy']:
      logging.info('Replacing patterns in file %s.', tex_file)
      content = '\n'.join(tex_contents[tex_file])
      content = _find_and_replace_patterns(
          content, parameters.get('patterns_and_insertions', list())
      )
      tex_contents[tex_file] = content
      new_path = os.path.join(parameters['output_folder'], tex_file)
      logging.info('Writing modified contents to %s.', new_path)
      _write_file_content(
          content,
          new_path,
      )

    full_content = '\n'.join(
        ''.join(tex_contents[fn]) for fn in splits['tex_to_copy']
    )
    _copy_only_referenced_non_tex_not_in_root(parameters, full_content, splits)
    for non_tex_file in splits['non_tex_in_root']:
      logging.info('Copying non-tex file %s.', non_tex_file)
      _copy_file(non_tex_file, parameters)

    _resize_and_copy_figures_if_referenced(parameters, full_content, splits)
    logging.info('Outputs written to %s', parameters['output_folder'])

run_arxiv_cleaner(parameters={'input_folder': 'tex', 'images_allowlist': {'images/im2_included.jpg': 200, 'images/im3_included.png': 400}, 'resize_images': True, 'im_size': 100, 'compress_pdf': False, 'pdf_im_resolution': 500, 'commands_to_delete': ['mytodo'], 'commands_only_to_delete': ['red'], 'environments_to_delete': ['mynote'], 'use_external_tikz': 'ext_tikz', 'keep_bib': False})

{'main.tex': ['\\begin{document}', 'Text', '', 'Text%', '', '', 'This is a percent \\%.', '\\includegraphics{images/im1_included.png}', '\\includegraphics{images/im3_included.png}', '\\includegraphics{%', '  images/im4_included.png%', '  }', '\\includegraphics[width=.5\\linewidth]{%', '  images/im5_included.jpg}', '', '\\includegraphics{./images/im3_included.png}', '', 'This line should not be separated', '%', 'from this one.', '', '', '', '\\newif\\ifvar', '', '\\ifvar', '\\fi', '', '\\newcommand{\\red}[1]{{\\color{red} #1}}', 'hello test hello', 'test hello', 'test', '', '', '\\input{figures/figure_included.tex}', '', '\\includegraphics{ext_tikz/test1.pdf}', '', '\\input{figures/figure_included.tikz}', '', '\\begin{tikzpicture}', '    \\node (test) at (0,0) {Test3};', '\\end{tikzpicture}', '', '\\tikzsetnextfilename{test_no_match}', '\\begin{tikzpicture}', '    \\node (test) at (0,0) {Test4};', '\\end{tikzpicture}', '', '\\end{document}', ''], 'figures/figure_not_included.tex': ['\\addplot{figures/data_not_included.txt}', '\\input{figures/figure_not_included_2.tex}', ''], 'figures/figure_not_included_2.tex': [''], 'figures/figure_included.tikz': ['\ufeff\\includegraphics{ext_tikz/test2.pdf}', ''], 'figures/figure_included.tex': ['\\includegraphics{images/im2_included.jpg}', '\\addplot{figures/data_included.txt}', '']}

{'all': ['main.bib', 'main.bbl', 'main.tex', 'main.aux', 'ext_tikz/test2.pdf', 'ext_tikz/test1.pdf', 'ext_tikz/figure_not_included.pdf', 'figures/data_not_included.txt', 'figures/figure_not_included.tex', 'figures/figure_not_included_2.tex', 'figures/figure_included.tikz', 'figures/figure_included.tex', 'figures/data_included.txt', 'not_included/figures/data_included.txt', 'images/im4_included.png', 'images/im1.png', 'images/im4_not_included.png', 'images/im3_included.png', 'images/im2_included.jpg', 'images/im5_not_included.jpg', 'images/im5_included.jpg', 'images/im1_included.png', 'images/im_not_included.png', 'images/include/images/im3_included.png'], 'in_root': ['main.bib', 'main.bbl', 'main.tex', 'main.aux'], 'not_in_root': ['ext_tikz/test2.pdf', 'ext_tikz/test1.pdf', 'ext_tikz/figure_not_included.pdf', 'figures/data_not_included.txt', 'figures/figure_not_included.tex', 'figures/figure_not_included_2.tex', 'figures/figure_included.tikz', 'figures/figure_included.tex', 'figures/data_included.txt', 'not_included/figures/data_included.txt', 'images/im4_included.png', 'images/im1.png', 'images/im4_not_included.png', 'images/im3_included.png', 'images/im2_included.jpg', 'images/im5_not_included.jpg', 'images/im5_included.jpg', 'images/im1_included.png', 'images/im_not_included.png', 'images/include/images/im3_included.png'], 'to_copy_in_root': ['main.bbl', 'main.tex'], 'to_copy_not_in_root': ['figures/data_not_included.txt', 'figures/figure_not_included.tex', 'figures/figure_not_included_2.tex', 'figures/figure_included.tikz', 'figures/figure_included.tex', 'figures/data_included.txt', 'not_included/figures/data_included.txt'], 'figures': ['ext_tikz/test2.pdf', 'ext_tikz/test1.pdf', 'ext_tikz/figure_not_included.pdf', 'images/im4_included.png', 'images/im1.png', 'images/im4_not_included.png', 'images/im3_included.png', 'images/im2_included.jpg', 'images/im5_not_included.jpg', 'images/im5_included.jpg', 'images/im1_included.png', 'images/im_not_included.png', 'images/include/images/im3_included.png'], 'tex_in_root': ['main.tex'], 'tex_not_in_root': ['figures/figure_not_included.tex', 'figures/figure_not_included_2.tex', 'figures/figure_included.tikz', 'figures/figure_included.tex'], 'non_tex_in_root': ['main.bbl'], 'non_tex_not_in_root': ['figures/data_not_included.txt', 'figures/data_included.txt', 'not_included/figures/data_included.txt'], 'external_tikz_figures': ['ext_tikz/test2.pdf', 'ext_tikz/test1.pdf', 'ext_tikz/figure_not_included.pdf'], 'svg_inkscape': []}

{'all': ['main.bib', 'main.bbl', 'main.tex', 'main.aux', 'ext_tikz/test2.pdf', 'ext_tikz/test1.pdf', 'ext_tikz/figure_not_included.pdf', 'figures/data_not_included.txt', 'figures/figure_not_included.tex', 'figures/figure_not_included_2.tex', 'figures/figure_included.tikz', 'figures/figure_included.tex', 'figures/data_included.txt', 'not_included/figures/data_included.txt', 'images/im4_included.png', 'images/im1.png', 'images/im4_not_included.png', 'images/im3_included.png', 'images/im2_included.jpg', 'images/im5_not_included.jpg', 'images/im5_included.jpg', 'images/im1_included.png', 'images/im_not_included.png', 'images/include/images/im3_included.png'], 'in_root': ['main.bib', 'main.bbl', 'main.tex', 'main.aux'], 'not_in_root': ['ext_tikz/test2.pdf', 'ext_tikz/test1.pdf', 'ext_tikz/figure_not_included.pdf', 'figures/data_not_included.txt', 'figures/figure_not_included.tex', 'figures/figure_not_included_2.tex', 'figures/figure_included.tikz', 'figures/figure_included.tex', 'figures/data_included.txt', 'not_included/figures/data_included.txt', 'images/im4_included.png', 'images/im1.png', 'images/im4_not_included.png', 'images/im3_included.png', 'images/im2_included.jpg', 'images/im5_not_included.jpg', 'images/im5_included.jpg', 'images/im1_included.png', 'images/im_not_included.png', 'images/include/images/im3_included.png'], 'to_copy_in_root': ['main.bbl', 'main.tex'], 'to_copy_not_in_root': ['figures/data_not_included.txt', 'figures/figure_not_included.tex', 'figures/figure_not_included_2.tex', 'figures/figure_included.tikz', 'figures/figure_included.tex', 'figures/data_included.txt', 'not_included/figures/data_included.txt'], 'figures': ['ext_tikz/test2.pdf', 'ext_tikz/test1.pdf', 'ext_tikz/figure_not_included.pdf', 'images/im4_included.png', 'images/im1.png', 'images/im4_not_included.png', 'images/im3_included.png', 'images/im2_included.jpg', 'images/im5_not_included.jpg', 'images/im5_included.jpg', 'images/im1_included.png', 'images/im_not_included.png', 'images/include/images/im3_included.png'], 'tex_in_root': ['main.tex'], 'tex_not_in_root': ['figures/figure_not_included.tex', 'figures/figure_not_included_2.tex', 'figures/figure_included.tikz', 'figures/figure_included.tex'], 'non_tex_in_root': ['main.bbl'], 'non_tex_not_in_root': ['figures/data_not_included.txt', 'figures/data_included.txt', 'not_included/figures/data_included.txt'], 'external_tikz_figures': ['ext_tikz/test2.pdf', 'ext_tikz/test1.pdf', 'ext_tikz/figure_not_included.pdf'], 'svg_inkscape': [], 'tex_to_copy': ['figures/figure_included.tex', 'figures/figure_included.tikz', 'main.tex']}

{'input_folder': 'tex', 'images_allowlist': {'images/im2_included.jpg': 200, 'images/im3_included.png': 400}, 'resize_images': True, 'im_size': 100, 'compress_pdf': False, 'pdf_im_resolution': 500, 'commands_to_delete': ['mytodo'], 'commands_only_to_delete': ['red'], 'environments_to_delete': ['mynote'], 'use_external_tikz': 'ext_tikz', 'keep_bib': False, 'to_delete': ['\\.aux$', '\\.sh$', '\\.blg$', '\\.brf$', '\\.log$', '\\.out$', '\\.ps$', '\\.dvi$', '\\.synctex.gz$', '~$', '\\.backup$', '\\.gitignore$', '\\.DS_Store$', '\\.svg$', '^\\.idea', '\\.dpth$', '\\.md5$', '\\.dep$', '\\.auxlock$', '\\.fls$', '\\.fdb_latexmk$', '\\.bib$'], 'figures_to_copy_if_referenced': ['\\.png$', '\\.jpg$', '\\.jpeg$', '\\.pdf$']}

{'input_folder': 'tex', 'images_allowlist': {'images/im2_included.jpg': 200, 'images/im3_included.png': 400}, 'resize_images': True, 'im_size': 100, 'compress_pdf': False, 'pdf_im_resolution': 500, 'commands_to_delete': ['mytodo'], 'commands_only_to_delete': ['red'], 'environments_to_delete': ['mynote'], 'use_external_tikz': 'ext_tikz', 'keep_bib': False, 'to_delete': ['\\.aux$', '\\.sh$', '\\.blg$', '\\.brf$', '\\.log$', '\\.out$', '\\.ps$', '\\.dvi$', '\\.synctex.gz$', '~$', '\\.backup$', '\\.gitignore$', '\\.DS_Store$', '\\.svg$', '^\\.idea', '\\.dpth$', '\\.md5$', '\\.dep$', '\\.auxlock$', '\\.fls$', '\\.fdb_latexmk$', '\\.bib$'], 'figures_to_copy_if_referenced': ['\\.png$', '\\.jpg$', '\\.jpeg$', '\\.pdf$'], 'output_folder': '/local/rcs/jinjun/code/pytrace-collector/logs/self_collected/tried/google-research+arxiv-latex-cleaner/google-research+arxiv-latex-cleaner/tex_arXiv'}

{'main.tex': ['\\begin{document}\n', 'Text\n', '% Whole line comment\n', '\n', 'Text% Inline comment\n', '\\begin{comment}\n', 'This is an environment comment.\n', '\\end{comment}\n', '\n', 'This is a percent \\%.\n', '% Whole line comment without newline\n', '\\includegraphics{images/im1_included.png}\n', '%\\includegraphics{images/im_not_included}\n', '\\includegraphics{images/im3_included.png}\n', '\\includegraphics{%\n', '  images/im4_included.png%\n', '  }\n', '\\includegraphics[width=.5\\linewidth]{%\n', '  images/im5_included.jpg}\n', '%\\includegraphics{%\n', '%  images/im4_not_included.png\n', '%  }\n', '%\\includegraphics[width=.5\\linewidth]{%\n', '%  images/im5_not_included.jpg}\n', '\n', '% test whatever the path satrting with dot works when include graphics\n', '\\includegraphics{./images/im3_included.png}\n', '\n', 'This line should\\mytodo{Do this later} not be separated\n', '\\mytodo{This is a todo command with a nested \\textit{command}.\n', 'Please remember that up to \\texttt{2 levels} of \\textit{nesting} are supported.}\n', 'from this one.\n', '\n', '\\begin{mynote}\n', '  This is a custom environment that could be excluded.\n', '\\end{mynote}\n', '\n', '\\newif\\ifvar\n', '\n', '\\ifvar\n', '\\if    false\n', '\\if false\n', '\\if 0\n', '\\iffalse\n', '\\ifvar\n', 'Text\n', '\\fi\n', '\\fi\n', '\\fi\n', '\\fi\n', '\\fi\n', '\\fi\n', '\n', '\\newcommand{\\red}[1]{{\\color{red} #1}}\n', 'hello test \\red{hello\n', 'test \\red{hello}}\n', 'test\n', '\n', '% content after this line should not be cleaned if \\end{document} is in a comment\n', '\n', '\\input{figures/figure_included.tex}\n', '% \\input{figures/figure_not_included.tex}\n', '\n', '% Test for tikzpicture feature\n', '% should be replaced\n', '\\tikzsetnextfilename{test1}\n', '\\begin{tikzpicture}\n', '    \\node (test) at (0,0) {Test1};\n', '\\end{tikzpicture}\n', '\n', '% should be replaced in included file\n', '\\input{figures/figure_included.tikz}\n', '\n', '% should not be be replaced - no preceding tikzsetnextfilename command\n', '\\begin{tikzpicture}\n', '    \\node (test) at (0,0) {Test3};\n', '\\end{tikzpicture}\n', '\n', '\\tikzsetnextfilename{test_no_match}\n', '\\begin{tikzpicture}\n', '    \\node (test) at (0,0) {Test4};\n', '\\end{tikzpicture}\n', '\n', '\\end{document}\n'], 'figures/figure_not_included.tex': ['\\addplot{figures/data_not_included.txt}\n', '\\input{figures/figure_not_included_2.tex}\n'], 'figures/figure_not_included_2.tex': [], 'figures/figure_included.tikz': ['\ufeff\\tikzsetnextfilename{test2}\n', '\\begin{tikzpicture}\n', '\\node {root}\n', 'child {node {left}}\n', 'child {node {right}\n', 'child {node {child}}\n', 'child {node {child}}\n', '};\n', '\\end{tikzpicture}'], 'figures/figure_included.tex': ['\\includegraphics{images/im2_included.jpg}\n', '\\addplot{figures/data_included.txt}\n']}

{'main.tex': '\\begin{document}\nText\n\nText%\n\n\nThis is a percent \\%.\n\\includegraphics{images/im1_included.png}\n\\includegraphics{images/im3_included.png}\n\\includegraphics{%\n  images/im4_included.png%\n  }\n\\includegraphics[width=.5\\linewidth]{%\n  images/im5_included.jpg}\n\n\\includegraphics{./images/im3_included.png}\n\nThis line should not be separated\n%\nfrom this one.\n\n\n\n\\newif\\ifvar\n\n\\ifvar\n\\fi\n\n\\newcommand{\\red}[1]{{\\color{red} #1}}\nhello test hello\ntest hello\ntest\n\n\n\\input{figures/figure_included.tex}\n\n\\tikzsetnextfilename{test1}\n\\begin{tikzpicture}\n    \\node (test) at (0,0) {Test1};\n\\end{tikzpicture}\n\n\\input{figures/figure_included.tikz}\n\n\\begin{tikzpicture}\n    \\node (test) at (0,0) {Test3};\n\\end{tikzpicture}\n\n\\tikzsetnextfilename{test_no_match}\n\\begin{tikzpicture}\n    \\node (test) at (0,0) {Test4};\n\\end{tikzpicture}\n\n\\end{document}\n', 'figures/figure_not_included.tex': ['\\addplot{figures/data_not_included.txt}\n', '\\input{figures/figure_not_included_2.tex}\n'], 'figures/figure_not_included_2.tex': [], 'figures/figure_included.tikz': ['\ufeff\\tikzsetnextfilename{test2}\n', '\\begin{tikzpicture}\n', '\\node {root}\n', 'child {node {left}}\n', 'child {node {right}\n', 'child {node {child}}\n', 'child {node {child}}\n', '};\n', '\\end{tikzpicture}'], 'figures/figure_included.tex': ['\\includegraphics{images/im2_included.jpg}\n', '\\addplot{figures/data_included.txt}\n']}

{'main.tex': '\\begin{document}\nText\n\nText%\n\n\nThis is a percent \\%.\n\\includegraphics{images/im1_included.png}\n\\includegraphics{images/im3_included.png}\n\\includegraphics{%\n  images/im4_included.png%\n  }\n\\includegraphics[width=.5\\linewidth]{%\n  images/im5_included.jpg}\n\n\\includegraphics{./images/im3_included.png}\n\nThis line should not be separated\n%\nfrom this one.\n\n\n\n\\newif\\ifvar\n\n\\ifvar\n\\fi\n\n\\newcommand{\\red}[1]{{\\color{red} #1}}\nhello test hello\ntest hello\ntest\n\n\n\\input{figures/figure_included.tex}\n\n\\tikzsetnextfilename{test1}\n\\begin{tikzpicture}\n    \\node (test) at (0,0) {Test1};\n\\end{tikzpicture}\n\n\\input{figures/figure_included.tikz}\n\n\\begin{tikzpicture}\n    \\node (test) at (0,0) {Test3};\n\\end{tikzpicture}\n\n\\tikzsetnextfilename{test_no_match}\n\\begin{tikzpicture}\n    \\node (test) at (0,0) {Test4};\n\\end{tikzpicture}\n\n\\end{document}\n', 'figures/figure_not_included.tex': '\\addplot{figures/data_not_included.txt}\n\\input{figures/figure_not_included_2.tex}\n', 'figures/figure_not_included_2.tex': [], 'figures/figure_included.tikz': ['\ufeff\\tikzsetnextfilename{test2}\n', '\\begin{tikzpicture}\n', '\\node {root}\n', 'child {node {left}}\n', 'child {node {right}\n', 'child {node {child}}\n', 'child {node {child}}\n', '};\n', '\\end{tikzpicture}'], 'figures/figure_included.tex': ['\\includegraphics{images/im2_included.jpg}\n', '\\addplot{figures/data_included.txt}\n']}

{'main.tex': '\\begin{document}\nText\n\nText%\n\n\nThis is a percent \\%.\n\\includegraphics{images/im1_included.png}\n\\includegraphics{images/im3_included.png}\n\\includegraphics{%\n  images/im4_included.png%\n  }\n\\includegraphics[width=.5\\linewidth]{%\n  images/im5_included.jpg}\n\n\\includegraphics{./images/im3_included.png}\n\nThis line should not be separated\n%\nfrom this one.\n\n\n\n\\newif\\ifvar\n\n\\ifvar\n\\fi\n\n\\newcommand{\\red}[1]{{\\color{red} #1}}\nhello test hello\ntest hello\ntest\n\n\n\\input{figures/figure_included.tex}\n\n\\tikzsetnextfilename{test1}\n\\begin{tikzpicture}\n    \\node (test) at (0,0) {Test1};\n\\end{tikzpicture}\n\n\\input{figures/figure_included.tikz}\n\n\\begin{tikzpicture}\n    \\node (test) at (0,0) {Test3};\n\\end{tikzpicture}\n\n\\tikzsetnextfilename{test_no_match}\n\\begin{tikzpicture}\n    \\node (test) at (0,0) {Test4};\n\\end{tikzpicture}\n\n\\end{document}\n', 'figures/figure_not_included.tex': '\\addplot{figures/data_not_included.txt}\n\\input{figures/figure_not_included_2.tex}\n', 'figures/figure_not_included_2.tex': '', 'figures/figure_included.tikz': ['\ufeff\\tikzsetnextfilename{test2}\n', '\\begin{tikzpicture}\n', '\\node {root}\n', 'child {node {left}}\n', 'child {node {right}\n', 'child {node {child}}\n', 'child {node {child}}\n', '};\n', '\\end{tikzpicture}'], 'figures/figure_included.tex': ['\\includegraphics{images/im2_included.jpg}\n', '\\addplot{figures/data_included.txt}\n']}

{'main.tex': '\\begin{document}\nText\n\nText%\n\n\nThis is a percent \\%.\n\\includegraphics{images/im1_included.png}\n\\includegraphics{images/im3_included.png}\n\\includegraphics{%\n  images/im4_included.png%\n  }\n\\includegraphics[width=.5\\linewidth]{%\n  images/im5_included.jpg}\n\n\\includegraphics{./images/im3_included.png}\n\nThis line should not be separated\n%\nfrom this one.\n\n\n\n\\newif\\ifvar\n\n\\ifvar\n\\fi\n\n\\newcommand{\\red}[1]{{\\color{red} #1}}\nhello test hello\ntest hello\ntest\n\n\n\\input{figures/figure_included.tex}\n\n\\tikzsetnextfilename{test1}\n\\begin{tikzpicture}\n    \\node (test) at (0,0) {Test1};\n\\end{tikzpicture}\n\n\\input{figures/figure_included.tikz}\n\n\\begin{tikzpicture}\n    \\node (test) at (0,0) {Test3};\n\\end{tikzpicture}\n\n\\tikzsetnextfilename{test_no_match}\n\\begin{tikzpicture}\n    \\node (test) at (0,0) {Test4};\n\\end{tikzpicture}\n\n\\end{document}\n', 'figures/figure_not_included.tex': '\\addplot{figures/data_not_included.txt}\n\\input{figures/figure_not_included_2.tex}\n', 'figures/figure_not_included_2.tex': '', 'figures/figure_included.tikz': '\ufeff\\tikzsetnextfilename{test2}\n\\begin{tikzpicture}\n\\node {root}\nchild {node {left}}\nchild {node {right}\nchild {node {child}}\nchild {node {child}}\n};\n\\end{tikzpicture}\n', 'figures/figure_included.tex': ['\\includegraphics{images/im2_included.jpg}\n', '\\addplot{figures/data_included.txt}\n']}

{'main.tex': '\\begin{document}\nText\n\nText%\n\n\nThis is a percent \\%.\n\\includegraphics{images/im1_included.png}\n\\includegraphics{images/im3_included.png}\n\\includegraphics{%\n  images/im4_included.png%\n  }\n\\includegraphics[width=.5\\linewidth]{%\n  images/im5_included.jpg}\n\n\\includegraphics{./images/im3_included.png}\n\nThis line should not be separated\n%\nfrom this one.\n\n\n\n\\newif\\ifvar\n\n\\ifvar\n\\fi\n\n\\newcommand{\\red}[1]{{\\color{red} #1}}\nhello test hello\ntest hello\ntest\n\n\n\\input{figures/figure_included.tex}\n\n\\tikzsetnextfilename{test1}\n\\begin{tikzpicture}\n    \\node (test) at (0,0) {Test1};\n\\end{tikzpicture}\n\n\\input{figures/figure_included.tikz}\n\n\\begin{tikzpicture}\n    \\node (test) at (0,0) {Test3};\n\\end{tikzpicture}\n\n\\tikzsetnextfilename{test_no_match}\n\\begin{tikzpicture}\n    \\node (test) at (0,0) {Test4};\n\\end{tikzpicture}\n\n\\end{document}\n', 'figures/figure_not_included.tex': '\\addplot{figures/data_not_included.txt}\n\\input{figures/figure_not_included_2.tex}\n', 'figures/figure_not_included_2.tex': '', 'figures/figure_included.tikz': '\ufeff\\tikzsetnextfilename{test2}\n\\begin{tikzpicture}\n\\node {root}\nchild {node {left}}\nchild {node {right}\nchild {node {child}}\nchild {node {child}}\n};\n\\end{tikzpicture}\n', 'figures/figure_included.tex': '\\includegraphics{images/im2_included.jpg}\n\\addplot{figures/data_included.txt}\n'}

{'main.tex': ['\\begin{document}', 'Text', '', 'Text%', '', '', 'This is a percent \\%.', '\\includegraphics{images/im1_included.png}', '\\includegraphics{images/im3_included.png}', '\\includegraphics{%', '  images/im4_included.png%', '  }', '\\includegraphics[width=.5\\linewidth]{%', '  images/im5_included.jpg}', '', '\\includegraphics{./images/im3_included.png}', '', 'This line should not be separated', '%', 'from this one.', '', '', '', '\\newif\\ifvar', '', '\\ifvar', '\\fi', '', '\\newcommand{\\red}[1]{{\\color{red} #1}}', 'hello test hello', 'test hello', 'test', '', '', '\\input{figures/figure_included.tex}', '', '\\includegraphics{ext_tikz/test1.pdf}', '', '\\input{figures/figure_included.tikz}', '', '\\begin{tikzpicture}', '    \\node (test) at (0,0) {Test3};', '\\end{tikzpicture}', '', '\\tikzsetnextfilename{test_no_match}', '\\begin{tikzpicture}', '    \\node (test) at (0,0) {Test4};', '\\end{tikzpicture}', '', '\\end{document}', ''], 'figures/figure_not_included.tex': '\\addplot{figures/data_not_included.txt}\n\\input{figures/figure_not_included_2.tex}\n', 'figures/figure_not_included_2.tex': '', 'figures/figure_included.tikz': '\ufeff\\tikzsetnextfilename{test2}\n\\begin{tikzpicture}\n\\node {root}\nchild {node {left}}\nchild {node {right}\nchild {node {child}}\nchild {node {child}}\n};\n\\end{tikzpicture}\n', 'figures/figure_included.tex': '\\includegraphics{images/im2_included.jpg}\n\\addplot{figures/data_included.txt}\n'}

{'main.tex': ['\\begin{document}', 'Text', '', 'Text%', '', '', 'This is a percent \\%.', '\\includegraphics{images/im1_included.png}', '\\includegraphics{images/im3_included.png}', '\\includegraphics{%', '  images/im4_included.png%', '  }', '\\includegraphics[width=.5\\linewidth]{%', '  images/im5_included.jpg}', '', '\\includegraphics{./images/im3_included.png}', '', 'This line should not be separated', '%', 'from this one.', '', '', '', '\\newif\\ifvar', '', '\\ifvar', '\\fi', '', '\\newcommand{\\red}[1]{{\\color{red} #1}}', 'hello test hello', 'test hello', 'test', '', '', '\\input{figures/figure_included.tex}', '', '\\includegraphics{ext_tikz/test1.pdf}', '', '\\input{figures/figure_included.tikz}', '', '\\begin{tikzpicture}', '    \\node (test) at (0,0) {Test3};', '\\end{tikzpicture}', '', '\\tikzsetnextfilename{test_no_match}', '\\begin{tikzpicture}', '    \\node (test) at (0,0) {Test4};', '\\end{tikzpicture}', '', '\\end{document}', ''], 'figures/figure_not_included.tex': ['\\addplot{figures/data_not_included.txt}', '\\input{figures/figure_not_included_2.tex}', ''], 'figures/figure_not_included_2.tex': '', 'figures/figure_included.tikz': '\ufeff\\tikzsetnextfilename{test2}\n\\begin{tikzpicture}\n\\node {root}\nchild {node {left}}\nchild {node {right}\nchild {node {child}}\nchild {node {child}}\n};\n\\end{tikzpicture}\n', 'figures/figure_included.tex': '\\includegraphics{images/im2_included.jpg}\n\\addplot{figures/data_included.txt}\n'}

{'main.tex': ['\\begin{document}', 'Text', '', 'Text%', '', '', 'This is a percent \\%.', '\\includegraphics{images/im1_included.png}', '\\includegraphics{images/im3_included.png}', '\\includegraphics{%', '  images/im4_included.png%', '  }', '\\includegraphics[width=.5\\linewidth]{%', '  images/im5_included.jpg}', '', '\\includegraphics{./images/im3_included.png}', '', 'This line should not be separated', '%', 'from this one.', '', '', '', '\\newif\\ifvar', '', '\\ifvar', '\\fi', '', '\\newcommand{\\red}[1]{{\\color{red} #1}}', 'hello test hello', 'test hello', 'test', '', '', '\\input{figures/figure_included.tex}', '', '\\includegraphics{ext_tikz/test1.pdf}', '', '\\input{figures/figure_included.tikz}', '', '\\begin{tikzpicture}', '    \\node (test) at (0,0) {Test3};', '\\end{tikzpicture}', '', '\\tikzsetnextfilename{test_no_match}', '\\begin{tikzpicture}', '    \\node (test) at (0,0) {Test4};', '\\end{tikzpicture}', '', '\\end{document}', ''], 'figures/figure_not_included.tex': ['\\addplot{figures/data_not_included.txt}', '\\input{figures/figure_not_included_2.tex}', ''], 'figures/figure_not_included_2.tex': [''], 'figures/figure_included.tikz': '\ufeff\\tikzsetnextfilename{test2}\n\\begin{tikzpicture}\n\\node {root}\nchild {node {left}}\nchild {node {right}\nchild {node {child}}\nchild {node {child}}\n};\n\\end{tikzpicture}\n', 'figures/figure_included.tex': '\\includegraphics{images/im2_included.jpg}\n\\addplot{figures/data_included.txt}\n'}

{'main.tex': ['\\begin{document}', 'Text', '', 'Text%', '', '', 'This is a percent \\%.', '\\includegraphics{images/im1_included.png}', '\\includegraphics{images/im3_included.png}', '\\includegraphics{%', '  images/im4_included.png%', '  }', '\\includegraphics[width=.5\\linewidth]{%', '  images/im5_included.jpg}', '', '\\includegraphics{./images/im3_included.png}', '', 'This line should not be separated', '%', 'from this one.', '', '', '', '\\newif\\ifvar', '', '\\ifvar', '\\fi', '', '\\newcommand{\\red}[1]{{\\color{red} #1}}', 'hello test hello', 'test hello', 'test', '', '', '\\input{figures/figure_included.tex}', '', '\\includegraphics{ext_tikz/test1.pdf}', '', '\\input{figures/figure_included.tikz}', '', '\\begin{tikzpicture}', '    \\node (test) at (0,0) {Test3};', '\\end{tikzpicture}', '', '\\tikzsetnextfilename{test_no_match}', '\\begin{tikzpicture}', '    \\node (test) at (0,0) {Test4};', '\\end{tikzpicture}', '', '\\end{document}', ''], 'figures/figure_not_included.tex': ['\\addplot{figures/data_not_included.txt}', '\\input{figures/figure_not_included_2.tex}', ''], 'figures/figure_not_included_2.tex': [''], 'figures/figure_included.tikz': ['\ufeff\\includegraphics{ext_tikz/test2.pdf}', ''], 'figures/figure_included.tex': '\\includegraphics{images/im2_included.jpg}\n\\addplot{figures/data_included.txt}\n'}

{'main.tex': ['\\begin{document}', 'Text', '', 'Text%', '', '', 'This is a percent \\%.', '\\includegraphics{images/im1_included.png}', '\\includegraphics{images/im3_included.png}', '\\includegraphics{%', '  images/im4_included.png%', '  }', '\\includegraphics[width=.5\\linewidth]{%', '  images/im5_included.jpg}', '', '\\includegraphics{./images/im3_included.png}', '', 'This line should not be separated', '%', 'from this one.', '', '', '', '\\newif\\ifvar', '', '\\ifvar', '\\fi', '', '\\newcommand{\\red}[1]{{\\color{red} #1}}', 'hello test hello', 'test hello', 'test', '', '', '\\input{figures/figure_included.tex}', '', '\\includegraphics{ext_tikz/test1.pdf}', '', '\\input{figures/figure_included.tikz}', '', '\\begin{tikzpicture}', '    \\node (test) at (0,0) {Test3};', '\\end{tikzpicture}', '', '\\tikzsetnextfilename{test_no_match}', '\\begin{tikzpicture}', '    \\node (test) at (0,0) {Test4};', '\\end{tikzpicture}', '', '\\end{document}', ''], 'figures/figure_not_included.tex': ['\\addplot{figures/data_not_included.txt}', '\\input{figures/figure_not_included_2.tex}', ''], 'figures/figure_not_included_2.tex': [''], 'figures/figure_included.tikz': '\ufeff\\includegraphics{ext_tikz/test2.pdf}\n', 'figures/figure_included.tex': '\\includegraphics{images/im2_included.jpg}\n\\addplot{figures/data_included.txt}\n'}

{'main.tex': '\\begin{document}\nText\n\nText%\n\n\nThis is a percent \\%.\n\\includegraphics{images/im1_included.png}\n\\includegraphics{images/im3_included.png}\n\\includegraphics{%\n  images/im4_included.png%\n  }\n\\includegraphics[width=.5\\linewidth]{%\n  images/im5_included.jpg}\n\n\\includegraphics{./images/im3_included.png}\n\nThis line should not be separated\n%\nfrom this one.\n\n\n\n\\newif\\ifvar\n\n\\ifvar\n\\fi\n\n\\newcommand{\\red}[1]{{\\color{red} #1}}\nhello test hello\ntest hello\ntest\n\n\n\\input{figures/figure_included.tex}\n\n\\includegraphics{ext_tikz/test1.pdf}\n\n\\input{figures/figure_included.tikz}\n\n\\begin{tikzpicture}\n    \\node (test) at (0,0) {Test3};\n\\end{tikzpicture}\n\n\\tikzsetnextfilename{test_no_match}\n\\begin{tikzpicture}\n    \\node (test) at (0,0) {Test4};\n\\end{tikzpicture}\n\n\\end{document}\n', 'figures/figure_not_included.tex': ['\\addplot{figures/data_not_included.txt}', '\\input{figures/figure_not_included_2.tex}', ''], 'figures/figure_not_included_2.tex': [''], 'figures/figure_included.tikz': '\ufeff\\includegraphics{ext_tikz/test2.pdf}\n', 'figures/figure_included.tex': '\\includegraphics{images/im2_included.jpg}\n\\addplot{figures/data_included.txt}\n'}

'\\includegraphics{images/im2_included.jpg}\n\\addplot{figures/data_included.txt}\n\n\ufeff\\includegraphics{ext_tikz/test2.pdf}\n\n\\begin{document}\nText\n\nText%\n\n\nThis is a percent \\%.\n\\includegraphics{images/im1_included.png}\n\\includegraphics{images/im3_included.png}\n\\includegraphics{%\n  images/im4_included.png%\n  }\n\\includegraphics[width=.5\\linewidth]{%\n  images/im5_included.jpg}\n\n\\includegraphics{./images/im3_included.png}\n\nThis line should not be separated\n%\nfrom this one.\n\n\n\n\\newif\\ifvar\n\n\\ifvar\n\\fi\n\n\\newcommand{\\red}[1]{{\\color{red} #1}}\nhello test hello\ntest hello\ntest\n\n\n\\input{figures/figure_included.tex}\n\n\\includegraphics{ext_tikz/test1.pdf}\n\n\\input{figures/figure_included.tikz}\n\n\\begin{tikzpicture}\n    \\node (test) at (0,0) {Test3};\n\\end{tikzpicture}\n\n\\tikzsetnextfilename{test_no_match}\n\\begin{tikzpicture}\n    \\node (test) at (0,0) {Test4};\n\\end{tikzpicture}\n\n\\end{document}\n'


def _read_file_content(filename):
  with open(filename, 'r', encoding='utf-8') as fp:
    lines = fp.readlines()
    lines = _strip_tex_contents(lines, '\\end{document}')
    return lines

_read_file_content(filename='tex/main.tex')

<_io.TextIOWrapper name='tex/main.tex' mode='r' encoding='utf-8'>

['\\begin{document}\n', 'Text\n', '% Whole line comment\n', '\n', 'Text% Inline comment\n', '\\begin{comment}\n', 'This is an environment comment.\n', '\\end{comment}\n', '\n', 'This is a percent \\%.\n', '% Whole line comment without newline\n', '\\includegraphics{images/im1_included.png}\n', '%\\includegraphics{images/im_not_included}\n', '\\includegraphics{images/im3_included.png}\n', '\\includegraphics{%\n', '  images/im4_included.png%\n', '  }\n', '\\includegraphics[width=.5\\linewidth]{%\n', '  images/im5_included.jpg}\n', '%\\includegraphics{%\n', '%  images/im4_not_included.png\n', '%  }\n', '%\\includegraphics[width=.5\\linewidth]{%\n', '%  images/im5_not_included.jpg}\n', '\n', '% test whatever the path satrting with dot works when include graphics\n', '\\includegraphics{./images/im3_included.png}\n', '\n', 'This line should\\mytodo{Do this later} not be separated\n', '\\mytodo{This is a todo command with a nested \\textit{command}.\n', 'Please remember that up to \\texttt{2 levels} of \\textit{nesting} are supported.}\n', 'from this one.\n', '\n', '\\begin{mynote}\n', '  This is a custom environment that could be excluded.\n', '\\end{mynote}\n', '\n', '\\newif\\ifvar\n', '\n', '\\ifvar\n', '\\if    false\n', '\\if false\n', '\\if 0\n', '\\iffalse\n', '\\ifvar\n', 'Text\n', '\\fi\n', '\\fi\n', '\\fi\n', '\\fi\n', '\\fi\n', '\\fi\n', '\n', '\\newcommand{\\red}[1]{{\\color{red} #1}}\n', 'hello test \\red{hello\n', 'test \\red{hello}}\n', 'test\n', '\n', '% content after this line should not be cleaned if \\end{document} is in a comment\n', '\n', '\\input{figures/figure_included.tex}\n', '% \\input{figures/figure_not_included.tex}\n', '\n', '% Test for tikzpicture feature\n', '% should be replaced\n', '\\tikzsetnextfilename{test1}\n', '\\begin{tikzpicture}\n', '    \\node (test) at (0,0) {Test1};\n', '\\end{tikzpicture}\n', '\n', '% should be replaced in included file\n', '\\input{figures/figure_included.tikz}\n', '\n', '% should not be be replaced - no preceding tikzsetnextfilename command\n', '\\begin{tikzpicture}\n', '    \\node (test) at (0,0) {Test3};\n', '\\end{tikzpicture}\n', '\n', '\\tikzsetnextfilename{test_no_match}\n', '\\begin{tikzpicture}\n', '    \\node (test) at (0,0) {Test4};\n', '\\end{tikzpicture}\n', '\n', '\\end{document}\n', '\n', 'This should be ignored.\n']

['\\begin{document}\n', 'Text\n', '% Whole line comment\n', '\n', 'Text% Inline comment\n', '\\begin{comment}\n', 'This is an environment comment.\n', '\\end{comment}\n', '\n', 'This is a percent \\%.\n', '% Whole line comment without newline\n', '\\includegraphics{images/im1_included.png}\n', '%\\includegraphics{images/im_not_included}\n', '\\includegraphics{images/im3_included.png}\n', '\\includegraphics{%\n', '  images/im4_included.png%\n', '  }\n', '\\includegraphics[width=.5\\linewidth]{%\n', '  images/im5_included.jpg}\n', '%\\includegraphics{%\n', '%  images/im4_not_included.png\n', '%  }\n', '%\\includegraphics[width=.5\\linewidth]{%\n', '%  images/im5_not_included.jpg}\n', '\n', '% test whatever the path satrting with dot works when include graphics\n', '\\includegraphics{./images/im3_included.png}\n', '\n', 'This line should\\mytodo{Do this later} not be separated\n', '\\mytodo{This is a todo command with a nested \\textit{command}.\n', 'Please remember that up to \\texttt{2 levels} of \\textit{nesting} are supported.}\n', 'from this one.\n', '\n', '\\begin{mynote}\n', '  This is a custom environment that could be excluded.\n', '\\end{mynote}\n', '\n', '\\newif\\ifvar\n', '\n', '\\ifvar\n', '\\if    false\n', '\\if false\n', '\\if 0\n', '\\iffalse\n', '\\ifvar\n', 'Text\n', '\\fi\n', '\\fi\n', '\\fi\n', '\\fi\n', '\\fi\n', '\\fi\n', '\n', '\\newcommand{\\red}[1]{{\\color{red} #1}}\n', 'hello test \\red{hello\n', 'test \\red{hello}}\n', 'test\n', '\n', '% content after this line should not be cleaned if \\end{document} is in a comment\n', '\n', '\\input{figures/figure_included.tex}\n', '% \\input{figures/figure_not_included.tex}\n', '\n', '% Test for tikzpicture feature\n', '% should be replaced\n', '\\tikzsetnextfilename{test1}\n', '\\begin{tikzpicture}\n', '    \\node (test) at (0,0) {Test1};\n', '\\end{tikzpicture}\n', '\n', '% should be replaced in included file\n', '\\input{figures/figure_included.tikz}\n', '\n', '% should not be be replaced - no preceding tikzsetnextfilename command\n', '\\begin{tikzpicture}\n', '    \\node (test) at (0,0) {Test3};\n', '\\end{tikzpicture}\n', '\n', '\\tikzsetnextfilename{test_no_match}\n', '\\begin{tikzpicture}\n', '    \\node (test) at (0,0) {Test4};\n', '\\end{tikzpicture}\n', '\n', '\\end{document}\n']


def _keep_only_referenced_tex(contents, splits):
  """Returns the filenames referenced from the tex files themselves.

  It needs various iterations in case one file is referenced from an
  unreferenced file.
  """
  old_referenced = set(splits['tex_in_root'] + splits['tex_not_in_root'])
  while True:
    referenced = set(splits['tex_in_root'])
    for fn in old_referenced:
      for fn2 in old_referenced:
        if regex.search(
            r'(' + os.path.splitext(fn)[0] + r'[.}])', '\n'.join(contents[fn2])
        ):
          referenced.add(fn)

    if referenced == old_referenced:
      splits['tex_to_copy'] = list(referenced)
      return

    old_referenced = referenced.copy()

_keep_only_referenced_tex(contents={'main.tex': ['\\begin{document}', 'Text', '', 'Text%', '', '', 'This is a percent \\%.', '\\includegraphics{images/im1_included.png}', '\\includegraphics{images/im3_included.png}', '\\includegraphics{%', '  images/im4_included.png%', '  }', '\\includegraphics[width=.5\\linewidth]{%', '  images/im5_included.jpg}', '', '\\includegraphics{./images/im3_included.png}', '', 'This line should not be separated', '%', 'from this one.', '', '', '', '\\newif\\ifvar', '', '\\ifvar', '\\fi', '', '\\newcommand{\\red}[1]{{\\color{red} #1}}', 'hello test hello', 'test hello', 'test', '', '', '\\input{figures/figure_included.tex}', '', '\\includegraphics{ext_tikz/test1.pdf}', '', '\\input{figures/figure_included.tikz}', '', '\\begin{tikzpicture}', '    \\node (test) at (0,0) {Test3};', '\\end{tikzpicture}', '', '\\tikzsetnextfilename{test_no_match}', '\\begin{tikzpicture}', '    \\node (test) at (0,0) {Test4};', '\\end{tikzpicture}', '', '\\end{document}', ''], 'figures/figure_not_included.tex': ['\\addplot{figures/data_not_included.txt}', '\\input{figures/figure_not_included_2.tex}', ''], 'figures/figure_not_included_2.tex': [''], 'figures/figure_included.tikz': ['\ufeff\\includegraphics{ext_tikz/test2.pdf}', ''], 'figures/figure_included.tex': ['\\includegraphics{images/im2_included.jpg}', '\\addplot{figures/data_included.txt}', '']}, splits={'all': ['main.bib', 'main.bbl', 'main.tex', 'main.aux', 'ext_tikz/test2.pdf', 'ext_tikz/test1.pdf', 'ext_tikz/figure_not_included.pdf', 'figures/data_not_included.txt', 'figures/figure_not_included.tex', 'figures/figure_not_included_2.tex', 'figures/figure_included.tikz', 'figures/figure_included.tex', 'figures/data_included.txt', 'not_included/figures/data_included.txt', 'images/im4_included.png', 'images/im1.png', 'images/im4_not_included.png', 'images/im3_included.png', 'images/im2_included.jpg', 'images/im5_not_included.jpg', 'images/im5_included.jpg', 'images/im1_included.png', 'images/im_not_included.png', 'images/include/images/im3_included.png'], 'in_root': ['main.bib', 'main.bbl', 'main.tex', 'main.aux'], 'not_in_root': ['ext_tikz/test2.pdf', 'ext_tikz/test1.pdf', 'ext_tikz/figure_not_included.pdf', 'figures/data_not_included.txt', 'figures/figure_not_included.tex', 'figures/figure_not_included_2.tex', 'figures/figure_included.tikz', 'figures/figure_included.tex', 'figures/data_included.txt', 'not_included/figures/data_included.txt', 'images/im4_included.png', 'images/im1.png', 'images/im4_not_included.png', 'images/im3_included.png', 'images/im2_included.jpg', 'images/im5_not_included.jpg', 'images/im5_included.jpg', 'images/im1_included.png', 'images/im_not_included.png', 'images/include/images/im3_included.png'], 'to_copy_in_root': ['main.bbl', 'main.tex'], 'to_copy_not_in_root': ['figures/data_not_included.txt', 'figures/figure_not_included.tex', 'figures/figure_not_included_2.tex', 'figures/figure_included.tikz', 'figures/figure_included.tex', 'figures/data_included.txt', 'not_included/figures/data_included.txt'], 'figures': ['ext_tikz/test2.pdf', 'ext_tikz/test1.pdf', 'ext_tikz/figure_not_included.pdf', 'images/im4_included.png', 'images/im1.png', 'images/im4_not_included.png', 'images/im3_included.png', 'images/im2_included.jpg', 'images/im5_not_included.jpg', 'images/im5_included.jpg', 'images/im1_included.png', 'images/im_not_included.png', 'images/include/images/im3_included.png'], 'tex_in_root': ['main.tex'], 'tex_not_in_root': ['figures/figure_not_included.tex', 'figures/figure_not_included_2.tex', 'figures/figure_included.tikz', 'figures/figure_included.tex'], 'non_tex_in_root': ['main.bbl'], 'non_tex_not_in_root': ['figures/data_not_included.txt', 'figures/data_included.txt', 'not_included/figures/data_included.txt'], 'external_tikz_figures': ['ext_tikz/test2.pdf', 'ext_tikz/test1.pdf', 'ext_tikz/figure_not_included.pdf'], 'svg_inkscape': []})

{'figures/figure_included.tex', 'figures/figure_not_included.tex', 'figures/figure_included.tikz', 'main.tex', 'figures/figure_not_included_2.tex'}

{'figures/figure_included.tex', 'figures/figure_included.tikz', 'main.tex', 'figures/figure_not_included_2.tex'}


def apply_changes(file_path: str, changes: List, confirm: bool = False):
    """
    Pass changes as loaded json (list of dicts)
    """
    with open(file_path) as f:
        original_file_lines = f.readlines()

    # Filter out explanation elements
    operation_changes = [change for change in changes if "operation" in change]
    explanations = [
        change["explanation"] for change in changes if "explanation" in change
    ]

    # Sort the changes in reverse line order
    operation_changes.sort(key=lambda x: x["line"], reverse=True)

    file_lines = original_file_lines.copy()
    for change in operation_changes:
        operation = change["operation"]
        line = change["line"]
        content = change["content"]

        if operation == "Replace":
            file_lines[line - 1] = content + "\n"
        elif operation == "Delete":
            del file_lines[line - 1]
        elif operation == "InsertAfter":
            file_lines.insert(line, content + "\n")

    # Print explanations
    cprint("Explanations:", "blue")
    for explanation in explanations:
        cprint(f"- {explanation}", "blue")

    # Display changes diff
    print("\nChanges to be made:")
    diff = difflib.unified_diff(original_file_lines, file_lines, lineterm="")
    for line in diff:
        if line.startswith("+"):
            cprint(line, "green", end="")
        elif line.startswith("-"):
            cprint(line, "red", end="")
        else:
            print(line, end="")

    if confirm:
        # check if user wants to apply changes or exit
        confirmation = input("Do you want to apply these changes? (y/n): ")
        if confirmation.lower() != "y":
            print("Changes not applied")
            sys.exit(0)

    with open(file_path, "w") as f:
        f.writelines(file_lines)
    print("Changes applied.")

apply_changes(file_path='/tmp/tmp6qrrn_j9', changes=[{'operation': 'Replace', 'line': 2, 'content': 'new second line'}], confirm=False)

<_io.TextIOWrapper name='/tmp/tmp6qrrn_j9' mode='r' encoding='UTF-8'>

<_io.TextIOWrapper name='/tmp/tmp6qrrn_j9' mode='w' encoding='UTF-8'>

['first line\n', 'second line\n', 'third line']

['first line\n', 'new second line\n', 'third line']

'new second line'

<generator object unified_diff at 0x7f5b71b2cba0>


def return_logger(name: str) -> logging.Logger:
    # Set log message format

    logger = logging.getLogger(name)

    if not len(logger.handlers):
        log_formatter = logging.Formatter('%(asctime)-s: %(levelname)-s %(message)s')
        # set logging level
        logger.setLevel(logging.DEBUG)

        # Direct logs to stderr
        console_handler = logging.StreamHandler()
        console_handler.setFormatter(log_formatter)
        logger.addHandler(console_handler)

    return logger

return_logger(name='imagededup.handlers.metrics.classification')

{_style=<logging.PercentStyle object at 0x7fcf18561f10>, _fmt='%(asctime)-s: %(levelname)-s %(message)s', datefmt=None}


def transform_path_to_dotted(sys_path, module_path):
    """
    Returns the dotted path inside a sys.path as a list of names. e.g.

    >>> transform_path_to_dotted([str(Path("/foo").absolute())], Path('/foo/bar/baz.py').absolute())
    (('bar', 'baz'), False)

    Returns (None, False) if the path doesn't really resolve to anything.
    The second return part is if it is a package.
    """
    # First remove the suffix.
    module_path = remove_python_path_suffix(module_path)
    if module_path.name.startswith('.'):
        return None, False

    # Once the suffix was removed we are using the files as we know them. This
    # means that if someone uses an ending like .vim for a Python file, .vim
    # will be part of the returned dotted part.

    is_package = module_path.name == '__init__'
    if is_package:
        module_path = module_path.parent

    def iter_potential_solutions():
        for p in sys_path:
            if str(module_path).startswith(p):
                # Strip the trailing slash/backslash
                rest = str(module_path)[len(p):]
                # On Windows a path can also use a slash.
                if rest.startswith(os.path.sep) or rest.startswith('/'):
                    # Remove a slash in cases it's still there.
                    rest = rest[1:]

                if rest:
                    split = rest.split(os.path.sep)
                    if not all(split):
                        # This means that part of the file path was empty, this
                        # is very strange and is probably a file that is called
                        # `.py`.
                        return
                    # Stub folders for foo can end with foo-stubs. Just remove
                    # it.
                    yield tuple(re.sub(r'-stubs$', '', s) for s in split)

    potential_solutions = tuple(iter_potential_solutions())
    if not potential_solutions:
        return None, False
    # Try to find the shortest path, this makes more sense usually, because the
    # user usually has venvs somewhere. This means that a path like
    # .tox/py37/lib/python3.7/os.py can be normal for a file. However in that
    # case we definitely want to return ['os'] as a path and not a crazy
    # ['.tox', 'py37', 'lib', 'python3.7', 'os']. Keep in mind that this is a
    # heuristic and there's now ay to "always" do it right.
    return sorted(potential_solutions, key=lambda p: len(p))[0], is_package

transform_path_to_dotted(sys_path=['/local/rcs/jinjun/code/pytrace-collector/logs/self_collected/tried/davidhalter+jedi/davidhalter+jedi', '/local/rcs/jinjun/code/pytrace-collector', '/local/rcs/jinjun/miniforge3/envs/davidhalter+jedi/lib/python39.zip', '/local/rcs/jinjun/miniforge3/envs/davidhalter+jedi/lib/python3.9', '/local/rcs/jinjun/miniforge3/envs/davidhalter+jedi/lib/python3.9/lib-dynload', '/local/rcs/jinjun/miniforge3/envs/davidhalter+jedi/lib/python3.9/site-packages', '/local/rcs/jinjun/miniforge3/envs/davidhalter+jedi/lib/python3.9/site-packages/PySnooper-1.2.0-py3.9.egg'], module_path=PosixPath('/local/rcs/jinjun/code/pytrace-collector/logs/self_collected/tried/davidhalter+jedi/davidhalter+jedi/example.py'))

<function transform_path_to_dotted.<locals>.iter_potential_solutions at 0x7f5855feec10>

(('example',), ('logs', 'self_collected', 'tried', 'davidhalter+jedi', 'davidhalter+jedi', 'example'))


def infer_node(context, element):
    if isinstance(context, CompForContext):
        return _infer_node(context, element)

    if_stmt = element
    while if_stmt is not None:
        if_stmt = if_stmt.parent
        if if_stmt.type in ('if_stmt', 'for_stmt'):
            break
        if parser_utils.is_scope(if_stmt):
            if_stmt = None
            break
    predefined_if_name_dict = context.predefined_names.get(if_stmt)
    # TODO there's a lot of issues with this one. We actually should do
    # this in a different way. Caching should only be active in certain
    # cases and this all sucks.
    if predefined_if_name_dict is None and if_stmt \
            and if_stmt.type == 'if_stmt' and context.inference_state.is_analysis:
        if_stmt_test = if_stmt.children[1]
        name_dicts = [{}]
        # If we already did a check, we don't want to do it again -> If
        # value.predefined_names is filled, we stop.
        # We don't want to check the if stmt itself, it's just about
        # the content.
        if element.start_pos > if_stmt_test.end_pos:
            # Now we need to check if the names in the if_stmt match the
            # names in the suite.
            if_names = get_names_of_node(if_stmt_test)
            element_names = get_names_of_node(element)
            str_element_names = [e.value for e in element_names]
            if any(i.value in str_element_names for i in if_names):
                for if_name in if_names:
                    definitions = context.inference_state.infer(context, if_name)
                    # Every name that has multiple different definitions
                    # causes the complexity to rise. The complexity should
                    # never fall below 1.
                    if len(definitions) > 1:
                        if len(name_dicts) * len(definitions) > 16:
                            debug.dbg('Too many options for if branch inference %s.', if_stmt)
                            # There's only a certain amount of branches
                            # Jedi can infer, otherwise it will take to
                            # long.
                            name_dicts = [{}]
                            break

                        original_name_dicts = list(name_dicts)
                        name_dicts = []
                        for definition in definitions:
                            new_name_dicts = list(original_name_dicts)
                            for i, name_dict in enumerate(new_name_dicts):
                                new_name_dicts[i] = name_dict.copy()
                                new_name_dicts[i][if_name.value] = ValueSet([definition])

                            name_dicts += new_name_dicts
                    else:
                        for name_dict in name_dicts:
                            name_dict[if_name.value] = definitions
        if len(name_dicts) > 1:
            result = NO_VALUES
            for name_dict in name_dicts:
                with context.predefine_names(if_stmt, name_dict):
                    result |= _infer_node(context, element)
            return result
        else:
            return _infer_node_if_inferred(context, element)
    else:
        if predefined_if_name_dict:
            return _infer_node(context, element)
        else:
            return _infer_node_if_inferred(context, element)

infer_node(context=ModuleContext(<ModuleValue: __main__@2-8 is_stub=False>), element=PythonNode(test, [<Name: f@8,11>, <Keyword: if>, PythonNode(atom_expr, [<Name: random@8,16>, PythonNode(trailer, [<Operator: .>, <Name: choice@8,23>]), PythonNode(trailer, [<Operator: (>, PythonNode(atom, [<Operator: [>, PythonNode(testlist_comp, [<Number: 0>, <Operator: ,>, <Number: 1>]), <Operator: ]>]), <Operator: )>])]), <Keyword: else>, <Name: C@8,42>]))

<ExprStmt: variable = f if random.choice([0,1]) else C@8,0>


def _search_param_in_docstr(docstr, param_str):
    """
    Search `docstr` for type(-s) of `param_str`.

    >>> _search_param_in_docstr(':type param: int', 'param')
    ['int']
    >>> _search_param_in_docstr('@type param: int', 'param')
    ['int']
    >>> _search_param_in_docstr(
    ...   ':type param: :class:`threading.Thread`', 'param')
    ['threading.Thread']
    >>> bool(_search_param_in_docstr('no document', 'param'))
    False
    >>> _search_param_in_docstr(':param int param: some description', 'param')
    ['int']

    """
    # look at #40 to see definitions of those params
    patterns = [re.compile(p % re.escape(param_str))
                for p in DOCSTRING_PARAM_PATTERNS]
    for pattern in patterns:
        match = pattern.search(docstr)
        if match:
            return [_strip_rst_role(match.group(1))]

    return _search_param_in_numpydocstr(docstr, param_str)

_search_param_in_docstr(docstr=':type param: int', param_str='param')

<re.Match object; span=(0, 16), match=':type param: int'>


def generate(resource_types=()):
    resource_defs = {}
    definitions = {
        'resources': resource_defs,
        'string_dict': {
            "type": "object",
            "patternProperties": {
                "": {"type": "string"},
            },
        },
        'basic_dict': {
            "type": "object",
            "patternProperties": {
                "": {
                    'oneOf': [
                        {"type": "string"},
                        {"type": "boolean"},
                        {"type": "number"},
                    ],
                }
            },
        },
        'iam-statement': {
            'additionalProperties': False,
            'type': 'object',
            'properties': {
                'Sid': {'type': 'string'},
                'Effect': {'type': 'string', 'enum': ['Allow', 'Deny']},
                'Principal': {'anyOf': [
                    {'type': 'string'},
                    {'type': 'object'}, {'type': 'array'}]},
                'NotPrincipal': {'anyOf': [{'type': 'object'}, {'type': 'array'}]},
                'Action': {'anyOf': [{'type': 'string'}, {'type': 'array'}]},
                'NotAction': {'anyOf': [{'type': 'string'}, {'type': 'array'}]},
                'Resource': {'anyOf': [{'type': 'string'}, {'type': 'array'}]},
                'NotResource': {'anyOf': [{'type': 'string'}, {'type': 'array'}]},
                'Condition': {'type': 'object'}
            },
            'required': ['Sid', 'Effect'],
            'oneOf': [
                {'required': ['Principal', 'Action', 'Resource']},
                {'required': ['NotPrincipal', 'Action', 'Resource']},
                {'required': ['Principal', 'NotAction', 'Resource']},
                {'required': ['NotPrincipal', 'NotAction', 'Resource']},
                {'required': ['Principal', 'Action', 'NotResource']},
                {'required': ['NotPrincipal', 'Action', 'NotResource']},
                {'required': ['Principal', 'NotAction', 'NotResource']},
                {'required': ['NotPrincipal', 'NotAction', 'NotResource']}
            ]
        },
        'actions': {},
        'filters': {
            'value': ValueFilter.schema,
            'event': EventFilter.schema,
            'age': AgeFilter.schema,
            'reduce': ReduceFilter.schema,
            # Shortcut form of value filter as k=v
            'valuekv': {
                'type': 'object',
                'additionalProperties': {'oneOf': [{'type': 'number'}, {'type': 'null'},
                    {'type': 'array', 'maxItems': 0}, {'type': 'string'}, {'type': 'boolean'}]},
                'minProperties': 1,
                'maxProperties': 1},
        },
        'filters_common': {
            'list_item_attrs': _get_attr_schema(),
            'comparison_operators': {
                'enum': list(OPERATORS.keys())},
            'value_types': {'enum': VALUE_TYPES},
            'value_from': ValuesFrom.schema,
            'value': {'oneOf': [
                {'type': 'array'},
                {'type': 'string'},
                {'type': 'boolean'},
                {'type': 'number'},
                {'type': 'null'}]},
        },
        'policy': {
            'type': 'object',
            'required': ['name', 'resource'],
            'additionalProperties': False,
            'properties': {
                'name': {
                    'type': 'string',
                    'pattern': "^[A-z][A-z0-9]*(-[A-z0-9]+)*$"},
                'conditions': {
                    'type': 'array',
                    'items': {'anyOf': [
                        {'type': 'object', 'additionalProperties': False,
                         'properties': {'or': {
                             '$ref': '#/definitions/policy/properties/conditions'}}},
                        {'type': 'object', 'additionalProperties': False,
                         'properties': {'not': {
                             '$ref': '#/definitions/policy/properties/conditions'}}},
                        {'type': 'object', 'additionalProperties': False,
                         'properties': {'and': {
                             '$ref': '#/definitions/policy/properties/conditions'}}},
                        {'$ref': '#/definitions/filters/value'},
                        {'$ref': '#/definitions/filters/event'},
                        {'$ref': '#/definitions/filters/valuekv'}]}},
                # these should be deprecated for conditions
                'region': {'type': 'string'},
                'tz': {'type': 'string'},
                'start': {'format': 'date-time'},
                'end': {'format': 'date-time'},
                'resource': {'oneOf': [
                    {'type': 'string'},
                    {'type': 'array', 'items': {'type': 'string'}}]},
                'max-resources': {'anyOf': [
                    {'type': 'integer', 'minimum': 1},
                    {'$ref': '#/definitions/max-resources-properties'}
                ]},
                'max-resources-percent': {'type': 'number', 'minimum': 0, 'maximum': 100},
                'comment': {'type': 'string'},
                'comments': {'type': 'string'},
                'description': {'type': 'string'},
                'tags': {'type': 'array', 'items': {'type': 'string'}},
                'metadata': {'type': 'object'},
                'mode': {'$ref': '#/definitions/policy-mode'},
                'source': {'enum': list(sources.keys())},
                'actions': {
                    'type': 'array',
                },
                'filters': {
                    'type': 'array'
                },
                #
                # TODO: source queries should really move under
                # source. This was initially used for describe sources
                # to expose server side query mechanisms, however its
                # important to note it also prevents resource cache
                # utilization between policies that have different
                # queries.
                'query': {
                    'type': 'array', 'items': {'type': 'object'}}

            },
        },
        'policy-mode': {
            'anyOf': [e.schema for _, e in execution.items()],
        },
        'max-resources-properties': {
            'type': 'object',
            'additionalProperties': False,
            'properties': {
                'amount': {"type": 'integer', 'minimum': 1},
                'op': {'enum': ['or', 'and']},
                'percent': {'type': 'number', 'minimum': 0, 'maximum': 100}
            }
        }
    }

    resource_refs = []
    for cloud_name, cloud_type in sorted(clouds.items()):
        for type_name, resource_type in sorted(cloud_type.resources.items()):
            r_type_name = "%s.%s" % (cloud_name, type_name)
            if resource_types and r_type_name not in resource_types:
                if not resource_type.type_aliases:
                    continue
                elif not {"%s.%s" % (cloud_name, ralias) for ralias
                        in resource_type.type_aliases}.intersection(
                        resource_types):
                    continue

            aliases = []
            if resource_type.type_aliases:
                aliases.extend(["%s.%s" % (cloud_name, a) for a in resource_type.type_aliases])
                # aws gets legacy aliases with no cloud prefix
                if cloud_name == 'aws':
                    aliases.extend(resource_type.type_aliases)

            # aws gets additional alias for default name
            if cloud_name == 'aws':
                aliases.append(type_name)

            resource_refs.append(
                process_resource(
                    r_type_name,
                    resource_type,
                    resource_defs,
                    aliases,
                    definitions,
                    cloud_name
                ))

    schema = {
        "$schema": "http://json-schema.org/draft-07/schema#",
        'id': 'http://schema.cloudcustodian.io/v0/custodian.json',
        'definitions': definitions,
        'type': 'object',
        'required': ['policies'],
        'additionalProperties': False,
        'properties': {
            'vars': {'type': 'object'},
            'policies': {
                'type': 'array',
                'additionalItems': False,
                'items': {'anyOf': resource_refs}
            }
        }
    }

    # allow empty policies with lazy load
    if not resource_refs:
        schema['properties']['policies']['items'] = {'type': 'object'}
    return schema

generate(resource_types=())

{'resources': {}, 'string_dict': {'type': 'object', 'patternProperties': {'': {'type': 'string'}}}, 'basic_dict': {'type': 'object', 'patternProperties': {'': {'oneOf': [{'type': 'string'}, {'type': 'boolean'}, {'type': 'number'}]}}}, 'iam-statement': {'additionalProperties': False, 'type': 'object', 'properties': {'Sid': {'type': 'string'}, 'Effect': {'type': 'string', 'enum': ['Allow', 'Deny']}, 'Principal': {'anyOf': [{'type': 'string'}, {'type': 'object'}, {'type': 'array'}]}, 'NotPrincipal': {'anyOf': [{'type': 'object'}, {'type': 'array'}]}, 'Action': {'anyOf': [{'type': 'string'}, {'type': 'array'}]}, 'NotAction': {'anyOf': [{'type': 'string'}, {'type': 'array'}]}, 'Resource': {'anyOf': [{'type': 'string'}, {'type': 'array'}]}, 'NotResource': {'anyOf': [{'type': 'string'}, {'type': 'array'}]}, 'Condition': {'type': 'object'}}, 'required': ['Sid', 'Effect'], 'oneOf': [{'required': ['Principal', 'Action', 'Resource']}, {'required': ['NotPrincipal', 'Action', 'Resource']}, {'required': ['Principal', 'NotAction', 'Resource']}, {'required': ['NotPrincipal', 'NotAction', 'Resource']}, {'required': ['Principal', 'Action', 'NotResource']}, {'required': ['NotPrincipal', 'Action', 'NotResource']}, {'required': ['Principal', 'NotAction', 'NotResource']}, {'required': ['NotPrincipal', 'NotAction', 'NotResource']}]}, 'actions': {}, 'filters': {'value': {'type': 'object', 'additionalProperties': False, 'required': ['type'], 'properties': {'type': {'enum': ['value']}, 'key': {'type': 'string'}, 'value_type': {'$ref': '#/definitions/filters_common/value_types'}, 'default': {'type': 'object'}, 'value_regex': {'type': 'string'}, 'value_from': {'$ref': '#/definitions/filters_common/value_from'}, 'value': {'$ref': '#/definitions/filters_common/value'}, 'op': {'$ref': '#/definitions/filters_common/comparison_operators'}, 'value_path': {'type': 'string'}}}, 'event': {'type': 'object', 'additionalProperties': False, 'required': ['type'], 'properties': {'type': {'enum': ['event']}, 'key': {'type': 'string'}, 'value_type': {'$ref': '#/definitions/filters_common/value_types'}, 'default': {'type': 'object'}, 'value_regex': {'type': 'string'}, 'value_from': {'$ref': '#/definitions/filters_common/value_from'}, 'value': {'$ref': '#/definitions/filters_common/value'}, 'op': {'$ref': '#/definitions/filters_common/comparison_operators'}, 'value_path': {'type': 'string'}}}, 'age': None, 'reduce': {'type': 'object', 'additionalProperties': False, 'required': ['type'], 'properties': {'type': {'enum': ['reduce']}, 'group-by': {'oneOf': [{'type': 'string'}, {'type': 'object', 'key': {'type': 'string'}, 'value_type': {'enum': ['string', 'number', 'date']}, 'value_regex': 'string'}]}, 'sort-by': {'oneOf': [{'type': 'string'}, {'type': 'object', 'key': {'type': 'string'}, 'value_type': {'enum': ['string', 'number', 'date']}, 'value_regex': 'string'}]}, 'order': {'enum': ['asc', 'desc', 'reverse', 'randomize']}, 'null-order': {'enum': ['first', 'last']}, 'limit': {'type': 'number', 'minimum': 0}, 'limit-percent': {'type': 'number', 'minimum': 0, 'maximum': 100}, 'discard': {'type': 'number', 'minimum': 0}, 'discard-percent': {'type': 'number', 'minimum': 0, 'maximum': 100}}}, 'valuekv': {'type': 'object', 'additionalProperties': {'oneOf': [{'type': 'number'}, {'type': 'null'}, {'type': 'array', 'maxItems': 0}, {'type': 'string'}, {'type': 'boolean'}]}, 'minProperties': 1, 'maxProperties': 1}}, 'filters_common': {'list_item_attrs': {'items': {'anyOf': [{'$ref': '#/definitions/filters/value'}, {'$ref': '#/definitions/filters/valuekv'}, {'additional_properties': False, 'properties': {'and': {'type': 'array', 'items': {'anyOf': [{'$ref': '#/definitions/filters/value'}, {'$ref': '#/definitions/filters/valuekv'}]}}}, 'type': 'object'}, {'additional_properties': False, 'properties': {'or': {'type': 'array', 'items': {'anyOf': [{'$ref': '#/definitions/filters/value'}, {'$ref': '#/definitions/filters/valuekv'}]}}}, 'type': 'object'}, {'additional_properties': False, 'properties': {'not': {'type': 'array', 'items': {'anyOf': [{'$ref': '#/definitions/filters/value...onalProperties': False, 'properties': {'execution-options': {'type': 'object'}, 'function-prefix': {'type': 'string'}, 'member-role': {'type': 'string'}, 'packages': {'type': 'array', 'items': {'type': 'string'}}, 'layers': {'type': 'array', 'items': {'type': 'string'}}, 'concurrency': {'type': 'integer'}, 'runtime': {'enum': ['python3.8', 'python3.9', 'python3.10', 'python3.11', 'python3.12']}, 'role': {'type': 'string'}, 'handler': {'type': 'string'}, 'pattern': {'type': 'object', 'minProperties': 1}, 'timeout': {'type': 'number'}, 'memory': {'type': 'number'}, 'environment': {'type': 'object'}, 'tags': {'type': 'object'}, 'dead_letter_config': {'type': 'object'}, 'kms_key_arn': {'type': 'string'}, 'tracing_config': {'type': 'object'}, 'security_groups': {'type': 'array'}, 'subnets': {'type': 'array'}, 'type': {'enum': ['asg-instance-state']}, 'events': {'type': 'array', 'items': {'enum': ['launch-success', 'launch-failure', 'terminate-success', 'terminate-failure']}}}, 'required': ['type']}, {'type': 'object', 'additionalProperties': False, 'properties': {'execution-options': {'type': 'object'}, 'function-prefix': {'type': 'string'}, 'member-role': {'type': 'string'}, 'packages': {'type': 'array', 'items': {'type': 'string'}}, 'layers': {'type': 'array', 'items': {'type': 'string'}}, 'concurrency': {'type': 'integer'}, 'runtime': {'enum': ['python3.8', 'python3.9', 'python3.10', 'python3.11', 'python3.12']}, 'role': {'type': 'string'}, 'handler': {'type': 'string'}, 'pattern': {'type': 'object', 'minProperties': 1}, 'timeout': {'type': 'number'}, 'memory': {'type': 'number'}, 'environment': {'type': 'object'}, 'tags': {'type': 'object'}, 'dead_letter_config': {'type': 'object'}, 'kms_key_arn': {'type': 'string'}, 'tracing_config': {'type': 'object'}, 'security_groups': {'type': 'array'}, 'subnets': {'type': 'array'}, 'type': {'enum': ['guard-duty']}}, 'required': ['type']}, {'type': 'object', 'additionalProperties': False, 'properties': {'execution-options': {'type': 'object'}, 'function-prefix': {'type': 'string'}, 'member-role': {'type': 'string'}, 'packages': {'type': 'array', 'items': {'type': 'string'}}, 'layers': {'type': 'array', 'items': {'type': 'string'}}, 'concurrency': {'type': 'integer'}, 'runtime': {'enum': ['python3.8', 'python3.9', 'python3.10', 'python3.11', 'python3.12']}, 'role': {'type': 'string'}, 'handler': {'type': 'string'}, 'pattern': {'type': 'object', 'minProperties': 1}, 'timeout': {'type': 'number'}, 'memory': {'type': 'number'}, 'environment': {'type': 'object'}, 'tags': {'type': 'object'}, 'dead_letter_config': {'type': 'object'}, 'kms_key_arn': {'type': 'string'}, 'tracing_config': {'type': 'object'}, 'security_groups': {'type': 'array'}, 'subnets': {'type': 'array'}, 'type': {'enum': ['config-poll-rule']}, 'schedule': {'enum': ['One_Hour', 'Three_Hours', 'Six_Hours', 'Twelve_Hours', 'TwentyFour_Hours']}, 'ignore-support-check': {'type': 'boolean'}}, 'required': ['type']}, {'type': 'object', 'additionalProperties': False, 'properties': {'execution-options': {'type': 'object'}, 'function-prefix': {'type': 'string'}, 'member-role': {'type': 'string'}, 'packages': {'type': 'array', 'items': {'type': 'string'}}, 'layers': {'type': 'array', 'items': {'type': 'string'}}, 'concurrency': {'type': 'integer'}, 'runtime': {'enum': ['python3.8', 'python3.9', 'python3.10', 'python3.11', 'python3.12']}, 'role': {'type': 'string'}, 'handler': {'type': 'string'}, 'pattern': {'type': 'object', 'minProperties': 1}, 'timeout': {'type': 'number'}, 'memory': {'type': 'number'}, 'environment': {'type': 'object'}, 'tags': {'type': 'object'}, 'dead_letter_config': {'type': 'object'}, 'kms_key_arn': {'type': 'string'}, 'tracing_config': {'type': 'object'}, 'security_groups': {'type': 'array'}, 'subnets': {'type': 'array'}, 'type': {'enum': ['config-rule']}}, 'required': ['type']}]}, 'max-resources-properties': {'type': 'object', 'additionalProperties': False, 'properties': {'amount': {'type': 'integer', 'minimum': 1}, 'op': {'enum': ['or', 'and']}, 'percent': {'type': 'number', 'minimum': 0, 'maximum': 100}}}}

{'resources': {'gcp.region': {'actions': {}, 'filters': {}, 'policy': {'allOf': [{'$ref': '#/definitions/policy'}, {'properties': {'resource': {'enum': ['gcp.region']}, 'filters': {'type': 'array', 'items': {'anyOf': [{'enum': []}, {'type': 'object', 'additionalProperties': False, 'properties': {'or': {'$ref': '#/definitions/resources/gcp.region/policy/allOf/1/properties/filters'}}}, {'type': 'object', 'additionalProperties': False, 'properties': {'and': {'$ref': '#/definitions/resources/gcp.region/policy/allOf/1/properties/filters'}}}, {'type': 'object', 'additionalProperties': False, 'properties': {'not': {'$ref': '#/definitions/resources/gcp.region/policy/allOf/1/properties/filters'}}}]}}, 'actions': {'type': 'array', 'items': {'anyOf': [{'enum': []}]}}}}]}}}, 'string_dict': {'type': 'object', 'patternProperties': {'': {'type': 'string'}}}, 'basic_dict': {'type': 'object', 'patternProperties': {'': {'oneOf': [{'type': 'string'}, {'type': 'boolean'}, {'type': 'number'}]}}}, 'iam-statement': {'additionalProperties': False, 'type': 'object', 'properties': {'Sid': {'type': 'string'}, 'Effect': {'type': 'string', 'enum': ['Allow', 'Deny']}, 'Principal': {'anyOf': [{'type': 'string'}, {'type': 'object'}, {'type': 'array'}]}, 'NotPrincipal': {'anyOf': [{'type': 'object'}, {'type': 'array'}]}, 'Action': {'anyOf': [{'type': 'string'}, {'type': 'array'}]}, 'NotAction': {'anyOf': [{'type': 'string'}, {'type': 'array'}]}, 'Resource': {'anyOf': [{'type': 'string'}, {'type': 'array'}]}, 'NotResource': {'anyOf': [{'type': 'string'}, {'type': 'array'}]}, 'Condition': {'type': 'object'}}, 'required': ['Sid', 'Effect'], 'oneOf': [{'required': ['Principal', 'Action', 'Resource']}, {'required': ['NotPrincipal', 'Action', 'Resource']}, {'required': ['Principal', 'NotAction', 'Resource']}, {'required': ['NotPrincipal', 'NotAction', 'Resource']}, {'required': ['Principal', 'Action', 'NotResource']}, {'required': ['NotPrincipal', 'Action', 'NotResource']}, {'required': ['Principal', 'NotAction', 'NotResource']}, {'required': ['NotPrincipal', 'NotAction', 'NotResource']}]}, 'actions': {}, 'filters': {'value': {'type': 'object', 'additionalProperties': False, 'required': ['type'], 'properties': {'type': {'enum': ['value']}, 'key': {'type': 'string'}, 'value_type': {'$ref': '#/definitions/filters_common/value_types'}, 'default': {'type': 'object'}, 'value_regex': {'type': 'string'}, 'value_from': {'$ref': '#/definitions/filters_common/value_from'}, 'value': {'$ref': '#/definitions/filters_common/value'}, 'op': {'$ref': '#/definitions/filters_common/comparison_operators'}, 'value_path': {'type': 'string'}}}, 'event': {'type': 'object', 'additionalProperties': False, 'required': ['type'], 'properties': {'type': {'enum': ['event']}, 'key': {'type': 'string'}, 'value_type': {'$ref': '#/definitions/filters_common/value_types'}, 'default': {'type': 'object'}, 'value_regex': {'type': 'string'}, 'value_from': {'$ref': '#/definitions/filters_common/value_from'}, 'value': {'$ref': '#/definitions/filters_common/value'}, 'op': {'$ref': '#/definitions/filters_common/comparison_operators'}, 'value_path': {'type': 'string'}}}, 'age': None, 'reduce': {'type': 'object', 'additionalProperties': False, 'required': ['type'], 'properties': {'type': {'enum': ['reduce']}, 'group-by': {'oneOf': [{'type': 'string'}, {'type': 'object', 'key': {'type': 'string'}, 'value_type': {'enum': ['string', 'number', 'date']}, 'value_regex': 'string'}]}, 'sort-by': {'oneOf': [{'type': 'string'}, {'type': 'object', 'key': {'type': 'string'}, 'value_type': {'enum': ['string', 'number', 'date']}, 'value_regex': 'string'}]}, 'order': {'enum': ['asc', 'desc', 'reverse', 'randomize']}, 'null-order': {'enum': ['first', 'last']}, 'limit': {'type': 'number', 'minimum': 0}, 'limit-percent': {'type': 'number', 'minimum': 0, 'maximum': 100}, 'discard': {'type': 'number', 'minimum': 0}, 'discard-percent': {'type': 'number', 'minimum': 0, 'maximum': 100}}}, 'valuekv': {'type': 'object', 'additionalProperties': {'oneOf': [{'type': 'number'}, {'type': 'null'}, {'type': 'array', 'maxItems': 0}, {...onalProperties': False, 'properties': {'execution-options': {'type': 'object'}, 'function-prefix': {'type': 'string'}, 'member-role': {'type': 'string'}, 'packages': {'type': 'array', 'items': {'type': 'string'}}, 'layers': {'type': 'array', 'items': {'type': 'string'}}, 'concurrency': {'type': 'integer'}, 'runtime': {'enum': ['python3.8', 'python3.9', 'python3.10', 'python3.11', 'python3.12']}, 'role': {'type': 'string'}, 'handler': {'type': 'string'}, 'pattern': {'type': 'object', 'minProperties': 1}, 'timeout': {'type': 'number'}, 'memory': {'type': 'number'}, 'environment': {'type': 'object'}, 'tags': {'type': 'object'}, 'dead_letter_config': {'type': 'object'}, 'kms_key_arn': {'type': 'string'}, 'tracing_config': {'type': 'object'}, 'security_groups': {'type': 'array'}, 'subnets': {'type': 'array'}, 'type': {'enum': ['asg-instance-state']}, 'events': {'type': 'array', 'items': {'enum': ['launch-success', 'launch-failure', 'terminate-success', 'terminate-failure']}}}, 'required': ['type']}, {'type': 'object', 'additionalProperties': False, 'properties': {'execution-options': {'type': 'object'}, 'function-prefix': {'type': 'string'}, 'member-role': {'type': 'string'}, 'packages': {'type': 'array', 'items': {'type': 'string'}}, 'layers': {'type': 'array', 'items': {'type': 'string'}}, 'concurrency': {'type': 'integer'}, 'runtime': {'enum': ['python3.8', 'python3.9', 'python3.10', 'python3.11', 'python3.12']}, 'role': {'type': 'string'}, 'handler': {'type': 'string'}, 'pattern': {'type': 'object', 'minProperties': 1}, 'timeout': {'type': 'number'}, 'memory': {'type': 'number'}, 'environment': {'type': 'object'}, 'tags': {'type': 'object'}, 'dead_letter_config': {'type': 'object'}, 'kms_key_arn': {'type': 'string'}, 'tracing_config': {'type': 'object'}, 'security_groups': {'type': 'array'}, 'subnets': {'type': 'array'}, 'type': {'enum': ['guard-duty']}}, 'required': ['type']}, {'type': 'object', 'additionalProperties': False, 'properties': {'execution-options': {'type': 'object'}, 'function-prefix': {'type': 'string'}, 'member-role': {'type': 'string'}, 'packages': {'type': 'array', 'items': {'type': 'string'}}, 'layers': {'type': 'array', 'items': {'type': 'string'}}, 'concurrency': {'type': 'integer'}, 'runtime': {'enum': ['python3.8', 'python3.9', 'python3.10', 'python3.11', 'python3.12']}, 'role': {'type': 'string'}, 'handler': {'type': 'string'}, 'pattern': {'type': 'object', 'minProperties': 1}, 'timeout': {'type': 'number'}, 'memory': {'type': 'number'}, 'environment': {'type': 'object'}, 'tags': {'type': 'object'}, 'dead_letter_config': {'type': 'object'}, 'kms_key_arn': {'type': 'string'}, 'tracing_config': {'type': 'object'}, 'security_groups': {'type': 'array'}, 'subnets': {'type': 'array'}, 'type': {'enum': ['config-poll-rule']}, 'schedule': {'enum': ['One_Hour', 'Three_Hours', 'Six_Hours', 'Twelve_Hours', 'TwentyFour_Hours']}, 'ignore-support-check': {'type': 'boolean'}}, 'required': ['type']}, {'type': 'object', 'additionalProperties': False, 'properties': {'execution-options': {'type': 'object'}, 'function-prefix': {'type': 'string'}, 'member-role': {'type': 'string'}, 'packages': {'type': 'array', 'items': {'type': 'string'}}, 'layers': {'type': 'array', 'items': {'type': 'string'}}, 'concurrency': {'type': 'integer'}, 'runtime': {'enum': ['python3.8', 'python3.9', 'python3.10', 'python3.11', 'python3.12']}, 'role': {'type': 'string'}, 'handler': {'type': 'string'}, 'pattern': {'type': 'object', 'minProperties': 1}, 'timeout': {'type': 'number'}, 'memory': {'type': 'number'}, 'environment': {'type': 'object'}, 'tags': {'type': 'object'}, 'dead_letter_config': {'type': 'object'}, 'kms_key_arn': {'type': 'string'}, 'tracing_config': {'type': 'object'}, 'security_groups': {'type': 'array'}, 'subnets': {'type': 'array'}, 'type': {'enum': ['config-rule']}}, 'required': ['type']}]}, 'max-resources-properties': {'type': 'object', 'additionalProperties': False, 'properties': {'amount': {'type': 'integer', 'minimum': 1}, 'op': {'enum': ['or', 'and']}, 'percent': {'type': 'number', 'minimum': 0, 'maximum': 100}}}}

{'$schema': 'http://json-schema.org/draft-07/schema#', 'id': 'http://schema.cloudcustodian.io/v0/custodian.json', 'definitions': {'resources': {'gcp.region': {'actions': {}, 'filters': {}, 'policy': {'allOf': [{'$ref': '#/definitions/policy'}, {'properties': {'resource': {'enum': ['gcp.region']}, 'filters': {'type': 'array', 'items': {'anyOf': [{'enum': []}, {'type': 'object', 'additionalProperties': False, 'properties': {'or': {'$ref': '#/definitions/resources/gcp.region/policy/allOf/1/properties/filters'}}}, {'type': 'object', 'additionalProperties': False, 'properties': {'and': {'$ref': '#/definitions/resources/gcp.region/policy/allOf/1/properties/filters'}}}, {'type': 'object', 'additionalProperties': False, 'properties': {'not': {'$ref': '#/definitions/resources/gcp.region/policy/allOf/1/properties/filters'}}}]}}, 'actions': {'type': 'array', 'items': {'anyOf': [{'enum': []}]}}}}]}}}, 'string_dict': {'type': 'object', 'patternProperties': {'': {'type': 'string'}}}, 'basic_dict': {'type': 'object', 'patternProperties': {'': {'oneOf': [{'type': 'string'}, {'type': 'boolean'}, {'type': 'number'}]}}}, 'iam-statement': {'additionalProperties': False, 'type': 'object', 'properties': {'Sid': {'type': 'string'}, 'Effect': {'type': 'string', 'enum': ['Allow', 'Deny']}, 'Principal': {'anyOf': [{'type': 'string'}, {'type': 'object'}, {'type': 'array'}]}, 'NotPrincipal': {'anyOf': [{'type': 'object'}, {'type': 'array'}]}, 'Action': {'anyOf': [{'type': 'string'}, {'type': 'array'}]}, 'NotAction': {'anyOf': [{'type': 'string'}, {'type': 'array'}]}, 'Resource': {'anyOf': [{'type': 'string'}, {'type': 'array'}]}, 'NotResource': {'anyOf': [{'type': 'string'}, {'type': 'array'}]}, 'Condition': {'type': 'object'}}, 'required': ['Sid', 'Effect'], 'oneOf': [{'required': ['Principal', 'Action', 'Resource']}, {'required': ['NotPrincipal', 'Action', 'Resource']}, {'required': ['Principal', 'NotAction', 'Resource']}, {'required': ['NotPrincipal', 'NotAction', 'Resource']}, {'required': ['Principal', 'Action', 'NotResource']}, {'required': ['NotPrincipal', 'Action', 'NotResource']}, {'required': ['Principal', 'NotAction', 'NotResource']}, {'required': ['NotPrincipal', 'NotAction', 'NotResource']}]}, 'actions': {}, 'filters': {'value': {'type': 'object', 'additionalProperties': False, 'required': ['type'], 'properties': {'type': {'enum': ['value']}, 'key': {'type': 'string'}, 'value_type': {'$ref': '#/definitions/filters_common/value_types'}, 'default': {'type': 'object'}, 'value_regex': {'type': 'string'}, 'value_from': {'$ref': '#/definitions/filters_common/value_from'}, 'value': {'$ref': '#/definitions/filters_common/value'}, 'op': {'$ref': '#/definitions/filters_common/comparison_operators'}, 'value_path': {'type': 'string'}}}, 'event': {'type': 'object', 'additionalProperties': False, 'required': ['type'], 'properties': {'type': {'enum': ['event']}, 'key': {'type': 'string'}, 'value_type': {'$ref': '#/definitions/filters_common/value_types'}, 'default': {'type': 'object'}, 'value_regex': {'type': 'string'}, 'value_from': {'$ref': '#/definitions/filters_common/value_from'}, 'value': {'$ref': '#/definitions/filters_common/value'}, 'op': {'$ref': '#/definitions/filters_common/comparison_operators'}, 'value_path': {'type': 'string'}}}, 'age': None, 'reduce': {'type': 'object', 'additionalProperties': False, 'required': ['type'], 'properties': {'type': {'enum': ['reduce']}, 'group-by': {'oneOf': [{'type': 'string'}, {'type': 'object', 'key': {'type': 'string'}, 'value_type': {'enum': ['string', 'number', 'date']}, 'value_regex': 'string'}]}, 'sort-by': {'oneOf': [{'type': 'string'}, {'type': 'object', 'key': {'type': 'string'}, 'value_type': {'enum': ['string', 'number', 'date']}, 'value_regex': 'string'}]}, 'order': {'enum': ['asc', 'desc', 'reverse', 'randomize']}, 'null-order': {'enum': ['first', 'last']}, 'limit': {'type': 'number', 'minimum': 0}, 'limit-percent': {'type': 'number', 'minimum': 0, 'maximum': 100}, 'discard': {'type': 'number', 'minimum': 0}, 'discard-percent': {'type': 'number', 'minimum': 0, 'maximum': 100}}}, 'valuekv'...ype': 'string'}}, 'concurrency': {'type': 'integer'}, 'runtime': {'enum': ['python3.8', 'python3.9', 'python3.10', 'python3.11', 'python3.12']}, 'role': {'type': 'string'}, 'handler': {'type': 'string'}, 'pattern': {'type': 'object', 'minProperties': 1}, 'timeout': {'type': 'number'}, 'memory': {'type': 'number'}, 'environment': {'type': 'object'}, 'tags': {'type': 'object'}, 'dead_letter_config': {'type': 'object'}, 'kms_key_arn': {'type': 'string'}, 'tracing_config': {'type': 'object'}, 'security_groups': {'type': 'array'}, 'subnets': {'type': 'array'}, 'type': {'enum': ['asg-instance-state']}, 'events': {'type': 'array', 'items': {'enum': ['launch-success', 'launch-failure', 'terminate-success', 'terminate-failure']}}}, 'required': ['type']}, {'type': 'object', 'additionalProperties': False, 'properties': {'execution-options': {'type': 'object'}, 'function-prefix': {'type': 'string'}, 'member-role': {'type': 'string'}, 'packages': {'type': 'array', 'items': {'type': 'string'}}, 'layers': {'type': 'array', 'items': {'type': 'string'}}, 'concurrency': {'type': 'integer'}, 'runtime': {'enum': ['python3.8', 'python3.9', 'python3.10', 'python3.11', 'python3.12']}, 'role': {'type': 'string'}, 'handler': {'type': 'string'}, 'pattern': {'type': 'object', 'minProperties': 1}, 'timeout': {'type': 'number'}, 'memory': {'type': 'number'}, 'environment': {'type': 'object'}, 'tags': {'type': 'object'}, 'dead_letter_config': {'type': 'object'}, 'kms_key_arn': {'type': 'string'}, 'tracing_config': {'type': 'object'}, 'security_groups': {'type': 'array'}, 'subnets': {'type': 'array'}, 'type': {'enum': ['guard-duty']}}, 'required': ['type']}, {'type': 'object', 'additionalProperties': False, 'properties': {'execution-options': {'type': 'object'}, 'function-prefix': {'type': 'string'}, 'member-role': {'type': 'string'}, 'packages': {'type': 'array', 'items': {'type': 'string'}}, 'layers': {'type': 'array', 'items': {'type': 'string'}}, 'concurrency': {'type': 'integer'}, 'runtime': {'enum': ['python3.8', 'python3.9', 'python3.10', 'python3.11', 'python3.12']}, 'role': {'type': 'string'}, 'handler': {'type': 'string'}, 'pattern': {'type': 'object', 'minProperties': 1}, 'timeout': {'type': 'number'}, 'memory': {'type': 'number'}, 'environment': {'type': 'object'}, 'tags': {'type': 'object'}, 'dead_letter_config': {'type': 'object'}, 'kms_key_arn': {'type': 'string'}, 'tracing_config': {'type': 'object'}, 'security_groups': {'type': 'array'}, 'subnets': {'type': 'array'}, 'type': {'enum': ['config-poll-rule']}, 'schedule': {'enum': ['One_Hour', 'Three_Hours', 'Six_Hours', 'Twelve_Hours', 'TwentyFour_Hours']}, 'ignore-support-check': {'type': 'boolean'}}, 'required': ['type']}, {'type': 'object', 'additionalProperties': False, 'properties': {'execution-options': {'type': 'object'}, 'function-prefix': {'type': 'string'}, 'member-role': {'type': 'string'}, 'packages': {'type': 'array', 'items': {'type': 'string'}}, 'layers': {'type': 'array', 'items': {'type': 'string'}}, 'concurrency': {'type': 'integer'}, 'runtime': {'enum': ['python3.8', 'python3.9', 'python3.10', 'python3.11', 'python3.12']}, 'role': {'type': 'string'}, 'handler': {'type': 'string'}, 'pattern': {'type': 'object', 'minProperties': 1}, 'timeout': {'type': 'number'}, 'memory': {'type': 'number'}, 'environment': {'type': 'object'}, 'tags': {'type': 'object'}, 'dead_letter_config': {'type': 'object'}, 'kms_key_arn': {'type': 'string'}, 'tracing_config': {'type': 'object'}, 'security_groups': {'type': 'array'}, 'subnets': {'type': 'array'}, 'type': {'enum': ['config-rule']}}, 'required': ['type']}]}, 'max-resources-properties': {'type': 'object', 'additionalProperties': False, 'properties': {'amount': {'type': 'integer', 'minimum': 1}, 'op': {'enum': ['or', 'and']}, 'percent': {'type': 'number', 'minimum': 0, 'maximum': 100}}}}, 'type': 'object', 'required': ['policies'], 'additionalProperties': False, 'properties': {'vars': {'type': 'object'}, 'policies': {'type': 'array', 'additionalItems': False, 'items': {'anyOf': [{'$ref': '#/definitions/resources/gcp.region/policy'}]}}}}


def _get_attr_schema():
    base_filters = [
        {'$ref': '#/definitions/filters/value'},
        {'$ref': '#/definitions/filters/valuekv'},
    ]
    any_of = []
    any_of.extend(base_filters)

    for op in ('and', 'or', 'not',):
        any_of.append(
            {
                'additional_properties': False,
                'properties': {
                    op: {
                        'type': 'array',
                        'items': {
                            'anyOf': base_filters
                        }
                    }
                },
                'type': 'object'
            }
        )

    attr_schema = {
        'items': {
            'anyOf': any_of
        },
        'type': 'array',
    }
    return attr_schema

_get_attr_schema()

[{'$ref': '#/definitions/filters/value'}, {'$ref': '#/definitions/filters/valuekv'}, {'additional_properties': False, 'properties': {'and': {'type': 'array', 'items': {'anyOf': [{'$ref': '#/definitions/filters/value'}, {'$ref': '#/definitions/filters/valuekv'}]}}}, 'type': 'object'}]

[{'$ref': '#/definitions/filters/value'}, {'$ref': '#/definitions/filters/valuekv'}, {'additional_properties': False, 'properties': {'and': {'type': 'array', 'items': {'anyOf': [{'$ref': '#/definitions/filters/value'}, {'$ref': '#/definitions/filters/valuekv'}]}}}, 'type': 'object'}, {'additional_properties': False, 'properties': {'or': {'type': 'array', 'items': {'anyOf': [{'$ref': '#/definitions/filters/value'}, {'$ref': '#/definitions/filters/valuekv'}]}}}, 'type': 'object'}]

{'items': {'anyOf': [{'$ref': '#/definitions/filters/value'}, {'$ref': '#/definitions/filters/valuekv'}, {'additional_properties': False, 'properties': {'and': {'type': 'array', 'items': {'anyOf': [{'$ref': '#/definitions/filters/value'}, {'$ref': '#/definitions/filters/valuekv'}]}}}, 'type': 'object'}, {'additional_properties': False, 'properties': {'or': {'type': 'array', 'items': {'anyOf': [{'$ref': '#/definitions/filters/value'}, {'$ref': '#/definitions/filters/valuekv'}]}}}, 'type': 'object'}, {'additional_properties': False, 'properties': {'not': {'type': 'array', 'items': {'anyOf': [{'$ref': '#/definitions/filters/value'}, {'$ref': '#/definitions/filters/valuekv'}]}}}, 'type': 'object'}]}, 'type': 'array'}


def _underscore(word: str) -> str:
    # https://github.com/jpvanhal/inflection/blob/master/inflection.py
    word = re.sub(r"([A-Z]+)([A-Z][a-z])", r'\1_\2', word)
    word = re.sub(r"([a-z\d])([A-Z])", r'\1_\2', word)
    word = word.replace("-", "_")
    return word.lower()

_underscore(word='TestKlass')


def timed_frames(records, min_frame_dur=1, max_frame_dur=None, last_frame_dur=1000):
    """Return a tuple made of the geometry of the screen and a generator of
    instances of TimedFrame computed from asciicast records

    Asciicast records are first coalesced so that the mininum duration between
    two frames is at least `min_frame_dur` milliseconds. Events with a duration
    greater than `max_frame_dur` will see their duration reduced to that value.

    The duration of all frames lasting until the end of the animation
    will be adjusted so that the last frame of the animation lasts
    `last_frame_dur`

    :param records: Terminal session record in Asciicast v2 format
    :param min_frame_dur: Minimum frame duration in milliseconds (integer)
    :param min_frame_dur: Minimum frame duration in milliseconds (integer)
    :param max_frame_dur: Maximum frame duration in milliseconds (None or
    integer)
    :param last_frame_dur: Duration of the last frame of the animation
    (integer)
    """
    if not isinstance(records, Iterator):
        records = iter(records)

    header = next(records)
    assert isinstance(header, AsciiCastV2Header)

    if not max_frame_dur and header.idle_time_limit:
        max_frame_dur = int(header.idle_time_limit * 1000)

    def generator():
        screen = pyte.Screen(header.width, header.height)
        stream = pyte.Stream(screen)
        timed_records = _group_by_time(records, min_frame_dur, max_frame_dur,
                                       last_frame_dur)

        for record_ in timed_records:
            assert isinstance(record_, AsciiCastV2Event)
            for char in record_.event_data:
                stream.feed(char)
            yield TimedFrame(int(1000 * record_.time),
                             int(1000 * record_.duration),
                             _screen_buffer(screen))

    return (header.width, header.height), generator()

timed_frames(records=[AsciiCastV2Header(version=2, width=80, height=24, theme=AsciiCastV2Theme(fg='#000000', bg='#FFFFFF', palette='#123456:#123456:#123456:#123456:#123456:#123456:#123456:#123456:#123456:#123456:#123456:#123456:#123456:#123456:#123456:#123456'), idle_time_limit=None), AsciiCastV2Event(time=0, event_type='o', event_data='0\r\n', duration=None), AsciiCastV2Event(time=1, event_type='o', event_data='1\r\n', duration=None)], min_frame_dur=1, max_frame_dur=None, last_frame_dur=42)

<function timed_frames.<locals>.generator at 0x7ff5e9385dc0>


def __get_command_output(command, cwd=None):
    """ Execute arbitrary commands.

    Parameters
    ----------
    command : list
        the command and its arguments

    cwd : string
        the directory where the command should be executed

    Returns
    -------
    output : string
        the raw output of the command executed
    """

    p = subprocess.Popen(command, stdout=subprocess.PIPE,
            stderr=subprocess.PIPE, cwd=cwd)
    p.wait()
    return p.returncode, p.stdout.read(), p.stderr.read()

__get_command_output(command=['git', 'rev-parse'], cwd='/local/rcs/jinjun/code/pytrace-collector/logs/pypibugs/tried/ASPP+pelita/ASPP+pelita/pelita')

<Popen: returncode: None args: ['git', 'rev-parse']>

<Popen: returncode: 0 args: ['git', 'rev-parse']>


def parse_layout(layout_str):
    """Parse a layout string

    Return a dict
        {'walls': list_of_wall_coordinates,
         'food' : list_of_food_coordinates,
         'bot'  : list_of_4_bot_coordinate}

    A layout string is composed of wall characters '#', food characters '.', and
    bot characters '0', '1', '2', and '3'.

    Valid layouts must be enclosed by walls and be of rectangular shape. Example:

     ########
     #0  .  #
     #2    1#
     #  .  3#
     ########

    If items are overlapping, several layout strings can be concateneted:
     ########
     #0  .  #
     #     1#
     #  .  3#
     ########
     ########
     #2  .  #
     #     1#
     #  .  3#
     ########

     In this case, bot '0' and bot '2' are on top of each other at position (1,1)
    """
    layout_list = []
    start = False
    for i, line in enumerate(layout_str.splitlines()):
        row = line.strip()
        if not row:
            # ignore emptylines
            continue
        if not start:
            # start a new layout
            # check that row is a valid opening string
            if row.count('#') != len(row):
                raise ValueError(f"Layout does not start with a row of walls (line: {i})!")
            current_layout = [row]
            start = True
            continue
        # we are in the middle of a layout, just append to the current
        # layout unless we detect the closing string
        current_layout.append(row)
        if row.count('#') == len(row):
            # this is a closing string
            # append the layout to tha layout list
            layout_list.append('\n'.join(current_layout))
            start = False

    if start:
        # the last layout has not been closed, complain here!
        raise ValueError(f"Layout does not end with a row of walls (line: {i})!")

    # initialize walls, food and bots from the first layout
    out = parse_single_layout(layout_list.pop(0))
    for layout in layout_list:
        items = parse_layout(layout)
        # walls should always be the same
        if items['walls'] != out['walls']:
            raise ValueError('Walls are not equal in all layouts!')
        # add the food, removing duplicates
        out['food'] = list(set(out['food'] + items['food']))
        # add the bots
        for bot_idx, bot_pos in enumerate(items['bots']):
            if bot_pos:
                # this bot position is not None, overwrite whatever we had before
                out['bots'][bot_idx] = bot_pos

    return out

parse_layout(layout_str='\n        ##################\n        #. ... .##.     3#\n        # # #  .  .### #1#\n        # # ##.   .      #\n        #      .   .## # #\n        #0# ###.  .  # # #\n        #2     .##. ... .#\n        ################## ')


def parse_single_layout(layout_str):
    """Parse a single layout from a string

    See parse_layout for details about valid layout strings.
    """
    # width of the layout (x-axis)
    width = None
    # list of layout rows
    rows = []
    start = False
    for i, line in enumerate(layout_str.splitlines()):
        row = line.strip()
        if not row:
            # always ignore empty lines
            continue
        # a layout is always started by a full row of walls
        if not start:
            if row.count('#') != len(row):
                raise ValueError(f"Layout must be enclosed by walls (line: {i})!")
            else:
                # start the layout parsing
                start = True
                # set width of layout
                width = len(row)
                # check that width is even
                if width % 2:
                    raise ValueError(f"Layout width must be even (found {width})!")
                rows.append(row)
                continue
        # Here we are within the layout
        # every row must have the same length
        if len(row) != width:
            raise ValueError(f"Layout rows have differing widths (line: {i})!")
        # rows are always enclosed by walls
        if row[0] != '#' or row[-1] != '#':
            raise ValueError(f"Layout must be enclosed by walls (line:{i})!")
        # append current row to the list of rows
        rows.append(row)
        # detect closing row and ignore whatever follows
        if row.count('#') == len(row):
            start = False
            break

    if start:
        # layout has not been closed!
        raise ValueError(f"Layout must be enclosed by walls (line:{i})!")

    # height of the layout (y-axis)
    height = len(rows)
    walls = []
    food = []
    # bot positions (we assume 4 bots)
    bots = [None]*4

    # iterate through the grid of characters
    for y, row in enumerate(rows):
        for x, char in enumerate(row):
            coord = (x, y)
            # assign the char to the corresponding list
            if char == '#':
                # wall
                walls.append(coord)
            elif char == '.':
                # food
                food.append(coord)
            elif char == ' ':
                # empty
                continue
            else:
                # bot
                try:
                    # we expect an 0<=index<=3
                    bot_idx = int(char)
                    if bot_idx >= len(bots):
                        # reuse the except below
                        raise ValueError
                except ValueError:
                    raise ValueError(f"Unknown character {char} in maze!")
                bots[bot_idx] = coord
    walls.sort()
    food.sort()
    return {'walls':walls, 'food':food, 'bots':bots}

parse_single_layout(layout_str='##################\n#. ... .##.     3#\n# # #  .  .### #1#\n# # ##.   .      #\n#      .   .## # #\n#0# ###.  .  # # #\n#2     .##. ... .#\n##################')


def initial_positions(walls):
    """Calculate initial positions.

    Given the list of walls, returns the free positions that are closest to the
    bottom left and top right corner. The algorithm starts searching from
    (1, height-2) and (width-2, 1) respectively and uses the Manhattan distance
    for judging what is closest. On equal distances, a smaller distance in the
    x value is preferred.
    """
    width = max(walls)[0] + 1
    height = max(walls)[1] + 1

    left_start = (1, height - 2)
    left = []
    right_start = (width - 2, 1)
    right = []

    dist = 0
    while len(left) < 2:
        # iterate through all possible x distances (inclusive)
        for x_dist in range(dist + 1):
            y_dist = dist - x_dist
            pos = (left_start[0] + x_dist, left_start[1] - y_dist)
            # if both coordinates are out of bounds, we stop
            if not (0 <= pos[0] < width) and not (0 <= pos[1] < height):
                raise ValueError("Not enough free initial positions.")
            # if one coordinate is out of bounds, we just continue
            if not (0 <= pos[0] < width) or not (0 <= pos[1] < height):
                continue
            # check if the new value is free
            if pos not in walls:
                left.append(pos)

            if len(left) == 2:
                break

        dist += 1

    dist = 0
    while len(right) < 2:
        # iterate through all possible x distances (inclusive)
        for x_dist in range(dist + 1):
            y_dist = dist - x_dist
            pos = (right_start[0] - x_dist, right_start[1] + y_dist)
            # if both coordinates are out of bounds, we stop
            if not (0 <= pos[0] < width) and not (0 <= pos[1] < height):
                raise ValueError("Not enough free initial positions.")
            # if one coordinate is out of bounds, we just continue
            if not (0 <= pos[0] < width) or not (0 <= pos[1] < height):
                continue
            # check if the new value is free
            if pos not in walls:
                right.append(pos)

            if len(right) == 2:
                break

        dist += 1

    # lower indices start further away
    left.reverse()
    right.reverse()
    return [left[0], right[0], left[1], right[1]]

initial_positions(walls=[(0, 0), (0, 1), (0, 2), (0, 3), (1, 0), (1, 3), (2, 0), (2, 1), (2, 3), (3, 0), (3, 1), (3, 3), (4, 0), (4, 1), (4, 3), (5, 0), (5, 3), (6, 0), (6, 3), (7, 0), (7, 1), (7, 2), (7, 3)])