| |
| """ |
| Junos book/reference parser. Handles three file shapes: |
| |
| 1. Day-One book format (encor-style): |
| - ## Chapter N: TOPIC |
| - ### SECTION HEADING (uppercase / short) |
| - ```code blocks``` |
| - ### Description sentence (sentence-case verb) |
| Used by: Beginner's Guide, Exploring the Junos CLI |
| |
| 2. CLI Reference index format: |
| - ## category — description (e.g. "## show — Display device information") |
| - one big code block listing one command per line |
| - no per-command descriptions |
| Used by: Junos OS CLI Reference |
| |
| 3. Both files may also have "Operational Mode (`>`)" / "Configuration Mode (`#`)" |
| subheadings indicating prompt context. |
| |
| Output: junos.json with {os, role, vendor, cat, title, cmd, desc} |
| """ |
| import re, json, pathlib |
| from collections import Counter |
|
|
| SRC_DIR = pathlib.Path(__file__).parent / "sources" |
| OUT = pathlib.Path(__file__).parent / "junos.json" |
| FILES = [ |
| "junos-dayone-beginners.md", |
| "junos-dayone-cli.md", |
| "junos-os-cli-reference.md", |
| ] |
|
|
| |
| SECTION_RECAT = [ |
| ("bgp", "BGP"), |
| ("ospf", "OSPF"), |
| ("isis", "Routing"), |
| ("is-is", "Routing"), |
| ("mpls", "MPLS"), |
| ("rsvp", "MPLS"), |
| ("ldp", "MPLS"), |
| ("vpls", "MPLS"), |
| ("vrf", "MPLS"), |
| ("evpn", "BGP"), |
| ("vxlan", "VLAN"), |
| ("vlan", "VLAN"), |
| ("ethernet-switching", "VLAN"), |
| ("spanning", "Spanning-Tree"), |
| ("rstp", "Spanning-Tree"), |
| ("mstp", "Spanning-Tree"), |
| ("lacp", "EtherChannel"), |
| ("aggregated", "EtherChannel"), |
| ("ae0", "EtherChannel"), |
| ("link-aggregation", "EtherChannel"), |
| ("multicast", "Multicast"), |
| ("pim", "Multicast"), |
| ("igmp", "Multicast"), |
| ("msdp", "Multicast"), |
| ("policy-statement", "Routing"), |
| ("policy-options", "Routing"), |
| ("route-filter", "Routing"), |
| ("prefix-list", "Routing"), |
| ("routing-options", "Routing"), |
| ("static route", "Static"), |
| ("routing policies", "Routing"), |
| ("firewall filter", "ACL"), |
| ("firewall family", "ACL"), |
| ("filter-list", "ACL"), |
| ("policer", "QoS"), |
| ("qos", "QoS"), |
| ("class-of-service", "QoS"), |
| ("scheduler", "QoS"), |
| ("forwarding-class", "QoS"), |
| ("security policy", "Firewall"), |
| ("security zone", "Firewall"), |
| ("security policies", "Firewall"), |
| ("security zones", "Firewall"), |
| ("security nat", "NAT"), |
| ("security ike", "VPN"), |
| ("security ipsec", "VPN"), |
| ("ipsec", "VPN"), |
| ("ike", "VPN"), |
| ("nat source", "NAT"), |
| ("nat destination", "NAT"), |
| ("nat static", "NAT"), |
| ("source nat", "NAT"), |
| ("destination nat", "NAT"), |
| ("static nat", "NAT"), |
| ("snmp", "SNMP"), |
| ("ntp", "NTP"), |
| ("system syslog", "Logging"), |
| ("syslog", "Logging"), |
| ("logging", "Logging"), |
| ("dhcp", "DHCP"), |
| ("radius", "AAA"), |
| ("tacacs", "AAA"), |
| ("authentication", "AAA"), |
| ("login class", "AAA"), |
| ("login user", "AAA"), |
| ("monitor traffic", "Troubleshooting"), |
| ("traceroute", "Troubleshooting"), |
| ("ping", "Troubleshooting"), |
| ("traceoptions", "Troubleshooting"), |
| ("show log", "Troubleshooting"), |
| ("show route", "Routing"), |
| ("show ospf", "OSPF"), |
| ("show bgp", "BGP"), |
| ("show isis", "Routing"), |
| ("show mpls", "MPLS"), |
| ("show interface", "Interfaces"), |
| ("show interfaces", "Interfaces"), |
| ("show chassis", "System"), |
| ("show system", "System"), |
| ("show security", "Firewall"), |
| ("show services", "Troubleshooting"), |
| ("show snmp", "SNMP"), |
| ("show ntp", "NTP"), |
| ("show vlans", "VLAN"), |
| ("show ethernet-switching", "VLAN"), |
| ("show spanning", "Spanning-Tree"), |
| ("show lldp", "Interfaces"), |
| ("show arp", "Interfaces"), |
| ("show ipv6 neighbors", "Interfaces"), |
| ("show dhcp", "DHCP"), |
| ("show pim", "Multicast"), |
| ("show igmp", "Multicast"), |
| ("show route protocol bgp", "BGP"), |
| ("show route protocol ospf", "OSPF"), |
| ("policies", "Routing"), |
| ("firewall filter concepts", "ACL"), |
| ("set interfaces", "Interfaces"), |
| ("set protocols ospf", "OSPF"), |
| ("set protocols bgp", "BGP"), |
| ("set protocols rstp", "Spanning-Tree"), |
| ("set protocols isis", "Routing"), |
| ("set protocols mpls", "MPLS"), |
| ("set protocols ldp", "MPLS"), |
| ("set protocols rsvp", "MPLS"), |
| ("set protocols igmp", "Multicast"), |
| ("set protocols pim", "Multicast"), |
| ("set firewall", "ACL"), |
| ("set vlans", "VLAN"), |
| ("set policy-options", "Routing"), |
| ("set routing-instances", "MPLS"), |
| ("set routing-options", "Routing"), |
| ("set security", "Firewall"), |
| ("set chassis cluster", "HA"), |
| ("set system", "System"), |
| ("set snmp", "SNMP"), |
| ("user interface", "System"), |
| ("operational monitoring", "Troubleshooting"), |
| ("routing fundamentals", "Routing"), |
| ("routing policies", "Routing"), |
| ("troubleshooting policies", "Routing"), |
| ("firewall filter", "ACL"), |
| ("fundamentals", "System"), |
| ("configuration basics", "System"), |
| ] |
|
|
| SKIP_DESC = {"note", "caution", "tip", "warning", "important", "best practice"} |
| DESC_STARTERS = re.compile( |
| r"^(creates?|configures?|configuring|sets?|setting|displays?|displaying|" |
| r"enables?|enabling|disables?|disabling|removes?|removing|specifies?|" |
| r"moves?|returns?|exits?|adds?|adding|assigns?|permits?|denies?|" |
| r"defines?|verif|matches?|matching|forces?|forcing|notes?|caution|" |
| r"this\b|to\b|use\b|when\b|if\b|after\b|step|increase|propagat|maps?|" |
| r"identif|associat|filters?|advertis|generat|propagates?|clears?|saves?|" |
| r"reduces?|lowers?|raises?|loads?|writes?|copies|sends?|forwards?|" |
| r"allows?|prevents?|restricts?|limits?|tracks?|checks?|monitors?|tells?|" |
| r"resets?|restarts?|reloads?|reboots?|powers?|requires?|let'?s|" |
| r"the\b|now\b|here|in this|by default|provides?|introduc|because|since|" |
| r"in fact|imagine|notice|but|so\b|that's|there|with|you\b|do\b|using|" |
| r"first|next|then|finally|enters?|switches?|navigates?|jumps?|select" |
| r")\b", |
| re.IGNORECASE, |
| ) |
|
|
| def looks_like_section(h: str) -> bool: |
| h = h.strip() |
| if not h: return True |
| if h.lower() in SKIP_DESC: return False |
| letters = [c for c in h if c.isalpha()] |
| if not letters: return True |
| uc_ratio = sum(1 for c in letters if c.isupper()) / len(letters) |
| if uc_ratio > 0.55: return True |
| |
| for sig in ("CONFIGURING","VERIFYING","TROUBLESHOOTING","COMMANDS","CONFIGURATION","EXAMPLE","OVERVIEW","FUNDAMENTALS","MONITORING","BUILDING","MODES"): |
| if sig in h.upper() and uc_ratio > 0.35: |
| return True |
| return False |
|
|
| def looks_like_description(h: str) -> bool: |
| if not h.strip(): return False |
| if h.strip().lower() in SKIP_DESC: return False |
| if looks_like_section(h): return False |
| if DESC_STARTERS.match(h.strip()): return True |
| if len(h) < 240 and h[0:1].isupper() and " " in h: |
| return True |
| return False |
|
|
| def categorize(default_cat: str, section_hdr: str, cmd: str) -> str: |
| hay = (section_hdr + " " + cmd).lower() |
| for k, c in SECTION_RECAT: |
| if k in hay: |
| return c |
| return default_cat |
|
|
| def role_of(cmd: str) -> str: |
| c = cmd.lower() |
| if any(k in c for k in ["security zone","security policies","security nat","security ike","security ipsec","chassis cluster","srx"]): |
| return "firewall" |
| if any(k in c for k in ["ethernet-switching","vlans vlan","spanning-tree","rstp","mstp","set vlans","virtual-chassis","poe"]): |
| return "switch" |
| return "router" |
|
|
| def is_command_line(line: str) -> bool: |
| l = line.strip() |
| if not l: return False |
| |
| l = re.sub(r"^\[edit[^\]]*\]\s*", "", l) |
| if len(l) > 280: return False |
| |
| if re.search(r"[.!?,]\s+\w+", l) and not l.startswith(("set ","show ","run ","delete ","edit ","commit","rollback","load","save","help ","monitor","ping","traceroute","clear","request","restart","configure","activate","deactivate","annotate","insert","copy","protect","rename","replace","wildcard","top","up","exit","end","file ","start ","ssh ","telnet ","test ","scp ")): |
| |
| if not re.match(r"^(set|show|run|delete|edit|commit|rollback|load|save|monitor|ping|traceroute|clear|request|restart|configure|help|file|test|start|telnet|ssh|scp|annotate|protect|wildcard|deactivate|activate|insert|copy|rename|replace|top\s|up$|exit$|end$)", l, re.IGNORECASE): |
| return False |
| if re.match(r"^(set|show|run|delete|edit|commit|rollback|load|save|monitor|ping|traceroute|clear|request|restart|configure|help|file|test|start|telnet|ssh|scp|annotate|protect|wildcard|deactivate|activate|insert|copy|rename|replace|top|up|exit|end|\?|<|>)", l, re.IGNORECASE): |
| return True |
| |
| if re.match(r"^(authentication|family|address|description|unit|disable|tunnel|local|remote|set\b)", l, re.IGNORECASE): |
| return True |
| return False |
|
|
| def parse_dayone(path: pathlib.Path, default_cat: str): |
| """encor-style parser: description heading AFTER code block.""" |
| text = path.read_text(encoding="utf-8", errors="ignore") |
| lines = text.split("\n") |
| out = [] |
| chapter_label = "" |
| section_hdr = "" |
| mode_prompt = "" |
| i, n = 0, len(lines) |
| while i < n: |
| line = lines[i] |
| m = re.match(r"^## .*Chapter\s+\d+:\s*(.+?)\s*$", line) |
| if m: |
| chapter_label = m.group(1) |
| section_hdr = chapter_label |
| i += 1; continue |
| m = re.match(r"^#{2,3}\s+(.+?)\s*$", line) |
| if m: |
| h = m.group(1).strip() |
| |
| if "operational mode" in h.lower(): mode_prompt = ">" |
| elif "configuration mode" in h.lower(): mode_prompt = "#" |
| elif looks_like_section(h): |
| section_hdr = h |
| i += 1; continue |
| |
| bm = re.match(r"^\*\*(.+?)\*\*\s*$", line) |
| if bm: |
| h = bm.group(1).strip() |
| if "operational mode" in h.lower(): mode_prompt = ">" |
| elif "configuration mode" in h.lower(): mode_prompt = "#" |
| i += 1; continue |
| if line.strip() == "```" or re.match(r"^```\s*\S*\s*$", line) or re.match(r"^```\s*`\[", line): |
| |
| block = [] |
| i += 1 |
| while i < n and not lines[i].strip().startswith("```"): |
| block.append(lines[i]) |
| i += 1 |
| i += 1 |
| |
| desc = "" |
| j = i |
| while j < n and j < i + 6: |
| ln = lines[j].strip() |
| if not ln: j += 1; continue |
| hm = re.match(r"^#{2,4}\s+(.+?)\s*$", ln) |
| if hm: |
| cand = hm.group(1).strip() |
| if looks_like_description(cand): |
| desc = cand |
| break |
| qm = re.match(r"^>\s+(.+)$", lines[j]) |
| if qm: |
| desc = qm.group(1).strip() |
| break |
| break |
| cleaned = [l for l in block if l.strip() and not l.strip().startswith("`[edit")] |
| if not cleaned: continue |
| has_indent = any(l.startswith(" ") or l.startswith("\t") for l in cleaned) |
| if has_indent or len(cleaned) == 1: |
| cmd = "\n".join(cleaned).rstrip() |
| if is_command_line(cleaned[0]): |
| cat = categorize(default_cat, section_hdr, cmd) |
| out.append({ |
| "os":"junos","role":role_of(cmd),"vendor":"Juniper","cat":cat, |
| "title": cmd.split("\n")[0][:80].strip(), |
| "cmd": cmd, |
| "desc": (desc or section_hdr)[:240], |
| }) |
| else: |
| for cl in cleaned: |
| if not is_command_line(cl): continue |
| cmd = cl.strip() |
| cat = categorize(default_cat, section_hdr, cmd) |
| out.append({ |
| "os":"junos","role":role_of(cmd),"vendor":"Juniper","cat":cat, |
| "title": cmd[:80], |
| "cmd": cmd, |
| "desc": (desc or section_hdr)[:240], |
| }) |
| continue |
| i += 1 |
| return out |
|
|
| def parse_index_ref(path: pathlib.Path): |
| """For the CLI Reference: ## category — desc, then one big code block of commands. |
| Each non-empty line = one command. No per-command description. |
| """ |
| text = path.read_text(encoding="utf-8", errors="ignore") |
| lines = text.split("\n") |
| out = [] |
| section_hdr = "" |
| default_cat = "System" |
| i, n = 0, len(lines) |
| while i < n: |
| line = lines[i] |
| m = re.match(r"^##\s+(\S+)\s*[—-]+\s*(.+?)\s*$", line) |
| if m: |
| section_hdr = m.group(1) + " - " + m.group(2) |
| verb = m.group(1).lower() |
| |
| if verb in {"show", "clear", "request", "restart", "monitor", "test", "file", "ssh", "telnet", "start", "help"}: |
| default_cat = "System" |
| elif verb in {"ping","traceroute"}: |
| default_cat = "Troubleshooting" |
| elif verb in {"set","delete","deactivate","activate","commit","rollback","load","save","annotate","copy","insert","protect","rename","replace","wildcard"}: |
| default_cat = "System" |
| i += 1; continue |
| m = re.match(r"^###?\s+(.+?)\s*$", line) |
| if m: |
| h = m.group(1).strip() |
| if looks_like_section(h): |
| section_hdr = h |
| i += 1; continue |
| if line.strip().startswith("```"): |
| block = []; i += 1 |
| while i < n and not lines[i].strip().startswith("```"): |
| block.append(lines[i]); i += 1 |
| i += 1 |
| |
| for cl in block: |
| cmd = cl.strip() |
| if not cmd: continue |
| |
| if '"' in cmd: |
| for part in cmd.split('"'): |
| part = part.strip() |
| if part and is_command_line(part) and len(part) >= 4: |
| cat = categorize(default_cat, section_hdr, part) |
| out.append({ |
| "os":"junos","role":role_of(part),"vendor":"Juniper","cat":cat, |
| "title": part[:80], |
| "cmd": part, |
| "desc": section_hdr[:240], |
| }) |
| continue |
| if is_command_line(cmd) and len(cmd) >= 3: |
| cat = categorize(default_cat, section_hdr, cmd) |
| out.append({ |
| "os":"junos","role":role_of(cmd),"vendor":"Juniper","cat":cat, |
| "title": cmd[:80], |
| "cmd": cmd, |
| "desc": section_hdr[:240], |
| }) |
| continue |
| i += 1 |
| return out |
|
|
| def main(): |
| items = [] |
| |
| for fname, defcat in [ |
| ("junos-dayone-beginners.md", "System"), |
| ("junos-dayone-cli.md", "System"), |
| ]: |
| p = SRC_DIR / fname |
| if not p.exists(): continue |
| new = parse_dayone(p, defcat) |
| items.extend(new) |
| print(f" parsed {fname}: {len(new)} entries") |
| |
| p = SRC_DIR / "junos-os-cli-reference.md" |
| if p.exists(): |
| new = parse_index_ref(p) |
| items.extend(new) |
| print(f" parsed junos-os-cli-reference.md: {len(new)} entries") |
| |
| seen = set(); deduped = [] |
| for it in items: |
| key = re.sub(r"\s+", " ", it["cmd"]).lower() |
| if key in seen: continue |
| seen.add(key); deduped.append(it) |
| OUT.write_text(json.dumps(deduped, indent=1, ensure_ascii=False)) |
| print(f"\nwrote {len(deduped)} Junos entries -> {OUT}") |
| print("by cat:", dict(Counter(c["cat"] for c in deduped))) |
| print("by role:", dict(Counter(c["role"] for c in deduped))) |
|
|
| if __name__ == "__main__": |
| main() |
|
|
