Download scripts/clean_labour_act.py from Sad1m/legallens: direct link, hf CLI and curl.
- Browser
- Download file 5.31 kB
-
https://huggingface.co/spaces/Sad1m/legallens/resolve/main/scripts/clean_labour_act.py
- Command line
-
hf download hf://spaces/Sad1m/legallens/scripts/clean_labour_act.py
-
curl -L -o clean_labour_act.py https://huggingface.co/spaces/Sad1m/legallens/resolve/main/scripts/clean_labour_act.py
5.31 kB
| """ | |
| Load PDF, extract main text, clean, parse into sections, save sections to disk | |
| """ | |
| import json | |
| import re | |
| import tiktoken | |
| from langchain_community.document_loaders import PyPDFLoader | |
| # Load raw PDF text | |
| def load_pdf_text(pdf_path): | |
| loader = PyPDFLoader(pdf_path) | |
| pages = loader.load() | |
| full_text = "\n".join([p.page_content for p in pages]) | |
| return full_text | |
| def parse_arrangement(raw_text): | |
| """ | |
| Extract official section titles from the Arrangement of Sections block. | |
| The real law starts at the second 'PART I' (after the preamble). | |
| Everything before that is front matter. | |
| """ | |
| # Find the start of the main Act: look for "PART I" that appears after "Commencement.]" | |
| start_marker = "Commencement.]" | |
| commence_idx = raw_text.find(start_marker) | |
| if commence_idx == -1: | |
| raise ValueError("Cannot find 'Commencement.]' marker") | |
| second_part = raw_text.find("PART I", commence_idx) | |
| if second_part == -1: | |
| raise ValueError("Cannot find second 'PART I' after Commencement]") | |
| front_matter = raw_text[:second_part] | |
| start = front_matter.upper().find("ARRANGEMENT OF SECTIONS") | |
| if start == -1: | |
| raise ValueError("Cannot find Arrangement of Sections in front matter") | |
| arrangement_text = front_matter[start:] | |
| pattern = r'^\s*(?P<num>\d+)\.\s+(?P<title>.+?)[.]?\s*$' | |
| mapping = {} | |
| for line in arrangement_text.split('\n'): | |
| m = re.match(pattern, line.strip()) | |
| if m: | |
| num = m.group('num') | |
| title = m.group('title').strip().rstrip('.') | |
| mapping[num] = title | |
| return mapping | |
| # Extract only the actual text | |
| def extract_main_text(raw_text): | |
| # Find "Commencement.]" then the following "PART I" | |
| start_marker = "Commencement.]" | |
| commence_idx = raw_text.find(start_marker) | |
| if commence_idx == -1: | |
| raise ValueError("Cannot find 'Commencement.]'") | |
| second_part = raw_text.find("PART I", commence_idx) | |
| if second_part == -1: | |
| raise ValueError("Cannot find second 'PART I'") | |
| main_text = raw_text[second_part:].strip() | |
| # Remove everything from the Schedule heading onwards | |
| # Look for "SCHEDULE" that appears after the last section (92) | |
| # Use regex to find a line that starts with "SCHEDULE" (possibly with spaces) | |
| schedule_match = re.search(r'\n\s*SCHEDULE\s*\n', main_text) | |
| if schedule_match: | |
| main_text = main_text[:schedule_match.start()].strip() | |
| return main_text | |
| # Clean headers and page numbers | |
| def clean_labour_act(text): | |
| # Remove page headers | |
| text = re.sub(r'\n?CAP\.\s*L\d+\s*\nLabour Act\s*\n', '\n', text, flags=re.IGNORECASE) | |
| text = re.sub(r'\n?CHAPTER\s*L\d+\s*\nLABOUR ACT\s*\n', '\n', text, flags=re.IGNORECASE) | |
| # Remove issue markers like "Ll-51 [Issue 1]" or "Ll-4" | |
| text = re.sub(r'\nLl\-\d+\s*\[Issue \d+\]\s*\n', '\n', text) | |
| text = re.sub(r'\nLl\-\d+\s*\n', '\n', text) | |
| # Remove standalone "[Issue 1]" lines | |
| text = re.sub(r'\n\[Issue \d+\]\s*\n', '\n', text) | |
| # Collapse excessive newlines | |
| text = re.sub(r'\n{3,}', '\n\n', text) | |
| return text.strip() | |
| # Parse sections using regex | |
| def parse_labour_sections(clean_text, title_map): | |
| text = "\n" + clean_text | |
| pattern = r'\n(?P<num>\d+)\.\s+(?P<body>.*?)(?=\n\d+\.\s+(?:[^\n]+\n)?|\Z)' | |
| matches = list(re.finditer(pattern, text, re.DOTALL)) | |
| sections = [] | |
| for m in matches: | |
| num = m.group('num') | |
| body = m.group('body').strip() | |
| if not body: | |
| continue | |
| # Stop if we have already processed section 92 (the last real section) | |
| if num == '92': | |
| # Add section 92 then break | |
| official_title = title_map.get(num, f"Section {num}") | |
| # Remove duplicated title line if present | |
| if body.startswith(official_title): | |
| body = body[len(official_title):].lstrip('\n').strip() | |
| sections.append({ | |
| 'source': 'Labour Act', | |
| 'section_number': num, | |
| 'title': official_title, | |
| 'content': body | |
| }) | |
| break | |
| official_title = title_map.get(num, f"Section {num}") | |
| if body.startswith(official_title): | |
| body = body[len(official_title):].lstrip('\n').strip() | |
| sections.append({ | |
| 'source': 'Labour Act', | |
| 'section_number': num, | |
| 'title': official_title, | |
| 'content': body | |
| }) | |
| return sections | |
| # Main execution, test | |
| if __name__ == "__main__": | |
| pdf_path = "data/raw/Labour_Act.pdf" | |
| raw_text = load_pdf_text(pdf_path) | |
| # Get the official titles | |
| title_map = parse_arrangement(raw_text) | |
| # Extract only the real provisions | |
| main_text = extract_main_text(raw_text) | |
| cleaned = clean_labour_act(main_text) | |
| # Parse sections with the correct titles | |
| sections = parse_labour_sections(cleaned, title_map) | |
| print(f"Found {len(sections)} sections.") | |
| for sec in sections[:5]: | |
| print(f"Section {sec['section_number']}: {sec['title']}") | |
| # Save | |
| import json | |
| with open("data/cleaned/labour_act_sections.json", "w", encoding="utf-8") as f: | |
| json.dump(sections, f, indent=2) | |
| print(f"Saved {len(sections)} sections to data/cleaned/labour_act_sections.json") |