Update shoalhaven_da_scraper.py

shoalhaven_da_scraper.py

@@ -55,9 +55,10 @@ def scrape():
         page.click("text=DA Tracking")
         time.sleep(4)
 
-        # Step 3: Advanced Search (robust selector)
-        page.locator("a:has-text('Advanced')").click()
-        time.sleep(3)
+        # Step 3: Advanced Search (robust ASP.NET tab click)
+        page.wait_for_selector("ul[role='tablist']")
+        page.locator("ul[role='tablist'] >> li").nth(1).click()
+        page.wait_for_timeout(2000)
 
         # Step 4: Date range
         page.fill("input[name='ctl00$ContentPlaceHolder1$txtFromDate']", "01/10/2025")
@@ -65,11 +66,13 @@ def scrape():
         page.click("text=Search")
         time.sleep(4)
 
-        # Show results
+        # Step 5: Show results
         page.click("text=Show")
         time.sleep(4)
 
+        # Step 6: Loop through result pages
         while True:
+            # Get all rows except header
             rows = page.query_selector_all("table tr")[1:]
 
             for row in rows:
@@ -82,9 +85,13 @@ def scrape():
                     continue
                 seen_da.add(da_number)
 
-                link = cols[0].query_selector("a").get_attribute("href")
+                link_tag = cols[0].query_selector("a")
+                if not link_tag:
+                    continue
+                link = link_tag.get_attribute("href")
                 detail_url = f"https://www3.shoalhaven.nsw.gov.au{link}"
 
+                # Open detail page
                 detail_page = context.new_page()
                 detail_page.goto(detail_url, timeout=60000)
                 time.sleep(3)
@@ -92,6 +99,8 @@ def scrape():
                 def get_value(label):
                     try:
                         el = detail_page.query_selector(f"text={label}")
+                        if not el:
+                            return ""
                         return el.evaluate(
                             "node => node.parentElement.nextElementSibling.innerText"
                         ).strip()
@@ -116,7 +125,7 @@ def scrape():
                 records.append(record)
                 detail_page.close()
 
-            # Pagination
+            # Pagination: check if "Next" button exists
             next_btn = page.query_selector("text=Next")
             if next_btn and next_btn.is_enabled():
                 next_btn.click()
@@ -126,6 +135,7 @@ def scrape():
 
         browser.close()
 
+    # Step 7: Save CSV
     df = pd.DataFrame(records, columns=HEADERS)
     df.to_csv(OUTPUT_FILE, index=False)
     print(f"Saved {len(records)} records to {OUTPUT_FILE}")