clementBE commited on
Commit
eece3fe
Β·
verified Β·
1 Parent(s): 805171c

Update app.py

Browse files
Files changed (1) hide show
  1. app.py +2981 -796
app.py CHANGED
@@ -1,203 +1,166 @@
1
- # api.py
2
- #
3
- # Flickr API helper for Hugging Face / Gradio image scraper
4
- #
5
- # Supports:
6
- # 1. Flickr search URLs
7
- # https://www.flickr.com/search/?text=ai
8
- #
9
- # 2. Individual photo URLs
10
- # https://www.flickr.com/photos/USERNAME/55555288566/
11
- #
12
- # 3. Flickr "with" URLs
13
- # https://www.flickr.com/photos/USERNAME/with/55555288566
14
- #
15
- # Main date fields:
16
- # - publication_date = Flickr upload/post date
17
- # - flickr_date_upload
18
- # - flickr_date_taken
19
- # - flickr_last_update
20
- #
21
- # Requires:
22
- # FLICKR_API_KEY environment variable
23
-
24
  import os
25
  import re
 
 
26
  import time
27
- from datetime import datetime, timezone
28
- from urllib.parse import urlparse, parse_qs, unquote
 
 
 
 
 
 
 
 
 
 
29
 
30
  import requests
 
31
 
 
 
32
 
33
- FLICKR_API_ENDPOINT = "https://www.flickr.com/services/rest/"
34
 
35
- FLICKR_API_KEY = os.getenv("FLICKR_API_KEY", "").strip()
 
 
36
 
37
- REQUEST_TIMEOUT = 30
38
 
39
- DEFAULT_EXTRAS = ",".join([
40
- "description",
41
- "license",
42
- "date_upload",
43
- "date_taken",
44
- "owner_name",
45
- "icon_server",
46
- "original_format",
47
- "last_update",
48
- "geo",
49
- "tags",
50
- "machine_tags",
51
- "o_dims",
52
- "views",
53
- "media",
54
- "path_alias",
55
- "url_sq",
56
- "url_t",
57
- "url_s",
58
- "url_q",
59
- "url_m",
60
- "url_n",
61
- "url_z",
62
- "url_c",
63
- "url_l",
64
- "url_o",
65
- ])
66
-
67
-
68
- # ---------------------------------------------------------------------
69
- # Basic helpers
70
- # ---------------------------------------------------------------------
71
-
72
- def require_api_key():
73
- """
74
- Raise a clear error if FLICKR_API_KEY is not configured.
75
- """
76
- if not FLICKR_API_KEY:
77
- raise RuntimeError(
78
- "FLICKR_API_KEY is not configured. "
79
- "Add it as a Hugging Face Space Secret."
80
- )
81
 
 
 
 
82
 
83
- def flickr_timestamp_to_utc(value):
84
- """
85
- Convert Flickr Unix timestamp to UTC datetime string.
86
 
87
- Flickr's 'posted' / 'dateupload' timestamps are Unix timestamps
88
- expressed in GMT/UTC.
89
- """
90
- if value in (None, "", 0, "0"):
91
- return None
92
 
93
- try:
94
- dt = datetime.fromtimestamp(
95
- int(float(value)),
96
- tz=timezone.utc
97
- )
98
 
99
- return dt.strftime("%Y-%m-%d %H:%M:%S UTC")
 
 
 
 
100
 
101
- except Exception:
102
- return None
 
 
 
 
 
 
 
 
103
 
104
 
105
- def flickr_taken_to_string(value):
106
- """
107
- Flickr's date taken is already a MySQL-style datetime.
108
 
109
- IMPORTANT:
110
- We do not convert this timezone because Flickr states that
111
- taken dates are represented in the photo owner's timezone.
112
- """
113
- if not value:
114
- return None
115
 
116
- value = str(value).strip()
 
 
 
117
 
118
- if value in ("0000-00-00 00:00:00", "0000-00-00"):
119
- return None
120
 
121
- return value
 
 
122
 
 
 
 
 
 
123
 
124
- def extract_flickr_photo_id(url):
125
- """
126
- Extract a Flickr photo ID from URLs such as:
127
 
128
- /photos/user/55555288566/
129
- /photos/user/with/55555288566
130
- /photos/user/55555288566/in/dateposted/
131
- """
132
 
133
- if not url:
134
- return None
 
135
 
136
- try:
137
- parsed = urlparse(url)
138
- path = unquote(parsed.path)
139
 
140
- # /photos/user/with/55555288566
141
- match = re.search(
142
- r"/photos/[^/]+/with/(\d+)",
143
- path,
144
- flags=re.IGNORECASE
145
- )
146
 
147
- if match:
148
- return match.group(1)
 
149
 
150
- # /photos/user/55555288566
151
- match = re.search(
152
- r"/photos/[^/]+/(\d+)",
153
- path,
154
- flags=re.IGNORECASE
155
- )
156
 
157
- if match:
158
- return match.group(1)
159
 
160
- except Exception:
161
- pass
 
 
 
162
 
163
- return None
164
 
 
 
 
165
 
166
- def extract_flickr_username(url):
167
- """
168
- Extract username/path alias from a Flickr photo URL.
169
- """
170
  if not url:
171
- return None
172
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
173
  try:
 
174
  parsed = urlparse(url)
175
- path = unquote(parsed.path)
176
 
177
- match = re.search(
178
- r"/photos/([^/]+)/",
179
- path,
180
- flags=re.IGNORECASE
181
  )
182
 
183
- if match:
184
- return match.group(1)
185
-
186
  except Exception:
187
- pass
188
 
189
- return None
190
 
191
 
192
  def is_flickr_url(url):
193
- """
194
- True for Flickr URLs.
195
- """
196
- if not url:
197
- return False
198
-
199
  try:
200
- host = urlparse(url).netloc.lower()
 
 
 
 
 
201
 
202
  return (
203
  host == "flickr.com"
@@ -205,903 +168,3125 @@ def is_flickr_url(url):
205
  )
206
 
207
  except Exception:
 
208
  return False
209
 
210
 
211
  def extract_flickr_search_text(url):
212
- """
213
- Extract ?text=... from Flickr search URLs.
214
- """
215
- if not url:
216
- return None
217
 
218
  try:
 
219
  parsed = urlparse(url)
220
- params = parse_qs(parsed.query)
221
 
222
- text = params.get("text", [None])[0]
 
 
 
 
 
 
 
223
 
224
- if text:
225
- return unquote(text).strip()
 
226
 
227
  except Exception:
228
- pass
229
 
230
- return None
231
 
232
 
233
- # ---------------------------------------------------------------------
234
- # API request
235
- # ---------------------------------------------------------------------
 
236
 
237
- def flickr_api_call(method, params=None, retries=3):
238
- """
239
- Generic Flickr REST API call.
240
 
241
- Returns decoded JSON.
242
 
243
- Raises RuntimeError with useful information on API errors.
244
- """
 
 
 
245
 
246
- require_api_key()
 
 
247
 
248
- payload = {
249
- "method": method,
250
- "api_key": FLICKR_API_KEY,
251
- "format": "json",
252
- "nojsoncallback": 1,
253
- }
254
 
255
- if params:
256
- payload.update(params)
257
 
258
- last_error = None
259
 
260
- for attempt in range(1, retries + 1):
261
 
262
- try:
 
 
 
 
 
263
 
264
- response = requests.get(
265
- FLICKR_API_ENDPOINT,
266
- params=payload,
267
- timeout=REQUEST_TIMEOUT,
268
- headers={
269
- "User-Agent": (
270
- "Mozilla/5.0 "
271
- "(compatible; FlickrResearchImageDownloader/1.0)"
272
- )
273
- },
274
- )
275
 
276
- response.raise_for_status()
 
 
277
 
278
- data = response.json()
 
279
 
280
- if data.get("stat") != "ok":
 
 
 
281
 
282
- code = data.get("code", "unknown")
283
 
284
- message = data.get(
285
- "message",
286
- "Unknown Flickr API error"
287
- )
 
 
 
 
288
 
289
- raise RuntimeError(
290
- f"Flickr API error {code}: {message}"
291
- )
292
 
293
- return data
 
 
 
294
 
295
- except Exception as exc:
296
 
297
- last_error = exc
 
 
298
 
299
- if attempt < retries:
300
- time.sleep(1.5 * attempt)
301
 
302
- raise RuntimeError(
303
- f"Flickr API request failed after {retries} attempts: "
304
- f"{last_error}"
305
- )
306
 
 
 
 
 
 
307
 
308
- # ---------------------------------------------------------------------
309
- # Photo normalization
310
- # ---------------------------------------------------------------------
 
311
 
312
- def choose_best_flickr_image_url(photo):
313
- """
314
- Select the highest available Flickr image URL.
315
- """
316
 
317
- # Highest β†’ lowest priority
318
- candidates = [
319
- "url_o",
320
- "url_l",
321
- "url_c",
322
- "url_z",
323
- "url_n",
324
- "url_m",
325
- "url_q",
326
- "url_s",
327
- "url_t",
328
- ]
329
 
330
- for field in candidates:
331
 
332
- value = photo.get(field)
 
 
 
 
333
 
334
- if value:
335
- return value
336
 
337
- return None
 
338
 
 
 
339
 
340
- def build_flickr_photo_record(photo):
341
- """
342
- Convert Flickr API photo object into the standardized metadata
343
- structure used by the scraper.
344
- """
345
 
346
- photo_id = str(photo.get("id", "")).strip()
347
 
348
- owner_id = photo.get("owner")
 
 
 
 
349
 
350
- owner_name = photo.get("ownername")
 
351
 
352
- title = photo.get("title", "")
 
 
 
353
 
354
- description = photo.get("description", "")
355
 
356
- if isinstance(description, dict):
357
- description = description.get("_content", "")
358
 
359
- image_url = choose_best_flickr_image_url(photo)
360
 
361
- # -------------------------------------------------------------
362
- # Dates
363
- # -------------------------------------------------------------
364
 
365
- date_upload_raw = photo.get("dateupload")
 
 
366
 
367
- date_taken_raw = photo.get("datetaken")
368
 
369
- last_update_raw = photo.get("lastupdate")
 
 
370
 
371
- flickr_date_upload = flickr_timestamp_to_utc(
372
- date_upload_raw
373
- )
374
 
375
- flickr_date_taken = flickr_taken_to_string(
376
- date_taken_raw
377
- )
378
 
379
- flickr_last_update = flickr_timestamp_to_utc(
380
- last_update_raw
381
- )
382
 
383
- # Publication date means upload/post date.
384
- publication_date = flickr_date_upload
 
 
385
 
386
- # -------------------------------------------------------------
387
- # Dimensions
388
- # -------------------------------------------------------------
389
 
390
- width = photo.get("width_o")
391
 
392
- height = photo.get("height_o")
 
 
 
393
 
394
- if not width:
395
- width = photo.get("width_l")
 
 
396
 
397
- if not height:
398
- height = photo.get("height_l")
 
 
399
 
400
- # Some API responses provide o_dims="1234x5678"
401
- if (not width or not height) and photo.get("o_dims"):
 
 
402
 
403
- try:
404
- dimensions = str(photo["o_dims"])
405
 
406
- match = re.match(
407
- r"^\s*(\d+)\s*x\s*(\d+)\s*$",
408
- dimensions
409
  )
410
 
411
- if match:
412
- width = int(match.group(1))
413
- height = int(match.group(2))
414
 
415
- except Exception:
416
- pass
 
417
 
418
- # -------------------------------------------------------------
419
- # Photo page
420
- # -------------------------------------------------------------
 
 
421
 
422
- photo_page_url = (
423
- f"https://www.flickr.com/photos/"
424
- f"{owner_id}/{photo_id}/"
425
- if owner_id and photo_id
426
- else None
427
- )
428
 
429
- # -------------------------------------------------------------
430
- # Tags
431
- # -------------------------------------------------------------
432
 
433
- tags = photo.get("tags", "")
434
 
435
- if isinstance(tags, list):
436
- tags = ", ".join(
437
- str(x) for x in tags
438
- )
439
 
440
- # -------------------------------------------------------------
441
- # Geographic information
442
- # -------------------------------------------------------------
443
 
444
- latitude = photo.get("latitude")
445
 
446
- longitude = photo.get("longitude")
 
 
447
 
448
- try:
449
- latitude = float(latitude) if latitude not in (None, "") else None
450
- except Exception:
451
- latitude = None
452
 
453
- try:
454
- longitude = (
455
- float(longitude)
456
- if longitude not in (None, "")
457
- else None
458
- )
459
- except Exception:
460
- longitude = None
 
 
 
 
 
 
 
 
 
461
 
462
- # -------------------------------------------------------------
463
- # Result
464
- # -------------------------------------------------------------
465
 
466
- record = {
467
 
468
- # Identification
469
- "source": "Flickr API",
470
- "flickr_photo_id": photo_id,
471
- "flickr_owner_id": owner_id,
472
- "flickr_owner_name": owner_name,
473
 
474
- # URLs
475
- "url": image_url,
476
- "image_url": image_url,
477
- "page_url": photo_page_url,
478
 
479
- # Basic metadata
480
- "title": title,
481
- "description": description,
482
 
483
- # Dates
484
- "publication_date": publication_date,
485
- "publication_date_source": (
486
- "Flickr API dateupload"
487
- if publication_date
488
- else None
489
- ),
490
-
491
- "flickr_date_upload": flickr_date_upload,
492
- "flickr_date_upload_timestamp": date_upload_raw,
493
-
494
- "flickr_date_taken": flickr_date_taken,
495
- "flickr_date_taken_granularity": (
496
- photo.get("datetakengranularity")
497
- ),
498
-
499
- "flickr_last_update": flickr_last_update,
500
- "flickr_last_update_timestamp": last_update_raw,
501
-
502
- # Image
503
- "width": width,
504
- "height": height,
505
- "original_format": photo.get("originalformat"),
506
- "media": photo.get("media"),
507
-
508
- # Flickr statistics
509
- "views": photo.get("views"),
510
- "license": photo.get("license"),
511
-
512
- # Tags
513
- "tags": tags,
514
- "machine_tags": photo.get("machine_tags"),
515
-
516
- # Location
517
- "latitude": latitude,
518
- "longitude": longitude,
519
-
520
- # Download fields populated later
521
- "local_file": None,
522
- "filename": None,
523
- "file_size": None,
524
- "final_url": image_url,
525
- "download_status": None,
526
 
527
- }
 
 
 
528
 
529
- return record
530
 
 
 
 
 
 
 
 
 
531
 
532
- # ---------------------------------------------------------------------
533
- # Get one Flickr photo
534
- # ---------------------------------------------------------------------
 
 
 
 
 
535
 
536
- def flickr_get_photo(photo_id):
537
- """
538
- Retrieve complete metadata for one Flickr photo.
 
 
 
 
 
539
 
540
- Uses:
541
- flickr.photos.getInfo
542
- """
 
 
 
 
 
543
 
544
- if not photo_id:
545
- raise ValueError("Missing Flickr photo ID.")
 
 
 
 
 
 
546
 
547
- data = flickr_api_call(
548
- "flickr.photos.getInfo",
549
- {
550
- "photo_id": str(photo_id),
551
- }
552
- )
 
 
553
 
554
- photo = data.get("photo")
 
 
 
 
 
 
 
555
 
556
- if not photo:
557
- raise RuntimeError(
558
- f"No photo data returned for Flickr ID {photo_id}"
 
 
 
 
559
  )
560
 
561
- # getInfo has a nested structure which differs from search.
562
- normalized = {
563
- "id": photo.get("id"),
564
 
565
- "owner": (
566
- photo.get("owner", {}).get("nsid")
567
- if isinstance(photo.get("owner"), dict)
568
- else None
569
- ),
570
 
571
- "ownername": (
572
- photo.get("owner", {}).get("username")
573
- if isinstance(photo.get("owner"), dict)
574
- else None
575
- ),
576
 
577
- "title": (
578
- photo.get("title", {}).get("_content", "")
579
- if isinstance(photo.get("title"), dict)
580
- else photo.get("title", "")
581
- ),
582
 
583
- "description": (
584
- photo.get("description", {}).get("_content", "")
585
- if isinstance(photo.get("description"), dict)
586
- else photo.get("description", "")
587
- ),
588
 
589
- "license": photo.get("license"),
 
 
 
 
 
 
 
 
 
 
 
590
 
591
- "rotation": photo.get("rotation"),
592
 
593
- "originalformat": photo.get("originalformat"),
 
 
 
 
594
 
595
- }
596
 
597
- # -------------------------------------------------------------
598
- # Dates from <dates ... />
599
- # -------------------------------------------------------------
 
 
600
 
601
- dates = photo.get("dates", {})
 
 
 
 
 
 
 
 
 
 
602
 
603
- if isinstance(dates, dict):
 
 
 
 
 
 
 
604
 
605
- normalized["dateupload"] = dates.get("posted")
 
 
 
 
 
 
 
606
 
607
- normalized["datetaken"] = dates.get("taken")
 
 
608
 
609
- normalized["datetakengranularity"] = dates.get(
610
- "takengranularity"
611
  )
612
 
613
- normalized["lastupdate"] = dates.get(
614
- "lastupdate"
615
- )
616
 
617
- # -------------------------------------------------------------
618
- # URLs
619
- # -------------------------------------------------------------
620
 
621
- urls = photo.get("urls", {}).get("url", [])
622
 
623
- photo_page_url = None
624
 
625
- if isinstance(urls, list):
 
 
 
626
 
627
- for item in urls:
 
 
628
 
629
- if (
630
- isinstance(item, dict)
631
- and item.get("type") == "photopage"
632
- ):
633
- photo_page_url = item.get("_content")
634
- break
635
 
636
- normalized["photo_page_url"] = photo_page_url
 
 
637
 
638
- # -------------------------------------------------------------
639
- # Tags
640
- # -------------------------------------------------------------
641
 
642
- tag_list = photo.get("tags", {}).get("tag", [])
 
 
643
 
644
- tags = []
 
 
645
 
646
- if isinstance(tag_list, list):
 
 
 
647
 
648
- for tag in tag_list:
 
 
 
649
 
650
- if isinstance(tag, dict):
651
 
652
- raw = tag.get("raw")
 
 
 
 
 
653
 
654
- content = tag.get("_content")
655
 
656
- tags.append(
657
- raw or content or ""
 
 
 
658
  )
659
 
660
- normalized["tags"] = ", ".join(
661
- x for x in tags if x
662
- )
663
 
664
- # -------------------------------------------------------------
665
- # Location
666
- # -------------------------------------------------------------
667
 
668
- location = photo.get("location", {})
669
 
670
- if isinstance(location, dict):
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
671
 
672
- normalized["latitude"] = location.get("latitude")
673
 
674
- normalized["longitude"] = location.get("longitude")
 
 
 
 
675
 
676
- # -------------------------------------------------------------
677
- # Get image URL through the size API
678
- # -------------------------------------------------------------
679
 
680
- try:
681
 
682
- sizes_data = flickr_api_call(
683
- "flickr.photos.getSizes",
684
- {
685
- "photo_id": str(photo_id)
686
- }
687
- )
688
 
689
- sizes = (
690
- sizes_data
691
- .get("sizes", {})
692
- .get("size", [])
693
- )
694
 
695
- best_url = None
696
- best_width = None
697
- best_height = None
698
 
699
- if isinstance(sizes, list):
 
 
 
 
 
700
 
701
- for size in sizes:
702
 
703
- if not isinstance(size, dict):
704
- continue
 
 
 
705
 
706
- source = size.get("source")
 
 
 
707
 
708
- if not source:
709
- continue
710
 
711
- try:
712
- w = int(size.get("width", 0))
713
- except Exception:
714
- w = 0
715
 
716
- try:
717
- h = int(size.get("height", 0))
718
- except Exception:
719
- h = 0
720
 
721
- if (
722
- best_width is None
723
- or w * h > best_width * best_height
724
- ):
725
- best_url = source
726
- best_width = w
727
- best_height = h
728
 
729
- normalized["url_o"] = best_url
730
- normalized["width_o"] = best_width
731
- normalized["height_o"] = best_height
732
 
733
- except Exception as exc:
 
 
 
 
 
 
734
 
735
- normalized["url_o"] = None
736
- normalized["width_o"] = None
737
- normalized["height_o"] = None
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
738
 
739
- normalized["_sizes_error"] = str(exc)
 
 
 
 
740
 
741
- return build_flickr_photo_record(normalized)
742
 
 
743
 
744
- # ---------------------------------------------------------------------
745
- # Flickr search
746
- # ---------------------------------------------------------------------
747
 
748
- def flickr_search(
749
- text=None,
750
- user_id=None,
751
- tags=None,
752
- max_images=100,
753
- sort="date-posted-desc",
754
- ):
755
- """
756
- Search Flickr photos.
757
 
758
- Parameters
759
- ----------
760
- text:
761
- Flickr free-text search.
762
 
763
- user_id:
764
- Flickr NSID if searching a specific account.
 
 
765
 
766
- tags:
767
- Flickr tag search.
 
 
768
 
769
- max_images:
770
- Maximum number of returned photos.
771
 
772
- sort:
773
- Flickr sorting mode.
774
 
775
- Returns
776
- -------
777
- list[dict]
778
- """
779
 
780
- require_api_key()
 
 
 
 
 
 
 
 
781
 
782
- max_images = max(
783
- 1,
784
- min(int(max_images), 500)
785
- )
786
 
787
- data = flickr_api_call(
788
- "flickr.photos.search",
789
- {
790
- "text": text or "",
791
- "user_id": user_id or "",
792
- "tags": tags or "",
793
- "sort": sort,
794
- "media": "all",
795
- "per_page": max_images,
796
- "page": 1,
797
- "extras": DEFAULT_EXTRAS,
798
- }
799
- )
800
 
801
- photos_container = data.get(
802
- "photos",
803
- {}
804
- )
805
 
806
- photos = photos_container.get(
807
- "photo",
808
- []
809
- )
810
 
811
- results = []
 
 
812
 
813
- for photo in photos:
814
 
815
- try:
 
816
 
817
- record = build_flickr_photo_record(
818
- photo
 
 
 
 
819
  )
820
 
821
- if record.get("url"):
 
 
 
 
 
 
822
 
823
- results.append(record)
 
 
 
824
 
825
- except Exception as exc:
 
 
 
826
 
827
- # Keep failed API records instead of crashing
828
- results.append({
829
- "source": "Flickr API",
830
- "download_status": (
831
- f"metadata_error: {exc}"
832
- ),
833
- })
834
 
835
- return results
 
 
 
 
 
836
 
 
 
 
 
 
 
837
 
838
- # ---------------------------------------------------------------------
839
- # Parse Flickr URL
840
- # ---------------------------------------------------------------------
 
 
841
 
842
- def parse_flickr_url(url):
843
- """
844
- Determine what type of Flickr URL we have.
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
845
 
846
- Returns:
 
847
 
848
- {
849
- "type": "photo",
850
- "photo_id": "...",
851
- "username": "..."
852
- }
 
853
 
854
- or
 
855
 
856
- {
857
- "type": "search",
858
- "text": "..."
859
- }
860
 
861
- """
862
 
863
- if not is_flickr_url(url):
864
 
865
- return {
866
- "type": "unknown",
867
- "photo_id": None,
868
- "username": None,
869
- "text": None,
870
- }
871
 
872
- photo_id = extract_flickr_photo_id(url)
 
 
 
873
 
874
- if photo_id:
 
 
 
875
 
876
- return {
877
- "type": "photo",
878
- "photo_id": photo_id,
879
- "username": extract_flickr_username(url),
880
- "text": None,
881
- }
882
 
883
- text = extract_flickr_search_text(url)
 
 
884
 
885
- if text:
 
 
886
 
887
- return {
888
- "type": "search",
889
- "photo_id": None,
890
- "username": None,
891
- "text": text,
892
- }
893
 
894
- # Generic Flickr search URL without text
895
- parsed = urlparse(url)
 
 
 
 
 
896
 
897
- if "/search" in parsed.path.lower():
898
 
899
- return {
900
- "type": "search",
901
- "photo_id": None,
902
- "username": None,
903
- "text": None,
904
- }
905
 
906
- return {
907
- "type": "unknown",
908
- "photo_id": None,
909
- "username": None,
910
- "text": None,
911
- }
912
 
 
 
913
 
914
- # ---------------------------------------------------------------------
915
- # Main URL handler
916
- # ---------------------------------------------------------------------
917
 
918
- def flickr_from_url(
919
- url,
920
- max_images=100,
921
- ):
922
- """
923
- Main entry point.
924
 
925
- Automatically decides whether to use:
926
 
927
- flickr.photos.getInfo
 
 
928
 
929
- or:
930
 
931
- flickr.photos.search
932
- """
 
 
 
 
 
 
 
933
 
934
- parsed = parse_flickr_url(url)
935
 
936
- url_type = parsed["type"]
 
 
 
 
 
 
 
 
937
 
938
- # -------------------------------------------------------------
939
- # Individual photo / /with/ URL
940
- # -------------------------------------------------------------
941
 
942
- if url_type == "photo":
 
 
943
 
944
- record = flickr_get_photo(
945
- parsed["photo_id"]
946
  )
947
 
948
- return [record]
 
949
 
950
- # -------------------------------------------------------------
951
- # Search URL
952
- # -------------------------------------------------------------
953
 
954
- if url_type == "search":
 
 
 
955
 
956
- return flickr_search(
957
- text=parsed.get("text"),
958
- max_images=max_images,
959
- )
960
 
961
- raise ValueError(
962
- "The Flickr URL could not be recognized. "
963
- "Supported formats include Flickr search URLs, "
964
- "individual photo URLs, and /with/PHOTO_ID URLs."
965
- )
 
966
 
 
 
 
967
 
968
- # ---------------------------------------------------------------------
969
- # Convenience function for your app.py
970
- # ---------------------------------------------------------------------
 
 
 
971
 
972
- def get_flickr_images(
973
- url,
974
- max_images=100,
975
- ):
976
- """
977
- Compatibility wrapper.
978
 
979
- Your app.py can simply call:
980
 
981
- records = get_flickr_images(url, max_images)
 
 
 
 
 
982
 
983
- """
 
 
984
 
985
- return flickr_from_url(
986
- url=url,
987
- max_images=max_images,
988
- )
989
 
 
990
 
991
- # ---------------------------------------------------------------------
992
- # Test / diagnostics
993
- # ---------------------------------------------------------------------
994
 
995
- def diagnose_flickr_url(url):
996
- """
997
- Return diagnostic information without downloading images.
998
- Useful for debugging a Hugging Face Space.
999
- """
1000
 
1001
- result = {
1002
- "url": url,
1003
- "is_flickr": is_flickr_url(url),
1004
- "url_type": None,
1005
- "photo_id": None,
1006
- "username": None,
1007
- "search_text": None,
1008
- "api_key_configured": bool(FLICKR_API_KEY),
1009
- }
1010
 
1011
- parsed = parse_flickr_url(url)
 
 
 
1012
 
1013
- result["url_type"] = parsed.get("type")
1014
- result["photo_id"] = parsed.get("photo_id")
1015
- result["username"] = parsed.get("username")
1016
- result["search_text"] = parsed.get("text")
1017
 
1018
- return result
 
1019
 
 
 
1020
 
1021
- # ---------------------------------------------------------------------
1022
- # Direct test
1023
- # ---------------------------------------------------------------------
1024
 
1025
- if __name__ == "__main__":
 
1026
 
1027
- TEST_URL = (
1028
- "https://www.flickr.com/"
1029
- "photos/snapnpiks0304/"
1030
- "with/55555288566"
1031
- )
1032
 
1033
- print("=" * 70)
1034
- print("FLICKR API TEST")
1035
- print("=" * 70)
1036
 
1037
- print("\nAPI key configured:", bool(FLICKR_API_KEY))
 
1038
 
1039
- diagnostic = diagnose_flickr_url(
1040
- TEST_URL
1041
- )
1042
 
1043
- print("\nURL diagnostic:")
 
1044
 
1045
- for key, value in diagnostic.items():
1046
- print(f" {key}: {value}")
1047
 
1048
- if FLICKR_API_KEY:
 
1049
 
1050
- print("\nRetrieving Flickr metadata...")
 
1051
 
1052
- try:
 
1053
 
1054
- records = get_flickr_images(
1055
- TEST_URL,
1056
- max_images=1
1057
- )
1058
-
1059
- if records:
1060
-
1061
- record = records[0]
1062
-
1063
- print("\nRESULT")
1064
- print("-" * 70)
1065
-
1066
- important_fields = [
1067
- "flickr_photo_id",
1068
- "flickr_owner_name",
1069
- "title",
1070
- "publication_date",
1071
- "publication_date_source",
1072
- "flickr_date_upload",
1073
- "flickr_date_taken",
1074
- "flickr_date_taken_granularity",
1075
- "flickr_last_update",
1076
- "width",
1077
- "height",
1078
- "views",
1079
- "url",
1080
- "page_url",
1081
- ]
1082
 
1083
- for field in important_fields:
 
1084
 
1085
- print(
1086
- f"{field}: "
1087
- f"{record.get(field)}"
1088
- )
1089
 
1090
- else:
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1091
 
1092
- print(
1093
- "No Flickr record returned."
 
 
 
 
 
 
 
1094
  )
1095
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1096
  except Exception as exc:
1097
 
1098
- print(
1099
- "\nERROR:",
1100
- exc
1101
  )
1102
 
1103
- else:
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1104
 
1105
- print(
1106
- "\nSet FLICKR_API_KEY before testing."
1107
- )
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
  import os
2
  import re
3
+ import csv
4
+ import io
5
  import time
6
+ import hashlib
7
+ import zipfile
8
+ import tempfile
9
+ import shutil
10
+ import math
11
+ from pathlib import Path
12
+ from urllib.parse import (
13
+ urlparse,
14
+ parse_qs,
15
+ unquote,
16
+ urljoin,
17
+ )
18
 
19
  import requests
20
+ from bs4 import BeautifulSoup
21
 
22
+ import pandas as pd
23
+ import numpy as np
24
 
25
+ from PIL import Image, ExifTags
26
 
27
+ import matplotlib
28
+ matplotlib.use("Agg")
29
+ import matplotlib.pyplot as plt
30
 
31
+ import gradio as gr
32
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
33
 
34
+ # ============================================================
35
+ # CONFIGURATION
36
+ # ============================================================
37
 
38
+ APP_TITLE = "Web & Flickr Image Downloader + Metadata"
 
 
39
 
40
+ MAX_IMAGES_HARD_LIMIT = 500
 
 
 
 
41
 
42
+ REQUEST_TIMEOUT = 30
43
+ IMAGE_TIMEOUT = 45
 
 
 
44
 
45
+ USER_AGENT = (
46
+ "Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
47
+ "AppleWebKit/537.36 (KHTML, like Gecko) "
48
+ "Chrome/140.0 Safari/537.36"
49
+ )
50
 
51
+ HEADERS = {
52
+ "User-Agent": USER_AGENT,
53
+ "Accept": (
54
+ "text/html,application/xhtml+xml,"
55
+ "application/xml;q=0.9,"
56
+ "image/avif,image/webp,image/apng,"
57
+ "*/*;q=0.8"
58
+ ),
59
+ "Accept-Language": "en-US,en;q=0.9,fr;q=0.8",
60
+ }
61
 
62
 
63
+ # ============================================================
64
+ # TEMPORARY STORAGE
65
+ # ============================================================
66
 
67
+ BASE_DIR = (
68
+ Path(tempfile.gettempdir())
69
+ / "gradio_web_image_downloader"
70
+ )
 
 
71
 
72
+ BASE_DIR.mkdir(
73
+ parents=True,
74
+ exist_ok=True
75
+ )
76
 
 
 
77
 
78
+ # ============================================================
79
+ # FLICKR API
80
+ # ============================================================
81
 
82
+ FLICKR_API_KEY = (
83
+ os.environ
84
+ .get("FLICKR_API_KEY", "")
85
+ .strip()
86
+ )
87
 
88
+ FLICKR_API_URL = (
89
+ "https://api.flickr.com/services/rest/"
90
+ )
91
 
 
 
 
 
92
 
93
+ # ============================================================
94
+ # HTTP SESSION
95
+ # ============================================================
96
 
97
+ session = requests.Session()
98
+ session.headers.update(HEADERS)
 
99
 
 
 
 
 
 
 
100
 
101
+ # ============================================================
102
+ # EXIF TAGS
103
+ # ============================================================
104
 
105
+ EXIF_TAGS = {
106
+ tag_id: tag_name
107
+ for tag_id, tag_name
108
+ in ExifTags.TAGS.items()
109
+ }
 
110
 
 
 
111
 
112
+ GPS_TAGS = {
113
+ tag_id: tag_name
114
+ for tag_id, tag_name
115
+ in ExifTags.GPSTAGS.items()
116
+ }
117
 
 
118
 
119
+ # ============================================================
120
+ # HELPERS
121
+ # ============================================================
122
 
123
+ def clean_url(url):
 
 
 
124
  if not url:
125
+ return ""
126
 
127
+ url = str(url).strip()
128
+
129
+ if not url:
130
+ return ""
131
+
132
+ if not url.startswith(
133
+ ("http://", "https://")
134
+ ):
135
+ url = "https://" + url
136
+
137
+ return url
138
+
139
+
140
+ def valid_http_url(url):
141
  try:
142
+
143
  parsed = urlparse(url)
 
144
 
145
+ return (
146
+ parsed.scheme in
147
+ ("http", "https")
148
+ and bool(parsed.netloc)
149
  )
150
 
 
 
 
151
  except Exception:
 
152
 
153
+ return False
154
 
155
 
156
  def is_flickr_url(url):
 
 
 
 
 
 
157
  try:
158
+
159
+ host = (
160
+ urlparse(url)
161
+ .netloc
162
+ .lower()
163
+ )
164
 
165
  return (
166
  host == "flickr.com"
 
168
  )
169
 
170
  except Exception:
171
+
172
  return False
173
 
174
 
175
  def extract_flickr_search_text(url):
 
 
 
 
 
176
 
177
  try:
178
+
179
  parsed = urlparse(url)
 
180
 
181
+ query = parse_qs(
182
+ parsed.query
183
+ )
184
+
185
+ text = query.get(
186
+ "text",
187
+ [""]
188
+ )[0]
189
 
190
+ return unquote(
191
+ text
192
+ ).strip()
193
 
194
  except Exception:
 
195
 
196
+ return ""
197
 
198
 
199
+ def safe_filename(
200
+ value,
201
+ default="image"
202
+ ):
203
 
204
+ if not value:
205
+ value = default
 
206
 
207
+ value = str(value)
208
 
209
+ value = re.sub(
210
+ r'[<>:"/\\|?*\x00-\x1f]',
211
+ "_",
212
+ value
213
+ )
214
 
215
+ value = value.strip(
216
+ " ."
217
+ )
218
 
219
+ if not value:
220
+ value = default
 
 
 
 
221
 
222
+ return value[:180]
 
223
 
 
224
 
225
+ def make_hash(value):
226
 
227
+ return hashlib.sha1(
228
+ value.encode(
229
+ "utf-8",
230
+ errors="ignore"
231
+ )
232
+ ).hexdigest()[:12]
233
 
 
 
 
 
 
 
 
 
 
 
 
234
 
235
+ def extension_from_content_type(
236
+ content_type
237
+ ):
238
 
239
+ if not content_type:
240
+ return ".jpg"
241
 
242
+ content_type = (
243
+ content_type
244
+ .lower()
245
+ )
246
 
247
+ mapping = {
248
 
249
+ "image/jpeg": ".jpg",
250
+ "image/jpg": ".jpg",
251
+ "image/png": ".png",
252
+ "image/webp": ".webp",
253
+ "image/gif": ".gif",
254
+ "image/bmp": ".bmp",
255
+ "image/tiff": ".tif",
256
+ "image/avif": ".avif",
257
 
258
+ }
 
 
259
 
260
+ return mapping.get(
261
+ content_type.split(";")[0],
262
+ ".jpg"
263
+ )
264
 
 
265
 
266
+ # ============================================================
267
+ # DATE HELPERS
268
+ # ============================================================
269
 
270
+ def parse_date(value):
 
271
 
272
+ if value is None:
273
+ return pd.NaT
 
 
274
 
275
+ if isinstance(
276
+ value,
277
+ pd.Timestamp
278
+ ):
279
+ return value
280
 
281
+ if isinstance(
282
+ value,
283
+ (int, float)
284
+ ):
285
 
286
+ if value > 100000000000:
 
 
 
287
 
288
+ return pd.to_datetime(
289
+ value,
290
+ unit="ms",
291
+ errors="coerce"
292
+ )
 
 
 
 
 
 
 
293
 
294
+ if value > 1000000000:
295
 
296
+ return pd.to_datetime(
297
+ value,
298
+ unit="s",
299
+ errors="coerce"
300
+ )
301
 
302
+ value = str(value).strip()
 
303
 
304
+ if not value:
305
+ return pd.NaT
306
 
307
+ # EXIF format:
308
+ # 2025:09:28 14:32:11
309
 
310
+ if re.match(
311
+ r"^\d{4}:\d{2}:\d{2}",
312
+ value
313
+ ):
 
314
 
315
+ try:
316
 
317
+ return pd.to_datetime(
318
+ value,
319
+ format="%Y:%m:%d %H:%M:%S",
320
+ errors="coerce"
321
+ )
322
 
323
+ except Exception:
324
+ pass
325
 
326
+ return pd.to_datetime(
327
+ value,
328
+ errors="coerce"
329
+ )
330
 
 
331
 
332
+ def iso_or_empty(value):
 
333
 
334
+ dt = parse_date(value)
335
 
336
+ if pd.isna(dt):
337
+ return ""
 
338
 
339
+ return dt.strftime(
340
+ "%Y-%m-%d %H:%M:%S"
341
+ )
342
 
 
343
 
344
+ # ============================================================
345
+ # GPS HELPERS
346
+ # ============================================================
347
 
348
+ def gps_to_decimal(value, ref=None):
 
 
349
 
350
+ try:
 
 
351
 
352
+ if value is None:
353
+ return None
 
354
 
355
+ if isinstance(
356
+ value,
357
+ (list, tuple)
358
+ ):
359
 
360
+ if len(value) != 3:
361
+ return None
 
362
 
363
+ def convert_component(x):
364
 
365
+ if hasattr(
366
+ x,
367
+ "numerator"
368
+ ):
369
 
370
+ return (
371
+ float(x.numerator)
372
+ / float(x.denominator)
373
+ )
374
 
375
+ if isinstance(
376
+ x,
377
+ tuple
378
+ ) and len(x) == 2:
379
 
380
+ return (
381
+ float(x[0])
382
+ / float(x[1])
383
+ )
384
 
385
+ return float(x)
 
386
 
387
+ degrees = convert_component(
388
+ value[0]
 
389
  )
390
 
391
+ minutes = convert_component(
392
+ value[1]
393
+ )
394
 
395
+ seconds = convert_component(
396
+ value[2]
397
+ )
398
 
399
+ decimal = (
400
+ degrees
401
+ + minutes / 60
402
+ + seconds / 3600
403
+ )
404
 
405
+ if ref in ("S", "W"):
406
+ decimal = -decimal
 
 
 
 
407
 
408
+ return decimal
 
 
409
 
410
+ return None
411
 
412
+ except Exception:
 
 
 
413
 
414
+ return None
 
 
415
 
 
416
 
417
+ # ============================================================
418
+ # EXIF EXTRACTION
419
+ # ============================================================
420
 
421
+ def extract_exif_metadata(
422
+ image
423
+ ):
 
424
 
425
+ metadata = {
426
+
427
+ "exif_make": "",
428
+ "exif_model": "",
429
+ "exif_software": "",
430
+ "exif_datetime": "",
431
+ "exif_datetime_original": "",
432
+ "exif_datetime_digitized": "",
433
+ "exif_orientation": "",
434
+ "exif_lens_model": "",
435
+ "exif_focal_length": "",
436
+ "exif_iso": "",
437
+ "exif_exposure_time": "",
438
+ "exif_f_number": "",
439
+ "exif_gps_latitude": "",
440
+ "exif_gps_longitude": "",
441
+ "exif_gps_altitude": "",
442
 
443
+ }
 
 
444
 
445
+ try:
446
 
447
+ exif = image.getexif()
 
 
 
 
448
 
449
+ if not exif:
450
+ return metadata
 
 
451
 
452
+ decoded = {}
 
 
453
 
454
+ for key, value in exif.items():
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
455
 
456
+ name = EXIF_TAGS.get(
457
+ key,
458
+ str(key)
459
+ )
460
 
461
+ decoded[name] = value
462
 
463
+ metadata[
464
+ "exif_make"
465
+ ] = str(
466
+ decoded.get(
467
+ "Make",
468
+ ""
469
+ )
470
+ )
471
 
472
+ metadata[
473
+ "exif_model"
474
+ ] = str(
475
+ decoded.get(
476
+ "Model",
477
+ ""
478
+ )
479
+ )
480
 
481
+ metadata[
482
+ "exif_software"
483
+ ] = str(
484
+ decoded.get(
485
+ "Software",
486
+ ""
487
+ )
488
+ )
489
 
490
+ metadata[
491
+ "exif_datetime"
492
+ ] = str(
493
+ decoded.get(
494
+ "DateTime",
495
+ ""
496
+ )
497
+ )
498
 
499
+ metadata[
500
+ "exif_datetime_original"
501
+ ] = str(
502
+ decoded.get(
503
+ "DateTimeOriginal",
504
+ ""
505
+ )
506
+ )
507
 
508
+ metadata[
509
+ "exif_datetime_digitized"
510
+ ] = str(
511
+ decoded.get(
512
+ "DateTimeDigitized",
513
+ ""
514
+ )
515
+ )
516
 
517
+ metadata[
518
+ "exif_orientation"
519
+ ] = str(
520
+ decoded.get(
521
+ "Orientation",
522
+ ""
523
+ )
524
+ )
525
 
526
+ metadata[
527
+ "exif_lens_model"
528
+ ] = str(
529
+ decoded.get(
530
+ "LensModel",
531
+ ""
532
+ )
533
  )
534
 
535
+ # Focal length
 
 
536
 
537
+ focal = decoded.get(
538
+ "FocalLength"
539
+ )
 
 
540
 
541
+ if focal is not None:
 
 
 
 
542
 
543
+ try:
 
 
 
 
544
 
545
+ if hasattr(
546
+ focal,
547
+ "numerator"
548
+ ):
 
549
 
550
+ metadata[
551
+ "exif_focal_length"
552
+ ] = round(
553
+ float(
554
+ focal.numerator
555
+ )
556
+ /
557
+ float(
558
+ focal.denominator
559
+ ),
560
+ 2
561
+ )
562
 
563
+ else:
564
 
565
+ metadata[
566
+ "exif_focal_length"
567
+ ] = str(
568
+ focal
569
+ )
570
 
571
+ except Exception:
572
 
573
+ metadata[
574
+ "exif_focal_length"
575
+ ] = str(
576
+ focal
577
+ )
578
 
579
+ metadata[
580
+ "exif_iso"
581
+ ] = str(
582
+ decoded.get(
583
+ "ISOSpeedRatings",
584
+ decoded.get(
585
+ "PhotographicSensitivity",
586
+ ""
587
+ )
588
+ )
589
+ )
590
 
591
+ metadata[
592
+ "exif_exposure_time"
593
+ ] = str(
594
+ decoded.get(
595
+ "ExposureTime",
596
+ ""
597
+ )
598
+ )
599
 
600
+ metadata[
601
+ "exif_f_number"
602
+ ] = str(
603
+ decoded.get(
604
+ "FNumber",
605
+ ""
606
+ )
607
+ )
608
 
609
+ # ----------------------------------------------------
610
+ # GPS
611
+ # ----------------------------------------------------
612
 
613
+ gps_info = decoded.get(
614
+ "GPSInfo"
615
  )
616
 
617
+ if gps_info:
 
 
618
 
619
+ gps_decoded = {}
 
 
620
 
621
+ try:
622
 
623
+ for key, value in gps_info.items():
624
 
625
+ gps_name = GPS_TAGS.get(
626
+ key,
627
+ str(key)
628
+ )
629
 
630
+ gps_decoded[
631
+ gps_name
632
+ ] = value
633
 
634
+ except Exception:
635
+ gps_decoded = {}
 
 
 
 
636
 
637
+ lat = gps_decoded.get(
638
+ "GPSLatitude"
639
+ )
640
 
641
+ lat_ref = gps_decoded.get(
642
+ "GPSLatitudeRef"
643
+ )
644
 
645
+ lon = gps_decoded.get(
646
+ "GPSLongitude"
647
+ )
648
 
649
+ lon_ref = gps_decoded.get(
650
+ "GPSLongitudeRef"
651
+ )
652
 
653
+ lat_decimal = gps_to_decimal(
654
+ lat,
655
+ lat_ref
656
+ )
657
 
658
+ lon_decimal = gps_to_decimal(
659
+ lon,
660
+ lon_ref
661
+ )
662
 
663
+ if lat_decimal is not None:
664
 
665
+ metadata[
666
+ "exif_gps_latitude"
667
+ ] = round(
668
+ lat_decimal,
669
+ 7
670
+ )
671
 
672
+ if lon_decimal is not None:
673
 
674
+ metadata[
675
+ "exif_gps_longitude"
676
+ ] = round(
677
+ lon_decimal,
678
+ 7
679
  )
680
 
681
+ altitude = gps_decoded.get(
682
+ "GPSAltitude"
683
+ )
684
 
685
+ if altitude is not None:
 
 
686
 
687
+ try:
688
 
689
+ if hasattr(
690
+ altitude,
691
+ "numerator"
692
+ ):
693
+
694
+ metadata[
695
+ "exif_gps_altitude"
696
+ ] = round(
697
+ float(
698
+ altitude.numerator
699
+ )
700
+ /
701
+ float(
702
+ altitude.denominator
703
+ ),
704
+ 2
705
+ )
706
+
707
+ else:
708
+
709
+ metadata[
710
+ "exif_gps_altitude"
711
+ ] = str(
712
+ altitude
713
+ )
714
 
715
+ except Exception:
716
 
717
+ metadata[
718
+ "exif_gps_altitude"
719
+ ] = str(
720
+ altitude
721
+ )
722
 
723
+ except Exception:
 
 
724
 
725
+ pass
726
 
727
+ return metadata
 
 
 
 
 
728
 
 
 
 
 
 
729
 
730
+ # ============================================================
731
+ # FLICKR API
732
+ # ============================================================
733
 
734
+ def flickr_search(
735
+ search_text,
736
+ max_images=50,
737
+ min_width=0,
738
+ min_height=0,
739
+ ):
740
 
741
+ if not FLICKR_API_KEY:
742
 
743
+ raise RuntimeError(
744
+ "No FLICKR_API_KEY was found. "
745
+ "Add FLICKR_API_KEY under "
746
+ "Space Settings β†’ Secrets."
747
+ )
748
 
749
+ search_text = (
750
+ search_text
751
+ .strip()
752
+ )
753
 
754
+ if not search_text:
 
755
 
756
+ raise ValueError(
757
+ "No Flickr search term was found."
758
+ )
 
759
 
760
+ max_images = min(
761
+ int(max_images),
762
+ MAX_IMAGES_HARD_LIMIT
763
+ )
764
 
765
+ results = []
 
 
 
 
 
 
766
 
767
+ page = 1
 
 
768
 
769
+ per_page = min(
770
+ 500,
771
+ max(
772
+ 100,
773
+ max_images
774
+ )
775
+ )
776
 
777
+ while len(results) < max_images:
778
+
779
+ params = {
780
+
781
+ "method":
782
+ "flickr.photos.search",
783
+
784
+ "api_key":
785
+ FLICKR_API_KEY,
786
+
787
+ "text":
788
+ search_text,
789
+
790
+ "format":
791
+ "json",
792
+
793
+ "nojsoncallback":
794
+ "1",
795
+
796
+ "safe_search":
797
+ "1",
798
+
799
+ "extras": (
800
+ "description,"
801
+ "license,"
802
+ "date_upload,"
803
+ "date_taken,"
804
+ "owner_name,"
805
+ "tags,"
806
+ "views,"
807
+ "url_sq,"
808
+ "url_t,"
809
+ "url_s,"
810
+ "url_q,"
811
+ "url_m,"
812
+ "url_n,"
813
+ "url_z,"
814
+ "url_c,"
815
+ "url_l,"
816
+ "url_o,"
817
+ "o_dims,"
818
+ "original_format"
819
+ ),
820
+
821
+ "per_page":
822
+ per_page,
823
+
824
+ "page":
825
+ page,
826
+ }
827
 
828
+ response = session.get(
829
+ FLICKR_API_URL,
830
+ params=params,
831
+ timeout=REQUEST_TIMEOUT,
832
+ )
833
 
834
+ response.raise_for_status()
835
 
836
+ data = response.json()
837
 
838
+ if data.get("stat") != "ok":
 
 
839
 
840
+ message = data.get(
841
+ "message",
842
+ "Unknown Flickr API error."
843
+ )
 
 
 
 
 
844
 
845
+ raise RuntimeError(
846
+ f"Flickr API error: {message}"
847
+ )
 
848
 
849
+ photos = data.get(
850
+ "photos",
851
+ {}
852
+ )
853
 
854
+ photo_list = photos.get(
855
+ "photo",
856
+ []
857
+ )
858
 
859
+ if not photo_list:
860
+ break
861
 
862
+ for photo in photo_list:
 
863
 
864
+ candidate_fields = [
 
 
 
865
 
866
+ "url_o",
867
+ "url_l",
868
+ "url_c",
869
+ "url_z",
870
+ "url_n",
871
+ "url_m",
872
+ "url_s",
873
+ "url_q",
874
+ "url_t",
875
 
876
+ ]
 
 
 
877
 
878
+ image_url = None
 
 
 
 
 
 
 
 
 
 
 
 
879
 
880
+ for field in candidate_fields:
 
 
 
881
 
882
+ if photo.get(field):
 
 
 
883
 
884
+ image_url = (
885
+ photo[field]
886
+ )
887
 
888
+ break
889
 
890
+ if not image_url:
891
+ continue
892
 
893
+ width = (
894
+ photo.get("width_o")
895
+ or photo.get("width_l")
896
+ or photo.get("width_c")
897
+ or photo.get("width_z")
898
+ or 0
899
  )
900
 
901
+ height = (
902
+ photo.get("height_o")
903
+ or photo.get("height_l")
904
+ or photo.get("height_c")
905
+ or photo.get("height_z")
906
+ or 0
907
+ )
908
 
909
+ try:
910
+ width = int(width or 0)
911
+ except Exception:
912
+ width = 0
913
 
914
+ try:
915
+ height = int(height or 0)
916
+ except Exception:
917
+ height = 0
918
 
919
+ if (
920
+ min_width
921
+ and width
922
+ and width < min_width
923
+ ):
924
+ continue
 
925
 
926
+ if (
927
+ min_height
928
+ and height
929
+ and height < min_height
930
+ ):
931
+ continue
932
 
933
+ photo_id = str(
934
+ photo.get(
935
+ "id",
936
+ ""
937
+ )
938
+ )
939
 
940
+ page_url = (
941
+ "https://www.flickr.com/photos/"
942
+ f"{photo.get('owner', '')}/"
943
+ f"{photo_id}/"
944
+ )
945
 
946
+ results.append(
947
+ {
948
+
949
+ "image_url":
950
+ image_url,
951
+
952
+ "page_url":
953
+ page_url,
954
+
955
+ "title":
956
+ photo.get(
957
+ "title",
958
+ ""
959
+ ),
960
+
961
+ "owner":
962
+ photo.get(
963
+ "ownername",
964
+ ""
965
+ ),
966
+
967
+ "photo_id":
968
+ photo_id,
969
+
970
+ "width":
971
+ width,
972
+
973
+ "height":
974
+ height,
975
+
976
+ "source":
977
+ "Flickr API",
978
+
979
+ "license":
980
+ photo.get(
981
+ "license",
982
+ ""
983
+ ),
984
+
985
+ "date_taken":
986
+ photo.get(
987
+ "datetaken",
988
+ ""
989
+ ),
990
+
991
+ "date_uploaded":
992
+ photo.get(
993
+ "dateupload",
994
+ ""
995
+ ),
996
+
997
+ "views":
998
+ photo.get(
999
+ "views",
1000
+ ""
1001
+ ),
1002
+
1003
+ "tags":
1004
+ photo.get(
1005
+ "tags",
1006
+ ""
1007
+ ),
1008
+
1009
+ "description":
1010
+ photo.get(
1011
+ "description",
1012
+ ""
1013
+ ),
1014
+
1015
+ "alt":
1016
+ "",
1017
+
1018
+ }
1019
+ )
1020
 
1021
+ if len(results) >= max_images:
1022
+ break
1023
 
1024
+ total_pages = int(
1025
+ photos.get(
1026
+ "pages",
1027
+ page
1028
+ )
1029
+ )
1030
 
1031
+ if page >= total_pages:
1032
+ break
1033
 
1034
+ page += 1
 
 
 
1035
 
1036
+ return results
1037
 
 
1038
 
1039
+ # ============================================================
1040
+ # GENERIC HTML EXTRACTION
1041
+ # ============================================================
 
 
 
1042
 
1043
+ def extract_image_candidates_from_html(
1044
+ page_url,
1045
+ html,
1046
+ ):
1047
 
1048
+ soup = BeautifulSoup(
1049
+ html,
1050
+ "html.parser"
1051
+ )
1052
 
1053
+ candidates = []
 
 
 
 
 
1054
 
1055
+ # --------------------------------------------------------
1056
+ # IMG
1057
+ # --------------------------------------------------------
1058
 
1059
+ for img in soup.find_all(
1060
+ "img"
1061
+ ):
1062
 
1063
+ attributes = [
 
 
 
 
 
1064
 
1065
+ "src",
1066
+ "data-src",
1067
+ "data-original",
1068
+ "data-lazy-src",
1069
+ "data-image",
1070
+ "data-url",
1071
+ "data-srcset",
1072
 
1073
+ ]
1074
 
1075
+ for attribute in attributes:
 
 
 
 
 
1076
 
1077
+ value = img.get(
1078
+ attribute
1079
+ )
 
 
 
1080
 
1081
+ if not value:
1082
+ continue
1083
 
1084
+ if attribute == "data-srcset":
 
 
1085
 
1086
+ parts = [
1087
+ p.strip()
1088
+ for p in value.split(",")
1089
+ ]
 
 
1090
 
1091
+ for part in parts:
1092
 
1093
+ pieces = (
1094
+ part.split()
1095
+ )
1096
 
1097
+ if pieces:
1098
 
1099
+ candidates.append(
1100
+ (
1101
+ pieces[0],
1102
+ img.get(
1103
+ "alt",
1104
+ ""
1105
+ )
1106
+ )
1107
+ )
1108
 
1109
+ else:
1110
 
1111
+ candidates.append(
1112
+ (
1113
+ value,
1114
+ img.get(
1115
+ "alt",
1116
+ ""
1117
+ )
1118
+ )
1119
+ )
1120
 
1121
+ # --------------------------------------------------------
1122
+ # SOURCE SRCSET
1123
+ # --------------------------------------------------------
1124
 
1125
+ for source in soup.find_all(
1126
+ "source"
1127
+ ):
1128
 
1129
+ srcset = source.get(
1130
+ "srcset"
1131
  )
1132
 
1133
+ if not srcset:
1134
+ continue
1135
 
1136
+ for part in srcset.split(","):
 
 
1137
 
1138
+ pieces = (
1139
+ part.strip()
1140
+ .split()
1141
+ )
1142
 
1143
+ if pieces:
 
 
 
1144
 
1145
+ candidates.append(
1146
+ (
1147
+ pieces[0],
1148
+ ""
1149
+ )
1150
+ )
1151
 
1152
+ # --------------------------------------------------------
1153
+ # OG IMAGE
1154
+ # --------------------------------------------------------
1155
 
1156
+ for meta in soup.find_all(
1157
+ "meta",
1158
+ attrs={
1159
+ "property": "og:image"
1160
+ }
1161
+ ):
1162
 
1163
+ value = meta.get(
1164
+ "content"
1165
+ )
 
 
 
1166
 
1167
+ if value:
1168
 
1169
+ candidates.append(
1170
+ (
1171
+ value,
1172
+ "og:image"
1173
+ )
1174
+ )
1175
 
1176
+ # --------------------------------------------------------
1177
+ # Resolve URLs
1178
+ # --------------------------------------------------------
1179
 
1180
+ resolved = []
 
 
 
1181
 
1182
+ for value, alt in candidates:
1183
 
1184
+ if not value:
1185
+ continue
 
1186
 
1187
+ value = value.strip()
 
 
 
 
1188
 
1189
+ if value.startswith(
1190
+ "data:"
1191
+ ):
1192
+ continue
 
 
 
 
 
1193
 
1194
+ absolute = urljoin(
1195
+ page_url,
1196
+ value
1197
+ )
1198
 
1199
+ if not valid_http_url(
1200
+ absolute
1201
+ ):
1202
+ continue
1203
 
1204
+ resolved.append(
1205
+ {
1206
 
1207
+ "image_url":
1208
+ absolute,
1209
 
1210
+ "page_url":
1211
+ page_url,
 
1212
 
1213
+ "title":
1214
+ "",
1215
 
1216
+ "owner":
1217
+ "",
 
 
 
1218
 
1219
+ "photo_id":
1220
+ "",
 
1221
 
1222
+ "width":
1223
+ 0,
1224
 
1225
+ "height":
1226
+ 0,
 
1227
 
1228
+ "source":
1229
+ "HTML",
1230
 
1231
+ "license":
1232
+ "",
1233
 
1234
+ "date_taken":
1235
+ "",
1236
 
1237
+ "date_uploaded":
1238
+ "",
1239
 
1240
+ "views":
1241
+ "",
1242
 
1243
+ "tags":
1244
+ "",
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1245
 
1246
+ "description":
1247
+ "",
1248
 
1249
+ "alt":
1250
+ alt,
 
 
1251
 
1252
+ }
1253
+ )
1254
+
1255
+ return resolved
1256
+
1257
+
1258
+ def generic_page_images(
1259
+ page_url,
1260
+ max_images=50,
1261
+ ):
1262
+
1263
+ response = session.get(
1264
+ page_url,
1265
+ timeout=REQUEST_TIMEOUT,
1266
+ allow_redirects=True,
1267
+ )
1268
+
1269
+ response.raise_for_status()
1270
+
1271
+ content_type = (
1272
+ response.headers
1273
+ .get(
1274
+ "Content-Type",
1275
+ ""
1276
+ )
1277
+ .lower()
1278
+ )
1279
+
1280
+ if (
1281
+ "html" not in content_type
1282
+ and "xhtml" not in content_type
1283
+ ):
1284
+
1285
+ if content_type.startswith(
1286
+ "image/"
1287
+ ):
1288
+
1289
+ return [
1290
+
1291
+ {
1292
+
1293
+ "image_url":
1294
+ response.url,
1295
+
1296
+ "page_url":
1297
+ page_url,
1298
+
1299
+ "title":
1300
+ "",
1301
+
1302
+ "owner":
1303
+ "",
1304
+
1305
+ "photo_id":
1306
+ "",
1307
+
1308
+ "width":
1309
+ 0,
1310
+
1311
+ "height":
1312
+ 0,
1313
+
1314
+ "source":
1315
+ "Direct image",
1316
+
1317
+ "license":
1318
+ "",
1319
+
1320
+ "date_taken":
1321
+ "",
1322
+
1323
+ "date_uploaded":
1324
+ "",
1325
+
1326
+ "views":
1327
+ "",
1328
+
1329
+ "tags":
1330
+ "",
1331
+
1332
+ "description":
1333
+ "",
1334
+
1335
+ "alt":
1336
+ "",
1337
+
1338
+ }
1339
+
1340
+ ]
1341
+
1342
+ raise RuntimeError(
1343
+ "The page did not return HTML. "
1344
+ f"Content-Type: {content_type}"
1345
+ )
1346
+
1347
+ candidates = (
1348
+ extract_image_candidates_from_html(
1349
+ response.url,
1350
+ response.text,
1351
+ )
1352
+ )
1353
+
1354
+ seen = set()
1355
+
1356
+ unique = []
1357
+
1358
+ for item in candidates:
1359
+
1360
+ image_url = item[
1361
+ "image_url"
1362
+ ]
1363
+
1364
+ normalized = (
1365
+ image_url
1366
+ .split("#")[0]
1367
+ )
1368
+
1369
+ if normalized in seen:
1370
+ continue
1371
+
1372
+ seen.add(
1373
+ normalized
1374
+ )
1375
+
1376
+ unique.append(
1377
+ item
1378
+ )
1379
+
1380
+ if len(unique) >= max_images:
1381
+ break
1382
+
1383
+ return unique
1384
+
1385
+
1386
+ # ============================================================
1387
+ # DOWNLOAD + METADATA
1388
+ # ============================================================
1389
+
1390
+ def download_image(
1391
+ item,
1392
+ output_dir,
1393
+ index,
1394
+ min_width=0,
1395
+ min_height=0,
1396
+ ):
1397
+
1398
+ image_url = item[
1399
+ "image_url"
1400
+ ]
1401
+
1402
+ try:
1403
+
1404
+ response = session.get(
1405
+ image_url,
1406
+ timeout=IMAGE_TIMEOUT,
1407
+ stream=True,
1408
+ allow_redirects=True,
1409
+ )
1410
+
1411
+ response.raise_for_status()
1412
+
1413
+ content_type = (
1414
+ response.headers
1415
+ .get(
1416
+ "Content-Type",
1417
+ ""
1418
+ )
1419
+ .lower()
1420
+ )
1421
+
1422
+ if (
1423
+ content_type
1424
+ and not content_type.startswith(
1425
+ "image/"
1426
+ )
1427
+ ):
1428
 
1429
+ if (
1430
+ "octet-stream"
1431
+ not in content_type
1432
+ ):
1433
+
1434
+ return (
1435
+ None,
1436
+ f"Not an image: "
1437
+ f"{content_type}"
1438
  )
1439
 
1440
+ data = response.content
1441
+
1442
+ if not data:
1443
+
1444
+ return (
1445
+ None,
1446
+ "Empty response"
1447
+ )
1448
+
1449
+ # ----------------------------------------------------
1450
+ # PIL
1451
+ # ----------------------------------------------------
1452
+
1453
+ try:
1454
+
1455
+ image = Image.open(
1456
+ io.BytesIO(data)
1457
+ )
1458
+
1459
+ width, height = (
1460
+ image.size
1461
+ )
1462
+
1463
+ image_format = (
1464
+ image.format
1465
+ or "JPEG"
1466
+ ).lower()
1467
+
1468
  except Exception as exc:
1469
 
1470
+ return (
1471
+ None,
1472
+ f"Invalid image: {exc}"
1473
  )
1474
 
1475
+ # ----------------------------------------------------
1476
+ # Dimension filtering
1477
+ # ----------------------------------------------------
1478
+
1479
+ if (
1480
+ min_width
1481
+ and width < min_width
1482
+ ):
1483
+
1484
+ return (
1485
+ None,
1486
+ f"Too narrow: "
1487
+ f"{width}px"
1488
+ )
1489
+
1490
+ if (
1491
+ min_height
1492
+ and height < min_height
1493
+ ):
1494
+
1495
+ return (
1496
+ None,
1497
+ f"Too short: "
1498
+ f"{height}px"
1499
+ )
1500
+
1501
+ # ----------------------------------------------------
1502
+ # EXIF
1503
+ # ----------------------------------------------------
1504
+
1505
+ exif_metadata = (
1506
+ extract_exif_metadata(
1507
+ image
1508
+ )
1509
+ )
1510
+
1511
+ # ----------------------------------------------------
1512
+ # Extension
1513
+ # ----------------------------------------------------
1514
+
1515
+ extension_map = {
1516
+
1517
+ "jpeg": ".jpg",
1518
+ "jpg": ".jpg",
1519
+ "png": ".png",
1520
+ "webp": ".webp",
1521
+ "gif": ".gif",
1522
+ "bmp": ".bmp",
1523
+ "tiff": ".tif",
1524
+ "avif": ".avif",
1525
+
1526
+ }
1527
+
1528
+ extension = (
1529
+ extension_map.get(
1530
+ image_format,
1531
+ extension_from_content_type(
1532
+ content_type
1533
+ )
1534
+ )
1535
+ )
1536
+
1537
+ # ----------------------------------------------------
1538
+ # Filename
1539
+ # ----------------------------------------------------
1540
+
1541
+ original_name = ""
1542
+
1543
+ try:
1544
+
1545
+ original_name = Path(
1546
+ urlparse(
1547
+ image_url
1548
+ ).path
1549
+ ).name
1550
+
1551
+ except Exception:
1552
+ pass
1553
+
1554
+ if original_name:
1555
+
1556
+ original_stem = Path(
1557
+ original_name
1558
+ ).stem
1559
+
1560
+ original_stem = (
1561
+ safe_filename(
1562
+ original_stem,
1563
+ f"image_{index:05d}"
1564
+ )
1565
+ )
1566
+
1567
+ else:
1568
+
1569
+ original_stem = (
1570
+ f"image_{index:05d}"
1571
+ )
1572
+
1573
+ filename = (
1574
+ f"{index:05d}_"
1575
+ f"{original_stem}"
1576
+ f"{extension}"
1577
+ )
1578
+
1579
+ filename = safe_filename(
1580
+ filename,
1581
+ f"image_{index:05d}"
1582
+ f"{extension}"
1583
+ )
1584
+
1585
+ output_path = (
1586
+ Path(output_dir)
1587
+ / filename
1588
+ )
1589
+
1590
+ with open(
1591
+ output_path,
1592
+ "wb"
1593
+ ) as f:
1594
+
1595
+ f.write(data)
1596
+
1597
+ # ----------------------------------------------------
1598
+ # Dates
1599
+ # ----------------------------------------------------
1600
+
1601
+ flickr_date_taken = (
1602
+ item.get(
1603
+ "date_taken",
1604
+ ""
1605
+ )
1606
+ )
1607
+
1608
+ flickr_date_uploaded = (
1609
+ item.get(
1610
+ "date_uploaded",
1611
+ ""
1612
+ )
1613
+ )
1614
+
1615
+ exif_original = (
1616
+ exif_metadata.get(
1617
+ "exif_datetime_original",
1618
+ ""
1619
+ )
1620
+ )
1621
+
1622
+ exif_datetime = (
1623
+ exif_metadata.get(
1624
+ "exif_datetime",
1625
+ ""
1626
+ )
1627
+ )
1628
+
1629
+ # Best available date
1630
+
1631
+ if flickr_date_taken:
1632
+
1633
+ date_used = (
1634
+ flickr_date_taken
1635
+ )
1636
+
1637
+ date_source = (
1638
+ "Flickr date taken"
1639
+ )
1640
+
1641
+ elif exif_original:
1642
+
1643
+ date_used = (
1644
+ exif_original
1645
+ )
1646
+
1647
+ date_source = (
1648
+ "EXIF DateTimeOriginal"
1649
+ )
1650
+
1651
+ elif exif_datetime:
1652
+
1653
+ date_used = (
1654
+ exif_datetime
1655
+ )
1656
+
1657
+ date_source = (
1658
+ "EXIF DateTime"
1659
+ )
1660
+
1661
+ elif flickr_date_uploaded:
1662
+
1663
+ date_used = (
1664
+ flickr_date_uploaded
1665
+ )
1666
+
1667
+ date_source = (
1668
+ "Flickr upload date"
1669
+ )
1670
+
1671
+ else:
1672
+
1673
+ date_used = ""
1674
+ date_source = ""
1675
+
1676
+ parsed_date = parse_date(
1677
+ date_used
1678
+ )
1679
+
1680
+ # ----------------------------------------------------
1681
+ # Aspect ratio
1682
+ # ----------------------------------------------------
1683
+
1684
+ if height:
1685
+
1686
+ aspect_ratio = round(
1687
+ width / height,
1688
+ 3
1689
+ )
1690
+
1691
+ else:
1692
+
1693
+ aspect_ratio = ""
1694
+
1695
+ # ----------------------------------------------------
1696
+ # File hash
1697
+ # ----------------------------------------------------
1698
+
1699
+ file_sha256 = hashlib.sha256(
1700
+ data
1701
+ ).hexdigest()
1702
+
1703
+ # ----------------------------------------------------
1704
+ # Build record
1705
+ # ----------------------------------------------------
1706
+
1707
+ record = dict(
1708
+ item
1709
+ )
1710
+
1711
+ record.update(
1712
+ {
1713
+
1714
+ "filename":
1715
+ filename,
1716
+
1717
+ "local_file":
1718
+ str(output_path),
1719
+
1720
+ "final_url":
1721
+ response.url,
1722
+
1723
+ "width":
1724
+ width,
1725
+
1726
+ "height":
1727
+ height,
1728
+
1729
+ "aspect_ratio":
1730
+ aspect_ratio,
1731
+
1732
+ "format":
1733
+ image_format,
1734
+
1735
+ "file_size_bytes":
1736
+ len(data),
1737
+
1738
+ "file_size_mb":
1739
+ round(
1740
+ len(data)
1741
+ / 1024
1742
+ / 1024,
1743
+ 3
1744
+ ),
1745
+
1746
+ "sha256":
1747
+ file_sha256,
1748
+
1749
+ "date_used":
1750
+ date_used,
1751
+
1752
+ "date_source":
1753
+ date_source,
1754
+
1755
+ "date_parsed":
1756
+ (
1757
+ parsed_date.strftime(
1758
+ "%Y-%m-%d"
1759
+ )
1760
+ if not pd.isna(
1761
+ parsed_date
1762
+ )
1763
+ else ""
1764
+ ),
1765
+
1766
+ "week":
1767
+ (
1768
+ parsed_date.strftime(
1769
+ "%Y-W%V"
1770
+ )
1771
+ if not pd.isna(
1772
+ parsed_date
1773
+ )
1774
+ else ""
1775
+ ),
1776
+
1777
+ "download_status":
1778
+ "OK",
1779
+
1780
+ }
1781
+ )
1782
+
1783
+ record.update(
1784
+ exif_metadata
1785
+ )
1786
+
1787
+ return (
1788
+ record,
1789
+ None
1790
+ )
1791
+
1792
+ except Exception as exc:
1793
+
1794
+ return (
1795
+ None,
1796
+ str(exc)
1797
+ )
1798
+
1799
+
1800
+ # ============================================================
1801
+ # DATAFRAME CREATION
1802
+ # ============================================================
1803
+
1804
+ def records_to_dataframe(
1805
+ records
1806
+ ):
1807
+
1808
+ if not records:
1809
+
1810
+ return pd.DataFrame()
1811
+
1812
+ df = pd.DataFrame(
1813
+ records
1814
+ )
1815
+
1816
+ preferred_columns = [
1817
+
1818
+ "filename",
1819
+
1820
+ "title",
1821
+ "owner",
1822
+ "photo_id",
1823
+
1824
+ "source",
1825
+
1826
+ "page_url",
1827
+ "image_url",
1828
+ "final_url",
1829
+
1830
+ "date_used",
1831
+ "date_source",
1832
+ "date_parsed",
1833
+ "week",
1834
+
1835
+ "date_taken",
1836
+ "date_uploaded",
1837
+
1838
+ "width",
1839
+ "height",
1840
+ "aspect_ratio",
1841
+
1842
+ "format",
1843
+
1844
+ "file_size_bytes",
1845
+ "file_size_mb",
1846
+
1847
+ "license",
1848
+ "tags",
1849
+ "description",
1850
+ "views",
1851
+ "alt",
1852
+
1853
+ "exif_make",
1854
+ "exif_model",
1855
+ "exif_software",
1856
+
1857
+ "exif_datetime",
1858
+ "exif_datetime_original",
1859
+ "exif_datetime_digitized",
1860
+
1861
+ "exif_orientation",
1862
+ "exif_lens_model",
1863
+ "exif_focal_length",
1864
+ "exif_iso",
1865
+ "exif_exposure_time",
1866
+ "exif_f_number",
1867
+
1868
+ "exif_gps_latitude",
1869
+ "exif_gps_longitude",
1870
+ "exif_gps_altitude",
1871
+
1872
+ "sha256",
1873
+
1874
+ "download_status",
1875
+
1876
+ ]
1877
+
1878
+ existing = [
1879
+ col
1880
+ for col in preferred_columns
1881
+ if col in df.columns
1882
+ ]
1883
+
1884
+ remaining = [
1885
+ col
1886
+ for col in df.columns
1887
+ if col not in existing
1888
+ and col != "local_file"
1889
+ ]
1890
+
1891
+ df = df[
1892
+ existing + remaining
1893
+ ]
1894
+
1895
+ return df
1896
+
1897
+
1898
+ # ============================================================
1899
+ # WEEKLY DATA
1900
+ # ============================================================
1901
+
1902
+ def create_weekly_dataframe(
1903
+ df
1904
+ ):
1905
+
1906
+ if df.empty:
1907
+
1908
+ return pd.DataFrame(
1909
+ columns=[
1910
+ "week",
1911
+ "week_start",
1912
+ "number_of_pictures",
1913
+ ]
1914
+ )
1915
+
1916
+ temp = df.copy()
1917
+
1918
+ temp[
1919
+ "_date"
1920
+ ] = pd.to_datetime(
1921
+ temp["date_parsed"],
1922
+ errors="coerce"
1923
+ )
1924
+
1925
+ temp = temp[
1926
+ temp["_date"].notna()
1927
+ ].copy()
1928
+
1929
+ if temp.empty:
1930
+
1931
+ return pd.DataFrame(
1932
+ columns=[
1933
+ "week",
1934
+ "week_start",
1935
+ "number_of_pictures",
1936
+ ]
1937
+ )
1938
+
1939
+ # Monday-based week
1940
+
1941
+ temp[
1942
+ "week_start"
1943
+ ] = (
1944
+ temp["_date"]
1945
+ - pd.to_timedelta(
1946
+ temp["_date"].dt.weekday,
1947
+ unit="D"
1948
+ )
1949
+ ).dt.normalize()
1950
+
1951
+ weekly = (
1952
+ temp
1953
+ .groupby(
1954
+ "week_start"
1955
+ )
1956
+ .size()
1957
+ .reset_index(
1958
+ name="number_of_pictures"
1959
+ )
1960
+ )
1961
+
1962
+ weekly[
1963
+ "week"
1964
+ ] = weekly[
1965
+ "week_start"
1966
+ ].dt.strftime(
1967
+ "%Y-W%V"
1968
+ )
1969
+
1970
+ weekly = weekly[
1971
+ [
1972
+ "week",
1973
+ "week_start",
1974
+ "number_of_pictures",
1975
+ ]
1976
+ ]
1977
+
1978
+ weekly = weekly.sort_values(
1979
+ "week_start"
1980
+ )
1981
+
1982
+ return weekly
1983
+
1984
+
1985
+ # ============================================================
1986
+ # WEEKLY PLOT
1987
+ # ============================================================
1988
+
1989
+ def create_weekly_plot(
1990
+ weekly_df,
1991
+ output_path
1992
+ ):
1993
+
1994
+ if weekly_df.empty:
1995
+
1996
+ return None
1997
+
1998
+ plt.figure(
1999
+ figsize=(12, 6)
2000
+ )
2001
+
2002
+ x = np.arange(
2003
+ len(weekly_df)
2004
+ )
2005
+
2006
+ y = (
2007
+ weekly_df[
2008
+ "number_of_pictures"
2009
+ ]
2010
+ .astype(int)
2011
+ .values
2012
+ )
2013
+
2014
+ plt.bar(
2015
+ x,
2016
+ y
2017
+ )
2018
+
2019
+ labels = (
2020
+ weekly_df[
2021
+ "week"
2022
+ ]
2023
+ .tolist()
2024
+ )
2025
+
2026
+ # Avoid unreadable labels
2027
+ # when there are many weeks.
2028
+
2029
+ if len(labels) > 30:
2030
+
2031
+ step = math.ceil(
2032
+ len(labels) / 30
2033
+ )
2034
+
2035
+ shown_labels = [
2036
+ label if i % step == 0
2037
+ else ""
2038
+ for i, label
2039
+ in enumerate(labels)
2040
+ ]
2041
+
2042
+ else:
2043
+
2044
+ shown_labels = labels
2045
+
2046
+ plt.xticks(
2047
+ x,
2048
+ shown_labels,
2049
+ rotation=45,
2050
+ ha="right"
2051
+ )
2052
+
2053
+ plt.xlabel(
2054
+ "Week"
2055
+ )
2056
+
2057
+ plt.ylabel(
2058
+ "Number of pictures"
2059
+ )
2060
+
2061
+ plt.title(
2062
+ "Number of pictures per week"
2063
+ )
2064
+
2065
+ plt.grid(
2066
+ axis="y",
2067
+ alpha=0.25
2068
+ )
2069
+
2070
+ plt.tight_layout()
2071
+
2072
+ plt.savefig(
2073
+ output_path,
2074
+ dpi=180,
2075
+ bbox_inches="tight"
2076
+ )
2077
+
2078
+ plt.close()
2079
+
2080
+ return str(
2081
+ output_path
2082
+ )
2083
+
2084
+
2085
+ # ============================================================
2086
+ # XLSX EXPORT
2087
+ # ============================================================
2088
+
2089
+ def create_xlsx(
2090
+ df,
2091
+ weekly_df,
2092
+ output_path
2093
+ ):
2094
+
2095
+ summary_rows = [
2096
+
2097
+ {
2098
+ "metric":
2099
+ "Number of pictures",
2100
+ "value":
2101
+ len(df),
2102
+ },
2103
+
2104
+ {
2105
+ "metric":
2106
+ "Pictures with date",
2107
+ "value":
2108
+ int(
2109
+ df[
2110
+ "date_parsed"
2111
+ ]
2112
+ .notna()
2113
+ .sum()
2114
+ )
2115
+ if (
2116
+ not df.empty
2117
+ and "date_parsed"
2118
+ in df.columns
2119
+ )
2120
+ else 0,
2121
+ },
2122
+
2123
+ {
2124
+ "metric":
2125
+ "Pictures with EXIF",
2126
+ "value":
2127
+ int(
2128
+ (
2129
+ df[
2130
+ "exif_model"
2131
+ ]
2132
+ .fillna("")
2133
+ .astype(str)
2134
+ .str.len()
2135
+ > 0
2136
+ ).sum()
2137
+ )
2138
+ if (
2139
+ not df.empty
2140
+ and "exif_model"
2141
+ in df.columns
2142
+ )
2143
+ else 0,
2144
+ },
2145
+
2146
+ {
2147
+ "metric":
2148
+ "Pictures with GPS",
2149
+ "value":
2150
+ int(
2151
+ (
2152
+ df[
2153
+ "exif_gps_latitude"
2154
+ ]
2155
+ .astype(str)
2156
+ .str.len()
2157
+ > 0
2158
+ ).sum()
2159
+ )
2160
+ if (
2161
+ not df.empty
2162
+ and "exif_gps_latitude"
2163
+ in df.columns
2164
+ )
2165
+ else 0,
2166
+ },
2167
+
2168
+ ]
2169
+
2170
+ summary_df = pd.DataFrame(
2171
+ summary_rows
2172
+ )
2173
+
2174
+ with pd.ExcelWriter(
2175
+ output_path,
2176
+ engine="openpyxl"
2177
+ ) as writer:
2178
+
2179
+ df.to_excel(
2180
+ writer,
2181
+ sheet_name="Metadata",
2182
+ index=False
2183
+ )
2184
+
2185
+ weekly_df.to_excel(
2186
+ writer,
2187
+ sheet_name="Weekly_counts",
2188
+ index=False
2189
+ )
2190
+
2191
+ summary_df.to_excel(
2192
+ writer,
2193
+ sheet_name="Summary",
2194
+ index=False
2195
+ )
2196
+
2197
+ # ----------------------------------------------------
2198
+ # Formatting
2199
+ # ----------------------------------------------------
2200
+
2201
+ workbook = writer.book
2202
+
2203
+ for sheet_name in [
2204
+ "Metadata",
2205
+ "Weekly_counts",
2206
+ "Summary",
2207
+ ]:
2208
+
2209
+ worksheet = workbook[
2210
+ sheet_name
2211
+ ]
2212
+
2213
+ worksheet.freeze_panes = "A2"
2214
+
2215
+ # Auto width, capped
2216
+ for column_cells in (
2217
+ worksheet.columns
2218
+ ):
2219
+
2220
+ max_length = 0
2221
+
2222
+ column_letter = (
2223
+ column_cells[0]
2224
+ .column_letter
2225
+ )
2226
+
2227
+ for cell in column_cells:
2228
+
2229
+ try:
2230
+
2231
+ value_length = len(
2232
+ str(
2233
+ cell.value
2234
+ )
2235
+ )
2236
+
2237
+ max_length = max(
2238
+ max_length,
2239
+ value_length
2240
+ )
2241
+
2242
+ except Exception:
2243
+ pass
2244
+
2245
+ worksheet.column_dimensions[
2246
+ column_letter
2247
+ ].width = min(
2248
+ max(
2249
+ max_length + 2,
2250
+ 10
2251
+ ),
2252
+ 45
2253
+ )
2254
+
2255
+
2256
+ # ============================================================
2257
+ # ZIP CREATION
2258
+ # ============================================================
2259
+
2260
+ def build_zip(
2261
+ records,
2262
+ zip_path,
2263
+ csv_path,
2264
+ xlsx_path,
2265
+ plot_path=None
2266
+ ):
2267
+
2268
+ with zipfile.ZipFile(
2269
+ zip_path,
2270
+ "w",
2271
+ compression=zipfile.ZIP_DEFLATED
2272
+ ) as z:
2273
+
2274
+ for record in records:
2275
+
2276
+ local_file = record.get(
2277
+ "local_file"
2278
+ )
2279
+
2280
+ if not local_file:
2281
+ continue
2282
+
2283
+ if not os.path.exists(
2284
+ local_file
2285
+ ):
2286
+ continue
2287
+
2288
+ filename = record.get(
2289
+ "filename",
2290
+ Path(
2291
+ local_file
2292
+ ).name
2293
+ )
2294
+
2295
+ z.write(
2296
+ local_file,
2297
+ arcname=(
2298
+ f"images/{filename}"
2299
+ )
2300
+ )
2301
+
2302
+ if os.path.exists(
2303
+ csv_path
2304
+ ):
2305
+
2306
+ z.write(
2307
+ csv_path,
2308
+ arcname="metadata.csv"
2309
+ )
2310
+
2311
+ if os.path.exists(
2312
+ xlsx_path
2313
+ ):
2314
+
2315
+ z.write(
2316
+ xlsx_path,
2317
+ arcname="metadata.xlsx"
2318
+ )
2319
+
2320
+ if (
2321
+ plot_path
2322
+ and os.path.exists(
2323
+ plot_path
2324
+ )
2325
+ ):
2326
+
2327
+ z.write(
2328
+ plot_path,
2329
+ arcname=(
2330
+ "weekly_picture_count.png"
2331
+ )
2332
+ )
2333
+
2334
+
2335
+ # ============================================================
2336
+ # MAIN FUNCTION
2337
+ # ============================================================
2338
+
2339
+ def find_images(
2340
+ url,
2341
+ max_images,
2342
+ min_width,
2343
+ min_height,
2344
+ remove_duplicates,
2345
+ ):
2346
+
2347
+ url = clean_url(
2348
+ url
2349
+ )
2350
+
2351
+ if not url:
2352
+
2353
+ return (
2354
+ [],
2355
+ "❌ Please enter a URL.",
2356
+ [],
2357
+ pd.DataFrame(),
2358
+ None,
2359
+ None,
2360
+ None,
2361
+ ""
2362
+ )
2363
+
2364
+ if not valid_http_url(
2365
+ url
2366
+ ):
2367
+
2368
+ return (
2369
+ [],
2370
+ "❌ Invalid HTTP/HTTPS URL.",
2371
+ [],
2372
+ pd.DataFrame(),
2373
+ None,
2374
+ None,
2375
+ None,
2376
+ ""
2377
+ )
2378
+
2379
+ max_images = min(
2380
+ int(max_images),
2381
+ MAX_IMAGES_HARD_LIMIT
2382
+ )
2383
+
2384
+ min_width = int(
2385
+ min_width or 0
2386
+ )
2387
+
2388
+ min_height = int(
2389
+ min_height or 0
2390
+ )
2391
+
2392
+ # --------------------------------------------------------
2393
+ # Job directory
2394
+ # --------------------------------------------------------
2395
+
2396
+ job_id = (
2397
+ f"{int(time.time())}_"
2398
+ f"{make_hash(url)}"
2399
+ )
2400
+
2401
+ job_dir = (
2402
+ BASE_DIR
2403
+ / job_id
2404
+ )
2405
+
2406
+ job_dir.mkdir(
2407
+ parents=True,
2408
+ exist_ok=True
2409
+ )
2410
+
2411
+ # --------------------------------------------------------
2412
+ # Find candidates
2413
+ # --------------------------------------------------------
2414
+
2415
+ try:
2416
+
2417
+ if is_flickr_url(
2418
+ url
2419
+ ):
2420
+
2421
+ search_text = (
2422
+ extract_flickr_search_text(
2423
+ url
2424
+ )
2425
+ )
2426
+
2427
+ if (
2428
+ FLICKR_API_KEY
2429
+ and search_text
2430
+ ):
2431
+
2432
+ mode_message = (
2433
+ "Flickr API mode"
2434
+ )
2435
+
2436
+ candidates = flickr_search(
2437
+ search_text,
2438
+ max_images=max_images,
2439
+ min_width=min_width,
2440
+ min_height=min_height,
2441
+ )
2442
+
2443
+ else:
2444
+
2445
+ mode_message = (
2446
+ "HTML webpage mode"
2447
+ )
2448
+
2449
+ candidates = (
2450
+ generic_page_images(
2451
+ url,
2452
+ max_images=max_images,
2453
+ )
2454
+ )
2455
+
2456
+ else:
2457
+
2458
+ mode_message = (
2459
+ "Generic webpage mode"
2460
+ )
2461
+
2462
+ candidates = (
2463
+ generic_page_images(
2464
+ url,
2465
+ max_images=max_images,
2466
+ )
2467
+ )
2468
+
2469
+ except Exception as exc:
2470
+
2471
+ return (
2472
+ [],
2473
+ (
2474
+ "❌ Could not retrieve the page.\n\n"
2475
+ f"{type(exc).__name__}: {exc}"
2476
+ ),
2477
+ [],
2478
+ pd.DataFrame(),
2479
+ None,
2480
+ None,
2481
+ None,
2482
+ ""
2483
+ )
2484
+
2485
+ if not candidates:
2486
+
2487
+ return (
2488
+ [],
2489
+ "⚠️ No image candidates were found.",
2490
+ [],
2491
+ pd.DataFrame(),
2492
+ None,
2493
+ None,
2494
+ None,
2495
+ ""
2496
+ )
2497
+
2498
+ # --------------------------------------------------------
2499
+ # URL deduplication
2500
+ # --------------------------------------------------------
2501
+
2502
+ if remove_duplicates:
2503
+
2504
+ seen = set()
2505
+
2506
+ unique = []
2507
+
2508
+ for item in candidates:
2509
+
2510
+ image_url = item.get(
2511
+ "image_url",
2512
+ ""
2513
+ )
2514
+
2515
+ normalized = (
2516
+ image_url
2517
+ .split("#")[0]
2518
+ )
2519
+
2520
+ if normalized in seen:
2521
+ continue
2522
+
2523
+ seen.add(
2524
+ normalized
2525
+ )
2526
+
2527
+ unique.append(
2528
+ item
2529
+ )
2530
+
2531
+ candidates = unique
2532
+
2533
+ # --------------------------------------------------------
2534
+ # Download
2535
+ # --------------------------------------------------------
2536
+
2537
+ downloaded = []
2538
+
2539
+ errors = []
2540
+
2541
+ for index, item in enumerate(
2542
+ candidates,
2543
+ start=1
2544
+ ):
2545
+
2546
+ result, error = (
2547
+ download_image(
2548
+ item,
2549
+ job_dir,
2550
+ index,
2551
+ min_width=min_width,
2552
+ min_height=min_height,
2553
+ )
2554
+ )
2555
+
2556
+ if result:
2557
+
2558
+ downloaded.append(
2559
+ result
2560
+ )
2561
+
2562
+ else:
2563
+
2564
+ errors.append(
2565
+ {
2566
+ "url":
2567
+ item.get(
2568
+ "image_url",
2569
+ ""
2570
+ ),
2571
+ "error":
2572
+ error,
2573
+ }
2574
+ )
2575
+
2576
+ if len(downloaded) >= max_images:
2577
+ break
2578
+
2579
+ # --------------------------------------------------------
2580
+ # Exact duplicate detection
2581
+ # --------------------------------------------------------
2582
+
2583
+ if remove_duplicates and downloaded:
2584
+
2585
+ seen_hashes = set()
2586
+
2587
+ unique_downloaded = []
2588
+
2589
+ duplicate_records = []
2590
+
2591
+ for record in downloaded:
2592
+
2593
+ file_hash = record.get(
2594
+ "sha256"
2595
+ )
2596
+
2597
+ if file_hash in seen_hashes:
2598
+
2599
+ duplicate_records.append(
2600
+ record
2601
+ )
2602
+
2603
+ try:
2604
+
2605
+ os.remove(
2606
+ record[
2607
+ "local_file"
2608
+ ]
2609
+ )
2610
+
2611
+ except Exception:
2612
+ pass
2613
+
2614
+ continue
2615
+
2616
+ seen_hashes.add(
2617
+ file_hash
2618
+ )
2619
+
2620
+ unique_downloaded.append(
2621
+ record
2622
+ )
2623
+
2624
+ downloaded = (
2625
+ unique_downloaded
2626
+ )
2627
+
2628
+ # --------------------------------------------------------
2629
+ # DataFrame
2630
+ # --------------------------------------------------------
2631
+
2632
+ df = records_to_dataframe(
2633
+ downloaded
2634
+ )
2635
+
2636
+ weekly_df = create_weekly_dataframe(
2637
+ df
2638
+ )
2639
+
2640
+ # --------------------------------------------------------
2641
+ # Files
2642
+ # --------------------------------------------------------
2643
+
2644
+ csv_path = (
2645
+ job_dir
2646
+ / "metadata.csv"
2647
+ )
2648
+
2649
+ xlsx_path = (
2650
+ job_dir
2651
+ / "metadata.xlsx"
2652
+ )
2653
+
2654
+ plot_path = (
2655
+ job_dir
2656
+ / "weekly_picture_count.png"
2657
+ )
2658
+
2659
+ df.to_csv(
2660
+ csv_path,
2661
+ index=False,
2662
+ encoding="utf-8-sig"
2663
+ )
2664
+
2665
+ create_xlsx(
2666
+ df,
2667
+ weekly_df,
2668
+ xlsx_path
2669
+ )
2670
+
2671
+ create_weekly_plot(
2672
+ weekly_df,
2673
+ plot_path
2674
+ )
2675
+
2676
+ # --------------------------------------------------------
2677
+ # Gallery
2678
+ # --------------------------------------------------------
2679
+
2680
+ gallery = []
2681
+
2682
+ for record in downloaded:
2683
+
2684
+ local_file = record.get(
2685
+ "local_file"
2686
+ )
2687
+
2688
+ if not local_file:
2689
+ continue
2690
+
2691
+ caption = (
2692
+ record.get(
2693
+ "title"
2694
+ )
2695
+ or record.get(
2696
+ "filename"
2697
+ )
2698
+ )
2699
+
2700
+ gallery.append(
2701
+ (
2702
+ local_file,
2703
+ caption
2704
+ )
2705
+ )
2706
+
2707
+ # --------------------------------------------------------
2708
+ # Status
2709
+ # --------------------------------------------------------
2710
+
2711
+ dated_count = 0
2712
+
2713
+ if not df.empty:
2714
+
2715
+ dated_count = int(
2716
+ df[
2717
+ "date_parsed"
2718
+ ]
2719
+ .fillna("")
2720
+ .astype(str)
2721
+ .str.len()
2722
+ .gt(0)
2723
+ .sum()
2724
+ )
2725
+
2726
+ exif_count = 0
2727
+
2728
+ if not df.empty:
2729
+
2730
+ exif_count = int(
2731
+ df[
2732
+ "exif_model"
2733
+ ]
2734
+ .fillna("")
2735
+ .astype(str)
2736
+ .str.len()
2737
+ .gt(0)
2738
+ .sum()
2739
+ )
2740
+
2741
+ gps_count = 0
2742
+
2743
+ if not df.empty:
2744
+
2745
+ gps_count = int(
2746
+ df[
2747
+ "exif_gps_latitude"
2748
+ ]
2749
+ .fillna("")
2750
+ .astype(str)
2751
+ .str.len()
2752
+ .gt(0)
2753
+ .sum()
2754
+ )
2755
+
2756
+ status_lines = [
2757
+
2758
+ f"### {mode_message}",
2759
+
2760
+ "",
2761
+
2762
+ f"**Candidates found:** "
2763
+ f"{len(candidates)}",
2764
+
2765
+ f"**Images downloaded:** "
2766
+ f"{len(downloaded)}",
2767
+
2768
+ f"**Failed downloads:** "
2769
+ f"{len(errors)}",
2770
+
2771
+ "",
2772
+
2773
+ f"**Images with usable date:** "
2774
+ f"{dated_count}",
2775
+
2776
+ f"**Images with camera EXIF:** "
2777
+ f"{exif_count}",
2778
+
2779
+ f"**Images with GPS EXIF:** "
2780
+ f"{gps_count}",
2781
+
2782
+ "",
2783
+
2784
+ f"**Weeks represented:** "
2785
+ f"{len(weekly_df)}",
2786
+
2787
+ "",
2788
+
2789
+ "Metadata XLSX and weekly plot "
2790
+ "are ready.",
2791
+
2792
+ ]
2793
+
2794
+ if errors:
2795
+
2796
+ status_lines.extend(
2797
+ [
2798
+ "",
2799
+ "### Download errors",
2800
+ ]
2801
+ )
2802
+
2803
+ for error in errors[:10]:
2804
+
2805
+ status_lines.append(
2806
+ f"- {error['error']}"
2807
+ )
2808
+
2809
+ status = "\n".join(
2810
+ status_lines
2811
+ )
2812
+
2813
+ return (
2814
+ gallery,
2815
+ status,
2816
+ downloaded,
2817
+ df,
2818
+ weekly_df,
2819
+ str(plot_path)
2820
+ if os.path.exists(
2821
+ plot_path
2822
+ )
2823
+ else None,
2824
+ str(xlsx_path),
2825
+ str(csv_path),
2826
+ )
2827
+
2828
+
2829
+ # ============================================================
2830
+ # ZIP CREATION
2831
+ # ============================================================
2832
+
2833
+ def create_zip(
2834
+ items,
2835
+ xlsx_path,
2836
+ csv_path,
2837
+ plot_path,
2838
+ ):
2839
+
2840
+ if not items:
2841
+
2842
+ return (
2843
+ None,
2844
+ "❌ There are no downloaded images."
2845
+ )
2846
+
2847
+ try:
2848
+
2849
+ first_file = Path(
2850
+ items[0]["local_file"]
2851
+ )
2852
+
2853
+ job_dir = (
2854
+ first_file.parent
2855
+ )
2856
+
2857
+ timestamp = time.strftime(
2858
+ "%Y%m%d_%H%M%S"
2859
+ )
2860
+
2861
+ zip_path = (
2862
+ job_dir.parent
2863
+ / f"images_{timestamp}.zip"
2864
+ )
2865
+
2866
+ build_zip(
2867
+ items,
2868
+ zip_path,
2869
+ csv_path,
2870
+ xlsx_path,
2871
+ plot_path,
2872
+ )
2873
+
2874
+ total_size = (
2875
+ zip_path.stat().st_size
2876
+ )
2877
+
2878
+ size_mb = (
2879
+ total_size
2880
+ / 1024
2881
+ / 1024
2882
+ )
2883
+
2884
+ return (
2885
+ str(zip_path),
2886
+ (
2887
+ "### βœ… ZIP created\n\n"
2888
+ f"Images: {len(items)}\n\n"
2889
+ f"ZIP size: {size_mb:.2f} MB\n\n"
2890
+ "Contents:\n"
2891
+ "- `images/`\n"
2892
+ "- `metadata.csv`\n"
2893
+ "- `metadata.xlsx`\n"
2894
+ "- `weekly_picture_count.png`"
2895
+ )
2896
+ )
2897
+
2898
+ except Exception as exc:
2899
+
2900
+ return (
2901
+ None,
2902
+ f"❌ ZIP creation failed: {exc}"
2903
+ )
2904
+
2905
+
2906
+ # ============================================================
2907
+ # CLEAR
2908
+ # ============================================================
2909
+
2910
+ def clear_all():
2911
+
2912
+ return (
2913
+ "",
2914
+ [],
2915
+ "Ready.",
2916
+ [],
2917
+ pd.DataFrame(),
2918
+ None,
2919
+ None,
2920
+ None,
2921
+ None,
2922
+ "",
2923
+ )
2924
+
2925
+
2926
+ # ============================================================
2927
+ # DESCRIPTION
2928
+ # ============================================================
2929
+
2930
+ DESCRIPTION = """
2931
+ ## πŸ–ΌοΈ Web & Flickr Image Downloader
2932
+ Download images from Flickr or ordinary webpages and automatically
2933
+ extract image metadata.
2934
+ ### Flickr
2935
+ Add a Hugging Face Space Secret:
2936
+ `FLICKR_API_KEY`
2937
+ Then use for example:
2938
+ `https://www.flickr.com/search/?text=ai`
2939
+ The application uses the Flickr API when the key is available.
2940
+ ### Metadata
2941
+ The application collects:
2942
+ **Flickr**
2943
+ - title
2944
+ - photographer
2945
+ - Flickr ID
2946
+ - license
2947
+ - tags
2948
+ - views
2949
+ - date taken
2950
+ - date uploaded
2951
+ - description
2952
+ **Image / EXIF**
2953
+ - width
2954
+ - height
2955
+ - aspect ratio
2956
+ - format
2957
+ - file size
2958
+ - SHA-256
2959
+ - camera manufacturer
2960
+ - camera model
2961
+ - software
2962
+ - original date/time
2963
+ - lens
2964
+ - focal length
2965
+ - ISO
2966
+ - exposure
2967
+ - aperture
2968
+ - GPS latitude/longitude/altitude
2969
+ ### Date used for the weekly analysis
2970
+ The application uses the first available date in this order:
2971
+ 1. Flickr date taken
2972
+ 2. EXIF DateTimeOriginal
2973
+ 3. EXIF DateTime
2974
+ 4. Flickr upload date
2975
+ The weekly chart uses Monday as the beginning of the week.
2976
+ ### Exports
2977
+ You receive:
2978
+ - metadata table
2979
+ - `metadata.csv`
2980
+ - `metadata.xlsx`
2981
+ - weekly bar chart
2982
+ - ZIP containing everything
2983
+ """
2984
+
2985
+
2986
+ # ============================================================
2987
+ # GRADIO UI
2988
+ # ============================================================
2989
+
2990
+ with gr.Blocks(
2991
+ title=APP_TITLE
2992
+ ) as demo:
2993
+
2994
+ gr.Markdown(
2995
+ "# πŸ–ΌοΈ Web & Flickr Image Downloader"
2996
+ )
2997
+
2998
+ gr.Markdown(
2999
+ DESCRIPTION
3000
+ )
3001
+
3002
+ # --------------------------------------------------------
3003
+ # INPUT
3004
+ # --------------------------------------------------------
3005
+
3006
+ with gr.Row():
3007
+
3008
+ with gr.Column(
3009
+ scale=4
3010
+ ):
3011
+
3012
+ url_input = gr.Textbox(
3013
+ label="Webpage or Flickr URL",
3014
+ placeholder=(
3015
+ "https://www.flickr.com/search/?text=ai"
3016
+ ),
3017
+ lines=2,
3018
+ )
3019
+
3020
+ with gr.Column(
3021
+ scale=1
3022
+ ):
3023
+
3024
+ max_images = gr.Slider(
3025
+ minimum=1,
3026
+ maximum=500,
3027
+ value=50,
3028
+ step=1,
3029
+ label="Maximum images",
3030
+ )
3031
+
3032
+ with gr.Row():
3033
+
3034
+ min_width = gr.Number(
3035
+ value=0,
3036
+ minimum=0,
3037
+ maximum=10000,
3038
+ step=100,
3039
+ label="Minimum width (px)",
3040
+ )
3041
+
3042
+ min_height = gr.Number(
3043
+ value=0,
3044
+ minimum=0,
3045
+ maximum=10000,
3046
+ step=100,
3047
+ label="Minimum height (px)",
3048
+ )
3049
+
3050
+ remove_duplicates = gr.Checkbox(
3051
+ value=True,
3052
+ label="Remove duplicates",
3053
+ )
3054
+
3055
+ with gr.Row():
3056
+
3057
+ find_button = gr.Button(
3058
+ "πŸ”Ž Find & Analyze Images",
3059
+ variant="primary",
3060
+ )
3061
+
3062
+ clear_button = gr.Button(
3063
+ "🧹 Clear",
3064
+ )
3065
+
3066
+ # --------------------------------------------------------
3067
+ # STATUS
3068
+ # --------------------------------------------------------
3069
+
3070
+ status = gr.Markdown(
3071
+ "Ready."
3072
+ )
3073
+
3074
+ # --------------------------------------------------------
3075
+ # GALLERY
3076
+ # --------------------------------------------------------
3077
+
3078
+ gallery = gr.Gallery(
3079
+ label="Downloaded images",
3080
+ show_label=True,
3081
+ columns=5,
3082
+ rows=3,
3083
+ height="auto",
3084
+ object_fit="contain",
3085
+ allow_preview=True,
3086
+ )
3087
+
3088
+ # --------------------------------------------------------
3089
+ # STATE
3090
+ # --------------------------------------------------------
3091
+
3092
+ image_state = gr.State(
3093
+ []
3094
+ )
3095
+
3096
+ xlsx_state = gr.State(
3097
+ None
3098
+ )
3099
+
3100
+ csv_state = gr.State(
3101
+ None
3102
+ )
3103
+
3104
+ plot_state = gr.State(
3105
+ None
3106
+ )
3107
+
3108
+ # --------------------------------------------------------
3109
+ # METADATA
3110
+ # --------------------------------------------------------
3111
+
3112
+ gr.Markdown(
3113
+ "## πŸ“‹ Image metadata"
3114
+ )
3115
+
3116
+ metadata_table = gr.Dataframe(
3117
+ headers=None,
3118
+ value=pd.DataFrame(),
3119
+ datatype=None,
3120
+ interactive=False,
3121
+ wrap=True,
3122
+ label="Metadata",
3123
+ )
3124
+
3125
+ # --------------------------------------------------------
3126
+ # WEEKLY
3127
+ # --------------------------------------------------------
3128
+
3129
+ gr.Markdown(
3130
+ "## πŸ“… Pictures per week"
3131
+ )
3132
+
3133
+ weekly_table = gr.Dataframe(
3134
+ headers=None,
3135
+ value=pd.DataFrame(),
3136
+ datatype=None,
3137
+ interactive=False,
3138
+ label="Weekly counts",
3139
+ )
3140
+
3141
+ weekly_plot = gr.Image(
3142
+ label="Number of pictures per week",
3143
+ type="filepath",
3144
+ )
3145
+
3146
+ # --------------------------------------------------------
3147
+ # DOWNLOADS
3148
+ # --------------------------------------------------------
3149
+
3150
+ gr.Markdown(
3151
+ "## πŸ“₯ Downloads"
3152
+ )
3153
+
3154
+ with gr.Row():
3155
+
3156
+ xlsx_file = gr.File(
3157
+ label="Excel metadata",
3158
+ )
3159
+
3160
+ csv_file = gr.File(
3161
+ label="CSV metadata",
3162
+ )
3163
+
3164
+ plot_file = gr.File(
3165
+ label="Weekly plot PNG",
3166
+ )
3167
+
3168
+ # --------------------------------------------------------
3169
+ # ZIP
3170
+ # --------------------------------------------------------
3171
+
3172
+ gr.Markdown(
3173
+ "## πŸ“¦ Complete ZIP"
3174
+ )
3175
+
3176
+ with gr.Row():
3177
+
3178
+ zip_button = gr.Button(
3179
+ "πŸ“¦ Create Complete ZIP",
3180
+ variant="primary",
3181
+ )
3182
+
3183
+ zip_file = gr.File(
3184
+ label="Complete ZIP",
3185
+ )
3186
+
3187
+ zip_status = gr.Markdown(
3188
+ ""
3189
+ )
3190
+
3191
+ # --------------------------------------------------------
3192
+ # EVENTS
3193
+ # --------------------------------------------------------
3194
+
3195
+ find_button.click(
3196
+ fn=find_images,
3197
+ inputs=[
3198
+ url_input,
3199
+ max_images,
3200
+ min_width,
3201
+ min_height,
3202
+ remove_duplicates,
3203
+ ],
3204
+ outputs=[
3205
+ gallery,
3206
+ status,
3207
+ image_state,
3208
+ metadata_table,
3209
+ weekly_table,
3210
+ weekly_plot,
3211
+ xlsx_file,
3212
+ csv_file,
3213
+ ],
3214
+ )
3215
+
3216
+ zip_button.click(
3217
+ fn=create_zip,
3218
+ inputs=[
3219
+ image_state,
3220
+ xlsx_state,
3221
+ csv_state,
3222
+ plot_state,
3223
+ ],
3224
+ outputs=[
3225
+ zip_file,
3226
+ zip_status,
3227
+ ],
3228
+ )
3229
+
3230
+ # --------------------------------------------------------
3231
+ # IMPORTANT:
3232
+ # State values need to be populated independently.
3233
+ # --------------------------------------------------------
3234
+
3235
+ def store_export_paths(
3236
+ xlsx_path,
3237
+ csv_path,
3238
+ plot_path,
3239
+ ):
3240
+
3241
+ return (
3242
+ xlsx_path,
3243
+ csv_path,
3244
+ plot_path,
3245
+ )
3246
+
3247
+ # The output files from find_images are also stored
3248
+ # through these hidden State components.
3249
+
3250
+ find_button.click(
3251
+ fn=store_export_paths,
3252
+ inputs=[
3253
+ xlsx_file,
3254
+ csv_file,
3255
+ plot_file,
3256
+ ],
3257
+ outputs=[
3258
+ xlsx_state,
3259
+ csv_state,
3260
+ plot_state,
3261
+ ],
3262
+ )
3263
+
3264
+ # --------------------------------------------------------
3265
+ # CLEAR
3266
+ # --------------------------------------------------------
3267
+
3268
+ clear_button.click(
3269
+ fn=clear_all,
3270
+ inputs=[],
3271
+ outputs=[
3272
+ url_input,
3273
+ gallery,
3274
+ status,
3275
+ image_state,
3276
+ metadata_table,
3277
+ weekly_plot,
3278
+ xlsx_file,
3279
+ csv_file,
3280
+ plot_file,
3281
+ zip_file,
3282
+ ],
3283
+ )
3284
+
3285
+
3286
+ # ============================================================
3287
+ # LAUNCH
3288
+ # ============================================================
3289
+
3290
+ if __name__ == "__main__":
3291
 
3292
+ demo.launch()