Spaces:
Sleeping
Sleeping
Download apps/api/src/lib/extract/helpers/deduplicate-objs-array.ts from s-eto-hub/firecrawl-api: direct link, hf CLI and curl.
- Browser
- Download file 908 Bytes
-
https://huggingface.co/spaces/s-eto-hub/firecrawl-api/resolve/main/apps/api/src/lib/extract/helpers/deduplicate-objs-array.ts
- Command line
-
hf download hf://spaces/s-eto-hub/firecrawl-api/apps/api/src/lib/extract/helpers/deduplicate-objs-array.ts
-
curl -L -o deduplicate-objs-array.ts https://huggingface.co/spaces/s-eto-hub/firecrawl-api/resolve/main/apps/api/src/lib/extract/helpers/deduplicate-objs-array.ts
908 Bytes
| export function deduplicateObjectsArray(objArray: { [key: string]: any[] }): { | |
| [key: string]: any[]; | |
| } { | |
| const deduplicatedObjArray: { [key: string]: any[] } = {}; | |
| for (const key in objArray) { | |
| if (Array.isArray(objArray[key])) { | |
| const seen = new Set(); | |
| deduplicatedObjArray[key] = objArray[key].filter(item => { | |
| // Create a unique identifier for each item based on its properties | |
| const identifier = JSON.stringify(item); | |
| // Check if this identifier has been seen before | |
| if (seen.has(identifier)) { | |
| return false; // Duplicate found, filter it out | |
| } | |
| // Add the identifier to the set and keep the item | |
| seen.add(identifier); | |
| return true; | |
| }); | |
| } else { | |
| // If the value is not an array, just copy it as is | |
| deduplicatedObjArray[key] = objArray[key]; | |
| } | |
| } | |
| return deduplicatedObjArray; | |
| } | |