Spaces:
Sleeping
Sleeping
Download apps/api/src/lib/permu-refactor.test.ts from s-eto-hub/firecrawl-api: direct link, hf CLI and curl.
- Browser
- Download file 15.3 kB
-
https://huggingface.co/spaces/s-eto-hub/firecrawl-api/resolve/main/apps/api/src/lib/permu-refactor.test.ts
- Command line
-
hf download hf://spaces/s-eto-hub/firecrawl-api/apps/api/src/lib/permu-refactor.test.ts
-
curl -L -o permu-refactor.test.ts https://huggingface.co/spaces/s-eto-hub/firecrawl-api/resolve/main/apps/api/src/lib/permu-refactor.test.ts
15.3 kB
| function generateURLPermutations(url: string | URL): URL[] { | |
| const urlO = new URL(url); | |
| // Construct two versions, one with www., one without | |
| const urlWithWWW = new URL(urlO); | |
| const urlWithoutWWW = new URL(urlO); | |
| if (urlO.hostname.startsWith("www.")) { | |
| urlWithoutWWW.hostname = urlWithWWW.hostname.slice(4); | |
| } else { | |
| urlWithWWW.hostname = "www." + urlWithoutWWW.hostname; | |
| } | |
| let permutations = [urlWithWWW, urlWithoutWWW]; | |
| // Construct more versions for http/https | |
| permutations = permutations.flatMap(urlO => { | |
| if (!["http:", "https:"].includes(urlO.protocol)) { | |
| return [urlO]; | |
| } | |
| const urlWithHTTP = new URL(urlO); | |
| const urlWithHTTPS = new URL(urlO); | |
| urlWithHTTP.protocol = "http:"; | |
| urlWithHTTPS.protocol = "https:"; | |
| return [urlWithHTTP, urlWithHTTPS]; | |
| }); | |
| // Construct more versions for index.html/index.php | |
| permutations = permutations.flatMap(urlO => { | |
| const urlWithHTML = new URL(urlO); | |
| const urlWithPHP = new URL(urlO); | |
| const urlWithBare = new URL(urlO); | |
| const urlWithSlash = new URL(urlO); | |
| if (urlO.pathname.endsWith("/")) { | |
| urlWithBare.pathname = | |
| urlWithBare.pathname.length === 1 | |
| ? urlWithBare.pathname | |
| : urlWithBare.pathname.slice(0, -1); | |
| urlWithHTML.pathname += "index.html"; | |
| urlWithPHP.pathname += "index.php"; | |
| } else if (urlO.pathname.endsWith("/index.html")) { | |
| urlWithPHP.pathname = | |
| urlWithPHP.pathname.slice(0, -"index.html".length) + "index.php"; | |
| urlWithSlash.pathname = urlWithSlash.pathname.slice( | |
| 0, | |
| -"index.html".length, | |
| ); | |
| urlWithBare.pathname = urlWithBare.pathname.slice( | |
| 0, | |
| -"/index.html".length, | |
| ); | |
| } else if (urlO.pathname.endsWith("/index.php")) { | |
| urlWithHTML.pathname = | |
| urlWithHTML.pathname.slice(0, -"index.php".length) + "index.html"; | |
| urlWithSlash.pathname = urlWithSlash.pathname.slice( | |
| 0, | |
| -"index.php".length, | |
| ); | |
| urlWithBare.pathname = urlWithBare.pathname.slice( | |
| 0, | |
| -"/index.php".length, | |
| ); | |
| } else { | |
| urlWithSlash.pathname += "/"; | |
| urlWithHTML.pathname += "/index.html"; | |
| urlWithPHP.pathname += "/index.php"; | |
| } | |
| return [urlWithHTML, urlWithPHP, urlWithSlash, urlWithBare]; | |
| }); | |
| return [...new Set(permutations.map(x => x.href))].map(x => new URL(x)); | |
| } | |
| // Random URLs generated by Claude. | |
| // This should suffice. - mogery | |
| const urls = `http://www.example.com/index.html | |
| https://subdomain.test.co.uk/path/to/resource?id=123&name=test | |
| https://api.github.com/users/john-doe/repos?sort=created&direction=desc | |
| http://shop.example.net/products/category/subcategory/item-name?color=blue&size=large&discount=20#reviews | |
| https://www.google.com/search?q=url+parsing&hl=en&source=hp&ei=abc123 | |
| https://en.wikipedia.org/wiki/URL#History | |
| http://mail.google.com/mail/u/0/#inbox/FMfcgzGpg | |
| https://www.amazon.co.jp/dp/B08N5WRWNW/ref=sr_1_1?keywords=echo&qid=1614556 | |
| https://subdomain1.subdomain2.example.com/path/index.html?param1=value1¶m2=value2 | |
| https://www.youtube.com/watch?v=dQw4w9WgXcQ&list=PLx123&index=5 | |
| https://docs.google.com/spreadsheets/d/1BxiMVs0XRA5nFMdKvBdBZjgmUUqptlbs74OgvE2upms/edit#gid=0 | |
| http://stackoverflow.com/questions/3809401/what-is-a-good-regular-expression-to-match-a-url | |
| https://www.linkedin.com/in/john-doe-123456789/ | |
| https://app.slack.com/client/T0123456/C0123456789/thread/C0123456789-1614556789.123456 | |
| http://www.example.com/page.html?utm_source=google&utm_medium=cpc&utm_campaign=spring_sale | |
| https://github.com/user/repo/blob/main/src/components/Header.jsx#L42 | |
| https://www.reddit.com/r/programming/comments/abc123/title_of_post/?sort=top | |
| http://bit.ly/3abc123 | |
| https://tinyurl.com/yckten7u | |
| https://www.example.com/index.php?id=123&action=view&session=abc123def456 | |
| http://sub-domain.example-site.co.uk/path/to/file.html | |
| https://www.airbnb.com/rooms/123456?check_in=2024-03-15&check_out=2024-03-20&guests=2 | |
| https://twitter.com/username/status/1234567890123456789 | |
| http://www.example.com/search?q=test+query+with+spaces&filter=recent | |
| https://drive.google.com/file/d/1a2b3c4d5e6f7g8h9i0j/view?usp=sharing | |
| http://example.com/index.html | |
| https://example.com/secure/page.html | |
| https://www.example.com/api/v2/users/123/posts?limit=10&offset=20&sort=date:desc | |
| https://maps.google.com/maps?q=40.7128,-74.0060&z=15 | |
| http://news.ycombinator.com/item?id=123456 | |
| https://www.facebook.com/photo.php?fbid=123456789&set=a.987654321&type=3 | |
| http://example.org/path/to/deeply/nested/resource/index.html | |
| https://medium.com/@username/article-title-here-abc123def456 | |
| http://localhost/api/v1/users | |
| https://www.instagram.com/p/ABC123def456/ | |
| http://test.example.com/~user/personal/page.html | |
| https://www.npmjs.com/package/express?activeTab=readme | |
| http://blog.example.net/2024/03/15/post-title-with-dashes/ | |
| https://codepen.io/username/pen/abcDEF?editors=1111 | |
| http://www.example.edu/students/enrollment.php?semester=fall2024&course=CS101 | |
| https://www.ebay.com/itm/123456789012?var=987654321098 | |
| http://archive.org/web/20210301000000/http://example.com/ | |
| https://www.twitch.tv/username/clip/ClipNameHere-abc123def456 | |
| http://example.com/products/electronics/phones/iphone.html?model=15&color=blue | |
| https://gitlab.com/group/project/-/merge_requests/123 | |
| http://www.example.com/blog/index.html | |
| https://store.steampowered.com/app/730/Counter-Strike_Global_Offensive/ | |
| http://example.net/api/users.json?page=1&per_page=50 | |
| https://www.spotify.com/us/track/4u7EnebtmKWzUH433cf5Qv | |
| http://subdomain.example.com/path%20with%20spaces/file.html | |
| https://www.goodreads.com/book/show/12345.Book_Title_Here | |
| http://example.com/search?q=test&lang=en&safe=on&start=10 | |
| https://marketplace.visualstudio.com/items?itemName=ms-python.python | |
| http://www.example.org/downloads/software-v1.2.3.tar.gz | |
| https://codesandbox.io/s/react-example-abc123 | |
| http://example.com/category/subcategory/index.html?sort=price&order=asc | |
| https://www.udemy.com/course/course-name-here/?couponCode=DISCOUNT50 | |
| http://api.example.com/v3/data.xml?format=json&key=public123 | |
| https://www.etsy.com/listing/123456789/handmade-item-title | |
| http://cdn.example.net/images/photo_2024_03_15.jpg?w=800&h=600&crop=true | |
| https://dev.to/username/article-slug-here-abc1 | |
| http://example.com/path/../another/path/file.html | |
| https://www.behance.net/gallery/123456789/Project-Title | |
| http://www.example.com/form.html?name=John+Doe&email=john%40example.com | |
| https://stackexchange.com/questions/123456/question-title-with-words | |
| http://staging.example.com/test/environment/page.html | |
| https://www.coursera.org/learn/machine-learning?action=enroll | |
| http://example.net/~user/public_html/index.html | |
| https://vimeo.com/123456789 | |
| http://blog.example.com/feed.rss | |
| https://www.pinterest.com/pin/123456789012345678/ | |
| http://example.org/cgi-bin/script.cgi?action=list&type=all | |
| https://unsplash.com/photos/abc123def456 | |
| http://www.example.com/products.php?category=1&brand=2&price_min=10&price_max=100 | |
| https://jsfiddle.net/username/abc123/ | |
| http://mirror.example.net/pub/software/package-1.0.0.rpm | |
| https://www.figma.com/file/abc123def456/Design-File-Name | |
| http://example.com/api/endpoint?callback=jsonpCallback123 | |
| https://open.spotify.com/playlist/37i9dQZF1DXcBWIGoYBM5M | |
| http://download.example.org/files/document.pdf?token=temp123 | |
| https://chrome.google.com/webstore/detail/extension-name/abcdefghijklmnop | |
| http://www.example.com/path/to/page.html#section-heading | |
| https://hub.docker.com/r/library/nginx | |
| http://example.net/search?q=query&filters[price][min]=10&filters[price][max]=100 | |
| https://www.patreon.com/username/posts?tag=tutorial | |
| http://static.example.com/assets/css/style.min.css?v=1.2.3 | |
| https://soundcloud.com/artist-name/track-name | |
| http://www.example.org/docs/api/reference.html#authentication | |
| https://codeberg.org/username/repository | |
| http://example.com/product/item-name/reviews?page=2&rating=5 | |
| https://www.kickstarter.com/projects/creator/project-name | |
| http://api.example.net/graphql?query={user(id:123){name,email}} | |
| https://gist.github.com/username/abc123def456789012345678901234567890 | |
| http://www.example.com/articles/2024/march/article-title.html | |
| https://discord.com/channels/123456789/987654321 | |
| http://example.org/download.php?file=document.docx&version=2 | |
| https://observablehq.com/@username/notebook-title | |
| http://test.example.com/api/v1/status.json | |
| https://www.deviantart.com/username/art/Art-Title-123456789 | |
| http://example.net/path/to/resource/index.html?debug=true | |
| https://repl.it/@username/project-name | |
| http://www.example.com/shop/cart.php?action=add&product_id=456 | |
| https://www.last.fm/music/Artist+Name/Album+Name | |
| http://cdn.example.org/media/video.mp4?quality=1080p | |
| https://exercism.io/tracks/javascript/exercises/hello-world | |
| http://example.com/api/search?query=test&fields=id,name,description | |
| https://www.meetup.com/group-name/events/123456789/ | |
| http://archive.example.net/2024/03/15/snapshot.html | |
| https://airtable.com/shrABC123DEF456 | |
| http://www.example.com/form?step=1&ref=homepage | |
| https://www.producthunt.com/posts/product-name | |
| http://api.example.org/rest/v2/users/123.json | |
| https://glitch.com/~project-name | |
| http://example.net/catalog/item.html?id=789&variant=red | |
| https://www.skillshare.com/classes/Class-Title/123456789 | |
| http://demo.example.com/preview/template-1/index.html | |
| https://dribbble.com/shots/123456-Shot-Title | |
| http://www.example.org/newsletter/subscribe.html?email=test%40test.com | |
| https://www.npmjs.com/search?q=keywords:react | |
| http://example.com/api/data?format=csv&delimiter=comma | |
| https://colab.research.google.com/drive/1abc123def456 | |
| http://media.example.net/thumbnails/image_300x200.jpg | |
| https://www.notion.so/Page-Title-abc123def456 | |
| http://example.org/path/to/file.html?lang=en-US | |
| https://www.canva.com/design/DAE123abc/edit | |
| http://beta.example.com/new-features/test.html | |
| https://www.freecodecamp.org/learn/javascript-algorithms-and-data-structures/ | |
| http://example.net/api/v1/auth/login.json | |
| https://www.hackerrank.com/challenges/challenge-name/problem | |
| http://www.example.com/blog/tags/javascript/page/2 | |
| https://replit.com/@username/project-name#file.js | |
| http://assets.example.org/fonts/font-family.woff2 | |
| https://www.kaggle.com/datasets/username/dataset-name | |
| http://example.com/products/index.html?sort=newest&limit=20 | |
| https://zenodo.org/record/123456 | |
| http://www.example.net/documentation/v2.0/getting-started.html | |
| https://www.openstreetmap.org/#map=15/40.7128/-74.0060 | |
| http://example.org/api/export?format=pdf&orientation=landscape | |
| https://codewars.com/kata/kata-id-here | |
| http://staging.example.com/test?env=development&debug=1 | |
| https://www.atlassian.com/software/jira/pricing | |
| http://example.com/path/to/page.aspx?id=123&mode=edit | |
| https://bundlephobia.com/package/lodash@4.17.21 | |
| http://www.example.org/events/2024/conference.html#speakers | |
| https://regex101.com/r/abc123/1 | |
| http://api.example.net/v1/search?q=keyword&limit=10&offset=0 | |
| https://www.typescriptlang.org/docs/handbook/intro.html | |
| http://example.com/user/profile.php?username=johndoe | |
| https://marketplace.zoom.us/apps/abc123 | |
| http://www.example.net/resources/whitepaper.pdf?download=true | |
| https://www.pluralsight.com/courses/course-name | |
| http://example.org/api/webhook?event=user.created | |
| https://httpbin.org/get?test=value | |
| http://demo.example.com/sandbox/index.html | |
| https://www.jetbrains.com/idea/download/#section=windows | |
| http://example.net/path/to/deeply/nested/folder/file.html | |
| https://www.heroku.com/platform | |
| http://www.example.com/contact.html?subject=inquiry&priority=high | |
| https://material-ui.com/components/buttons/ | |
| http://api.example.org/oauth/authorize?client_id=123&redirect_uri=http://example.com | |
| https://www.digitalocean.com/community/tutorials/tutorial-slug | |
| http://example.com/search/results.php?query=test&page=3 | |
| https://vercel.com/new/templates | |
| http://www.example.net/downloads/index.html?category=software | |
| https://www.postman.com/downloads/ | |
| http://example.org/api/v2/data.json?include=metadata | |
| https://tailwindcss.com/docs/installation | |
| http://media.example.com/stream/video.m3u8 | |
| https://www.elastic.co/guide/index.html | |
| http://example.net/admin/dashboard/index.html | |
| https://nodejs.org/dist/latest-v18.x/docs/api/ | |
| http://www.example.com/shop/checkout.html?coupon=SAVE20 | |
| https://www.mongodb.com/cloud/atlas/register | |
| http://example.org/files/archive.zip?extract=true | |
| https://bundler.io/guides/getting_started.html | |
| http://api.example.net/graphql/playground | |
| https://www.selenium.dev/documentation/ | |
| http://example.com/blog/post.html?id=456&comments=true | |
| https://flutter.dev/docs/get-started/install | |
| http://www.example.org/gallery/photos/2024/index.html | |
| https://www.djangoproject.com/start/ | |
| http://example.net/api/status?format=json&verbose=true | |
| https://reactjs.org/tutorial/tutorial.html | |
| http://cdn.example.com/js/app.bundle.js?v=3.2.1 | |
| https://www.ruby-lang.org/en/documentation/ | |
| http://example.org/search?facets[category][]=tech&facets[category][]=science | |
| https://pytorch.org/get-started/locally/ | |
| http://www.example.com/faq.html#payment-methods | |
| https://www.tensorflow.org/tutorials | |
| http://example.net/export/data.csv?encoding=utf-8 | |
| https://vuejs.org/guide/introduction.html | |
| http://api.example.org/v3/metrics?period=daily&start=2024-01-01 | |
| https://symfony.com/doc/current/index.html | |
| http://example.com/products/sale/index.html?discount=50 | |
| https://laravel.com/docs/10.x/installation | |
| http://www.example.net/api/batch?operations=create,update,delete | |
| https://angular.io/guide/setup-local | |
| http://example.org/tools/calculator.html?mode=scientific | |
| https://expressjs.com/en/starter/installing.html | |
| http://static.example.com/images/logo.svg?color=blue | |
| https://spring.io/guides/gs/spring-boot/ | |
| http://example.net/path/to/file.xml?transform=json | |
| https://www.rust-lang.org/learn/get-started | |
| http://www.example.com/newsletter/archive/2024/march.html | |
| https://kotlinlang.org/docs/getting-started.html | |
| http://example.org/api/search?facets=true&hl=true&hl.fl=title,description | |
| https://dart.dev/guides/language/language-tour | |
| http://media.example.net/audio/podcast-episode-123.mp3 | |
| https://golang.org/doc/install | |
| http://example.com/user/settings.html?tab=privacy | |
| https://www.scala-lang.org/download/ | |
| http://www.example.org/reports/annual/2024/index.html | |
| https://elixir-lang.org/getting-started/introduction.html | |
| http://api.example.net/v1/aggregate?groupby=category&metric=count | |
| https://www.haskell.org/platform/ | |
| http://example.org/documentation/api/v2/index.html#endpoints | |
| https://julialang.org/downloads/ | |
| http://www.example.com/store/product.php?sku=ABC123&reviews=show | |
| https://www.erlang.org/downloads | |
| http://example.net/calendar/events.html?month=march&year=2024 | |
| https://clojure.org/guides/getting_started | |
| http://cdn.example.org/videos/tutorial-01.webm | |
| https://fsharp.org/learn/`.split("\n"); | |
| describe("Proof that generateURLPermutations is stable", () => { | |
| describe.each(urls)("%s", url => { | |
| it("any permutation of a URL will produce the same permutations (content, length, order!) as the permutations of the original URL", () => { | |
| const permutations = generateURLPermutations(new URL(url)); | |
| expect(permutations).toBeInstanceOf(Array); | |
| expect(permutations.length).toBeGreaterThan(0); | |
| permutations.forEach(newBase => { | |
| const newPermutations = generateURLPermutations(newBase); | |
| expect(newPermutations.length).toBe(permutations.length); | |
| expect( | |
| newPermutations.every((p, i) => p.href === permutations[i].href), | |
| ).toBe(true); | |
| }); | |
| }); | |
| }); | |
| }); | |