| import sys, collections |
| sys.path.insert(0, "/job/work") |
| from label.common import load_forms |
| C = collections.Counter |
| rows = list(load_forms(sys.argv[1])) |
| sigs = C(r["sig"] for r in rows) |
| hosts = C(r["host"] for r in rows) |
| print("forms", len(rows), "unique sigs", len(sigs), "hosts", len(hosts)) |
| uniq = {} |
| for r in rows: |
| uniq.setdefault(r["sig"], r) |
| u = list(uniq.values()) |
| pw = [r for r in u if any(f["type"] == "input password" for f in r["fields"])] |
| print("unique with password", len(pw), "html ok", sum(1 for r in u if r.get("html"))) |
| print("nfields", sorted(C(min(len(r["fields"]), 12) for r in u).items())) |
| print("types", C(f["type"] for r in u for f in r["fields"]).most_common(12)) |
| print("top hosts", hosts.most_common(5)) |
| single_search = sum(1 for r in u if len(r["fields"]) == 1 and (r["fields"][0]["type"] == "input search" or r["fields"][0]["name"] in ("q", "s", "search", "query", "keyword", "keywords"))) |
| print("single-field search-like", single_search) |
|
|
