webtraffic commited on
Commit
d5ff03b
·
verified ·
1 Parent(s): f47f42e

Update main.py

Browse files
Files changed (1) hide show
  1. main.py +14 -20
main.py CHANGED
@@ -21,14 +21,10 @@ class TextPayload(BaseModel):
21
  async def analyze_text(payload: TextPayload):
22
  doc = nlp(payload.text)
23
 
24
- # We use the raw tokens to verify frequencies accurately
25
  entity_freq = Counter([ent.text.lower() for ent in doc.ents])
26
 
27
- # BRAND GUARDIAN: Names in this list will NEVER be split
28
  brand_safety = [
29
- 'johnson & johnson', 'ben & jerry', 'tiffany & co', 'h&m', 'at&t',
30
- 'dolce & gabbana', 'marks & spencer', 'bed bath & beyond', 'a&p', 'c&a',
31
- 'p&g', 'procter & gamble'
32
  ]
33
 
34
  processed_entities = []
@@ -38,15 +34,10 @@ async def analyze_text(payload: TextPayload):
38
  raw_text = ent.text.strip()
39
  e_lower = raw_text.lower()
40
 
41
- # Decide if we should split
42
  sub_items = [raw_text]
43
-
44
- # Only split if NOT a protected brand and contains 'and' or '&'
45
  if e_lower not in brand_safety:
46
  if " and " in e_lower or " & " in raw_text:
47
- # Split the text into parts
48
  parts = raw_text.replace(" and ", " & ").split(" & ")
49
- # Only split if each part is a short name (1-2 words)
50
  if all(1 <= len(p.strip().split()) <= 2 for p in parts):
51
  sub_items = [p.strip() for p in parts if len(p.strip()) > 1]
52
 
@@ -57,16 +48,20 @@ async def analyze_text(payload: TextPayload):
57
 
58
  label = ent.label_
59
 
60
- # Smart Category Overrides
61
- tech_orgs = ['gravatar', 'hostwinds', 'facebook', 'instagram', 'bing', 'yandex', 'google']
62
- tech_prods = ['vps', 'server', 'hosting', 'cloud', 'schema', 'json', 'api', 'plugin']
63
- concepts = ['free', 'booster', 'get', 'noticed', 'top', 'best', 'simple']
64
-
65
- if any(t in f_lower for t in tech_orgs): label = "ORG"
66
- elif any(p in f_lower for p in tech_prods): label = "PRODUCT"
67
- elif any(c in f_lower for c in concepts): label = "CONCEPT"
68
 
69
- # Length filter to catch those long "Privacy & Cookie" strings
 
 
 
 
 
 
 
 
 
70
  if len(final_text.split()) >= 4:
71
  continue
72
 
@@ -78,7 +73,6 @@ async def analyze_text(payload: TextPayload):
78
  })
79
  seen.add(f_lower)
80
 
81
- # Return up to 50 entities so the list feels rich and full
82
  sorted_entities = sorted(processed_entities, key=lambda x: x['frequency'], reverse=True)
83
 
84
  return {
 
21
  async def analyze_text(payload: TextPayload):
22
  doc = nlp(payload.text)
23
 
 
24
  entity_freq = Counter([ent.text.lower() for ent in doc.ents])
25
 
 
26
  brand_safety = [
27
+ 'johnson & johnson', 'ben & jerry', 'tiffany & co', 'h&m', 'at&t'
 
 
28
  ]
29
 
30
  processed_entities = []
 
34
  raw_text = ent.text.strip()
35
  e_lower = raw_text.lower()
36
 
 
37
  sub_items = [raw_text]
 
 
38
  if e_lower not in brand_safety:
39
  if " and " in e_lower or " & " in raw_text:
 
40
  parts = raw_text.replace(" and ", " & ").split(" & ")
 
41
  if all(1 <= len(p.strip().split()) <= 2 for p in parts):
42
  sub_items = [p.strip() for p in parts if len(p.strip()) > 1]
43
 
 
48
 
49
  label = ent.label_
50
 
51
+ # --- UPDATED LOGIC HERE ---
52
+ # 1. Marketing/General Topics (Force to CONCEPT)
53
+ concepts = ['privacy', 'cookies', 'cookie', 'booster', 'get', 'noticed', 'free', 'terms']
 
 
 
 
 
54
 
55
+ # 2. Technical Data/Tools (Force to PRODUCT)
56
+ tech_items = ['exif', 'mediaif', 'cookiesif', 'json', 'api', 'schema', 'vps', 'server']
57
+
58
+ if any(c in f_lower for c in concepts):
59
+ label = "CONCEPT"
60
+ elif any(t in f_lower for t in tech_items):
61
+ label = "PRODUCT"
62
+ # ---------------------------
63
+
64
+ # Keep the length filter to avoid messy UI strings
65
  if len(final_text.split()) >= 4:
66
  continue
67
 
 
73
  })
74
  seen.add(f_lower)
75
 
 
76
  sorted_entities = sorted(processed_entities, key=lambda x: x['frequency'], reverse=True)
77
 
78
  return {