HasanSabour's picture
download
raw
5.33 kB
import json
import gzip
from tqdm import tqdm
import pickle
from collections import defaultdict
from sklearn.model_selection import train_test_split
import re
import sys
class Preprocess():
def __init__(self, dataset):
self.dataset = dataset
def parse(self, file_path):
g = gzip.open(file_path, 'rb')
for l in tqdm(g):
yield json.loads(l)
def count(self, topk):
f = open(f'./data/{self.dataset}/meta_{self.dataset}.json', 'r')
json_data = f.readlines()
f.close()
metadata = {}
item_with_meta = set()
data_list = [json.loads(line[:-1]) for line in json_data]
for l in data_list:
asin = l['asin']
if len(l['description']) == 0 and len(l['title']) == 0:
continue
elif len(l['description']) == 0:
metadata[asin] = l['title']
# metadata[asin] = metadata[asin].replace('\n', ' ')
else:
title = l['title']
description = l['description'][0]
metadata[asin] = title + ' ' + description
# metadata[asin] = metadata[asin].replace('\n', ' ')
metadata[asin] = re.sub(r'[\n\t\s\"\'\r]+', ' ', metadata[asin])
metadata[asin] = re.sub(r'<.*?>', '', metadata[asin])
item_with_meta.add(asin)
# first count: item
countU = defaultdict(lambda: 0)
countI = defaultdict(lambda: 0)
line = 0
frequent_item = []
review_text = defaultdict(lambda: defaultdict(lambda: ''))
file_path = f'./data/{self.dataset}/{self.dataset}.json.gz'
empty_review = 0
for l in self.parse(file_path):
if l['asin'] not in item_with_meta:
continue
line += 1
asin = l['asin'] # item id
rev = l['reviewerID']
# countU[rev] += 1
countI[asin] += 1
reviewText = ""
summary = ""
try:
reviewText = l['reviewText']
except:
pass
try:
summary = l['summary']
except:
pass
review_text[rev][asin] = reviewText + ' ' + summary
if(review_text[rev][asin] == ' '):
empty_review += 1
review_text[rev][asin] = re.sub(r'[\n\t\s\"\'\r]+', ' ', review_text[rev][asin])
review_text[rev][asin] = re.sub(r'<.*?>', '', review_text[rev][asin])
print(f'Empty review: {empty_review}')
for key, value in countI.items():
if value > topk:
frequent_item.append(key)
# second count: user
for l in self.parse(file_path):
asin = l['asin']
if asin not in frequent_item:
continue
rev = l['reviewerID']
countU[rev] += 1
# third: record
empty_review = 0
user_id = {}
item_id = {}
item_meta = {}
user_num = 0
item_num = 0
interactions = defaultdict(list)
kept_reviews = defaultdict(lambda: defaultdict(lambda: ''))
for l in self.parse(file_path):
asin = l['asin']
if asin not in frequent_item:
continue
rev = l['reviewerID']
if countU[rev] > topk:
if rev in user_id:
userid = user_id[rev]
else:
userid = user_num
user_id[rev] = userid
interactions[userid] = []
user_num += 1
if asin in item_id:
itemid = item_id[asin]
else:
itemid = item_num
item_id[asin] = itemid
item_meta[itemid] = metadata[asin]
item_num += 1
interactions[userid].append(itemid)
kept_reviews[userid][itemid] = review_text[rev][asin]
if review_text[rev][asin] == '':
empty_review += 1
print(f'Empty review: {empty_review}')
with open (f'./data/{self.dataset}/train.txt', 'w+') as trainf:
with open (f'./data/{self.dataset}/test.txt', 'w+') as testf:
for user in interactions.keys():
try:
train, test = train_test_split(interactions[user], test_size=0.2, shuffle=True)
except:
print(user)
print(interactions[user])
trainf.write(f'{user} ' + ' '.join(map(str, train)) + '\n')
testf.write(f'{user} ' + ' '.join(map(str, test)) + '\n')
with open (f'./data/{self.dataset}/item_dict.txt', 'w+') as itemf:
for key, value in item_meta.items():
itemf.write(f'{key} {value}\n')
with open (f'./data/{self.dataset}/review_dict.txt', 'w+') as reviewf:
for key, value in kept_reviews.items():
for k, v in value.items():
reviewf.write(f'{key} {k} {v}\n')
if __name__ == '__main__':
dataset = str(sys.argv[1])
preprocess = Preprocess(dataset)
preprocess.count(10)

Xet Storage Details

Size:
5.33 kB
·
Xet hash:
e4a661f1fe03c98a3af0ff1d9f89b80977a4686ee183d9f3381197508f99aaa9

Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.