| import pandas as pd |
| import numpy as np |
| import requests |
| from bs4 import BeautifulSoup |
| import urllib.parse |
|
|
| import streamlit as st |
| import uuid |
|
|
| import warnings |
|
|
| warnings.filterwarnings('ignore') |
|
|
| |
| from folder_management import create_folder, remove_files_folder |
| from sqlite_database import create_insert_table, query_table |
|
|
| ''' |
| This Module will, for a particular keyword and publisherid combo query data from bizrate.com store it |
| in bizrate.db. And then generate Top 50 recommendation from it and Store it in RecSysData.db |
| |
| ''' |
|
|
|
|
| |
| def tag_to_list(tag_name, soup): |
| if tag_name == 'Image': |
| tag_list = soup.find_all(tag_name, {'xsize': '400'}) |
| else: |
| tag_list = soup.find_all(tag_name) |
|
|
| return pd.Series([item.text for item in tag_list]) |
|
|
|
|
| def loadRSS(filepath, keyword, publisherid): |
| url = filepath |
| resp = requests.get(url) |
| with open('bizrate/' + keyword + "_" + publisherid + '.xml', 'wb') as f: |
| f.write(resp.content) |
|
|
|
|
| def remove_dollar_comma(row): |
| try: |
| row = row.split('$')[1] |
| row = row.replace(',', '') |
| row = float(row) |
| except AttributeError: |
| return row |
| return row |
|
|
|
|
| def url_decode(url): |
| return urllib.parse.unquote(url.split('?t=')[1]) |
|
|
|
|
| def query_bizrate(keyword, publisherid='725895', search_results=500): |
| file_path = 'http://catalog.bizrate.com/services/catalog/v1/api/product?apiKey=c942e4e24d0859a748b4d1c07c1c3df1' \ |
| '&publisherId={}&placementId=1&categoryId=&keyword={' \ |
| '}&productId=&productIdType=&offersOnly=true&merchantId=&brandId=&biddedOnly=&minPrice=&maxPrice' \ |
| '=&minMarkdown=&zipCode=&freeShipping=&start=0&results={' \ |
| '}&startOffers=0&resultsOffers=0&sort=relevancy_desc&attFilter=&attWeights=&attributeId' \ |
| '=&resultsAttribute=10&resultsAttributeValues=10&showAttributes=&showProductAttributes' \ |
| '=&minRelevancyScore=1000&maxAge=&showRawUrl=&showUnitPricing=&useSecureImageDomain' \ |
| '=&useSecureLinkDomain=&reviews=none&format=xml&callback=callback'.format(publisherid, |
| keyword, |
| search_results) |
|
|
| loadRSS(file_path, keyword, publisherid) |
|
|
| with open('bizrate/' + keyword + "_" + publisherid + '.xml', 'r', errors='ignore') as f: |
| file = f.read() |
|
|
| soup = BeautifulSoup(file, 'xml') |
|
|
| |
| |
| |
| |
|
|
| cols = ['title', 'Brand', 'url', 'Image', 'Skus', 'price', 'originalPrice', 'markdownPercent', 'totalPrice', |
| 'condition', 'stock', 'relevancy'] |
|
|
| df_product = pd.DataFrame({'title': pd.Series(dtype='object'), |
| 'Brand': pd.Series(dtype='object'), |
| 'url': pd.Series(dtype='object'), |
| 'Image': pd.Series(dtype='object'), |
| 'Skus': pd.Series(dtype='object'), |
| 'price': pd.Series(dtype='object'), |
| 'originalPrice': pd.Series(dtype='object'), |
| 'markdownPercent': pd.Series(dtype='object'), |
| 'totalPrice': pd.Series(dtype='object'), |
| 'condition': pd.Series(dtype='object'), |
| 'stock': pd.Series(dtype='object'), |
| 'relevancy': pd.Series(dtype='object')}) |
| for col in cols: |
| df_product[col] = tag_to_list(col, soup) |
| df_product.dropna(subset=['Skus'], inplace=True) |
| df_product['Skus'] = df_product['Skus'].astype('object') |
|
|
| |
| create_insert_table(db_name='bizrate', table_name=keyword + "_" + publisherid, df=df_product) |
|
|
|
|
| def recommend(keyword, publisherid='725895', relevancy_filter=False, price_filter=True, discount_filter=False, |
| condition_filter='NEW', stock_filter='IN', n_rec=50): |
|
|
| |
| df_product = query_table(db_name='bizrate', table_name=keyword + "_" + publisherid) |
|
|
| '''df_sku = pd.read_excel('df_sku.xlsx') |
| sku_list = list(df_product['Skus'].unique()) |
| df_session = pd.read_excel('df_session.xlsx') |
| session_id = list(df_session['ID'])[-1] |
| for sku in sku_list: |
| df_row = pd.DataFrame({'SessionID': session_id, 'Keyword': selected_keyword, 'Skus': sku, 'Count': 0}, index=[0]) |
| df_sku = pd.concat([df_sku, df_row], ignore_index=True) |
| df_sku.to_excel('df_sku.xlsx', index = False)''' |
|
|
| df_product['price'] = df_product['price'].map(remove_dollar_comma) |
| df_product['originalPrice'] = df_product['originalPrice'].map(remove_dollar_comma) |
| df_product['totalPrice'] = df_product['totalPrice'].map(remove_dollar_comma) |
| |
| df_product['markdownPercent'] = df_product['markdownPercent'].astype('float') |
| df_product['relevancy'] = df_product['relevancy'].astype('float') |
|
|
| |
| df_product = df_product.sort_values(by=['relevancy', 'price', 'markdownPercent'], |
| ascending=[relevancy_filter, price_filter, discount_filter], na_position='last') |
|
|
| |
| |
| |
|
|
| filter_condition = (df_product['condition'] == condition_filter) & (df_product['stock'] == stock_filter) |
| df_product = df_product[filter_condition] |
|
|
| |
| top_n_rec = df_product.head(n_rec) |
|
|
| |
| create_insert_table(db_name='RecSysData', table_name=keyword + "_" + publisherid, df=top_n_rec) |
|
|
|
|
| |
| |
| |
| |
| |
|
|
| |
| |
| |
| |
| |
|
|
