Spaces:
Sleeping
Sleeping
File size: 4,690 Bytes
695b43d | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 | #%%
from bs4 import BeautifulSoup
import requests
import matplotlib.pyplot as plt
import io
import base64
import re
# get URL
def scrape(commodity="Apple"):
#list
list1 = ['rice','maize','jute','cotton','coconut','papaya','orange_(fruit)','apple','cantaloupe','watermelon',
'grapes','mango','banana','pomegranate','lentil','blackgram','mungbean','mothbean','pigeonpea',
'chickpea','coffee']
link = "https://en.wikipedia.org/wiki/"
page = requests.get(link+(commodity.capitalize()))
# scrape webpage
soup = BeautifulSoup(page.content, 'lxml')
tags = soup.find_all('h2')
headings = ['description', 'agronomy', 'pests', 'structure and physiology', 'pests and diseases',
'cultivation', 'uses','use', 'environmental impact', 'types','varieties','breeding']
# for head in tags:
# print(head.text)
infopage = ""
textjson = {}
l=[]
i,j,k=1,1,1
for head in tags:
if any(sub in head.text.lower() for sub in headings):
#tempjson = {}
#print(head.text)
#infopage+=head.text
head_text = head.text
head_text = re.sub(r'\[\d+\]', '', head_text)
textjson[f"heading{i}"] = head_text
l.append(f"heading{i}")
i+=1
content = head.find_next_sibling()
#tempjson['heading'] = head_text
while content and content.name!='h2':
if content.name =='h3' :
subheading = content.text
#print(content.text)
subheading = re.sub(r'\[\d+\]', '', subheading)
#print(subheading)
textjson[f"subheading{j}"] = subheading
l.append(f"subheading{j}")
j+=1
#infopage+=content.text
if content.name == 'p':
para =content.text
para = re.sub(r'\[\d+\]', '', para)
textjson[f"para{k}"] = para
l.append(f"para{k}")
k+=1
content = content.find_next_sibling()
# textjson[f'heading{i}']= head_text
# textjson[f'content{i}'] = tempjson
#text = re.sub(r'\[\d+\]', '', infopage)
#for commodity prices
link2 = "https://api.data.gov.in/resource/9ef84268-d588-465a-a308-a864a43d0070?api-key=579b464db66ec23bdd000001cdd3946e44ce4aad7209ff7b23ac571b&format=json&filters%5Bcommodity%5D="
#commodity = "Apple"
info = requests.get(link2+(commodity.capitalize()))
# print(link2+(commodity.capitalize()))
# print(info)
if info.status_code == 200:
data = info.json()
data = data['records']
if data == []:
print("no data available")
else:
print("Failed to retrieve data. Status code:", info.status_code)
# Extracting data for plotting
markets = [entry['market'] for entry in data]
min_prices = [float(entry['min_price']) for entry in data]
max_prices = [float(entry['max_price']) for entry in data]
modal_prices = [float(entry['modal_price']) for entry in data]
#modifications
# for i in range(len(markets)):
# plt.text(modal_prices[i], i, f"{modal_prices[i]}", ha='left', va='center', fontsize=8, color='black')
# plt.text(max_prices[i], i, f"{max_prices[i]}", ha='left', va='center', fontsize=8, color='black')
# plt.text(min_prices[i], i, f"{min_prices[i]}", ha='left', va='center', fontsize=8, color='black')
# for index, value in enumerate(modal_prices):
# plt.text(value, index, str(value), va='center', fontsize=10, color='black')
# Plotting
plt.figure(figsize=(10, 6))
plt.barh(markets, min_prices, color='lightblue', label='Min Price')
plt.barh(markets, max_prices, color='skyblue', label='Max Price', left=min_prices)
plt.barh(markets, modal_prices, color='dodgerblue', label='Modal Price', left=[min + max for min, max in zip(min_prices, max_prices)])
plt.xlabel('Price')
plt.ylabel('Market')
plt.title(f'{commodity} Prices in Different Markets')
plt.legend()
plt.tight_layout()
#plt.show()
# Encode the plot image as a bytes object
buffer = io.BytesIO()
plt.savefig(buffer, format='png')
buffer.seek(0)
plot_image = buffer.getvalue()
buffer.close()
# Encode the plot image as base64
plot_image_base64 = base64.b64encode(plot_image).decode('utf-8')
response_data = plot_image_base64
# {
# 'plot_image':
# }
return textjson,response_data,l
# display scraped data
#print(soup.prettify())
# %%
|