| import streamlit as st |
| from dotenv import load_dotenv |
| import os |
| from langchain.llms import OpenAI |
| from langchain.chat_models import ChatOpenAI |
| from langchain.schema import AIMessage, HumanMessage, SystemMessage |
| from langchain.prompts.chat import ( |
| ChatMessagePromptTemplate, |
| SystemMessagePromptTemplate, |
| AIMessagePromptTemplate, |
| HumanMessagePromptTemplate |
| ) |
| from langchain.callbacks import get_openai_callback |
|
|
| |
| from langchain.document_loaders import AsyncChromiumLoader |
| from langchain.document_transformers import BeautifulSoupTransformer |
| import requests |
|
|
| |
| import pandas as pd |
| import json |
| from pandas import json_normalize |
|
|
| def rewrite(text, language='Bahasa Indonesia'): |
| llm = ChatOpenAI(model='gpt-3.5-turbo-16k', temperature=0) |
| message = [ |
| SystemMessage(content='Your are expert blogger'), |
| HumanMessage(content=f'write the following text into 1000 words blog article, use language: {language}, use casual tone. and give clickbait title. TEXT: {text}') |
| ] |
|
|
| with get_openai_callback() as cost: |
| output = llm(message).content |
|
|
| return output, cost |
|
|
| def extract_seo_keywords(text): |
| llm = ChatOpenAI(model='gpt-3.5-turbo-16k', temperature=0) |
| message = [ |
| SystemMessage(content='Your are an SEO expert'), |
| HumanMessage(content=f'indentify 10 SEO keywords in this text. order by the most relevance keyword. output format: comma separated value. example output: keyword,another keyword,some keyword. TEXT: {text}') |
| ] |
|
|
| with get_openai_callback() as cost: |
| keywords = llm(message).content.split(',') |
| array_keywords = [i.lstrip() for i in keywords] |
|
|
| return array_keywords, cost |
|
|
| def summarize(text, language='Bahasa Indonesia'): |
| llm = ChatOpenAI(model='gpt-3.5-turbo-16k', temperature=0) |
| message = [ |
| SystemMessage(content='Your are expert copywriter'), |
| HumanMessage(content=f'Extract main idea the following article and summarize into 10 bullet point or more. Use language: {language}. TEXT: {text}') |
| ] |
|
|
| with get_openai_callback() as cost: |
| summary = llm(message).content |
| return summary, cost |
|
|
| def scrap(url): |
| scraper = AsyncChromiumLoader(url) |
| html = scraper.load() |
|
|
| extractor = BeautifulSoupTransformer() |
| content = extractor.transform_documents(html, tags_to_extract=['h1', 'h2', 'h3', 'h4','h5', 'h6','p']) |
| print(content[0].page_content) |
| return content[0].page_content |
|
|
| def get_keywords_data(keywords): |
| my_data = { |
| 'currency': 'USD', |
| 'dataSource': 'gkp', |
| 'kw[]': keywords |
| } |
|
|
| kw_api_key = os.environ.get('KEYWORD_API') |
| auth = f'Bearer {kw_api_key}' |
|
|
| my_headers = { |
| 'Accept': 'application/json', |
| 'Authorization': auth |
| } |
| response = requests.post('https://api.keywordseverywhere.com/v1/get_keyword_data', data=my_data, headers=my_headers) |
| if response.status_code == 200: |
| data = body=response.content.decode('utf-8') |
| return data |
| else: |
| st.write(body=response.content.decode('utf-8')) |
|
|
| def main(): |
|
|
| load_dotenv() |
|
|
| st.subheader("Summarize, Extract Keywords, Rewrite") |
|
|
| url = st.text_input("Enter URL, start with http:// or https://") |
| options = ['Bahasa Indonesia', 'English', 'Japanese', 'France', 'Germany', 'Dutch'] |
| language = st.selectbox("Output language", options=options) |
| submit = st.button("submit") |
| content = None |
|
|
| if url and submit: |
| |
| |
| with st.spinner(f'Processing: {url}'): |
| content = scrap([url]) |
|
|
| if content is not None: |
|
|
| tab1, tab2, = st.tabs(["Summary", "Rewritten content"]) |
| |
| with tab1: |
| |
| with st.spinner('summarizing content..'): |
| st.subheader('Summary') |
| summary, cost = summarize(content, language) |
| st.write(summary) |
| st.write(cost) |
|
|
| with st.spinner('extracing keywords..'): |
| st.subheader('SEO Keywords ') |
| keywords, cost = extract_seo_keywords(content) |
|
|
| |
| st.write(cost) |
|
|
| data = get_keywords_data(keywords) |
|
|
| |
| dict= json.loads(data) |
|
|
| |
| vol_list = [] |
| cpc_value_list = [] |
| keyword_list = [] |
| competition_list = [] |
| trend_value_list = [] |
|
|
| |
| for item in dict["data"]: |
| vol_list.append(item["vol"]) |
| cpc_value_list.append(float(item["cpc"]["value"])) |
| keyword_list.append(item["keyword"]) |
| competition_list.append(item["competition"]) |
| trend_values = [trend["value"] for trend in item["trend"]] |
| trend_value_list.append(trend_values) |
|
|
| |
| df = pd.DataFrame({ |
| 'Keyword': keyword_list, |
| 'Vol': vol_list, |
| 'CPC': cpc_value_list, |
| 'Competition': competition_list, |
| 'Trend (last 12 months)': trend_value_list |
| }) |
|
|
| |
| st.dataframe( |
| df, |
| column_config={ |
| "CPC": st.column_config.NumberColumn( |
| format="$%f", |
| ), |
| "Trend (last 12 months)": st.column_config.LineChartColumn(y_min=0) |
| }, |
| hide_index=True, |
| width=800 |
| ) |
|
|
| with st.expander("see extracted content"): |
| st.write(content) |
|
|
| with tab2: |
| st.subheader("Rewrite content") |
|
|
| rewrite_button = st.button("Rewrite!") |
|
|
| if rewrite_button: |
| with st.spinner('rewriting content..'): |
| st.subheader('rewritten content') |
| output, cost = rewrite(summary, language) |
|
|
| st.write(output) |
| st.write(cost) |
|
|
|
|
| if __name__ == "__main__": |
| main() |
|
|
|
|
|
|
|
|
|
|
