Upload 2 files

app.py

@@ -0,0 +1,193 @@
+import streamlit as st
+from dotenv import load_dotenv
+import os
+from langchain.llms import OpenAI
+from langchain.chat_models import ChatOpenAI
+from langchain.schema import AIMessage, HumanMessage, SystemMessage
+from langchain.prompts.chat import (
+   ChatMessagePromptTemplate, 
+   SystemMessagePromptTemplate, 
+   AIMessagePromptTemplate, 
+   HumanMessagePromptTemplate
+)
+from langchain.callbacks import get_openai_callback
+
+# scrapper
+from langchain.document_loaders import AsyncChromiumLoader
+from langchain.document_transformers import BeautifulSoupTransformer
+import requests
+
+#data
+import pandas as pd
+import json
+from pandas import json_normalize
+
+def rewrite(text, language='Bahasa Indonesia'):
+  llm = ChatOpenAI(model='gpt-3.5-turbo-16k', temperature=0)
+  message = [
+    SystemMessage(content='Your are expert blogger'),
+    HumanMessage(content=f'write the following text into 1000 words blog article, use language: {language}, use casual tone. and give clickbait title. TEXT: {text}')
+  ]
+
+  with get_openai_callback() as cost:
+    output = llm(message).content
+
+  return output, cost
+
+def extract_seo_keywords(text):
+  llm = ChatOpenAI(model='gpt-3.5-turbo-16k', temperature=0)
+  message = [
+    SystemMessage(content='Your are an SEO expert'),
+    HumanMessage(content=f'indentify 10 SEO keywords in this text. order by the most relevance keyword. output format: comma separated value. example output: keyword,another keyword,some keyword. TEXT: {text}')
+  ]
+
+  with get_openai_callback() as cost:
+    keywords = llm(message).content.split(',')
+    array_keywords = [i.lstrip() for i in keywords]
+
+  return array_keywords, cost
+
+def summarize(text, language='Bahasa Indonesia'):
+  llm = ChatOpenAI(model='gpt-3.5-turbo-16k', temperature=0)
+  message = [
+    SystemMessage(content='Your are expert copywriter'),
+    HumanMessage(content=f'Extract main idea the following article and summarize into 10 bullet point or more. Use language: {language}. TEXT: {text}')
+  ]
+
+  with get_openai_callback() as cost:
+    summary = llm(message).content
+    return summary, cost
+
+def scrap(url):
+  scraper = AsyncChromiumLoader(url)
+  html = scraper.load()
+
+  extractor = BeautifulSoupTransformer()
+  content = extractor.transform_documents(html, tags_to_extract=['h1', 'h2', 'h3', 'h4','h5', 'h6','p'])
+  print(content[0].page_content)
+  return content[0].page_content
+
+def get_keywords_data(keywords):
+  my_data = {
+      'currency': 'USD',
+      'dataSource': 'gkp',
+      'kw[]': keywords
+  }
+
+  kw_api_key = os.environ.get('KEYWORD_API')
+  auth = f'Bearer {kw_api_key}'
+
+  my_headers = {
+      'Accept': 'application/json',
+      'Authorization': auth
+  }
+  response = requests.post('https://api.keywordseverywhere.com/v1/get_keyword_data', data=my_data, headers=my_headers)
+  if response.status_code == 200:
+      data = body=response.content.decode('utf-8')
+      return data
+  else:
+      st.write(body=response.content.decode('utf-8'))
+
+def main():
+
+  load_dotenv()
+
+  st.subheader("Summarize, Extract Keywords, Rewrite")
+
+  url = st.text_input("Enter URL, start with http:// or https://")
+  options = ['Bahasa Indonesia', 'English', 'Japanese', 'France', 'Germany', 'Dutch']
+  language = st.selectbox("Output language", options=options)
+  submit = st.button("submit")
+  content = None
+
+  if url and submit:
+    
+  
+    with st.spinner(f'Processing: {url}'):
+      content = scrap([url])
+
+      if content is not None:
+
+        tab1, tab2, = st.tabs(["Summary", "Rewritten content"])
+            
+        with tab1:
+          
+          with st.spinner('summarizing content..'):
+            st.subheader('Summary')
+            summary, cost = summarize(content, language)
+            st.write(summary)
+            st.write(cost)
+
+          with st.spinner('extracing keywords..'):
+            st.subheader('SEO Keywords ')
+            keywords, cost = extract_seo_keywords(content)
+
+            #st.write(keywords)
+            st.write(cost)
+
+            data = get_keywords_data(keywords)
+
+            # Use json_normalize() to convert JSON to DataFrame
+            dict= json.loads(data)
+
+            # Initialize empty lists to store data
+            vol_list = []
+            cpc_value_list = []
+            keyword_list = []
+            competition_list = []
+            trend_value_list = []
+
+            # Iterate through the data and extract relevant information
+            for item in dict["data"]:
+                vol_list.append(item["vol"])
+                cpc_value_list.append(float(item["cpc"]["value"]))
+                keyword_list.append(item["keyword"])
+                competition_list.append(item["competition"])
+                trend_values = [trend["value"] for trend in item["trend"]]
+                trend_value_list.append(trend_values)
+
+            # Create a DataFrame
+            df = pd.DataFrame({
+              'Keyword': keyword_list,
+              'Vol': vol_list,
+              'CPC': cpc_value_list,
+              'Competition': competition_list,
+              'Trend (last 12 months)': trend_value_list
+            })
+
+            # Display the DataFrame
+            st.dataframe(
+              df,
+              column_config={
+                "CPC": st.column_config.NumberColumn(
+                  format="$%f",
+                ),
+                "Trend (last 12 months)": st.column_config.LineChartColumn(y_min=0)
+              },
+              hide_index=True,
+              width=800
+            )
+
+          with st.expander("see extracted content"):
+            st.write(content)
+
+        with tab2:
+          st.subheader("Rewrite content")
+
+          rewrite_button = st.button("Rewrite!")
+
+          if rewrite_button:
+            with st.spinner('rewriting content..'):
+              st.subheader('rewritten content')
+              output, cost = rewrite(summary, language)
+
+              st.write(output)
+              st.write(cost)
+
+
+if __name__ == "__main__":
+    main()
+
+
+
+

requirements.txt

@@ -0,0 +1,5 @@
+openai
+langchain
+python-dotenv
+tiktoken
+streamlit