import streamlit as st
from dotenv import load_dotenv
import os
from langchain.llms import OpenAI
from langchain.chat_models import ChatOpenAI
from langchain.schema import AIMessage, HumanMessage, SystemMessage
from langchain.prompts.chat import (
   ChatMessagePromptTemplate, 
   SystemMessagePromptTemplate, 
   AIMessagePromptTemplate, 
   HumanMessagePromptTemplate
)
from langchain.callbacks import get_openai_callback

# scrapper
from langchain.document_loaders import AsyncChromiumLoader
from langchain.document_transformers import BeautifulSoupTransformer
import requests

#data
import pandas as pd
import json
from pandas import json_normalize

def rewrite(text, language='Bahasa Indonesia'):
  llm = ChatOpenAI(model='gpt-3.5-turbo-16k', temperature=0)
  message = [
    SystemMessage(content='Your are expert blogger'),
    HumanMessage(content=f'write the following text into 1000 words blog article, use language: {language}, use casual tone. and give clickbait title. TEXT: {text}')
  ]

  with get_openai_callback() as cost:
    output = llm(message).content

  return output, cost

def extract_seo_keywords(text):
  llm = ChatOpenAI(model='gpt-3.5-turbo-16k', temperature=0)
  message = [
    SystemMessage(content='Your are an SEO expert'),
    HumanMessage(content=f'indentify 10 SEO keywords in this text. order by the most relevance keyword. output format: comma separated value. example output: keyword,another keyword,some keyword. TEXT: {text}')
  ]

  with get_openai_callback() as cost:
    keywords = llm(message).content.split(',')
    array_keywords = [i.lstrip() for i in keywords]

  return array_keywords, cost

def summarize(text, language='Bahasa Indonesia'):
  llm = ChatOpenAI(model='gpt-3.5-turbo-16k', temperature=0)
  message = [
    SystemMessage(content='Your are expert copywriter'),
    HumanMessage(content=f'Extract main idea the following article and summarize into 10 bullet point or more. Use language: {language}. TEXT: {text}')
  ]

  with get_openai_callback() as cost:
    summary = llm(message).content
    return summary, cost

def scrap(url):
  scraper = AsyncChromiumLoader(url)
  html = scraper.load()

  extractor = BeautifulSoupTransformer()
  content = extractor.transform_documents(html, tags_to_extract=['h1', 'h2', 'h3', 'h4','h5', 'h6','p'])
  print(content[0].page_content)
  return content[0].page_content

def get_keywords_data(keywords):
  my_data = {
      'currency': 'USD',
      'dataSource': 'gkp',
      'kw[]': keywords
  }

  kw_api_key = os.environ.get('KEYWORD_API')
  auth = f'Bearer {kw_api_key}'

  my_headers = {
      'Accept': 'application/json',
      'Authorization': auth
  }
  response = requests.post('https://api.keywordseverywhere.com/v1/get_keyword_data', data=my_data, headers=my_headers)
  if response.status_code == 200:
      data = body=response.content.decode('utf-8')
      return data
  else:
      st.write(body=response.content.decode('utf-8'))

def main():

  load_dotenv()

  st.subheader("Summarize, Extract Keywords, Rewrite")

  url = st.text_input("Enter URL, start with http:// or https://")
  options = ['Bahasa Indonesia', 'English', 'Japanese', 'France', 'Germany', 'Dutch']
  language = st.selectbox("Output language", options=options)
  submit = st.button("submit")
  content = None

  if url and submit:
    
  
    with st.spinner(f'Processing: {url}'):
      content = scrap([url])

      if content is not None:

        tab1, tab2, = st.tabs(["Summary", "Rewritten content"])
            
        with tab1:
          
          with st.spinner('summarizing content..'):
            st.subheader('Summary')
            summary, cost = summarize(content, language)
            st.write(summary)
            st.write(cost)

          with st.spinner('extracing keywords..'):
            st.subheader('SEO Keywords ')
            keywords, cost = extract_seo_keywords(content)

            #st.write(keywords)
            st.write(cost)

            data = get_keywords_data(keywords)

            # Use json_normalize() to convert JSON to DataFrame
            dict= json.loads(data)

            # Initialize empty lists to store data
            vol_list = []
            cpc_value_list = []
            keyword_list = []
            competition_list = []
            trend_value_list = []

            # Iterate through the data and extract relevant information
            for item in dict["data"]:
                vol_list.append(item["vol"])
                cpc_value_list.append(float(item["cpc"]["value"]))
                keyword_list.append(item["keyword"])
                competition_list.append(item["competition"])
                trend_values = [trend["value"] for trend in item["trend"]]
                trend_value_list.append(trend_values)

            # Create a DataFrame
            df = pd.DataFrame({
              'Keyword': keyword_list,
              'Vol': vol_list,
              'CPC': cpc_value_list,
              'Competition': competition_list,
              'Trend (last 12 months)': trend_value_list
            })

            # Display the DataFrame
            st.dataframe(
              df,
              column_config={
                "CPC": st.column_config.NumberColumn(
                  format="$%f",
                ),
                "Trend (last 12 months)": st.column_config.LineChartColumn(y_min=0)
              },
              hide_index=True,
              width=800
            )

          with st.expander("see extracted content"):
            st.write(content)

        with tab2:
          st.subheader("Rewrite content")

          rewrite_button = st.button("Rewrite!")

          if rewrite_button:
            with st.spinner('rewriting content..'):
              st.subheader('rewritten content')
              output, cost = rewrite(summary, language)

              st.write(output)
              st.write(cost)


if __name__ == "__main__":
    main()