Wikidata_Text_Parser.py

import numpy as np
from tqdm import tqdm
import pandas as pd
import os, sqlite3, traceback, ast, requests, fasttext, re, time, string, spacy, pysbd
from requests.exceptions import ReadTimeout, TooManyRedirects, ConnectionError, ConnectTimeout, InvalidSchema, InvalidURL
from qwikidata.linked_data_interface import get_entity_dict_from_api
from datetime import datetime
import utils.wikidata_utils as wdutils
from importlib import reload  
from urllib.parse import urlparse, unquote
from urllib import parse
from bs4 import BeautifulSoup
from IPython.display import clear_output
from os.path import exists
from pathlib import Path
from nltk.tokenize import sent_tokenize
from sentence_splitter import SentenceSplitter, split_text_into_sentences
import nltk
nltk.download('punkt')

class DatabaseExtractor():
    def __init__(self, dbname='wikidata_claims_refs_parsed.db'):
        self.dbname = dbname
        self.prepare_extraction()
        
    def finish_extraction(self):
        self.db.commit()
        
    def prepare_extraction(self):
        self.db = sqlite3.connect(self.dbname)
        self.cursor = self.db.cursor()

        self.cursor.execute('''
            CREATE TABLE IF NOT EXISTS claims(
                entity_id TEXT,
                claim_id TEXT,
                rank TEXT,
                property_id TEXT,
                datatype TEXT,
                datavalue TEXT,
                PRIMARY KEY (
                    claim_id
                )
        )''')

        self.cursor.execute('''
            CREATE TABLE IF NOT EXISTS claims_refs(
                claim_id TEXT,
                reference_id TEXT,
                PRIMARY KEY (
                    claim_id,
                    reference_id
                )
        )''')

        self.cursor.execute('''
            CREATE TABLE IF NOT EXISTS refs(
                reference_id TEXT,
                reference_property_id TEXT,
                reference_index TEXT,
                reference_datatype TEXT,
                reference_value TEXT,
                PRIMARY KEY (
                    reference_id,
                    reference_property_id,
                    reference_index
                )
        )''')
        self.db.commit()  
        
    def extract_claim(self, entity_id, claim):
        if claim['mainsnak']['snaktype'] == 'value':
            value = str(claim['mainsnak']['datavalue'])
        else:
            value = claim['mainsnak']['snaktype']
        try:
            self.cursor.execute('''
            INSERT INTO claims(entity_id, claim_id, rank, property_id, datatype, datavalue)
            VALUES($var,$var,$var,$var,$var,$var)'''.replace('$var','?'), (
                entity_id,claim['id'],claim['rank'],
                claim['mainsnak']['property'],claim['mainsnak']['datatype'],value
            ))
        except UnicodeEncodeError:
            print(entity_id,claim['id'],claim['rank'],
                claim['mainsnak']['property'],claim['mainsnak']['datatype'],value)
            raise
        except sqlite3.IntegrityError as err:
            #self.db.rollback()
            self.cursor.execute(
                '''SELECT *
                FROM claims 
                WHERE claim_id=$var
                '''.replace('$var','?'), (claim['id'],)
            )
            conflicted_value = self.cursor.fetchone()
            if conflicted_value == (entity_id,claim['id'],claim['rank'],
                    claim['mainsnak']['property'],claim['mainsnak']['datatype'],value):
                pass
            else:
                print(err, claim['id'])
                traceback.print_exc()
                raise err
        finally:
            #self.db.commit()
            pass

    def extract_reference(self, ref):
        for snaks in ref['snaks'].values():
            for i, snak in enumerate(snaks):
                if snak['snaktype'] == 'value':
                    value = str(snak['datavalue'])
                else:
                    value = snak['snaktype']
                try:
                    self.cursor.execute('''
                    INSERT INTO refs(reference_id, reference_property_id, reference_index,
                    reference_datatype, reference_value)
                    VALUES($var,$var,$var,$var,$var)'''.replace('$var','?'), (
                        ref['hash'],snak['property'],str(i),snak['datatype'],value
                    ))
                except sqlite3.IntegrityError as err:
                    #self.db.rollback()
                    self.cursor.execute(# WE DONT USE THE INDEX HERE, THEY TEND TO COME SHUFFLED FROM API AND SORTING TAKES TOO LONG
                        '''SELECT reference_id, reference_property_id, reference_datatype, reference_value
                        FROM refs 
                        WHERE reference_id = $var
                        AND reference_property_id = $var
                        '''.replace('$var','?'), (ref['hash'],snak['property'])
                    )
                    conflicted_values = self.cursor.fetchall()
                    if  (ref['hash'],snak['property'],snak['datatype'],value) in conflicted_values:
                        pass
                    else:
                        print(err, ref['hash'],snak['property'],i)
                        print('trying to insert:',(ref['hash'],snak['property'],str(i),snak['datatype'],value))
                        traceback.print_exc()
                        raise err
                finally:
                    #self.db.commit()
                    pass
            
    def extract_claim_reference(self, claim, ref):
        claim['id'],ref['hash']
        try:
            self.cursor.execute('''
            INSERT INTO claims_refs(claim_id, reference_id)
            VALUES($var,$var)'''.replace('$var','?'), (
                claim['id'],ref['hash']
            ))
        except sqlite3.IntegrityError as err:
            #db.rollback()
            pass
        finally:
            #self.db.commit()
            pass
    
    def extract_entity(self, e):
        for outgoing_property_id in e['claims'].values():
            for claim in outgoing_property_id:
                self.extract_claim(e['id'],claim)
                if 'references' in claim:
                    for ref in claim['references']: 
                        self.extract_claim_reference(claim, ref)
                        self.extract_reference(ref)

def claimParser(QID):
    entity_id = QID
    print('Setting up database ...')
    extractor = DatabaseExtractor()

    print('Fetching entity from API ...')
    entity = get_entity_dict_from_api(entity_id)

    if entity:
        print(f'Parsing entity: {entity_id}')
        extractor.extract_entity(entity)
    else:
        print(f'Failed to fetch entity: {entity_id}')

    extractor.finish_extraction()

def propertyFiltering(QID):
    reload(wdutils)
    DB_PATH = 'wikidata_claims_refs_parsed.db'
    claims_columns = ['entity_id','claim_id','rank','property_id','datatype','datavalue']

    properties_to_remove = {
        'general':[
            'P31', # - instance of
            'P279',# - subclass of
            'P373',# - commons category
            'P910',# - Topic's main category
            'P7561',# - category for the interior of the item
            'P5008',# - on focus list of Wikimedia project
            'P2670',# -  has parts of the class
            'P1740',# -  category for films shot at this location
            'P1612',# -  Commons Institution page
            'P8989',# -  category for the view of the item
            'P2959',# -  permanent duplicated item
            'P7867',# -  category for maps
            'P935' ,# -  Commons gallery
            'P1472',#  -  Commons Creator page
            'P8596',# category for the exterior of the item
            'P5105',# Deutsche Bahn station category
            'P8933',# category for the view from the item
            'P642',# of
            'P3876',# category for alumni of educational institution
            'P1791',# category of people buried here
            'P7084',# related category
            'P1465',# category for people who died here
            'P1687',# Wikidata property
            'P6104',# maintained by WikiProject
            'P4195',# category for employees of the organization
            'P1792',# category of associated people
            'P5869',# model item
            'P1659',# see also
            'P1464',# category for people born here
            'P2354',# has list
            'P1424',# topic's main template
            'P7782',# category for ship name
            'P179',# part of the series
            'P7888',# merged into
            'P6365',# member category
            'P8464',# content partnership category
            'P360',# is a list of
            'P805',# statement is subject of
            'P8703',# entry in abbreviations table
            'P1456',# list of monuments
            'P1012',# including
            'P1151',# topic's main Wikimedia portal
            'P2490',# page at OSTIS Belarus Wiki
            'P593',# HomoloGene ID
            'P8744',# economy of topic
            'P2614',# World Heritage criteria
            'P2184',# history of topic
            'P9241',# demographics of topic
            'P487',#Unicode character
            'P1754',#category related to list
            'P2559',#Wikidata usage instructions
            'P2517',#category for recipients of this award
            'P971',#category combines topics
            'P6112',# category for members of a team
            'P4224',#category contains
            'P301',#category's main topic
            'P1753',#list related to category
            'P1423',#template has topic
            'P1204',#Wikimedia portal's main topic
            'P3921',#Wikidata SPARQL query equivalent
            'P1963',#properties for this type
            'P5125',#Wikimedia outline
            'P3176',#uses property
            'P8952',#inappropriate property for this type
            'P2306',#property
            'P5193',#Wikidata property example for forms
            'P5977',#Wikidata property example for senses
        ],
        'specific': {}
    }

    db = sqlite3.connect(DB_PATH)
    cursor = db.cursor()
    # To see how many out of the total number of stored claims we are excluding by removing the general properties
    sql_query = "select count(*) from claims where property_id in $1;"
    sql_query = sql_query.replace('$1', '(' + ','.join([('"' + e + '"') for e in properties_to_remove['general']]) + ')')
    cursor.execute(sql_query)
    print('Removing the',len(properties_to_remove['general']),'properties deemed as ontological or unverbalisable')
    cursor = db.cursor()

    sql_query = "select * from claims where entity_id in $1;"
    sql_query = sql_query.replace('$1', '(' + ','.join([('"' + e + '"') for e in [QID]]) + ')')

    cursor.execute(sql_query)
    theme_df = pd.DataFrame(cursor.fetchall())
    theme_df.columns = claims_columns

    original_theme_df_size = theme_df.shape[0]
    last_stage_theme_df_size = original_theme_df_size

    print('-    Removing deprecated')

    # Remove deprecated
    theme_df = theme_df[theme_df['rank'] != 'deprecated'].reset_index(drop=True)
    print(
        '    -    Percentage of deprecated:',
        round((last_stage_theme_df_size-theme_df.shape[0])/original_theme_df_size*100, 2), '%'
    )
    last_stage_theme_df_size = theme_df.shape[0]

    print('-    Removing bad datatypes')

    # Remove external_ids, commonsMedia (e.g. photos), globe-coordinates, urls
    bad_datatypes = ['commonsMedia','external-id','globe-coordinate','url', 'wikibase-form',
                        'geo-shape', 'math', 'musical-notation', 'tabular-data', 'wikibase-sense']
    theme_df = theme_df[
        theme_df['datatype'].apply(
            lambda x : x not in bad_datatypes
        )
    ].reset_index(drop=True)
    print(
        '    -    Percentage of bad datatypes:',
        round((last_stage_theme_df_size-theme_df.shape[0])/original_theme_df_size*100, 2), '%'
    )
    last_stage_theme_df_size = theme_df.shape[0]

    print('-    Removing bad properties')

    # Remove specific properties such as P31 and P279
    theme_df = theme_df[
        theme_df['property_id'].apply(
            lambda x : (x not in properties_to_remove['general']))
        
    ].reset_index(drop=True)
    print(
        '    -    Percentage of ontology (non-domain) properties:',
        round((last_stage_theme_df_size-theme_df.shape[0])/original_theme_df_size*100, 2), '%'
    )
    last_stage_theme_df_size = theme_df.shape[0]

    print('-    Removing somevalue/novalue')

    # Remove novalue and somevalue
    theme_df = theme_df[
        theme_df['datavalue'].apply(
            lambda x : x not in ['somevalue', 'novalue']
        )
    ].reset_index(drop=True)
    print(
        '    -    Percentage of somevalue/novalue:',
        round((last_stage_theme_df_size-theme_df.shape[0])/original_theme_df_size*100, 2), '%'
    )
    last_stage_theme_df_size = theme_df.shape[0]

    print(
        'After all removals, we keep',
        round(last_stage_theme_df_size/original_theme_df_size*100, 2),
    )
    theme_df.to_sql('claims', db, if_exists='replace', index=False)

    return theme_df

def get_object_label_given_datatype(row):
    Wd_API = wdutils.CachedWikidataAPI()
    Wd_API.languages = ['en']
    def turn_to_century_or_millennium(y, mode):
        y = str(y)
        if mode == 'C':
            div = 100
            group = int(y.rjust(3, '0')[:-2])
            mode_name = 'century'
        elif mode == 'M':
            div = 1000
            group = int(y.rjust(4, '0')[:-3])
            mode_name = 'millenium'
        else:        
            raise ValueError('Use mode = C for century and M for millennium')
            
        if int(y)%div != 0:
            group += 1
        group = str(group)

        group_suffix = (
            'st' if group[-1] == '1' else (
                'nd' if group[-1] == '2' else (
                    'rd' if group[-1] == '3' else 'th'
                )
            )
        )

        return ' '.join([group+group_suffix, mode_name])

    dt = row['datatype']
    dv = row['datavalue']
    
    dt_types = ['wikibase-item', 'monolingualtext', 'quantity', 'time', 'string']
    if dt not in dt_types:
        print(dt)
        raise ValueError
    else:
        try:
            if dt == dt_types[0]:
                return Wd_API.get_label(ast.literal_eval(dv)['value']['id'], True) #get label here
            elif dt == dt_types[1]:
                dv = ast.literal_eval(dv)
                return (dv['value']['text'], dv['value']['language'])
            elif dt == dt_types[2]:
                dv = ast.literal_eval(dv)
                amount, unit = dv['value']['amount'], dv['value']['unit']
                if amount[0] == '+':
                    amount = amount[1:]
                if str(unit) == '1':
                    return (str(amount), 'en')
                else:
                    unit_entity_id = unit.split('/')[-1]
                    unit = Wd_API.get_label(unit_entity_id, True)#get label here
                    return (' '.join([amount, unit[0]]), unit[1])
            elif dt == dt_types[3]:
                dv = ast.literal_eval(dv)
                time = dv['value']['time']
                timezone = dv['value']['timezone']
                precision = dv['value']['precision']
                assert dv['value']['after'] == 0 and dv['value']['before'] == 0

                sufix = 'BC' if time[0] == '-' else ''
                time = time[1:]

                if precision == 11: #date
                    return (datetime.strptime(time, '%Y-%m-%dT00:00:%SZ').strftime('%d/%m/%Y') + sufix, 'en')
                elif precision == 10: #month
                    try:
                        return (datetime.strptime(time, '%Y-%m-00T00:00:%SZ').strftime("%B of %Y") + sufix, 'en')
                    except ValueError:
                        return (datetime.strptime(time, '%Y-%m-%dT00:00:%SZ').strftime("%B of %Y") + sufix, 'en')
                elif precision == 9: #year
                    try:
                        return (datetime.strptime(time, '%Y-00-00T00:00:%SZ').strftime('%Y') + sufix, 'en')
                    except ValueError:
                        return (datetime.strptime(time, '%Y-%m-%dT00:00:%SZ').strftime('%Y') + sufix, 'en')
                elif precision == 8: #decade
                    try:
                        return (datetime.strptime(time, '%Y-00-00T00:00:%SZ').strftime('%Y')[:-1] +'0s' + sufix, 'en')
                    except ValueError:
                        return (datetime.strptime(time, '%Y-%m-%dT00:00:%SZ').strftime('%Y')[:-1] +'0s' + sufix, 'en')
                elif precision == 7: #century
                    try:
                        parsed_time = datetime.strptime(time, '%Y-00-00T00:00:%SZ')
                    except ValueError:
                        parsed_time = datetime.strptime(time, '%Y-%m-%dT00:00:%SZ')
                    finally:                        
                        return (turn_to_century_or_millennium(
                            parsed_time.strftime('%Y'), mode='C'
                        ) + sufix, 'en')
                elif precision == 6: #millennium
                    try:
                        parsed_time = datetime.strptime(time, '%Y-00-00T00:00:%SZ')
                    except ValueError:
                        parsed_time = datetime.strptime(time, '%Y-%m-%dT00:00:%SZ')
                    finally:                        
                        return (turn_to_century_or_millennium(
                            parsed_time.strftime('%Y'), mode='M'
                        ) + sufix, 'en')
                elif precision == 4: #hundred thousand years 
                    timeint = int(datetime.strptime(time, '%Y-00-00T00:00:%SZ').strftime('%Y'))
                    timeint = round(timeint/1e5,1)
                    return (str(timeint) + 'hundred thousand years' + sufix, 'en')
                elif precision == 3: #million years 
                    timeint = int(datetime.strptime(time, '%Y-00-00T00:00:%SZ').strftime('%Y'))
                    timeint = round(timeint/1e6,1)
                    return (str(timeint) + 'million years' + sufix, 'en')
                elif precision == 0: #billion years 
                    timeint = int(datetime.strptime(time, '%Y-00-00T00:00:%SZ').strftime('%Y'))
                    timeint = round(timeint/1e9,1)
                    return (str(timeint) + 'billion years' +sufix, 'en')
            elif dt == dt_types[4]:
                return (ast.literal_eval(dv)['value'], 'en')
        except ValueError as e:
            #pdb.set_trace()
            raise e
            
def get_object_desc_given_datatype(row):
    Wd_API = wdutils.CachedWikidataAPI()
    Wd_API.languages = ['en']
    dt = row['datatype']
    dv = row['datavalue']
    
    dt_types = ['wikibase-item', 'monolingualtext', 'quantity', 'time', 'string']
    if dt not in dt_types:
        print(dt)
        raise ValueError
    else:
        try:
            if dt == dt_types[0]:
                return Wd_API.get_desc(ast.literal_eval(dv)['value']['id']) #get label here
            elif dt == dt_types[1]:
                return ('no-desc', 'none')
            elif dt == dt_types[2]:
                dv = ast.literal_eval(dv)
                amount, unit = dv['value']['amount'], dv['value']['unit']
                if amount[0] == '+':
                    amount = amount[1:]
                if str(unit) == '1':
                    return ('no-desc', 'none')
                else:
                    unit_entity_id = unit.split('/')[-1]
                    return Wd_API.get_desc(unit_entity_id)
            elif dt == dt_types[3]:
                return ('no-desc', 'none')
            elif dt == dt_types[4]:
                return ('no-desc', 'none')
        except ValueError as e:
            #pdb.set_trace()
            raise e
            
def get_object_alias_given_datatype(row):
    Wd_API = wdutils.CachedWikidataAPI()
    Wd_API.languages = ['en']
    dt = row['datatype']
    dv = row['datavalue']
    
    dt_types = ['wikibase-item', 'monolingualtext', 'quantity', 'time', 'string']
    if dt not in dt_types:
        print(dt)
        raise ValueError
    else:
        try:
            if dt == dt_types[0]:
                return Wd_API.get_alias(ast.literal_eval(dv)['value']['id']) #get label here
            elif dt == dt_types[1]:
                return ('no-alias', 'none')
            elif dt == dt_types[2]:
                dv = ast.literal_eval(dv)
                amount, unit = dv['value']['amount'], dv['value']['unit']
                if amount[0] == '+':
                    amount = amount[1:]
                if str(unit) == '1':
                    return ('no-alias', 'none')
                else:
                    unit_entity_id = unit.split('/')[-1]
                    return Wd_API.get_alias(unit_entity_id)
            elif dt == dt_types[3]:
                dv = ast.literal_eval(dv)
                time = dv['value']['time']
                timezone = dv['value']['timezone']
                precision = dv['value']['precision']
                assert dv['value']['after'] == 0 and dv['value']['before'] == 0

                sufix = 'BC' if time[0] == '-' else ''
                time = time[1:]

                if precision == 11: #date
                    return ([
                        datetime.strptime(time, '%Y-%m-%dT00:00:%SZ').strftime('%-d of %B, %Y') + sufix,
                        datetime.strptime(time, '%Y-%m-%dT00:00:%SZ').strftime('%d/%m/%Y (dd/mm/yyyy)') + sufix,
                        datetime.strptime(time, '%Y-%m-%dT00:00:%SZ').strftime('%b %-d, %Y') + sufix
                    ], 'en')
                else: #month
                    return ('no-alias', 'none')
            elif dt == dt_types[4]:
                return ('no-alias', 'none')
        except ValueError as e:
            #pdb.set_trace()
            raise e

def textualAugmentation(filtered_df):

    Wd_API = wdutils.CachedWikidataAPI()
    Wd_API.languages = ['en']

    filtered_df['entity_label'] = filtered_df['entity_id'].apply(lambda x: Wd_API.get_label(x, True))
    filtered_df['entity_desc'] = filtered_df['entity_id'].apply(lambda x: Wd_API.get_desc(x))
    filtered_df['entity_alias'] = filtered_df['entity_id'].apply(lambda x: Wd_API.get_alias(x))

    print(' - Predicate augmentation...')
    filtered_df['property_label'] = filtered_df['property_id'].apply(lambda x: Wd_API.get_label(x, True))
    filtered_df['property_desc'] = filtered_df['property_id'].apply(lambda x: Wd_API.get_desc(x))
    filtered_df['property_alias'] = filtered_df['property_id'].apply(lambda x: Wd_API.get_alias(x))

    print(' - Object augmentation...')
    filtered_df['object_label'] = filtered_df.apply(get_object_label_given_datatype, axis=1)
    filtered_df['object_desc'] = filtered_df.apply(get_object_desc_given_datatype, axis=1)
    filtered_df['object_alias'] = filtered_df.apply(get_object_alias_given_datatype, axis=1)


    no_subject_label_perc = filtered_df[filtered_df['entity_label'].apply(lambda x: x[0] == 'no-label')].shape[0] / filtered_df.shape[0] * 100
    print(' - No subject label %:', no_subject_label_perc, '%')

    no_predicate_label_perc = filtered_df[filtered_df['property_label'].apply(lambda x: x[0] == 'no-label')].shape[0] / filtered_df.shape[0] * 100
    print(' - No predicate label %:', no_predicate_label_perc, '%')

    no_object_label_perc = filtered_df[filtered_df['object_label'].apply(lambda x: x[0] == 'no-label')].shape[0] / filtered_df.shape[0] * 100
    print(' - No object label %:', no_object_label_perc, '%')
    return filtered_df

def urlParser(target_QID):
    Wd_API = wdutils.CachedWikidataAPI()
    Wd_API.languages = ['en']
    db = sqlite3.connect('wikidata_claims_refs_parsed.db')
    cursor = db.cursor()
    refs_columns = ['reference_id','reference_property_id', 'reference_index', 'reference_datatype', 'reference_value']
    cursor.execute('select * from refs where reference_datatype="url";')
    url_df = pd.DataFrame(cursor.fetchall())
    url_df.columns = refs_columns
    def reference_value_to_url(reference_value):
        if reference_value in ['novalue','somevalue']:
            return reference_value
        reference_value = ast.literal_eval(reference_value)
        assert reference_value['type'] == 'string'
        return reference_value['value']
    def reference_value_to_external_id(reference_value):
        if reference_value in ['novalue','somevalue']:
            return reference_value
        reference_value = ast.literal_eval(reference_value)
        assert reference_value['type'] == 'string'
        return reference_value['value']
    def get_formatter_url(entity_id):
        try:
            sparql_query = '''
                SELECT ?item ?itemLabel 
                WHERE 
                {
                wd:$1 wdt:P1630 ?item.
                SERVICE wikibase:label { bd:serviceParam wikibase:language "[AUTO_LANGUAGE],en". }
                }
            '''.replace('$1',entity_id)
            sparql_results = Wd_API.query_sparql_endpoint(sparql_query)
            if len(sparql_results['results']['bindings']) > 0:
                return sparql_results['results']['bindings'][0]['item']['value']
            else:
                return 'no_formatter_url'
        except Exception:
            print(entity_id)
            print(sparql_results)
            raise
    url_df['url'] = url_df.reference_value.apply(reference_value_to_url)
    cursor.execute('select * from refs where reference_datatype="url";')
    ext_id_df = pd.DataFrame(cursor.fetchall())
    ext_id_df.columns = refs_columns
    ext_id_df['ext_id'] = ext_id_df.reference_value.apply(reference_value_to_external_id)
    ext_id_df['formatter_url'] = ext_id_df['reference_property_id'].apply(get_formatter_url)
    ext_id_df['url'] = ext_id_df.apply(lambda x : x['formatter_url'].replace('$1', x['ext_id']), axis=1)
    columns_for_join = ['reference_id', 'reference_property_id','reference_index','reference_datatype','url']
    url_df_pre_join = url_df[columns_for_join]
    ext_id_df_pre_join = ext_id_df[columns_for_join]
    all_url_df = pd.concat([url_df_pre_join,ext_id_df_pre_join])
    all_url_df = all_url_df.sort_values(['reference_id','reference_index'])
    # drop those with url = 'no_formatter_url'
    all_url_df = all_url_df[all_url_df['url'] != 'no_formatter_url'].reset_index(drop=True)
    # drop those with url = somevalue and novalue
    all_url_df = all_url_df[~all_url_df['url'].isin(['somevalue','novalue'])]
    reference_id_counts = all_url_df.reference_id.value_counts().reset_index()
    reference_id_counts.columns = ['reference_id', 'counts']
    reference_id_counts_equal_1 = reference_id_counts[reference_id_counts['counts'] == 1].reference_id.tolist()
    all_url_df_eq1 = all_url_df[all_url_df.reference_id.isin(reference_id_counts_equal_1)]
    all_url_df_eq1 = all_url_df_eq1.reset_index(drop=True).drop('reference_index', axis=1)
    return all_url_df_eq1

def htmlParser(url_set, qid):
    text_reference_sampled_df = url_set
    _RE_COMBINE_WHITESPACE = re.compile(r"\s+")
    text_reference_sampled_df['html'] = None
    for i, row in text_reference_sampled_df.iterrows():

        print(i, row.url)
        try:
            response = requests.get(row.url, timeout=10)
            if response.status_code == 200:
                html = response.text
                text_reference_sampled_df.loc[i, 'html'] = html
            else:
                print(f"not response, {response.status_code}")
                text_reference_sampled_df.loc[i, 'html'] = response.status_code
        except requests.exceptions.Timeout:
            print("Timeout occurred while fetching the URL:", row.url)
            text_reference_sampled_df.loc[i, 'html'] = 'TimeOut'
            pass  
        except Exception as e:
            print("An error occurred:", str(e))
            pass 
    text_reference_sampled_df_html = text_reference_sampled_df.copy()
    text_reference_sampled_df_html['entity_id'] = qid
    return text_reference_sampled_df_html

def claim2text(html_set):
    text_reference_sampled_df_html = html_set
    Wd_API = wdutils.CachedWikidataAPI()
    Wd_API.languages = ['en']
    db = sqlite3.connect('wikidata_claims_refs_parsed.db')
    cursor = db.cursor()
    claims_columns = ['entity_id','claim_id','rank','property_id','datatype','datavalue']
    refs_columns = ['reference_id', 'reference_property_id', 'reference_index', 'reference_datatype', 'reference_value']

    def reference_id_to_claim_id(reference_id):
        cursor.execute(f'select claim_id from claims_refs where reference_id="{reference_id}"')
        sql_result = cursor.fetchall()
        #return sql_result
        randomly_chosen_claim_id = np.array(sql_result).reshape(-1)
        return randomly_chosen_claim_id
            
    def reference_id_to_claim_data(reference_id):
        claim_ids = reference_id_to_claim_id(reference_id)
        r = []
        for claim_id in claim_ids:
            #print(claim_id)
            cursor.execute(f'select * from claims where claim_id="{claim_id}";')
            d = cursor.fetchall()
            r = r + d
        return r

    claim_data = []
    for reference_id in text_reference_sampled_df_html.reference_id:
        data = reference_id_to_claim_data(reference_id)    
        #print(data)
        data = [(reference_id,) + t for t in data]
        claim_data = claim_data + data
        #break

    claim_df = pd.DataFrame(claim_data, columns = ['reference_id'] + claims_columns)
    claim_df

    def claim_id_to_claim_url(claim_id):
        claim_id_parts = claim_id.split('$')
        return f'https://www.wikidata.org/wiki/{claim_id_parts[0]}#{claim_id}'

    BAD_DATATYPES = ['external-id','commonsMedia','url', 'globe-coordinate', 'wikibase-lexeme', 'wikibase-property']

    assert claim_df[~claim_df.datatype.isin(BAD_DATATYPES)].reference_id.unique().shape\
        == claim_df.reference_id.unique().shape

    print(claim_df.reference_id.unique().shape[0])
    claim_df = claim_df[~claim_df.datatype.isin(BAD_DATATYPES)].reset_index(drop=True)

    from tqdm.auto import tqdm
    tqdm.pandas()

    claim_df[['entity_label','entity_label_lan']] = pd.DataFrame(
        claim_df.entity_id.progress_apply(Wd_API.get_label, non_language_set=True).tolist()
    )
    claim_df[['property_label','property_label_lan']] = pd.DataFrame(
        claim_df.property_id.progress_apply(Wd_API.get_label, non_language_set=True).tolist()
    )

    claim_df[['entity_alias','entity_alias_lan']] = pd.DataFrame(
        claim_df.entity_id.progress_apply(Wd_API.get_alias, non_language_set=True).tolist()
    )
    claim_df[['property_alias','property_alias_lan']] = pd.DataFrame(
        claim_df.property_id.progress_apply(Wd_API.get_alias, non_language_set=True).tolist()
    )

    claim_df[['entity_desc','entity_desc_lan']] = pd.DataFrame(
        claim_df.entity_id.progress_apply(Wd_API.get_desc, non_language_set=True).tolist()
    )
    claim_df[['property_desc','property_desc_lan']] = pd.DataFrame(
        claim_df.property_id.progress_apply(Wd_API.get_desc, non_language_set=True).tolist()
    )

    claim_df['object_label'] = claim_df.apply(get_object_label_given_datatype, axis=1)
    claim_df['object_alias'] = claim_df.apply(get_object_alias_given_datatype, axis=1)
    claim_df['object_desc'] = claim_df.apply(get_object_desc_given_datatype, axis=1)

    claim_df['object_label'], claim_df['object_label_lan'] = zip(*claim_df['object_label'].apply(lambda x: x if isinstance(x, tuple) else (x, '')))
    claim_df['object_alias'], claim_df['object_alias_lan'] = zip(*claim_df['object_alias'].apply(lambda x: x if isinstance(x, tuple) else (x, '')))
    claim_df['object_desc'], claim_df['object_desc_lan'] = zip(*claim_df['object_desc'].apply(lambda x: x if isinstance(x, tuple) else (x, '')))

    # Removing bad object labels
    claim_df = claim_df[claim_df['object_label_lan'] != 'none'].reset_index(drop=True)
    return claim_df

def html2text(html_set):
    reference_html_df = html_set
    _RE_COMBINE_WHITESPACE = re.compile(r"\s+")
    ft_model = fasttext.load_model('base/lid.176.ftz')
    def predict_language(text, k=20):
        ls, scores = ft_model.predict(text, k=k)  # top 20 matching languages
        ls = [l.replace('__label__','') for l in ls]
        return list(zip(ls,scores))
    def get_url_language(html):
        try:
            soup = BeautifulSoup(html, "lxml")
            [s.decompose() for s in soup("script")]  # remove <script> elements
            if soup.body == None:
                return ('no body', None)
            body_text = _RE_COMBINE_WHITESPACE.sub(" ", soup.body.get_text(' ')).strip()
            return predict_language(body_text, k=1)[0]
        except Exception:
            raise     
    def get_text_p_tags(soup):
        p_tags = soup.find_all('p')
        text = [p.getText().strip() for p in p_tags if p.getText()]
        return '\n'.join(text)
    def clean_text_line_by_line(text, join=True, ch_join = ' ', verb=True):
        # text = soup.body.get_text()
        # break into lines and remove leading and trailing space on each
        lines = list(text.splitlines())
        lines = (line.strip() for line in lines)
        # for each line, lets correct double spaces into single space
        lines = (re.sub(r' {2,}', ' ', line) for line in lines)
        # for each line, lets correct punctuation spaced to the left    
        lines = (re.sub(r' ([.,:;!?\\-])', r'\1', line) for line in lines)       
        # put periods if missing    
        lines = [line+'.' if line and line[-1] not in string.punctuation else line for i, line in enumerate(lines)]  
        
        if verb:
            for i, line in enumerate(lines):
                print(i,line)
        # drop blank lines
        if join:
            return ch_join.join([line for line in lines if line])
        else:
            return [line for line in lines if line]

    def apply_manual_rules(text):
        # RULE: A line ending with a ':' followed by whitespaces and a newline is likely a continuing line and should be joined
        #text = re.sub(
        #   r':\s*\n', 
        #   r': ', 
        #   text
        #)
        # RULE: Remove [1] reference numbers
        text = re.sub(r'\[[0-9]+\]', '', text)
        return text
    def retrieve_text_from_html(html, soup_parser = 'lxml', verb=True, join=True):
        if not isinstance(html, str) or 'DOCTYPE html' not in html:
            return 'No body'
        soup = BeautifulSoup(html, soup_parser)
        for script in soup(["script", "style"]):
            script.decompose()
        if soup.body == None:
            return 'No body'
        [s.unwrap() for s in soup.body.find_all('strong')]

        for p in soup.body.find_all('p'):
            p.string = _RE_COMBINE_WHITESPACE.sub(" ", p.get_text('')).strip()
        
        #DECOMPOSE ALL BAD TAGS
        #--------------
        #for c in ['warningbox', 'metadata', 'references', 'navbox', 'toc', 'catlinks']:
        #    for e in soup.body.find_all(class_=c):
        #        print('decomposed',e)
        #        e.decompose()
        
        # DECOMPOSE INVISIBLE ELEMENTS
        #for e in soup.body.find_all():
        #    if e.hidden:
        #        print('decomposed',e)
        #        e.decompose()
        #    else:
        #        if e.attrs is not None:
        #            #print(e)
        #            #print('-')
        #            style = e.get('style')
        #            if style and 'display' in style and 'none' in style:
        #                print('decomposed',e)
        #                e.decompose()
        #                #print(e, style)
        #--------------
        
        #print(soup.body)
        
        # BOILERPLATE REMOVAL OPTIONS
        #1. jusText
        #text = justext.justext(html, justext.get_stoplist("English"))
        #text = '\n'.join([paragraph.text for paragraph in text if not paragraph.is_boilerplate])

        #2. boilerpy3
        #html = soup.body
        #text = extractor.get_content(soup.prettify())

        #3. Just extracting from 'text tags' like p
        #simple rules (does not work depending on website, like on artgallery.yale, anything without clear paragraphic style)
        #text = get_text_p_tags(soup)

        #4. NONE
        text = soup.body.get_text(' ').strip() # NOT GETTING FROM THE WHOLE SOUP, JUST BODY TO AVOID TITLES
        
        #POST PROCESSING
        text = apply_manual_rules(text)
        text = clean_text_line_by_line(text, ch_join = ' ', verb=verb, join=join)

        if not text:
            return 'No extractable text' if join else ['No extractable text']
        else:
            return text
    i=0
    print(i)
    print(reference_html_df.url.iloc[i])

    reference_html_df['extracted_sentences'] = reference_html_df.html.progress_apply(retrieve_text_from_html, join=False, verb=False)

    join_ch = ' '
    reference_html_df['extracted_text'] = reference_html_df.extracted_sentences.apply(lambda x : join_ch.join(x))

    splitter = SentenceSplitter(language='en')

    seg = pysbd.Segmenter(language="en", clean=False)

    if not spacy.util.is_package("en_core_web_lg"):
        os.system("python -m spacy download en_core_web_lg")

    nlp = spacy.load("en_core_web_lg")

    text = reference_html_df.loc[0,'extracted_text']

    # OPTION 1
    # This gets some things wrong, such as Smt.=Shrimati ending a sentence, or any
    # initials like P. N. Nampoothiri or Lt. Col.
    #sents = sent_tokenize(text)

    # OPTION 2
    # Also breaks titles and initials like above, but additionally gets parenthesis wrong, like
    # Amma Maharani [break](queen mother) [break] of Travancore.
    #sents = seg.segment(text)

    # OPTION 3
    # Same as above plus new ones, like breaking contractions (like m. for married)
    #sents = splitter.split(text)

    # OPTION 4
    # By far the best option, makes way less of the mistakes above, but not none. So let's adopt a strategy so ease this.
    sents =  [s for s in nlp(text).sents]


    reference_html_df['nlp_sentences'] = reference_html_df.extracted_text.progress_apply(lambda x : [str(s) for s in nlp(x).sents])
    reference_html_df['nlp_sentences_slide_2'] = reference_html_df['nlp_sentences'].progress_apply(
        lambda x : [' '.join([a,b]) for a,b in list(zip(x,x[1:]+['']))]
    )

    assert type(reference_html_df.loc[0,'nlp_sentences']) == list
    assert type(reference_html_df.loc[0,'nlp_sentences'][0]) == str
    assert type(reference_html_df.loc[0,'nlp_sentences_slide_2']) == list
    assert type(reference_html_df.loc[0,'nlp_sentences_slide_2'][0]) == str
    return reference_html_df

if __name__ == '__main__':
    conn = sqlite3.connect('wikidata_claims_refs_parsed.db')
    target_QID = 'Q3621696'
    claimParser(target_QID) #save results in .db
    filtered_df = propertyFiltering(target_QID) #update db and return dataframe after filtering
    url_set = urlParser(target_QID) #from ref table in .db
    html_set = htmlParser(url_set, target_QID) #Original html docs collection
    try:
        claim_text = claim2text(html_set) #Claims generation
        html_text = html2text(html_set)
        claim_text = claim_text.astype(str)
        html_text = html_text.astype(str)
        claim_text.to_sql('claim_text', conn, if_exists='replace', index=False)
        html_text.to_sql('html_text', conn, if_exists='replace', index=False)
    except Exception as e:
        print(f"No accessible html documents")
        

    conn.commit()
    conn.close()
    #augmented_df = textualAugmentation(filtered_df) #textual information augmentation including label, desc, and alias