robert.elder commited on
Commit ·
3a67458
1
Parent(s): 4e5a7f9
bug fixes for chemicals package, density lookup
Browse files- ChemID.py +31 -7
- custom_chemicals_db.tsv +3 -0
ChemID.py
CHANGED
|
@@ -4,10 +4,15 @@ import pandas as pd
|
|
| 4 |
import numpy as np
|
| 5 |
import pubchempy as pcp
|
| 6 |
import cirpy
|
|
|
|
| 7 |
import chemicals
|
|
|
|
|
|
|
|
|
|
| 8 |
|
| 9 |
import bs4
|
| 10 |
import urllib
|
|
|
|
| 11 |
import json
|
| 12 |
|
| 13 |
from io import BytesIO
|
|
@@ -476,17 +481,19 @@ def is_cas(cas):
|
|
| 476 |
except:
|
| 477 |
return False
|
| 478 |
|
| 479 |
-
def string2density(
|
| 480 |
rho, rho_origin = None, None
|
| 481 |
# try to scrape from PubChem
|
| 482 |
if not rho:
|
| 483 |
try:
|
| 484 |
content = None
|
| 485 |
-
compounds = pcp.get_compounds(
|
| 486 |
c = compounds[0]
|
| 487 |
cid = c.cid
|
| 488 |
-
|
| 489 |
-
|
|
|
|
|
|
|
| 490 |
content = json.loads(webpage)
|
| 491 |
except:
|
| 492 |
pass
|
|
@@ -501,12 +508,29 @@ def string2density(string):
|
|
| 501 |
for ii in k['Information']:
|
| 502 |
try:
|
| 503 |
rho_string = ii['Value']['StringWithMarkup'][0]['String']
|
| 504 |
-
|
| 505 |
-
|
| 506 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 507 |
except:
|
| 508 |
continue
|
| 509 |
if rho_list:
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 510 |
rho = np.mean(rho_list)
|
| 511 |
rho_origin = 'pubchem'
|
| 512 |
else:
|
|
|
|
| 4 |
import numpy as np
|
| 5 |
import pubchempy as pcp
|
| 6 |
import cirpy
|
| 7 |
+
|
| 8 |
import chemicals
|
| 9 |
+
## add custom chemical definitions (i.e., to correct confusion between methane and carbon)
|
| 10 |
+
db = chemicals.identifiers.get_pubchem_db()
|
| 11 |
+
db.load('custom_chemicals_db.tsv')
|
| 12 |
|
| 13 |
import bs4
|
| 14 |
import urllib
|
| 15 |
+
import requests
|
| 16 |
import json
|
| 17 |
|
| 18 |
from io import BytesIO
|
|
|
|
| 481 |
except:
|
| 482 |
return False
|
| 483 |
|
| 484 |
+
def string2density(name):
|
| 485 |
rho, rho_origin = None, None
|
| 486 |
# try to scrape from PubChem
|
| 487 |
if not rho:
|
| 488 |
try:
|
| 489 |
content = None
|
| 490 |
+
compounds = pcp.get_compounds(name, namespace='name')
|
| 491 |
c = compounds[0]
|
| 492 |
cid = c.cid
|
| 493 |
+
url = f'https://pubchem.ncbi.nlm.nih.gov/rest/pug_view/data/compound/{cid}/JSON'
|
| 494 |
+
#fid = urllib.request.urlopen(url)
|
| 495 |
+
#webpage = fid.read().decode('utf-8')
|
| 496 |
+
webpage = requests.get(url).text
|
| 497 |
content = json.loads(webpage)
|
| 498 |
except:
|
| 499 |
pass
|
|
|
|
| 508 |
for ii in k['Information']:
|
| 509 |
try:
|
| 510 |
rho_string = ii['Value']['StringWithMarkup'][0]['String']
|
| 511 |
+
rho_string = rho_string.replace('Relative density (water = 1): ', '')
|
| 512 |
+
#print(rho_string)
|
| 513 |
+
#tmp_rho = re.match('(?:\d+(?:\.\d*)?|\.\d+)',rho_string)
|
| 514 |
+
m = re.match('((?:\d+(?:\.\d*)?|\.\d+))(?:-((?:\d+(?:\.\d*)?|\.\d+)))?',rho_string)
|
| 515 |
+
groups = m.groups()
|
| 516 |
+
if len(groups):
|
| 517 |
+
for g in groups:
|
| 518 |
+
try:
|
| 519 |
+
tmp_rho = float(g)
|
| 520 |
+
rho_list.append(tmp_rho)
|
| 521 |
+
except:
|
| 522 |
+
continue
|
| 523 |
except:
|
| 524 |
continue
|
| 525 |
if rho_list:
|
| 526 |
+
## remove outliers using interquartile range (IQR)
|
| 527 |
+
rho_list = np.array(rho_list)
|
| 528 |
+
q75,q25 = np.percentile(rho_list,[75,25])
|
| 529 |
+
intr_qr = q75-q25
|
| 530 |
+
hi = q75+(1.5*intr_qr)
|
| 531 |
+
lo = q25-(1.5*intr_qr)
|
| 532 |
+
mask = (rho_list < hi) & (rho_list > lo)
|
| 533 |
+
rho_list = rho_list[mask]
|
| 534 |
rho = np.mean(rho_list)
|
| 535 |
rho_origin = 'pubchem'
|
| 536 |
else:
|
custom_chemicals_db.tsv
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:c46fe917b6bf96481b6d3dfd4613b60103affab34ede2863ecf16b8425385316
|
| 3 |
+
size 7805
|